diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index 27595a959841..dd7bbce8bad7 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -38,6 +38,123 @@ # xargs -n1 | sort | uniq -d; # done +# 20180114: new clang import which bumps version from 5.0.1 to 6.0.0. +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/allocator_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/asan_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/common_interface_defs.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/coverage_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/dfsan_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/esan_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/linux_syscall_hooks.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/lsan_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/msan_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/tsan_interface.h +OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/tsan_interface_atomic.h +OLD_DIRS+=usr/lib/clang/5.0.1/include/sanitizer +OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_builtin_vars.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_cmath.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_complex_builtins.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_intrinsics.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_math_forward_declares.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_runtime_wrapper.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__stddef_max_align_t.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__wmmintrin_aes.h +OLD_FILES+=usr/lib/clang/5.0.1/include/__wmmintrin_pclmul.h +OLD_FILES+=usr/lib/clang/5.0.1/include/adxintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/altivec.h +OLD_FILES+=usr/lib/clang/5.0.1/include/ammintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/arm_acle.h +OLD_FILES+=usr/lib/clang/5.0.1/include/arm_neon.h +OLD_FILES+=usr/lib/clang/5.0.1/include/armintr.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx2intrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512bwintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512cdintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512dqintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512erintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512fintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512ifmaintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512ifmavlintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512pfintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vbmiintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vbmivlintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlbwintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlcdintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vldqintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vpopcntdqintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/avxintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/bmi2intrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/bmiintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/clflushoptintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/clzerointrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/cpuid.h +OLD_FILES+=usr/lib/clang/5.0.1/include/emmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/f16cintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/fma4intrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/fmaintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/fxsrintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/htmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/htmxlintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/ia32intrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/immintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/lwpintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/lzcntintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/mm3dnow.h +OLD_FILES+=usr/lib/clang/5.0.1/include/mm_malloc.h +OLD_FILES+=usr/lib/clang/5.0.1/include/mmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/module.modulemap +OLD_FILES+=usr/lib/clang/5.0.1/include/msa.h +OLD_FILES+=usr/lib/clang/5.0.1/include/mwaitxintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/nmmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/opencl-c.h +OLD_FILES+=usr/lib/clang/5.0.1/include/pkuintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/pmmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/popcntintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/prfchwintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/rdseedintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/rtmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/s390intrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/shaintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/smmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/tbmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/tmmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/vadefs.h +OLD_FILES+=usr/lib/clang/5.0.1/include/vecintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/wmmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/x86intrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xmmintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xopintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xsavecintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xsaveintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xsaveoptintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xsavesintrin.h +OLD_FILES+=usr/lib/clang/5.0.1/include/xtestintrin.h +OLD_DIRS+=usr/lib/clang/5.0.1/include +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-i386.so +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-x86_64.so +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-arm.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-armhf.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.safestack-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats_client-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a +OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a +OLD_DIRS+=usr/lib/clang/5.0.1/lib/freebsd +OLD_DIRS+=usr/lib/clang/5.0.1/lib +OLD_DIRS+=usr/lib/clang/5.0.1 # 20180109: Remove vestiges of digi(4) driver OLD_FILES+=usr/include/sys/digiio.h OLD_FILES+=usr/sbin/digictl diff --git a/UPDATING b/UPDATING index c24e88df08f7..29e941e9601c 100644 --- a/UPDATING +++ b/UPDATING @@ -51,6 +51,12 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 12.x IS SLOW: ****************************** SPECIAL WARNING: ****************************** +20180114: + Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to + 6.0.0. Please see the 20141231 entry below for information about + prerequisites and upgrading, if you are not already using clang 3.5.0 + or higher. + 20180110: LLVM's lld linker is now used as the FreeBSD/amd64 bootstrap linker. This means it is used to link the kernel and userland libraries and diff --git a/contrib/compiler-rt/include/sanitizer/allocator_interface.h b/contrib/compiler-rt/include/sanitizer/allocator_interface.h index 5220631619fc..89f328301db3 100644 --- a/contrib/compiler-rt/include/sanitizer/allocator_interface.h +++ b/contrib/compiler-rt/include/sanitizer/allocator_interface.h @@ -32,7 +32,7 @@ extern "C" { size_t __sanitizer_get_allocated_size(const volatile void *p); /* Number of bytes, allocated and not yet freed by the application. */ - size_t __sanitizer_get_current_allocated_bytes(); + size_t __sanitizer_get_current_allocated_bytes(void); /* Number of bytes, mmaped by the allocator to fulfill allocation requests. Generally, for request of X bytes, allocator can reserve and add to free @@ -40,17 +40,17 @@ extern "C" { All these chunks count toward the heap size. Currently, allocator never releases memory to OS (instead, it just puts freed chunks to free lists). */ - size_t __sanitizer_get_heap_size(); + size_t __sanitizer_get_heap_size(void); /* Number of bytes, mmaped by the allocator, which can be used to fulfill allocation requests. When a user program frees memory chunk, it can first fall into quarantine and will count toward __sanitizer_get_free_bytes() later. */ - size_t __sanitizer_get_free_bytes(); + size_t __sanitizer_get_free_bytes(void); /* Number of bytes in unmapped pages, that are released to OS. Currently, always returns 0. */ - size_t __sanitizer_get_unmapped_bytes(); + size_t __sanitizer_get_unmapped_bytes(void); /* Malloc hooks that may be optionally provided by user. __sanitizer_malloc_hook(ptr, size) is called immediately after @@ -76,6 +76,13 @@ extern "C" { void (*malloc_hook)(const volatile void *, size_t), void (*free_hook)(const volatile void *)); + /* Drains allocator quarantines (calling thread's and global ones), returns + freed memory back to OS and releases other non-essential internal allocator + resources in attempt to reduce process RSS. + Currently available with ASan only. + */ + void __sanitizer_purge_allocator(void); + #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/asan_interface.h b/contrib/compiler-rt/include/sanitizer/asan_interface.h index 97ba0ceb0b23..f2d77143b931 100644 --- a/contrib/compiler-rt/include/sanitizer/asan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/asan_interface.h @@ -64,19 +64,19 @@ extern "C" { // Useful for calling from a debugger to get information about an ASan error. // Returns 1 if an error has been (or is being) reported, otherwise returns 0. - int __asan_report_present(); + int __asan_report_present(void); // Useful for calling from a debugger to get information about an ASan error. // If an error has been (or is being) reported, the following functions return // the pc, bp, sp, address, access type (0 = read, 1 = write), access size and // bug description (e.g. "heap-use-after-free"). Otherwise they return 0. - void *__asan_get_report_pc(); - void *__asan_get_report_bp(); - void *__asan_get_report_sp(); - void *__asan_get_report_address(); - int __asan_get_report_access_type(); - size_t __asan_get_report_access_size(); - const char *__asan_get_report_description(); + void *__asan_get_report_pc(void); + void *__asan_get_report_bp(void); + void *__asan_get_report_sp(void); + void *__asan_get_report_address(void); + int __asan_get_report_access_type(void); + size_t __asan_get_report_access_size(void); + const char *__asan_get_report_description(void); // Useful for calling from the debugger to get information about a pointer. // Returns the category of the given pointer as a constant string. @@ -118,21 +118,21 @@ extern "C" { // User may provide function that would be called right when ASan detects // an error. This can be used to notice cases when ASan detects an error, but // the program crashes before ASan report is printed. - void __asan_on_error(); + void __asan_on_error(void); // Prints accumulated stats to stderr. Used for debugging. - void __asan_print_accumulated_stats(); + void __asan_print_accumulated_stats(void); // This function may be optionally provided by user and should return // a string containing ASan runtime options. See asan_flags.h for details. - const char* __asan_default_options(); + const char* __asan_default_options(void); // The following 2 functions facilitate garbage collection in presence of // asan's fake stack. // Returns an opaque handler to be used later in __asan_addr_is_in_fake_stack. // Returns NULL if the current thread does not have a fake stack. - void *__asan_get_current_fake_stack(); + void *__asan_get_current_fake_stack(void); // If fake_stack is non-NULL and addr belongs to a fake frame in // fake_stack, returns the address on real stack that corresponds to @@ -144,6 +144,10 @@ extern "C" { void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg, void **end); + // Performs cleanup before a [[noreturn]] function. Must be called + // before things like _exit and execl to avoid false positives on stack. + void __asan_handle_no_return(void); + #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h index 4a1de968b0ee..6d4326f74021 100644 --- a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h +++ b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h @@ -115,7 +115,7 @@ extern "C" { const void *beg, const void *mid, const void *end); // Print the stack trace leading to this call. Useful for debugging user code. - void __sanitizer_print_stack_trace(); + void __sanitizer_print_stack_trace(void); // Symbolizes the supplied 'pc' using the format string 'fmt'. // Outputs at most 'out_buf_size' bytes into 'out_buf'. diff --git a/contrib/compiler-rt/include/sanitizer/coverage_interface.h b/contrib/compiler-rt/include/sanitizer/coverage_interface.h index 637379d47c41..bc95a5caf82c 100644 --- a/contrib/compiler-rt/include/sanitizer/coverage_interface.h +++ b/contrib/compiler-rt/include/sanitizer/coverage_interface.h @@ -20,10 +20,13 @@ extern "C" { #endif // Record and dump coverage info. - void __sanitizer_cov_dump(); + void __sanitizer_cov_dump(void); - // Dump collected coverage info. Sorts pcs by module into individual - // .sancov files. + // Clear collected coverage info. + void __sanitizer_cov_reset(void); + + // Dump collected coverage info. Sorts pcs by module into individual .sancov + // files. void __sanitizer_dump_coverage(const uintptr_t *pcs, uintptr_t len); #ifdef __cplusplus diff --git a/contrib/compiler-rt/include/sanitizer/esan_interface.h b/contrib/compiler-rt/include/sanitizer/esan_interface.h index 4aff8d47bbd1..c755ed330000 100644 --- a/contrib/compiler-rt/include/sanitizer/esan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/esan_interface.h @@ -37,11 +37,11 @@ extern "C" { // This function can be called mid-run (or at the end of a run for // a server process that doesn't shut down normally) to request that // data for that point in the run be reported from the tool. -void COMPILER_RT_WEAK __esan_report(); +void COMPILER_RT_WEAK __esan_report(void); // This function returns the number of samples that the esan tool has collected // to this point. This is useful for testing. -unsigned int COMPILER_RT_WEAK __esan_get_sample_count(); +unsigned int COMPILER_RT_WEAK __esan_get_sample_count(void); #ifdef __cplusplus } // extern "C" diff --git a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h new file mode 100644 index 000000000000..0c306cf279e2 --- /dev/null +++ b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h @@ -0,0 +1,33 @@ +//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_HWASAN_INTERFACE_H +#define SANITIZER_HWASAN_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + // This function may be optionally provided by user and should return + // a string containing HWASan runtime options. See asan_flags.h for details. + const char* __hwasan_default_options(void); + + void __hwasan_enable_allocator_tagging(void); + void __hwasan_disable_allocator_tagging(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_HWASAN_INTERFACE_H diff --git a/contrib/compiler-rt/include/sanitizer/lsan_interface.h b/contrib/compiler-rt/include/sanitizer/lsan_interface.h index 8fb8e756da26..731b37314125 100644 --- a/contrib/compiler-rt/include/sanitizer/lsan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/lsan_interface.h @@ -21,8 +21,8 @@ extern "C" { #endif // Allocations made between calls to __lsan_disable() and __lsan_enable() will // be treated as non-leaks. Disable/enable pairs may be nested. - void __lsan_disable(); - void __lsan_enable(); + void __lsan_disable(void); + void __lsan_enable(void); // The heap object into which p points will be treated as a non-leak. void __lsan_ignore_object(const void *p); @@ -49,7 +49,7 @@ extern "C" { // the time of first invocation of this function. // By calling this function early during process shutdown, you can instruct // LSan to ignore shutdown-only leaks which happen later on. - void __lsan_do_leak_check(); + void __lsan_do_leak_check(void); // Check for leaks now. Returns zero if no leaks have been found or if leak // detection is disabled, non-zero otherwise. @@ -58,17 +58,23 @@ extern "C" { // terminate the process. It does not affect the behavior of // __lsan_do_leak_check() or the end-of-process leak check, and is not // affected by them. - int __lsan_do_recoverable_leak_check(); + int __lsan_do_recoverable_leak_check(void); // The user may optionally provide this function to disallow leak checking // for the program it is linked into (if the return value is non-zero). This // function must be defined as returning a constant value; any behavior beyond // that is unsupported. - int __lsan_is_turned_off(); + // To avoid dead stripping, you may need to define this function with + // __attribute__((used)) + int __lsan_is_turned_off(void); + + // This function may be optionally provided by user and should return + // a string containing LSan runtime options. See lsan_flags.inc for details. + const char *__lsan_default_options(void); // This function may be optionally provided by the user and should return // a string containing LSan suppressions. - const char *__lsan_default_suppressions(); + const char *__lsan_default_suppressions(void); #ifdef __cplusplus } // extern "C" diff --git a/contrib/compiler-rt/include/sanitizer/msan_interface.h b/contrib/compiler-rt/include/sanitizer/msan_interface.h index 6d6a3765241b..a87c954c15ec 100644 --- a/contrib/compiler-rt/include/sanitizer/msan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/msan_interface.h @@ -31,10 +31,10 @@ extern "C" { int __msan_origin_is_descendant_or_same(uint32_t this_id, uint32_t prev_id); /* Returns non-zero if tracking origins. */ - int __msan_get_track_origins(); + int __msan_get_track_origins(void); /* Returns the origin id of the latest UMR in the calling thread. */ - uint32_t __msan_get_umr_origin(); + uint32_t __msan_get_umr_origin(void); /* Make memory region fully initialized (without changing its contents). */ void __msan_unpoison(const volatile void *a, size_t size); @@ -82,7 +82,7 @@ extern "C" { void __msan_dump_shadow(const volatile void *x, size_t size); /* Returns true if running under a dynamic tool (DynamoRio-based). */ - int __msan_has_dynamic_component(); + int __msan_has_dynamic_component(void); /* Tell MSan about newly allocated memory (ex.: custom allocator). Memory will be marked uninitialized, with origin at the call site. */ @@ -93,7 +93,7 @@ extern "C" { /* This function may be optionally provided by user and should return a string containing Msan runtime options. See msan_flags.h for details. */ - const char* __msan_default_options(); + const char* __msan_default_options(void); /* Deprecated. Call __sanitizer_set_death_callback instead. */ void __msan_set_death_callback(void (*callback)(void)); diff --git a/contrib/compiler-rt/include/sanitizer/scudo_interface.h b/contrib/compiler-rt/include/sanitizer/scudo_interface.h new file mode 100644 index 000000000000..ec7a9e4e150c --- /dev/null +++ b/contrib/compiler-rt/include/sanitizer/scudo_interface.h @@ -0,0 +1,34 @@ +//===-- sanitizer/scudo_interface.h -----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Public Scudo interface header. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_SCUDO_INTERFACE_H_ +#define SANITIZER_SCUDO_INTERFACE_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + // This function may be optionally provided by a user and should return + // a string containing Scudo runtime options. See scudo_flags.h for details. + const char* __scudo_default_options(void); + + // This function allows to set the RSS limit at runtime. This can be either + // the hard limit (HardLimit=1) or the soft limit (HardLimit=0). The limit + // can be removed by setting LimitMb to 0. This function's parameters should + // be fully trusted to avoid security mishaps. + void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_SCUDO_INTERFACE_H_ diff --git a/contrib/compiler-rt/include/sanitizer/tsan_interface.h b/contrib/compiler-rt/include/sanitizer/tsan_interface.h index 5ea09ab5c1ff..530885014ac3 100644 --- a/contrib/compiler-rt/include/sanitizer/tsan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/tsan_interface.h @@ -44,6 +44,11 @@ const unsigned __tsan_mutex_linker_init = 1 << 0; const unsigned __tsan_mutex_write_reentrant = 1 << 1; // Mutex is read reentrant. const unsigned __tsan_mutex_read_reentrant = 1 << 2; +// Mutex does not have static storage duration, and must not be used after +// its destructor runs. The opposite of __tsan_mutex_linker_init. +// If this flag is passed to __tsan_mutex_destroy, then the destruction +// is ignored unless this flag was previously set on the mutex. +const unsigned __tsan_mutex_not_static = 1 << 8; // Mutex operation flags: @@ -70,6 +75,7 @@ void __tsan_mutex_create(void *addr, unsigned flags); // Annotate destruction of a mutex. // Supported flags: // - __tsan_mutex_linker_init +// - __tsan_mutex_not_static void __tsan_mutex_destroy(void *addr, unsigned flags); // Annotate start of lock operation. diff --git a/contrib/compiler-rt/include/xray/xray_interface.h b/contrib/compiler-rt/include/xray/xray_interface.h index 564613417069..d08039a67fa2 100644 --- a/contrib/compiler-rt/include/xray/xray_interface.h +++ b/contrib/compiler-rt/include/xray/xray_interface.h @@ -106,6 +106,14 @@ extern uintptr_t __xray_function_address(int32_t FuncId); /// encounter errors (when there are no instrumented functions, etc.). extern size_t __xray_max_function_id(); +/// Initialize the required XRay data structures. This is useful in cases where +/// users want to control precisely when the XRay instrumentation data +/// structures are initialized, for example when the XRay library is built with +/// the XRAY_NO_PREINIT preprocessor definition. +/// +/// Calling __xray_init() more than once is safe across multiple threads. +extern void __xray_init(); + } // end extern "C" #endif // XRAY_XRAY_INTERFACE_H diff --git a/contrib/compiler-rt/include/xray/xray_log_interface.h b/contrib/compiler-rt/include/xray/xray_log_interface.h index cdb20094dc26..6c53cdaebfe2 100644 --- a/contrib/compiler-rt/include/xray/xray_log_interface.h +++ b/contrib/compiler-rt/include/xray/xray_log_interface.h @@ -128,6 +128,16 @@ enum XRayLogFlushStatus { XRAY_LOG_FLUSHED = 2, }; +/// This enum indicates the installation state of a logging implementation, when +/// associating a mode to a particular logging implementation through +/// `__xray_log_register_impl(...)` or through `__xray_log_select_mode(...`. +enum XRayLogRegisterStatus { + XRAY_REGISTRATION_OK = 0, + XRAY_DUPLICATE_MODE = 1, + XRAY_MODE_NOT_FOUND = 2, + XRAY_INCOMPLETE_IMPL = 3, +}; + /// A valid XRay logging implementation MUST provide all of the function /// pointers in XRayLogImpl when being installed through `__xray_set_log_impl`. /// To be precise, ALL the functions pointers MUST NOT be nullptr. @@ -159,6 +169,9 @@ struct XRayLogImpl { /// always have a handler for function entry and exit events. In case the /// implementation wants to support arg1 (or other future extensions to XRay /// logging) those MUST be installed by the installed 'log_init' handler. + /// + /// Because we didn't want to change the ABI of this struct, the arg1 handler + /// may be silently overwritten during initialization as well. void (*handle_arg0)(int32_t, XRayEntryType); /// The log implementation provided routine for when __xray_log_flushLog() is @@ -186,6 +199,34 @@ struct XRayLogImpl { /// called while in any other states. void __xray_set_log_impl(XRayLogImpl Impl); +/// This function registers a logging implementation against a "mode" +/// identifier. This allows multiple modes to be registered, and chosen at +/// runtime using the same mode identifier through +/// `__xray_log_select_mode(...)`. +/// +/// We treat the Mode identifier as a null-terminated byte string, as the +/// identifier used when retrieving the log impl. +/// +/// Returns: +/// - XRAY_REGISTRATION_OK on success. +/// - XRAY_DUPLICATE_MODE when an implementation is already associated with +/// the provided Mode; does not update the already-registered +/// implementation. +XRayLogRegisterStatus __xray_log_register_mode(const char *Mode, + XRayLogImpl Impl); + +/// This function selects the implementation associated with Mode that has been +/// registered through __xray_log_register_mode(...) and installs that +/// implementation (as if through calling __xray_set_log_impl(...)). The same +/// caveats apply to __xray_log_select_mode(...) as with +/// __xray_log_set_log_impl(...). +/// +/// Returns: +/// - XRAY_REGISTRATION_OK on success. +/// - XRAY_MODE_NOT_FOUND if there is no implementation associated with Mode; +/// does not update the currently installed implementation. +XRayLogRegisterStatus __xray_log_select_mode(const char *Mode); + /// This function removes the currently installed implementation. It will also /// uninstall any handlers that have been previously installed. It does NOT /// unpatch the instrumentation sleds. @@ -220,12 +261,19 @@ XRayLogFlushStatus __xray_log_flushLog(); namespace __xray { -// Options used by the LLVM XRay FDR implementation. +/// Options used by the LLVM XRay FDR logging implementation. struct FDRLoggingOptions { bool ReportErrors = false; int Fd = -1; }; +/// Options used by the LLVM XRay Basic (Naive) logging implementation. +struct BasicLoggingOptions { + int DurationFilterMicros = 0; + size_t MaxStackDepth = 0; + size_t ThreadBufferSize = 0; +}; + } // namespace __xray #endif // XRAY_XRAY_LOG_INTERFACE_H diff --git a/contrib/compiler-rt/include/xray/xray_records.h b/contrib/compiler-rt/include/xray/xray_records.h index feb8d228b2fd..d4b7b4c31a9a 100644 --- a/contrib/compiler-rt/include/xray/xray_records.h +++ b/contrib/compiler-rt/include/xray/xray_records.h @@ -17,6 +17,8 @@ #ifndef XRAY_XRAY_RECORDS_H #define XRAY_XRAY_RECORDS_H +#include + namespace __xray { enum FileTypes { @@ -65,18 +67,23 @@ static_assert(sizeof(XRayFileHeader) == 32, "XRayFileHeader != 32 bytes"); enum RecordTypes { NORMAL = 0, + ARG_PAYLOAD = 1, }; struct alignas(32) XRayRecord { // This is the type of the record being written. We use 16 bits to allow us to // treat this as a discriminant, and so that the first 4 bytes get packed // properly. See RecordTypes for more supported types. - uint16_t RecordType = 0; + uint16_t RecordType = RecordTypes::NORMAL; // The CPU where the thread is running. We assume number of CPUs <= 256. uint8_t CPU = 0; - // The type of the event. Usually either ENTER = 0 or EXIT = 1. + // The type of the event. One of the following: + // ENTER = 0 + // EXIT = 1 + // TAIL_EXIT = 2 + // ENTER_ARG = 3 uint8_t Type = 0; // The function ID for the record. @@ -94,6 +101,32 @@ struct alignas(32) XRayRecord { static_assert(sizeof(XRayRecord) == 32, "XRayRecord != 32 bytes"); +struct alignas(32) XRayArgPayload { + // We use the same 16 bits as a discriminant for the records in the log here + // too, and so that the first 4 bytes are packed properly. + uint16_t RecordType = RecordTypes::ARG_PAYLOAD; + + // Add a few bytes to pad. + uint8_t Padding[2] = {}; + + // The function ID for the record. + int32_t FuncId = 0; + + // The thread ID for the currently running thread. + uint32_t TId = 0; + + // Add more padding. + uint8_t Padding2[4] = {}; + + // The argument payload. + uint64_t Arg = 0; + + // The rest of this record ought to be left as padding. + uint8_t TailPadding[8] = {}; +} __attribute__((packed)); + +static_assert(sizeof(XRayArgPayload) == 32, "XRayArgPayload != 32 bytes"); + } // namespace __xray #endif // XRAY_XRAY_RECORDS_H diff --git a/contrib/compiler-rt/lib/BlocksRuntime/Block.h b/contrib/compiler-rt/lib/BlocksRuntime/Block.h index 55cdd01a9123..e6cb142fd755 100644 --- a/contrib/compiler-rt/lib/BlocksRuntime/Block.h +++ b/contrib/compiler-rt/lib/BlocksRuntime/Block.h @@ -27,7 +27,7 @@ #if !defined(BLOCK_EXPORT) # if defined(__cplusplus) -# define BLOCK_EXPORT extern "C" +# define BLOCK_EXPORT extern "C" # else # define BLOCK_EXPORT extern # endif diff --git a/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h b/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h index 8ae821815ebe..91d8d8a9850b 100644 --- a/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h +++ b/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h @@ -27,7 +27,7 @@ #if !defined(BLOCK_EXPORT) # if defined(__cplusplus) -# define BLOCK_EXPORT extern "C" +# define BLOCK_EXPORT extern "C" # else # define BLOCK_EXPORT extern # endif diff --git a/contrib/compiler-rt/lib/asan/asan_activation.cc b/contrib/compiler-rt/lib/asan/asan_activation.cc index 66eba9ce2748..d642be93488d 100644 --- a/contrib/compiler-rt/lib/asan/asan_activation.cc +++ b/contrib/compiler-rt/lib/asan/asan_activation.cc @@ -16,8 +16,10 @@ #include "asan_allocator.h" #include "asan_flags.h" #include "asan_internal.h" +#include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_stack.h" +#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" namespace __asan { @@ -110,8 +112,9 @@ void AsanDeactivate() { AllocatorOptions disabled = asan_deactivated_flags.allocator_options; disabled.quarantine_size_mb = 0; disabled.thread_local_quarantine_size_kb = 0; - disabled.min_redzone = 16; // Redzone must be at least 16 bytes long. - disabled.max_redzone = 16; + // Redzone must be at least Max(16, granularity) bytes long. + disabled.min_redzone = Max(16, (int)SHADOW_GRANULARITY); + disabled.max_redzone = disabled.min_redzone; disabled.alloc_dealloc_mismatch = false; disabled.may_return_null = true; ReInitializeAllocator(disabled); diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.cc b/contrib/compiler-rt/lib/asan/asan_allocator.cc index 92963ddfc4da..a437ae1cd3b1 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.cc +++ b/contrib/compiler-rt/lib/asan/asan_allocator.cc @@ -84,7 +84,10 @@ struct ChunkHeader { // This field is used for small sizes. For large sizes it is equal to // SizeClassMap::kMaxSize and the actual size is stored in the // SecondaryAllocator's metadata. - u32 user_requested_size; + u32 user_requested_size : 29; + // align < 8 -> 0 + // else -> log2(min(align, 512)) - 2 + u32 user_requested_alignment_log : 3; u32 alloc_context_id; }; @@ -271,9 +274,9 @@ struct Allocator { atomic_store(&max_redzone, options.max_redzone, memory_order_release); } - void Initialize(const AllocatorOptions &options) { + void InitLinkerInitialized(const AllocatorOptions &options) { SetAllocatorMayReturnNull(options.may_return_null); - allocator.Init(options.release_to_os_interval_ms); + allocator.InitLinkerInitialized(options.release_to_os_interval_ms); SharedInitCode(options); } @@ -351,6 +354,20 @@ struct Allocator { return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz)); } + static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) { + if (user_requested_alignment < 8) + return 0; + if (user_requested_alignment > 512) + user_requested_alignment = 512; + return Log2(user_requested_alignment) - 2; + } + + static uptr ComputeUserAlignment(uptr user_requested_alignment_log) { + if (user_requested_alignment_log == 0) + return 0; + return 1LL << (user_requested_alignment_log + 2); + } + // We have an address between two chunks, and we want to report just one. AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk, AsanChunk *right_chunk) { @@ -385,6 +402,8 @@ struct Allocator { Flags &fl = *flags(); CHECK(stack); const uptr min_alignment = SHADOW_GRANULARITY; + const uptr user_requested_alignment_log = + ComputeUserRequestedAlignmentLog(alignment); if (alignment < min_alignment) alignment = min_alignment; if (size == 0) { @@ -472,6 +491,7 @@ struct Allocator { meta[0] = size; meta[1] = chunk_beg; } + m->user_requested_alignment_log = user_requested_alignment_log; m->alloc_context_id = StackDepotPut(*stack); @@ -573,8 +593,8 @@ struct Allocator { } } - void Deallocate(void *ptr, uptr delete_size, BufferedStackTrace *stack, - AllocType alloc_type) { + void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment, + BufferedStackTrace *stack, AllocType alloc_type) { uptr p = reinterpret_cast(ptr); if (p == 0) return; @@ -601,11 +621,14 @@ struct Allocator { ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type, (AllocType)alloc_type); } - } - - if (delete_size && flags()->new_delete_type_mismatch && - delete_size != m->UsedSize()) { - ReportNewDeleteSizeMismatch(p, delete_size, stack); + } else { + if (flags()->new_delete_type_mismatch && + (alloc_type == FROM_NEW || alloc_type == FROM_NEW_BR) && + ((delete_size && delete_size != m->UsedSize()) || + ComputeUserRequestedAlignmentLog(delete_alignment) != + m->user_requested_alignment_log)) { + ReportNewDeleteTypeMismatch(p, delete_size, delete_alignment, stack); + } } QuarantineChunk(m, ptr, stack); @@ -631,7 +654,7 @@ struct Allocator { // If realloc() races with free(), we may start copying freed memory. // However, we will report racy double-free later anyway. REAL(memcpy)(new_ptr, old_ptr, memcpy_size); - Deallocate(old_ptr, 0, stack, FROM_MALLOC); + Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC); } return new_ptr; } @@ -716,6 +739,22 @@ struct Allocator { return AsanChunkView(m1); } + void Purge() { + AsanThread *t = GetCurrentThread(); + if (t) { + AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); + quarantine.DrainAndRecycle(GetQuarantineCache(ms), + QuarantineCallback(GetAllocatorCache(ms))); + } + { + SpinMutexLock l(&fallback_mutex); + quarantine.DrainAndRecycle(&fallback_quarantine_cache, + QuarantineCallback(&fallback_allocator_cache)); + } + + allocator.ForceReleaseToOS(); + } + void PrintStats() { allocator.PrintStats(); quarantine.PrintStats(); @@ -750,6 +789,9 @@ bool AsanChunkView::IsQuarantined() const { uptr AsanChunkView::Beg() const { return chunk_->Beg(); } uptr AsanChunkView::End() const { return Beg() + UsedSize(); } uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); } +u32 AsanChunkView::UserRequestedAlignment() const { + return Allocator::ComputeUserAlignment(chunk_->user_requested_alignment_log); +} uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; } uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; } AllocType AsanChunkView::GetAllocType() const { @@ -775,7 +817,7 @@ StackTrace AsanChunkView::GetFreeStack() const { } void InitializeAllocator(const AllocatorOptions &options) { - instance.Initialize(options); + instance.InitLinkerInitialized(options); } void ReInitializeAllocator(const AllocatorOptions &options) { @@ -802,12 +844,12 @@ void PrintInternalAllocatorStats() { } void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { - instance.Deallocate(ptr, 0, stack, alloc_type); + instance.Deallocate(ptr, 0, 0, stack, alloc_type); } -void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, - AllocType alloc_type) { - instance.Deallocate(ptr, size, stack, alloc_type); +void asan_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, size, alignment, stack, alloc_type); } void *asan_malloc(uptr size, BufferedStackTrace *stack) { @@ -823,7 +865,7 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true)); if (size == 0) { if (flags()->allocator_frees_and_returns_null_on_realloc_zero) { - instance.Deallocate(p, 0, stack, FROM_MALLOC); + instance.Deallocate(p, 0, 0, stack, FROM_MALLOC); return nullptr; } // Allocate a size of 1 if we shouldn't free() on Realloc to 0 @@ -839,6 +881,10 @@ void *asan_valloc(uptr size, BufferedStackTrace *stack) { void *asan_pvalloc(uptr size, BufferedStackTrace *stack) { uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + return AsanAllocator::FailureHandler::OnBadRequest(); + } // pvalloc(0) should allocate one page. size = size ? RoundUpTo(size, PageSize) : PageSize; return SetErrnoOnNull( @@ -1007,6 +1053,10 @@ uptr __sanitizer_get_allocated_size(const void *p) { return allocated_size; } +void __sanitizer_purge_allocator() { + instance.Purge(); +} + #if !SANITIZER_SUPPORTS_WEAK_HOOKS // Provide default (no-op) implementation of malloc hooks. SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.h b/contrib/compiler-rt/lib/asan/asan_allocator.h index ad1aeb58a86b..26483db4c60e 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.h +++ b/contrib/compiler-rt/lib/asan/asan_allocator.h @@ -58,6 +58,7 @@ class AsanChunkView { uptr Beg() const; // First byte of user memory. uptr End() const; // Last byte of user memory. uptr UsedSize() const; // Size requested by the user. + u32 UserRequestedAlignment() const; // Originally requested alignment. uptr AllocTid() const; uptr FreeTid() const; bool Eq(const AsanChunkView &c) const { return chunk_ == c.chunk_; } @@ -119,7 +120,11 @@ struct AsanMapUnmapCallback { }; #if SANITIZER_CAN_USE_ALLOCATOR64 -# if defined(__powerpc64__) +# if SANITIZER_FUCHSIA +const uptr kAllocatorSpace = ~(uptr)0; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +typedef DefaultSizeClassMap SizeClassMap; +# elif defined(__powerpc64__) const uptr kAllocatorSpace = 0xa0000000000ULL; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. typedef DefaultSizeClassMap SizeClassMap; @@ -193,8 +198,8 @@ struct AsanThreadLocalMallocStorage { void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, AllocType alloc_type); void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type); -void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, - AllocType alloc_type); +void asan_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type); void *asan_malloc(uptr size, BufferedStackTrace *stack); void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); diff --git a/contrib/compiler-rt/lib/asan/asan_descriptions.cc b/contrib/compiler-rt/lib/asan/asan_descriptions.cc index 822a6a62d64e..86c6af7d9f75 100644 --- a/contrib/compiler-rt/lib/asan/asan_descriptions.cc +++ b/contrib/compiler-rt/lib/asan/asan_descriptions.cc @@ -122,6 +122,7 @@ static void GetAccessToHeapChunkInformation(ChunkAccess *descr, } descr->chunk_begin = chunk.Beg(); descr->chunk_size = chunk.UsedSize(); + descr->user_requested_alignment = chunk.UserRequestedAlignment(); descr->alloc_type = chunk.GetAllocType(); } @@ -150,7 +151,7 @@ static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) { str.append(" %zu-byte region [%p,%p)\n", descr.chunk_size, (void *)descr.chunk_begin, (void *)(descr.chunk_begin + descr.chunk_size)); - str.append("%s", d.EndLocation()); + str.append("%s", d.Default()); Printf("%s", str.data()); } @@ -260,7 +261,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr, // FIXME: we may want to also print the size of the access here, // but in case of accesses generated by memset it may be confusing. str.append("%s <== Memory access at offset %zd %s this variable%s\n", - d.Location(), addr, pos_descr, d.EndLocation()); + d.Location(), addr, pos_descr, d.Default()); } else { str.append("\n"); } @@ -295,7 +296,7 @@ static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size, MaybeDemangleGlobalName(g.name)); PrintGlobalLocation(&str, g); str.append("' (0x%zx) of size %zu\n", g.beg, g.size); - str.append("%s", d.EndLocation()); + str.append("%s", d.Default()); PrintGlobalNameIfASCII(&str, g); Printf("%s", str.data()); } @@ -335,6 +336,26 @@ void GlobalAddressDescription::Print(const char *bug_type) const { } } +bool GlobalAddressDescription::PointsInsideTheSameVariable( + const GlobalAddressDescription &other) const { + if (size == 0 || other.size == 0) return false; + + for (uptr i = 0; i < size; i++) { + const __asan_global &a = globals[i]; + for (uptr j = 0; j < other.size; j++) { + const __asan_global &b = other.globals[j]; + if (a.beg == b.beg && + a.beg <= addr && + b.beg <= other.addr && + (addr + access_size) < (a.beg + a.size) && + (other.addr + other.access_size) < (b.beg + b.size)) + return true; + } + } + + return false; +} + void StackAddressDescription::Print() const { Decorator d; char tname[128]; @@ -343,10 +364,10 @@ void StackAddressDescription::Print() const { ThreadNameWithParenthesis(tid, tname, sizeof(tname))); if (!frame_descr) { - Printf("%s\n", d.EndLocation()); + Printf("%s\n", d.Default()); return; } - Printf(" at offset %zu in frame%s\n", offset, d.EndLocation()); + Printf(" at offset %zu in frame%s\n", offset, d.Default()); // Now we print the frame where the alloca has happened. // We print this frame as a stack trace with one element. @@ -355,7 +376,7 @@ void StackAddressDescription::Print() const { // previously. That's unfortunate, but I have no better solution, // especially given that the alloca may be from entirely different place // (e.g. use-after-scope, or different thread's stack). - Printf("%s", d.EndLocation()); + Printf("%s", d.Default()); StackTrace alloca_stack(&frame_pc, 1); alloca_stack.Print(); @@ -405,18 +426,18 @@ void HeapAddressDescription::Print() const { Printf("%sfreed by thread T%d%s here:%s\n", d.Allocation(), free_thread->tid, ThreadNameWithParenthesis(free_thread, tname, sizeof(tname)), - d.EndAllocation()); + d.Default()); StackTrace free_stack = GetStackTraceFromId(free_stack_id); free_stack.Print(); Printf("%spreviously allocated by thread T%d%s here:%s\n", d.Allocation(), alloc_thread->tid, ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)), - d.EndAllocation()); + d.Default()); } else { Printf("%sallocated by thread T%d%s here:%s\n", d.Allocation(), alloc_thread->tid, ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)), - d.EndAllocation()); + d.Default()); } alloc_stack.Print(); DescribeThread(GetCurrentThread()); diff --git a/contrib/compiler-rt/lib/asan/asan_descriptions.h b/contrib/compiler-rt/lib/asan/asan_descriptions.h index 0ee677eb7d0e..1a1b01cf20cd 100644 --- a/contrib/compiler-rt/lib/asan/asan_descriptions.h +++ b/contrib/compiler-rt/lib/asan/asan_descriptions.h @@ -34,11 +34,8 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() {} const char *Access() { return Blue(); } - const char *EndAccess() { return Default(); } const char *Location() { return Green(); } - const char *EndLocation() { return Default(); } const char *Allocation() { return Magenta(); } - const char *EndAllocation() { return Default(); } const char *ShadowByte(u8 byte) { switch (byte) { @@ -72,9 +69,6 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator { return Default(); } } - const char *EndShadowByte() { return Default(); } - const char *MemoryByte() { return Magenta(); } - const char *EndMemoryByte() { return Default(); } }; enum ShadowKind : u8 { @@ -108,6 +102,7 @@ struct ChunkAccess { sptr offset; uptr chunk_begin; uptr chunk_size; + u32 user_requested_alignment : 12; u32 access_type : 2; u32 alloc_type : 2; }; @@ -151,6 +146,10 @@ struct GlobalAddressDescription { u8 size; void Print(const char *bug_type = "") const; + + // Returns true when this descriptions points inside the same global variable + // as other. Descriptions can have different address within the variable + bool PointsInsideTheSameVariable(const GlobalAddressDescription &other) const; }; bool GetGlobalAddressInformation(uptr addr, uptr access_size, diff --git a/contrib/compiler-rt/lib/asan/asan_errors.cc b/contrib/compiler-rt/lib/asan/asan_errors.cc index b7a38eb7cece..0f4a3abff9a7 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.cc +++ b/contrib/compiler-rt/lib/asan/asan_errors.cc @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "asan_errors.h" -#include #include "asan_descriptions.h" #include "asan_mapping.h" #include "asan_report.h" @@ -22,82 +21,26 @@ namespace __asan { -void ErrorStackOverflow::Print() { - Decorator d; - Printf("%s", d.Warning()); - Report( - "ERROR: AddressSanitizer: %s on address %p" - " (pc %p bp %p sp %p T%d)\n", scariness.GetDescription(), - (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid); - Printf("%s", d.EndWarning()); - scariness.Print(); - BufferedStackTrace stack; - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context, - common_flags()->fast_unwind_on_fatal); - stack.Print(); - ReportErrorSummary(scariness.GetDescription(), &stack); -} - -static void MaybeDumpInstructionBytes(uptr pc) { - if (!flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) return; - InternalScopedString str(1024); - str.append("First 16 instruction bytes at pc: "); - if (IsAccessibleMemoryRange(pc, 16)) { - for (int i = 0; i < 16; ++i) { - PrintMemoryByte(&str, "", ((u8 *)pc)[i], /*in_shadow*/ false, " "); - } - str.append("\n"); - } else { - str.append("unaccessible\n"); - } - Report("%s", str.data()); -} - -static void MaybeDumpRegisters(void *context) { - if (!flags()->dump_registers) return; - SignalContext::DumpAllRegisters(context); -} - -static void MaybeReportNonExecRegion(uptr pc) { -#if SANITIZER_FREEBSD || SANITIZER_LINUX - MemoryMappingLayout proc_maps(/*cache_enabled*/ true); - MemoryMappedSegment segment; - while (proc_maps.Next(&segment)) { - if (pc >= segment.start && pc < segment.end && !segment.IsExecutable()) - Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n"); - } +static void OnStackUnwind(const SignalContext &sig, + const void *callback_context, + BufferedStackTrace *stack) { + bool fast = common_flags()->fast_unwind_on_fatal; +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace() + // yields the call stack of the signal's handler and not of the code + // that raised the signal (as it does on Linux). + fast = true; #endif + // Tests and maybe some users expect that scariness is going to be printed + // just before the stack. As only asan has scariness score we have no + // corresponding code in the sanitizer_common and we use this callback to + // print it. + static_cast(callback_context)->Print(); + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, fast); } void ErrorDeadlySignal::Print() { - Decorator d; - Printf("%s", d.Warning()); - const char *description = __sanitizer::DescribeSignalOrException(signo); - Report( - "ERROR: AddressSanitizer: %s on unknown address %p (pc %p bp %p sp %p " - "T%d)\n", - description, (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid); - Printf("%s", d.EndWarning()); - if (pc < GetPageSizeCached()) Report("Hint: pc points to the zero page.\n"); - if (is_memory_access) { - const char *access_type = - write_flag == SignalContext::WRITE - ? "WRITE" - : (write_flag == SignalContext::READ ? "READ" : "UNKNOWN"); - Report("The signal is caused by a %s memory access.\n", access_type); - if (addr < GetPageSizeCached()) - Report("Hint: address points to the zero page.\n"); - } - MaybeReportNonExecRegion(pc); - scariness.Print(); - BufferedStackTrace stack; - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context, - common_flags()->fast_unwind_on_fatal); - stack.Print(); - MaybeDumpInstructionBytes(pc); - MaybeDumpRegisters(context); - Printf("AddressSanitizer can not provide additional info.\n"); - ReportErrorSummary(description, &stack); + ReportDeadlySignal(signal, tid, &OnStackUnwind, &scariness); } void ErrorDoubleFree::Print() { @@ -109,7 +52,7 @@ void ErrorDoubleFree::Print() { "thread T%d%s:\n", scariness.GetDescription(), addr_description.addr, tid, ThreadNameWithParenthesis(tid, tname, sizeof(tname))); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); scariness.Print(); GET_STACK_TRACE_FATAL(second_free_stack->trace[0], second_free_stack->top_frame_bp); @@ -118,7 +61,7 @@ void ErrorDoubleFree::Print() { ReportErrorSummary(scariness.GetDescription(), &stack); } -void ErrorNewDeleteSizeMismatch::Print() { +void ErrorNewDeleteTypeMismatch::Print() { Decorator d; Printf("%s", d.Warning()); char tname[128]; @@ -127,11 +70,29 @@ void ErrorNewDeleteSizeMismatch::Print() { "T%d%s:\n", scariness.GetDescription(), addr_description.addr, tid, ThreadNameWithParenthesis(tid, tname, sizeof(tname))); - Printf("%s object passed to delete has wrong type:\n", d.EndWarning()); - Printf( - " size of the allocated type: %zd bytes;\n" - " size of the deallocated type: %zd bytes.\n", - addr_description.chunk_access.chunk_size, delete_size); + Printf("%s object passed to delete has wrong type:\n", d.Default()); + if (delete_size != 0) { + Printf( + " size of the allocated type: %zd bytes;\n" + " size of the deallocated type: %zd bytes.\n", + addr_description.chunk_access.chunk_size, delete_size); + } + const uptr user_alignment = + addr_description.chunk_access.user_requested_alignment; + if (delete_alignment != user_alignment) { + char user_alignment_str[32]; + char delete_alignment_str[32]; + internal_snprintf(user_alignment_str, sizeof(user_alignment_str), + "%zd bytes", user_alignment); + internal_snprintf(delete_alignment_str, sizeof(delete_alignment_str), + "%zd bytes", delete_alignment); + static const char *kDefaultAlignment = "default-aligned"; + Printf( + " alignment of the allocated type: %s;\n" + " alignment of the deallocated type: %s.\n", + user_alignment > 0 ? user_alignment_str : kDefaultAlignment, + delete_alignment > 0 ? delete_alignment_str : kDefaultAlignment); + } CHECK_GT(free_stack->size, 0); scariness.Print(); GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp); @@ -152,7 +113,7 @@ void ErrorFreeNotMalloced::Print() { "which was not malloc()-ed: %p in thread T%d%s\n", addr_description.Address(), tid, ThreadNameWithParenthesis(tid, tname, sizeof(tname))); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); CHECK_GT(free_stack->size, 0); scariness.Print(); GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp); @@ -173,7 +134,7 @@ void ErrorAllocTypeMismatch::Print() { scariness.GetDescription(), alloc_names[alloc_type], dealloc_names[dealloc_type], addr_description.addr); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); CHECK_GT(dealloc_stack->size, 0); scariness.Print(); GET_STACK_TRACE_FATAL(dealloc_stack->trace[0], dealloc_stack->top_frame_bp); @@ -192,7 +153,7 @@ void ErrorMallocUsableSizeNotOwned::Print() { "ERROR: AddressSanitizer: attempting to call malloc_usable_size() for " "pointer which is not owned: %p\n", addr_description.Address()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); stack->Print(); addr_description.Print(); ReportErrorSummary(scariness.GetDescription(), stack); @@ -205,7 +166,7 @@ void ErrorSanitizerGetAllocatedSizeNotOwned::Print() { "ERROR: AddressSanitizer: attempting to call " "__sanitizer_get_allocated_size() for pointer which is not owned: %p\n", addr_description.Address()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); stack->Print(); addr_description.Print(); ReportErrorSummary(scariness.GetDescription(), stack); @@ -222,7 +183,7 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() { bug_type, addr1_description.Address(), addr1_description.Address() + length1, addr2_description.Address(), addr2_description.Address() + length2); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); scariness.Print(); stack->Print(); addr1_description.Print(); @@ -235,7 +196,7 @@ void ErrorStringFunctionSizeOverflow::Print() { Printf("%s", d.Warning()); Report("ERROR: AddressSanitizer: %s: (size=%zd)\n", scariness.GetDescription(), size); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); scariness.Print(); stack->Print(); addr_description.Print(); @@ -263,7 +224,7 @@ void ErrorODRViolation::Print() { Printf("%s", d.Warning()); Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(), global1.beg); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); InternalScopedString g1_loc(256), g2_loc(256); PrintGlobalLocation(&g1_loc, global1); PrintGlobalLocation(&g2_loc, global2); @@ -292,7 +253,7 @@ void ErrorInvalidPointerPair::Print() { Printf("%s", d.Warning()); Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(), addr1_description.Address(), addr2_description.Address()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); GET_STACK_TRACE_FATAL(pc, bp); stack.Print(); addr1_description.Print(); @@ -477,9 +438,14 @@ static void PrintShadowMemoryForAddress(uptr addr) { InternalScopedString str(4096 * 8); str.append("Shadow bytes around the buggy address:\n"); for (int i = -5; i <= 5; i++) { + uptr row_shadow_addr = aligned_shadow + i * n_bytes_per_row; + // Skip rows that would be outside the shadow range. This can happen when + // the user address is near the bottom, top, or shadow gap of the address + // space. + if (!AddrIsInShadow(row_shadow_addr)) continue; const char *prefix = (i == 0) ? "=>" : " "; - PrintShadowBytes(&str, prefix, (u8 *)(aligned_shadow + i * n_bytes_per_row), - (u8 *)shadow_addr, n_bytes_per_row); + PrintShadowBytes(&str, prefix, (u8 *)row_shadow_addr, (u8 *)shadow_addr, + n_bytes_per_row); } if (flags()->print_legend) PrintLegend(&str); Printf("%s", str.data()); @@ -491,13 +457,13 @@ void ErrorGeneric::Print() { uptr addr = addr_description.Address(); Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n", bug_descr, (void *)addr, pc, bp, sp); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); char tname[128]; Printf("%s%s of size %zu at %p thread T%d%s%s\n", d.Access(), access_size ? (is_write ? "WRITE" : "READ") : "ACCESS", access_size, (void *)addr, tid, - ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.EndAccess()); + ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.Default()); scariness.Print(); GET_STACK_TRACE_FATAL(pc, bp); diff --git a/contrib/compiler-rt/lib/asan/asan_errors.h b/contrib/compiler-rt/lib/asan/asan_errors.h index 9a124924470e..518ba0c6945e 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.h +++ b/contrib/compiler-rt/lib/asan/asan_errors.h @@ -27,61 +27,28 @@ struct ErrorBase { u32 tid; }; -struct ErrorStackOverflow : ErrorBase { - uptr addr, pc, bp, sp; - // ErrorStackOverflow never owns the context. - void *context; - // VS2013 doesn't implement unrestricted unions, so we need a trivial default - // constructor - ErrorStackOverflow() = default; - ErrorStackOverflow(u32 tid, const SignalContext &sig) - : ErrorBase(tid), - addr(sig.addr), - pc(sig.pc), - bp(sig.bp), - sp(sig.sp), - context(sig.context) { - scariness.Clear(); - scariness.Scare(10, "stack-overflow"); - } - void Print(); -}; - struct ErrorDeadlySignal : ErrorBase { - uptr addr, pc, bp, sp; - // ErrorDeadlySignal never owns the context. - void *context; - int signo; - SignalContext::WriteFlag write_flag; - bool is_memory_access; + SignalContext signal; // VS2013 doesn't implement unrestricted unions, so we need a trivial default // constructor ErrorDeadlySignal() = default; - ErrorDeadlySignal(u32 tid, const SignalContext &sig, int signo_) - : ErrorBase(tid), - addr(sig.addr), - pc(sig.pc), - bp(sig.bp), - sp(sig.sp), - context(sig.context), - signo(signo_), - write_flag(sig.write_flag), - is_memory_access(sig.is_memory_access) { + ErrorDeadlySignal(u32 tid, const SignalContext &sig) + : ErrorBase(tid), signal(sig) { scariness.Clear(); - if (is_memory_access) { - if (addr < GetPageSizeCached()) { - scariness.Scare(10, "null-deref"); - } else if (addr == pc) { - scariness.Scare(60, "wild-jump"); - } else if (write_flag == SignalContext::WRITE) { - scariness.Scare(30, "wild-addr-write"); - } else if (write_flag == SignalContext::READ) { - scariness.Scare(20, "wild-addr-read"); - } else { - scariness.Scare(25, "wild-addr"); - } - } else { + if (signal.IsStackOverflow()) { + scariness.Scare(10, "stack-overflow"); + } else if (!signal.is_memory_access) { scariness.Scare(10, "signal"); + } else if (signal.addr < GetPageSizeCached()) { + scariness.Scare(10, "null-deref"); + } else if (signal.addr == signal.pc) { + scariness.Scare(60, "wild-jump"); + } else if (signal.write_flag == SignalContext::WRITE) { + scariness.Scare(30, "wild-addr-write"); + } else if (signal.write_flag == SignalContext::READ) { + scariness.Scare(20, "wild-addr-read"); + } else { + scariness.Scare(25, "wild-addr"); } } void Print(); @@ -104,17 +71,19 @@ struct ErrorDoubleFree : ErrorBase { void Print(); }; -struct ErrorNewDeleteSizeMismatch : ErrorBase { - // ErrorNewDeleteSizeMismatch doesn't own the stack trace. +struct ErrorNewDeleteTypeMismatch : ErrorBase { + // ErrorNewDeleteTypeMismatch doesn't own the stack trace. const BufferedStackTrace *free_stack; HeapAddressDescription addr_description; uptr delete_size; + uptr delete_alignment; // VS2013 doesn't implement unrestricted unions, so we need a trivial default // constructor - ErrorNewDeleteSizeMismatch() = default; - ErrorNewDeleteSizeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr, - uptr delete_size_) - : ErrorBase(tid), free_stack(stack), delete_size(delete_size_) { + ErrorNewDeleteTypeMismatch() = default; + ErrorNewDeleteTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr, + uptr delete_size_, uptr delete_alignment_) + : ErrorBase(tid), free_stack(stack), delete_size(delete_size_), + delete_alignment(delete_alignment_) { GetHeapAddressInformation(addr, 1, &addr_description); scariness.Clear(); scariness.Scare(10, "new-delete-type-mismatch"); @@ -324,10 +293,9 @@ struct ErrorGeneric : ErrorBase { // clang-format off #define ASAN_FOR_EACH_ERROR_KIND(macro) \ - macro(StackOverflow) \ macro(DeadlySignal) \ macro(DoubleFree) \ - macro(NewDeleteSizeMismatch) \ + macro(NewDeleteTypeMismatch) \ macro(FreeNotMalloced) \ macro(AllocTypeMismatch) \ macro(MallocUsableSizeNotOwned) \ diff --git a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc index 017b7d2af129..1c6184e3c7fc 100644 --- a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc +++ b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc @@ -28,9 +28,9 @@ static const u64 kAllocaRedzoneMask = 31UL; // For small size classes inline PoisonShadow for better performance. ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) { - CHECK_EQ(SHADOW_SCALE, 3); // This code expects SHADOW_SCALE=3. u64 *shadow = reinterpret_cast(MemToShadow(ptr)); - if (class_id <= 6) { + if (SHADOW_SCALE == 3 && class_id <= 6) { + // This code expects SHADOW_SCALE=3. for (uptr i = 0; i < (((uptr)1) << class_id); i++) { shadow[i] = magic; // Make sure this does not become memset. @@ -171,7 +171,7 @@ void FakeStack::ForEachFakeFrame(RangeIteratorCallback callback, void *arg) { } } -#if SANITIZER_LINUX && !SANITIZER_ANDROID +#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA static THREADLOCAL FakeStack *fake_stack_tls; FakeStack *GetTLSFakeStack() { @@ -183,7 +183,7 @@ void SetTLSFakeStack(FakeStack *fs) { #else FakeStack *GetTLSFakeStack() { return 0; } void SetTLSFakeStack(FakeStack *fs) { } -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA static FakeStack *GetFakeStack() { AsanThread *t = GetCurrentThread(); diff --git a/contrib/compiler-rt/lib/asan/asan_flags.cc b/contrib/compiler-rt/lib/asan/asan_flags.cc index 6be0d6e94b9a..562168e6d428 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.cc +++ b/contrib/compiler-rt/lib/asan/asan_flags.cc @@ -118,6 +118,10 @@ void InitializeFlags() { const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); ubsan_parser.ParseString(ubsan_default_options); #endif +#if CAN_SANITIZE_LEAKS + const char *lsan_default_options = __lsan::MaybeCallLsanDefaultOptions(); + lsan_parser.ParseString(lsan_default_options); +#endif // Override from command line. asan_parser.ParseString(GetEnv("ASAN_OPTIONS")); @@ -144,6 +148,9 @@ void InitializeFlags() { SanitizerToolName); Die(); } + // Ensure that redzone is at least SHADOW_GRANULARITY. + if (f->redzone < (int)SHADOW_GRANULARITY) + f->redzone = SHADOW_GRANULARITY; // Make "strict_init_order" imply "check_initialization_order". // TODO(samsonov): Use a single runtime flag for an init-order checker. if (f->strict_init_order) { diff --git a/contrib/compiler-rt/lib/asan/asan_flags.inc b/contrib/compiler-rt/lib/asan/asan_flags.inc index f2216c2e9b3b..00071d39f041 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.inc +++ b/contrib/compiler-rt/lib/asan/asan_flags.inc @@ -79,6 +79,10 @@ ASAN_FLAG( "Number of seconds to sleep between printing an error report and " "terminating the program. Useful for debugging purposes (e.g. when one " "needs to attach gdb).") +ASAN_FLAG( + int, sleep_after_init, 0, + "Number of seconds to sleep after AddressSanitizer is initialized. " + "Useful for debugging purposes (e.g. when one needs to attach gdb).") ASAN_FLAG(bool, check_malloc_usable_size, true, "Allows the users to work around the bug in Nvidia drivers prior to " "295.*.") @@ -143,11 +147,6 @@ ASAN_FLAG(int, detect_odr_violation, 2, "If >=2, detect violation of One-Definition-Rule (ODR); " "If ==1, detect ODR-violation only if the two variables " "have different sizes") -ASAN_FLAG(bool, dump_instruction_bytes, false, - "If true, dump 16 bytes starting at the instruction that caused SEGV") -ASAN_FLAG(bool, dump_registers, true, - "If true, dump values of CPU registers when SEGV happens. Only " - "available on OS X for now.") ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.") ASAN_FLAG(bool, halt_on_error, true, "Crash the program after printing the first error report " diff --git a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc new file mode 100644 index 000000000000..0b5bff4f565e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc @@ -0,0 +1,218 @@ +//===-- asan_fuchsia.cc --------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Fuchsia-specific details. +//===---------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "asan_interceptors.h" +#include "asan_internal.h" +#include "asan_stack.h" +#include "asan_thread.h" + +#include +#include +#include +#include + +namespace __asan { + +// The system already set up the shadow memory for us. +// __sanitizer::GetMaxUserVirtualAddress has already been called by +// AsanInitInternal->InitializeHighMemEnd (asan_rtl.cc). +// Just do some additional sanity checks here. +void InitializeShadowMemory() { + if (Verbosity()) PrintAddressSpaceLayout(); + + // Make sure SHADOW_OFFSET doesn't use __asan_shadow_memory_dynamic_address. + __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; + DCHECK(kLowShadowBeg != kDefaultShadowSentinel); + __asan_shadow_memory_dynamic_address = kLowShadowBeg; + + CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); + CHECK_EQ(kHighMemEnd, __sanitizer::ShadowBounds.memory_limit - 1); + CHECK_EQ(kHighMemBeg, __sanitizer::ShadowBounds.shadow_limit); + CHECK_EQ(kHighShadowBeg, __sanitizer::ShadowBounds.shadow_base); + CHECK_EQ(kShadowGapEnd, __sanitizer::ShadowBounds.shadow_base - 1); + CHECK_EQ(kLowShadowEnd, 0); + CHECK_EQ(kLowShadowBeg, 0); +} + +void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { + UNIMPLEMENTED(); +} + +void AsanCheckDynamicRTPrereqs() {} +void AsanCheckIncompatibleRT() {} +void InitializeAsanInterceptors() {} + +void *AsanDoesNotSupportStaticLinkage() { return nullptr; } + +void InitializePlatformExceptionHandlers() {} +void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { + UNIMPLEMENTED(); +} + +// We can use a plain thread_local variable for TSD. +static thread_local void *per_thread; + +void *AsanTSDGet() { return per_thread; } + +void AsanTSDSet(void *tsd) { per_thread = tsd; } + +// There's no initialization needed, and the passed-in destructor +// will never be called. Instead, our own thread destruction hook +// (below) will call AsanThread::TSDDtor directly. +void AsanTSDInit(void (*destructor)(void *tsd)) { + DCHECK(destructor == &PlatformTSDDtor); +} + +void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); } + +static inline size_t AsanThreadMmapSize() { + return RoundUpTo(sizeof(AsanThread), PAGE_SIZE); +} + +struct AsanThread::InitOptions { + uptr stack_bottom, stack_size; +}; + +// Shared setup between thread creation and startup for the initial thread. +static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid, + uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + // In lieu of AsanThread::Create. + AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__); + + AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; + u32 tid = + asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args); + asanThreadRegistry().SetThreadName(tid, name); + + // On other systems, AsanThread::Init() is called from the new + // thread itself. But on Fuchsia we already know the stack address + // range beforehand, so we can do most of the setup right now. + const AsanThread::InitOptions options = {stack_bottom, stack_size}; + thread->Init(&options); + + return thread; +} + +// This gets the same arguments passed to Init by CreateAsanThread, above. +// We're in the creator thread before the new thread is actually started, +// but its stack address range is already known. We don't bother tracking +// the static TLS address range because the system itself already uses an +// ASan-aware allocator for that. +void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) { + DCHECK_NE(GetCurrentThread(), this); + DCHECK_NE(GetCurrentThread(), nullptr); + CHECK_NE(options->stack_bottom, 0); + CHECK_NE(options->stack_size, 0); + stack_bottom_ = options->stack_bottom; + stack_top_ = options->stack_bottom + options->stack_size; +} + +// Called by __asan::AsanInitInternal (asan_rtl.c). +AsanThread *CreateMainThread() { + thrd_t self = thrd_current(); + char name[ZX_MAX_NAME_LEN]; + CHECK_NE(__sanitizer::MainThreadStackBase, 0); + CHECK_GT(__sanitizer::MainThreadStackSize, 0); + AsanThread *t = CreateAsanThread( + nullptr, 0, reinterpret_cast(self), true, + _zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name, + sizeof(name)) == ZX_OK + ? name + : nullptr, + __sanitizer::MainThreadStackBase, __sanitizer::MainThreadStackSize); + SetCurrentThread(t); + return t; +} + +// This is called before each thread creation is attempted. So, in +// its first call, the calling thread is the initial and sole thread. +static void *BeforeThreadCreateHook(uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + EnsureMainThreadIDIsCorrect(); + // Strict init-order checking is thread-hostile. + if (flags()->strict_init_order) StopInitOrderChecking(); + + GET_STACK_TRACE_THREAD; + u32 parent_tid = GetCurrentTidOrInvalid(); + + return CreateAsanThread(&stack, parent_tid, user_id, detached, name, + stack_bottom, stack_size); +} + +// This is called after creating a new thread (in the creating thread), +// with the pointer returned by BeforeThreadCreateHook (above). +static void ThreadCreateHook(void *hook, bool aborted) { + AsanThread *thread = static_cast(hook); + if (!aborted) { + // The thread was created successfully. + // ThreadStartHook is already running in the new thread. + } else { + // The thread wasn't created after all. + // Clean up everything we set up in BeforeThreadCreateHook. + asanThreadRegistry().FinishThread(thread->tid()); + UnmapOrDie(thread, AsanThreadMmapSize()); + } +} + +// This is called in the newly-created thread before it runs anything else, +// with the pointer returned by BeforeThreadCreateHook (above). +// cf. asan_interceptors.cc:asan_thread_start +static void ThreadStartHook(void *hook, uptr os_id) { + AsanThread *thread = static_cast(hook); + SetCurrentThread(thread); + + // In lieu of AsanThread::ThreadStart. + asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false, + nullptr); +} + +// Each thread runs this just before it exits, +// with the pointer returned by BeforeThreadCreateHook (above). +// All per-thread destructors have already been called. +static void ThreadExitHook(void *hook, uptr os_id) { + AsanThread::TSDDtor(per_thread); +} + +} // namespace __asan + +// These are declared (in extern "C") by . +// The system runtime will call our definitions directly. + +void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached, + const char *name, void *stack_base, + size_t stack_size) { + return __asan::BeforeThreadCreateHook( + reinterpret_cast(thread), detached, name, + reinterpret_cast(stack_base), stack_size); +} + +void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) { + __asan::ThreadCreateHook(hook, error != thrd_success); +} + +void __sanitizer_thread_start_hook(void *hook, thrd_t self) { + __asan::ThreadStartHook(hook, reinterpret_cast(self)); +} + +void __sanitizer_thread_exit_hook(void *hook, thrd_t self) { + __asan::ThreadExitHook(hook, reinterpret_cast(self)); +} + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_globals.cc b/contrib/compiler-rt/lib/asan/asan_globals.cc index eebada804f07..0db65d010349 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals.cc @@ -384,6 +384,10 @@ void __asan_register_globals(__asan_global *globals, uptr n) { } RegisterGlobal(&globals[i]); } + + // Poison the metadata. It should not be accessible to user code. + PoisonShadow(reinterpret_cast(globals), n * sizeof(__asan_global), + kAsanGlobalRedzoneMagic); } // Unregister an array of globals. @@ -399,6 +403,9 @@ void __asan_unregister_globals(__asan_global *globals, uptr n) { } UnregisterGlobal(&globals[i]); } + + // Unpoison the metadata. + PoisonShadow(reinterpret_cast(globals), n * sizeof(__asan_global), 0); } // This method runs immediately prior to dynamic initialization in each TU, diff --git a/contrib/compiler-rt/lib/asan/asan_init_version.h b/contrib/compiler-rt/lib/asan/asan_init_version.h index f48cc19cc515..c49fcd740248 100644 --- a/contrib/compiler-rt/lib/asan/asan_init_version.h +++ b/contrib/compiler-rt/lib/asan/asan_init_version.h @@ -15,6 +15,8 @@ #ifndef ASAN_INIT_VERSION_H #define ASAN_INIT_VERSION_H +#include "sanitizer_common/sanitizer_platform.h" + extern "C" { // Every time the ASan ABI changes we also change the version number in the // __asan_init function name. Objects built with incompatible ASan ABI @@ -32,7 +34,12 @@ extern "C" { // v6=>v7: added 'odr_indicator' to __asan_global // v7=>v8: added '__asan_(un)register_image_globals' functions for dead // stripping support on Mach-O platforms +#if SANITIZER_WORDSIZE == 32 && SANITIZER_ANDROID + // v8=>v9: 32-bit Android switched to dynamic shadow + #define __asan_version_mismatch_check __asan_version_mismatch_check_v9 +#else #define __asan_version_mismatch_check __asan_version_mismatch_check_v8 +#endif } #endif // ASAN_INIT_VERSION_H diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.cc b/contrib/compiler-rt/lib/asan/asan_interceptors.cc index 34ca22b8616e..cb7dcb3b1ca8 100644 --- a/contrib/compiler-rt/lib/asan/asan_interceptors.cc +++ b/contrib/compiler-rt/lib/asan/asan_interceptors.cc @@ -24,6 +24,11 @@ #include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_libc.h" +// There is no general interception at all on Fuchsia. +// Only the functions in asan_interceptors_memintrinsics.cc are +// really defined to replace libc functions. +#if !SANITIZER_FUCHSIA + #if SANITIZER_POSIX #include "sanitizer_common/sanitizer_posix.h" #endif @@ -36,108 +41,6 @@ namespace __asan { -// Return true if we can quickly decide that the region is unpoisoned. -// We assume that a redzone is at least 16 bytes. -static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) { - if (size == 0) return true; - if (size <= 32) - return !AddressIsPoisoned(beg) && - !AddressIsPoisoned(beg + size - 1) && - !AddressIsPoisoned(beg + size / 2); - if (size <= 64) - return !AddressIsPoisoned(beg) && - !AddressIsPoisoned(beg + size / 4) && - !AddressIsPoisoned(beg + size - 1) && - !AddressIsPoisoned(beg + 3 * size / 4) && - !AddressIsPoisoned(beg + size / 2); - return false; -} - -struct AsanInterceptorContext { - const char *interceptor_name; -}; - -// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE, -// and ASAN_WRITE_RANGE as macro instead of function so -// that no extra frames are created, and stack trace contains -// relevant information only. -// We check all shadow bytes. -#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \ - uptr __offset = (uptr)(offset); \ - uptr __size = (uptr)(size); \ - uptr __bad = 0; \ - if (__offset > __offset + __size) { \ - GET_STACK_TRACE_FATAL_HERE; \ - ReportStringFunctionSizeOverflow(__offset, __size, &stack); \ - } \ - if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \ - (__bad = __asan_region_is_poisoned(__offset, __size))) { \ - AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \ - bool suppressed = false; \ - if (_ctx) { \ - suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \ - if (!suppressed && HaveStackTraceBasedSuppressions()) { \ - GET_STACK_TRACE_FATAL_HERE; \ - suppressed = IsStackTraceSuppressed(&stack); \ - } \ - } \ - if (!suppressed) { \ - GET_CURRENT_PC_BP_SP; \ - ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\ - } \ - } \ - } while (0) - -// memcpy is called during __asan_init() from the internals of printf(...). -// We do not treat memcpy with to==from as a bug. -// See http://llvm.org/bugs/show_bug.cgi?id=11763. -#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \ - if (asan_init_is_running) { \ - return REAL(memcpy)(to, from, size); \ - } \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - if (to != from) { \ - CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ - } \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return REAL(memcpy)(to, from, size); \ - } while (0) - -// memset is called inside Printf. -#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \ - if (asan_init_is_running) { \ - return REAL(memset)(block, c, size); \ - } \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - ASAN_WRITE_RANGE(ctx, block, size); \ - } \ - return REAL(memset)(block, c, size); \ - } while (0) - -#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return internal_memmove(to, from, size); \ - } while (0) - -#define ASAN_READ_RANGE(ctx, offset, size) \ - ACCESS_MEMORY_RANGE(ctx, offset, size, false) -#define ASAN_WRITE_RANGE(ctx, offset, size) \ - ACCESS_MEMORY_RANGE(ctx, offset, size, true) - #define ASAN_READ_STRING_OF_LEN(ctx, s, len, n) \ ASAN_READ_RANGE((ctx), (s), \ common_flags()->strict_string_checks ? (len) + 1 : (n)) @@ -145,23 +48,6 @@ struct AsanInterceptorContext { #define ASAN_READ_STRING(ctx, s, n) \ ASAN_READ_STRING_OF_LEN((ctx), (s), REAL(strlen)(s), (n)) -// Behavior of functions like "memcpy" or "strcpy" is undefined -// if memory intervals overlap. We report error in this case. -// Macro is used to avoid creation of new frames. -static inline bool RangesOverlap(const char *offset1, uptr length1, - const char *offset2, uptr length2) { - return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1)); -} -#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \ - const char *offset1 = (const char*)_offset1; \ - const char *offset2 = (const char*)_offset2; \ - if (RangesOverlap(offset1, length1, offset2, length2)) { \ - GET_STACK_TRACE_FATAL_HERE; \ - ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \ - offset2, length2, &stack); \ - } \ -} while (0) - static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) { #if SANITIZER_INTERCEPT_STRNLEN if (REAL(strnlen)) { @@ -275,6 +161,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) } while (false) #include "sanitizer_common/sanitizer_common_interceptors.inc" +#include "sanitizer_common/sanitizer_signal_interceptors.inc" // Syscall interceptors don't have contexts, we don't support suppressions // for them. @@ -356,42 +243,6 @@ INTERCEPTOR(int, pthread_join, void *t, void **arg) { DEFINE_REAL_PTHREAD_FUNCTIONS #endif // ASAN_INTERCEPT_PTHREAD_CREATE -#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION - -#if SANITIZER_ANDROID -INTERCEPTOR(void*, bsd_signal, int signum, void *handler) { - if (GetHandleSignalMode(signum) != kHandleSignalExclusive) - return REAL(bsd_signal)(signum, handler); - return 0; -} -#endif - -INTERCEPTOR(void*, signal, int signum, void *handler) { - if (GetHandleSignalMode(signum) != kHandleSignalExclusive) - return REAL(signal)(signum, handler); - return nullptr; -} - -INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) { - if (GetHandleSignalMode(signum) != kHandleSignalExclusive) - return REAL(sigaction)(signum, act, oldact); - return 0; -} - -namespace __sanitizer { -int real_sigaction(int signum, const void *act, void *oldact) { - return REAL(sigaction)(signum, (const struct sigaction *)act, - (struct sigaction *)oldact); -} -} // namespace __sanitizer - -#elif SANITIZER_POSIX -// We need to have defined REAL(sigaction) on posix systems. -DEFINE_REAL(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) -#endif // ASAN_INTERCEPT_SIGNAL_AND_SIGACTION - #if ASAN_INTERCEPT_SWAPCONTEXT static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) { // Align to page size. @@ -428,6 +279,11 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp, } #endif // ASAN_INTERCEPT_SWAPCONTEXT +#if SANITIZER_NETBSD +#define longjmp __longjmp14 +#define siglongjmp __siglongjmp14 +#endif + INTERCEPTOR(void, longjmp, void *env, int val) { __asan_handle_no_return(); REAL(longjmp)(env, val); @@ -462,18 +318,6 @@ INTERCEPTOR(void, __cxa_throw, void *a, void *b, void *c) { } #endif -void *__asan_memcpy(void *to, const void *from, uptr size) { - ASAN_MEMCPY_IMPL(nullptr, to, from, size); -} - -void *__asan_memset(void *block, int c, uptr size) { - ASAN_MEMSET_IMPL(nullptr, block, c, size); -} - -void *__asan_memmove(void *to, const void *from, uptr size) { - ASAN_MEMMOVE_IMPL(nullptr, to, from, size); -} - #if ASAN_INTERCEPT_INDEX # if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX INTERCEPTOR(char*, index, const char *string, int c) @@ -711,6 +555,7 @@ void InitializeAsanInterceptors() { CHECK(!was_called_once); was_called_once = true; InitializeCommonInterceptors(); + InitializeSignalInterceptors(); // Intercept str* functions. ASAN_INTERCEPT_FUNC(strcat); // NOLINT @@ -733,15 +578,9 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(strtoll); #endif - // Intecept signal- and jump-related functions. + // Intecept jump-related functions. ASAN_INTERCEPT_FUNC(longjmp); -#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION - ASAN_INTERCEPT_FUNC(sigaction); -#if SANITIZER_ANDROID - ASAN_INTERCEPT_FUNC(bsd_signal); -#endif - ASAN_INTERCEPT_FUNC(signal); -#endif + #if ASAN_INTERCEPT_SWAPCONTEXT ASAN_INTERCEPT_FUNC(swapcontext); #endif @@ -785,3 +624,5 @@ void InitializeAsanInterceptors() { } } // namespace __asan + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.h b/contrib/compiler-rt/lib/asan/asan_interceptors.h index 93fca4f67366..e13bdecfdede 100644 --- a/contrib/compiler-rt/lib/asan/asan_interceptors.h +++ b/contrib/compiler-rt/lib/asan/asan_interceptors.h @@ -15,9 +15,30 @@ #define ASAN_INTERCEPTORS_H #include "asan_internal.h" +#include "asan_interceptors_memintrinsics.h" #include "interception/interception.h" #include "sanitizer_common/sanitizer_platform_interceptors.h" +namespace __asan { + +void InitializeAsanInterceptors(); +void InitializePlatformInterceptors(); + +#define ENSURE_ASAN_INITED() \ + do { \ + CHECK(!asan_init_is_running); \ + if (UNLIKELY(!asan_inited)) { \ + AsanInitFromRtl(); \ + } \ + } while (0) + +} // namespace __asan + +// There is no general interception at all on Fuchsia. +// Only the functions in asan_interceptors_memintrinsics.h are +// really defined to replace libc functions. +#if !SANITIZER_FUCHSIA + // Use macro to describe if specific function should be // intercepted on a given platform. #if !SANITIZER_WINDOWS @@ -34,24 +55,19 @@ # define ASAN_INTERCEPT_FORK 0 #endif -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS # define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 1 #else # define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 0 #endif -#if SANITIZER_LINUX && !SANITIZER_ANDROID +#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_SOLARIS # define ASAN_INTERCEPT_SWAPCONTEXT 1 #else # define ASAN_INTERCEPT_SWAPCONTEXT 0 #endif -#if !SANITIZER_WINDOWS -# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 1 -#else -# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 0 -#endif - #if !SANITIZER_WINDOWS # define ASAN_INTERCEPT_SIGLONGJMP 1 #else @@ -66,7 +82,8 @@ // Android bug: https://code.google.com/p/android/issues/detail?id=61799 #if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && \ - !(SANITIZER_ANDROID && defined(__i386)) + !(SANITIZER_ANDROID && defined(__i386)) && \ + !SANITIZER_SOLARIS # define ASAN_INTERCEPT___CXA_THROW 1 #else # define ASAN_INTERCEPT___CXA_THROW 0 @@ -85,16 +102,11 @@ #endif DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size) -DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size) -DECLARE_REAL(void*, memset, void *block, int c, uptr size) DECLARE_REAL(char*, strchr, const char *str, int c) DECLARE_REAL(SIZE_T, strlen, const char *s) DECLARE_REAL(char*, strncpy, char *to, const char *from, uptr size) DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen) DECLARE_REAL(char*, strstr, const char *s1, const char *s2) -struct sigaction; -DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, - struct sigaction *oldact) #if !SANITIZER_MAC #define ASAN_INTERCEPT_FUNC(name) \ @@ -113,18 +125,6 @@ DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, #define ASAN_INTERCEPT_FUNC(name) #endif // SANITIZER_MAC -namespace __asan { - -void InitializeAsanInterceptors(); -void InitializePlatformInterceptors(); - -#define ENSURE_ASAN_INITED() do { \ - CHECK(!asan_init_is_running); \ - if (UNLIKELY(!asan_inited)) { \ - AsanInitFromRtl(); \ - } \ -} while (0) - -} // namespace __asan +#endif // !SANITIZER_FUCHSIA #endif // ASAN_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc new file mode 100644 index 000000000000..c89cb011492e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc @@ -0,0 +1,44 @@ +//===-- asan_interceptors_memintrinsics.cc --------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// ASan versions of memcpy, memmove, and memset. +//===---------------------------------------------------------------------===// + +#include "asan_interceptors_memintrinsics.h" +#include "asan_report.h" +#include "asan_stack.h" +#include "asan_suppressions.h" + +using namespace __asan; // NOLINT + +void *__asan_memcpy(void *to, const void *from, uptr size) { + ASAN_MEMCPY_IMPL(nullptr, to, from, size); +} + +void *__asan_memset(void *block, int c, uptr size) { + ASAN_MEMSET_IMPL(nullptr, block, c, size); +} + +void *__asan_memmove(void *to, const void *from, uptr size) { + ASAN_MEMMOVE_IMPL(nullptr, to, from, size); +} + +#if SANITIZER_FUCHSIA + +// Fuchsia doesn't use sanitizer_common_interceptors.inc, but the only +// things there it wants are these three. Just define them as aliases +// here rather than repeating the contents. + +decltype(memcpy) memcpy[[gnu::alias("__asan_memcpy")]]; +decltype(memmove) memmove[[gnu::alias("__asan_memmove")]]; +decltype(memset) memset[[gnu::alias("__asan_memset")]]; + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h new file mode 100644 index 000000000000..5a8339a23e97 --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h @@ -0,0 +1,148 @@ +//===-- asan_interceptors_memintrinsics.h -----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// ASan-private header for asan_memintrin.cc +//===---------------------------------------------------------------------===// +#ifndef ASAN_MEMINTRIN_H +#define ASAN_MEMINTRIN_H + +#include "asan_interface_internal.h" +#include "asan_internal.h" +#include "asan_mapping.h" +#include "interception/interception.h" + +DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size) +DECLARE_REAL(void*, memset, void *block, int c, uptr size) + +namespace __asan { + +// Return true if we can quickly decide that the region is unpoisoned. +// We assume that a redzone is at least 16 bytes. +static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) { + if (size == 0) return true; + if (size <= 32) + return !AddressIsPoisoned(beg) && + !AddressIsPoisoned(beg + size - 1) && + !AddressIsPoisoned(beg + size / 2); + if (size <= 64) + return !AddressIsPoisoned(beg) && + !AddressIsPoisoned(beg + size / 4) && + !AddressIsPoisoned(beg + size - 1) && + !AddressIsPoisoned(beg + 3 * size / 4) && + !AddressIsPoisoned(beg + size / 2); + return false; +} + +struct AsanInterceptorContext { + const char *interceptor_name; +}; + +// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE, +// and ASAN_WRITE_RANGE as macro instead of function so +// that no extra frames are created, and stack trace contains +// relevant information only. +// We check all shadow bytes. +#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \ + uptr __offset = (uptr)(offset); \ + uptr __size = (uptr)(size); \ + uptr __bad = 0; \ + if (__offset > __offset + __size) { \ + GET_STACK_TRACE_FATAL_HERE; \ + ReportStringFunctionSizeOverflow(__offset, __size, &stack); \ + } \ + if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \ + (__bad = __asan_region_is_poisoned(__offset, __size))) { \ + AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \ + bool suppressed = false; \ + if (_ctx) { \ + suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \ + if (!suppressed && HaveStackTraceBasedSuppressions()) { \ + GET_STACK_TRACE_FATAL_HERE; \ + suppressed = IsStackTraceSuppressed(&stack); \ + } \ + } \ + if (!suppressed) { \ + GET_CURRENT_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\ + } \ + } \ + } while (0) + +// memcpy is called during __asan_init() from the internals of printf(...). +// We do not treat memcpy with to==from as a bug. +// See http://llvm.org/bugs/show_bug.cgi?id=11763. +#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \ + if (asan_init_is_running) { \ + return REAL(memcpy)(to, from, size); \ + } \ + ENSURE_ASAN_INITED(); \ + if (flags()->replace_intrin) { \ + if (to != from) { \ + CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ + } \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return REAL(memcpy)(to, from, size); \ + } while (0) + +// memset is called inside Printf. +#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \ + if (asan_init_is_running) { \ + return REAL(memset)(block, c, size); \ + } \ + ENSURE_ASAN_INITED(); \ + if (flags()->replace_intrin) { \ + ASAN_WRITE_RANGE(ctx, block, size); \ + } \ + return REAL(memset)(block, c, size); \ + } while (0) + +#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \ + ENSURE_ASAN_INITED(); \ + if (flags()->replace_intrin) { \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return internal_memmove(to, from, size); \ + } while (0) + +#define ASAN_READ_RANGE(ctx, offset, size) \ + ACCESS_MEMORY_RANGE(ctx, offset, size, false) +#define ASAN_WRITE_RANGE(ctx, offset, size) \ + ACCESS_MEMORY_RANGE(ctx, offset, size, true) + +// Behavior of functions like "memcpy" or "strcpy" is undefined +// if memory intervals overlap. We report error in this case. +// Macro is used to avoid creation of new frames. +static inline bool RangesOverlap(const char *offset1, uptr length1, + const char *offset2, uptr length2) { + return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1)); +} +#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \ + const char *offset1 = (const char*)_offset1; \ + const char *offset2 = (const char*)_offset2; \ + if (RangesOverlap(offset1, length1, offset2, length2)) { \ + GET_STACK_TRACE_FATAL_HERE; \ + ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \ + offset2, length2, &stack); \ + } \ +} while (0) + +} // namespace __asan + +#endif // ASAN_MEMINTRIN_H diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h index f09bbd83af25..19133e5291a9 100644 --- a/contrib/compiler-rt/lib/asan/asan_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_internal.h @@ -69,8 +69,12 @@ void InitializePlatformExceptionHandlers(); bool IsSystemHeapAddress(uptr addr); // asan_rtl.cc +void PrintAddressSpaceLayout(); void NORETURN ShowStatsAndAbort(); +// asan_shadow_setup.cc +void InitializeShadowMemory(); + // asan_malloc_linux.cc / asan_malloc_mac.cc void ReplaceSystemMalloc(); @@ -80,6 +84,9 @@ void *AsanDoesNotSupportStaticLinkage(); void AsanCheckDynamicRTPrereqs(); void AsanCheckIncompatibleRT(); +// asan_thread.cc +AsanThread *CreateMainThread(); + // Support function for __asan_(un)register_image_globals. Searches for the // loaded image containing `needle' and then enumerates all global metadata // structures declared in that image, applying `op' (e.g., diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc index 6d47ba432a61..047e1dbb72fa 100644 --- a/contrib/compiler-rt/lib/asan/asan_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_linux.cc @@ -13,10 +13,12 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "asan_interceptors.h" #include "asan_internal.h" +#include "asan_premap_shadow.h" #include "asan_thread.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_freebsd.h" @@ -39,9 +41,17 @@ #include #endif -#if SANITIZER_ANDROID || SANITIZER_FREEBSD +#if SANITIZER_SOLARIS +#include +#endif + +#if SANITIZER_ANDROID || SANITIZER_FREEBSD || SANITIZER_SOLARIS #include extern "C" void* _DYNAMIC; +#elif SANITIZER_NETBSD +#include +#include +extern Elf_Dyn _DYNAMIC; #else #include #include @@ -77,9 +87,51 @@ void *AsanDoesNotSupportStaticLinkage() { return &_DYNAMIC; // defined in link.h } +static void UnmapFromTo(uptr from, uptr to) { + CHECK(to >= from); + if (to == from) return; + uptr res = internal_munmap(reinterpret_cast(from), to - from); + if (UNLIKELY(internal_iserror(res))) { + Report( + "ERROR: AddresSanitizer failed to unmap 0x%zx (%zd) bytes at address " + "%p\n", + to - from, to - from, from); + CHECK("unable to unmap" && 0); + } +} + +#if ASAN_PREMAP_SHADOW +uptr FindPremappedShadowStart() { + uptr granularity = GetMmapGranularity(); + uptr shadow_start = reinterpret_cast(&__asan_shadow); + uptr premap_shadow_size = PremapShadowSize(); + uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity); + // We may have mapped too much. Release extra memory. + UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size); + return shadow_start; +} +#endif + uptr FindDynamicShadowStart() { - UNREACHABLE("FindDynamicShadowStart is not available"); - return 0; +#if ASAN_PREMAP_SHADOW + if (!PremapShadowFailed()) + return FindPremappedShadowStart(); +#endif + + uptr granularity = GetMmapGranularity(); + uptr alignment = granularity * 8; + uptr left_padding = granularity; + uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity); + uptr map_size = shadow_size + left_padding + alignment; + + uptr map_start = (uptr)MmapNoAccess(map_size); + CHECK_NE(map_start, ~(uptr)0); + + uptr shadow_start = RoundUpTo(map_start + left_padding, alignment); + UnmapFromTo(map_start, shadow_start - left_padding); + UnmapFromTo(shadow_start + shadow_size, map_start + map_size); + + return shadow_start; } void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { @@ -93,6 +145,9 @@ void AsanCheckIncompatibleRT() {} #else static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size, void *data) { + VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n", + info->dlpi_name, info->dlpi_addr); + // Continue until the first dynamic library is found if (!info->dlpi_name || info->dlpi_name[0] == 0) return 0; @@ -101,6 +156,21 @@ static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size, if (internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0) return 0; +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + // Ignore first entry (the main program) + char **p = (char **)data; + if (!(*p)) { + *p = (char *)-1; + return 0; + } +#endif + +#if SANITIZER_SOLARIS + // Ignore executable on Solaris + if (info->dlpi_addr == 0) + return 0; +#endif + *(const char **)data = info->dlpi_name; return 1; } @@ -179,4 +249,5 @@ void *AsanDlSymNext(const char *sym) { } // namespace __asan -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc index fd40f47db1c4..6697ff8764ed 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc @@ -15,7 +15,8 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \ + SANITIZER_NETBSD || SANITIZER_SOLARIS #include "sanitizer_common/sanitizer_tls_get_addr.h" #include "asan_allocator.h" @@ -30,9 +31,9 @@ static uptr allocated_for_dlsym; static const uptr kDlsymAllocPoolSize = 1024; static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; -static bool IsInDlsymAllocPool(const void *ptr) { +static INLINE bool IsInDlsymAllocPool(const void *ptr) { uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); + return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); } static void *AllocateFromLocalPool(uptr size_in_bytes) { @@ -43,6 +44,26 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { return mem; } +static INLINE bool MaybeInDlsym() { + // Fuchsia doesn't use dlsym-based interceptors. + return !SANITIZER_FUCHSIA && asan_init_is_running; +} + +static void *ReallocFromLocalPool(void *ptr, uptr size) { + const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(MaybeInDlsym())) { + new_ptr = AllocateFromLocalPool(size); + } else { + ENSURE_ASAN_INITED(); + GET_STACK_TRACE_MALLOC; + new_ptr = asan_malloc(size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; +} + INTERCEPTOR(void, free, void *ptr) { GET_STACK_TRACE_FREE; if (UNLIKELY(IsInDlsymAllocPool(ptr))) @@ -60,7 +81,7 @@ INTERCEPTOR(void, cfree, void *ptr) { #endif // SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(asan_init_is_running)) + if (UNLIKELY(MaybeInDlsym())) // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. return AllocateFromLocalPool(size); ENSURE_ASAN_INITED(); @@ -69,7 +90,7 @@ INTERCEPTOR(void*, malloc, uptr size) { } INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { - if (UNLIKELY(asan_init_is_running)) + if (UNLIKELY(MaybeInDlsym())) // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. return AllocateFromLocalPool(nmemb * size); ENSURE_ASAN_INITED(); @@ -78,21 +99,9 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { } INTERCEPTOR(void*, realloc, void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(asan_init_is_running)) { - new_ptr = AllocateFromLocalPool(size); - } else { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - new_ptr = asan_malloc(size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } - if (UNLIKELY(asan_init_is_running)) + if (UNLIKELY(IsInDlsymAllocPool(ptr))) + return ReallocFromLocalPool(ptr, size); + if (UNLIKELY(MaybeInDlsym())) return AllocateFromLocalPool(size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; @@ -226,4 +235,5 @@ void ReplaceSystemMalloc() { } // namespace __asan #endif // SANITIZER_ANDROID -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || + // SANITIZER_NETBSD || SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/asan/asan_mapping.h b/contrib/compiler-rt/lib/asan/asan_mapping.h index 695740cd982f..d3f360ffc969 100644 --- a/contrib/compiler-rt/lib/asan/asan_mapping.h +++ b/contrib/compiler-rt/lib/asan/asan_mapping.h @@ -115,6 +115,13 @@ // || `[0x40000000, 0x47ffffff]` || LowShadow || // || `[0x00000000, 0x3fffffff]` || LowMem || // +// Shadow mapping on NetBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: +// || `[0x4feffffffe01, 0x7f7ffffff000]` || HighMem || +// || `[0x49fdffffffc0, 0x4feffffffe00]` || HighShadow || +// || `[0x480000000000, 0x49fdffffffbf]` || ShadowGap || +// || `[0x400000000000, 0x47ffffffffff]` || LowShadow || +// || `[0x000000000000, 0x3fffffffffff]` || LowMem || +// // Default Windows/i386 mapping: // (the exact location of HighShadow/HighMem may vary depending // on WoW64, /LARGEADDRESSAWARE, etc). @@ -124,11 +131,16 @@ // || `[0x30000000, 0x35ffffff]` || LowShadow || // || `[0x00000000, 0x2fffffff]` || LowMem || +#if defined(ASAN_SHADOW_SCALE) +static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE; +#else static const u64 kDefaultShadowScale = 3; +#endif static const u64 kDefaultShadowSentinel = ~(uptr)0; static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000 static const u64 kDefaultShadowOffset64 = 1ULL << 44; -static const u64 kDefaultShort64bitShadowOffset = 0x7FFF8000; // < 2G. +static const u64 kDefaultShort64bitShadowOffset = + 0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G. static const u64 kIosShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kIosShadowOffset64 = 0x120200000; static const u64 kIosSimShadowOffset32 = 1ULL << 30; @@ -136,18 +148,20 @@ static const u64 kIosSimShadowOffset64 = kDefaultShadowOffset64; static const u64 kAArch64_ShadowOffset64 = 1ULL << 36; static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; -static const u64 kPPC64_ShadowOffset64 = 1ULL << 41; +static const u64 kPPC64_ShadowOffset64 = 1ULL << 44; static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52; static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 +static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 #define SHADOW_SCALE kDefaultShadowScale - -#if SANITIZER_WORDSIZE == 32 +#if SANITIZER_FUCHSIA +# define SHADOW_OFFSET (0) +#elif SANITIZER_WORDSIZE == 32 # if SANITIZER_ANDROID -# define SHADOW_OFFSET (0) +# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # elif defined(__mips__) # define SHADOW_OFFSET kMIPS32_ShadowOffset32 # elif SANITIZER_FREEBSD @@ -178,6 +192,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 # define SHADOW_OFFSET kSystemZ_ShadowOffset64 # elif SANITIZER_FREEBSD # define SHADOW_OFFSET kFreeBSD_ShadowOffset64 +# elif SANITIZER_NETBSD +# define SHADOW_OFFSET kNetBSD_ShadowOffset64 # elif SANITIZER_MAC # define SHADOW_OFFSET kDefaultShadowOffset64 # elif defined(__mips64) @@ -189,6 +205,12 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 # endif #endif +#if SANITIZER_ANDROID && defined(__arm__) +# define ASAN_PREMAP_SHADOW 1 +#else +# define ASAN_PREMAP_SHADOW 0 +#endif + #define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) diff --git a/contrib/compiler-rt/lib/asan/asan_memory_profile.cc b/contrib/compiler-rt/lib/asan/asan_memory_profile.cc index 05846c37cb6d..603284c8c688 100644 --- a/contrib/compiler-rt/lib/asan/asan_memory_profile.cc +++ b/contrib/compiler-rt/lib/asan/asan_memory_profile.cc @@ -107,6 +107,9 @@ static void MemoryProfileCB(const SuspendedThreadsList &suspended_threads_list, __lsan::ForEachChunk(ChunkCallback, &hp); uptr *Arg = reinterpret_cast(argument); hp.Print(Arg[0], Arg[1]); + + if (Verbosity()) + __asan_print_accumulated_stats(); } } // namespace __asan diff --git a/contrib/compiler-rt/lib/asan/asan_new_delete.cc b/contrib/compiler-rt/lib/asan/asan_new_delete.cc index e68c7f3e2400..072f027addd6 100644 --- a/contrib/compiler-rt/lib/asan/asan_new_delete.cc +++ b/contrib/compiler-rt/lib/asan/asan_new_delete.cc @@ -125,77 +125,69 @@ INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) { INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); } -#endif +#endif // !SANITIZER_MAC #define OPERATOR_DELETE_BODY(type) \ GET_STACK_TRACE_FREE;\ - asan_free(ptr, &stack, type); + asan_delete(ptr, 0, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE(type) \ + GET_STACK_TRACE_FREE;\ + asan_delete(ptr, size, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_ALIGN(type) \ + GET_STACK_TRACE_FREE;\ + asan_delete(ptr, 0, static_cast(align), &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \ + GET_STACK_TRACE_FREE;\ + asan_delete(ptr, size, static_cast(align), &stack, type); #if !SANITIZER_MAC CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr) NOEXCEPT +{ OPERATOR_DELETE_BODY(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr) NOEXCEPT +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW); -} +void operator delete(void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW_BR); -} +void operator delete[](void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::align_val_t) NOEXCEPT { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW); -} +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} +void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW); -} +void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT { - GET_STACK_TRACE_FREE; - asan_sized_free(ptr, size, &stack, FROM_NEW_BR); -} +void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); } #else // SANITIZER_MAC -INTERCEPTOR(void, _ZdlPv, void *ptr) { - OPERATOR_DELETE_BODY(FROM_NEW); -} -INTERCEPTOR(void, _ZdaPv, void *ptr) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} -INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW); -} -INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(FROM_NEW_BR); -} -#endif +INTERCEPTOR(void, _ZdlPv, void *ptr) +{ OPERATOR_DELETE_BODY(FROM_NEW); } +INTERCEPTOR(void, _ZdaPv, void *ptr) +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } +INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW); } +INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FROM_NEW_BR); } +#endif // !SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.cc b/contrib/compiler-rt/lib/asan/asan_poisoning.cc index abb75ab3bf92..c3a82aa0049e 100644 --- a/contrib/compiler-rt/lib/asan/asan_poisoning.cc +++ b/contrib/compiler-rt/lib/asan/asan_poisoning.cc @@ -217,7 +217,7 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) { uptr __bad = __asan_region_is_poisoned(__p, __size); \ __asan_report_error(pc, bp, sp, __bad, isWrite, __size, 0);\ } \ - } while (false); \ + } while (false) extern "C" SANITIZER_INTERFACE_ATTRIBUTE diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.h b/contrib/compiler-rt/lib/asan/asan_poisoning.h index cc3281e08a71..1e00070bcf63 100644 --- a/contrib/compiler-rt/lib/asan/asan_poisoning.h +++ b/contrib/compiler-rt/lib/asan/asan_poisoning.h @@ -46,8 +46,11 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, // for mapping shadow and zeroing out pages doesn't "just work", so we should // probably provide higher-level interface for these operations. // For now, just memset on Windows. - if (value || - SANITIZER_WINDOWS == 1 || + if (value || SANITIZER_WINDOWS == 1 || + // TODO(mcgrathr): Fuchsia doesn't allow the shadow mapping to be + // changed at all. It doesn't currently have an efficient means + // to zero a bunch of pages, but maybe we should add one. + SANITIZER_FUCHSIA == 1 || shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg); } else { diff --git a/contrib/compiler-rt/lib/asan/asan_posix.cc b/contrib/compiler-rt/lib/asan/asan_posix.cc index added746ace8..17c28b0aea2a 100644 --- a/contrib/compiler-rt/lib/asan/asan_posix.cc +++ b/contrib/compiler-rt/lib/asan/asan_posix.cc @@ -25,7 +25,6 @@ #include "sanitizer_common/sanitizer_procmaps.h" #include -#include #include #include #include @@ -34,58 +33,9 @@ namespace __asan { void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { - ScopedDeadlySignal signal_scope(GetCurrentThread()); - int code = (int)((siginfo_t*)siginfo)->si_code; - // Write the first message using fd=2, just in case. - // It may actually fail to write in case stderr is closed. - internal_write(2, "ASAN:DEADLYSIGNAL\n", 18); - SignalContext sig = SignalContext::Create(siginfo, context); - - // Access at a reasonable offset above SP, or slightly below it (to account - // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is - // probably a stack overflow. -#ifdef __s390__ - // On s390, the fault address in siginfo points to start of the page, not - // to the precise word that was accessed. Mask off the low bits of sp to - // take it into account. - bool IsStackAccess = sig.addr >= (sig.sp & ~0xFFF) && - sig.addr < sig.sp + 0xFFFF; -#else - bool IsStackAccess = sig.addr + 512 > sig.sp && sig.addr < sig.sp + 0xFFFF; -#endif - -#if __powerpc__ - // Large stack frames can be allocated with e.g. - // lis r0,-10000 - // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000 - // If the store faults then sp will not have been updated, so test above - // will not work, because the fault address will be more than just "slightly" - // below sp. - if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) { - u32 inst = *(unsigned *)sig.pc; - u32 ra = (inst >> 16) & 0x1F; - u32 opcd = inst >> 26; - u32 xo = (inst >> 1) & 0x3FF; - // Check for store-with-update to sp. The instructions we accept are: - // stbu rs,d(ra) stbux rs,ra,rb - // sthu rs,d(ra) sthux rs,ra,rb - // stwu rs,d(ra) stwux rs,ra,rb - // stdu rs,ds(ra) stdux rs,ra,rb - // where ra is r1 (the stack pointer). - if (ra == 1 && - (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 || - (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181)))) - IsStackAccess = true; - } -#endif // __powerpc__ - - // We also check si_code to filter out SEGV caused by something else other - // then hitting the guard page or unmapped memory, like, for example, - // unaligned memory access. - if (IsStackAccess && (code == si_SEGV_MAPERR || code == si_SEGV_ACCERR)) - ReportStackOverflow(sig); - else - ReportDeadlySignal(signo, sig); + StartReportDeadlySignal(); + SignalContext sig(siginfo, context); + ReportDeadlySignal(sig); } // ---------------------- TSD ---------------- {{{1 diff --git a/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc b/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc new file mode 100644 index 000000000000..229eba99fe0e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc @@ -0,0 +1,79 @@ +//===-- asan_premap_shadow.cc ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Reserve shadow memory with an ifunc resolver. +//===----------------------------------------------------------------------===// + +#include "asan_mapping.h" + +#if ASAN_PREMAP_SHADOW + +#include "asan_premap_shadow.h" +#include "sanitizer_common/sanitizer_posix.h" + +namespace __asan { + +// The code in this file needs to run in an unrelocated binary. It may not +// access any external symbol, including its own non-hidden globals. + +// Conservative upper limit. +uptr PremapShadowSize() { + uptr granularity = GetMmapGranularity(); + return RoundUpTo(GetMaxVirtualAddress() >> SHADOW_SCALE, granularity); +} + +// Returns an address aligned to 8 pages, such that one page on the left and +// PremapShadowSize() bytes on the right of it are mapped r/o. +uptr PremapShadow() { + uptr granularity = GetMmapGranularity(); + uptr alignment = granularity * 8; + uptr left_padding = granularity; + uptr shadow_size = PremapShadowSize(); + uptr map_size = shadow_size + left_padding + alignment; + + uptr map_start = (uptr)MmapNoAccess(map_size); + CHECK_NE(map_start, ~(uptr)0); + + uptr shadow_start = RoundUpTo(map_start + left_padding, alignment); + uptr shadow_end = shadow_start + shadow_size; + internal_munmap(reinterpret_cast(map_start), + shadow_start - left_padding - map_start); + internal_munmap(reinterpret_cast(shadow_end), + map_start + map_size - shadow_end); + return shadow_start; +} + +bool PremapShadowFailed() { + uptr shadow = reinterpret_cast(&__asan_shadow); + uptr resolver = reinterpret_cast(&__asan_premap_shadow); + // shadow == resolver is how Android KitKat and older handles ifunc. + // shadow == 0 just in case. + if (shadow == 0 || shadow == resolver) + return true; + return false; +} +} // namespace __asan + +extern "C" { +decltype(__asan_shadow)* __asan_premap_shadow() { + // The resolver may be called multiple times. Map the shadow just once. + static uptr premapped_shadow = 0; + if (!premapped_shadow) premapped_shadow = __asan::PremapShadow(); + return reinterpret_cast(premapped_shadow); +} + +// __asan_shadow is a "function" that has the same address as the first byte of +// the shadow mapping. +INTERFACE_ATTRIBUTE __attribute__((ifunc("__asan_premap_shadow"))) void +__asan_shadow(); +} + +#endif // ASAN_PREMAP_SHADOW diff --git a/contrib/compiler-rt/lib/asan/asan_premap_shadow.h b/contrib/compiler-rt/lib/asan/asan_premap_shadow.h new file mode 100644 index 000000000000..41acbdbbb69f --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_premap_shadow.h @@ -0,0 +1,30 @@ +//===-- asan_mapping.h ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Premap shadow range with an ifunc resolver. +//===----------------------------------------------------------------------===// + + +#ifndef ASAN_PREMAP_SHADOW_H +#define ASAN_PREMAP_SHADOW_H + +#if ASAN_PREMAP_SHADOW +namespace __asan { +// Conservative upper limit. +uptr PremapShadowSize(); +bool PremapShadowFailed(); +} +#endif + +extern "C" INTERFACE_ATTRIBUTE void __asan_shadow(); +extern "C" decltype(__asan_shadow)* __asan_premap_shadow(); + +#endif // ASAN_PREMAP_SHADOW_H diff --git a/contrib/compiler-rt/lib/asan/asan_report.cc b/contrib/compiler-rt/lib/asan/asan_report.cc index 2e477f258b8d..e3bc02994efd 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.cc +++ b/contrib/compiler-rt/lib/asan/asan_report.cc @@ -60,9 +60,8 @@ void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte, bool in_shadow, const char *after) { Decorator d; str->append("%s%s%x%x%s%s", before, - in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), - byte >> 4, byte & 15, - in_shadow ? d.EndShadowByte() : d.EndMemoryByte(), after); + in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), byte >> 4, + byte & 15, d.Default(), after); } static void PrintZoneForPointer(uptr ptr, uptr zone_ptr, @@ -123,53 +122,15 @@ bool ParseFrameDescription(const char *frame_descr, // immediately after printing error report. class ScopedInErrorReport { public: - explicit ScopedInErrorReport(bool fatal = false) { - halt_on_error_ = fatal || flags()->halt_on_error; - - if (lock_.TryLock()) { - StartReporting(); - return; - } - - // ASan found two bugs in different threads simultaneously. - - u32 current_tid = GetCurrentTidOrInvalid(); - if (reporting_thread_tid_ == current_tid || - reporting_thread_tid_ == kInvalidTid) { - // This is either asynch signal or nested error during error reporting. - // Fail simple to avoid deadlocks in Report(). - - // Can't use Report() here because of potential deadlocks - // in nested signal handlers. - const char msg[] = "AddressSanitizer: nested bug in the same thread, " - "aborting.\n"; - WriteToFile(kStderrFd, msg, sizeof(msg)); - - internal__exit(common_flags()->exitcode); - } - - if (halt_on_error_) { - // Do not print more than one report, otherwise they will mix up. - // Error reporting functions shouldn't return at this situation, as - // they are effectively no-returns. - - Report("AddressSanitizer: while reporting a bug found another one. " - "Ignoring.\n"); - - // Sleep long enough to make sure that the thread which started - // to print an error report will finish doing it. - SleepForSeconds(Max(100, flags()->sleep_before_dying + 1)); - - // If we're still not dead for some reason, use raw _exit() instead of - // Die() to bypass any additional checks. - internal__exit(common_flags()->exitcode); - } else { - // The other thread will eventually finish reporting - // so it's safe to wait - lock_.Lock(); - } - - StartReporting(); + explicit ScopedInErrorReport(bool fatal = false) + : halt_on_error_(fatal || flags()->halt_on_error) { + // Make sure the registry and sanitizer report mutexes are locked while + // we're printing an error report. + // We can lock them only here to avoid self-deadlock in case of + // recursive reports. + asanThreadRegistry().Lock(); + Printf( + "=================================================================\n"); } ~ScopedInErrorReport() { @@ -217,9 +178,6 @@ class ScopedInErrorReport { if (!halt_on_error_) internal_memset(¤t_error_, 0, sizeof(current_error_)); - CommonSanitizerReportMutex.Unlock(); - reporting_thread_tid_ = kInvalidTid; - lock_.Unlock(); if (halt_on_error_) { Report("ABORTING\n"); Die(); @@ -237,39 +195,18 @@ class ScopedInErrorReport { } private: - void StartReporting() { - // Make sure the registry and sanitizer report mutexes are locked while - // we're printing an error report. - // We can lock them only here to avoid self-deadlock in case of - // recursive reports. - asanThreadRegistry().Lock(); - CommonSanitizerReportMutex.Lock(); - reporting_thread_tid_ = GetCurrentTidOrInvalid(); - Printf("====================================================" - "=============\n"); - } - - static StaticSpinMutex lock_; - static u32 reporting_thread_tid_; + ScopedErrorReportLock error_report_lock_; // Error currently being reported. This enables the destructor to interact // with the debugger and point it to an error description. static ErrorDescription current_error_; bool halt_on_error_; }; -StaticSpinMutex ScopedInErrorReport::lock_; -u32 ScopedInErrorReport::reporting_thread_tid_ = kInvalidTid; ErrorDescription ScopedInErrorReport::current_error_; -void ReportStackOverflow(const SignalContext &sig) { +void ReportDeadlySignal(const SignalContext &sig) { ScopedInErrorReport in_report(/*fatal*/ true); - ErrorStackOverflow error(GetCurrentTidOrInvalid(), sig); - in_report.ReportError(error); -} - -void ReportDeadlySignal(int signo, const SignalContext &sig) { - ScopedInErrorReport in_report(/*fatal*/ true); - ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig, signo); + ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig); in_report.ReportError(error); } @@ -279,11 +216,12 @@ void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack) { in_report.ReportError(error); } -void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, +void ReportNewDeleteTypeMismatch(uptr addr, uptr delete_size, + uptr delete_alignment, BufferedStackTrace *free_stack) { ScopedInErrorReport in_report; - ErrorNewDeleteSizeMismatch error(GetCurrentTidOrInvalid(), free_stack, addr, - delete_size); + ErrorNewDeleteTypeMismatch error(GetCurrentTidOrInvalid(), free_stack, addr, + delete_size, delete_alignment); in_report.ReportError(error); } @@ -360,17 +298,58 @@ static NOINLINE void ReportInvalidPointerPair(uptr pc, uptr bp, uptr sp, in_report.ReportError(error); } +static bool IsInvalidPointerPair(uptr a1, uptr a2) { + if (a1 == a2) + return false; + + // 256B in shadow memory can be iterated quite fast + static const uptr kMaxOffset = 2048; + + uptr left = a1 < a2 ? a1 : a2; + uptr right = a1 < a2 ? a2 : a1; + uptr offset = right - left; + if (offset <= kMaxOffset) + return __asan_region_is_poisoned(left, offset); + + AsanThread *t = GetCurrentThread(); + + // check whether left is a stack memory pointer + if (uptr shadow_offset1 = t->GetStackVariableShadowStart(left)) { + uptr shadow_offset2 = t->GetStackVariableShadowStart(right); + return shadow_offset2 == 0 || shadow_offset1 != shadow_offset2; + } + + // check whether left is a heap memory address + HeapAddressDescription hdesc1, hdesc2; + if (GetHeapAddressInformation(left, 0, &hdesc1) && + hdesc1.chunk_access.access_type == kAccessTypeInside) + return !GetHeapAddressInformation(right, 0, &hdesc2) || + hdesc2.chunk_access.access_type != kAccessTypeInside || + hdesc1.chunk_access.chunk_begin != hdesc2.chunk_access.chunk_begin; + + // check whether left is an address of a global variable + GlobalAddressDescription gdesc1, gdesc2; + if (GetGlobalAddressInformation(left, 0, &gdesc1)) + return !GetGlobalAddressInformation(right - 1, 0, &gdesc2) || + !gdesc1.PointsInsideTheSameVariable(gdesc2); + + if (t->GetStackVariableShadowStart(right) || + GetHeapAddressInformation(right, 0, &hdesc2) || + GetGlobalAddressInformation(right - 1, 0, &gdesc2)) + return true; + + // At this point we know nothing about both a1 and a2 addresses. + return false; +} + static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) { if (!flags()->detect_invalid_pointer_pairs) return; uptr a1 = reinterpret_cast(p1); uptr a2 = reinterpret_cast(p2); - AsanChunkView chunk1 = FindHeapChunkByAddress(a1); - AsanChunkView chunk2 = FindHeapChunkByAddress(a2); - bool valid1 = chunk1.IsAllocated(); - bool valid2 = chunk2.IsAllocated(); - if (!valid1 || !valid2 || !chunk1.Eq(chunk2)) { + + if (IsInvalidPointerPair(a1, a2)) { GET_CALLER_PC_BP_SP; - return ReportInvalidPointerPair(pc, bp, sp, a1, a2); + ReportInvalidPointerPair(pc, bp, sp, a1, a2); } } // ----------------------- Mac-specific reports ----------------- {{{1 diff --git a/contrib/compiler-rt/lib/asan/asan_report.h b/contrib/compiler-rt/lib/asan/asan_report.h index 5a3533a319af..f2cdad47ee81 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.h +++ b/contrib/compiler-rt/lib/asan/asan_report.h @@ -46,9 +46,9 @@ bool ParseFrameDescription(const char *frame_descr, // Different kinds of error reports. void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write, uptr access_size, u32 exp, bool fatal); -void ReportStackOverflow(const SignalContext &sig); -void ReportDeadlySignal(int signo, const SignalContext &sig); -void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, +void ReportDeadlySignal(const SignalContext &sig); +void ReportNewDeleteTypeMismatch(uptr addr, uptr delete_size, + uptr delete_alignment, BufferedStackTrace *free_stack); void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack); void ReportFreeNotMalloced(uptr addr, BufferedStackTrace *free_stack); diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc index 5ae3568ae04a..21fd0e240708 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtl.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc @@ -84,26 +84,6 @@ void ShowStatsAndAbort() { Die(); } -// ---------------------- mmap -------------------- {{{1 -// Reserve memory range [beg, end]. -// We need to use inclusive range because end+1 may not be representable. -void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { - CHECK_EQ((beg % GetMmapGranularity()), 0); - CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); - uptr size = end - beg + 1; - DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. - void *res = MmapFixedNoReserve(beg, size, name); - if (res != (void*)beg) { - Report("ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " - "Perhaps you're using ulimit -v\n", size); - Abort(); - } - if (common_flags()->no_huge_pages_for_shadow) - NoHugePagesInRegion(beg, size); - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(beg, size); -} - // --------------- LowLevelAllocateCallbac ---------- {{{1 static void OnLowLevelAllocate(uptr ptr, uptr size) { PoisonShadow(ptr, size, kAsanInternalHeapMagic); @@ -327,7 +307,7 @@ static void asan_atexit() { static void InitializeHighMemEnd() { #if !ASAN_FIXED_MAPPING - kHighMemEnd = GetMaxVirtualAddress(); + kHighMemEnd = GetMaxUserVirtualAddress(); // Increase kHighMemEnd to make sure it's properly // aligned together with kHighMemBeg: kHighMemEnd |= SHADOW_GRANULARITY * GetMmapGranularity() - 1; @@ -335,46 +315,7 @@ static void InitializeHighMemEnd() { CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0); } -static void ProtectGap(uptr addr, uptr size) { - if (!flags()->protect_shadow_gap) { - // The shadow gap is unprotected, so there is a chance that someone - // is actually using this memory. Which means it needs a shadow... - uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached()); - uptr GapShadowEnd = - RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1; - if (Verbosity()) - Printf("protect_shadow_gap=0:" - " not protecting shadow gap, allocating gap's shadow\n" - "|| `[%p, %p]` || ShadowGap's shadow ||\n", GapShadowBeg, - GapShadowEnd); - ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd, - "unprotected gap shadow"); - return; - } - void *res = MmapFixedNoAccess(addr, size, "shadow gap"); - if (addr == (uptr)res) - return; - // A few pages at the start of the address space can not be protected. - // But we really want to protect as much as possible, to prevent this memory - // being returned as a result of a non-FIXED mmap(). - if (addr == kZeroBaseShadowStart) { - uptr step = GetMmapGranularity(); - while (size > step && addr < kZeroBaseMaxShadowStart) { - addr += step; - size -= step; - void *res = MmapFixedNoAccess(addr, size, "shadow gap"); - if (addr == (uptr)res) - return; - } - } - - Report("ERROR: Failed to protect the shadow gap. " - "ASan cannot proceed correctly. ABORTING.\n"); - DumpProcessMap(); - Die(); -} - -static void PrintAddressSpaceLayout() { +void PrintAddressSpaceLayout() { Printf("|| `[%p, %p]` || HighMem ||\n", (void*)kHighMemBeg, (void*)kHighMemEnd); Printf("|| `[%p, %p]` || HighShadow ||\n", @@ -426,71 +367,6 @@ static void PrintAddressSpaceLayout() { kHighShadowBeg > kMidMemEnd); } -static void InitializeShadowMemory() { - // Set the shadow memory address to uninitialized. - __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; - - uptr shadow_start = kLowShadowBeg; - // Detect if a dynamic shadow address must used and find a available location - // when necessary. When dynamic address is used, the macro |kLowShadowBeg| - // expands to |__asan_shadow_memory_dynamic_address| which is - // |kDefaultShadowSentinel|. - if (shadow_start == kDefaultShadowSentinel) { - __asan_shadow_memory_dynamic_address = 0; - CHECK_EQ(0, kLowShadowBeg); - shadow_start = FindDynamicShadowStart(); - } - // Update the shadow memory address (potentially) used by instrumentation. - __asan_shadow_memory_dynamic_address = shadow_start; - - if (kLowShadowBeg) - shadow_start -= GetMmapGranularity(); - bool full_shadow_is_available = - MemoryRangeIsAvailable(shadow_start, kHighShadowEnd); - -#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \ - !ASAN_FIXED_MAPPING - if (!full_shadow_is_available) { - kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0; - kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0; - } -#endif - - if (Verbosity()) PrintAddressSpaceLayout(); - - if (full_shadow_is_available) { - // mmap the low shadow plus at least one page at the left. - if (kLowShadowBeg) - ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); - // mmap the high shadow. - ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); - // protect the gap. - ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); - CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); - } else if (kMidMemBeg && - MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) && - MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) { - CHECK(kLowShadowBeg != kLowShadowEnd); - // mmap the low shadow plus at least one page at the left. - ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); - // mmap the mid shadow. - ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow"); - // mmap the high shadow. - ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); - // protect the gaps. - ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); - ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1); - ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1); - } else { - Report("Shadow memory range interleaves with an existing memory mapping. " - "ASan cannot proceed correctly. ABORTING.\n"); - Report("ASan shadow was supposed to be located in the [%p-%p] range.\n", - shadow_start, kHighShadowEnd); - DumpProcessMap(); - Die(); - } -} - static void AsanInitInternal() { if (LIKELY(asan_inited)) return; SanitizerToolName = "AddressSanitizer"; @@ -531,6 +407,7 @@ static void AsanInitInternal() { MaybeReexec(); // Setup internal allocator callback. + SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); SetLowLevelAllocateCallback(OnLowLevelAllocate); InitializeAsanInterceptors(); @@ -575,20 +452,18 @@ static void AsanInitInternal() { InitTlsSize(); // Create main thread. - AsanThread *main_thread = AsanThread::Create( - /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0, - /* stack */ nullptr, /* detached */ true); + AsanThread *main_thread = CreateMainThread(); CHECK_EQ(0, main_thread->tid()); - SetCurrentThread(main_thread); - main_thread->ThreadStart(internal_getpid(), - /* signal_thread_is_registered */ nullptr); force_interface_symbols(); // no-op. SanitizerInitializeUnwinder(); if (CAN_SANITIZE_LEAKS) { __lsan::InitCommonLsan(); if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) { - Atexit(__lsan::DoLeakCheck); + if (flags()->halt_on_error) + Atexit(__lsan::DoLeakCheck); + else + Atexit(__lsan::DoRecoverableLeakCheckVoid); } } @@ -608,6 +483,11 @@ static void AsanInitInternal() { } VReport(1, "AddressSanitizer Init done\n"); + + if (flags()->sleep_after_init) { + Report("Sleeping for %d second(s)\n", flags()->sleep_after_init); + SleepForSeconds(flags()->sleep_after_init); + } } // Initialize as requested from some part of ASan runtime library (interceptors, @@ -647,6 +527,7 @@ void NOINLINE __asan_handle_no_return() { top = curr_thread->stack_top(); bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1); } else { + CHECK(!SANITIZER_FUCHSIA); // If we haven't seen this thread, try asking the OS for stack bounds. uptr tls_addr, tls_size, stack_size; GetThreadStackAndTls(/*main=*/false, &bottom, &stack_size, &tls_addr, diff --git a/contrib/compiler-rt/lib/asan/asan_scariness_score.h b/contrib/compiler-rt/lib/asan/asan_scariness_score.h index 7f1571416fd5..7f095dd29f29 100644 --- a/contrib/compiler-rt/lib/asan/asan_scariness_score.h +++ b/contrib/compiler-rt/lib/asan/asan_scariness_score.h @@ -47,7 +47,7 @@ struct ScarinessScoreBase { }; int GetScore() const { return score; } const char *GetDescription() const { return descr; } - void Print() { + void Print() const { if (score && flags()->print_scariness) Printf("SCARINESS: %d (%s)\n", score, descr); } diff --git a/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc b/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc new file mode 100644 index 000000000000..b3cf0b5c1ce2 --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc @@ -0,0 +1,165 @@ +//===-- asan_shadow_setup.cc ----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Set up the shadow memory. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" + +// asan_fuchsia.cc has its own InitializeShadowMemory implementation. +#if !SANITIZER_FUCHSIA + +#include "asan_internal.h" +#include "asan_mapping.h" + +namespace __asan { + +// ---------------------- mmap -------------------- {{{1 +// Reserve memory range [beg, end]. +// We need to use inclusive range because end+1 may not be representable. +void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { + CHECK_EQ((beg % GetMmapGranularity()), 0); + CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); + uptr size = end - beg + 1; + DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. + void *res = MmapFixedNoReserve(beg, size, name); + if (res != (void *)beg) { + Report( + "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " + "Perhaps you're using ulimit -v\n", + size); + Abort(); + } + if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); +} + +static void ProtectGap(uptr addr, uptr size) { + if (!flags()->protect_shadow_gap) { + // The shadow gap is unprotected, so there is a chance that someone + // is actually using this memory. Which means it needs a shadow... + uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached()); + uptr GapShadowEnd = + RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1; + if (Verbosity()) + Printf( + "protect_shadow_gap=0:" + " not protecting shadow gap, allocating gap's shadow\n" + "|| `[%p, %p]` || ShadowGap's shadow ||\n", + GapShadowBeg, GapShadowEnd); + ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd, + "unprotected gap shadow"); + return; + } + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + // A few pages at the start of the address space can not be protected. + // But we really want to protect as much as possible, to prevent this memory + // being returned as a result of a non-FIXED mmap(). + if (addr == kZeroBaseShadowStart) { + uptr step = GetMmapGranularity(); + while (size > step && addr < kZeroBaseMaxShadowStart) { + addr += step; + size -= step; + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + } + } + + Report( + "ERROR: Failed to protect the shadow gap. " + "ASan cannot proceed correctly. ABORTING.\n"); + DumpProcessMap(); + Die(); +} + +static void MaybeReportLinuxPIEBug() { +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__aarch64__)) + Report("This might be related to ELF_ET_DYN_BASE change in Linux 4.12.\n"); + Report( + "See https://github.com/google/sanitizers/issues/856 for possible " + "workarounds.\n"); +#endif +} + +void InitializeShadowMemory() { + // Set the shadow memory address to uninitialized. + __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; + + uptr shadow_start = kLowShadowBeg; + // Detect if a dynamic shadow address must used and find a available location + // when necessary. When dynamic address is used, the macro |kLowShadowBeg| + // expands to |__asan_shadow_memory_dynamic_address| which is + // |kDefaultShadowSentinel|. + bool full_shadow_is_available = false; + if (shadow_start == kDefaultShadowSentinel) { + __asan_shadow_memory_dynamic_address = 0; + CHECK_EQ(0, kLowShadowBeg); + shadow_start = FindDynamicShadowStart(); + if (SANITIZER_LINUX) full_shadow_is_available = true; + } + // Update the shadow memory address (potentially) used by instrumentation. + __asan_shadow_memory_dynamic_address = shadow_start; + + if (kLowShadowBeg) shadow_start -= GetMmapGranularity(); + + if (!full_shadow_is_available) + full_shadow_is_available = + MemoryRangeIsAvailable(shadow_start, kHighShadowEnd); + +#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \ + !ASAN_FIXED_MAPPING + if (!full_shadow_is_available) { + kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0; + kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0; + } +#endif + + if (Verbosity()) PrintAddressSpaceLayout(); + + if (full_shadow_is_available) { + // mmap the low shadow plus at least one page at the left. + if (kLowShadowBeg) + ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); + // mmap the high shadow. + ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); + // protect the gap. + ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); + CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); + } else if (kMidMemBeg && + MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) && + MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) { + CHECK(kLowShadowBeg != kLowShadowEnd); + // mmap the low shadow plus at least one page at the left. + ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); + // mmap the mid shadow. + ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow"); + // mmap the high shadow. + ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); + // protect the gaps. + ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); + ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1); + ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1); + } else { + Report( + "Shadow memory range interleaves with an existing memory mapping. " + "ASan cannot proceed correctly. ABORTING.\n"); + Report("ASan shadow was supposed to be located in the [%p-%p] range.\n", + shadow_start, kHighShadowEnd); + MaybeReportLinuxPIEBug(); + DumpProcessMap(); + Die(); + } +} + +} // namespace __asan + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/asan/asan_stack.h b/contrib/compiler-rt/lib/asan/asan_stack.h index cc95e0f30a33..8e9df888d798 100644 --- a/contrib/compiler-rt/lib/asan/asan_stack.h +++ b/contrib/compiler-rt/lib/asan/asan_stack.h @@ -31,9 +31,8 @@ u32 GetMallocContextSize(); // The pc will be in the position 0 of the resulting stack trace. // The bp may refer to the current frame or to the caller's frame. ALWAYS_INLINE -void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, - uptr pc, uptr bp, void *context, - bool fast) { +void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, + void *context, bool fast) { #if SANITIZER_WINDOWS stack->Unwind(max_depth, pc, bp, context, 0, 0, fast); #else @@ -41,10 +40,6 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, stack->size = 0; if (LIKELY(asan_inited)) { if ((t = GetCurrentThread()) && !t->isUnwinding()) { - // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace() - // yields the call stack of the signal's handler and not of the code - // that raised the signal (as it does on Linux). - if (SANITIZER_FREEBSD && t->isInDeadlySignal()) fast = true; uptr stack_top = t->stack_top(); uptr stack_bottom = t->stack_bottom(); ScopedUnwinding unwind_scope(t); @@ -66,32 +61,29 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, // as early as possible (in functions exposed to the user), as we generally // don't want stack trace to contain functions from ASan internals. -#define GET_STACK_TRACE(max_size, fast) \ - BufferedStackTrace stack; \ - if (max_size <= 2) { \ - stack.size = max_size; \ - if (max_size > 0) { \ - stack.top_frame_bp = GET_CURRENT_FRAME(); \ - stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ - if (max_size > 1) \ - stack.trace_buffer[1] = GET_CALLER_PC(); \ - } \ - } else { \ - GetStackTraceWithPcBpAndContext(&stack, max_size, \ - StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), 0, fast); \ +#define GET_STACK_TRACE(max_size, fast) \ + BufferedStackTrace stack; \ + if (max_size <= 2) { \ + stack.size = max_size; \ + if (max_size > 0) { \ + stack.top_frame_bp = GET_CURRENT_FRAME(); \ + stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ + if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \ + } \ + } else { \ + GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), 0, fast); \ } -#define GET_STACK_TRACE_FATAL(pc, bp) \ - BufferedStackTrace stack; \ - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, 0, \ - common_flags()->fast_unwind_on_fatal) +#define GET_STACK_TRACE_FATAL(pc, bp) \ + BufferedStackTrace stack; \ + GetStackTrace(&stack, kStackTraceMax, pc, bp, 0, \ + common_flags()->fast_unwind_on_fatal) -#define GET_STACK_TRACE_SIGNAL(sig) \ - BufferedStackTrace stack; \ - GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, \ - (sig).pc, (sig).bp, (sig).context, \ - common_flags()->fast_unwind_on_fatal) +#define GET_STACK_TRACE_SIGNAL(sig) \ + BufferedStackTrace stack; \ + GetStackTrace(&stack, kStackTraceMax, (sig).pc, (sig).bp, (sig).context, \ + common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_FATAL_HERE \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) diff --git a/contrib/compiler-rt/lib/asan/asan_thread.cc b/contrib/compiler-rt/lib/asan/asan_thread.cc index b1a0d9a3b37f..ad81512dff08 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.cc +++ b/contrib/compiler-rt/lib/asan/asan_thread.cc @@ -27,11 +27,6 @@ namespace __asan { // AsanThreadContext implementation. -struct CreateThreadContextArgs { - AsanThread *thread; - StackTrace *stack; -}; - void AsanThreadContext::OnCreated(void *arg) { CreateThreadContextArgs *args = static_cast(arg); if (args->stack) @@ -88,7 +83,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg, AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__); thread->start_routine_ = start_routine; thread->arg_ = arg; - CreateThreadContextArgs args = { thread, stack }; + AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; asanThreadRegistry().CreateThread(*reinterpret_cast(thread), detached, parent_tid, &args); @@ -223,12 +218,12 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { return nullptr; } -void AsanThread::Init() { +void AsanThread::Init(const InitOptions *options) { next_stack_top_ = next_stack_bottom_ = 0; atomic_store(&stack_switching_, false, memory_order_release); fake_stack_ = nullptr; // Will be initialized lazily if needed. CHECK_EQ(this->stack_size(), 0U); - SetThreadStackAndTls(); + SetThreadStackAndTls(options); CHECK_GT(this->stack_size(), 0U); CHECK(AddrIsInMem(stack_bottom_)); CHECK(AddrIsInMem(stack_top_ - 1)); @@ -239,6 +234,10 @@ void AsanThread::Init() { &local); } +// Fuchsia doesn't use ThreadStart. +// asan_fuchsia.c defines CreateMainThread and SetThreadStackAndTls. +#if !SANITIZER_FUCHSIA + thread_return_t AsanThread::ThreadStart( tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) { Init(); @@ -270,7 +269,21 @@ thread_return_t AsanThread::ThreadStart( return res; } -void AsanThread::SetThreadStackAndTls() { +AsanThread *CreateMainThread() { + AsanThread *main_thread = AsanThread::Create( + /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0, + /* stack */ nullptr, /* detached */ true); + SetCurrentThread(main_thread); + main_thread->ThreadStart(internal_getpid(), + /* signal_thread_is_registered */ nullptr); + return main_thread; +} + +// This implementation doesn't use the argument, which is just passed down +// from the caller of Init (which see, above). It's only there to support +// OS-specific implementations that need more information passed through. +void AsanThread::SetThreadStackAndTls(const InitOptions *options) { + DCHECK_EQ(options, nullptr); uptr tls_size = 0; uptr stack_size = 0; GetThreadStackAndTls(tid() == 0, const_cast(&stack_bottom_), @@ -283,6 +296,8 @@ void AsanThread::SetThreadStackAndTls() { CHECK(AddrIsInStack((uptr)&local)); } +#endif // !SANITIZER_FUCHSIA + void AsanThread::ClearShadowForThreadStackAndTLS() { PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) @@ -302,7 +317,7 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, access->frame_descr = (const char *)((uptr*)bottom)[1]; return true; } - uptr aligned_addr = addr & ~(SANITIZER_WORDSIZE/8 - 1); // align addr. + uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY); u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); u8 *shadow_bottom = (u8*)MemToShadow(bottom); @@ -331,6 +346,29 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, return true; } +uptr AsanThread::GetStackVariableShadowStart(uptr addr) { + uptr bottom = 0; + if (AddrIsInStack(addr)) { + bottom = stack_bottom(); + } else if (has_fake_stack()) { + bottom = fake_stack()->AddrIsInFakeStack(addr); + CHECK(bottom); + } else + return 0; + + uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. + u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); + u8 *shadow_bottom = (u8*)MemToShadow(bottom); + + while (shadow_ptr >= shadow_bottom && + (*shadow_ptr != kAsanStackLeftRedzoneMagic && + *shadow_ptr != kAsanStackMidRedzoneMagic && + *shadow_ptr != kAsanStackRightRedzoneMagic)) + shadow_ptr--; + + return (uptr)shadow_ptr + 1; +} + bool AsanThread::AddrIsInStack(uptr addr) { const auto bounds = GetStackBounds(); return addr >= bounds.bottom && addr < bounds.top; diff --git a/contrib/compiler-rt/lib/asan/asan_thread.h b/contrib/compiler-rt/lib/asan/asan_thread.h index 424f9e68dfea..66091921101f 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.h +++ b/contrib/compiler-rt/lib/asan/asan_thread.h @@ -49,6 +49,11 @@ class AsanThreadContext : public ThreadContextBase { void OnCreated(void *arg) override; void OnFinished() override; + + struct CreateThreadContextArgs { + AsanThread *thread; + StackTrace *stack; + }; }; // AsanThreadContext objects are never freed, so we need many of them. @@ -62,7 +67,9 @@ class AsanThread { static void TSDDtor(void *tsd); void Destroy(); - void Init(); // Should be called from the thread itself. + struct InitOptions; + void Init(const InitOptions *options = nullptr); + thread_return_t ThreadStart(tid_t os_id, atomic_uintptr_t *signal_thread_is_registered); @@ -83,6 +90,9 @@ class AsanThread { }; bool GetStackFrameAccessByAddr(uptr addr, StackFrameAccess *access); + // Returns a pointer to the start of the stack variable's shadow memory. + uptr GetStackVariableShadowStart(uptr addr); + bool AddrIsInStack(uptr addr); void DeleteFakeStack(int tid) { @@ -118,17 +128,15 @@ class AsanThread { bool isUnwinding() const { return unwinding_; } void setUnwinding(bool b) { unwinding_ = b; } - // True if we are in a deadly signal handler. - bool isInDeadlySignal() const { return in_deadly_signal_; } - void setInDeadlySignal(bool b) { in_deadly_signal_ = b; } - AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } AsanStats &stats() { return stats_; } private: // NOTE: There is no AsanThread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. - void SetThreadStackAndTls(); + + void SetThreadStackAndTls(const InitOptions *options); + void ClearShadowForThreadStackAndTLS(); FakeStack *AsyncSignalSafeLazyInitFakeStack(); @@ -158,7 +166,6 @@ class AsanThread { AsanThreadLocalMallocStorage malloc_storage_; AsanStats stats_; bool unwinding_; - bool in_deadly_signal_; }; // ScopedUnwinding is a scope for stacktracing member of a context @@ -173,20 +180,6 @@ class ScopedUnwinding { AsanThread *thread; }; -// ScopedDeadlySignal is a scope for handling deadly signals. -class ScopedDeadlySignal { - public: - explicit ScopedDeadlySignal(AsanThread *t) : thread(t) { - if (thread) thread->setInDeadlySignal(true); - } - ~ScopedDeadlySignal() { - if (thread) thread->setInDeadlySignal(false); - } - - private: - AsanThread *thread; -}; - // Returns a single instance of registry. ThreadRegistry &asanThreadRegistry(); diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc index 8a839d913f95..68eedd1863bc 100644 --- a/contrib/compiler-rt/lib/asan/asan_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_win.cc @@ -57,8 +57,8 @@ long __asan_unhandled_exception_filter(EXCEPTION_POINTERS *info) { // FIXME: Handle EXCEPTION_STACK_OVERFLOW here. - SignalContext sig = SignalContext::Create(exception_record, context); - ReportDeadlySignal(exception_record->ExceptionCode, sig); + SignalContext sig(exception_record, context); + ReportDeadlySignal(sig); UNREACHABLE("returned from reporting deadly signal"); } diff --git a/contrib/compiler-rt/lib/builtins/aarch64/chkstk.S b/contrib/compiler-rt/lib/builtins/aarch64/chkstk.S new file mode 100644 index 000000000000..89ec90b08a13 --- /dev/null +++ b/contrib/compiler-rt/lib/builtins/aarch64/chkstk.S @@ -0,0 +1,34 @@ +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. + +#include "../assembly.h" + +// __chkstk routine +// This routine is windows specific. +// http://msdn.microsoft.com/en-us/library/ms648426.aspx + +// This clobbers registers x16 and x17. +// Does not modify any memory or the stack pointer. + +// mov x15, #256 // Number of bytes of stack, in units of 16 byte +// bl __chkstk +// sub sp, sp, x15, lsl #4 + +#ifdef __aarch64__ + +#define PAGE_SIZE 4096 + + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__chkstk) + lsl x16, x15, #4 + mov x17, sp +1: + sub x17, x17, #PAGE_SIZE + subs x16, x16, #PAGE_SIZE + ldr xzr, [x17] + b.gt 1b + + ret +END_COMPILERRT_FUNCTION(__chkstk) + +#endif // __aarch64__ diff --git a/contrib/compiler-rt/lib/builtins/adddf3.c b/contrib/compiler-rt/lib/builtins/adddf3.c index c528e9e21f51..9a3901312e51 100644 --- a/contrib/compiler-rt/lib/builtins/adddf3.c +++ b/contrib/compiler-rt/lib/builtins/adddf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI double __adddf3(double a, double b){ } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_dadd(double a, double b) { return __adddf3(a, b); } +#else +AEABI_RTABI double __aeabi_dadd(double a, double b) COMPILER_RT_ALIAS(__adddf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/addsf3.c b/contrib/compiler-rt/lib/builtins/addsf3.c index fe570687a25e..c5c1a41c3611 100644 --- a/contrib/compiler-rt/lib/builtins/addsf3.c +++ b/contrib/compiler-rt/lib/builtins/addsf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI float __addsf3(float a, float b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_fadd(float a, float b) { return __addsf3(a, b); } +#else +AEABI_RTABI float __aeabi_fadd(float a, float b) COMPILER_RT_ALIAS(__addsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S index 3e7a8b86b739..87dd03dce94d 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S @@ -30,7 +30,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) push {r0-r3, lr} bl __aeabi_cdcmpeq_check_nan cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) beq 1f // NaN has been ruled out, so __aeabi_cdcmple can't trap mov r0, sp @@ -46,9 +46,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) pop {r0-r3, lr} // NaN has been ruled out, so __aeabi_cdcmple can't trap + // Use "it ne" + unconditional branch to guarantee a supported relocation if + // __aeabi_cdcmple is in a different section for some builds. + IT(ne) bne __aeabi_cdcmple -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) +#if defined(USE_THUMB_2) mov ip, #APSR_C msr APSR_nzcvq, ip #else @@ -78,7 +81,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple) bl __aeabi_dcmplt cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) bne 1f // Z = 0, C = 0 movs r0, #1 diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S index 1f304ffd964c..c5fee6b6a08e 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S @@ -30,7 +30,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) push {r0-r3, lr} bl __aeabi_cfcmpeq_check_nan cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) beq 1f // NaN has been ruled out, so __aeabi_cfcmple can't trap mov r0, sp @@ -46,9 +46,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) pop {r0-r3, lr} // NaN has been ruled out, so __aeabi_cfcmple can't trap + // Use "it ne" + unconditional branch to guarantee a supported relocation if + // __aeabi_cfcmple is in a different section for some builds. + IT(ne) bne __aeabi_cfcmple -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) +#if defined(USE_THUMB_2) mov ip, #APSR_C msr APSR_nzcvq, ip #else @@ -78,7 +81,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple) bl __aeabi_fcmplt cmp r0, #1 -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) bne 1f // Z = 0, C = 0 movs r0, #1 diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S index 4419929f62ac..9c9c80ab5a7b 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S @@ -19,18 +19,11 @@ #define __aeabi_idivmod __rt_sdiv #endif - .syntax unified .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__aeabi_idivmod) -#else DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod) -#endif #if defined(USE_THUMB_1) push {r0, r1, lr} bl SYMBOL_NAME(__divsi3) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S index 33ea54848b26..e86d6113760e 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S @@ -14,7 +14,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memcmp) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memcmp + pop {r7, pc} +#else b memcmp +#endif END_COMPILERRT_FUNCTION(__aeabi_memcmp) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcmp4, __aeabi_memcmp) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S index eabfa490494c..e83c5fd4dbb3 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S @@ -14,7 +14,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memcpy) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memcpy + pop {r7, pc} +#else b memcpy +#endif END_COMPILERRT_FUNCTION(__aeabi_memcpy) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcpy4, __aeabi_memcpy) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S index 1bf08c0d5b75..ee28300e46f2 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S @@ -13,7 +13,13 @@ .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memmove) +#ifdef USE_THUMB_1 + push {r7, lr} + bl memmove + pop {r7, pc} +#else b memmove +#endif END_COMPILERRT_FUNCTION(__aeabi_memmove) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memmove4, __aeabi_memmove) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S index 633f592279b5..0a678d7627e7 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S @@ -18,16 +18,29 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_memset) mov r3, r1 mov r1, r2 mov r2, r3 +#ifdef USE_THUMB_1 + push {r7, lr} + bl memset + pop {r7, pc} +#else b memset +#endif END_COMPILERRT_FUNCTION(__aeabi_memset) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset4, __aeabi_memset) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset8, __aeabi_memset) + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__aeabi_memclr) mov r2, r1 movs r1, #0 +#ifdef USE_THUMB_1 + push {r7, lr} + bl memset + pop {r7, pc} +#else b memset +#endif END_COMPILERRT_FUNCTION(__aeabi_memclr) DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memclr4, __aeabi_memclr) diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S index 37dae4a105cc..88a4a6d8bc12 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S @@ -22,16 +22,10 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__aeabi_uidivmod) -#else DEFINE_COMPILERRT_FUNCTION(__aeabi_uidivmod) -#endif -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) cmp r0, r1 bcc LOCAL_LABEL(case_denom_larger) push {r0, r1, lr} @@ -44,7 +38,7 @@ LOCAL_LABEL(case_denom_larger): movs r1, r0 movs r0, #0 JMP (lr) -#else +#else // defined(USE_THUMB_1) push { lr } sub sp, sp, #4 mov r2, sp diff --git a/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S b/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S index 4e5a579cf16a..e9db8bac7994 100644 --- a/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S @@ -11,9 +11,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE // // extern uint64_t __bswapdi2(uint64_t); @@ -21,11 +19,7 @@ // Reverse all the bytes in a 64-bit integer. // .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__bswapdi2) -#else DEFINE_COMPILERRT_FUNCTION(__bswapdi2) -#endif #if __ARM_ARCH < 6 // before armv6 does not have "rev" instruction // r2 = rev(r0) diff --git a/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S b/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S index 60342ae0f2fb..1f6eed5c1bbf 100644 --- a/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S @@ -11,9 +11,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE // // extern uint32_t __bswapsi2(uint32_t); @@ -21,11 +19,7 @@ // Reverse all the bytes in a 32-bit integer. // .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__bswapsi2) -#else DEFINE_COMPILERRT_FUNCTION(__bswapsi2) -#endif #if __ARM_ARCH < 6 // before armv6 does not have "rev" instruction eor r1, r0, r0, ror #16 diff --git a/contrib/compiler-rt/lib/builtins/arm/clzdi2.S b/contrib/compiler-rt/lib/builtins/arm/clzdi2.S index fe56a183f1b1..fc03b385cdfa 100644 --- a/contrib/compiler-rt/lib/builtins/arm/clzdi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/clzdi2.S @@ -15,17 +15,10 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif - + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__clzdi2) -#else DEFINE_COMPILERRT_FUNCTION(__clzdi2) -#endif #ifdef __ARM_FEATURE_CLZ #ifdef __ARMEB__ cmp r0, 0 diff --git a/contrib/compiler-rt/lib/builtins/arm/clzsi2.S b/contrib/compiler-rt/lib/builtins/arm/clzsi2.S index 28d43041eaf2..f2ce59c90119 100644 --- a/contrib/compiler-rt/lib/builtins/arm/clzsi2.S +++ b/contrib/compiler-rt/lib/builtins/arm/clzsi2.S @@ -15,16 +15,10 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__clzsi2) -#else DEFINE_COMPILERRT_FUNCTION(__clzsi2) -#endif #ifdef __ARM_FEATURE_CLZ clz r0, r0 JMP(lr) diff --git a/contrib/compiler-rt/lib/builtins/arm/comparesf2.S b/contrib/compiler-rt/lib/builtins/arm/comparesf2.S index 1f7031cbf665..c6c4cc067f07 100644 --- a/contrib/compiler-rt/lib/builtins/arm/comparesf2.S +++ b/contrib/compiler-rt/lib/builtins/arm/comparesf2.S @@ -40,25 +40,19 @@ #include "../assembly.h" .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ int __eqsf2(float a, float b) .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__eqsf2) -#else DEFINE_COMPILERRT_FUNCTION(__eqsf2) -#endif #if defined(COMPILER_RT_ARMHF_TARGET) vmov r0, s0 vmov r1, s1 #endif // Make copies of a and b with the sign bit shifted off the top. These will // be used to detect zeros and NaNs. -#if __ARM_ARCH_ISA_THUMB == 1 +#if defined(USE_THUMB_1) push {r6, lr} lsls r2, r0, #1 lsls r3, r1, #1 @@ -150,7 +144,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqsf2) // If a == b, then the Z flag is set, so we can get the correct final value // into r0 by simply or'ing with 1 if Z is clear. // For Thumb-1, r0 contains -1 if a < b, 0 if a > b and 0 if a == b. -#if defined(USE_THUMB_1) +#if !defined(USE_THUMB_1) it ne orrne r0, r0, #1 #endif @@ -185,11 +179,7 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2) @ int __gtsf2(float a, float b) .p2align 2 -#if defined(USE_THUMB) -DEFINE_COMPILERRT_THUMB_FUNCTION(__gtsf2) -#else DEFINE_COMPILERRT_FUNCTION(__gtsf2) -#endif // Identical to the preceding except in that we return -1 for NaN values. // Given that the two paths share so much code, one might be tempted to // unify them; however, the extra code needed to do so makes the code size @@ -263,11 +253,7 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__gesf2, __gtsf2) @ int __unordsf2(float a, float b) .p2align 2 -#if defined(USE_THUMB) -DEFINE_COMPILERRT_THUMB_FUNCTION(__unordsf2) -#else DEFINE_COMPILERRT_FUNCTION(__unordsf2) -#endif #if defined(COMPILER_RT_ARMHF_TARGET) vmov r0, s0 diff --git a/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S b/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S index 3c8359852009..8a027b741efe 100644 --- a/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S +++ b/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S @@ -23,20 +23,14 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ int __divmodsi4(int divident, int divisor, int *remainder) @ Calculate the quotient and remainder of the (signed) division. The return @ value is the quotient, the remainder is placed in the variable. .p2align 3 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__divmodsi4) -#else DEFINE_COMPILERRT_FUNCTION(__divmodsi4) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divzero) diff --git a/contrib/compiler-rt/lib/builtins/arm/divsi3.S b/contrib/compiler-rt/lib/builtins/arm/divsi3.S index ad96c9e5bfa9..19757af177eb 100644 --- a/contrib/compiler-rt/lib/builtins/arm/divsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/divsi3.S @@ -22,9 +22,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE .p2align 3 // Ok, APCS and AAPCS agree on 32 bit args, so it's safe to use the same routine. @@ -33,11 +31,7 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_idiv, __divsi3) @ int __divsi3(int divident, int divisor) @ Calculate and return the quotient of the (signed) division. -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__divsi3) -#else DEFINE_COMPILERRT_FUNCTION(__divsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1,r1 beq LOCAL_LABEL(divzero) @@ -49,14 +43,14 @@ LOCAL_LABEL(divzero): #else ESTABLISH_FRAME // Set aside the sign of the quotient. -# if __ARM_ARCH_ISA_THUMB == 1 +# if defined(USE_THUMB_1) movs r4, r0 eors r4, r1 # else eor r4, r0, r1 # endif // Take absolute value of a and b via abs(x) = (x^(x >> 31)) - (x >> 31). -# if __ARM_ARCH_ISA_THUMB == 1 +# if defined(USE_THUMB_1) asrs r2, r0, #31 asrs r3, r1, #31 eors r0, r2 diff --git a/contrib/compiler-rt/lib/builtins/arm/modsi3.S b/contrib/compiler-rt/lib/builtins/arm/modsi3.S index 0986d84945a6..be263834d7f1 100644 --- a/contrib/compiler-rt/lib/builtins/arm/modsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/modsi3.S @@ -22,19 +22,13 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ int __modsi3(int divident, int divisor) @ Calculate and return the remainder of the (signed) division. .p2align 3 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__modsi3) -#else DEFINE_COMPILERRT_FUNCTION(__modsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divzero) diff --git a/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S b/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S index 860fced63085..ee3950c9b0eb 100644 --- a/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S +++ b/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S @@ -16,9 +16,7 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ unsigned int __udivmodsi4(unsigned int divident, unsigned int divisor, @ unsigned int *remainder) @@ -26,11 +24,7 @@ @ value is the quotient, the remainder is placed in the variable. .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__udivmodsi4) -#else DEFINE_COMPILERRT_FUNCTION(__udivmodsi4) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divby0) diff --git a/contrib/compiler-rt/lib/builtins/arm/udivsi3.S b/contrib/compiler-rt/lib/builtins/arm/udivsi3.S index 82db2cbb395f..6dea27d404ff 100644 --- a/contrib/compiler-rt/lib/builtins/arm/udivsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/udivsi3.S @@ -16,9 +16,8 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + +DEFINE_CODE_STATE .p2align 2 DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_uidiv, __udivsi3) @@ -26,11 +25,7 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_uidiv, __udivsi3) @ unsigned int __udivsi3(unsigned int divident, unsigned int divisor) @ Calculate and return the quotient of the (unsigned) division. -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__udivsi3) -#else DEFINE_COMPILERRT_FUNCTION(__udivsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divby0) diff --git a/contrib/compiler-rt/lib/builtins/arm/umodsi3.S b/contrib/compiler-rt/lib/builtins/arm/umodsi3.S index 90f3e8575c77..069fad34cb9c 100644 --- a/contrib/compiler-rt/lib/builtins/arm/umodsi3.S +++ b/contrib/compiler-rt/lib/builtins/arm/umodsi3.S @@ -16,19 +16,13 @@ .syntax unified .text -#if defined(USE_THUMB_PROLOGUE) - .thumb -#endif + DEFINE_CODE_STATE @ unsigned int __umodsi3(unsigned int divident, unsigned int divisor) @ Calculate and return the remainder of the (unsigned) division. .p2align 2 -#if defined(USE_THUMB_PROLOGUE) -DEFINE_COMPILERRT_THUMB_FUNCTION(__umodsi3) -#else DEFINE_COMPILERRT_FUNCTION(__umodsi3) -#endif #if __ARM_ARCH_EXT_IDIV__ tst r1, r1 beq LOCAL_LABEL(divby0) diff --git a/contrib/compiler-rt/lib/builtins/ashldi3.c b/contrib/compiler-rt/lib/builtins/ashldi3.c index fcb0abdb1fce..a5c1836006b9 100644 --- a/contrib/compiler-rt/lib/builtins/ashldi3.c +++ b/contrib/compiler-rt/lib/builtins/ashldi3.c @@ -41,8 +41,5 @@ __ashldi3(di_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) { - return __ashldi3(a, b); -} +AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) COMPILER_RT_ALIAS(__ashldi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/ashrdi3.c b/contrib/compiler-rt/lib/builtins/ashrdi3.c index b4ab4c617ba0..84619965eca0 100644 --- a/contrib/compiler-rt/lib/builtins/ashrdi3.c +++ b/contrib/compiler-rt/lib/builtins/ashrdi3.c @@ -42,8 +42,5 @@ __ashrdi3(di_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) { - return __ashrdi3(a, b); -} +AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) COMPILER_RT_ALIAS(__ashrdi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/assembly.h b/contrib/compiler-rt/lib/builtins/assembly.h index 4893d483fd82..3f5e59b25442 100644 --- a/contrib/compiler-rt/lib/builtins/assembly.h +++ b/contrib/compiler-rt/lib/builtins/assembly.h @@ -68,10 +68,42 @@ #endif #if defined(__arm__) + +/* + * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: + * - for '-mthumb -march=armv6' compiler defines '__thumb__' + * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' + */ +#if defined(__thumb2__) || defined(__thumb__) +#define DEFINE_CODE_STATE .thumb SEPARATOR +#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR +#if defined(__thumb2__) +#define USE_THUMB_2 +#define IT(cond) it cond +#define ITT(cond) itt cond +#define ITE(cond) ite cond +#else +#define USE_THUMB_1 +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif // defined(__thumb__2) +#else // !defined(__thumb2__) && !defined(__thumb__) +#define DEFINE_CODE_STATE .arm SEPARATOR +#define DECLARE_FUNC_ENCODING +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif + +#if defined(USE_THUMB_1) && defined(USE_THUMB_2) +#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together." +#endif + #if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5 #define ARM_HAS_BX #endif -#if !defined(__ARM_FEATURE_CLZ) && __ARM_ARCH_ISA_THUMB != 1 && \ +#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \ (__ARM_ARCH >= 6 || (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__))) #define __ARM_FEATURE_CLZ #endif @@ -93,37 +125,15 @@ JMP(ip) #endif -/* - * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: - * - for '-mthumb -march=armv6' compiler defines '__thumb__' - * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' - */ -#if defined(__thumb2__) -#define USE_THUMB_2 1 -#elif defined(__thumb__) -#define USE_THUMB_1 1 -#endif - -#if defined(USE_THUMB_1) && defined(USE_THUMB_2) -#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together." -#endif - -#if defined(USE_THUMB_1) || defined(USE_THUMB_1) -#define USE_THUMB_PROLOGUE 1 -#endif - #if defined(USE_THUMB_2) -#define IT(cond) it cond -#define ITT(cond) itt cond -#define ITE(cond) ite cond #define WIDE(op) op.w #else -#define IT(cond) -#define ITT(cond) -#define ITE(cond) #define WIDE(op) op #endif -#endif /* defined(__arm__) */ +#else // !defined(__arm) +#define DECLARE_FUNC_ENCODING +#define DEFINE_CODE_STATE +#endif #define GLUE2(a, b) a##b #define GLUE(a, b) GLUE2(a, b) @@ -137,13 +147,16 @@ #endif #define DEFINE_COMPILERRT_FUNCTION(name) \ + DEFINE_CODE_STATE \ FILE_LEVEL_DIRECTIVE SEPARATOR \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ DECLARE_SYMBOL_VISIBILITY(name) \ + DECLARE_FUNC_ENCODING \ SYMBOL_NAME(name): #define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \ + DEFINE_CODE_STATE \ FILE_LEVEL_DIRECTIVE SEPARATOR \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ @@ -152,16 +165,20 @@ SYMBOL_NAME(name): #define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \ + DEFINE_CODE_STATE \ FILE_LEVEL_DIRECTIVE SEPARATOR \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ HIDDEN(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_FUNC_ENCODING \ SYMBOL_NAME(name): #define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \ + DEFINE_CODE_STATE \ .globl name SEPARATOR \ SYMBOL_IS_FUNC(name) SEPARATOR \ HIDDEN(name) SEPARATOR \ + DECLARE_FUNC_ENCODING \ name: #define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \ diff --git a/contrib/compiler-rt/lib/builtins/clear_cache.c b/contrib/compiler-rt/lib/builtins/clear_cache.c index 7ccbe019dba9..4a01cb46d4ac 100644 --- a/contrib/compiler-rt/lib/builtins/clear_cache.c +++ b/contrib/compiler-rt/lib/builtins/clear_cache.c @@ -9,6 +9,7 @@ */ #include "int_lib.h" +#include #include #if __APPLE__ @@ -23,7 +24,7 @@ uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress, uintptr_t GetCurrentProcess(void); #endif -#if (defined(__FreeBSD__) || defined(__Bitrig__)) && defined(__arm__) +#if defined(__FreeBSD__) && defined(__arm__) #include #include #endif @@ -32,7 +33,7 @@ uintptr_t GetCurrentProcess(void); #include #endif -#if defined(__mips__) && !defined(__FreeBSD__) +#if defined(__linux__) && defined(__mips__) #include #include #include @@ -41,7 +42,7 @@ uintptr_t GetCurrentProcess(void); * clear_mips_cache - Invalidates instruction cache for Mips. */ static void clear_mips_cache(const void* Addr, size_t Size) { - asm volatile ( + __asm__ volatile ( ".set push\n" ".set noreorder\n" ".set noat\n" @@ -96,7 +97,7 @@ void __clear_cache(void *start, void *end) { * so there is nothing to do */ #elif defined(__arm__) && !defined(__APPLE__) - #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__Bitrig__) + #if defined(__FreeBSD__) || defined(__NetBSD__) struct arm_sync_icache_args arg; arg.addr = (uintptr_t)start; @@ -121,15 +122,13 @@ void __clear_cache(void *start, void *end) { : "=r"(start_reg) : "r"(syscall_nr), "r"(start_reg), "r"(end_reg), "r"(flags)); - if (start_reg != 0) { - compilerrt_abort(); - } + assert(start_reg == 0 && "Cache flush syscall failed."); #elif defined(_WIN32) FlushInstructionCache(GetCurrentProcess(), start, end - start); #else compilerrt_abort(); #endif -#elif defined(__mips__) && !defined(__FreeBSD__) +#elif defined(__linux__) && defined(__mips__) const uintptr_t start_int = (uintptr_t) start; const uintptr_t end_int = (uintptr_t) end; #if defined(__ANDROID__) && defined(__LP64__) @@ -165,6 +164,21 @@ void __clear_cache(void *start, void *end) { for (addr = xstart; addr < xend; addr += icache_line_size) __asm __volatile("ic ivau, %0" :: "r"(addr)); __asm __volatile("isb sy"); +#elif defined (__powerpc64__) + const size_t line_size = 32; + const size_t len = (uintptr_t)end - (uintptr_t)start; + + const uintptr_t mask = ~(line_size - 1); + const uintptr_t start_line = ((uintptr_t)start) & mask; + const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask; + + for (uintptr_t line = start_line; line < end_line; line += line_size) + __asm__ volatile("dcbf 0, %0" : : "r"(line)); + __asm__ volatile("sync"); + + for (uintptr_t line = start_line; line < end_line; line += line_size) + __asm__ volatile("icbi 0, %0" : : "r"(line)); + __asm__ volatile("isync"); #else #if __APPLE__ /* On Darwin, sys_icache_invalidate() provides this functionality */ @@ -174,4 +188,3 @@ void __clear_cache(void *start, void *end) { #endif #endif } - diff --git a/contrib/compiler-rt/lib/builtins/comparedf2.c b/contrib/compiler-rt/lib/builtins/comparedf2.c index c5bb169d0021..44e5d2b288a6 100644 --- a/contrib/compiler-rt/lib/builtins/comparedf2.c +++ b/contrib/compiler-rt/lib/builtins/comparedf2.c @@ -143,8 +143,11 @@ __gtdf2(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { return __unorddf2(a, b); } +#else +AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unorddf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/comparesf2.c b/contrib/compiler-rt/lib/builtins/comparesf2.c index 4badb5e1b9f7..43cd6a6a7003 100644 --- a/contrib/compiler-rt/lib/builtins/comparesf2.c +++ b/contrib/compiler-rt/lib/builtins/comparesf2.c @@ -143,8 +143,11 @@ __gtsf2(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { return __unordsf2(a, b); } +#else +AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unordsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/cpu_model.c b/contrib/compiler-rt/lib/builtins/cpu_model.c index 83ea7a49faf7..4c96e9cd85d5 100644 --- a/contrib/compiler-rt/lib/builtins/cpu_model.c +++ b/contrib/compiler-rt/lib/builtins/cpu_model.c @@ -54,6 +54,7 @@ enum ProcessorTypes { AMD_BTVER1, AMD_BTVER2, AMDFAM17H, + INTEL_KNM, CPU_TYPE_MAX }; @@ -74,6 +75,7 @@ enum ProcessorSubtypes { INTEL_COREI7_BROADWELL, INTEL_COREI7_SKYLAKE, INTEL_COREI7_SKYLAKE_AVX512, + INTEL_COREI7_CANNONLAKE, CPU_SUBTYPE_MAX }; @@ -339,6 +341,12 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" break; + // Cannonlake: + case 0x66: + *Type = INTEL_COREI7; + *Subtype = INTEL_COREI7_CANNONLAKE; // "cannonlake" + break; + case 0x1c: // Most 45 nm Intel Atom processors case 0x26: // 45 nm Atom Lincroft case 0x27: // 32 nm Atom Medfield @@ -361,6 +369,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, *Type = INTEL_KNL; // knl break; + case 0x85: + *Type = INTEL_KNM; // knm + break; + default: // Unknown family 6 CPU. break; break; diff --git a/contrib/compiler-rt/lib/builtins/divdf3.c b/contrib/compiler-rt/lib/builtins/divdf3.c index 492e32b851e9..04a4dc5571ca 100644 --- a/contrib/compiler-rt/lib/builtins/divdf3.c +++ b/contrib/compiler-rt/lib/builtins/divdf3.c @@ -183,8 +183,11 @@ __divdf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) { return __divdf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divdf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/divsf3.c b/contrib/compiler-rt/lib/builtins/divsf3.c index aa6289a6d70a..65294d70fc61 100644 --- a/contrib/compiler-rt/lib/builtins/divsf3.c +++ b/contrib/compiler-rt/lib/builtins/divsf3.c @@ -167,8 +167,11 @@ __divsf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) { return __divsf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/divsi3.c b/contrib/compiler-rt/lib/builtins/divsi3.c index 3852e3990b5b..75aea008ddc1 100644 --- a/contrib/compiler-rt/lib/builtins/divsi3.c +++ b/contrib/compiler-rt/lib/builtins/divsi3.c @@ -35,8 +35,5 @@ __divsi3(si_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) { - return __divsi3(a, b); -} +AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) COMPILER_RT_ALIAS(__divsi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/emutls.c b/contrib/compiler-rt/lib/builtins/emutls.c index 12aad3a42b76..5dd8dd154771 100644 --- a/contrib/compiler-rt/lib/builtins/emutls.c +++ b/contrib/compiler-rt/lib/builtins/emutls.c @@ -102,7 +102,6 @@ static __inline emutls_address_array* emutls_getspecific() { #include #include #include -#include static LPCRITICAL_SECTION emutls_mutex; static DWORD emutls_tls_index = TLS_OUT_OF_INDEXES; @@ -203,25 +202,24 @@ static __inline emutls_address_array* emutls_getspecific() { /* Provide atomic load/store functions for emutls_get_index if built with MSVC. */ #if !defined(__ATOMIC_RELEASE) +#include enum { __ATOMIC_ACQUIRE = 2, __ATOMIC_RELEASE = 3 }; static __inline uintptr_t __atomic_load_n(void *ptr, unsigned type) { assert(type == __ATOMIC_ACQUIRE); + // These return the previous value - but since we do an OR with 0, + // it's equivalent to a plain load. #ifdef _WIN64 - return (uintptr_t) _load_be_u64(ptr); + return InterlockedOr64(ptr, 0); #else - return (uintptr_t) _load_be_u32(ptr); + return InterlockedOr(ptr, 0); #endif } static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) { assert(type == __ATOMIC_RELEASE); -#ifdef _WIN64 - _store_be_u64(ptr, val); -#else - _store_be_u32(ptr, val); -#endif + InterlockedExchangePointer((void *volatile *)ptr, (void *)val); } #endif diff --git a/contrib/compiler-rt/lib/builtins/enable_execute_stack.c b/contrib/compiler-rt/lib/builtins/enable_execute_stack.c index 0dc3482c4467..327d460b4253 100644 --- a/contrib/compiler-rt/lib/builtins/enable_execute_stack.c +++ b/contrib/compiler-rt/lib/builtins/enable_execute_stack.c @@ -22,7 +22,7 @@ #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN -#include +#include #else #ifndef __APPLE__ #include diff --git a/contrib/compiler-rt/lib/builtins/extendhfsf2.c b/contrib/compiler-rt/lib/builtins/extendhfsf2.c index e7d9fde8abfc..d9c0db84b0ce 100644 --- a/contrib/compiler-rt/lib/builtins/extendhfsf2.c +++ b/contrib/compiler-rt/lib/builtins/extendhfsf2.c @@ -23,8 +23,11 @@ COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); } +#else +AEABI_RTABI float __aeabi_h2f(uint16_t a) COMPILER_RT_ALIAS(__extendhfsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/extendsfdf2.c b/contrib/compiler-rt/lib/builtins/extendsfdf2.c index b9e7a7471a98..3d84529a6c53 100644 --- a/contrib/compiler-rt/lib/builtins/extendsfdf2.c +++ b/contrib/compiler-rt/lib/builtins/extendsfdf2.c @@ -17,8 +17,11 @@ COMPILER_RT_ABI double __extendsfdf2(float a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_f2d(float a) { return __extendsfdf2(a); } +#else +AEABI_RTABI double __aeabi_f2d(float a) COMPILER_RT_ALIAS(__extendsfdf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixdfdi.c b/contrib/compiler-rt/lib/builtins/fixdfdi.c index 31d76df28255..54e312d3c8f7 100644 --- a/contrib/compiler-rt/lib/builtins/fixdfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixdfdi.c @@ -45,13 +45,11 @@ __fixdfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI di_int -#if defined(__SOFT_FP__) -__aeabi_d2lz(fp_t a) { -#else -__aeabi_d2lz(double a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI di_int __aeabi_d2lz(fp_t a) { return __fixdfdi(a); } +#else +AEABI_RTABI di_int __aeabi_d2lz(fp_t a) COMPILER_RT_ALIAS(__fixdfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixdfsi.c b/contrib/compiler-rt/lib/builtins/fixdfsi.c index fc316dcd0545..5b9588175717 100644 --- a/contrib/compiler-rt/lib/builtins/fixdfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixdfsi.c @@ -20,8 +20,11 @@ __fixdfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI si_int __aeabi_d2iz(fp_t a) { return __fixdfsi(a); } +#else +AEABI_RTABI si_int __aeabi_d2iz(fp_t a) COMPILER_RT_ALIAS(__fixdfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixsfdi.c b/contrib/compiler-rt/lib/builtins/fixsfdi.c index c43473637d60..32e87c60889f 100644 --- a/contrib/compiler-rt/lib/builtins/fixsfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixsfdi.c @@ -45,13 +45,11 @@ __fixsfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI di_int -#if defined(__SOFT_FP__) -__aeabi_f2lz(fp_t a) { -#else -__aeabi_f2lz(float a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI di_int __aeabi_f2lz(fp_t a) { return __fixsfdi(a); } +#else +AEABI_RTABI di_int __aeabi_f2lz(fp_t a) COMPILER_RT_ALIAS(__fixsfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixsfsi.c b/contrib/compiler-rt/lib/builtins/fixsfsi.c index 3276df966460..e94e5f3dcd68 100644 --- a/contrib/compiler-rt/lib/builtins/fixsfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixsfsi.c @@ -20,8 +20,11 @@ __fixsfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI si_int __aeabi_f2iz(fp_t a) { return __fixsfsi(a); } +#else +AEABI_RTABI si_int __aeabi_f2iz(fp_t a) COMPILER_RT_ALIAS(__fixsfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunsdfdi.c b/contrib/compiler-rt/lib/builtins/fixunsdfdi.c index b734409709bf..bfe4dbb25656 100644 --- a/contrib/compiler-rt/lib/builtins/fixunsdfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixunsdfdi.c @@ -42,13 +42,11 @@ __fixunsdfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI du_int -#if defined(__SOFT_FP__) -__aeabi_d2ulz(fp_t a) { -#else -__aeabi_d2ulz(double a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) { return __fixunsdfdi(a); } +#else +AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunsdfsi.c b/contrib/compiler-rt/lib/builtins/fixunsdfsi.c index bb3d8e0f831b..3c5355beae1a 100644 --- a/contrib/compiler-rt/lib/builtins/fixunsdfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixunsdfsi.c @@ -19,8 +19,11 @@ __fixunsdfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) { return __fixunsdfsi(a); } +#else +AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunssfdi.c b/contrib/compiler-rt/lib/builtins/fixunssfdi.c index 5d92245df0d9..080a25bb1e99 100644 --- a/contrib/compiler-rt/lib/builtins/fixunssfdi.c +++ b/contrib/compiler-rt/lib/builtins/fixunssfdi.c @@ -43,13 +43,11 @@ __fixunssfdi(fp_t a) { #endif #if defined(__ARM_EABI__) -AEABI_RTABI du_int -#if defined(__SOFT_FP__) -__aeabi_f2ulz(fp_t a) { -#else -__aeabi_f2ulz(float a) { -#endif +#if defined(COMPILER_RT_ARMHF_TARGET) +AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) { return __fixunssfdi(a); } +#else +AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunssfdi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/fixunssfsi.c b/contrib/compiler-rt/lib/builtins/fixunssfsi.c index 91d5e8ae5d7f..eca2916a5c88 100644 --- a/contrib/compiler-rt/lib/builtins/fixunssfsi.c +++ b/contrib/compiler-rt/lib/builtins/fixunssfsi.c @@ -23,8 +23,11 @@ __fixunssfsi(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) { return __fixunssfsi(a); } +#else +AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunssfsi); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatdidf.c b/contrib/compiler-rt/lib/builtins/floatdidf.c index 681fecef9682..36b856e078d4 100644 --- a/contrib/compiler-rt/lib/builtins/floatdidf.c +++ b/contrib/compiler-rt/lib/builtins/floatdidf.c @@ -105,8 +105,11 @@ __floatdidf(di_int a) #endif #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_l2d(di_int a) { return __floatdidf(a); } +#else +AEABI_RTABI double __aeabi_l2d(di_int a) COMPILER_RT_ALIAS(__floatdidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatdisf.c b/contrib/compiler-rt/lib/builtins/floatdisf.c index dd548165c373..a2f09eb2ed2c 100644 --- a/contrib/compiler-rt/lib/builtins/floatdisf.c +++ b/contrib/compiler-rt/lib/builtins/floatdisf.c @@ -78,8 +78,11 @@ __floatdisf(di_int a) } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_l2f(di_int a) { return __floatdisf(a); } +#else +AEABI_RTABI float __aeabi_l2f(di_int a) COMPILER_RT_ALIAS(__floatdisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatsidf.c b/contrib/compiler-rt/lib/builtins/floatsidf.c index 2ae395bdc1db..fe051123ce7c 100644 --- a/contrib/compiler-rt/lib/builtins/floatsidf.c +++ b/contrib/compiler-rt/lib/builtins/floatsidf.c @@ -51,8 +51,11 @@ __floatsidf(int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_i2d(int a) { return __floatsidf(a); } +#else +AEABI_RTABI fp_t __aeabi_i2d(int a) COMPILER_RT_ALIAS(__floatsidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatsisf.c b/contrib/compiler-rt/lib/builtins/floatsisf.c index 08891fcdf201..bf087ee3c295 100644 --- a/contrib/compiler-rt/lib/builtins/floatsisf.c +++ b/contrib/compiler-rt/lib/builtins/floatsisf.c @@ -57,8 +57,11 @@ __floatsisf(int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_i2f(int a) { return __floatsisf(a); } +#else +AEABI_RTABI fp_t __aeabi_i2f(int a) COMPILER_RT_ALIAS(__floatsisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatundidf.c b/contrib/compiler-rt/lib/builtins/floatundidf.c index 6c1a931ef2f3..8bc2a096324f 100644 --- a/contrib/compiler-rt/lib/builtins/floatundidf.c +++ b/contrib/compiler-rt/lib/builtins/floatundidf.c @@ -104,8 +104,11 @@ __floatundidf(du_int a) #endif #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI double __aeabi_ul2d(du_int a) { return __floatundidf(a); } +#else +AEABI_RTABI double __aeabi_ul2d(du_int a) COMPILER_RT_ALIAS(__floatundidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatundisf.c b/contrib/compiler-rt/lib/builtins/floatundisf.c index 86841a75dc66..844786ea7777 100644 --- a/contrib/compiler-rt/lib/builtins/floatundisf.c +++ b/contrib/compiler-rt/lib/builtins/floatundisf.c @@ -75,8 +75,11 @@ __floatundisf(du_int a) } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_ul2f(du_int a) { return __floatundisf(a); } +#else +AEABI_RTABI float __aeabi_ul2f(du_int a) COMPILER_RT_ALIAS(__floatundisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatunsidf.c b/contrib/compiler-rt/lib/builtins/floatunsidf.c index 8d4807194f0b..75cf6b9177df 100644 --- a/contrib/compiler-rt/lib/builtins/floatunsidf.c +++ b/contrib/compiler-rt/lib/builtins/floatunsidf.c @@ -40,8 +40,11 @@ __floatunsidf(unsigned int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) { return __floatunsidf(a); } +#else +AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) COMPILER_RT_ALIAS(__floatunsidf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/floatunsisf.c b/contrib/compiler-rt/lib/builtins/floatunsisf.c index f194c046d2fb..29525ccedbbe 100644 --- a/contrib/compiler-rt/lib/builtins/floatunsisf.c +++ b/contrib/compiler-rt/lib/builtins/floatunsisf.c @@ -48,8 +48,11 @@ __floatunsisf(unsigned int a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) { return __floatunsisf(a); } +#else +AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) COMPILER_RT_ALIAS(__floatunsisf); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/int_endianness.h b/contrib/compiler-rt/lib/builtins/int_endianness.h index 7995ddbb953c..e2586c56bac8 100644 --- a/contrib/compiler-rt/lib/builtins/int_endianness.h +++ b/contrib/compiler-rt/lib/builtins/int_endianness.h @@ -61,7 +61,7 @@ #endif /* *BSD */ -#if defined(__OpenBSD__) || defined(__Bitrig__) +#if defined(__OpenBSD__) #include #if _BYTE_ORDER == _BIG_ENDIAN @@ -72,7 +72,7 @@ #define _YUGA_BIG_ENDIAN 0 #endif /* _BYTE_ORDER */ -#endif /* OpenBSD and Bitrig. */ +#endif /* OpenBSD */ /* .. */ diff --git a/contrib/compiler-rt/lib/builtins/int_lib.h b/contrib/compiler-rt/lib/builtins/int_lib.h index 53caf3a53e63..93394cebf7ea 100644 --- a/contrib/compiler-rt/lib/builtins/int_lib.h +++ b/contrib/compiler-rt/lib/builtins/int_lib.h @@ -22,9 +22,11 @@ #if defined(__ELF__) #define FNALIAS(alias_name, original_name) \ - void alias_name() __attribute__((alias(#original_name))) + void alias_name() __attribute__((__alias__(#original_name))) +#define COMPILER_RT_ALIAS(aliasee) __attribute__((__alias__(#aliasee))) #else #define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")") +#define COMPILER_RT_ALIAS(aliasee) _Pragma("GCC error(\"alias unsupported on this file format\")") #endif /* ABI macro definitions */ diff --git a/contrib/compiler-rt/lib/builtins/lshrdi3.c b/contrib/compiler-rt/lib/builtins/lshrdi3.c index becbbef4eb09..67b2a7668345 100644 --- a/contrib/compiler-rt/lib/builtins/lshrdi3.c +++ b/contrib/compiler-rt/lib/builtins/lshrdi3.c @@ -41,8 +41,5 @@ __lshrdi3(di_int a, si_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) { - return __lshrdi3(a, b); -} +AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) COMPILER_RT_ALIAS(__lshrdi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/muldf3.c b/contrib/compiler-rt/lib/builtins/muldf3.c index 59a60190eba3..1bb103e38c13 100644 --- a/contrib/compiler-rt/lib/builtins/muldf3.c +++ b/contrib/compiler-rt/lib/builtins/muldf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) { return __muldf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__muldf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/muldi3.c b/contrib/compiler-rt/lib/builtins/muldi3.c index 6818a9e2f722..a187315e9165 100644 --- a/contrib/compiler-rt/lib/builtins/muldi3.c +++ b/contrib/compiler-rt/lib/builtins/muldi3.c @@ -54,8 +54,5 @@ __muldi3(di_int a, di_int b) } #if defined(__ARM_EABI__) -AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) { - return __muldi3(a, b); -} +AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) COMPILER_RT_ALIAS(__muldi3); #endif - diff --git a/contrib/compiler-rt/lib/builtins/mulsf3.c b/contrib/compiler-rt/lib/builtins/mulsf3.c index f141af1acc58..1e2cf3e717c9 100644 --- a/contrib/compiler-rt/lib/builtins/mulsf3.c +++ b/contrib/compiler-rt/lib/builtins/mulsf3.c @@ -20,8 +20,11 @@ COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) { return __mulsf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__mulsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/negdf2.c b/contrib/compiler-rt/lib/builtins/negdf2.c index 5e2544cdb4be..f0bfaad24743 100644 --- a/contrib/compiler-rt/lib/builtins/negdf2.c +++ b/contrib/compiler-rt/lib/builtins/negdf2.c @@ -20,8 +20,11 @@ __negdf2(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_dneg(fp_t a) { return __negdf2(a); } +#else +AEABI_RTABI fp_t __aeabi_dneg(fp_t a) COMPILER_RT_ALIAS(__negdf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/negsf2.c b/contrib/compiler-rt/lib/builtins/negsf2.c index f90b34335680..05c97d4d5a11 100644 --- a/contrib/compiler-rt/lib/builtins/negsf2.c +++ b/contrib/compiler-rt/lib/builtins/negsf2.c @@ -20,8 +20,11 @@ __negsf2(fp_t a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fneg(fp_t a) { return __negsf2(a); } +#else +AEABI_RTABI fp_t __aeabi_fneg(fp_t a) COMPILER_RT_ALIAS(__negsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/subdf3.c b/contrib/compiler-rt/lib/builtins/subdf3.c index 38340dfab1a6..a892fa603cf2 100644 --- a/contrib/compiler-rt/lib/builtins/subdf3.c +++ b/contrib/compiler-rt/lib/builtins/subdf3.c @@ -22,8 +22,11 @@ __subdf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) { return __subdf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subdf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/subsf3.c b/contrib/compiler-rt/lib/builtins/subsf3.c index 34276b1447ba..4b2786177dcc 100644 --- a/contrib/compiler-rt/lib/builtins/subsf3.c +++ b/contrib/compiler-rt/lib/builtins/subsf3.c @@ -22,8 +22,11 @@ __subsf3(fp_t a, fp_t b) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) { return __subsf3(a, b); } +#else +AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subsf3); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/truncdfhf2.c b/contrib/compiler-rt/lib/builtins/truncdfhf2.c index 4bb71aa178a0..8354a41b8b6f 100644 --- a/contrib/compiler-rt/lib/builtins/truncdfhf2.c +++ b/contrib/compiler-rt/lib/builtins/truncdfhf2.c @@ -16,8 +16,11 @@ COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); } +#else +AEABI_RTABI uint16_t __aeabi_d2h(double a) COMPILER_RT_ALIAS(__truncdfhf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/truncdfsf2.c b/contrib/compiler-rt/lib/builtins/truncdfsf2.c index 8bf58bb23a3b..195d3e0656e7 100644 --- a/contrib/compiler-rt/lib/builtins/truncdfsf2.c +++ b/contrib/compiler-rt/lib/builtins/truncdfsf2.c @@ -16,8 +16,11 @@ COMPILER_RT_ABI float __truncdfsf2(double a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI float __aeabi_d2f(double a) { return __truncdfsf2(a); } +#else +AEABI_RTABI float __aeabi_d2f(double a) COMPILER_RT_ALIAS(__truncdfsf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/truncsfhf2.c b/contrib/compiler-rt/lib/builtins/truncsfhf2.c index f6ce1fa1de05..9c84ab4f938a 100644 --- a/contrib/compiler-rt/lib/builtins/truncsfhf2.c +++ b/contrib/compiler-rt/lib/builtins/truncsfhf2.c @@ -22,8 +22,11 @@ COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { } #if defined(__ARM_EABI__) +#if defined(COMPILER_RT_ARMHF_TARGET) AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); } +#else +AEABI_RTABI uint16_t __aeabi_f2h(float a) COMPILER_RT_ALIAS(__truncsfhf2); +#endif #endif - diff --git a/contrib/compiler-rt/lib/builtins/udivsi3.c b/contrib/compiler-rt/lib/builtins/udivsi3.c index 8eccf102cc97..bb720f8c382b 100644 --- a/contrib/compiler-rt/lib/builtins/udivsi3.c +++ b/contrib/compiler-rt/lib/builtins/udivsi3.c @@ -64,8 +64,5 @@ __udivsi3(su_int n, su_int d) } #if defined(__ARM_EABI__) -AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) { - return __udivsi3(n, d); -} +AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) COMPILER_RT_ALIAS(__udivsi3); #endif - diff --git a/contrib/compiler-rt/lib/cfi/cfi.cc b/contrib/compiler-rt/lib/cfi/cfi.cc index f720230a70be..f7693b37d29c 100644 --- a/contrib/compiler-rt/lib/cfi/cfi.cc +++ b/contrib/compiler-rt/lib/cfi/cfi.cc @@ -280,7 +280,7 @@ void InitShadow() { CHECK_EQ(0, GetShadow()); CHECK_EQ(0, GetShadowSize()); - uptr vma = GetMaxVirtualAddress(); + uptr vma = GetMaxUserVirtualAddress(); // Shadow is 2 -> 2**kShadowGranularity. SetShadowSize((vma >> (kShadowGranularity - 1)) + 1); VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize()); diff --git a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt index cc111be8120e..d8c9d49e3cdd 100644 --- a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt +++ b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt @@ -1,18 +1,4 @@ -# Standard library types. -type:std::* - -# The stdext namespace contains Microsoft standard library extensions. -type:stdext::* - -# Types with a uuid attribute, i.e. COM types. -type:attr:uuid - -# STL allocators (T *allocator::allocate(size_type, const void*)). -# The type signature mandates a cast from uninitialized void* to T*. -# size_type can either be unsigned int (j) or unsigned long (m). -fun:*8allocateEjPKv -fun:*8allocateEmPKv - +[cfi-unrelated-cast] # std::get_temporary_buffer, likewise (libstdc++, libc++). fun:_ZSt20get_temporary_buffer* fun:_ZNSt3__120get_temporary_buffer* diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.cc b/contrib/compiler-rt/lib/dfsan/dfsan.cc index 3aa99b7f9c27..9e360c959fa7 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan.cc +++ b/contrib/compiler-rt/lib/dfsan/dfsan.cc @@ -21,6 +21,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_libc.h" diff --git a/contrib/compiler-rt/lib/esan/esan_sideline.h b/contrib/compiler-rt/lib/esan/esan_sideline.h index aa3fae1db318..04aff22f4cf0 100644 --- a/contrib/compiler-rt/lib/esan/esan_sideline.h +++ b/contrib/compiler-rt/lib/esan/esan_sideline.h @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" namespace __esan { @@ -46,7 +47,8 @@ class SidelineThread { private: static int runSideline(void *Arg); static void registerSignal(int SigNum); - static void handleSidelineSignal(int SigNum, void *SigInfo, void *Ctx); + static void handleSidelineSignal(int SigNum, __sanitizer_siginfo *SigInfo, + void *Ctx); char *Stack; SidelineFunc sampleFunc; diff --git a/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp b/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp index bc272dfe49f8..4a96910ece35 100644 --- a/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp +++ b/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp @@ -40,7 +40,8 @@ static const uptr SidelineIdUninitialized = 1; static SidelineThread *TheThread; // We aren't passing SA_NODEFER so the same signal is blocked while here. -void SidelineThread::handleSidelineSignal(int SigNum, void *SigInfo, +void SidelineThread::handleSidelineSignal(int SigNum, + __sanitizer_siginfo *SigInfo, void *Ctx) { VPrintf(3, "Sideline signal %d\n", SigNum); CHECK_EQ(SigNum, SIGALRM); diff --git a/contrib/compiler-rt/lib/esan/working_set_posix.cpp b/contrib/compiler-rt/lib/esan/working_set_posix.cpp index fcfa87128857..5ec53b959261 100644 --- a/contrib/compiler-rt/lib/esan/working_set_posix.cpp +++ b/contrib/compiler-rt/lib/esan/working_set_posix.cpp @@ -34,7 +34,7 @@ bool processWorkingSetSignal(int SigNum, void (*Handler)(int), VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum); if (SigNum == SIGSEGV) { *Result = AppSigAct.handler; - AppSigAct.sigaction = (void (*)(int, void*, void*))Handler; + AppSigAct.sigaction = (decltype(AppSigAct.sigaction))Handler; return false; // Skip real call. } return true; @@ -73,7 +73,7 @@ bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet) { static void reinstateDefaultHandler(int SigNum) { __sanitizer_sigaction SigAct; internal_memset(&SigAct, 0, sizeof(SigAct)); - SigAct.sigaction = (void (*)(int, void*, void*)) SIG_DFL; + SigAct.sigaction = (decltype(SigAct.sigaction))SIG_DFL; int Res = internal_sigaction(SigNum, &SigAct, nullptr); CHECK(Res == 0); VPrintf(1, "Unregistered for %d handler\n", SigNum); @@ -81,7 +81,8 @@ static void reinstateDefaultHandler(int SigNum) { // If this is a shadow fault, we handle it here; otherwise, we pass it to the // app to handle it just as the app would do without our tool in place. -static void handleMemoryFault(int SigNum, void *Info, void *Ctx) { +static void handleMemoryFault(int SigNum, __sanitizer_siginfo *Info, + void *Ctx) { if (SigNum == SIGSEGV) { // We rely on si_addr being filled in (thus we do not support old kernels). siginfo_t *SigInfo = (siginfo_t *)Info; diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.cc b/contrib/compiler-rt/lib/hwasan/hwasan.cc new file mode 100644 index 000000000000..8b1e5a7846af --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan.cc @@ -0,0 +1,375 @@ +//===-- hwasan.cc -----------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// HWAddressSanitizer runtime. +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "ubsan/ubsan_flags.h" +#include "ubsan/ubsan_init.h" + +// ACHTUNG! No system header includes in this file. + +using namespace __sanitizer; + +namespace __hwasan { + +void EnterSymbolizer() { + HwasanThread *t = GetCurrentThread(); + CHECK(t); + t->EnterSymbolizer(); +} +void ExitSymbolizer() { + HwasanThread *t = GetCurrentThread(); + CHECK(t); + t->LeaveSymbolizer(); +} +bool IsInSymbolizer() { + HwasanThread *t = GetCurrentThread(); + return t && t->InSymbolizer(); +} + +static Flags hwasan_flags; + +Flags *flags() { + return &hwasan_flags; +} + +int hwasan_inited = 0; +bool hwasan_init_is_running; + +int hwasan_report_count = 0; + +void Flags::SetDefaults() { +#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "hwasan_flags.inc" +#undef HWASAN_FLAG +} + +static void RegisterHwasanFlags(FlagParser *parser, Flags *f) { +#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "hwasan_flags.inc" +#undef HWASAN_FLAG +} + +static void InitializeFlags() { + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH"); + cf.malloc_context_size = 20; + cf.handle_ioctl = true; + // FIXME: test and enable. + cf.check_printf = false; + cf.intercept_tls_get_addr = true; + cf.exitcode = 99; + cf.handle_sigill = kHandleSignalExclusive; + OverrideCommonFlags(cf); + } + + Flags *f = flags(); + f->SetDefaults(); + + FlagParser parser; + RegisterHwasanFlags(&parser, f); + RegisterCommonFlags(&parser); + +#if HWASAN_CONTAINS_UBSAN + __ubsan::Flags *uf = __ubsan::flags(); + uf->SetDefaults(); + + FlagParser ubsan_parser; + __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); + RegisterCommonFlags(&ubsan_parser); +#endif + + // Override from user-specified string. + if (__hwasan_default_options) + parser.ParseString(__hwasan_default_options()); +#if HWASAN_CONTAINS_UBSAN + const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); + ubsan_parser.ParseString(ubsan_default_options); +#endif + + const char *hwasan_options = GetEnv("HWASAN_OPTIONS"); + parser.ParseString(hwasan_options); +#if HWASAN_CONTAINS_UBSAN + ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); +#endif + VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : ""); + + InitializeCommonFlags(); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); +} + +void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, + void *context, bool request_fast_unwind) { + HwasanThread *t = GetCurrentThread(); + if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); + } + stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), + request_fast_unwind); +} + +void PrintWarning(uptr pc, uptr bp) { + GET_FATAL_STACK_TRACE_PC_BP(pc, bp); + ReportInvalidAccess(&stack, 0); +} + +} // namespace __hwasan + +// Interface. + +using namespace __hwasan; + +void __hwasan_init() { + CHECK(!hwasan_init_is_running); + if (hwasan_inited) return; + hwasan_init_is_running = 1; + SanitizerToolName = "HWAddressSanitizer"; + + InitTlsSize(); + + CacheBinaryName(); + InitializeFlags(); + + __sanitizer_set_report_path(common_flags()->log_path); + + InitializeInterceptors(); + InstallDeadlySignalHandlers(HwasanOnDeadlySignal); + InstallAtExitHandler(); // Needs __cxa_atexit interceptor. + + DisableCoreDumperIfNecessary(); + if (!InitShadow()) { + Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n"); + Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); + Printf("FATAL: Disabling ASLR is known to cause this error.\n"); + Printf("FATAL: If running under GDB, try " + "'set disable-randomization off'.\n"); + DumpProcessMap(); + Die(); + } + + Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); + + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); + + HwasanTSDInit(HwasanTSDDtor); + + HwasanAllocatorInit(); + + HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr); + SetCurrentThread(main_thread); + main_thread->ThreadStart(); + +#if HWASAN_CONTAINS_UBSAN + __ubsan::InitAsPlugin(); +#endif + + VPrintf(1, "HWAddressSanitizer init done\n"); + + hwasan_init_is_running = 0; + hwasan_inited = 1; +} + +void __hwasan_print_shadow(const void *x, uptr size) { + // FIXME: + Printf("FIXME: __hwasan_print_shadow unimplemented\n"); +} + +sptr __hwasan_test_shadow(const void *p, uptr sz) { + if (sz == 0) + return -1; + tag_t ptr_tag = GetTagFromPointer((uptr)p); + if (ptr_tag == 0) + return -1; + uptr ptr_raw = GetAddressFromPointer((uptr)p); + uptr shadow_first = MEM_TO_SHADOW(ptr_raw); + uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1); + for (uptr s = shadow_first; s <= shadow_last; ++s) + if (*(tag_t*)s != ptr_tag) + return SHADOW_TO_MEM(s) - ptr_raw; + return -1; +} + +u16 __sanitizer_unaligned_load16(const uu16 *p) { + return *p; +} +u32 __sanitizer_unaligned_load32(const uu32 *p) { + return *p; +} +u64 __sanitizer_unaligned_load64(const uu64 *p) { + return *p; +} +void __sanitizer_unaligned_store16(uu16 *p, u16 x) { + *p = x; +} +void __sanitizer_unaligned_store32(uu32 *p, u32 x) { + *p = x; +} +void __sanitizer_unaligned_store64(uu64 *p, u64 x) { + *p = x; +} + +template +__attribute__((always_inline)) +static void SigIll() { +#if defined(__aarch64__) + asm("hlt %0\n\t" ::"n"(X)); +#elif defined(__x86_64__) || defined(__i386__) + asm("ud2\n\t"); +#else + // FIXME: not always sigill. + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + +enum class ErrorAction { Abort, Recover }; +enum class AccessType { Load, Store }; + +template +__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw); + if (UNLIKELY(ptr_tag != mem_tag)) { + SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + LogSize>(); + if (EA == ErrorAction::Abort) __builtin_unreachable(); + } +} + +template +__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, + uptr sz) { + CHECK_NE(0, sz); + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw); + tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1); + for (tag_t *t = shadow_first; t <= shadow_last; ++t) + if (UNLIKELY(ptr_tag != *t)) { + SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + 0xf>(); + if (EA == ErrorAction::Abort) __builtin_unreachable(); + } +} + +void __hwasan_load(uptr p, uptr sz) { + CheckAddressSized(p, sz); +} +void __hwasan_load1(uptr p) { + CheckAddress(p); +} +void __hwasan_load2(uptr p) { + CheckAddress(p); +} +void __hwasan_load4(uptr p) { + CheckAddress(p); +} +void __hwasan_load8(uptr p) { + CheckAddress(p); +} +void __hwasan_load16(uptr p) { + CheckAddress(p); +} + +void __hwasan_load_noabort(uptr p, uptr sz) { + CheckAddressSized(p, sz); +} +void __hwasan_load1_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_load2_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_load4_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_load8_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_load16_noabort(uptr p) { + CheckAddress(p); +} + +void __hwasan_store(uptr p, uptr sz) { + CheckAddressSized(p, sz); +} +void __hwasan_store1(uptr p) { + CheckAddress(p); +} +void __hwasan_store2(uptr p) { + CheckAddress(p); +} +void __hwasan_store4(uptr p) { + CheckAddress(p); +} +void __hwasan_store8(uptr p) { + CheckAddress(p); +} +void __hwasan_store16(uptr p) { + CheckAddress(p); +} + +void __hwasan_store_noabort(uptr p, uptr sz) { + CheckAddressSized(p, sz); +} +void __hwasan_store1_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_store2_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_store4_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_store8_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_store16_noabort(uptr p) { + CheckAddress(p); +} + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char* __hwasan_default_options() { return ""; } +} // extern "C" +#endif + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); + stack.Print(); +} +} // extern "C" diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.h b/contrib/compiler-rt/lib/hwasan/hwasan.h new file mode 100644 index 000000000000..bcf5282dce7c --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan.h @@ -0,0 +1,172 @@ +//===-- hwasan.h --------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Private Hwasan header. +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_H +#define HWASAN_H + +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "hwasan_interface_internal.h" +#include "hwasan_flags.h" +#include "ubsan/ubsan_platform.h" + +#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE +# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1 +#endif + +#ifndef HWASAN_CONTAINS_UBSAN +# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB +#endif + +typedef u8 tag_t; + +// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th). +const uptr kShadowScale = 4; +const uptr kShadowAlignment = 1UL << kShadowScale; + +#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale) +#define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale) +#define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale) + +#define MEM_IS_APP(mem) true + +// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address +// translation and can be used to store a tag. +const unsigned kAddressTagShift = 56; +const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; + +static inline tag_t GetTagFromPointer(uptr p) { + return p >> kAddressTagShift; +} + +static inline uptr GetAddressFromPointer(uptr p) { + return p & ~kAddressTagMask; +} + +static inline void * GetAddressFromPointer(const void *p) { + return (void *)((uptr)p & ~kAddressTagMask); +} + +static inline uptr AddTagToPointer(uptr p, tag_t tag) { + return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift); +} + +namespace __hwasan { + +extern int hwasan_inited; +extern bool hwasan_init_is_running; +extern int hwasan_report_count; + +bool ProtectRange(uptr beg, uptr end); +bool InitShadow(); +char *GetProcSelfMaps(); +void InitializeInterceptors(); + +void HwasanAllocatorInit(); +void HwasanAllocatorThreadFinish(); +void HwasanDeallocate(StackTrace *stack, void *ptr); + +void *hwasan_malloc(uptr size, StackTrace *stack); +void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); +void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack); +void *hwasan_valloc(uptr size, StackTrace *stack); +void *hwasan_pvalloc(uptr size, StackTrace *stack); +void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); +void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack); +int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, + StackTrace *stack); + +void InstallTrapHandler(); +void InstallAtExitHandler(); + +const char *GetStackOriginDescr(u32 id, uptr *pc); + +void EnterSymbolizer(); +void ExitSymbolizer(); +bool IsInSymbolizer(); + +struct SymbolizerScope { + SymbolizerScope() { EnterSymbolizer(); } + ~SymbolizerScope() { ExitSymbolizer(); } +}; + +void PrintWarning(uptr pc, uptr bp); + +void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, + void *context, bool request_fast_unwind); + +void ReportInvalidAccess(StackTrace *stack, u32 origin); +void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, + bool is_store); +void ReportStats(); +void ReportAtExitStatistics(); +void DescribeMemoryRange(const void *x, uptr size); +void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, uptr size, + uptr offset); + +// Returns a "chained" origin id, pointing to the given stack trace followed by +// the previous origin id. +u32 ChainOrigin(u32 id, StackTrace *stack); + +const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; + +#define GET_MALLOC_STACK_TRACE \ + BufferedStackTrace stack; \ + if (hwasan_inited) \ + GetStackTrace(&stack, common_flags()->malloc_context_size, \ + StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ + common_flags()->fast_unwind_on_malloc) + +#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (hwasan_inited) \ + GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ + common_flags()->fast_unwind_on_fatal) + +class ScopedThreadLocalStateBackup { + public: + ScopedThreadLocalStateBackup() { Backup(); } + ~ScopedThreadLocalStateBackup() { Restore(); } + void Backup(); + void Restore(); + private: + u64 va_arg_overflow_size_tls; +}; + +void HwasanTSDInit(void (*destructor)(void *tsd)); +void *HwasanTSDGet(); +void HwasanTSDSet(void *tsd); +void HwasanTSDDtor(void *tsd); + +void HwasanOnDeadlySignal(int signo, void *info, void *context); + +} // namespace __hwasan + +#define HWASAN_MALLOC_HOOK(ptr, size) \ + do { \ + if (&__sanitizer_malloc_hook) { \ + __sanitizer_malloc_hook(ptr, size); \ + } \ + RunMallocHooks(ptr, size); \ + } while (false) +#define HWASAN_FREE_HOOK(ptr) \ + do { \ + if (&__sanitizer_free_hook) { \ + __sanitizer_free_hook(ptr); \ + } \ + RunFreeHooks(ptr); \ + } while (false) + +#endif // HWASAN_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra b/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra new file mode 100644 index 000000000000..8738627e06e8 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra @@ -0,0 +1,2 @@ +__hwasan_* +__ubsan_* diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc new file mode 100644 index 000000000000..fbcaf78b88f0 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc @@ -0,0 +1,330 @@ +//===-- hwasan_allocator.cc --------------------------- ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// HWAddressSanitizer allocator. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "hwasan.h" +#include "hwasan_allocator.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" + +namespace __hwasan { + +enum { + CHUNK_INVALID = 0, + CHUNK_FREE = 1, + CHUNK_ALLOCATED = 2 +}; + +struct Metadata { + u64 state : 2; + u64 requested_size : 62; + u32 alloc_context_id; + u32 free_context_id; +}; + +bool HwasanChunkView::IsValid() const { + return metadata_ && metadata_->state != CHUNK_INVALID; +} +bool HwasanChunkView::IsAllocated() const { + return metadata_ && metadata_->state == CHUNK_ALLOCATED; +} +uptr HwasanChunkView::Beg() const { + return block_; +} +uptr HwasanChunkView::End() const { + return Beg() + UsedSize(); +} +uptr HwasanChunkView::UsedSize() const { + return metadata_->requested_size; +} +u32 HwasanChunkView::GetAllocStackId() const { + return metadata_->alloc_context_id; +} +u32 HwasanChunkView::GetFreeStackId() const { + return metadata_->free_context_id; +} + +struct HwasanMapUnmapCallback { + void OnMap(uptr p, uptr size) const {} + void OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // It can return as user-requested mmap() or another thread stack. + // Make it accessible with zero-tagged pointer. + TagMemory(p, size, 0); + } +}; + +#if !defined(__aarch64__) +#error unsupported platform +#endif + +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G +static const uptr kRegionSizeLog = 20; +static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; +typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog; + typedef __hwasan::ByteMap ByteMap; + typedef HwasanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32 PrimaryAllocator; +typedef SizeClassAllocatorLocalCache AllocatorCache; +typedef LargeMmapAllocator SecondaryAllocator; +typedef CombinedAllocator Allocator; + +static Allocator allocator; +static AllocatorCache fallback_allocator_cache; +static SpinMutex fallback_mutex; +static atomic_uint8_t hwasan_allocator_tagging_enabled; + +void HwasanAllocatorInit() { + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, + !flags()->disable_allocator_tagging); + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator.Init(common_flags()->allocator_release_to_os_interval_ms); +} + +AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) { + CHECK(ms); + CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); + return reinterpret_cast(ms->allocator_cache); +} + +void HwasanThreadLocalMallocStorage::CommitBack() { + allocator.SwallowCache(GetAllocatorCache(this)); +} + +static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, + bool zeroise) { + alignment = Max(alignment, kShadowAlignment); + size = RoundUpTo(size, kShadowAlignment); + + if (size > kMaxAllowedMallocSize) { + Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n", + (void *)size); + return Allocator::FailureHandler::OnBadRequest(); + } + HwasanThread *t = GetCurrentThread(); + void *allocated; + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocated = allocator.Allocate(cache, size, alignment); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocated = allocator.Allocate(cache, size, alignment); + } + Metadata *meta = + reinterpret_cast(allocator.GetMetaData(allocated)); + meta->state = CHUNK_ALLOCATED; + meta->requested_size = size; + meta->alloc_context_id = StackDepotPut(*stack); + if (zeroise) + internal_memset(allocated, 0, size); + + void *user_ptr = (flags()->tag_in_malloc && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB) + : allocated; + + HWASAN_MALLOC_HOOK(user_ptr, size); + return user_ptr; +} + +void HwasanDeallocate(StackTrace *stack, void *user_ptr) { + CHECK(user_ptr); + HWASAN_FREE_HOOK(user_ptr); + + void *p = GetAddressFromPointer(user_ptr); + Metadata *meta = reinterpret_cast(allocator.GetMetaData(p)); + uptr size = meta->requested_size; + meta->state = CHUNK_FREE; + meta->requested_size = 0; + meta->free_context_id = StackDepotPut(*stack); + // This memory will not be reused by anyone else, so we are free to keep it + // poisoned. + if (flags()->tag_in_free && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + TagMemoryAligned((uptr)p, size, 0xBC); + HwasanThread *t = GetCurrentThread(); + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocator.Deallocate(cache, p); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocator.Deallocate(cache, p); + } +} + +void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size, + uptr alignment) { + alignment = Max(alignment, kShadowAlignment); + new_size = RoundUpTo(new_size, kShadowAlignment); + + void *old_p = GetAddressFromPointer(user_old_p); + Metadata *meta = reinterpret_cast(allocator.GetMetaData(old_p)); + uptr old_size = meta->requested_size; + uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); + if (new_size <= actually_allocated_size) { + // We are not reallocating here. + // FIXME: update stack trace for the allocation? + meta->requested_size = new_size; + if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + return user_old_p; + if (flags()->retag_in_realloc) + return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC); + if (new_size > old_size) { + tag_t tag = GetTagFromPointer((uptr)user_old_p); + TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag); + } + return user_old_p; + } + uptr memcpy_size = Min(new_size, old_size); + void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); + if (new_p) { + internal_memcpy(new_p, old_p, memcpy_size); + HwasanDeallocate(stack, old_p); + } + return new_p; +} + +HwasanChunkView FindHeapChunkByAddress(uptr address) { + void *block = allocator.GetBlockBegin(reinterpret_cast(address)); + if (!block) + return HwasanChunkView(); + Metadata *metadata = + reinterpret_cast(allocator.GetMetaData(block)); + return HwasanChunkView(reinterpret_cast(block), metadata); +} + +static uptr AllocationSize(const void *user_ptr) { + const void *p = GetAddressFromPointer(user_ptr); + if (!p) return 0; + const void *beg = allocator.GetBlockBegin(p); + if (beg != p) return 0; + Metadata *b = (Metadata *)allocator.GetMetaData(p); + return b->requested_size; +} + +void *hwasan_malloc(uptr size, StackTrace *stack) { + return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); +} + +void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) + return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest()); + return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true)); +} + +void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { + if (!ptr) + return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + if (size == 0) { + HwasanDeallocate(stack, ptr); + return nullptr; + } + return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); +} + +void *hwasan_valloc(uptr size, StackTrace *stack) { + return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false)); +} + +void *hwasan_pvalloc(uptr size, StackTrace *stack) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + return Allocator::FailureHandler::OnBadRequest(); + } + // pvalloc(0) should allocate one page. + size = size ? RoundUpTo(size, PageSize) : PageSize; + return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false)); +} + +void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); +} + +void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) { + if (UNLIKELY(!IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); +} + +int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, + StackTrace *stack) { + if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { + Allocator::FailureHandler::OnBadRequest(); + return errno_EINVAL; + } + void *ptr = HwasanAllocate(stack, size, alignment, false); + if (UNLIKELY(!ptr)) + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, alignment)); + *memptr = ptr; + return 0; +} + +} // namespace __hwasan + +using namespace __hwasan; + +void __hwasan_enable_allocator_tagging() { + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1); +} + +void __hwasan_disable_allocator_tagging() { + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0); +} + +uptr __sanitizer_get_current_allocated_bytes() { + uptr stats[AllocatorStatCount]; + allocator.GetStats(stats); + return stats[AllocatorStatAllocated]; +} + +uptr __sanitizer_get_heap_size() { + uptr stats[AllocatorStatCount]; + allocator.GetStats(stats); + return stats[AllocatorStatMapped]; +} + +uptr __sanitizer_get_free_bytes() { return 1; } + +uptr __sanitizer_get_unmapped_bytes() { return 1; } + +uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } + +int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; } + +uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h new file mode 100644 index 000000000000..d025112e9773 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -0,0 +1,55 @@ +//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_ALLOCATOR_H +#define HWASAN_ALLOCATOR_H + +#include "sanitizer_common/sanitizer_common.h" + +namespace __hwasan { + +struct HwasanThreadLocalMallocStorage { + uptr quarantine_cache[16]; + // Allocator cache contains atomic_uint64_t which must be 8-byte aligned. + ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque. + void CommitBack(); + + private: + // These objects are allocated via mmap() and are zero-initialized. + HwasanThreadLocalMallocStorage() {} +}; + +struct Metadata; + +class HwasanChunkView { + public: + HwasanChunkView() : block_(0), metadata_(nullptr) {} + HwasanChunkView(uptr block, Metadata *metadata) + : block_(block), metadata_(metadata) {} + bool IsValid() const; // Checks if it points to a valid allocated chunk + bool IsAllocated() const; // Checks if the memory is currently allocated + uptr Beg() const; // First byte of user memory + uptr End() const; // Last byte of user memory + uptr UsedSize() const; // Size requested by the user + u32 GetAllocStackId() const; + u32 GetFreeStackId() const; + private: + uptr block_; + Metadata *const metadata_; +}; + +HwasanChunkView FindHeapChunkByAddress(uptr address); + +} // namespace __hwasan + +#endif // HWASAN_ALLOCATOR_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt b/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt new file mode 100644 index 000000000000..395ba28f0212 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt @@ -0,0 +1,7 @@ +# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular +# functions or sources. Use with care. You may set location of blacklist +# at compile-time using -fsanitize-blacklist= flag. + +# Example usage: +# fun:*bad_function_name* +# src:file_with_tricky_code.cc diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h new file mode 100644 index 000000000000..16d60c4d8ba5 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h @@ -0,0 +1,30 @@ +//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// +#ifndef HWASAN_FLAGS_H +#define HWASAN_FLAGS_H + +namespace __hwasan { + +struct Flags { +#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "hwasan_flags.inc" +#undef HWASAN_FLAG + + void SetDefaults(); +}; + +Flags *flags(); + +} // namespace __hwasan + +#endif // HWASAN_FLAGS_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc new file mode 100644 index 000000000000..a2cb701ae93d --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -0,0 +1,29 @@ +//===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Hwasan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef HWASAN_FLAG +# error "Define HWASAN_FLAG prior to including this file!" +#endif + +// HWASAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +HWASAN_FLAG(bool, tag_in_malloc, true, "") +HWASAN_FLAG(bool, tag_in_free, true, "") +HWASAN_FLAG(bool, retag_in_realloc, true, "") +HWASAN_FLAG(bool, print_stats, false, "") +HWASAN_FLAG(bool, halt_on_error, true, "") +HWASAN_FLAG(bool, atexit, false, "") + +// Test only flag to disable malloc/realloc/free memory tagging on startup. +// Tagging can be reenabled with __hwasan_enable_allocator_tagging(). +HWASAN_FLAG(bool, disable_allocator_tagging, false, "") diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc new file mode 100644 index 000000000000..fb39e9fdacf9 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc @@ -0,0 +1,483 @@ +//===-- hwasan_interceptors.cc ----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Interceptors for standard library functions. +// +// FIXME: move as many interceptors as possible into +// sanitizer_common/sanitizer_common_interceptors.h +//===----------------------------------------------------------------------===// + +#include "interception/interception.h" +#include "hwasan.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +#include +// ACHTUNG! No other system header includes in this file. +// Ideally, we should get rid of stdarg.h as well. + +using namespace __hwasan; + +using __sanitizer::memory_order; +using __sanitizer::atomic_load; +using __sanitizer::atomic_store; +using __sanitizer::atomic_uintptr_t; + +DECLARE_REAL(SIZE_T, strlen, const char *s) +DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen) +DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n) +DECLARE_REAL(void *, memset, void *dest, int c, uptr n) + +bool IsInInterceptorScope() { + HwasanThread *t = GetCurrentThread(); + return t && t->InInterceptorScope(); +} + +struct InterceptorScope { + InterceptorScope() { + HwasanThread *t = GetCurrentThread(); + if (t) + t->EnterInterceptorScope(); + } + ~InterceptorScope() { + HwasanThread *t = GetCurrentThread(); + if (t) + t->LeaveInterceptorScope(); + } +}; + +static uptr allocated_for_dlsym; +static const uptr kDlsymAllocPoolSize = 1024; +static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; + +static bool IsInDlsymAllocPool(const void *ptr) { + uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + return off < sizeof(alloc_memory_for_dlsym); +} + +static void *AllocateFromLocalPool(uptr size_in_bytes) { + uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; + void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; + allocated_for_dlsym += size_in_words; + CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); + return mem; +} + +#define ENSURE_HWASAN_INITED() do { \ + CHECK(!hwasan_init_is_running); \ + if (!hwasan_inited) { \ + __hwasan_init(); \ + } \ +} while (0) + + + +#define HWASAN_READ_RANGE(ctx, offset, size) \ + CHECK_UNPOISONED(offset, size) +#define HWASAN_WRITE_RANGE(ctx, offset, size) \ + CHECK_UNPOISONED(offset, size) + + + +// Check that [x, x+n) range is unpoisoned. +#define CHECK_UNPOISONED_0(x, n) \ + do { \ + sptr __offset = __hwasan_test_shadow(x, n); \ + if (__hwasan::IsInSymbolizer()) break; \ + if (__offset >= 0) { \ + GET_CALLER_PC_BP_SP; \ + (void)sp; \ + ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \ + __hwasan::PrintWarning(pc, bp); \ + if (__hwasan::flags()->halt_on_error) { \ + Printf("Exiting\n"); \ + Die(); \ + } \ + } \ + } while (0) + +// Check that [x, x+n) range is unpoisoned unless we are in a nested +// interceptor. +#define CHECK_UNPOISONED(x, n) \ + do { \ + if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \ + } while (0) + +#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \ + CHECK_UNPOISONED((x), \ + common_flags()->strict_string_checks ? (len) + 1 : (n) ) + + +INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + CHECK_NE(memptr, 0); + int res = hwasan_posix_memalign(memptr, alignment, size, &stack); + return res; +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_memalign(alignment, size, &stack); +} +#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign) +#else +#define HWASAN_MAYBE_INTERCEPT_MEMALIGN +#endif + +INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_aligned_alloc(alignment, size, &stack); +} + +INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + void *ptr = hwasan_memalign(alignment, size, &stack); + if (ptr) + DTLS_on_libc_memalign(ptr, size); + return ptr; +} + +INTERCEPTOR(void *, valloc, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_valloc(size, &stack); +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void *, pvalloc, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + return hwasan_pvalloc(size, &stack); +} +#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc) +#else +#define HWASAN_MAYBE_INTERCEPT_PVALLOC +#endif + +INTERCEPTOR(void, free, void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + HwasanDeallocate(&stack, ptr); +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void, cfree, void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + HwasanDeallocate(&stack, ptr); +} +#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) +#else +#define HWASAN_MAYBE_INTERCEPT_CFREE +#endif + +INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { + return __sanitizer_get_allocated_size(ptr); +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +// This function actually returns a struct by value, but we can't unpoison a +// temporary! The following is equivalent on all supported platforms but +// aarch64 (which uses a different register for sret value). We have a test +// to confirm that. +INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { +#ifdef __aarch64__ + uptr r8; + asm volatile("mov %0,x8" : "=r" (r8)); + sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); +#endif + REAL(memset)(sret, 0, sizeof(*sret)); +} +#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) +#else +#define HWASAN_MAYBE_INTERCEPT_MALLINFO +#endif + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(int, mallopt, int cmd, int value) { + return -1; +} +#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) +#else +#define HWASAN_MAYBE_INTERCEPT_MALLOPT +#endif + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void, malloc_stats, void) { + // FIXME: implement, but don't call REAL(malloc_stats)! +} +#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats) +#else +#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS +#endif + + +INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. + return AllocateFromLocalPool(nmemb * size); + return hwasan_calloc(nmemb, size, &stack); +} + +INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(IsInDlsymAllocPool(ptr))) { + uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(!hwasan_inited)) { + new_ptr = AllocateFromLocalPool(copy_size); + } else { + copy_size = size; + new_ptr = hwasan_malloc(copy_size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; + } + return hwasan_realloc(ptr, size, &stack); +} + +INTERCEPTOR(void *, malloc, SIZE_T size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. + return AllocateFromLocalPool(size); + return hwasan_malloc(size, &stack); +} + + +INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, + int fd, OFF_T offset) { + if (hwasan_init_is_running) + return REAL(mmap)(addr, length, prot, flags, fd, offset); + ENSURE_HWASAN_INITED(); + if (addr && !MEM_IS_APP(addr)) { + if (flags & map_fixed) { + errno = errno_EINVAL; + return (void *)-1; + } else { + addr = nullptr; + } + } + void *res = REAL(mmap)(addr, length, prot, flags, fd, offset); + return res; +} + +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags, + int fd, OFF64_T offset) { + ENSURE_HWASAN_INITED(); + if (addr && !MEM_IS_APP(addr)) { + if (flags & map_fixed) { + errno = errno_EINVAL; + return (void *)-1; + } else { + addr = nullptr; + } + } + void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset); + return res; +} +#define HWASAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64) +#else +#define HWASAN_MAYBE_INTERCEPT_MMAP64 +#endif + +extern "C" int pthread_attr_init(void *attr); +extern "C" int pthread_attr_destroy(void *attr); + +static void *HwasanThreadStartFunc(void *arg) { + HwasanThread *t = (HwasanThread *)arg; + SetCurrentThread(t); + return t->ThreadStart(); +} + +INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), + void * param) { + ENSURE_HWASAN_INITED(); // for GetTlsSize() + __sanitizer_pthread_attr_t myattr; + if (!attr) { + pthread_attr_init(&myattr); + attr = &myattr; + } + + AdjustStackSize(attr); + + HwasanThread *t = HwasanThread::Create(callback, param); + + int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t); + + if (attr == &myattr) + pthread_attr_destroy(&myattr); + return res; +} + +static void BeforeFork() { + StackDepotLockAll(); +} + +static void AfterFork() { + StackDepotUnlockAll(); +} + +INTERCEPTOR(int, fork, void) { + ENSURE_HWASAN_INITED(); + BeforeFork(); + int pid = REAL(fork)(); + AfterFork(); + return pid; +} + + +struct HwasanInterceptorContext { + bool in_interceptor_scope; +}; + +namespace __hwasan { + +int OnExit() { + // FIXME: ask frontend whether we need to return failure. + return 0; +} + +} // namespace __hwasan + +// A version of CHECK_UNPOISONED using a saved scope value. Used in common +// interceptors. +#define CHECK_UNPOISONED_CTX(ctx, x, n) \ + do { \ + if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \ + CHECK_UNPOISONED_0(x, n); \ + } while (0) + +#define HWASAN_INTERCEPT_FUNC(name) \ + do { \ + if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ + VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \ + } while (0) + +#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ + VReport( \ + 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ + } while (0) + +#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name) +#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ + HWASAN_INTERCEPT_FUNC_VER(name, ver) +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + CHECK_UNPOISONED_CTX(ctx, ptr, size) +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + CHECK_UNPOISONED_CTX(ctx, ptr, size) +#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \ + HWASAN_WRITE_RANGE(ctx, ptr, size) +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \ + ENSURE_HWASAN_INITED(); \ + HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \ + ctx = (void *)&hwasan_ctx; \ + (void)ctx; \ + InterceptorScope interceptor_scope; +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ + do { \ + } while (false) // FIXME +#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ + do { \ + } while (false) // FIXME +#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) +#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() + +#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ + if (HwasanThread *t = GetCurrentThread()) { \ + *begin = t->tls_begin(); \ + *end = t->tls_end(); \ + } else { \ + *begin = *end = 0; \ + } + +#include "sanitizer_common/sanitizer_platform_interceptors.h" +#include "sanitizer_common/sanitizer_common_interceptors.inc" +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) +#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#include "sanitizer_common/sanitizer_common_syscalls.inc" + + + +namespace __hwasan { + +void InitializeInterceptors() { + static int inited = 0; + CHECK_EQ(inited, 0); + InitializeCommonInterceptors(); + InitializeSignalInterceptors(); + + INTERCEPT_FUNCTION(mmap); + HWASAN_MAYBE_INTERCEPT_MMAP64; + INTERCEPT_FUNCTION(posix_memalign); + HWASAN_MAYBE_INTERCEPT_MEMALIGN; + INTERCEPT_FUNCTION(__libc_memalign); + INTERCEPT_FUNCTION(valloc); + HWASAN_MAYBE_INTERCEPT_PVALLOC; + INTERCEPT_FUNCTION(malloc); + INTERCEPT_FUNCTION(calloc); + INTERCEPT_FUNCTION(realloc); + INTERCEPT_FUNCTION(free); + HWASAN_MAYBE_INTERCEPT_CFREE; + INTERCEPT_FUNCTION(malloc_usable_size); + HWASAN_MAYBE_INTERCEPT_MALLINFO; + HWASAN_MAYBE_INTERCEPT_MALLOPT; + HWASAN_MAYBE_INTERCEPT_MALLOC_STATS; + INTERCEPT_FUNCTION(pthread_create); + INTERCEPT_FUNCTION(fork); + + inited = 1; +} +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h new file mode 100644 index 000000000000..7e95271ac0af --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -0,0 +1,123 @@ +//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Private Hwasan interface header. +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_INTERFACE_INTERNAL_H +#define HWASAN_INTERFACE_INTERNAL_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_init(); + +using __sanitizer::uptr; +using __sanitizer::sptr; +using __sanitizer::uu64; +using __sanitizer::uu32; +using __sanitizer::uu16; +using __sanitizer::u64; +using __sanitizer::u32; +using __sanitizer::u16; +using __sanitizer::u8; + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load(uptr, uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16(uptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load_noabort(uptr, uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16_noabort(uptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store(uptr, uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16(uptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store_noabort(uptr, uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16_noabort(uptr); + +// Returns the offset of the first tag mismatch or -1 if the whole range is +// good. +SANITIZER_INTERFACE_ATTRIBUTE +sptr __hwasan_test_shadow(const void *x, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +/* OPTIONAL */ const char* __hwasan_default_options(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_print_shadow(const void *x, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +u16 __sanitizer_unaligned_load16(const uu16 *p); + +SANITIZER_INTERFACE_ATTRIBUTE +u32 __sanitizer_unaligned_load32(const uu32 *p); + +SANITIZER_INTERFACE_ATTRIBUTE +u64 __sanitizer_unaligned_load64(const uu64 *p); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store16(uu16 *p, u16 x); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store32(uu32 *p, u32 x); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store64(uu64 *p, u64 x); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_enable_allocator_tagging(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_disable_allocator_tagging(); + +} // extern "C" + +#endif // HWASAN_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc new file mode 100644 index 000000000000..48dea8eb6ff4 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc @@ -0,0 +1,255 @@ +//===-- hwasan_linux.cc -----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Linux-, NetBSD- and FreeBSD-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD + +#include "hwasan.h" +#include "hwasan_thread.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_procmaps.h" + +namespace __hwasan { + +void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { + CHECK_EQ((beg % GetMmapGranularity()), 0); + CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); + uptr size = end - beg + 1; + DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. + void *res = MmapFixedNoReserve(beg, size, name); + if (res != (void *)beg) { + Report( + "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " + "Perhaps you're using ulimit -v\n", + size); + Abort(); + } + if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); +} + +static void ProtectGap(uptr addr, uptr size) { + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + // A few pages at the start of the address space can not be protected. + // But we really want to protect as much as possible, to prevent this memory + // being returned as a result of a non-FIXED mmap(). + if (addr == 0) { + uptr step = GetMmapGranularity(); + while (size > step) { + addr += step; + size -= step; + void *res = MmapFixedNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) return; + } + } + + Report( + "ERROR: Failed to protect the shadow gap. " + "ASan cannot proceed correctly. ABORTING.\n"); + DumpProcessMap(); + Die(); +} + +bool InitShadow() { + const uptr maxVirtualAddress = GetMaxUserVirtualAddress(); + + // LowMem covers as much of the first 4GB as possible. + const uptr kLowMemEnd = 1UL<<32; + const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale; + const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale; + + // HighMem covers the upper part of the address space. + const uptr kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1; + const uptr kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale); + CHECK(kHighShadowStart < kHighShadowEnd); + + const uptr kHighMemStart = kHighShadowStart << kShadowScale; + CHECK(kHighShadowEnd <= kHighMemStart); + + if (Verbosity()) { + Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemStart, + (void *)maxVirtualAddress); + if (kHighMemStart > kHighShadowEnd) + Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd, + (void *)kHighMemStart); + Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowStart, + (void *)kHighShadowEnd); + if (kHighShadowStart > kLowMemEnd) + Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd, + (void *)kHighMemStart); + Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowShadowEnd, + (void *)kLowMemEnd); + Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowStart, + (void *)kLowShadowEnd); + Printf("|| `[%p, %p]` || ShadowGap1 ||\n", (void *)0, + (void *)kLowShadowStart); + } + + ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd - 1, "low shadow"); + ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd - 1, "high shadow"); + ProtectGap(0, kLowShadowStart); + if (kHighShadowStart > kLowMemEnd) + ProtectGap(kLowMemEnd, kHighShadowStart - kLowMemEnd); + if (kHighMemStart > kHighShadowEnd) + ProtectGap(kHighShadowEnd, kHighMemStart - kHighShadowEnd); + + return true; +} + +static void HwasanAtExit(void) { + if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) + ReportStats(); + if (hwasan_report_count > 0) { + // ReportAtExitStatistics(); + if (common_flags()->exitcode) + internal__exit(common_flags()->exitcode); + } +} + +void InstallAtExitHandler() { + atexit(HwasanAtExit); +} + +// ---------------------- TSD ---------------- {{{1 + +static pthread_key_t tsd_key; +static bool tsd_key_inited = false; + +void HwasanTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_key_inited); + tsd_key_inited = true; + CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); +} + +HwasanThread *GetCurrentThread() { + return (HwasanThread*)pthread_getspecific(tsd_key); +} + +void SetCurrentThread(HwasanThread *t) { + // Make sure that HwasanTSDDtor gets called at the end. + CHECK(tsd_key_inited); + // Make sure we do not reset the current HwasanThread. + CHECK_EQ(0, pthread_getspecific(tsd_key)); + pthread_setspecific(tsd_key, (void *)t); +} + +void HwasanTSDDtor(void *tsd) { + HwasanThread *t = (HwasanThread*)tsd; + if (t->destructor_iterations_ > 1) { + t->destructor_iterations_--; + CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); + return; + } + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + HwasanThread::TSDDtor(tsd); +} + +struct AccessInfo { + uptr addr; + uptr size; + bool is_store; + bool is_load; + bool recover; +}; + +#if defined(__aarch64__) +static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { + // Access type is encoded in HLT immediate as 0x1XY, + // where X&1 is 1 for store, 0 for load, + // and X&2 is 1 if the error is recoverable. + // Valid values of Y are 0 to 4, which are interpreted as log2(access_size), + // and 0xF, which means that access size is stored in X1 register. + // Access address is always in X0 register. + AccessInfo ai; + uptr pc = (uptr)info->si_addr; + unsigned code = ((*(u32 *)pc) >> 5) & 0xffff; + if ((code & 0xff00) != 0x100) + return AccessInfo{0, 0, false, false}; // Not ours. + bool is_store = code & 0x10; + bool recover = code & 0x20; + unsigned size_log = code & 0xf; + if (size_log > 4 && size_log != 0xf) + return AccessInfo{0, 0, false, false}; // Not ours. + + ai.is_store = is_store; + ai.is_load = !is_store; + ai.addr = uc->uc_mcontext.regs[0]; + if (size_log == 0xf) + ai.size = uc->uc_mcontext.regs[1]; + else + ai.size = 1U << size_log; + ai.recover = recover; + return ai; +} +#else +static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { + return AccessInfo{0, 0, false, false}; +} +#endif + +static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) { + SignalContext sig{info, uc}; + AccessInfo ai = GetAccessInfo(info, uc); + if (!ai.is_store && !ai.is_load) + return false; + + InternalScopedBuffer stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc, + common_flags()->fast_unwind_on_fatal); + + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store); + + ++hwasan_report_count; + if (flags()->halt_on_error || !ai.recover) + Die(); + + uc->uc_mcontext.pc += 4; + return true; +} + +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +void HwasanOnDeadlySignal(int signo, void *info, void *context) { + // Probably a tag mismatch. + if (signo == SIGILL) + if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context)) + return; + + HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); +} + + +} // namespace __hwasan + +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc new file mode 100644 index 000000000000..3ccc26734bf6 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc @@ -0,0 +1,66 @@ +//===-- hwasan_new_delete.cc ------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Interceptors for operators new and delete. +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator.h" + +#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE + +#include + +using namespace __hwasan; // NOLINT + +// Fake std::nothrow_t to avoid including . +namespace std { + struct nothrow_t {}; +} // namespace std + + +// TODO(alekseys): throw std::bad_alloc instead of dying on OOM. +#define OPERATOR_NEW_BODY(nothrow) \ + GET_MALLOC_STACK_TRACE; \ + void *res = hwasan_malloc(size, &stack);\ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ + return res + +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(true /*nothrow*/); +} +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(true /*nothrow*/); +} + +#define OPERATOR_DELETE_BODY \ + GET_MALLOC_STACK_TRACE; \ + if (ptr) HwasanDeallocate(&stack, ptr) + +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const&) { + OPERATOR_DELETE_BODY; +} + +#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc new file mode 100644 index 000000000000..411fd05b10d5 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc @@ -0,0 +1,36 @@ +//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "hwasan_poisoning.h" + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace __hwasan { + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MEM_TO_SHADOW(p); + uptr shadow_size = MEM_TO_SHADOW_OFFSET(size); + internal_memset((void *)shadow_start, tag, shadow_size); + return AddTagToPointer(p, tag); +} + +uptr TagMemory(uptr p, uptr size, tag_t tag) { + uptr start = RoundDownTo(p, kShadowAlignment); + uptr end = RoundUpTo(p + size, kShadowAlignment); + return TagMemoryAligned(start, end - start, tag); +} + +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h new file mode 100644 index 000000000000..b44a91f975f5 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h @@ -0,0 +1,25 @@ +//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_POISONING_H +#define HWASAN_POISONING_H + +#include "hwasan.h" + +namespace __hwasan { +uptr TagMemory(uptr p, uptr size, tag_t tag); +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag); + +} // namespace __hwasan + +#endif // HWASAN_POISONING_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc new file mode 100644 index 000000000000..a3c6709491d3 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc @@ -0,0 +1,133 @@ +//===-- hwasan_report.cc ----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Error reporting. +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "hwasan_allocator.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_mutex.h" +#include "sanitizer_common/sanitizer_report_decorator.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_symbolizer.h" + +using namespace __sanitizer; + +namespace __hwasan { + +static StackTrace GetStackTraceFromId(u32 id) { + CHECK(id); + StackTrace res = StackDepotGet(id); + CHECK(res.trace); + return res; +} + +class Decorator: public __sanitizer::SanitizerCommonDecorator { + public: + Decorator() : SanitizerCommonDecorator() { } + const char *Allocation() { return Magenta(); } + const char *Origin() { return Magenta(); } + const char *Name() { return Green(); } +}; + +struct HeapAddressDescription { + uptr addr; + u32 alloc_stack_id; + u32 free_stack_id; + + void Print() const { + Decorator d; + if (free_stack_id) { + Printf("%sfreed here:%s\n", d.Allocation(), d.Default()); + GetStackTraceFromId(free_stack_id).Print(); + Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default()); + } else { + Printf("%sallocated here:%s\n", d.Allocation(), d.Default()); + } + GetStackTraceFromId(alloc_stack_id).Print(); + } +}; + +bool GetHeapAddressInformation(uptr addr, uptr access_size, + HeapAddressDescription *description) { + HwasanChunkView chunk = FindHeapChunkByAddress(addr); + if (!chunk.IsValid()) + return false; + description->addr = addr; + description->alloc_stack_id = chunk.GetAllocStackId(); + description->free_stack_id = chunk.GetFreeStackId(); + return true; +} + +void PrintAddressDescription(uptr addr, uptr access_size) { + HeapAddressDescription heap_description; + if (GetHeapAddressInformation(addr, access_size, &heap_description)) { + heap_description.Print(); + return; + } + // We exhausted our possibilities. Bail out. + Printf("HWAddressSanitizer can not describe address in more detail.\n"); +} + +void ReportInvalidAccess(StackTrace *stack, u32 origin) { + ScopedErrorReportLock l; + + Decorator d; + Printf("%s", d.Warning()); + Report("WARNING: HWAddressSanitizer: invalid access\n"); + Printf("%s", d.Default()); + stack->Print(); + ReportErrorSummary("invalid-access", stack); +} + +void ReportStats() {} + +void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, + uptr size, uptr offset) { + ScopedErrorReportLock l; + + Decorator d; + Printf("%s", d.Warning()); + Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n", + d.Warning(), d.Name(), what, d.Warning(), offset, start, size, + d.Default()); + PrintAddressDescription((uptr)start + offset, 1); + // if (__sanitizer::Verbosity()) + // DescribeMemoryRange(start, size); +} + +void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, + bool is_store) { + ScopedErrorReportLock l; + + Decorator d; + Printf("%s", d.Warning()); + uptr address = GetAddressFromPointer(addr); + Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size, + address); + + tag_t ptr_tag = GetTagFromPointer(addr); + tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address); + Printf("pointer tag 0x%x\nmemory tag 0x%x\n", ptr_tag, mem_tag); + Printf("%s", d.Default()); + + stack->Print(); + + PrintAddressDescription(address, access_size); + + ReportErrorSummary("tag-mismatch", stack); +} + + +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc new file mode 100644 index 000000000000..d6551ffc6fdc --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc @@ -0,0 +1,75 @@ + +#include "hwasan.h" +#include "hwasan_thread.h" +#include "hwasan_poisoning.h" +#include "hwasan_interface_internal.h" + +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +namespace __hwasan { + +HwasanThread *HwasanThread::Create(thread_callback_t start_routine, + void *arg) { + uptr PageSize = GetPageSizeCached(); + uptr size = RoundUpTo(sizeof(HwasanThread), PageSize); + HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__); + thread->start_routine_ = start_routine; + thread->arg_ = arg; + thread->destructor_iterations_ = GetPthreadDestructorIterations(); + + return thread; +} + +void HwasanThread::SetThreadStackAndTls() { + uptr tls_size = 0; + uptr stack_size = 0; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, + &tls_begin_, &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; + + int local; + CHECK(AddrIsInStack((uptr)&local)); +} + +void HwasanThread::Init() { + SetThreadStackAndTls(); + CHECK(MEM_IS_APP(stack_bottom_)); + CHECK(MEM_IS_APP(stack_top_ - 1)); +} + +void HwasanThread::TSDDtor(void *tsd) { + HwasanThread *t = (HwasanThread*)tsd; + t->Destroy(); +} + +void HwasanThread::ClearShadowForThreadStackAndTLS() { + TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); + if (tls_begin_ != tls_end_) + TagMemory(tls_begin_, tls_end_ - tls_begin_, 0); +} + +void HwasanThread::Destroy() { + malloc_storage().CommitBack(); + ClearShadowForThreadStackAndTLS(); + uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached()); + UnmapOrDie(this, size); + DTLS_Destroy(); +} + +thread_return_t HwasanThread::ThreadStart() { + Init(); + + if (!start_routine_) { + // start_routine_ == 0 if we're on the main thread or on one of the + // OS X libdispatch worker threads. But nobody is supposed to call + // ThreadStart() for the worker threads. + return 0; + } + + thread_return_t res = start_routine_(arg_); + + return res; +} + +} // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h new file mode 100644 index 000000000000..96f1bb813adf --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h @@ -0,0 +1,81 @@ +//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_THREAD_H +#define HWASAN_THREAD_H + +#include "hwasan_allocator.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace __hwasan { + +class HwasanThread { + public: + static HwasanThread *Create(thread_callback_t start_routine, void *arg); + static void TSDDtor(void *tsd); + void Destroy(); + + void Init(); // Should be called from the thread itself. + thread_return_t ThreadStart(); + + uptr stack_top() { return stack_top_; } + uptr stack_bottom() { return stack_bottom_; } + uptr tls_begin() { return tls_begin_; } + uptr tls_end() { return tls_end_; } + bool IsMainThread() { return start_routine_ == nullptr; } + + bool AddrIsInStack(uptr addr) { + return addr >= stack_bottom_ && addr < stack_top_; + } + + bool InSignalHandler() { return in_signal_handler_; } + void EnterSignalHandler() { in_signal_handler_++; } + void LeaveSignalHandler() { in_signal_handler_--; } + + bool InSymbolizer() { return in_symbolizer_; } + void EnterSymbolizer() { in_symbolizer_++; } + void LeaveSymbolizer() { in_symbolizer_--; } + + bool InInterceptorScope() { return in_interceptor_scope_; } + void EnterInterceptorScope() { in_interceptor_scope_++; } + void LeaveInterceptorScope() { in_interceptor_scope_--; } + + HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } + + int destructor_iterations_; + + private: + // NOTE: There is no HwasanThread constructor. It is allocated + // via mmap() and *must* be valid in zero-initialized state. + void SetThreadStackAndTls(); + void ClearShadowForThreadStackAndTLS(); + thread_callback_t start_routine_; + void *arg_; + uptr stack_top_; + uptr stack_bottom_; + uptr tls_begin_; + uptr tls_end_; + + unsigned in_signal_handler_; + unsigned in_symbolizer_; + unsigned in_interceptor_scope_; + + HwasanThreadLocalMallocStorage malloc_storage_; +}; + +HwasanThread *GetCurrentThread(); +void SetCurrentThread(HwasanThread *t); + +} // namespace __hwasan + +#endif // HWASAN_THREAD_H diff --git a/contrib/compiler-rt/lib/interception/interception.h b/contrib/compiler-rt/lib/interception/interception.h index d79fa67babfa..cba484936eac 100644 --- a/contrib/compiler-rt/lib/interception/interception.h +++ b/contrib/compiler-rt/lib/interception/interception.h @@ -15,13 +15,14 @@ #ifndef INTERCEPTION_H #define INTERCEPTION_H -#if !defined(__linux__) && !defined(__FreeBSD__) && \ - !defined(__APPLE__) && !defined(_WIN32) +#include "sanitizer_common/sanitizer_internal_defs.h" + +#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \ + !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \ + !SANITIZER_SOLARIS # error "Interception doesn't work on this operating system." #endif -#include "sanitizer_common/sanitizer_internal_defs.h" - // These typedefs should be used only in the interceptor definitions to replace // the standard system types (e.g. SSIZE_T instead of ssize_t) typedef __sanitizer::uptr SIZE_T; @@ -87,7 +88,7 @@ typedef __sanitizer::OFF64_T OFF64_T; // As it's decided at compile time which functions are to be intercepted on Mac, // INTERCEPT_FUNCTION() is effectively a no-op on this system. -#if defined(__APPLE__) +#if SANITIZER_MAC #include // For __DARWIN_ALIAS_C(). // Just a pair of pointers. @@ -121,7 +122,7 @@ const interpose_substitution substitution_##func_name[] \ # define INTERCEPTOR_ATTRIBUTE # define DECLARE_WRAPPER(ret_type, func, ...) -#elif defined(_WIN32) +#elif SANITIZER_WINDOWS # define WRAP(x) __asan_wrap_##x # define WRAPPER_NAME(x) "__asan_wrap_"#x # define INTERCEPTOR_ATTRIBUTE __declspec(dllexport) @@ -129,7 +130,7 @@ const interpose_substitution substitution_##func_name[] \ extern "C" ret_type func(__VA_ARGS__); # define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__); -#elif defined(__FreeBSD__) +#elif SANITIZER_FREEBSD || SANITIZER_NETBSD # define WRAP(x) __interceptor_ ## x # define WRAPPER_NAME(x) "__interceptor_" #x # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) @@ -139,7 +140,7 @@ const interpose_substitution substitution_##func_name[] \ # define DECLARE_WRAPPER(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__) \ __attribute__((alias("__interceptor_" #func), visibility("default"))); -#else +#elif !SANITIZER_FUCHSIA # define WRAP(x) __interceptor_ ## x # define WRAPPER_NAME(x) "__interceptor_" #x # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) @@ -148,7 +149,15 @@ const interpose_substitution substitution_##func_name[] \ __attribute__((weak, alias("__interceptor_" #func), visibility("default"))); #endif -#if !defined(__APPLE__) +#if SANITIZER_FUCHSIA +// There is no general interception at all on Fuchsia. +// Sanitizer runtimes just define functions directly to preempt them, +// and have bespoke ways to access the underlying libc functions. +# include +# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) +# define REAL(x) __unsanitized_##x +# define DECLARE_REAL(ret_type, func, ...) +#elif !SANITIZER_MAC # define PTR_TO_REAL(x) real_##x # define REAL(x) __interception::PTR_TO_REAL(x) # define FUNC_TYPE(x) x##_f @@ -159,22 +168,26 @@ const interpose_substitution substitution_##func_name[] \ extern FUNC_TYPE(func) PTR_TO_REAL(func); \ } # define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src) -#else // __APPLE__ +#else // SANITIZER_MAC # define REAL(x) x # define DECLARE_REAL(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__); # define ASSIGN_REAL(x, y) -#endif // __APPLE__ +#endif // SANITIZER_MAC +#if !SANITIZER_FUCHSIA #define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ DECLARE_REAL(ret_type, func, __VA_ARGS__) \ extern "C" ret_type WRAP(func)(__VA_ARGS__); +#else +#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) +#endif // Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR // macros does its job. In exceptional cases you may need to call REAL(foo) // without defining INTERCEPTOR(..., foo, ...). For example, if you override // foo with an interceptor for other function. -#if !defined(__APPLE__) +#if !SANITIZER_MAC && !SANITIZER_FUCHSIA # define DEFINE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ namespace __interception { \ @@ -184,7 +197,18 @@ const interpose_substitution substitution_##func_name[] \ # define DEFINE_REAL(ret_type, func, ...) #endif -#if !defined(__APPLE__) +#if SANITIZER_FUCHSIA + +// We need to define the __interceptor_func name just to get +// sanitizer_common/scripts/gen_dynamic_list.py to export func. +// But we don't need to export __interceptor_func to get that. +#define INTERCEPTOR(ret_type, func, ...) \ + extern "C"[[ gnu::alias(#func), gnu::visibility("hidden") ]] ret_type \ + __interceptor_##func(__VA_ARGS__); \ + extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__) + +#elif !SANITIZER_MAC + #define INTERCEPTOR(ret_type, func, ...) \ DEFINE_REAL(ret_type, func, __VA_ARGS__) \ DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \ @@ -196,7 +220,7 @@ const interpose_substitution substitution_##func_name[] \ #define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \ INTERCEPTOR(ret_type, func, __VA_ARGS__) -#else // __APPLE__ +#else // SANITIZER_MAC #define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__) suffix; \ @@ -215,7 +239,7 @@ const interpose_substitution substitution_##func_name[] \ INTERPOSER_2(overridee, WRAP(overrider)) #endif -#if defined(_WIN32) +#if SANITIZER_WINDOWS # define INTERCEPTOR_WINAPI(ret_type, func, ...) \ typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \ namespace __interception { \ @@ -241,17 +265,19 @@ typedef unsigned long uptr; // NOLINT #define INCLUDED_FROM_INTERCEPTION_LIB -#if defined(__linux__) || defined(__FreeBSD__) +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + # include "interception_linux.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) -#elif defined(__APPLE__) +#elif SANITIZER_MAC # include "interception_mac.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ INTERCEPT_FUNCTION_VER_MAC(func, symver) -#else // defined(_WIN32) +#elif SANITIZER_WINDOWS # include "interception_win.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_WIN(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ diff --git a/contrib/compiler-rt/lib/interception/interception_linux.cc b/contrib/compiler-rt/lib/interception/interception_linux.cc index 6e908ac017b2..c991550a4f72 100644 --- a/contrib/compiler-rt/lib/interception/interception_linux.cc +++ b/contrib/compiler-rt/lib/interception/interception_linux.cc @@ -12,25 +12,44 @@ // Linux-specific interception methods. //===----------------------------------------------------------------------===// -#if defined(__linux__) || defined(__FreeBSD__) #include "interception.h" +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + #include // for dlsym() and dlvsym() +#if SANITIZER_NETBSD +#include "sanitizer_common/sanitizer_libc.h" +#endif + namespace __interception { bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, uptr real, uptr wrapper) { +#if SANITIZER_NETBSD + // XXX: Find a better way to handle renames + if (internal_strcmp(func_name, "sigaction") == 0) func_name = "__sigaction14"; +#endif *func_addr = (uptr)dlsym(RTLD_NEXT, func_name); + if (!*func_addr) { + // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is + // later in the library search order than the DSO that we are trying to + // intercept, which means that we cannot intercept this function. We still + // want the address of the real definition, though, so look it up using + // RTLD_DEFAULT. + *func_addr = (uptr)dlsym(RTLD_DEFAULT, func_name); + } return real == wrapper; } -#if !defined(__ANDROID__) // android does not have dlvsym +// Android and Solaris do not have dlvsym +#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS void *GetFuncAddrVer(const char *func_name, const char *ver) { return dlvsym(RTLD_NEXT, func_name, ver); } -#endif // !defined(__ANDROID__) +#endif // !SANITIZER_ANDROID } // namespace __interception - -#endif // __linux__ || __FreeBSD__ +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/interception/interception_linux.h b/contrib/compiler-rt/lib/interception/interception_linux.h index 27a66c882041..98fe51b858fc 100644 --- a/contrib/compiler-rt/lib/interception/interception_linux.h +++ b/contrib/compiler-rt/lib/interception/interception_linux.h @@ -12,7 +12,8 @@ // Linux-specific interception methods. //===----------------------------------------------------------------------===// -#if defined(__linux__) || defined(__FreeBSD__) +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_linux.h should be included from interception library only" @@ -34,14 +35,16 @@ void *GetFuncAddrVer(const char *func_name, const char *ver); (::__interception::uptr) & (func), \ (::__interception::uptr) & WRAP(func)) -#if !defined(__ANDROID__) // android does not have dlvsym +// Android and Solaris do not have dlvsym +#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ (::__interception::real_##func = (func##_f)( \ unsigned long)::__interception::GetFuncAddrVer(#func, symver)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) -#endif // !defined(__ANDROID__) +#endif // !SANITIZER_ANDROID && !SANITIZER_SOLARIS #endif // INTERCEPTION_LINUX_H -#endif // __linux__ || __FreeBSD__ +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/interception/interception_mac.cc b/contrib/compiler-rt/lib/interception/interception_mac.cc index b035cf998140..ea8072f8dd73 100644 --- a/contrib/compiler-rt/lib/interception/interception_mac.cc +++ b/contrib/compiler-rt/lib/interception/interception_mac.cc @@ -12,9 +12,8 @@ // Mac-specific interception methods. //===----------------------------------------------------------------------===// -#ifdef __APPLE__ - #include "interception.h" +#if SANITIZER_MAC -#endif // __APPLE__ +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/interception/interception_mac.h b/contrib/compiler-rt/lib/interception/interception_mac.h index e5a35c6971cd..6f31fed47b84 100644 --- a/contrib/compiler-rt/lib/interception/interception_mac.h +++ b/contrib/compiler-rt/lib/interception/interception_mac.h @@ -12,7 +12,7 @@ // Mac-specific interception methods. //===----------------------------------------------------------------------===// -#ifdef __APPLE__ +#if SANITIZER_MAC #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_mac.h should be included from interception.h only" @@ -25,4 +25,4 @@ #define INTERCEPT_FUNCTION_VER_MAC(func, symver) #endif // INTERCEPTION_MAC_H -#endif // __APPLE__ +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/interception/interception_type_test.cc b/contrib/compiler-rt/lib/interception/interception_type_test.cc index 98ce2e6afc1b..2b3a6d509bfb 100644 --- a/contrib/compiler-rt/lib/interception/interception_type_test.cc +++ b/contrib/compiler-rt/lib/interception/interception_type_test.cc @@ -12,9 +12,10 @@ // Compile-time tests of the internal type definitions. //===----------------------------------------------------------------------===// -#if defined(__linux__) || defined(__APPLE__) - #include "interception.h" + +#if SANITIZER_LINUX || SANITIZER_MAC + #include #include #include @@ -24,14 +25,14 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t)); COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t)); COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t)); -#ifndef __APPLE__ +#if !SANITIZER_MAC COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t)); #endif // The following are the cases when pread (and friends) is used instead of // pread64. In those cases we need OFF_T to match off_t. We don't care about the // rest (they depend on _FILE_OFFSET_BITS setting when building an application). -# if defined(__ANDROID__) || !defined _FILE_OFFSET_BITS || \ +# if SANITIZER_ANDROID || !defined _FILE_OFFSET_BITS || \ _FILE_OFFSET_BITS != 64 COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t)); # endif diff --git a/contrib/compiler-rt/lib/interception/interception_win.cc b/contrib/compiler-rt/lib/interception/interception_win.cc index b2902d57f542..dc3fe35242ef 100644 --- a/contrib/compiler-rt/lib/interception/interception_win.cc +++ b/contrib/compiler-rt/lib/interception/interception_win.cc @@ -125,9 +125,9 @@ // addr2: .bytes //===----------------------------------------------------------------------===// -#ifdef _WIN32 - #include "interception.h" + +#if SANITIZER_WINDOWS #include "sanitizer_common/sanitizer_platform.h" #define WIN32_LEAN_AND_MEAN #include @@ -223,9 +223,8 @@ static bool IsMemoryPadding(uptr address, uptr size) { return true; } -static const u8 kHintNop10Bytes[] = { - 0x66, 0x66, 0x0F, 0x1F, 0x84, - 0x00, 0x00, 0x00, 0x00, 0x00 +static const u8 kHintNop9Bytes[] = { + 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; template @@ -240,8 +239,8 @@ static bool FunctionHasPrefix(uptr address, const T &pattern) { static bool FunctionHasPadding(uptr address, uptr size) { if (IsMemoryPadding(address - size, size)) return true; - if (size <= sizeof(kHintNop10Bytes) && - FunctionHasPrefix(address, kHintNop10Bytes)) + if (size <= sizeof(kHintNop9Bytes) && + FunctionHasPrefix(address, kHintNop9Bytes)) return true; return false; } @@ -554,7 +553,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi + case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx return 5; + case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY + return 6; } #else @@ -833,6 +835,7 @@ bool OverrideFunction( static void **InterestingDLLsAvailable() { static const char *InterestingDLLs[] = { "kernel32.dll", + "msvcr100.dll", // VS2010 "msvcr110.dll", // VS2012 "msvcr120.dll", // VS2013 "vcruntime140.dll", // VS2015 @@ -1010,4 +1013,4 @@ bool OverrideImportedFunction(const char *module_to_patch, } // namespace __interception -#endif // _WIN32 +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/interception/interception_win.h b/contrib/compiler-rt/lib/interception/interception_win.h index 9061f9ed4c21..71a44428f5c7 100644 --- a/contrib/compiler-rt/lib/interception/interception_win.h +++ b/contrib/compiler-rt/lib/interception/interception_win.h @@ -12,7 +12,7 @@ // Windows-specific interception methods. //===----------------------------------------------------------------------===// -#ifdef _WIN32 +#if SANITIZER_WINDOWS #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_win.h should be included from interception library only" @@ -81,4 +81,4 @@ void TestOnlyReleaseTrampolineRegions(); (::__interception::uptr *)&REAL(func)) #endif // INTERCEPTION_WIN_H -#endif // _WIN32 +#endif // SANITIZER_WINDOWS diff --git a/contrib/compiler-rt/lib/lsan/lsan.cc b/contrib/compiler-rt/lib/lsan/lsan.cc index 6c4767d61252..a9f7e399e14c 100644 --- a/contrib/compiler-rt/lib/lsan/lsan.cc +++ b/contrib/compiler-rt/lib/lsan/lsan.cc @@ -56,6 +56,9 @@ static void InitializeFlags() { RegisterLsanFlags(&parser, f); RegisterCommonFlags(&parser); + // Override from user-specified string. + const char *lsan_default_options = MaybeCallLsanDefaultOptions(); + parser.ParseString(lsan_default_options); parser.ParseString(GetEnv("LSAN_OPTIONS")); SetVerbosity(common_flags()->verbosity); @@ -65,6 +68,17 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void LsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind, + nullptr); +} + extern "C" void __lsan_init() { CHECK(!lsan_init_is_running); if (lsan_inited) @@ -80,6 +94,7 @@ extern "C" void __lsan_init() { InitTlsSize(); InitializeInterceptors(); InitializeThreadRegistry(); + InstallDeadlySignalHandlers(LsanOnDeadlySignal); u32 tid = ThreadCreate(0, 0, true); CHECK_EQ(tid, 0); ThreadStart(tid, GetTid()); diff --git a/contrib/compiler-rt/lib/lsan/lsan.h b/contrib/compiler-rt/lib/lsan/lsan.h index 7446e9507e0d..6baee81d00cd 100644 --- a/contrib/compiler-rt/lib/lsan/lsan.h +++ b/contrib/compiler-rt/lib/lsan/lsan.h @@ -12,24 +12,14 @@ // //===----------------------------------------------------------------------===// +#include "lsan_thread.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#define GET_STACK_TRACE(max_size, fast) \ - BufferedStackTrace stack; \ - { \ - uptr stack_top = 0, stack_bottom = 0; \ - ThreadContext *t; \ - if (fast && (t = CurrentThreadContext())) { \ - stack_top = t->stack_end(); \ - stack_bottom = t->stack_begin(); \ - } \ - if (!SANITIZER_MIPS || \ - IsValidFrame(GET_CURRENT_FRAME(), stack_top, stack_bottom)) { \ - stack.Unwind(max_size, StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ - /* context */ 0, stack_top, stack_bottom, fast); \ - } \ - } +#define GET_STACK_TRACE(max_size, fast) \ + __sanitizer::BufferedStackTrace stack; \ + GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \ + GET_CURRENT_FRAME(), nullptr, fast); #define GET_STACK_TRACE_FATAL \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) @@ -51,9 +41,27 @@ void ReplaceSystemMalloc(); __lsan_init(); \ } while (0) +// Get the stack trace with the given pc and bp. +// The pc will be in the position 0 of the resulting stack trace. +// The bp may refer to the current frame or to the caller's frame. +ALWAYS_INLINE +void GetStackTrace(__sanitizer::BufferedStackTrace *stack, + __sanitizer::uptr max_depth, __sanitizer::uptr pc, + __sanitizer::uptr bp, void *context, bool fast) { + uptr stack_top = 0, stack_bottom = 0; + ThreadContext *t; + if (fast && (t = CurrentThreadContext())) { + stack_top = t->stack_end(); + stack_bottom = t->stack_begin(); + } + if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { + stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast); + } +} + } // namespace __lsan extern bool lsan_inited; extern bool lsan_init_is_running; -extern "C" void __lsan_init(); +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __lsan_init(); diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.h b/contrib/compiler-rt/lib/lsan/lsan_allocator.h index 5a0d94c71415..4006f7929269 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.h +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.h @@ -68,9 +68,16 @@ struct AP32 { }; typedef SizeClassAllocator32 PrimaryAllocator; #elif defined(__x86_64__) || defined(__powerpc64__) +# if defined(__powerpc64__) +const uptr kAllocatorSpace = 0xa0000000000ULL; +const uptr kAllocatorSize = 0x20000000000ULL; // 2T. +# else +const uptr kAllocatorSpace = 0x600000000000ULL; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +# endif struct AP64 { // Allocator64 parameters. Deliberately using a short name. - static const uptr kSpaceBeg = 0x600000000000ULL; - static const uptr kSpaceSize = 0x40000000000ULL; // 4T. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = kAllocatorSize; static const uptr kMetadataSize = sizeof(ChunkMetadata); typedef DefaultSizeClassMap SizeClassMap; typedef NoOpMapUnmapCallback MapUnmapCallback; diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.cc b/contrib/compiler-rt/lib/lsan/lsan_common.cc index c121e6a8fb24..69ffda539a26 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common.cc @@ -57,12 +57,12 @@ void RegisterLsanFlags(FlagParser *parser, Flags *f) { #define LOG_POINTERS(...) \ do { \ if (flags()->log_pointers) Report(__VA_ARGS__); \ - } while (0); + } while (0) #define LOG_THREADS(...) \ do { \ if (flags()->log_threads) Report(__VA_ARGS__); \ - } while (0); + } while (0) ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; static SuppressionContext *suppression_ctx = nullptr; @@ -107,6 +107,10 @@ void InitializeRootRegions() { root_regions = new(placeholder) InternalMmapVector(1); } +const char *MaybeCallLsanDefaultOptions() { + return (&__lsan_default_options) ? __lsan_default_options() : ""; +} + void InitCommonLsan() { InitializeRootRegions(); if (common_flags()->detect_leaks) { @@ -122,7 +126,6 @@ class Decorator: public __sanitizer::SanitizerCommonDecorator { Decorator() : SanitizerCommonDecorator() { } const char *Error() { return Red(); } const char *Leak() { return Blue(); } - const char *End() { return Default(); } }; static inline bool CanBeAHeapPointer(uptr p) { @@ -408,8 +411,9 @@ static void MarkInvalidPCCb(uptr chunk, void *arg) { } } -// On Linux, handles dynamically allocated TLS blocks by treating all chunks -// allocated from ld-linux.so as reachable. +// On Linux, treats all chunks allocated from ld-linux.so as reachable, which +// covers dynamically allocated TLS blocks, internal dynamic loader's loaded +// modules accounting etc. // Dynamic TLS blocks contain the TLS variables of dynamically loaded modules. // They are allocated with a __libc_memalign() call in allocate_and_init() // (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those @@ -564,7 +568,7 @@ static bool CheckForLeaks() { "\n"); Printf("%s", d.Error()); Report("ERROR: LeakSanitizer: detected memory leaks\n"); - Printf("%s", d.End()); + Printf("%s", d.Default()); param.leak_report.ReportTopLeaks(flags()->max_leaks); } if (common_flags()->print_suppressions) @@ -594,6 +598,8 @@ static int DoRecoverableLeakCheck() { return have_leaks ? 1 : 0; } +void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); } + static Suppression *GetSuppressionForAddr(uptr addr) { Suppression *s = nullptr; @@ -698,7 +704,7 @@ void LeakReport::PrintReportForLeak(uptr index) { Printf("%s leak of %zu byte(s) in %zu object(s) allocated from:\n", leaks_[index].is_directly_leaked ? "Direct" : "Indirect", leaks_[index].total_size, leaks_[index].hit_count); - Printf("%s", d.End()); + Printf("%s", d.Default()); PrintStackTraceById(leaks_[index].stack_trace_id); @@ -756,6 +762,7 @@ uptr LeakReport::UnsuppressedLeakCount() { namespace __lsan { void InitCommonLsan() { } void DoLeakCheck() { } +void DoRecoverableLeakCheckVoid() { } void DisableInThisThread() { } void EnableInThisThread() { } } @@ -853,6 +860,11 @@ int __lsan_do_recoverable_leak_check() { } #if !SANITIZER_SUPPORTS_WEAK_HOOKS +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char * __lsan_default_options() { + return ""; +} + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int __lsan_is_turned_off() { return 0; diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.h b/contrib/compiler-rt/lib/lsan/lsan_common.h index 31bf3eb1df42..f3863309d893 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.h +++ b/contrib/compiler-rt/lib/lsan/lsan_common.h @@ -27,9 +27,9 @@ // because of "small" (4 bytes) pointer size that leads to high false negative // ratio on large leaks. But we still want to have it for some 32 bit arches // (e.g. x86), see https://github.com/google/sanitizers/issues/403. -// To enable LeakSanitizer on new architecture, one need to implement -// internal_clone function as well as (probably) adjust TLS machinery for -// new architecture inside sanitizer library. +// To enable LeakSanitizer on a new architecture, one needs to implement the +// internal_clone function as well as (probably) adjust the TLS machinery for +// the new architecture inside the sanitizer library. #if (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC) && \ (SANITIZER_WORDSIZE == 64) && \ (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \ @@ -143,8 +143,10 @@ enum IgnoreObjectResult { }; // Functions called from the parent tool. +const char *MaybeCallLsanDefaultOptions(); void InitCommonLsan(); void DoLeakCheck(); +void DoRecoverableLeakCheckVoid(); void DisableCounterUnderflow(); bool DisabledInThisThread(); @@ -249,6 +251,9 @@ class LsanMetadata { } // namespace __lsan extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char *__lsan_default_options(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int __lsan_is_turned_off(); diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc b/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc index 5042c7b3ada5..cdd7f032a192 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc @@ -20,6 +20,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_getauxval.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -30,8 +31,12 @@ static const char kLinkerName[] = "ld"; static char linker_placeholder[sizeof(LoadedModule)] ALIGNED(64); static LoadedModule *linker = nullptr; -static bool IsLinker(const char* full_name) { - return LibraryNameIs(full_name, kLinkerName); +static bool IsLinker(const LoadedModule& module) { +#if SANITIZER_USE_GETAUXVAL + return module.base_address() == getauxval(AT_BASE); +#else + return LibraryNameIs(module.full_name(), kLinkerName); +#endif // SANITIZER_USE_GETAUXVAL } __attribute__((tls_model("initial-exec"))) @@ -49,22 +54,25 @@ void InitializePlatformSpecificModules() { ListOfModules modules; modules.init(); for (LoadedModule &module : modules) { - if (!IsLinker(module.full_name())) continue; + if (!IsLinker(module)) + continue; if (linker == nullptr) { linker = reinterpret_cast(linker_placeholder); *linker = module; module = LoadedModule(); } else { VReport(1, "LeakSanitizer: Multiple modules match \"%s\". " - "TLS will not be handled correctly.\n", kLinkerName); + "TLS and other allocations originating from linker might be " + "falsely reported as leaks.\n", kLinkerName); linker->clear(); linker = nullptr; return; } } if (linker == nullptr) { - VReport(1, "LeakSanitizer: Dynamic linker not found. " - "TLS will not be handled correctly.\n"); + VReport(1, "LeakSanitizer: Dynamic linker not found. TLS and other " + "allocations originating from linker might be falsely reported " + "as leaks.\n"); } } diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc index ade94340ae81..ac27c7af6e35 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc @@ -92,8 +92,25 @@ LoadedModule *GetLinker() { return nullptr; } // required on Darwin. void InitializePlatformSpecificModules() {} +// Sections which can't contain contain global pointers. This list errs on the +// side of caution to avoid false positives, at the expense of performance. +// +// Other potentially safe sections include: +// __all_image_info, __crash_info, __const, __got, __interpose, __objc_msg_break +// +// Sections which definitely cannot be included here are: +// __objc_data, __objc_const, __data, __bss, __common, __thread_data, +// __thread_bss, __thread_vars, __objc_opt_rw, __objc_opt_ptrs +static const char *kSkippedSecNames[] = { + "__cfstring", "__la_symbol_ptr", "__mod_init_func", + "__mod_term_func", "__nl_symbol_ptr", "__objc_classlist", + "__objc_classrefs", "__objc_imageinfo", "__objc_nlclslist", + "__objc_protolist", "__objc_selrefs", "__objc_superrefs"}; + // Scans global variables for heap pointers. void ProcessGlobalRegions(Frontier *frontier) { + for (auto name : kSkippedSecNames) CHECK(ARRAY_SIZE(name) < kMaxSegName); + MemoryMappingLayout memory_mapping(false); InternalMmapVector modules(/*initial_capacity*/ 128); memory_mapping.DumpListOfModules(&modules); @@ -107,6 +124,10 @@ void ProcessGlobalRegions(Frontier *frontier) { // Sections storing global variables are writable and non-executable if (range.executable || !range.writable) continue; + for (auto name : kSkippedSecNames) { + if (!internal_strcmp(range.name, name)) continue; + } + ScanGlobalRange(range.beg, range.end, frontier); } } diff --git a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc index 168868b012bc..b3e73e3896d5 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc @@ -20,6 +20,7 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_interceptors.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" @@ -44,6 +45,7 @@ int pthread_setspecific(unsigned key, const void *v); namespace std { struct nothrow_t; + enum class align_val_t: size_t; } #if !SANITIZER_MAC @@ -203,13 +205,19 @@ INTERCEPTOR(int, mprobe, void *ptr) { #define OPERATOR_NEW_BODY(nothrow) \ ENSURE_LSAN_INITED; \ GET_STACK_TRACE_MALLOC; \ - void *res = Allocate(stack, size, 1, kAlwaysClearMemory);\ - if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ + void *res = lsan_malloc(size, stack); \ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM(); \ + return res; +#define OPERATOR_NEW_BODY_ALIGN(nothrow) \ + ENSURE_LSAN_INITED; \ + GET_STACK_TRACE_MALLOC; \ + void *res = lsan_memalign((uptr)align, size, stack); \ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM(); \ return res; #define OPERATOR_DELETE_BODY \ ENSURE_LSAN_INITED; \ - Deallocate(ptr); + lsan_free(ptr); // On OS X it's not enough to just provide our own 'operator new' and // 'operator delete' implementations, because they're going to be in the runtime @@ -229,6 +237,18 @@ void *operator new(size_t size, std::nothrow_t const&) INTERCEPTOR_ATTRIBUTE void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } @@ -239,6 +259,30 @@ void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE void operator delete[](void *ptr, std::nothrow_t const &) { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT +{ OPERATOR_DELETE_BODY; } #else // SANITIZER_MAC @@ -265,6 +309,7 @@ INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) ///// Thread initialization and finalization. ///// +#if !SANITIZER_NETBSD static unsigned g_thread_finalize_key; static void thread_finalize(void *v) { @@ -278,6 +323,18 @@ static void thread_finalize(void *v) { } ThreadFinish(); } +#endif + +#if SANITIZER_NETBSD +INTERCEPTOR(void, _lwp_exit) { + ENSURE_LSAN_INITED; + ThreadFinish(); + REAL(_lwp_exit)(); +} +#define LSAN_MAYBE_INTERCEPT__LWP_EXIT INTERCEPT_FUNCTION(_lwp_exit) +#else +#define LSAN_MAYBE_INTERCEPT__LWP_EXIT +#endif struct ThreadParam { void *(*callback)(void *arg); @@ -291,11 +348,13 @@ extern "C" void *__lsan_thread_start_func(void *arg) { void *param = p->param; // Wait until the last iteration to maximize the chance that we are the last // destructor to run. +#if !SANITIZER_NETBSD if (pthread_setspecific(g_thread_finalize_key, (void*)GetPthreadDestructorIterations())) { Report("LeakSanitizer: failed to set thread key.\n"); Die(); } +#endif int tid = 0; while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) internal_sched_yield(); @@ -357,9 +416,14 @@ INTERCEPTOR(void, _exit, int status) { REAL(_exit)(status); } +#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + namespace __lsan { void InitializeInterceptors() { + InitializeSignalInterceptors(); + INTERCEPT_FUNCTION(malloc); INTERCEPT_FUNCTION(free); LSAN_MAYBE_INTERCEPT_CFREE; @@ -378,10 +442,14 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(_exit); + LSAN_MAYBE_INTERCEPT__LWP_EXIT; + +#if !SANITIZER_NETBSD if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { Report("LeakSanitizer: failed to create thread key.\n"); Die(); } +#endif } } // namespace __lsan diff --git a/contrib/compiler-rt/lib/lsan/weak_symbols.txt b/contrib/compiler-rt/lib/lsan/weak_symbols.txt index da4f994da865..692255679bee 100644 --- a/contrib/compiler-rt/lib/lsan/weak_symbols.txt +++ b/contrib/compiler-rt/lib/lsan/weak_symbols.txt @@ -1,2 +1,3 @@ +___lsan_default_options ___lsan_default_suppressions ___lsan_is_turned_off diff --git a/contrib/compiler-rt/lib/msan/msan.cc b/contrib/compiler-rt/lib/msan/msan.cc index d2981f0b0edb..e6226ba7670f 100644 --- a/contrib/compiler-rt/lib/msan/msan.cc +++ b/contrib/compiler-rt/lib/msan/msan.cc @@ -218,14 +218,14 @@ static void InitializeFlags() { } void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - bool request_fast_unwind) { + void *context, bool request_fast_unwind) { MsanThread *t = GetCurrentThread(); if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { // Block reports from our interceptors during _Unwind_Backtrace. SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, nullptr, 0, 0, request_fast_unwind); + return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); } - stack->Unwind(max_s, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), + stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), request_fast_unwind); } @@ -369,6 +369,16 @@ void __msan_warning_noreturn() { Die(); } +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void MsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} + void __msan_init() { CHECK(!msan_init_is_running); if (msan_inited) return; @@ -384,6 +394,7 @@ void __msan_init() { __sanitizer_set_report_path(common_flags()->log_path); InitializeInterceptors(); + InstallDeadlySignalHandlers(MsanOnDeadlySignal); InstallAtExitHandler(); // Needs __cxa_atexit interceptor. DisableCoreDumperIfNecessary(); diff --git a/contrib/compiler-rt/lib/msan/msan.h b/contrib/compiler-rt/lib/msan/msan.h index fa9c15b88bef..cbae444127ee 100644 --- a/contrib/compiler-rt/lib/msan/msan.h +++ b/contrib/compiler-rt/lib/msan/msan.h @@ -160,21 +160,25 @@ const MappingDesc kMemoryLayout[] = { # define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x1000000000ULL) #elif SANITIZER_LINUX && SANITIZER_PPC64 - const MappingDesc kMemoryLayout[] = { - {0x000000000000ULL, 0x000100000000ULL, MappingDesc::APP, "low memory"}, - {0x000100000000ULL, 0x080000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x080000000000ULL, 0x180100000000ULL, MappingDesc::SHADOW, "shadow"}, - {0x180100000000ULL, 0x1C0000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x1C0000000000ULL, 0x2C0100000000ULL, MappingDesc::ORIGIN, "origin"}, - {0x2C0100000000ULL, 0x300000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x300000000000ULL, 0x400000000000ULL, MappingDesc::APP, "high memory"}}; + {0x000000000000ULL, 0x000200000000ULL, MappingDesc::APP, "low memory"}, + {0x000200000000ULL, 0x080000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x080000000000ULL, 0x180200000000ULL, MappingDesc::SHADOW, "shadow"}, + {0x180200000000ULL, 0x1C0000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x1C0000000000ULL, 0x2C0200000000ULL, MappingDesc::ORIGIN, "origin"}, + {0x2C0200000000ULL, 0x300000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x300000000000ULL, 0x800000000000ULL, MappingDesc::APP, "high memory"}}; +// Various kernels use different low end ranges but we can combine them into one +// big range. They also use different high end ranges but we can map them all to +// one range. // Maps low and high app ranges to contiguous space with zero base: -// Low: 0000 0000 0000 - 0000 ffff ffff -> 1000 0000 0000 - 1000 ffff ffff +// Low: 0000 0000 0000 - 0001 ffff ffff -> 1000 0000 0000 - 1001 ffff ffff // High: 3000 0000 0000 - 3fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff +// High: 4000 0000 0000 - 4fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff +// High: 7000 0000 0000 - 7fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff #define LINEARIZE_MEM(mem) \ - (((uptr)(mem) & ~0x200000000000ULL) ^ 0x100000000000ULL) + (((uptr)(mem) & ~0xE00000000000ULL) ^ 0x100000000000ULL) #define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x080000000000ULL) #define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x140000000000ULL) @@ -199,7 +203,7 @@ const MappingDesc kMemoryLayout[] = { #define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x100000000000ULL) #define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x280000000000) -#elif SANITIZER_LINUX && SANITIZER_WORDSIZE == 64 +#elif SANITIZER_NETBSD || (SANITIZER_LINUX && SANITIZER_WORDSIZE == 64) #ifdef MSAN_LINUX_X86_64_OLD_MAPPING // Requries PIE binary and ASLR enabled. @@ -310,7 +314,7 @@ void PrintWarning(uptr pc, uptr bp); void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin); void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - bool request_fast_unwind); + void *context, bool request_fast_unwind); void ReportUMR(StackTrace *stack, u32 origin); void ReportExpectedUMRNotFound(StackTrace *stack); @@ -330,32 +334,32 @@ u32 ChainOrigin(u32 id, StackTrace *stack); const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; -#define GET_MALLOC_STACK_TRACE \ - BufferedStackTrace stack; \ - if (__msan_get_track_origins() && msan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ +#define GET_MALLOC_STACK_TRACE \ + BufferedStackTrace stack; \ + if (__msan_get_track_origins() && msan_inited) \ + GetStackTrace(&stack, common_flags()->malloc_context_size, \ + StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ common_flags()->fast_unwind_on_malloc) // For platforms which support slow unwinder only, we restrict the store context // size to 1, basically only storing the current pc. We do this because the slow // unwinder which is based on libunwind is not async signal safe and causes // random freezes in forking applications as well as in signal handlers. -#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ - BufferedStackTrace stack; \ - if (__msan_get_track_origins() > 1 && msan_inited) { \ - if (!SANITIZER_CAN_FAST_UNWIND) \ - GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \ - false); \ - else \ - GetStackTrace(&stack, flags()->store_context_size, pc, bp, \ - common_flags()->fast_unwind_on_malloc); \ +#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (__msan_get_track_origins() > 1 && msan_inited) { \ + if (!SANITIZER_CAN_FAST_UNWIND) \ + GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \ + nullptr, false); \ + else \ + GetStackTrace(&stack, flags()->store_context_size, pc, bp, nullptr, \ + common_flags()->fast_unwind_on_malloc); \ } -#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ - BufferedStackTrace stack; \ - if (msan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, \ +#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ + BufferedStackTrace stack; \ + if (msan_inited) \ + GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ common_flags()->fast_unwind_on_fatal) #define GET_STORE_STACK_TRACE \ diff --git a/contrib/compiler-rt/lib/msan/msan_allocator.cc b/contrib/compiler-rt/lib/msan/msan_allocator.cc index 1034dbdf9b55..0f9942324931 100644 --- a/contrib/compiler-rt/lib/msan/msan_allocator.cc +++ b/contrib/compiler-rt/lib/msan/msan_allocator.cc @@ -62,7 +62,8 @@ struct MsanMapUnmapCallback { }; typedef SizeClassAllocator32 PrimaryAllocator; #elif defined(__x86_64__) -#if SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING) +#if SANITIZER_NETBSD || \ + (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING)) static const uptr kAllocatorSpace = 0x700000000000ULL; #else static const uptr kAllocatorSpace = 0x600000000000ULL; @@ -255,8 +256,12 @@ void *msan_valloc(uptr size, StackTrace *stack) { void *msan_pvalloc(uptr size, StackTrace *stack) { uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + return Allocator::FailureHandler::OnBadRequest(); + } // pvalloc(0) should allocate one page. - size = size == 0 ? PageSize : RoundUpTo(size, PageSize); + size = size ? RoundUpTo(size, PageSize) : PageSize; return SetErrnoOnNull(MsanAllocate(stack, size, PageSize, false)); } diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc index b5d22baca08d..a7fe09b25ffb 100644 --- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc +++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc @@ -22,6 +22,7 @@ #include "msan_thread.h" #include "msan_poisoning.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_allocator.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -33,6 +34,11 @@ #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" +#if SANITIZER_NETBSD +#define gettimeofday __gettimeofday50 +#define getrusage __getrusage50 +#endif + #include // ACHTUNG! No other system header includes in this file. // Ideally, we should get rid of stdarg.h as well. @@ -86,22 +92,21 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { } while (0) // Check that [x, x+n) range is unpoisoned. -#define CHECK_UNPOISONED_0(x, n) \ - do { \ - sptr offset = __msan_test_shadow(x, n); \ - if (__msan::IsInSymbolizer()) \ - break; \ - if (offset >= 0 && __msan::flags()->report_umrs) { \ - GET_CALLER_PC_BP_SP; \ - (void) sp; \ - ReportUMRInsideAddressRange(__func__, x, n, offset); \ - __msan::PrintWarningWithOrigin( \ - pc, bp, __msan_get_origin((const char *)x + offset)); \ - if (__msan::flags()->halt_on_error) { \ - Printf("Exiting\n"); \ - Die(); \ - } \ - } \ +#define CHECK_UNPOISONED_0(x, n) \ + do { \ + sptr __offset = __msan_test_shadow(x, n); \ + if (__msan::IsInSymbolizer()) break; \ + if (__offset >= 0 && __msan::flags()->report_umrs) { \ + GET_CALLER_PC_BP_SP; \ + (void)sp; \ + ReportUMRInsideAddressRange(__func__, x, n, __offset); \ + __msan::PrintWarningWithOrigin( \ + pc, bp, __msan_get_origin((const char *)x + __offset)); \ + if (__msan::flags()->halt_on_error) { \ + Printf("Exiting\n"); \ + Die(); \ + } \ + } \ } while (0) // Check that [x, x+n) range is unpoisoned unless we are in a nested @@ -109,7 +114,7 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { #define CHECK_UNPOISONED(x, n) \ do { \ if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \ - } while (0); + } while (0) #define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \ CHECK_UNPOISONED((x), \ @@ -118,7 +123,7 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { #define CHECK_UNPOISONED_STRING(x, n) \ CHECK_UNPOISONED_STRING_OF_LEN((x), internal_strlen(x), (n)) -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb, void *file) { ENSURE_MSAN_INITED(); @@ -134,16 +139,21 @@ INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb, INTERCEPTOR(SSIZE_T, readlink, const char *path, char *buf, SIZE_T bufsiz) { ENSURE_MSAN_INITED(); - CHECK_UNPOISONED_STRING(path, 0) + CHECK_UNPOISONED_STRING(path, 0); SSIZE_T res = REAL(readlink)(path, buf, bufsiz); if (res > 0) __msan_unpoison(buf, res); return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(void *, mempcpy, void *dest, const void *src, SIZE_T n) { return (char *)__msan_memcpy(dest, src, n) + n; } +#define MSAN_MAYBE_INTERCEPT_MEMPCPY INTERCEPT_FUNCTION(mempcpy) +#else +#define MSAN_MAYBE_INTERCEPT_MEMPCPY +#endif INTERCEPTOR(void *, memccpy, void *dest, const void *src, int c, SIZE_T n) { ENSURE_MSAN_INITED(); @@ -168,7 +178,7 @@ INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) { GET_MALLOC_STACK_TRACE; return msan_memalign(alignment, size, &stack); @@ -183,6 +193,7 @@ INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) { return msan_aligned_alloc(alignment, size, &stack); } +#if !SANITIZER_NETBSD INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { GET_MALLOC_STACK_TRACE; void *ptr = msan_memalign(alignment, size, &stack); @@ -190,13 +201,17 @@ INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { DTLS_on_libc_memalign(ptr, size); return ptr; } +#define MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN INTERCEPT_FUNCTION(__libc_memalign) +#else +#define MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN +#endif INTERCEPTOR(void *, valloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; return msan_valloc(size, &stack); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void *, pvalloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; return msan_pvalloc(size, &stack); @@ -212,7 +227,7 @@ INTERCEPTOR(void, free, void *ptr) { MsanDeallocate(&stack, ptr); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void, cfree, void *ptr) { GET_MALLOC_STACK_TRACE; if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; @@ -223,11 +238,17 @@ INTERCEPTOR(void, cfree, void *ptr) { #define MSAN_MAYBE_INTERCEPT_CFREE #endif +#if !SANITIZER_NETBSD INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { return __sanitizer_get_allocated_size(ptr); } +#define MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE \ + INTERCEPT_FUNCTION(malloc_usable_size) +#else +#define MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE +#endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD // This function actually returns a struct by value, but we can't unpoison a // temporary! The following is equivalent on all supported platforms but // aarch64 (which uses a different register for sret value). We have a test @@ -246,7 +267,7 @@ INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { #define MSAN_MAYBE_INTERCEPT_MALLINFO #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, mallopt, int cmd, int value) { return -1; } @@ -255,7 +276,7 @@ INTERCEPTOR(int, mallopt, int cmd, int value) { #define MSAN_MAYBE_INTERCEPT_MALLOPT #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void, malloc_stats, void) { // FIXME: implement, but don't call REAL(malloc_stats)! } @@ -286,6 +307,7 @@ INTERCEPTOR(char *, strncpy, char *dest, const char *src, SIZE_T n) { // NOLINT return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; @@ -295,6 +317,10 @@ INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT CopyShadowAndOrigin(dest, src, n + 1, &stack); return res; } +#define MSAN_MAYBE_INTERCEPT_STPCPY INTERCEPT_FUNCTION(stpcpy) +#else +#define MSAN_MAYBE_INTERCEPT_STPCPY +#endif INTERCEPTOR(char *, strdup, char *src) { ENSURE_MSAN_INITED(); @@ -308,7 +334,7 @@ INTERCEPTOR(char *, strdup, char *src) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(char *, __strdup, char *src) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; @@ -323,6 +349,7 @@ INTERCEPTOR(char *, __strdup, char *src) { #define MSAN_MAYBE_INTERCEPT___STRDUP #endif +#if !SANITIZER_NETBSD INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) { ENSURE_MSAN_INITED(); char *res = REAL(gcvt)(number, ndigit, buf); @@ -330,6 +357,10 @@ INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) { __msan_unpoison(buf, n + 1); return res; } +#define MSAN_MAYBE_INTERCEPT_GCVT INTERCEPT_FUNCTION(gcvt) +#else +#define MSAN_MAYBE_INTERCEPT_GCVT +#endif INTERCEPTOR(char *, strcat, char *dest, const char *src) { // NOLINT ENSURE_MSAN_INITED(); @@ -386,6 +417,16 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT INTERCEPTOR_STRTO_BODY(ret_type, func, nptr, endptr, base, loc); \ } +#if SANITIZER_NETBSD +#define INTERCEPTORS_STRTO(ret_type, func, char_type) \ + INTERCEPTOR_STRTO(ret_type, func, char_type) \ + INTERCEPTOR_STRTO_LOC(ret_type, func##_l, char_type) + +#define INTERCEPTORS_STRTO_BASE(ret_type, func, char_type) \ + INTERCEPTOR_STRTO_BASE(ret_type, func, char_type) \ + INTERCEPTOR_STRTO_BASE_LOC(ret_type, func##_l, char_type) + +#else #define INTERCEPTORS_STRTO(ret_type, func, char_type) \ INTERCEPTOR_STRTO(ret_type, func, char_type) \ INTERCEPTOR_STRTO_LOC(ret_type, func##_l, char_type) \ @@ -397,6 +438,7 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT INTERCEPTOR_STRTO_BASE_LOC(ret_type, func##_l, char_type) \ INTERCEPTOR_STRTO_BASE_LOC(ret_type, __##func##_l, char_type) \ INTERCEPTOR_STRTO_BASE_LOC(ret_type, __##func##_internal, char_type) +#endif INTERCEPTORS_STRTO(double, strtod, char) // NOLINT INTERCEPTORS_STRTO(float, strtof, char) // NOLINT @@ -405,6 +447,7 @@ INTERCEPTORS_STRTO_BASE(long, strtol, char) // NOLINT INTERCEPTORS_STRTO_BASE(long long, strtoll, char) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long, strtoul, char) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long long, strtoull, char) // NOLINT +INTERCEPTORS_STRTO_BASE(u64, strtouq, char) // NOLINT INTERCEPTORS_STRTO(double, wcstod, wchar_t) // NOLINT INTERCEPTORS_STRTO(float, wcstof, wchar_t) // NOLINT @@ -414,11 +457,17 @@ INTERCEPTORS_STRTO_BASE(long long, wcstoll, wchar_t) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long, wcstoul, wchar_t) // NOLINT INTERCEPTORS_STRTO_BASE(unsigned long long, wcstoull, wchar_t) // NOLINT +#if SANITIZER_NETBSD +#define INTERCEPT_STRTO(func) \ + INTERCEPT_FUNCTION(func); \ + INTERCEPT_FUNCTION(func##_l); +#else #define INTERCEPT_STRTO(func) \ INTERCEPT_FUNCTION(func); \ INTERCEPT_FUNCTION(func##_l); \ INTERCEPT_FUNCTION(__##func##_l); \ INTERCEPT_FUNCTION(__##func##_internal); +#endif // FIXME: support *wprintf in common format interceptors. @@ -457,6 +506,20 @@ INTERCEPTOR(SIZE_T, strxfrm_l, char *dest, const char *src, SIZE_T n, return res; } +#if SANITIZER_LINUX +INTERCEPTOR(SIZE_T, __strxfrm_l, char *dest, const char *src, SIZE_T n, + void *loc) { + ENSURE_MSAN_INITED(); + CHECK_UNPOISONED(src, REAL(strlen)(src) + 1); + SIZE_T res = REAL(__strxfrm_l)(dest, src, n, loc); + if (res < n) __msan_unpoison(dest, res + 1); + return res; +} +#define MSAN_MAYBE_INTERCEPT___STRXFRM_L INTERCEPT_FUNCTION(__strxfrm_l) +#else +#define MSAN_MAYBE_INTERCEPT___STRXFRM_L +#endif + #define INTERCEPTOR_STRFTIME_BODY(char_type, ret_type, func, s, ...) \ ENSURE_MSAN_INITED(); \ ret_type res = REAL(func)(s, __VA_ARGS__); \ @@ -473,7 +536,7 @@ INTERCEPTOR(SIZE_T, strftime_l, char *s, SIZE_T max, const char *format, INTERCEPTOR_STRFTIME_BODY(char, SIZE_T, strftime_l, s, max, format, tm, loc); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(SIZE_T, __strftime_l, char *s, SIZE_T max, const char *format, __sanitizer_tm *tm, void *loc) { INTERCEPTOR_STRFTIME_BODY(char, SIZE_T, __strftime_l, s, max, format, tm, @@ -495,7 +558,7 @@ INTERCEPTOR(SIZE_T, wcsftime_l, wchar_t *s, SIZE_T max, const wchar_t *format, loc); } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(SIZE_T, __wcsftime_l, wchar_t *s, SIZE_T max, const wchar_t *format, __sanitizer_tm *tm, void *loc) { INTERCEPTOR_STRFTIME_BODY(wchar_t, SIZE_T, __wcsftime_l, s, max, format, tm, @@ -513,7 +576,8 @@ INTERCEPTOR(int, mbtowc, wchar_t *dest, const char *src, SIZE_T n) { return res; } -INTERCEPTOR(int, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, void *ps) { +INTERCEPTOR(SIZE_T, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, + void *ps) { ENSURE_MSAN_INITED(); SIZE_T res = REAL(mbrtowc)(dest, src, n, ps); if (res != (SIZE_T)-1 && dest) __msan_unpoison(dest, sizeof(wchar_t)); @@ -529,6 +593,7 @@ INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; @@ -536,6 +601,10 @@ INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { CopyShadowAndOrigin(dest, src, n * sizeof(wchar_t), &stack); return res; } +#define MSAN_MAYBE_INTERCEPT_WMEMPCPY INTERCEPT_FUNCTION(wmempcpy) +#else +#define MSAN_MAYBE_INTERCEPT_WMEMPCPY +#endif INTERCEPTOR(wchar_t *, wmemset, wchar_t *s, wchar_t c, SIZE_T n) { CHECK(MEM_IS_APP(s)); @@ -569,6 +638,7 @@ INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { return res; } +#if !SANITIZER_NETBSD INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) { ENSURE_MSAN_INITED(); char *res = REAL(fcvt)(x, a, b, c); @@ -577,6 +647,10 @@ INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) { if (res) __msan_unpoison(res, REAL(strlen)(res) + 1); return res; } +#define MSAN_MAYBE_INTERCEPT_FCVT INTERCEPT_FUNCTION(fcvt) +#else +#define MSAN_MAYBE_INTERCEPT_FCVT +#endif INTERCEPTOR(char *, getenv, char *name) { if (msan_init_is_running) @@ -601,7 +675,7 @@ static void UnpoisonEnviron() { INTERCEPTOR(int, setenv, const char *name, const char *value, int overwrite) { ENSURE_MSAN_INITED(); - CHECK_UNPOISONED_STRING(name, 0) + CHECK_UNPOISONED_STRING(name, 0); int res = REAL(setenv)(name, value, overwrite); if (!res) UnpoisonEnviron(); return res; @@ -614,7 +688,7 @@ INTERCEPTOR(int, putenv, char *string) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) { ENSURE_MSAN_INITED(); int res = REAL(__fxstat)(magic, fd, buf); @@ -627,7 +701,7 @@ INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) { #define MSAN_MAYBE_INTERCEPT___FXSTAT #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) { ENSURE_MSAN_INITED(); int res = REAL(__fxstat64)(magic, fd, buf); @@ -640,7 +714,7 @@ INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) { #define MSAN_MAYBE_INTERCEPT___FXSTAT64 #endif -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD INTERCEPTOR(int, fstatat, int fd, char *pathname, void *buf, int flags) { ENSURE_MSAN_INITED(); int res = REAL(fstatat)(fd, pathname, buf, flags); @@ -659,7 +733,7 @@ INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf, # define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(__fxstatat) #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, __fxstatat64, int magic, int fd, char *pathname, void *buf, int flags) { ENSURE_MSAN_INITED(); @@ -706,7 +780,7 @@ INTERCEPTOR(char *, fgets, char *s, int size, void *stream) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(char *, fgets_unlocked, char *s, int size, void *stream) { ENSURE_MSAN_INITED(); char *res = REAL(fgets_unlocked)(s, size, stream); @@ -729,7 +803,7 @@ INTERCEPTOR(int, getrlimit, int resource, void *rlim) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, getrlimit64, int resource, void *rlim) { if (msan_init_is_running) return REAL(getrlimit64)(resource, rlim); ENSURE_MSAN_INITED(); @@ -805,7 +879,7 @@ INTERCEPTOR(int, gethostname, char *name, SIZE_T len) { return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents, int timeout) { ENSURE_MSAN_INITED(); @@ -820,7 +894,7 @@ INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents, #define MSAN_MAYBE_INTERCEPT_EPOLL_WAIT #endif -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, epoll_pwait, int epfd, void *events, int maxevents, int timeout, void *sigmask) { ENSURE_MSAN_INITED(); @@ -909,7 +983,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, return res; } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags, int fd, OFF64_T offset) { ENSURE_MSAN_INITED(); @@ -983,59 +1057,13 @@ static void SignalAction(int signo, void *si, void *uc) { cb(signo, si, uc); } -INTERCEPTOR(int, sigaction, int signo, const __sanitizer_sigaction *act, - __sanitizer_sigaction *oldact) { - ENSURE_MSAN_INITED(); - // FIXME: check that *act is unpoisoned. - // That requires intercepting all of sigemptyset, sigfillset, etc. - int res; - if (flags()->wrap_signals) { - SpinMutexLock lock(&sigactions_mu); - CHECK_LT(signo, kMaxSignals); - uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed); - __sanitizer_sigaction new_act; - __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr; - if (act) { - REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction)); - uptr cb = (uptr)pnew_act->sigaction; - uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo) - ? (uptr)SignalAction - : (uptr)SignalHandler; - if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { - atomic_store(&sigactions[signo], cb, memory_order_relaxed); - pnew_act->sigaction = (void (*)(int, void *, void *))new_cb; - } - } - res = REAL(sigaction)(signo, pnew_act, oldact); - if (res == 0 && oldact) { - uptr cb = (uptr)oldact->sigaction; - if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { - oldact->sigaction = (void (*)(int, void *, void *))old_cb; - } - } - } else { - res = REAL(sigaction)(signo, act, oldact); - } - - if (res == 0 && oldact) { - __msan_unpoison(oldact, sizeof(__sanitizer_sigaction)); - } - return res; -} - -INTERCEPTOR(int, signal, int signo, uptr cb) { - ENSURE_MSAN_INITED(); - if (flags()->wrap_signals) { - CHECK_LT(signo, kMaxSignals); - SpinMutexLock lock(&sigactions_mu); - if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { - atomic_store(&sigactions[signo], cb, memory_order_relaxed); - cb = (uptr) SignalHandler; - } - return REAL(signal)(signo, cb); - } else { - return REAL(signal)(signo, cb); - } +static void read_sigaction(const __sanitizer_sigaction *act) { + CHECK_UNPOISONED(&act->sa_flags, sizeof(act->sa_flags)); + if (act->sa_flags & __sanitizer::sa_siginfo) + CHECK_UNPOISONED(&act->sigaction, sizeof(act->sigaction)); + else + CHECK_UNPOISONED(&act->handler, sizeof(act->handler)); + CHECK_UNPOISONED(&act->sa_mask, sizeof(act->sa_mask)); } extern "C" int pthread_attr_init(void *attr); @@ -1080,6 +1108,11 @@ INTERCEPTOR(int, pthread_key_create, __sanitizer_pthread_key_t *key, return res; } +#if SANITIZER_NETBSD +INTERCEPTOR(void, __libc_thr_keycreate, void *m, void (*dtor)(void *value)) \ + ALIAS(WRAPPER_NAME(pthread_key_create)); +#endif + INTERCEPTOR(int, pthread_join, void *th, void **retval) { ENSURE_MSAN_INITED(); int res = REAL(pthread_join)(th, retval); @@ -1124,21 +1157,6 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, return REAL(__cxa_atexit)(MSanAtExitWrapper, r, dso_handle); } -DECLARE_REAL(int, shmctl, int shmid, int cmd, void *buf) - -INTERCEPTOR(void *, shmat, int shmid, const void *shmaddr, int shmflg) { - ENSURE_MSAN_INITED(); - void *p = REAL(shmat)(shmid, shmaddr, shmflg); - if (p != (void *)-1) { - __sanitizer_shmid_ds ds; - int res = REAL(shmctl)(shmid, shmctl_ipc_stat, &ds); - if (!res) { - __msan_unpoison(p, ds.shm_segsz); - } - } - return p; -} - static void BeforeFork() { StackDepotLockAll(); ChainedOriginDepotLockAll(); @@ -1293,6 +1311,73 @@ int OnExit() { #include "sanitizer_common/sanitizer_platform_interceptors.h" #include "sanitizer_common/sanitizer_common_interceptors.inc" +static uptr signal_impl(int signo, uptr cb); +static int sigaction_impl(int signo, const __sanitizer_sigaction *act, + __sanitizer_sigaction *oldact); + +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ + { return sigaction_impl(signo, act, oldact); } + +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ + { \ + handler = signal_impl(signo, handler); \ + return REAL(func)(signo, handler); \ + } + +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +static int sigaction_impl(int signo, const __sanitizer_sigaction *act, + __sanitizer_sigaction *oldact) { + ENSURE_MSAN_INITED(); + if (act) read_sigaction(act); + int res; + if (flags()->wrap_signals) { + SpinMutexLock lock(&sigactions_mu); + CHECK_LT(signo, kMaxSignals); + uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed); + __sanitizer_sigaction new_act; + __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr; + if (act) { + REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction)); + uptr cb = (uptr)pnew_act->sigaction; + uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo) + ? (uptr)SignalAction + : (uptr)SignalHandler; + if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { + atomic_store(&sigactions[signo], cb, memory_order_relaxed); + pnew_act->sigaction = (decltype(pnew_act->sigaction))new_cb; + } + } + res = REAL(SIGACTION_SYMNAME)(signo, pnew_act, oldact); + if (res == 0 && oldact) { + uptr cb = (uptr)oldact->sigaction; + if (cb == (uptr)SignalAction || cb == (uptr)SignalHandler) { + oldact->sigaction = (decltype(oldact->sigaction))old_cb; + } + } + } else { + res = REAL(SIGACTION_SYMNAME)(signo, act, oldact); + } + + if (res == 0 && oldact) { + __msan_unpoison(oldact, sizeof(__sanitizer_sigaction)); + } + return res; +} + +static uptr signal_impl(int signo, uptr cb) { + ENSURE_MSAN_INITED(); + if (flags()->wrap_signals) { + CHECK_LT(signo, kMaxSignals); + SpinMutexLock lock(&sigactions_mu); + if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) { + atomic_store(&sigactions[signo], cb, memory_order_relaxed); + cb = (uptr)&SignalHandler; + } + } + return cb; +} + #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ do { \ @@ -1353,6 +1438,19 @@ static int msan_dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, return cbdata->callback(info, size, cbdata->data); } +INTERCEPTOR(void *, shmat, int shmid, const void *shmaddr, int shmflg) { + ENSURE_MSAN_INITED(); + void *p = REAL(shmat)(shmid, shmaddr, shmflg); + if (p != (void *)-1) { + __sanitizer_shmid_ds ds; + int res = REAL(shmctl)(shmid, shmctl_ipc_stat, &ds); + if (!res) { + __msan_unpoison(p, ds.shm_segsz); + } + } + return p; +} + INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb callback, void *data) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, dl_iterate_phdr, callback, data); @@ -1455,12 +1553,13 @@ void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); InitializeCommonInterceptors(); + InitializeSignalInterceptors(); INTERCEPT_FUNCTION(mmap); MSAN_MAYBE_INTERCEPT_MMAP64; INTERCEPT_FUNCTION(posix_memalign); MSAN_MAYBE_INTERCEPT_MEMALIGN; - INTERCEPT_FUNCTION(__libc_memalign); + MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN; INTERCEPT_FUNCTION(valloc); MSAN_MAYBE_INTERCEPT_PVALLOC; INTERCEPT_FUNCTION(malloc); @@ -1468,7 +1567,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); MSAN_MAYBE_INTERCEPT_CFREE; - INTERCEPT_FUNCTION(malloc_usable_size); + MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE; MSAN_MAYBE_INTERCEPT_MALLINFO; MSAN_MAYBE_INTERCEPT_MALLOPT; MSAN_MAYBE_INTERCEPT_MALLOC_STATS; @@ -1476,18 +1575,18 @@ void InitializeInterceptors() { MSAN_MAYBE_INTERCEPT_FREAD_UNLOCKED; INTERCEPT_FUNCTION(readlink); INTERCEPT_FUNCTION(memccpy); - INTERCEPT_FUNCTION(mempcpy); + MSAN_MAYBE_INTERCEPT_MEMPCPY; INTERCEPT_FUNCTION(bcopy); INTERCEPT_FUNCTION(wmemset); INTERCEPT_FUNCTION(wmemcpy); - INTERCEPT_FUNCTION(wmempcpy); + MSAN_MAYBE_INTERCEPT_WMEMPCPY; INTERCEPT_FUNCTION(wmemmove); INTERCEPT_FUNCTION(strcpy); // NOLINT - INTERCEPT_FUNCTION(stpcpy); // NOLINT + MSAN_MAYBE_INTERCEPT_STPCPY; // NOLINT INTERCEPT_FUNCTION(strdup); MSAN_MAYBE_INTERCEPT___STRDUP; INTERCEPT_FUNCTION(strncpy); // NOLINT - INTERCEPT_FUNCTION(gcvt); + MSAN_MAYBE_INTERCEPT_GCVT; INTERCEPT_FUNCTION(strcat); // NOLINT INTERCEPT_FUNCTION(strncat); // NOLINT INTERCEPT_STRTO(strtod); @@ -1497,6 +1596,7 @@ void InitializeInterceptors() { INTERCEPT_STRTO(strtoul); INTERCEPT_STRTO(strtoll); INTERCEPT_STRTO(strtoull); + INTERCEPT_STRTO(strtouq); INTERCEPT_STRTO(wcstod); INTERCEPT_STRTO(wcstof); INTERCEPT_STRTO(wcstold); @@ -1513,6 +1613,7 @@ void InitializeInterceptors() { #endif INTERCEPT_FUNCTION(strxfrm); INTERCEPT_FUNCTION(strxfrm_l); + MSAN_MAYBE_INTERCEPT___STRXFRM_L; INTERCEPT_FUNCTION(strftime); INTERCEPT_FUNCTION(strftime_l); MSAN_MAYBE_INTERCEPT___STRFTIME_L; @@ -1531,7 +1632,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(setenv); INTERCEPT_FUNCTION(putenv); INTERCEPT_FUNCTION(gettimeofday); - INTERCEPT_FUNCTION(fcvt); + MSAN_MAYBE_INTERCEPT_FCVT; MSAN_MAYBE_INTERCEPT___FXSTAT; MSAN_INTERCEPT_FSTATAT; MSAN_MAYBE_INTERCEPT___FXSTAT64; @@ -1553,14 +1654,17 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(dlerror); INTERCEPT_FUNCTION(dl_iterate_phdr); INTERCEPT_FUNCTION(getrusage); - INTERCEPT_FUNCTION(sigaction); - INTERCEPT_FUNCTION(signal); #if defined(__mips__) INTERCEPT_FUNCTION_VER(pthread_create, "GLIBC_2.2"); #else INTERCEPT_FUNCTION(pthread_create); #endif INTERCEPT_FUNCTION(pthread_key_create); + +#if SANITIZER_NETBSD + INTERCEPT_FUNCTION(__libc_thr_keycreate); +#endif + INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(tzset); INTERCEPT_FUNCTION(__cxa_atexit); diff --git a/contrib/compiler-rt/lib/msan/msan_linux.cc b/contrib/compiler-rt/lib/msan/msan_linux.cc index 0a687f620c94..4e6321fcb918 100644 --- a/contrib/compiler-rt/lib/msan/msan_linux.cc +++ b/contrib/compiler-rt/lib/msan/msan_linux.cc @@ -9,11 +9,11 @@ // // This file is a part of MemorySanitizer. // -// Linux- and FreeBSD-specific code. +// Linux-, NetBSD- and FreeBSD-specific code. //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD #include "msan.h" #include "msan_thread.h" @@ -120,7 +120,7 @@ bool InitShadow(bool init_origins) { return false; } - const uptr maxVirtualAddress = GetMaxVirtualAddress(); + const uptr maxVirtualAddress = GetMaxUserVirtualAddress(); for (unsigned i = 0; i < kMemoryLayoutSize; ++i) { uptr start = kMemoryLayout[i].start; @@ -213,4 +213,4 @@ void MsanTSDDtor(void *tsd) { } // namespace __msan -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/msan/msan_new_delete.cc b/contrib/compiler-rt/lib/msan/msan_new_delete.cc index 721926791029..5cc76e4bc08c 100644 --- a/contrib/compiler-rt/lib/msan/msan_new_delete.cc +++ b/contrib/compiler-rt/lib/msan/msan_new_delete.cc @@ -22,9 +22,10 @@ using namespace __msan; // NOLINT -// Fake std::nothrow_t to avoid including . +// Fake std::nothrow_t and std::align_val_t to avoid including . namespace std { struct nothrow_t {}; + enum class align_val_t: size_t {}; } // namespace std @@ -34,6 +35,11 @@ namespace std { void *res = msan_malloc(size, &stack);\ if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ return res +#define OPERATOR_NEW_BODY_ALIGN(nothrow) \ + GET_MALLOC_STACK_TRACE;\ + void *res = msan_memalign((uptr)align, size, &stack);\ + if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ + return res; INTERCEPTOR_ATTRIBUTE void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } @@ -47,6 +53,18 @@ INTERCEPTOR_ATTRIBUTE void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); } +INTERCEPTOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); } #define OPERATOR_DELETE_BODY \ GET_MALLOC_STACK_TRACE; \ @@ -62,5 +80,29 @@ INTERCEPTOR_ATTRIBUTE void operator delete[](void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, size_t size) NOEXCEPT { OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY; } + #endif // MSAN_REPLACE_OPERATORS_NEW_AND_DELETE diff --git a/contrib/compiler-rt/lib/msan/msan_report.cc b/contrib/compiler-rt/lib/msan/msan_report.cc index 9a35c9c13de5..28c9bbabb3e9 100644 --- a/contrib/compiler-rt/lib/msan/msan_report.cc +++ b/contrib/compiler-rt/lib/msan/msan_report.cc @@ -30,10 +30,8 @@ namespace __msan { class Decorator: public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() { } - const char *Warning() { return Red(); } const char *Origin() { return Magenta(); } const char *Name() { return Green(); } - const char *End() { return Default(); } }; static void DescribeStackOrigin(const char *so, uptr pc) { @@ -47,7 +45,7 @@ static void DescribeStackOrigin(const char *so, uptr pc) { " %sUninitialized value was created by an allocation of '%s%s%s'" " in the stack frame of function '%s%s%s'%s\n", d.Origin(), d.Name(), s, d.Origin(), d.Name(), sep + 1, d.Origin(), - d.End()); + d.Default()); InternalFree(s); if (pc) { @@ -66,7 +64,7 @@ static void DescribeOrigin(u32 id) { StackTrace stack; o = o.getNextChainedOrigin(&stack); Printf(" %sUninitialized value was stored to memory at%s\n", d.Origin(), - d.End()); + d.Default()); stack.Print(); } if (o.isStackOrigin()) { @@ -78,18 +76,19 @@ static void DescribeOrigin(u32 id) { switch (stack.tag) { case StackTrace::TAG_ALLOC: Printf(" %sUninitialized value was created by a heap allocation%s\n", - d.Origin(), d.End()); + d.Origin(), d.Default()); break; case StackTrace::TAG_DEALLOC: Printf(" %sUninitialized value was created by a heap deallocation%s\n", - d.Origin(), d.End()); + d.Origin(), d.Default()); break; case STACK_TRACE_TAG_POISON: Printf(" %sMemory was marked as uninitialized%s\n", d.Origin(), - d.End()); + d.Default()); break; default: - Printf(" %sUninitialized value was created%s\n", d.Origin(), d.End()); + Printf(" %sUninitialized value was created%s\n", d.Origin(), + d.Default()); break; } stack.Print(); @@ -99,12 +98,12 @@ static void DescribeOrigin(u32 id) { void ReportUMR(StackTrace *stack, u32 origin) { if (!__msan::flags()->report_umrs) return; - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; Decorator d; Printf("%s", d.Warning()); Report("WARNING: MemorySanitizer: use-of-uninitialized-value\n"); - Printf("%s", d.End()); + Printf("%s", d.Default()); stack->Print(); if (origin) { DescribeOrigin(origin); @@ -113,14 +112,14 @@ void ReportUMR(StackTrace *stack, u32 origin) { } void ReportExpectedUMRNotFound(StackTrace *stack) { - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; Printf("WARNING: Expected use of uninitialized value not found\n"); stack->Print(); } void ReportStats() { - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; if (__msan_get_track_origins() > 0) { StackDepotStats *stack_depot_stats = StackDepotGetStats(); @@ -138,13 +137,13 @@ void ReportStats() { } void ReportAtExitStatistics() { - SpinMutexLock l(&CommonSanitizerReportMutex); + ScopedErrorReportLock l; if (msan_report_count > 0) { Decorator d; Printf("%s", d.Warning()); Printf("MemorySanitizer: %d warnings reported.\n", msan_report_count); - Printf("%s", d.End()); + Printf("%s", d.Default()); } } @@ -203,7 +202,7 @@ void DescribeMemoryRange(const void *x, uptr size) { Decorator d; Printf("%s", d.Warning()); Printf("Shadow map of [%p, %p), %zu bytes:\n", start, end, end - start); - Printf("%s", d.End()); + Printf("%s", d.Default()); while (s < e) { // Line start. if (pos % 16 == 0) { @@ -265,7 +264,7 @@ void ReportUMRInsideAddressRange(const char *what, const void *start, uptr size, Printf("%s", d.Warning()); Printf("%sUninitialized bytes in %s%s%s at offset %zu inside [%p, %zu)%s\n", d.Warning(), d.Name(), what, d.Warning(), offset, start, size, - d.End()); + d.Default()); if (__sanitizer::Verbosity()) DescribeMemoryRange(start, size); } diff --git a/contrib/compiler-rt/lib/profile/GCDAProfiling.c b/contrib/compiler-rt/lib/profile/GCDAProfiling.c index 138af6ec4033..bb70ff0cf8f1 100644 --- a/contrib/compiler-rt/lib/profile/GCDAProfiling.c +++ b/contrib/compiler-rt/lib/profile/GCDAProfiling.c @@ -20,9 +20,6 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfilingPort.h" -#include "InstrProfilingUtil.h" - #include #include #include @@ -34,9 +31,6 @@ #else #include #include -#ifndef MAP_FILE -#define MAP_FILE 0 -#endif #endif #if defined(__FreeBSD__) && defined(__i386__) @@ -62,6 +56,9 @@ typedef unsigned int uint32_t; typedef unsigned long long uint64_t; #endif +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + /* #define DEBUG_GCDAPROFILING */ /* @@ -231,6 +228,7 @@ static void unmap_file() { * profiling enabled will emit to a different file. Only one file may be * started at a time. */ +COMPILER_RT_VISIBILITY void llvm_gcda_start_file(const char *orig_filename, const char version[4], uint32_t checksum) { const char *mode = "r+b"; @@ -238,17 +236,17 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], /* Try just opening the file. */ new_file = 0; - fd = open(filename, O_RDWR); + fd = open(filename, O_RDWR | O_BINARY); if (fd == -1) { /* Try opening the file, creating it if necessary. */ new_file = 1; mode = "w+b"; - fd = open(filename, O_RDWR | O_CREAT, 0644); + fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); if (fd == -1) { /* Try creating the directories first then opening the file. */ __llvm_profile_recursive_mkdir(filename); - fd = open(filename, O_RDWR | O_CREAT, 0644); + fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); if (fd == -1) { /* Bah! It's hopeless. */ int errnum = errno; @@ -263,7 +261,7 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], * same GCDA. This can fail if the filesystem doesn't support it, but in that * case we'll just carry on with the old racy behaviour and hope for the best. */ - flock(fd, LOCK_EX); + lprofLockFd(fd); output_file = fdopen(fd, mode); /* Initialize the write buffer. */ @@ -298,6 +296,7 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], /* Given an array of pointers to counters (counters), increment the n-th one, * where we're also given a pointer to n (predecessor). */ +COMPILER_RT_VISIBILITY void llvm_gcda_increment_indirect_counter(uint32_t *predecessor, uint64_t **counters) { uint64_t *counter; @@ -320,6 +319,7 @@ void llvm_gcda_increment_indirect_counter(uint32_t *predecessor, #endif } +COMPILER_RT_VISIBILITY void llvm_gcda_emit_function(uint32_t ident, const char *function_name, uint32_t func_checksum, uint8_t use_extra_checksum, uint32_t cfg_checksum) { @@ -346,6 +346,7 @@ void llvm_gcda_emit_function(uint32_t ident, const char *function_name, write_string(function_name); } +COMPILER_RT_VISIBILITY void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { uint32_t i; uint64_t *old_ctrs = NULL; @@ -397,6 +398,7 @@ void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { #endif } +COMPILER_RT_VISIBILITY void llvm_gcda_summary_info() { const uint32_t obj_summary_len = 9; /* Length for gcov compatibility. */ uint32_t i; @@ -450,6 +452,7 @@ void llvm_gcda_summary_info() { #endif } +COMPILER_RT_VISIBILITY void llvm_gcda_end_file() { /* Write out EOF record. */ if (output_file) { @@ -462,7 +465,8 @@ void llvm_gcda_end_file() { unmap_file(); } - flock(fd, LOCK_UN); + fflush(output_file); + lprofUnlockFd(fd); fclose(output_file); output_file = NULL; write_buffer = NULL; @@ -474,6 +478,7 @@ void llvm_gcda_end_file() { #endif } +COMPILER_RT_VISIBILITY void llvm_register_writeout_function(writeout_fn fn) { struct writeout_fn_node *new_node = malloc(sizeof(struct writeout_fn_node)); new_node->fn = fn; @@ -487,6 +492,7 @@ void llvm_register_writeout_function(writeout_fn fn) { } } +COMPILER_RT_VISIBILITY void llvm_writeout_files(void) { struct writeout_fn_node *curr = writeout_fn_head; @@ -496,6 +502,7 @@ void llvm_writeout_files(void) { } } +COMPILER_RT_VISIBILITY void llvm_delete_writeout_function_list(void) { while (writeout_fn_head) { struct writeout_fn_node *node = writeout_fn_head; @@ -506,6 +513,7 @@ void llvm_delete_writeout_function_list(void) { writeout_fn_head = writeout_fn_tail = NULL; } +COMPILER_RT_VISIBILITY void llvm_register_flush_function(flush_fn fn) { struct flush_fn_node *new_node = malloc(sizeof(struct flush_fn_node)); new_node->fn = fn; @@ -519,6 +527,7 @@ void llvm_register_flush_function(flush_fn fn) { } } +COMPILER_RT_VISIBILITY void __gcov_flush() { struct flush_fn_node *curr = flush_fn_head; @@ -528,6 +537,7 @@ void __gcov_flush() { } } +COMPILER_RT_VISIBILITY void llvm_delete_flush_function_list(void) { while (flush_fn_head) { struct flush_fn_node *node = flush_fn_head; @@ -538,6 +548,7 @@ void llvm_delete_flush_function_list(void) { flush_fn_head = flush_fn_tail = NULL; } +COMPILER_RT_VISIBILITY void llvm_gcov_init(writeout_fn wfn, flush_fn ffn) { static int atexit_ran = 0; diff --git a/contrib/compiler-rt/lib/profile/InstrProfData.inc b/contrib/compiler-rt/lib/profile/InstrProfData.inc index be0dd4ad04bf..6a98dc7b9b85 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfData.inc +++ b/contrib/compiler-rt/lib/profile/InstrProfData.inc @@ -628,9 +628,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 4 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 4 +#define INSTR_PROF_INDEX_VERSION 5 /* Coverage mapping format vresion (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 1 +#define INSTR_PROF_COVMAP_VERSION 2 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 diff --git a/contrib/compiler-rt/lib/profile/InstrProfiling.c b/contrib/compiler-rt/lib/profile/InstrProfiling.c index fe66fec50658..00b31e1ee7d4 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfiling.c +++ b/contrib/compiler-rt/lib/profile/InstrProfiling.c @@ -7,12 +7,14 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" #include #include #include #include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + #define INSTR_PROF_VALUE_PROF_DATA #include "InstrProfData.inc" diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c index d038bb9cb5b0..d7c0abbc16e3 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c @@ -7,9 +7,6 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" -#include "InstrProfilingUtil.h" #include #include #include @@ -31,6 +28,10 @@ #endif #endif +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + /* From where is profile name specified. * The order the enumerators define their * precedence. Re-order them may lead to @@ -85,7 +86,6 @@ typedef struct lprofFilename { COMPILER_RT_WEAK lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0}, 0, 0, 0, PNS_unknown}; -int getpid(void); static int getCurFilenameLength(); static const char *getCurFilename(char *FilenameBuf); static unsigned doMerging() { return lprofCurFilename.MergePoolSize; } @@ -325,7 +325,7 @@ static int parseFilenamePattern(const char *FilenamePat, if (FilenamePat[I] == '%') { if (FilenamePat[++I] == 'p') { if (!NumPids++) { - if (snprintf(PidChars, MAX_PID_SIZE, "%d", getpid()) <= 0) { + if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) { PROF_WARN("Unable to get pid for filename pattern %s. Using the " "default name.", FilenamePat); @@ -519,8 +519,10 @@ void __llvm_profile_initialize_file(void) { EnvFilenamePat = getFilenamePatFromEnv(); if (EnvFilenamePat) { - SelectedPat = EnvFilenamePat; - PNS = PNS_environment; + /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid + at the moment when __llvm_profile_write_file() gets executed. */ + parseAndSetFilename(EnvFilenamePat, PNS_environment, 1); + return; } else if (hasCommandLineOverrider) { SelectedPat = INSTR_PROF_PROFILE_NAME_VAR; PNS = PNS_command_line; diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h b/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h index 36490ef7d433..40540ab5f205 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -10,8 +10,9 @@ #ifndef PROFILE_INSTRPROFILING_INTERNALH_ #define PROFILE_INSTRPROFILING_INTERNALH_ +#include + #include "InstrProfiling.h" -#include "stddef.h" /*! * \brief Write instrumentation data to the given buffer, given explicit diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index b6c780ff514f..89f1ab4cfeda 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -7,11 +7,13 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" +#if defined(__linux__) || defined(__FreeBSD__) || \ + (defined(__sun__) && defined(__svr4__)) -#if defined(__linux__) || defined(__FreeBSD__) #include +#include "InstrProfiling.h" + #define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) #define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME) #define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME) diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index b25966487e91..a339abc7f883 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -7,12 +7,13 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" - -#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !(defined(__sun__) && defined(__svr4__)) #include +#include "InstrProfiling.h" + static const __llvm_profile_data *DataFirst = NULL; static const __llvm_profile_data *DataLast = NULL; static const char *NamesFirst = NULL; diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPort.h b/contrib/compiler-rt/lib/profile/InstrProfilingPort.h index 5789351956b9..cd1264b0e575 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPort.h +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPort.h @@ -7,6 +7,9 @@ |* \*===----------------------------------------------------------------------===*/ +/* This header must be included after all others so it can provide fallback + definitions for stuff missing in system headers. */ + #ifndef PROFILE_INSTRPROFILING_PORT_H_ #define PROFILE_INSTRPROFILING_PORT_H_ @@ -44,9 +47,6 @@ #define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1)) #else #define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len) -#ifndef _MSC_VER -#define COMPILER_RT_HAS_UNAME 1 -#endif #endif #if COMPILER_RT_HAS_ATOMICS == 1 @@ -107,6 +107,14 @@ #define PROF_NOTE(Format, ...) \ fprintf(stderr, "LLVM Profile Note: " Format, __VA_ARGS__); +#ifndef MAP_FILE +#define MAP_FILE 0 +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + #if defined(__FreeBSD__) #include diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c index fb68f30a5e1f..7f49d0f9fae1 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c @@ -7,13 +7,11 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfilingUtil.h" -#include "InstrProfiling.h" - #ifdef _WIN32 #include -#include +#include #include +#include "WindowsMMap.h" #else #include #include @@ -34,6 +32,9 @@ #include #endif +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + COMPILER_RT_VISIBILITY void __llvm_profile_recursive_mkdir(char *path) { int i; @@ -86,16 +87,16 @@ COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { #elif defined(COMPILER_RT_HAS_UNAME) COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { struct utsname N; - int R; - if (!(R = uname(&N))) + int R = uname(&N); + if (R >= 0) { strncpy(Name, N.nodename, Len); + return 0; + } return R; } #endif -COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { - FILE *f; - int fd; +COMPILER_RT_VISIBILITY int lprofLockFd(int fd) { #ifdef COMPILER_RT_HAS_FCNTL_LCK struct flock s_flock; @@ -103,21 +104,59 @@ COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { s_flock.l_start = 0; s_flock.l_len = 0; /* Until EOF. */ s_flock.l_pid = getpid(); - s_flock.l_type = F_WRLCK; - fd = open(ProfileName, O_RDWR | O_CREAT, 0666); - if (fd < 0) - return NULL; while (fcntl(fd, F_SETLKW, &s_flock) == -1) { if (errno != EINTR) { if (errno == ENOLCK) { - PROF_WARN("Data may be corrupted during profile merging : %s\n", - "Fail to obtain file lock due to system limit."); + return -1; } break; } } + return 0; +#else + flock(fd, LOCK_EX); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { +#ifdef COMPILER_RT_HAS_FCNTL_LCK + struct flock s_flock; + + s_flock.l_whence = SEEK_SET; + s_flock.l_start = 0; + s_flock.l_len = 0; /* Until EOF. */ + s_flock.l_pid = getpid(); + s_flock.l_type = F_UNLCK; + + while (fcntl(fd, F_SETLKW, &s_flock) == -1) { + if (errno != EINTR) { + if (errno == ENOLCK) { + return -1; + } + break; + } + } + return 0; +#else + flock(fd, LOCK_UN); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { + FILE *f; + int fd; +#ifdef COMPILER_RT_HAS_FCNTL_LCK + fd = open(ProfileName, O_RDWR | O_CREAT, 0666); + if (fd < 0) + return NULL; + + if (lprofLockFd(fd) != 0) + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); f = fdopen(fd, "r+b"); #elif defined(_WIN32) diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h index 9698599606e0..2f2ea1b876a8 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h +++ b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -16,6 +16,9 @@ /*! \brief Create a directory tree. */ void __llvm_profile_recursive_mkdir(char *Pathname); +int lprofLockFd(int fd); +int lprofUnlockFd(int fd); + /*! Open file \c Filename for read+write with write * lock for exclusive access. The caller will block * if the lock is already held by another process. */ diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c index 44263da80097..3db0de8a6723 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c @@ -7,13 +7,15 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" -#include "InstrProfilingUtil.h" /* For PS4 getenv shim. */ #include #include #include #include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + #define INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_COMMON_API_IMPL #include "InstrProfData.inc" @@ -22,7 +24,7 @@ static int hasStaticCounters = 1; static int OutOfNodesWarnings = 0; static int hasNonDefaultValsPerSite = 0; #define INSTR_PROF_MAX_VP_WARNS 10 -#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 8 +#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16 #define INSTR_PROF_VNODE_POOL_SIZE 1024 #ifndef _MSC_VER diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c b/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c index d4c9b9bd663c..7c6061d2c65a 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -7,14 +7,15 @@ |* \*===----------------------------------------------------------------------===*/ -#include "InstrProfiling.h" -#include "InstrProfilingInternal.h" #ifdef _MSC_VER /* For _alloca */ #include #endif #include +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + #define INSTR_PROF_VALUE_PROF_DATA #include "InstrProfData.inc" diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.c b/contrib/compiler-rt/lib/profile/WindowsMMap.c index f81d7da5389e..dc87a888ae7b 100644 --- a/contrib/compiler-rt/lib/profile/WindowsMMap.c +++ b/contrib/compiler-rt/lib/profile/WindowsMMap.c @@ -18,11 +18,12 @@ #if defined(_WIN32) #include "WindowsMMap.h" -#include "InstrProfiling.h" #define WIN32_LEAN_AND_MEAN #include +#include "InstrProfiling.h" + #ifdef __USE_FILE_OFFSET64 # define DWORD_HI(x) (x >> 32) # define DWORD_LO(x) ((x) & 0xffffffff) @@ -120,9 +121,61 @@ int msync(void *addr, size_t length, int flags) } COMPILER_RT_VISIBILITY -int flock(int fd, int operation) -{ - return -1; /* Not supported. */ +int lock(HANDLE handle, DWORD lockType, BOOL blocking) { + DWORD flags = lockType; + if (!blocking) + flags |= LOCKFILE_FAIL_IMMEDIATELY; + + OVERLAPPED overlapped; + ZeroMemory(&overlapped, sizeof(OVERLAPPED)); + overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped); + if (!result) { + DWORD dw = GetLastError(); + + // In non-blocking mode, return an error if the file is locked. + if (!blocking && dw == ERROR_LOCK_VIOLATION) + return -1; // EWOULDBLOCK + + // If the error is ERROR_IO_PENDING, we need to wait until the operation + // finishes. Otherwise, we return an error. + if (dw != ERROR_IO_PENDING) + return -1; + + DWORD dwNumBytes; + if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE)) + return -1; + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int flock(int fd, int operation) { + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) + return -1; + + BOOL blocking = (operation & LOCK_NB) == 0; + int op = operation & ~LOCK_NB; + + switch (op) { + case LOCK_EX: + return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking); + + case LOCK_SH: + return lock(handle, 0, blocking); + + case LOCK_UN: + if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + break; + + default: + return -1; + } + + return 0; } #undef DWORD_HI diff --git a/contrib/compiler-rt/lib/safestack/safestack.cc b/contrib/compiler-rt/lib/safestack/safestack.cc index b194b6cfa8f9..d783cd5a9b29 100644 --- a/contrib/compiler-rt/lib/safestack/safestack.cc +++ b/contrib/compiler-rt/lib/safestack/safestack.cc @@ -21,7 +21,9 @@ #include #include #include +#if !defined(__NetBSD__) #include +#endif #include "interception/interception.h" #include "sanitizer_common/sanitizer_common.h" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sancov_begin.S b/contrib/compiler-rt/lib/sanitizer_common/sancov_begin.S new file mode 100644 index 000000000000..c8ad0a0bcb5c --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sancov_begin.S @@ -0,0 +1,5 @@ + .type __start___sancov_guards,@object + .globl __start___sancov_guards + .section __sancov_guards,"aw",@progbits + .p2align 2 +__start___sancov_guards: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sancov_end.S b/contrib/compiler-rt/lib/sanitizer_common/sancov_end.S new file mode 100644 index 000000000000..31117b1c0b56 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sancov_end.S @@ -0,0 +1,5 @@ + .type __stop___sancov_guards,@object + .globl __stop___sancov_guards + .section __sancov_guards,"aw",@progbits + .p2align 2 +__stop___sancov_guards: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc index 84f523c5e431..fc4f7a75ae34 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc @@ -178,11 +178,13 @@ void InternalFree(void *addr, InternalAllocatorCache *cache) { } // LowLevelAllocator +constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; +static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; static LowLevelAllocateCallback low_level_alloc_callback; void *LowLevelAllocator::Allocate(uptr size) { // Align allocation size. - size = RoundUpTo(size, 8); + size = RoundUpTo(size, low_level_alloc_min_alignment); if (allocated_end_ - allocated_current_ < (sptr)size) { uptr size_to_allocate = Max(size, GetPageSizeCached()); allocated_current_ = @@ -199,6 +201,11 @@ void *LowLevelAllocator::Allocate(uptr size) { return res; } +void SetLowLevelAllocateMinAlignment(uptr alignment) { + CHECK(IsPowerOfTwo(alignment)); + low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment); +} + void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { low_level_alloc_callback = callback; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index 8c5696ea789c..38368361de6f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -56,6 +56,19 @@ struct NoOpMapUnmapCallback { // Callback type for iterating over chunks. typedef void (*ForEachChunkCallback)(uptr chunk, void *arg); +INLINE u32 Rand(u32 *state) { // ANSI C linear congruential PRNG. + return (*state = *state * 1103515245 + 12345) >> 16; +} + +INLINE u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n) + +template +INLINE void RandomShuffle(T *a, u32 n, u32 *rand_state) { + if (n <= 1) return; + for (u32 i = n - 1; i > 0; i--) + Swap(a[i], a[RandN(rand_state, i + 1)]); +} + #include "sanitizer_allocator_size_class_map.h" #include "sanitizer_allocator_stats.h" #include "sanitizer_allocator_primary64.h" diff --git a/contrib/llvm/lib/Target/AMDGPU/CIInstructions.td b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc similarity index 51% rename from contrib/llvm/lib/Target/AMDGPU/CIInstructions.td rename to contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc index 26a483a8abf6..dc263dbef5fc 100644 --- a/contrib/llvm/lib/Target/AMDGPU/CIInstructions.td +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc @@ -1,4 +1,4 @@ -//===-- CIInstructions.td - CI Instruction Defintions ---------------------===// +//===-- sanitizer_allocator_checks.cc ---------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,10 +6,18 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// Instruction definitions for CI and newer. +// +// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory +// allocators. +// //===----------------------------------------------------------------------===// -// Remaining instructions: -// S_CBRANCH_CDBGUSER -// S_CBRANCH_CDBGSYS -// S_CBRANCH_CDBGSYS_OR_USER -// S_CBRANCH_CDBGSYS_AND_USER \ No newline at end of file + +#include "sanitizer_errno.h" + +namespace __sanitizer { + +void SetErrnoToENOMEM() { + errno = errno_ENOMEM; +} + +} // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h index 202916eae348..b61a8b2eb3b6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h @@ -15,17 +15,22 @@ #ifndef SANITIZER_ALLOCATOR_CHECKS_H #define SANITIZER_ALLOCATOR_CHECKS_H -#include "sanitizer_errno.h" #include "sanitizer_internal_defs.h" #include "sanitizer_common.h" #include "sanitizer_platform.h" namespace __sanitizer { +// The following is defined in a separate compilation unit to avoid pulling in +// sanitizer_errno.h in this header, which leads to conflicts when other system +// headers include errno.h. This is usually the result of an unlikely event, +// and as such we do not care as much about having it inlined. +void SetErrnoToENOMEM(); + // A common errno setting logic shared by almost all sanitizer allocator APIs. INLINE void *SetErrnoOnNull(void *ptr) { if (UNLIKELY(!ptr)) - errno = errno_ENOMEM; + SetErrnoToENOMEM(); return ptr; } @@ -59,6 +64,12 @@ INLINE bool CheckForCallocOverflow(uptr size, uptr n) { return (max / size) < n; } +// Returns true if the size passed to pvalloc overflows when rounded to the next +// multiple of page_size. +INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) { + return RoundUpTo(size, page_size) < size; +} + } // namespace __sanitizer #endif // SANITIZER_ALLOCATOR_CHECKS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h index efd25cadfe74..0d8a2a174bb5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -77,6 +77,10 @@ class CombinedAllocator { primary_.SetReleaseToOSIntervalMs(release_to_os_interval_ms); } + void ForceReleaseToOS() { + primary_.ForceReleaseToOS(); + } + void Deallocate(AllocatorCache *cache, void *p) { if (!p) return; if (primary_.PointerIsMine(p)) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h index 13910e719e78..2f5ce3151ee4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h @@ -38,6 +38,9 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_free_hook(void *ptr); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_purge_allocator(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_print_memory_profile(uptr top_percent, uptr max_number_of_contexts); } // extern "C" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h index ec0742c20be2..1b3c2c0c19d1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h @@ -26,9 +26,6 @@ struct SizeClassAllocatorLocalCache template struct SizeClassAllocator64LocalCache { typedef SizeClassAllocator Allocator; - static const uptr kNumClasses = SizeClassAllocator::kNumClasses; - typedef typename Allocator::SizeClassMapT SizeClassMap; - typedef typename Allocator::CompactPtrT CompactPtrT; void Init(AllocatorGlobalStats *s) { stats_.Init(); @@ -76,14 +73,18 @@ struct SizeClassAllocator64LocalCache { } void Drain(SizeClassAllocator *allocator) { - for (uptr class_id = 0; class_id < kNumClasses; class_id++) { - PerClass *c = &per_class_[class_id]; + for (uptr i = 0; i < kNumClasses; i++) { + PerClass *c = &per_class_[i]; while (c->count > 0) - Drain(c, allocator, class_id, c->count); + Drain(c, allocator, i, c->count); } } - // private: + private: + typedef typename Allocator::SizeClassMapT SizeClassMap; + static const uptr kNumClasses = SizeClassMap::kNumClasses; + typedef typename Allocator::CompactPtrT CompactPtrT; + struct PerClass { u32 count; u32 max_count; @@ -94,7 +95,7 @@ struct SizeClassAllocator64LocalCache { AllocatorStats stats_; void InitCache() { - if (per_class_[1].max_count) + if (LIKELY(per_class_[1].max_count)) return; for (uptr i = 0; i < kNumClasses; i++) { PerClass *c = &per_class_[i]; @@ -130,7 +131,6 @@ template struct SizeClassAllocator32LocalCache { typedef SizeClassAllocator Allocator; typedef typename Allocator::TransferBatch TransferBatch; - static const uptr kNumClasses = SizeClassAllocator::kNumClasses; void Init(AllocatorGlobalStats *s) { stats_.Init(); @@ -138,6 +138,21 @@ struct SizeClassAllocator32LocalCache { s->Register(&stats_); } + // Returns a TransferBatch suitable for class_id. + TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator, + TransferBatch *b) { + if (uptr batch_class_id = per_class_[class_id].batch_class_id) + return (TransferBatch*)Allocate(allocator, batch_class_id); + return b; + } + + // Destroys TransferBatch b. + void DestroyBatch(uptr class_id, SizeClassAllocator *allocator, + TransferBatch *b) { + if (uptr batch_class_id = per_class_[class_id].batch_class_id) + Deallocate(allocator, batch_class_id, b); + } + void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) { Drain(allocator); if (s) @@ -173,66 +188,57 @@ struct SizeClassAllocator32LocalCache { } void Drain(SizeClassAllocator *allocator) { - for (uptr class_id = 0; class_id < kNumClasses; class_id++) { - PerClass *c = &per_class_[class_id]; + for (uptr i = 0; i < kNumClasses; i++) { + PerClass *c = &per_class_[i]; while (c->count > 0) - Drain(allocator, class_id); + Drain(allocator, i); } } - // private: - typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap; + private: + typedef typename Allocator::SizeClassMapT SizeClassMap; + static const uptr kBatchClassID = SizeClassMap::kBatchClassID; + static const uptr kNumClasses = SizeClassMap::kNumClasses; + // If kUseSeparateSizeClassForBatch is true, all TransferBatch objects are + // allocated from kBatchClassID size class (except for those that are needed + // for kBatchClassID itself). The goal is to have TransferBatches in a totally + // different region of RAM to improve security. + static const bool kUseSeparateSizeClassForBatch = + Allocator::kUseSeparateSizeClassForBatch; + struct PerClass { uptr count; uptr max_count; uptr class_size; - uptr class_id_for_transfer_batch; + uptr batch_class_id; void *batch[2 * TransferBatch::kMaxNumCached]; }; PerClass per_class_[kNumClasses]; AllocatorStats stats_; void InitCache() { - if (per_class_[1].max_count) + if (LIKELY(per_class_[1].max_count)) return; - // TransferBatch class is declared in SizeClassAllocator. - uptr class_id_for_transfer_batch = - SizeClassMap::ClassID(sizeof(TransferBatch)); + const uptr batch_class_id = SizeClassMap::ClassID(sizeof(TransferBatch)); for (uptr i = 0; i < kNumClasses; i++) { PerClass *c = &per_class_[i]; uptr max_cached = TransferBatch::MaxCached(i); c->max_count = 2 * max_cached; c->class_size = Allocator::ClassIdToSize(i); - // We transfer chunks between central and thread-local free lists in - // batches. For small size classes we allocate batches separately. For - // large size classes we may use one of the chunks to store the batch. - // sizeof(TransferBatch) must be a power of 2 for more efficient - // allocation. - c->class_id_for_transfer_batch = (c->class_size < + // Precompute the class id to use to store batches for the current class + // id. 0 means the class size is large enough to store a batch within one + // of the chunks. If using a separate size class, it will always be + // kBatchClassID, except for kBatchClassID itself. + if (kUseSeparateSizeClassForBatch) { + c->batch_class_id = (i == kBatchClassID) ? 0 : kBatchClassID; + } else { + c->batch_class_id = (c->class_size < TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ? - class_id_for_transfer_batch : 0; + batch_class_id : 0; + } } } - // Returns a TransferBatch suitable for class_id. - // For small size classes allocates the batch from the allocator. - // For large size classes simply returns b. - TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator, - TransferBatch *b) { - if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch) - return (TransferBatch*)Allocate(allocator, batch_class_id); - return b; - } - - // Destroys TransferBatch b. - // For small size classes deallocates b to the allocator. - // Does notthing for large size classes. - void DestroyBatch(uptr class_id, SizeClassAllocator *allocator, - TransferBatch *b) { - if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch) - Deallocate(allocator, batch_class_id, b); - } - NOINLINE bool Refill(SizeClassAllocator *allocator, uptr class_id) { InitCache(); PerClass *c = &per_class_[class_id]; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h index e85821543d27..6c682aff64a2 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -41,6 +41,7 @@ template struct SizeClassAllocator32LocalCache; struct SizeClassAllocator32FlagMasks { // Bit masks. enum { kRandomShuffleChunks = 1, + kUseSeparateSizeClassForBatch = 2, }; }; @@ -55,8 +56,10 @@ class SizeClassAllocator32 { typedef typename Params::ByteMap ByteMap; typedef typename Params::MapUnmapCallback MapUnmapCallback; - static const bool kRandomShuffleChunks = - Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks; + static const bool kRandomShuffleChunks = Params::kFlags & + SizeClassAllocator32FlagMasks::kRandomShuffleChunks; + static const bool kUseSeparateSizeClassForBatch = Params::kFlags & + SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; struct TransferBatch { static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2; @@ -94,11 +97,11 @@ class SizeClassAllocator32 { static const uptr kBatchSize = sizeof(TransferBatch); COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0); - COMPILER_CHECK(sizeof(TransferBatch) == - SizeClassMap::kMaxNumCachedHint * sizeof(uptr)); + COMPILER_CHECK(kBatchSize == SizeClassMap::kMaxNumCachedHint * sizeof(uptr)); static uptr ClassIdToSize(uptr class_id) { - return SizeClassMap::Size(class_id); + return (class_id == SizeClassMap::kBatchClassID) ? + kBatchSize : SizeClassMap::Size(class_id); } typedef SizeClassAllocator32 ThisT; @@ -117,6 +120,10 @@ class SizeClassAllocator32 { // This is empty here. Currently only implemented in 64-bit allocator. } + void ForceReleaseToOS() { + // Currently implemented in 64-bit allocator only. + } + void *MapWithCallback(uptr size) { void *res = MmapOrDie(size, "SizeClassAllocator32"); MapUnmapCallback().OnMap((uptr)res, size); @@ -161,9 +168,9 @@ class SizeClassAllocator32 { NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id, TransferBatch *b) { CHECK_LT(class_id, kNumClasses); + CHECK_GT(b->Count(), 0); SizeClassInfo *sci = GetSizeClassInfo(class_id); SpinMutexLock l(&sci->mutex); - CHECK_GT(b->Count(), 0); sci->free_list.push_front(b); } @@ -261,7 +268,8 @@ class SizeClassAllocator32 { struct SizeClassInfo { SpinMutex mutex; IntrusiveList free_list; - char padding[kCacheLineSize - sizeof(uptr) - + u32 rand_state; + char padding[kCacheLineSize - 2 * sizeof(uptr) - sizeof(IntrusiveList)]; }; COMPILER_CHECK(sizeof(SizeClassInfo) == kCacheLineSize); @@ -294,29 +302,62 @@ class SizeClassAllocator32 { return &size_class_info_array[class_id]; } + bool PopulateBatches(AllocatorCache *c, SizeClassInfo *sci, uptr class_id, + TransferBatch **current_batch, uptr max_count, + uptr *pointers_array, uptr count) { + // If using a separate class for batches, we do not need to shuffle it. + if (kRandomShuffleChunks && (!kUseSeparateSizeClassForBatch || + class_id != SizeClassMap::kBatchClassID)) + RandomShuffle(pointers_array, count, &sci->rand_state); + TransferBatch *b = *current_batch; + for (uptr i = 0; i < count; i++) { + if (!b) { + b = c->CreateBatch(class_id, this, (TransferBatch*)pointers_array[i]); + if (UNLIKELY(!b)) + return false; + b->Clear(); + } + b->Add((void*)pointers_array[i]); + if (b->Count() == max_count) { + sci->free_list.push_back(b); + b = nullptr; + } + } + *current_batch = b; + return true; + } + bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c, SizeClassInfo *sci, uptr class_id) { uptr size = ClassIdToSize(class_id); uptr reg = AllocateRegion(stat, class_id); if (UNLIKELY(!reg)) return false; + if (kRandomShuffleChunks) + if (UNLIKELY(sci->rand_state == 0)) + // The random state is initialized from ASLR (PIE) and time. + sci->rand_state = reinterpret_cast(sci) ^ NanoTime(); uptr n_chunks = kRegionSize / (size + kMetadataSize); uptr max_count = TransferBatch::MaxCached(class_id); CHECK_GT(max_count, 0); TransferBatch *b = nullptr; + const uptr kShuffleArraySize = 48; + uptr shuffle_array[kShuffleArraySize]; + uptr count = 0; for (uptr i = reg; i < reg + n_chunks * size; i += size) { - if (!b) { - b = c->CreateBatch(class_id, this, (TransferBatch*)i); - if (UNLIKELY(!b)) + shuffle_array[count++] = i; + if (count == kShuffleArraySize) { + if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count, + shuffle_array, count))) return false; - b->Clear(); - } - b->Add((void*)i); - if (b->Count() == max_count) { - sci->free_list.push_back(b); - b = nullptr; + count = 0; } } + if (count) { + if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count, + shuffle_array, count))) + return false; + } if (b) { CHECK_GT(b->Count(), 0); sci->free_list.push_back(b); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h index 0c2e72ce7eb1..651a64b04f2e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -62,21 +62,20 @@ class SizeClassAllocator64 { // as a 4-byte integer (offset from the region start shifted right by 4). typedef u32 CompactPtrT; static const uptr kCompactPtrScale = 4; - CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) { + CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) const { return static_cast((ptr - base) >> kCompactPtrScale); } - uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) { + uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) const { return base + (static_cast(ptr32) << kCompactPtrScale); } void Init(s32 release_to_os_interval_ms) { uptr TotalSpaceSize = kSpaceSize + AdditionalSize(); if (kUsingConstantSpaceBeg) { - CHECK_EQ(kSpaceBeg, reinterpret_cast( - MmapFixedNoAccess(kSpaceBeg, TotalSpaceSize))); + CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize, AllocatorName(), + kSpaceBeg)); } else { - NonConstSpaceBeg = - reinterpret_cast(MmapNoAccess(TotalSpaceSize)); + NonConstSpaceBeg = address_range.Init(TotalSpaceSize, AllocatorName()); CHECK_NE(NonConstSpaceBeg, ~(uptr)0); } SetReleaseToOSIntervalMs(release_to_os_interval_ms); @@ -92,6 +91,13 @@ class SizeClassAllocator64 { memory_order_relaxed); } + void ForceReleaseToOS() { + for (uptr class_id = 1; class_id < kNumClasses; class_id++) { + BlockingMutexLock l(&GetRegionInfo(class_id)->mutex); + MaybeReleaseToOS(class_id, true /*force*/); + } + } + static bool CanAllocate(uptr size, uptr alignment) { return size <= SizeClassMap::kMaxSize && alignment <= SizeClassMap::kMaxSize; @@ -116,7 +122,7 @@ class SizeClassAllocator64 { region->num_freed_chunks = new_num_freed_chunks; region->stats.n_freed += n_chunks; - MaybeReleaseToOS(class_id); + MaybeReleaseToOS(class_id, false /*force*/); } NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id, @@ -155,7 +161,7 @@ class SizeClassAllocator64 { space_beg; } - uptr GetRegionBeginBySizeClass(uptr class_id) { + uptr GetRegionBeginBySizeClass(uptr class_id) const { return SpaceBeg() + kRegionSize * class_id; } @@ -223,30 +229,39 @@ class SizeClassAllocator64 { uptr avail_chunks = region->allocated_user / ClassIdToSize(class_id); Printf( "%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd " - "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd\n", + "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd " + "last released: %6zdK region: 0x%zx\n", region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id), region->mapped_user >> 10, region->stats.n_allocated, region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks, - rss >> 10, region->rtoi.num_releases); + rss >> 10, region->rtoi.num_releases, + region->rtoi.last_released_bytes >> 10, + SpaceBeg() + kRegionSize * class_id); } void PrintStats() { - uptr total_mapped = 0; - uptr n_allocated = 0; - uptr n_freed = 0; - for (uptr class_id = 1; class_id < kNumClasses; class_id++) { - RegionInfo *region = GetRegionInfo(class_id); - total_mapped += region->mapped_user; - n_allocated += region->stats.n_allocated; - n_freed += region->stats.n_freed; - } - Printf("Stats: SizeClassAllocator64: %zdM mapped in %zd allocations; " - "remains %zd\n", - total_mapped >> 20, n_allocated, n_allocated - n_freed); uptr rss_stats[kNumClasses]; for (uptr class_id = 0; class_id < kNumClasses; class_id++) rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id; GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses); + + uptr total_mapped = 0; + uptr total_rss = 0; + uptr n_allocated = 0; + uptr n_freed = 0; + for (uptr class_id = 1; class_id < kNumClasses; class_id++) { + RegionInfo *region = GetRegionInfo(class_id); + if (region->mapped_user != 0) { + total_mapped += region->mapped_user; + total_rss += rss_stats[class_id]; + } + n_allocated += region->stats.n_allocated; + n_freed += region->stats.n_freed; + } + + Printf("Stats: SizeClassAllocator64: %zdM mapped (%zdM rss) in " + "%zd allocations; remains %zd\n", total_mapped >> 20, + total_rss >> 20, n_allocated, n_allocated - n_freed); for (uptr class_id = 1; class_id < kNumClasses; class_id++) PrintStats(class_id, rss_stats[class_id]); } @@ -294,7 +309,243 @@ class SizeClassAllocator64 { static const uptr kNumClasses = SizeClassMap::kNumClasses; static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded; + // A packed array of counters. Each counter occupies 2^n bits, enough to store + // counter's max_value. Ctor will try to allocate the required buffer via + // mapper->MapPackedCounterArrayBuffer and the caller is expected to check + // whether the initialization was successful by checking IsAllocated() result. + // For the performance sake, none of the accessors check the validity of the + // arguments, it is assumed that index is always in [0, n) range and the value + // is not incremented past max_value. + template + class PackedCounterArray { + public: + PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper) + : n(num_counters), memory_mapper(mapper) { + CHECK_GT(num_counters, 0); + CHECK_GT(max_value, 0); + constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL; + // Rounding counter storage size up to the power of two allows for using + // bit shifts calculating particular counter's index and offset. + uptr counter_size_bits = + RoundUpToPowerOfTwo(MostSignificantSetBitIndex(max_value) + 1); + CHECK_LE(counter_size_bits, kMaxCounterBits); + counter_size_bits_log = Log2(counter_size_bits); + counter_mask = ~0ULL >> (kMaxCounterBits - counter_size_bits); + + uptr packing_ratio = kMaxCounterBits >> counter_size_bits_log; + CHECK_GT(packing_ratio, 0); + packing_ratio_log = Log2(packing_ratio); + bit_offset_mask = packing_ratio - 1; + + buffer_size = + (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) * + sizeof(*buffer); + buffer = reinterpret_cast( + memory_mapper->MapPackedCounterArrayBuffer(buffer_size)); + } + ~PackedCounterArray() { + if (buffer) { + memory_mapper->UnmapPackedCounterArrayBuffer( + reinterpret_cast(buffer), buffer_size); + } + } + + bool IsAllocated() const { + return !!buffer; + } + + u64 GetCount() const { + return n; + } + + uptr Get(uptr i) const { + DCHECK_LT(i, n); + uptr index = i >> packing_ratio_log; + uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log; + return (buffer[index] >> bit_offset) & counter_mask; + } + + void Inc(uptr i) const { + DCHECK_LT(Get(i), counter_mask); + uptr index = i >> packing_ratio_log; + uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log; + buffer[index] += 1ULL << bit_offset; + } + + void IncRange(uptr from, uptr to) const { + DCHECK_LE(from, to); + for (uptr i = from; i <= to; i++) + Inc(i); + } + + private: + const u64 n; + u64 counter_size_bits_log; + u64 counter_mask; + u64 packing_ratio_log; + u64 bit_offset_mask; + + MemoryMapperT* const memory_mapper; + u64 buffer_size; + u64* buffer; + }; + + template + class FreePagesRangeTracker { + public: + explicit FreePagesRangeTracker(MemoryMapperT* mapper) + : memory_mapper(mapper), + page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)), + in_the_range(false), current_page(0), current_range_start_page(0) {} + + void NextPage(bool freed) { + if (freed) { + if (!in_the_range) { + current_range_start_page = current_page; + in_the_range = true; + } + } else { + CloseOpenedRange(); + } + current_page++; + } + + void Done() { + CloseOpenedRange(); + } + + private: + void CloseOpenedRange() { + if (in_the_range) { + memory_mapper->ReleasePageRangeToOS( + current_range_start_page << page_size_scaled_log, + current_page << page_size_scaled_log); + in_the_range = false; + } + } + + MemoryMapperT* const memory_mapper; + const uptr page_size_scaled_log; + bool in_the_range; + uptr current_page; + uptr current_range_start_page; + }; + + // Iterates over the free_array to identify memory pages containing freed + // chunks only and returns these pages back to OS. + // allocated_pages_count is the total number of pages allocated for the + // current bucket. + template + static void ReleaseFreeMemoryToOS(CompactPtrT *free_array, + uptr free_array_count, uptr chunk_size, + uptr allocated_pages_count, + MemoryMapperT *memory_mapper) { + const uptr page_size = GetPageSizeCached(); + + // Figure out the number of chunks per page and whether we can take a fast + // path (the number of chunks per page is the same for all pages). + uptr full_pages_chunk_count_max; + bool same_chunk_count_per_page; + if (chunk_size <= page_size && page_size % chunk_size == 0) { + // Same number of chunks per page, no cross overs. + full_pages_chunk_count_max = page_size / chunk_size; + same_chunk_count_per_page = true; + } else if (chunk_size <= page_size && page_size % chunk_size != 0 && + chunk_size % (page_size % chunk_size) == 0) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks, but all pages contain the same + // number of chunks. + full_pages_chunk_count_max = page_size / chunk_size + 1; + same_chunk_count_per_page = true; + } else if (chunk_size <= page_size) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks. + full_pages_chunk_count_max = page_size / chunk_size + 2; + same_chunk_count_per_page = false; + } else if (chunk_size > page_size && chunk_size % page_size == 0) { + // One chunk covers multiple pages, no cross overs. + full_pages_chunk_count_max = 1; + same_chunk_count_per_page = true; + } else if (chunk_size > page_size) { + // One chunk covers multiple pages, Some chunks are crossing page + // boundaries. Some pages contain one chunk, some contain two. + full_pages_chunk_count_max = 2; + same_chunk_count_per_page = false; + } else { + UNREACHABLE("All chunk_size/page_size ratios must be handled."); + } + + PackedCounterArray counters(allocated_pages_count, + full_pages_chunk_count_max, + memory_mapper); + if (!counters.IsAllocated()) + return; + + const uptr chunk_size_scaled = chunk_size >> kCompactPtrScale; + const uptr page_size_scaled = page_size >> kCompactPtrScale; + const uptr page_size_scaled_log = Log2(page_size_scaled); + + // Iterate over free chunks and count how many free chunks affect each + // allocated page. + if (chunk_size <= page_size && page_size % chunk_size == 0) { + // Each chunk affects one page only. + for (uptr i = 0; i < free_array_count; i++) + counters.Inc(free_array[i] >> page_size_scaled_log); + } else { + // In all other cases chunks might affect more than one page. + for (uptr i = 0; i < free_array_count; i++) { + counters.IncRange( + free_array[i] >> page_size_scaled_log, + (free_array[i] + chunk_size_scaled - 1) >> page_size_scaled_log); + } + } + + // Iterate over pages detecting ranges of pages with chunk counters equal + // to the expected number of chunks for the particular page. + FreePagesRangeTracker range_tracker(memory_mapper); + if (same_chunk_count_per_page) { + // Fast path, every page has the same number of chunks affecting it. + for (uptr i = 0; i < counters.GetCount(); i++) + range_tracker.NextPage(counters.Get(i) == full_pages_chunk_count_max); + } else { + // Show path, go through the pages keeping count how many chunks affect + // each page. + const uptr pn = + chunk_size < page_size ? page_size_scaled / chunk_size_scaled : 1; + const uptr pnc = pn * chunk_size_scaled; + // The idea is to increment the current page pointer by the first chunk + // size, middle portion size (the portion of the page covered by chunks + // except the first and the last one) and then the last chunk size, adding + // up the number of chunks on the current page and checking on every step + // whether the page boundary was crossed. + uptr prev_page_boundary = 0; + uptr current_boundary = 0; + for (uptr i = 0; i < counters.GetCount(); i++) { + uptr page_boundary = prev_page_boundary + page_size_scaled; + uptr chunks_per_page = pn; + if (current_boundary < page_boundary) { + if (current_boundary > prev_page_boundary) + chunks_per_page++; + current_boundary += pnc; + if (current_boundary < page_boundary) { + chunks_per_page++; + current_boundary += chunk_size_scaled; + } + } + prev_page_boundary = page_boundary; + + range_tracker.NextPage(counters.Get(i) == chunks_per_page); + } + } + range_tracker.Done(); + } + private: + friend class MemoryMapper; + + ReservedAddressRange address_range; + static const char *AllocatorName() { return "sanitizer_allocator"; } + static const uptr kRegionSize = kSpaceSize / kNumClassesRounded; // FreeArray is the array of free-d chunks (stored as 4-byte offsets). // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize @@ -330,6 +581,7 @@ class SizeClassAllocator64 { uptr n_freed_at_last_release; uptr num_releases; u64 last_release_at_ns; + u64 last_released_bytes; }; struct RegionInfo { @@ -347,30 +599,18 @@ class SizeClassAllocator64 { }; COMPILER_CHECK(sizeof(RegionInfo) >= kCacheLineSize); - u32 Rand(u32 *state) { // ANSI C linear congruential PRNG. - return (*state = *state * 1103515245 + 12345) >> 16; - } - - u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n) - - void RandomShuffle(u32 *a, u32 n, u32 *rand_state) { - if (n <= 1) return; - for (u32 i = n - 1; i > 0; i--) - Swap(a[i], a[RandN(rand_state, i + 1)]); - } - - RegionInfo *GetRegionInfo(uptr class_id) { + RegionInfo *GetRegionInfo(uptr class_id) const { CHECK_LT(class_id, kNumClasses); RegionInfo *regions = reinterpret_cast(SpaceBeg() + kSpaceSize); return ®ions[class_id]; } - uptr GetMetadataEnd(uptr region_beg) { + uptr GetMetadataEnd(uptr region_beg) const { return region_beg + kRegionSize - kFreeArraySize; } - uptr GetChunkIdx(uptr chunk, uptr size) { + uptr GetChunkIdx(uptr chunk, uptr size) const { if (!kUsingConstantSpaceBeg) chunk -= SpaceBeg(); @@ -382,13 +622,12 @@ class SizeClassAllocator64 { return (u32)offset / (u32)size; } - CompactPtrT *GetFreeArray(uptr region_beg) { - return reinterpret_cast(region_beg + kRegionSize - - kFreeArraySize); + CompactPtrT *GetFreeArray(uptr region_beg) const { + return reinterpret_cast(GetMetadataEnd(region_beg)); } bool MapWithCallback(uptr beg, uptr size) { - uptr mapped = reinterpret_cast(MmapFixedOrDieOnFatalError(beg, size)); + uptr mapped = address_range.Map(beg, size); if (UNLIKELY(!mapped)) return false; CHECK_EQ(beg, mapped); @@ -397,21 +636,21 @@ class SizeClassAllocator64 { } void MapWithCallbackOrDie(uptr beg, uptr size) { - CHECK_EQ(beg, reinterpret_cast(MmapFixedOrDie(beg, size))); + CHECK_EQ(beg, address_range.MapOrDie(beg, size)); MapUnmapCallback().OnMap(beg, size); } void UnmapWithCallbackOrDie(uptr beg, uptr size) { MapUnmapCallback().OnUnmap(beg, size); - UnmapOrDie(reinterpret_cast(beg), size); + address_range.Unmap(beg, size); } bool EnsureFreeArraySpace(RegionInfo *region, uptr region_beg, uptr num_freed_chunks) { uptr needed_space = num_freed_chunks * sizeof(CompactPtrT); if (region->mapped_free_array < needed_space) { - CHECK_LE(needed_space, kFreeArraySize); uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize); + CHECK_LE(new_mapped_free_array, kFreeArraySize); uptr current_map_end = reinterpret_cast(GetFreeArray(region_beg)) + region->mapped_free_array; uptr new_map_size = new_mapped_free_array - region->mapped_free_array; @@ -422,55 +661,74 @@ class SizeClassAllocator64 { return true; } + // Check whether this size class is exhausted. + bool IsRegionExhausted(RegionInfo *region, uptr class_id, + uptr additional_map_size) { + if (LIKELY(region->mapped_user + region->mapped_meta + + additional_map_size <= kRegionSize - kFreeArraySize)) + return false; + if (!region->exhausted) { + region->exhausted = true; + Printf("%s: Out of memory. ", SanitizerToolName); + Printf("The process has exhausted %zuMB for size class %zu.\n", + kRegionSize >> 20, ClassIdToSize(class_id)); + } + return true; + } + NOINLINE bool PopulateFreeArray(AllocatorStats *stat, uptr class_id, RegionInfo *region, uptr requested_count) { // region->mutex is held. - const uptr size = ClassIdToSize(class_id); - const uptr new_space_beg = region->allocated_user; - const uptr new_space_end = new_space_beg + requested_count * size; const uptr region_beg = GetRegionBeginBySizeClass(class_id); + const uptr size = ClassIdToSize(class_id); + const uptr total_user_bytes = + region->allocated_user + requested_count * size; // Map more space for chunks, if necessary. - if (new_space_end > region->mapped_user) { - if (!kUsingConstantSpaceBeg && region->mapped_user == 0) - region->rand_state = static_cast(region_beg >> 12); // From ASLR. - // Do the mmap for the user memory. - uptr map_size = kUserMapSize; - while (new_space_end > region->mapped_user + map_size) - map_size += kUserMapSize; - CHECK_GE(region->mapped_user + map_size, new_space_end); - if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user, - map_size))) - return false; - stat->Add(AllocatorStatMapped, map_size); - region->mapped_user += map_size; - } - const uptr new_chunks_count = (region->mapped_user - new_space_beg) / size; - - // Calculate the required space for metadata. - const uptr requested_allocated_meta = - region->allocated_meta + new_chunks_count * kMetadataSize; - uptr requested_mapped_meta = region->mapped_meta; - while (requested_allocated_meta > requested_mapped_meta) - requested_mapped_meta += kMetaMapSize; - // Check whether this size class is exhausted. - if (region->mapped_user + requested_mapped_meta > - kRegionSize - kFreeArraySize) { - if (!region->exhausted) { - region->exhausted = true; - Printf("%s: Out of memory. ", SanitizerToolName); - Printf("The process has exhausted %zuMB for size class %zu.\n", - kRegionSize >> 20, size); + if (LIKELY(total_user_bytes > region->mapped_user)) { + if (UNLIKELY(region->mapped_user == 0)) { + if (!kUsingConstantSpaceBeg && kRandomShuffleChunks) + // The random state is initialized from ASLR. + region->rand_state = static_cast(region_beg >> 12); + // Postpone the first release to OS attempt for ReleaseToOSIntervalMs, + // preventing just allocated memory from being released sooner than + // necessary and also preventing extraneous ReleaseMemoryPagesToOS calls + // for short lived processes. + // Do it only when the feature is turned on, to avoid a potentially + // extraneous syscall. + if (ReleaseToOSIntervalMs() >= 0) + region->rtoi.last_release_at_ns = MonotonicNanoTime(); } - return false; - } - // Map more space for metadata, if necessary. - if (requested_mapped_meta > region->mapped_meta) { - if (UNLIKELY(!MapWithCallback( - GetMetadataEnd(region_beg) - requested_mapped_meta, - requested_mapped_meta - region->mapped_meta))) + // Do the mmap for the user memory. + const uptr user_map_size = + RoundUpTo(total_user_bytes - region->mapped_user, kUserMapSize); + if (UNLIKELY(IsRegionExhausted(region, class_id, user_map_size))) return false; - region->mapped_meta = requested_mapped_meta; + if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user, + user_map_size))) + return false; + stat->Add(AllocatorStatMapped, user_map_size); + region->mapped_user += user_map_size; + } + const uptr new_chunks_count = + (region->mapped_user - region->allocated_user) / size; + + if (kMetadataSize) { + // Calculate the required space for metadata. + const uptr total_meta_bytes = + region->allocated_meta + new_chunks_count * kMetadataSize; + const uptr meta_map_size = (total_meta_bytes > region->mapped_meta) ? + RoundUpTo(total_meta_bytes - region->mapped_meta, kMetaMapSize) : 0; + // Map more space for metadata, if necessary. + if (meta_map_size) { + if (UNLIKELY(IsRegionExhausted(region, class_id, meta_map_size))) + return false; + if (UNLIKELY(!MapWithCallback( + GetMetadataEnd(region_beg) - region->mapped_meta - meta_map_size, + meta_map_size))) + return false; + region->mapped_meta += meta_map_size; + } } // If necessary, allocate more space for the free array and populate it with @@ -479,7 +737,7 @@ class SizeClassAllocator64 { if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg, total_freed_chunks))) return false; CompactPtrT *free_array = GetFreeArray(region_beg); - for (uptr i = 0, chunk = new_space_beg; i < new_chunks_count; + for (uptr i = 0, chunk = region->allocated_user; i < new_chunks_count; i++, chunk += size) free_array[total_freed_chunks - 1 - i] = PointerToCompactPtr(0, chunk); if (kRandomShuffleChunks) @@ -491,27 +749,66 @@ class SizeClassAllocator64 { region->num_freed_chunks += new_chunks_count; region->allocated_user += new_chunks_count * size; CHECK_LE(region->allocated_user, region->mapped_user); - region->allocated_meta = requested_allocated_meta; + region->allocated_meta += new_chunks_count * kMetadataSize; CHECK_LE(region->allocated_meta, region->mapped_meta); region->exhausted = false; + // TODO(alekseyshl): Consider bumping last_release_at_ns here to prevent + // MaybeReleaseToOS from releasing just allocated pages or protect these + // not yet used chunks some other way. + return true; } - void MaybeReleaseChunkRange(uptr region_beg, uptr chunk_size, - CompactPtrT first, CompactPtrT last) { - uptr beg_ptr = CompactPtrToPointer(region_beg, first); - uptr end_ptr = CompactPtrToPointer(region_beg, last) + chunk_size; - ReleaseMemoryPagesToOS(beg_ptr, end_ptr); - } + class MemoryMapper { + public: + MemoryMapper(const ThisT& base_allocator, uptr class_id) + : allocator(base_allocator), + region_base(base_allocator.GetRegionBeginBySizeClass(class_id)), + released_ranges_count(0), + released_bytes(0) { + } - // Attempts to release some RAM back to OS. The region is expected to be - // locked. - // Algorithm: - // * Sort the chunks. - // * Find ranges fully covered by free-d chunks - // * Release them to OS with madvise. - void MaybeReleaseToOS(uptr class_id) { + uptr GetReleasedRangesCount() const { + return released_ranges_count; + } + + uptr GetReleasedBytes() const { + return released_bytes; + } + + uptr MapPackedCounterArrayBuffer(uptr buffer_size) { + // TODO(alekseyshl): The idea to explore is to check if we have enough + // space between num_freed_chunks*sizeof(CompactPtrT) and + // mapped_free_array to fit buffer_size bytes and use that space instead + // of mapping a temporary one. + return reinterpret_cast( + MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters")); + } + + void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) { + UnmapOrDie(reinterpret_cast(buffer), buffer_size); + } + + // Releases [from, to) range of pages back to OS. + void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) { + const uptr from_page = allocator.CompactPtrToPointer(region_base, from); + const uptr to_page = allocator.CompactPtrToPointer(region_base, to); + ReleaseMemoryPagesToOS(from_page, to_page); + released_ranges_count++; + released_bytes += to_page - from_page; + } + + private: + const ThisT& allocator; + const uptr region_base; + uptr released_ranges_count; + uptr released_bytes; + }; + + // Attempts to release RAM occupied by freed chunks back to OS. The region is + // expected to be locked. + void MaybeReleaseToOS(uptr class_id, bool force) { RegionInfo *region = GetRegionInfo(class_id); const uptr chunk_size = ClassIdToSize(class_id); const uptr page_size = GetPageSizeCached(); @@ -524,37 +821,29 @@ class SizeClassAllocator64 { return; // Nothing new to release. } - s32 interval_ms = ReleaseToOSIntervalMs(); - if (interval_ms < 0) - return; + if (!force) { + s32 interval_ms = ReleaseToOSIntervalMs(); + if (interval_ms < 0) + return; - u64 now_ns = NanoTime(); - if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > now_ns) - return; // Memory was returned recently. - region->rtoi.last_release_at_ns = now_ns; - - uptr region_beg = GetRegionBeginBySizeClass(class_id); - CompactPtrT *free_array = GetFreeArray(region_beg); - SortArray(free_array, n); - - const uptr scaled_chunk_size = chunk_size >> kCompactPtrScale; - const uptr kScaledGranularity = page_size >> kCompactPtrScale; - - uptr range_beg = free_array[0]; - uptr prev = free_array[0]; - for (uptr i = 1; i < n; i++) { - uptr chunk = free_array[i]; - CHECK_GT(chunk, prev); - if (chunk - prev != scaled_chunk_size) { - CHECK_GT(chunk - prev, scaled_chunk_size); - if (prev + scaled_chunk_size - range_beg >= kScaledGranularity) { - MaybeReleaseChunkRange(region_beg, chunk_size, range_beg, prev); - region->rtoi.n_freed_at_last_release = region->stats.n_freed; - region->rtoi.num_releases++; - } - range_beg = chunk; + if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > + MonotonicNanoTime()) { + return; // Memory was returned recently. } - prev = chunk; } + + MemoryMapper memory_mapper(*this, class_id); + + ReleaseFreeMemoryToOS( + GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size, + RoundUpTo(region->allocated_user, page_size) / page_size, + &memory_mapper); + + if (memory_mapper.GetReleasedRangesCount() > 0) { + region->rtoi.n_freed_at_last_release = region->stats.n_freed; + region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount(); + region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes(); + } + region->rtoi.last_release_at_ns = MonotonicNanoTime(); } }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h index 7151a4636056..2bd83b2eb5a4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -134,8 +134,9 @@ class SizeClassMap { static const uptr kMaxSize = 1UL << kMaxSizeLog; static const uptr kNumClasses = - kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1; + kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1 + 1; static const uptr kLargestClassID = kNumClasses - 2; + static const uptr kBatchClassID = kNumClasses - 1; COMPILER_CHECK(kNumClasses >= 16 && kNumClasses <= 256); static const uptr kNumClassesRounded = kNumClasses <= 32 ? 32 : @@ -143,6 +144,11 @@ class SizeClassMap { kNumClasses <= 128 ? 128 : 256; static uptr Size(uptr class_id) { + // Estimate the result for kBatchClassID because this class does not know + // the exact size of TransferBatch. It's OK since we are using the actual + // sizeof(TransferBatch) where it matters. + if (UNLIKELY(class_id == kBatchClassID)) + return kMaxNumCachedHint * sizeof(uptr); if (class_id <= kMidClass) return kMinSize * class_id; class_id -= kMidClass; @@ -151,9 +157,10 @@ class SizeClassMap { } static uptr ClassID(uptr size) { + if (UNLIKELY(size > kMaxSize)) + return 0; if (size <= kMidSize) return (size + kMinSize - 1) >> kMinSizeLog; - if (size > kMaxSize) return 0; uptr l = MostSignificantSetBitIndex(size); uptr hbits = (size >> (l - S)) & M; uptr lbits = size & ((1 << (l - S)) - 1); @@ -162,7 +169,13 @@ class SizeClassMap { } static uptr MaxCachedHint(uptr class_id) { - if (class_id == 0) return 0; + // Estimate the result for kBatchClassID because this class does not know + // the exact size of TransferBatch. We need to cache fewer batches than user + // chunks, so this number can be small. + if (UNLIKELY(class_id == kBatchClassID)) + return 16; + if (UNLIKELY(class_id == 0)) + return 0; uptr n = (1UL << kMaxBytesCachedLog) / Size(class_id); return Max(1, Min(kMaxNumCachedHint, n)); } @@ -178,6 +191,8 @@ class SizeClassMap { uptr p = prev_s ? (d * 100 / prev_s) : 0; uptr l = s ? MostSignificantSetBitIndex(s) : 0; uptr cached = MaxCachedHint(i) * s; + if (i == kBatchClassID) + d = p = l = 0; Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd " "cached: %zd %zd; id %zd\n", i, Size(i), d, p, l, MaxCachedHint(i), cached, ClassID(s)); @@ -192,12 +207,13 @@ class SizeClassMap { // Printf("Validate: c%zd\n", c); uptr s = Size(c); CHECK_NE(s, 0U); + if (c == kBatchClassID) + continue; CHECK_EQ(ClassID(s), c); - if (c != kNumClasses - 1) + if (c < kLargestClassID) CHECK_EQ(ClassID(s + 1), c + 1); CHECK_EQ(ClassID(s - 1), c); - if (c) - CHECK_GT(Size(c), Size(c-1)); + CHECK_GT(Size(c), Size(c - 1)); } CHECK_EQ(ClassID(kMaxSize + 1), 0); @@ -207,7 +223,7 @@ class SizeClassMap { CHECK_LT(c, kNumClasses); CHECK_GE(Size(c), s); if (c > 0) - CHECK_LT(Size(c-1), s); + CHECK_LT(Size(c - 1), s); } } }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h index 47c2b12a2049..76a0bf71425d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h @@ -47,12 +47,12 @@ # define ASM_HIDDEN(symbol) .hidden symbol # define ASM_TYPE_FUNCTION(symbol) .type symbol, @function # define ASM_SIZE(symbol) .size symbol, .-symbol -# define ASM_TSAN_SYMBOL(symbol) symbol -# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) symbol +# define ASM_SYMBOL(symbol) symbol +# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol #else # define ASM_HIDDEN(symbol) # define ASM_TYPE_FUNCTION(symbol) # define ASM_SIZE(symbol) -# define ASM_TSAN_SYMBOL(symbol) _##symbol -# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol +# define ASM_SYMBOL(symbol) _##symbol +# define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h index 65b3a38f0d51..cd41c920f1b1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h @@ -78,17 +78,7 @@ INLINE bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp, typedef typename T::Type Type; Type cmpv = *cmp; Type prev; -#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32 - if (sizeof(*a) == 8) { - Type volatile *val_ptr = const_cast(&a->val_dont_use); - prev = __mips_sync_val_compare_and_swap( - reinterpret_cast(val_ptr), (u64)cmpv, (u64)xchg); - } else { - prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg); - } -#else prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg); -#endif if (prev == cmpv) return true; *cmp = prev; return false; @@ -104,6 +94,13 @@ INLINE bool atomic_compare_exchange_weak(volatile T *a, } // namespace __sanitizer +// This include provides explicit template instantiations for atomic_uint64_t +// on MIPS32, which does not directly support 8 byte atomics. It has to +// proceed the template definitions above. +#if defined(_MIPS_SIM) && defined(_ABIO32) + #include "sanitizer_atomic_clang_mips.h" +#endif + #undef ATOMIC_ORDER #endif // SANITIZER_ATOMIC_CLANG_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h new file mode 100644 index 000000000000..c64e6e542798 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h @@ -0,0 +1,118 @@ +//===-- sanitizer_atomic_clang_mips.h ---------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer/AddressSanitizer runtime. +// Not intended for direct inclusion. Include sanitizer_atomic.h. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ATOMIC_CLANG_MIPS_H +#define SANITIZER_ATOMIC_CLANG_MIPS_H + +namespace __sanitizer { + +// MIPS32 does not support atomics > 4 bytes. To address this lack of +// functionality, the sanitizer library provides helper methods which use an +// internal spin lock mechanism to emulate atomic oprations when the size is +// 8 bytes. +static void __spin_lock(volatile int *lock) { + while (__sync_lock_test_and_set(lock, 1)) + while (*lock) { + } +} + +static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); } + +// Make sure the lock is on its own cache line to prevent false sharing. +// Put it inside a struct that is aligned and padded to the typical MIPS +// cacheline which is 32 bytes. +static struct { + int lock; + char pad[32 - sizeof(int)]; +} __attribute__((aligned(32))) lock = {0, {0}}; + +template <> +INLINE atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr, + atomic_uint64_t::Type val, + memory_order mo) { + DCHECK(mo & + (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst)); + DCHECK(!((uptr)ptr % sizeof(*ptr))); + + atomic_uint64_t::Type ret; + + __spin_lock(&lock.lock); + ret = *(const_cast(&ptr->val_dont_use)); + ptr->val_dont_use = ret + val; + __spin_unlock(&lock.lock); + + return ret; +} + +template <> +INLINE atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr, + atomic_uint64_t::Type val, + memory_order mo) { + return atomic_fetch_add(ptr, -val, mo); +} + +template <> +INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr, + atomic_uint64_t::Type *cmp, + atomic_uint64_t::Type xchg, + memory_order mo) { + DCHECK(mo & + (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst)); + DCHECK(!((uptr)ptr % sizeof(*ptr))); + + typedef atomic_uint64_t::Type Type; + Type cmpv = *cmp; + Type prev; + bool ret = false; + + __spin_lock(&lock.lock); + prev = *(const_cast(&ptr->val_dont_use)); + if (prev == cmpv) { + ret = true; + ptr->val_dont_use = xchg; + } + __spin_unlock(&lock.lock); + + return ret; +} + +template <> +INLINE atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr, + memory_order mo) { + DCHECK(mo & + (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst)); + DCHECK(!((uptr)ptr % sizeof(*ptr))); + + atomic_uint64_t::Type zero = 0; + volatile atomic_uint64_t *Newptr = + const_cast(ptr); + return atomic_fetch_add(Newptr, zero, mo); +} + +template <> +INLINE void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v, + memory_order mo) { + DCHECK(mo & + (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst)); + DCHECK(!((uptr)ptr % sizeof(*ptr))); + + __spin_lock(&lock.lock); + ptr->val_dont_use = v; + __spin_unlock(&lock.lock); +} + +} // namespace __sanitizer + +#endif // SANITIZER_ATOMIC_CLANG_MIPS_H + diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h index d2acc311bf7d..35e2d007eda8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h @@ -17,55 +17,6 @@ namespace __sanitizer { -// MIPS32 does not support atomic > 4 bytes. To address this lack of -// functionality, the sanitizer library provides helper methods which use an -// internal spin lock mechanism to emulate atomic oprations when the size is -// 8 bytes. -#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32 -static void __spin_lock(volatile int *lock) { - while (__sync_lock_test_and_set(lock, 1)) - while (*lock) { - } -} - -static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); } - - -// Make sure the lock is on its own cache line to prevent false sharing. -// Put it inside a struct that is aligned and padded to the typical MIPS -// cacheline which is 32 bytes. -static struct { - int lock; - char pad[32 - sizeof(int)]; -} __attribute__((aligned(32))) lock = {0}; - -template -T __mips_sync_fetch_and_add(volatile T *ptr, T val) { - T ret; - - __spin_lock(&lock.lock); - - ret = *ptr; - *ptr = ret + val; - - __spin_unlock(&lock.lock); - - return ret; -} - -template -T __mips_sync_val_compare_and_swap(volatile T *ptr, T oldval, T newval) { - T ret; - __spin_lock(&lock.lock); - - ret = *ptr; - if (ret == oldval) *ptr = newval; - - __spin_unlock(&lock.lock); - - return ret; -} -#endif INLINE void proc_yield(int cnt) { __asm__ __volatile__("" ::: "memory"); @@ -103,15 +54,8 @@ INLINE typename T::Type atomic_load( // 64-bit load on 32-bit platform. // Gross, but simple and reliable. // Assume that it is not in read-only memory. -#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32 - typename T::Type volatile *val_ptr = - const_cast(&a->val_dont_use); - v = __mips_sync_fetch_and_add( - reinterpret_cast(val_ptr), 0); -#else v = __sync_fetch_and_add( const_cast(&a->val_dont_use), 0); -#endif } return v; } @@ -141,14 +85,7 @@ INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { typename T::Type cmp = a->val_dont_use; typename T::Type cur; for (;;) { -#if defined(_MIPS_SIM) && _MIPS_SIM == _ABIO32 - typename T::Type volatile *val_ptr = - const_cast(&a->val_dont_use); - cur = __mips_sync_val_compare_and_swap( - reinterpret_cast(val_ptr), (u64)cmp, (u64)v); -#else cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v); -#endif if (cmp == v) break; cmp = cur; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h index d8472732ff21..0f65814c398f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h @@ -22,7 +22,7 @@ namespace __sanitizer { template class BasicBitVector { public: - enum SizeEnum { kSize = sizeof(basic_int_t) * 8 }; + enum SizeEnum : uptr { kSize = sizeof(basic_int_t) * 8 }; uptr size() const { return kSize; } // No CTOR. @@ -115,7 +115,7 @@ class TwoLevelBitVector { // This structure allows O(kLevel1Size) time for clear() and empty(), // as well fast handling of sparse BVs. public: - enum SizeEnum { kSize = BV::kSize * BV::kSize * kLevel1Size }; + enum SizeEnum : uptr { kSize = BV::kSize * BV::kSize * kLevel1Size }; // No CTOR. uptr size() const { return kSize; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h index df72f1c2d471..9c2db9ae4954 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h @@ -25,7 +25,7 @@ namespace __sanitizer { template class BVGraph { public: - enum SizeEnum { kSize = BV::kSize }; + enum SizeEnum : uptr { kSize = BV::kSize }; uptr size() const { return kSize; } // No CTOR. void clear() { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc index 7e6f8fce76d2..ae26d5efc24f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc @@ -26,72 +26,7 @@ const char *SanitizerToolName = "SanitizerTool"; atomic_uint32_t current_verbosity; uptr PageSizeCached; - -StaticSpinMutex report_file_mu; -ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0}; - -void RawWrite(const char *buffer) { - report_file.Write(buffer, internal_strlen(buffer)); -} - -void ReportFile::ReopenIfNecessary() { - mu->CheckLocked(); - if (fd == kStdoutFd || fd == kStderrFd) return; - - uptr pid = internal_getpid(); - // If in tracer, use the parent's file. - if (pid == stoptheworld_tracer_pid) - pid = stoptheworld_tracer_ppid; - if (fd != kInvalidFd) { - // If the report file is already opened by the current process, - // do nothing. Otherwise the report file was opened by the parent - // process, close it now. - if (fd_pid == pid) - return; - else - CloseFile(fd); - } - - const char *exe_name = GetProcessName(); - if (common_flags()->log_exe_name && exe_name) { - internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix, - exe_name, pid); - } else { - internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid); - } - fd = OpenFile(full_path, WrOnly); - if (fd == kInvalidFd) { - const char *ErrorMsgPrefix = "ERROR: Can't open file: "; - WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix)); - WriteToFile(kStderrFd, full_path, internal_strlen(full_path)); - Die(); - } - fd_pid = pid; -} - -void ReportFile::SetReportPath(const char *path) { - if (!path) - return; - uptr len = internal_strlen(path); - if (len > sizeof(path_prefix) - 100) { - Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n", - path[0], path[1], path[2], path[3], - path[4], path[5], path[6], path[7]); - Die(); - } - - SpinMutexLock l(mu); - if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd) - CloseFile(fd); - fd = kInvalidFd; - if (internal_strcmp(path, "stdout") == 0) { - fd = kStdoutFd; - } else if (internal_strcmp(path, "stderr") == 0) { - fd = kStderrFd; - } else { - internal_snprintf(path_prefix, kMaxPathLength, "%s", path); - } -} +u32 NumberOfCPUsCached; // PID of the tracer task in StopTheWorld. It shares the address space with the // main process, but has a different PID and thus requires special handling. @@ -120,42 +55,6 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, UNREACHABLE("unable to mmap"); } -bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, - uptr *read_len, uptr max_len, error_t *errno_p) { - uptr PageSize = GetPageSizeCached(); - uptr kMinFileLen = PageSize; - *buff = nullptr; - *buff_size = 0; - *read_len = 0; - // The files we usually open are not seekable, so try different buffer sizes. - for (uptr size = kMinFileLen; size <= max_len; size *= 2) { - fd_t fd = OpenFile(file_name, RdOnly, errno_p); - if (fd == kInvalidFd) return false; - UnmapOrDie(*buff, *buff_size); - *buff = (char*)MmapOrDie(size, __func__); - *buff_size = size; - *read_len = 0; - // Read up to one page at a time. - bool reached_eof = false; - while (*read_len + PageSize <= size) { - uptr just_read; - if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) { - UnmapOrDie(*buff, *buff_size); - return false; - } - if (just_read == 0) { - reached_eof = true; - break; - } - *read_len += just_read; - } - CloseFile(fd); - if (reached_eof) // We've read the whole file. - break; - } - return true; -} - typedef bool UptrComparisonFunction(const uptr &a, const uptr &b); typedef bool U32ComparisonFunction(const u32 &a, const u32 &b); @@ -285,9 +184,10 @@ void LoadedModule::clear() { } void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable, - bool writable) { + bool writable, const char *name) { void *mem = InternalAlloc(sizeof(AddressRange)); - AddressRange *r = new(mem) AddressRange(beg, end, executable, writable); + AddressRange *r = + new(mem) AddressRange(beg, end, executable, writable, name); ranges_.push_back(r); if (executable && end > max_executable_address_) max_executable_address_ = end; @@ -359,36 +259,6 @@ bool TemplateMatch(const char *templ, const char *str) { return true; } -static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':'; - -char *FindPathToBinary(const char *name) { - if (FileExists(name)) { - return internal_strdup(name); - } - - const char *path = GetEnv("PATH"); - if (!path) - return nullptr; - uptr name_len = internal_strlen(name); - InternalScopedBuffer buffer(kMaxPathLength); - const char *beg = path; - while (true) { - const char *end = internal_strchrnul(beg, kPathSeparator); - uptr prefix_len = end - beg; - if (prefix_len + name_len + 2 <= kMaxPathLength) { - internal_memcpy(buffer.data(), beg, prefix_len); - buffer[prefix_len] = '/'; - internal_memcpy(&buffer[prefix_len + 1], name, name_len); - buffer[prefix_len + 1 + name_len] = '\0'; - if (FileExists(buffer.data())) - return internal_strdup(buffer.data()); - } - if (*end == '\0') break; - beg = end + 1; - } - return nullptr; -} - static char binary_name_cache_str[kMaxPathLength]; static char process_name_cache_str[kMaxPathLength]; @@ -482,15 +352,6 @@ static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr), using namespace __sanitizer; // NOLINT extern "C" { -void __sanitizer_set_report_path(const char *path) { - report_file.SetReportPath(path); -} - -void __sanitizer_set_report_fd(void *fd) { - report_file.fd = (fd_t)reinterpret_cast(fd); - report_file.fd_pid = internal_getpid(); -} - SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_report_error_summary, const char *error_summary) { Printf("%s\n", error_summary); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 89aae579856a..1fbaee7e39a1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -29,8 +29,11 @@ extern "C" void _ReadWriteBarrier(); #endif namespace __sanitizer { -struct StackTrace; + struct AddressInfo; +struct BufferedStackTrace; +struct SignalContext; +struct StackTrace; // Constants. const uptr kWordSize = SANITIZER_WORDSIZE / 8; @@ -71,6 +74,7 @@ INLINE uptr GetPageSizeCached() { } uptr GetMmapGranularity(); uptr GetMaxVirtualAddress(); +uptr GetMaxUserVirtualAddress(); // Threads tid_t GetTid(); uptr GetThreadSelf(); @@ -125,6 +129,30 @@ void CheckVMASize(); void RunMallocHooks(const void *ptr, uptr size); void RunFreeHooks(const void *ptr); +class ReservedAddressRange { + public: + uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0); + uptr Map(uptr fixed_addr, uptr size); + uptr MapOrDie(uptr fixed_addr, uptr size); + void Unmap(uptr addr, uptr size); + void *base() const { return base_; } + uptr size() const { return size_; } + + private: + void* base_; + uptr size_; + const char* name_; + uptr os_handle_; +}; + +typedef void (*fill_profile_f)(uptr start, uptr rss, bool file, + /*out*/uptr *stats, uptr stats_size); + +// Parse the contents of /proc/self/smaps and generate a memory profile. +// |cb| is a tool-specific callback that fills the |stats| array containing +// |stats_size| elements. +void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size); + // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. // FIXME: use InternalAlloc instead of MmapOrDie once @@ -179,12 +207,15 @@ class LowLevelAllocator { char *allocated_end_; char *allocated_current_; }; +// Set the min alignment of LowLevelAllocator to at least alignment. +void SetLowLevelAllocateMinAlignment(uptr alignment); typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size); // Allows to register tool-specific callbacks for LowLevelAllocator. // Passing NULL removes the callback. void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback); // IO +void CatastrophicErrorWrite(const char *buffer, uptr length); void RawWrite(const char *buffer); bool ColorizeReports(); void RemoveANSIEscapeSequencesFromString(char *buffer); @@ -200,67 +231,18 @@ void SetPrintfAndReportCallback(void (*callback)(const char *)); if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \ } while (0) -// Can be used to prevent mixing error reports from different sanitizers. -extern StaticSpinMutex CommonSanitizerReportMutex; +// Lock sanitizer error reporting and protects against nested errors. +class ScopedErrorReportLock { + public: + ScopedErrorReportLock(); + ~ScopedErrorReportLock(); -struct ReportFile { - void Write(const char *buffer, uptr length); - bool SupportsColors(); - void SetReportPath(const char *path); - - // Don't use fields directly. They are only declared public to allow - // aggregate initialization. - - // Protects fields below. - StaticSpinMutex *mu; - // Opened file descriptor. Defaults to stderr. It may be equal to - // kInvalidFd, in which case new file will be opened when necessary. - fd_t fd; - // Path prefix of report file, set via __sanitizer_set_report_path. - char path_prefix[kMaxPathLength]; - // Full path to report, obtained as .PID - char full_path[kMaxPathLength]; - // PID of the process that opened fd. If a fork() occurs, - // the PID of child will be different from fd_pid. - uptr fd_pid; - - private: - void ReopenIfNecessary(); + static void CheckLocked(); }; -extern ReportFile report_file; extern uptr stoptheworld_tracer_pid; extern uptr stoptheworld_tracer_ppid; -enum FileAccessMode { - RdOnly, - WrOnly, - RdWr -}; - -// Returns kInvalidFd on error. -fd_t OpenFile(const char *filename, FileAccessMode mode, - error_t *errno_p = nullptr); -void CloseFile(fd_t); - -// Return true on success, false on error. -bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, - uptr *bytes_read = nullptr, error_t *error_p = nullptr); -bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, - uptr *bytes_written = nullptr, error_t *error_p = nullptr); - -bool RenameFile(const char *oldpath, const char *newpath, - error_t *error_p = nullptr); - -// Scoped file handle closer. -struct FileCloser { - explicit FileCloser(fd_t fd) : fd(fd) {} - ~FileCloser() { CloseFile(fd); } - fd_t fd; -}; - -bool SupportsColoredOutput(fd_t fd); - // Opens the file 'file_name" and reads up to 'max_len' bytes. // The resulting buffer is mmaped and stored in '*buff'. // The size of the mmaped region is stored in '*buff_size'. @@ -269,11 +251,6 @@ bool SupportsColoredOutput(fd_t fd); bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, uptr *read_len, uptr max_len = 1 << 26, error_t *errno_p = nullptr); -// Maps given file to virtual memory, and returns pointer to it -// (or NULL if mapping fails). Stores the size of mmaped region -// in '*buff_size'. -void *MapFileToMemory(const char *file_name, uptr *buff_size); -void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset); bool IsAccessibleMemoryRange(uptr beg, uptr size); @@ -293,27 +270,8 @@ void CacheBinaryName(); void DisableCoreDumperIfNecessary(); void DumpProcessMap(); void PrintModuleMap(); -bool FileExists(const char *filename); const char *GetEnv(const char *name); bool SetEnv(const char *name, const char *value); -const char *GetPwd(); -char *FindPathToBinary(const char *name); -bool IsPathSeparator(const char c); -bool IsAbsolutePath(const char *path); -// Starts a subprocess and returs its pid. -// If *_fd parameters are not kInvalidFd their corresponding input/output -// streams will be redirect to the file. The files will always be closed -// in parent process even in case of an error. -// The child process will close all fds after STDERR_FILENO -// before passing control to a program. -pid_t StartSubprocess(const char *filename, const char *const argv[], - fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd, - fd_t stderr_fd = kInvalidFd); -// Checks if specified process is still running -bool IsProcessRunning(pid_t pid); -// Waits for the process to finish and returns its exit code. -// Returns -1 in case of an error. -int WaitForProcess(pid_t pid); u32 GetUid(); void ReExec(); @@ -337,6 +295,7 @@ uptr GetTlsSize(); void SleepForSeconds(int seconds); void SleepForMillis(int millis); u64 NanoTime(); +u64 MonotonicNanoTime(); int Atexit(void (*function)(void)); void SortArray(uptr *array, uptr size); void SortArray(u32 *array, uptr size); @@ -385,7 +344,24 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)); typedef void (*SignalHandlerType)(int, void *, void *); HandleSignalMode GetHandleSignalMode(int signum); void InstallDeadlySignalHandlers(SignalHandlerType handler); -const char *DescribeSignalOrException(int signo); + +// Signal reporting. +// Each sanitizer uses slightly different implementation of stack unwinding. +typedef void (*UnwindSignalStackCallbackType)(const SignalContext &sig, + const void *callback_context, + BufferedStackTrace *stack); +// Print deadly signal report and die. +void HandleDeadlySignal(void *siginfo, void *context, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context); + +// Part of HandleDeadlySignal, exposed for asan. +void StartReportDeadlySignal(); +// Part of HandleDeadlySignal, exposed for asan. +void ReportDeadlySignal(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context); + // Alternative signal stack (POSIX-only). void SetAlternateSignalStack(); void UnsetAlternateSignalStack(); @@ -702,6 +678,7 @@ inline const char *ModuleArchToString(ModuleArch arch) { } const uptr kModuleUUIDSize = 16; +const uptr kMaxSegName = 16; // Represents a binary loaded into virtual memory (e.g. this can be an // executable or a shared object). @@ -720,7 +697,8 @@ class LoadedModule { void set(const char *module_name, uptr base_address, ModuleArch arch, u8 uuid[kModuleUUIDSize], bool instrumented); void clear(); - void addAddressRange(uptr beg, uptr end, bool executable, bool writable); + void addAddressRange(uptr beg, uptr end, bool executable, bool writable, + const char *name = nullptr); bool containsAddress(uptr address) const; const char *full_name() const { return full_name_; } @@ -736,13 +714,17 @@ class LoadedModule { uptr end; bool executable; bool writable; + char name[kMaxSegName]; - AddressRange(uptr beg, uptr end, bool executable, bool writable) + AddressRange(uptr beg, uptr end, bool executable, bool writable, + const char *name) : next(nullptr), beg(beg), end(end), executable(executable), - writable(writable) {} + writable(writable) { + internal_strncpy(this->name, (name ? name : ""), ARRAY_SIZE(this->name)); + } }; const IntrusiveList &ranges() const { return ranges_; } @@ -761,9 +743,10 @@ class LoadedModule { // filling this information. class ListOfModules { public: - ListOfModules() : modules_(kInitialCapacity) {} + ListOfModules() : initialized(false) {} ~ListOfModules() { clear(); } void init(); + void fallbackInit(); // Uses fallback init if available, otherwise clears const LoadedModule *begin() const { return modules_.begin(); } LoadedModule *begin() { return modules_.begin(); } const LoadedModule *end() const { return modules_.end(); } @@ -779,10 +762,15 @@ class ListOfModules { for (auto &module : modules_) module.clear(); modules_.clear(); } + void clearOrInit() { + initialized ? clear() : modules_.Initialize(kInitialCapacity); + initialized = true; + } - InternalMmapVector modules_; + InternalMmapVectorNoCtor modules_; // We rarely have more than 16K loaded modules. static const uptr kInitialCapacity = 1 << 14; + bool initialized; }; // Callback type for iterating over a set of memory ranges. @@ -858,36 +846,50 @@ static inline void SanitizerBreakOptimization(void *arg) { } struct SignalContext { + void *siginfo; void *context; uptr addr; uptr pc; uptr sp; uptr bp; bool is_memory_access; - enum WriteFlag { UNKNOWN, READ, WRITE } write_flag; - SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp, - bool is_memory_access, WriteFlag write_flag) - : context(context), - addr(addr), - pc(pc), - sp(sp), - bp(bp), - is_memory_access(is_memory_access), - write_flag(write_flag) {} + // VS2013 doesn't implement unrestricted unions, so we need a trivial default + // constructor + SignalContext() = default; + + // Creates signal context in a platform-specific manner. + // SignalContext is going to keep pointers to siginfo and context without + // owning them. + SignalContext(void *siginfo, void *context) + : siginfo(siginfo), + context(context), + addr(GetAddress()), + is_memory_access(IsMemoryAccess()), + write_flag(GetWriteFlag()) { + InitPcSpBp(); + } static void DumpAllRegisters(void *context); - // Creates signal context in a platform-specific manner. - static SignalContext Create(void *siginfo, void *context); + // Type of signal e.g. SIGSEGV or EXCEPTION_ACCESS_VIOLATION. + int GetType() const; - // Returns true if the "context" indicates a memory write. - static WriteFlag GetWriteFlag(void *context); + // String description of the signal. + const char *Describe() const; + + // Returns true if signal is stack overflow. + bool IsStackOverflow() const; + + private: + // Platform specific initialization. + void InitPcSpBp(); + uptr GetAddress() const; + WriteFlag GetWriteFlag() const; + bool IsMemoryAccess() const; }; -void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp); - void MaybeReexec(); template @@ -929,8 +931,17 @@ const s32 kReleaseToOSIntervalNever = -1; void CheckNoDeepBind(const char *filename, int flag); // Returns the requested amount of random data (up to 256 bytes) that can then -// be used to seed a PRNG. -bool GetRandom(void *buffer, uptr length); +// be used to seed a PRNG. Defaults to blocking like the underlying syscall. +bool GetRandom(void *buffer, uptr length, bool blocking = true); + +// Returns the number of logical processors on the system. +u32 GetNumberOfCPUs(); +extern u32 NumberOfCPUsCached; +INLINE u32 GetNumberOfCPUsCached() { + if (!NumberOfCPUsCached) + NumberOfCPUsCached = GetNumberOfCPUs(); + return NumberOfCPUsCached; +} } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 8607bf44902d..32342682a595 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -43,6 +43,7 @@ #include "sanitizer_errno.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_interceptors.h" +#include "sanitizer_symbolizer.h" #include "sanitizer_tls_get_addr.h" #include @@ -68,6 +69,46 @@ #define iconv __bsd_iconv #endif +#if SANITIZER_NETBSD +#define clock_getres __clock_getres50 +#define clock_gettime __clock_gettime50 +#define clock_settime __clock_settime50 +#define ctime __ctime50 +#define ctime_r __ctime_r50 +#define getitimer __getitimer50 +#define getpwent __getpwent50 +#define getpwnam __getpwnam50 +#define getpwnam_r __getpwnam_r50 +#define getpwuid __getpwuid50 +#define getpwuid_r __getpwuid_r50 +#define getutent __getutent50 +#define getutxent __getutxent50 +#define getutxid __getutxid50 +#define getutxline __getutxline50 +#define glob __glob30 +#define gmtime __gmtime50 +#define gmtime_r __gmtime_r50 +#define localtime_r __localtime_r50 +#define mktime __mktime50 +#define opendir __opendir30 +#define readdir __readdir30 +#define readdir_r __readdir_r30 +#define scandir __scandir30 +#define setitimer __setitimer50 +#define setlocale __setlocale50 +#define shmctl __shmctl50 +#define sigemptyset __sigemptyset14 +#define sigfillset __sigfillset14 +#define sigpending __sigpending14 +#define sigprocmask __sigprocmask14 +#define sigtimedwait __sigtimedwait50 +#define stat __stat50 +#define time __time50 +#define times __times13 +#define wait3 __wait350 +#define wait4 __wait450 +#endif + // Platform-specific options. #if SANITIZER_MAC namespace __sanitizer { @@ -258,7 +299,7 @@ typedef AddrHashMap MetadataHashMap; static MetadataHashMap *interceptor_metadata_map; -#if SI_NOT_WINDOWS +#if SI_POSIX UNUSED static void SetInterceptorMetadata(__sanitizer_FILE *addr, const FileMetadata &file) { MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr); @@ -272,7 +313,7 @@ UNUSED static const FileMetadata *GetInterceptorMetadata( MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, /* remove */ false, /* create */ false); - if (h.exists()) { + if (addr && h.exists()) { CHECK(!h.created()); CHECK(h->type == CommonInterceptorMetadata::CIMT_FILE); return &h->file; @@ -285,7 +326,7 @@ UNUSED static void DeleteInterceptorMetadata(void *addr) { MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, true); CHECK(h.exists()); } -#endif // SI_NOT_WINDOWS +#endif // SI_POSIX #if SANITIZER_INTERCEPT_STRLEN INTERCEPTOR(SIZE_T, strlen, const char *s) { @@ -886,7 +927,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) { #define INIT_FREXPF_FREXPL #endif // SANITIZER_INTERCEPT_FREXPF_FREXPL -#if SI_NOT_WINDOWS +#if SI_POSIX static void write_iovec(void *ctx, struct __sanitizer_iovec *iovec, SIZE_T iovlen, SIZE_T maxlen) { for (SIZE_T i = 0; i < iovlen && maxlen; ++i) { @@ -1176,12 +1217,14 @@ INTERCEPTOR(unsigned long, time, unsigned long *t) { #if SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS static void unpoison_tm(void *ctx, __sanitizer_tm *tm) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tm, sizeof(*tm)); +#if !SANITIZER_SOLARIS if (tm->tm_zone) { // Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone // can point to shared memory and tsan would report a data race. COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone, REAL(strlen(tm->tm_zone)) + 1); } +#endif } INTERCEPTOR(__sanitizer_tm *, localtime, unsigned long *timep) { void *ctx; @@ -1509,6 +1552,12 @@ INTERCEPTOR(int, vsnprintf, char *str, SIZE_T size, const char *format, va_list ap) VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __vsnprintf_chk, char *str, SIZE_T size, int flag, + SIZE_T size_to, const char *format, va_list ap) +VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap) +#endif + #if SANITIZER_INTERCEPT_PRINTF_L INTERCEPTOR(int, vsnprintf_l, char *str, SIZE_T size, void *loc, const char *format, va_list ap) @@ -1522,6 +1571,12 @@ FORMAT_INTERCEPTOR_IMPL(snprintf_l, vsnprintf_l, str, size, loc, format) INTERCEPTOR(int, vsprintf, char *str, const char *format, va_list ap) VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __vsprintf_chk, char *str, int flag, SIZE_T size_to, + const char *format, va_list ap) +VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap) +#endif + INTERCEPTOR(int, vasprintf, char **strp, const char *format, va_list ap) VASPRINTF_INTERCEPTOR_IMPL(vasprintf, strp, format, ap) @@ -1550,12 +1605,30 @@ FORMAT_INTERCEPTOR_IMPL(printf, vprintf, format) INTERCEPTOR(int, fprintf, __sanitizer_FILE *stream, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(fprintf, vfprintf, stream, format) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __fprintf_chk, __sanitizer_FILE *stream, SIZE_T size, + const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__fprintf_chk, vfprintf, stream, format) +#endif + INTERCEPTOR(int, sprintf, char *str, const char *format, ...) // NOLINT FORMAT_INTERCEPTOR_IMPL(sprintf, vsprintf, str, format) // NOLINT +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __sprintf_chk, char *str, int flag, SIZE_T size_to, + const char *format, ...) // NOLINT +FORMAT_INTERCEPTOR_IMPL(__sprintf_chk, vsprintf, str, format) // NOLINT +#endif + INTERCEPTOR(int, snprintf, char *str, SIZE_T size, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(snprintf, vsnprintf, str, size, format) +#if SANITIZER_INTERCEPT___PRINTF_CHK +INTERCEPTOR(int, __snprintf_chk, char *str, SIZE_T size, int flag, + SIZE_T size_to, const char *format, ...) // NOLINT +FORMAT_INTERCEPTOR_IMPL(__snprintf_chk, vsnprintf, str, size, format) // NOLINT +#endif + INTERCEPTOR(int, asprintf, char **strp, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(asprintf, vasprintf, strp, format) @@ -1595,6 +1668,17 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size, #define INIT_PRINTF #endif +#if SANITIZER_INTERCEPT___PRINTF_CHK +#define INIT___PRINTF_CHK \ + COMMON_INTERCEPT_FUNCTION(__sprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__snprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__vsprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__vsnprintf_chk); \ + COMMON_INTERCEPT_FUNCTION(__fprintf_chk); +#else +#define INIT___PRINTF_CHK +#endif + #if SANITIZER_INTERCEPT_PRINTF_L #define INIT_PRINTF_L \ COMMON_INTERCEPT_FUNCTION(snprintf_l); \ @@ -1723,7 +1807,8 @@ static void unpoison_group(void *ctx, __sanitizer_group *grp) { INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name); - COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); + if (name) + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_passwd *res = REAL(getpwnam)(name); if (res) unpoison_passwd(ctx, res); return res; @@ -2003,6 +2088,13 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) { } return res; } +namespace __sanitizer { +extern "C" { +int real_clock_gettime(u32 clk_id, void *tp) { + return REAL(clock_gettime)(clk_id, tp); +} +} // extern "C" +} // namespace __sanitizer INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp); @@ -2064,6 +2156,18 @@ static void unpoison_glob_t(void *ctx, __sanitizer_glob_t *pglob) { } } +#if SANITIZER_SOLARIS +INTERCEPTOR(int, glob, const char *pattern, int flags, + int (*errfunc)(const char *epath, int eerrno), + __sanitizer_glob_t *pglob) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob); + COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0); + int res = REAL(glob)(pattern, flags, errfunc, pglob); + if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob); + return res; +} +#else static THREADLOCAL __sanitizer_glob_t *pglob_copy; static void wrapped_gl_closedir(void *dir) { @@ -2127,7 +2231,14 @@ INTERCEPTOR(int, glob, const char *pattern, int flags, if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob); return res; } +#endif // SANITIZER_SOLARIS +#define INIT_GLOB \ + COMMON_INTERCEPT_FUNCTION(glob); +#else // SANITIZER_INTERCEPT_GLOB +#define INIT_GLOB +#endif // SANITIZER_INTERCEPT_GLOB +#if SANITIZER_INTERCEPT_GLOB64 INTERCEPTOR(int, glob64, const char *pattern, int flags, int (*errfunc)(const char *epath, int eerrno), __sanitizer_glob_t *pglob) { @@ -2156,12 +2267,11 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags, if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob); return res; } -#define INIT_GLOB \ - COMMON_INTERCEPT_FUNCTION(glob); \ +#define INIT_GLOB64 \ COMMON_INTERCEPT_FUNCTION(glob64); -#else // SANITIZER_INTERCEPT_GLOB -#define INIT_GLOB -#endif // SANITIZER_INTERCEPT_GLOB +#else // SANITIZER_INTERCEPT_GLOB64 +#define INIT_GLOB64 +#endif // SANITIZER_INTERCEPT_GLOB64 #if SANITIZER_INTERCEPT_WAIT // According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version @@ -2466,7 +2576,15 @@ INTERCEPTOR(struct __sanitizer_hostent *, gethostent, int fake) { if (res) write_hostent(ctx, res); return res; } +#define INIT_GETHOSTBYNAME \ + COMMON_INTERCEPT_FUNCTION(gethostent); \ + COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \ + COMMON_INTERCEPT_FUNCTION(gethostbyname); +#else +#define INIT_GETHOSTBYNAME +#endif // SANITIZER_INTERCEPT_GETHOSTBYNAME +#if SANITIZER_INTERCEPT_GETHOSTBYNAME2 INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname2, name, af); @@ -2474,14 +2592,10 @@ INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) { if (res) write_hostent(ctx, res); return res; } -#define INIT_GETHOSTBYNAME \ - COMMON_INTERCEPT_FUNCTION(gethostent); \ - COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \ - COMMON_INTERCEPT_FUNCTION(gethostbyname); \ - COMMON_INTERCEPT_FUNCTION(gethostbyname2); +#define INIT_GETHOSTBYNAME2 COMMON_INTERCEPT_FUNCTION(gethostbyname2); #else -#define INIT_GETHOSTBYNAME -#endif +#define INIT_GETHOSTBYNAME2 +#endif // SANITIZER_INTERCEPT_GETHOSTBYNAME2 #if SANITIZER_INTERCEPT_GETHOSTBYNAME_R INTERCEPTOR(int, gethostbyname_r, char *name, struct __sanitizer_hostent *ret, @@ -3371,7 +3485,7 @@ INTERCEPTOR(char *, strerror, int errnum) { // * GNU version returns message pointer, which points to either buf or some // static storage. #if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \ - SANITIZER_MAC || SANITIZER_ANDROID + SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD // POSIX version. Spec is not clear on whether buf is NULL-terminated. // At least on OSX, buf contents are valid even when the call fails. INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) { @@ -3589,7 +3703,7 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds, if (fds && nfds) read_pollfd(ctx, fds, nfds); if (timeout_ts) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout_ts, struct_timespec_sz); - // FIXME: read sigmask when all of sigemptyset, etc are intercepted. + if (sigmask) COMMON_INTERCEPTOR_READ_RANGE(ctx, sigmask, sizeof(*sigmask)); int res = COMMON_INTERCEPTOR_BLOCK_REAL(ppoll)(fds, nfds, timeout_ts, sigmask); if (fds && nfds) write_pollfd(ctx, fds, nfds); @@ -3630,7 +3744,7 @@ INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) { INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigwait, set, sig); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3647,7 +3761,7 @@ INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) { INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigwaitinfo, set, info); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3666,7 +3780,7 @@ INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info, void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigtimedwait, set, info, timeout); if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3729,7 +3843,7 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset); - // FIXME: read sigset_t when all of sigemptyset, etc are intercepted + if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. @@ -3827,6 +3941,15 @@ INTERCEPTOR(int, pthread_mutex_unlock, void *m) { #define INIT_PTHREAD_MUTEX_UNLOCK #endif +#if SANITIZER_NETBSD +INTERCEPTOR(int, __libc_mutex_lock, void *m) \ + ALIAS(WRAPPER_NAME(pthread_mutex_lock)); +INTERCEPTOR(int, __libc_mutex_unlock, void *m) \ + ALIAS(WRAPPER_NAME(pthread_mutex_unlock)); +INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate) \ + ALIAS(WRAPPER_NAME(pthread_setcancelstate)); +#endif + #if SANITIZER_INTERCEPT_GETMNTENT || SANITIZER_INTERCEPT_GETMNTENT_R static void write_mntent(void *ctx, __sanitizer_mntent *mnt) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt)); @@ -4404,7 +4527,7 @@ INTERCEPTOR(char *, tempnam, char *dir, char *pfx) { #define INIT_TEMPNAM #endif -#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP +#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && !SANITIZER_NETBSD INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name); @@ -4413,10 +4536,35 @@ INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) { return REAL(pthread_setname_np)(thread, name); } #define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np); +#elif SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && SANITIZER_NETBSD +INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name, void *arg) { + void *ctx; + char newname[32]; // PTHREAD_MAX_NAMELEN_NP=32 + COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name, arg); + COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0); + internal_snprintf(newname, sizeof(newname), name, arg); + COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, newname); + return REAL(pthread_setname_np)(thread, name, arg); +} +#define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np); #else #define INIT_PTHREAD_SETNAME_NP #endif +#if SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP +INTERCEPTOR(int, pthread_getname_np, uptr thread, char *name, SIZE_T len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pthread_getname_np, thread, name, len); + int res = REAL(pthread_getname_np)(thread, name, len); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strnlen(name, len) + 1); + return res; +} +#define INIT_PTHREAD_GETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_getname_np); +#else +#define INIT_PTHREAD_GETNAME_NP +#endif + #if SANITIZER_INTERCEPT_SINCOS INTERCEPTOR(void, sincos, double x, double *sin, double *cos) { void *ctx; @@ -5575,6 +5723,7 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) { if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0); COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag); void *res = REAL(dlopen)(filename, flag); + Symbolizer::GetOrInit()->InvalidateModuleList(); COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res); return res; } @@ -5583,6 +5732,7 @@ INTERCEPTOR(int, dlclose, void *handle) { void *ctx; COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlclose, handle); int res = REAL(dlclose)(handle); + Symbolizer::GetOrInit()->InvalidateModuleList(); COMMON_INTERCEPTOR_LIBRARY_UNLOADED(); return res; } @@ -5821,7 +5971,7 @@ INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcancelstate, state, oldstate); int res = REAL(pthread_setcancelstate)(state, oldstate); - if (res == 0) + if (res == 0 && oldstate != nullptr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldstate, sizeof(*oldstate)); return res; } @@ -5830,7 +5980,7 @@ INTERCEPTOR(int, pthread_setcanceltype, int type, int *oldtype) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcanceltype, type, oldtype); int res = REAL(pthread_setcanceltype)(type, oldtype); - if (res == 0) + if (res == 0 && oldtype != nullptr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldtype, sizeof(*oldtype)); return res; } @@ -6313,6 +6463,7 @@ static void InitializeCommonInterceptors() { INIT_GETITIMER; INIT_TIME; INIT_GLOB; + INIT_GLOB64; INIT_WAIT; INIT_WAIT4; INIT_INET; @@ -6321,6 +6472,7 @@ static void InitializeCommonInterceptors() { INIT_GETNAMEINFO; INIT_GETSOCKNAME; INIT_GETHOSTBYNAME; + INIT_GETHOSTBYNAME2; INIT_GETHOSTBYNAME_R; INIT_GETHOSTBYNAME2_R; INIT_GETHOSTBYADDR_R; @@ -6403,6 +6555,7 @@ static void InitializeCommonInterceptors() { INIT_TTYNAME_R; INIT_TEMPNAM; INIT_PTHREAD_SETNAME_NP; + INIT_PTHREAD_GETNAME_NP; INIT_SINCOS; INIT_REMQUO; INIT_LGAMMA; @@ -6457,4 +6610,12 @@ static void InitializeCommonInterceptors() { INIT_GETLOADAVG; INIT_WCSLEN; INIT_WCSCAT; + +#if SANITIZER_NETBSD + COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock); + COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock); + COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate); +#endif + + INIT___PRINTF_CHK; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index 4ed9afedf84a..24e7548a597e 100755 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -57,7 +57,9 @@ static void ioctl_table_fill() { _(SIOCGIFCONF, CUSTOM, 0); _(SIOCGPGRP, WRITE, sizeof(int)); _(SIOCSPGRP, READ, sizeof(int)); +#if !SANITIZER_SOLARIS _(TIOCCONS, NONE, 0); +#endif _(TIOCEXCL, NONE, 0); _(TIOCGETD, WRITE, sizeof(int)); _(TIOCGPGRP, WRITE, pid_t_sz); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc index 550427c906a6..2568df3a5bff 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc @@ -34,6 +34,7 @@ INTERFACE_FUNCTION(__sanitizer_get_heap_size) INTERFACE_FUNCTION(__sanitizer_get_ownership) INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes) INTERFACE_FUNCTION(__sanitizer_install_malloc_and_free_hooks) +INTERFACE_FUNCTION(__sanitizer_purge_allocator) INTERFACE_FUNCTION(__sanitizer_print_memory_profile) INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook) INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc index cf200512de84..5cdfbbb275f7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -14,7 +14,10 @@ #include "sanitizer_common.h" #include "sanitizer_allocator_interface.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" +#include "sanitizer_procmaps.h" +#include "sanitizer_report_decorator.h" #include "sanitizer_stackdepot.h" #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" @@ -25,12 +28,25 @@ namespace __sanitizer { +#if !SANITIZER_FUCHSIA + bool ReportFile::SupportsColors() { SpinMutexLock l(mu); ReopenIfNecessary(); return SupportsColoredOutput(fd); } +static INLINE bool ReportSupportsColors() { + return report_file.SupportsColors(); +} + +#else // SANITIZER_FUCHSIA + +// Fuchsia's logs always go through post-processing that handles colorization. +static INLINE bool ReportSupportsColors() { return true; } + +#endif // !SANITIZER_FUCHSIA + bool ColorizeReports() { // FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color // printing on Windows. @@ -39,7 +55,7 @@ bool ColorizeReports() { const char *flag = common_flags()->color; return internal_strcmp(flag, "always") == 0 || - (internal_strcmp(flag, "auto") == 0 && report_file.SupportsColors()); + (internal_strcmp(flag, "auto") == 0 && ReportSupportsColors()); } static void (*sandboxing_callback)(); @@ -131,6 +147,127 @@ void BackgroundThread(void *arg) { } #endif +#if !SANITIZER_FUCHSIA && !SANITIZER_GO +void StartReportDeadlySignal() { + // Write the first message using fd=2, just in case. + // It may actually fail to write in case stderr is closed. + CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName)); + static const char kDeadlySignal[] = ":DEADLYSIGNAL\n"; + CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1); +} + +static void MaybeReportNonExecRegion(uptr pc) { +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD + MemoryMappingLayout proc_maps(/*cache_enabled*/ true); + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (pc >= segment.start && pc < segment.end && !segment.IsExecutable()) + Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n"); + } +#endif +} + +static void PrintMemoryByte(InternalScopedString *str, const char *before, + u8 byte) { + SanitizerCommonDecorator d; + str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15, + d.Default()); +} + +static void MaybeDumpInstructionBytes(uptr pc) { + if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) + return; + InternalScopedString str(1024); + str.append("First 16 instruction bytes at pc: "); + if (IsAccessibleMemoryRange(pc, 16)) { + for (int i = 0; i < 16; ++i) { + PrintMemoryByte(&str, "", ((u8 *)pc)[i]); + } + str.append("\n"); + } else { + str.append("unaccessible\n"); + } + Report("%s", str.data()); +} + +static void MaybeDumpRegisters(void *context) { + if (!common_flags()->dump_registers) return; + SignalContext::DumpAllRegisters(context); +} + +static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + SanitizerCommonDecorator d; + Printf("%s", d.Warning()); + static const char kDescription[] = "stack-overflow"; + Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n", + SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc, + (void *)sig.bp, (void *)sig.sp, tid); + Printf("%s", d.Default()); + InternalScopedBuffer stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + unwind(sig, unwind_context, stack); + stack->Print(); + ReportErrorSummary(kDescription, stack); +} + +static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + SanitizerCommonDecorator d; + Printf("%s", d.Warning()); + const char *description = sig.Describe(); + Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n", + SanitizerToolName, description, (void *)sig.addr, (void *)sig.pc, + (void *)sig.bp, (void *)sig.sp, tid); + Printf("%s", d.Default()); + if (sig.pc < GetPageSizeCached()) + Report("Hint: pc points to the zero page.\n"); + if (sig.is_memory_access) { + const char *access_type = + sig.write_flag == SignalContext::WRITE + ? "WRITE" + : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN"); + Report("The signal is caused by a %s memory access.\n", access_type); + if (sig.addr < GetPageSizeCached()) + Report("Hint: address points to the zero page.\n"); + } + MaybeReportNonExecRegion(sig.pc); + InternalScopedBuffer stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + unwind(sig, unwind_context, stack); + stack->Print(); + MaybeDumpInstructionBytes(sig.pc); + MaybeDumpRegisters(sig.context); + Printf("%s can not provide additional info.\n", SanitizerToolName); + ReportErrorSummary(description, stack); +} + +void ReportDeadlySignal(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + if (sig.IsStackOverflow()) + ReportStackOverflowImpl(sig, tid, unwind, unwind_context); + else + ReportDeadlySignalImpl(sig, tid, unwind, unwind_context); +} + +void HandleDeadlySignal(void *siginfo, void *context, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) { + StartReportDeadlySignal(); + ScopedErrorReportLock rl; + SignalContext sig(siginfo, context); + ReportDeadlySignal(sig, tid, unwind, unwind_context); + Report("ABORTING\n"); + Die(); +} + +#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO + void WriteToSyslog(const char *msg) { InternalScopedString msg_copy(kErrorMessageBufferSize); msg_copy.append("%s", msg); @@ -161,6 +298,47 @@ void MaybeStartBackgroudThread() { #endif } +static atomic_uintptr_t reporting_thread = {0}; +static StaticSpinMutex CommonSanitizerReportMutex; + +ScopedErrorReportLock::ScopedErrorReportLock() { + uptr current = GetThreadSelf(); + for (;;) { + uptr expected = 0; + if (atomic_compare_exchange_strong(&reporting_thread, &expected, current, + memory_order_relaxed)) { + // We've claimed reporting_thread so proceed. + CommonSanitizerReportMutex.Lock(); + return; + } + + if (expected == current) { + // This is either asynch signal or nested error during error reporting. + // Fail simple to avoid deadlocks in Report(). + + // Can't use Report() here because of potential deadlocks in nested + // signal handlers. + CatastrophicErrorWrite(SanitizerToolName, + internal_strlen(SanitizerToolName)); + static const char msg[] = ": nested bug in the same thread, aborting.\n"; + CatastrophicErrorWrite(msg, sizeof(msg) - 1); + + internal__exit(common_flags()->exitcode); + } + + internal_sched_yield(); + } +} + +ScopedErrorReportLock::~ScopedErrorReportLock() { + CommonSanitizerReportMutex.Unlock(); + atomic_store_relaxed(&reporting_thread, 0); +} + +void ScopedErrorReportLock::CheckLocked() { + CommonSanitizerReportMutex.CheckLocked(); +} + } // namespace __sanitizer SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify, diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc new file mode 100644 index 000000000000..c5be48bceace --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc @@ -0,0 +1,240 @@ +//===-- sanitizer_coverage_fuchsia.cc ------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// Sanitizer Coverage Controller for Trace PC Guard, Fuchsia-specific version. +// +// This Fuchsia-specific implementation uses the same basic scheme and the +// same simple '.sancov' file format as the generic implementation. The +// difference is that we just produce a single blob of output for the whole +// program, not a separate one per DSO. We do not sort the PC table and do +// not prune the zeros, so the resulting file is always as large as it +// would be to report 100% coverage. Implicit tracing information about +// the address ranges of DSOs allows offline tools to split the one big +// blob into separate files that the 'sancov' tool can understand. +// +// Unlike the traditional implementation that uses an atexit hook to write +// out data files at the end, the results on Fuchsia do not go into a file +// per se. The 'coverage_dir' option is ignored. Instead, they are stored +// directly into a shared memory object (a Zircon VMO). At exit, that VMO +// is handed over to a system service that's responsible for getting the +// data out to somewhere that it can be fed into the sancov tool (where and +// how is not our problem). + +#include "sanitizer_platform.h" +#if SANITIZER_FUCHSIA +#include "sanitizer_atomic.h" +#include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" + +#include +#include +#include + +using namespace __sanitizer; // NOLINT + +namespace __sancov { +namespace { + +// TODO(mcgrathr): Move the constant into a header shared with other impls. +constexpr u64 Magic64 = 0xC0BFFFFFFFFFFF64ULL; +static_assert(SANITIZER_WORDSIZE == 64, "Fuchsia is always LP64"); + +constexpr const char kSancovSinkName[] = "sancov"; + +// Collects trace-pc guard coverage. +// This class relies on zero-initialization. +class TracePcGuardController { + public: + // For each PC location being tracked, there is a u32 reserved in global + // data called the "guard". At startup, we assign each guard slot a + // unique index into the big results array. Later during runtime, the + // first call to TracePcGuard (below) will store the corresponding PC at + // that index in the array. (Each later call with the same guard slot is + // presumed to be from the same PC.) Then it clears the guard slot back + // to zero, which tells the compiler not to bother calling in again. At + // the end of the run, we have a big array where each element is either + // zero or is a tracked PC location that was hit in the trace. + + // This is called from global constructors. Each translation unit has a + // contiguous array of guard slots, and a constructor that calls here + // with the bounds of its array. Those constructors are allowed to call + // here more than once for the same array. Usually all of these + // constructors run in the initial thread, but it's possible that a + // dlopen call on a secondary thread will run constructors that get here. + void InitTracePcGuard(u32 *start, u32 *end) { + if (end > start && *start == 0 && common_flags()->coverage) { + // Complete the setup before filling in any guards with indices. + // This avoids the possibility of code called from Setup reentering + // TracePcGuard. + u32 idx = Setup(end - start); + for (u32 *p = start; p < end; ++p) { + *p = idx++; + } + } + } + + void TracePcGuard(u32 *guard, uptr pc) { + atomic_uint32_t *guard_ptr = reinterpret_cast(guard); + u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed); + if (idx > 0) array_[idx] = pc; + } + + void Dump() { + BlockingMutexLock locked(&setup_lock_); + if (array_) { + CHECK_NE(vmo_, ZX_HANDLE_INVALID); + + // Publish the VMO to the system, where it can be collected and + // analyzed after this process exits. This always consumes the VMO + // handle. Any failure is just logged and not indicated to us. + __sanitizer_publish_data(kSancovSinkName, vmo_); + vmo_ = ZX_HANDLE_INVALID; + + // This will route to __sanitizer_log_write, which will ensure that + // information about shared libraries is written out. This message + // uses the `dumpfile` symbolizer markup element to highlight the + // dump. See the explanation for this in: + // https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md + Printf("SanitizerCoverage: {{{dumpfile:%s:%s}}} with up to %u PCs\n", + kSancovSinkName, vmo_name_, next_index_ - 1); + } + } + + private: + // We map in the largest possible view into the VMO: one word + // for every possible 32-bit index value. This avoids the need + // to change the mapping when increasing the size of the VMO. + // We can always spare the 32G of address space. + static constexpr size_t MappingSize = sizeof(uptr) << 32; + + BlockingMutex setup_lock_; + uptr *array_; + u32 next_index_; + zx_handle_t vmo_; + char vmo_name_[ZX_MAX_NAME_LEN]; + + size_t DataSize() const { return next_index_ * sizeof(uintptr_t); } + + u32 Setup(u32 num_guards) { + BlockingMutexLock locked(&setup_lock_); + DCHECK(common_flags()->coverage); + + if (next_index_ == 0) { + CHECK_EQ(vmo_, ZX_HANDLE_INVALID); + CHECK_EQ(array_, nullptr); + + // The first sample goes at [1] to reserve [0] for the magic number. + next_index_ = 1 + num_guards; + + zx_status_t status = _zx_vmo_create(DataSize(), 0, &vmo_); + CHECK_EQ(status, ZX_OK); + + // Give the VMO a name including our process KOID so it's easy to spot. + internal_snprintf(vmo_name_, sizeof(vmo_name_), "%s.%zu", kSancovSinkName, + internal_getpid()); + _zx_object_set_property(vmo_, ZX_PROP_NAME, vmo_name_, + internal_strlen(vmo_name_)); + + // Map the largest possible view we might need into the VMO. Later + // we might need to increase the VMO's size before we can use larger + // indices, but we'll never move the mapping address so we don't have + // any multi-thread synchronization issues with that. + uintptr_t mapping; + status = + _zx_vmar_map(_zx_vmar_root_self(), 0, vmo_, 0, MappingSize, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &mapping); + CHECK_EQ(status, ZX_OK); + + // Hereafter other threads are free to start storing into + // elements [1, next_index_) of the big array. + array_ = reinterpret_cast(mapping); + + // Store the magic number. + // Hereafter, the VMO serves as the contents of the '.sancov' file. + array_[0] = Magic64; + + return 1; + } else { + // The VMO is already mapped in, but it's not big enough to use the + // new indices. So increase the size to cover the new maximum index. + + CHECK_NE(vmo_, ZX_HANDLE_INVALID); + CHECK_NE(array_, nullptr); + + uint32_t first_index = next_index_; + next_index_ += num_guards; + + zx_status_t status = _zx_vmo_set_size(vmo_, DataSize()); + CHECK_EQ(status, ZX_OK); + + return first_index; + } + } +}; + +static TracePcGuardController pc_guard_controller; + +} // namespace +} // namespace __sancov + +namespace __sanitizer { +void InitializeCoverage(bool enabled, const char *dir) { + CHECK_EQ(enabled, common_flags()->coverage); + CHECK_EQ(dir, common_flags()->coverage_dir); + + static bool coverage_enabled = false; + if (!coverage_enabled) { + coverage_enabled = enabled; + Atexit(__sanitizer_cov_dump); + AddDieCallback(__sanitizer_cov_dump); + } +} +} // namespace __sanitizer + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage( // NOLINT + const uptr *pcs, uptr len) { + UNIMPLEMENTED(); +} + +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *guard) { + if (!*guard) return; + __sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1); +} + +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init, + u32 *start, u32 *end) { + if (start == end || *start) return; + __sancov::pc_guard_controller.InitTracePcGuard(start, end); +} + +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() { + __sancov::pc_guard_controller.Dump(); +} +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { + __sanitizer_dump_trace_pc_guard_coverage(); +} +// Default empty implementations (weak). Users should redefine them. +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {} +} // extern "C" + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc index d4749000d80a..f909b74c9752 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc @@ -9,6 +9,7 @@ // Sanitizer Coverage interface list. //===----------------------------------------------------------------------===// INTERFACE_FUNCTION(__sanitizer_cov_dump) +INTERFACE_FUNCTION(__sanitizer_cov_reset) INTERFACE_FUNCTION(__sanitizer_dump_coverage) INTERFACE_FUNCTION(__sanitizer_dump_trace_pc_guard_coverage) INTERFACE_WEAK_FUNCTION(__sancov_default_options) @@ -17,6 +18,10 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp1) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp2) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp4) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp8) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp1) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp2) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp4) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp8) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div4) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div8) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep) @@ -24,3 +29,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc index 24433356c63d..3c5f29b2899b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc @@ -8,10 +8,14 @@ //===----------------------------------------------------------------------===// // Sanitizer Coverage Controller for Trace PC Guard. +#include "sanitizer_platform.h" + +#if !SANITIZER_FUCHSIA #include "sancov_flags.h" #include "sanitizer_allocator_internal.h" #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_symbolizer.h" using namespace __sanitizer; @@ -124,11 +128,17 @@ class TracePcGuardController { } void TracePcGuard(u32* guard, uptr pc) { - atomic_uint32_t* guard_ptr = reinterpret_cast(guard); - u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed); + u32 idx = *guard; if (!idx) return; // we start indices from 1. - pc_vector[idx - 1] = pc; + atomic_uintptr_t* pc_ptr = + reinterpret_cast(&pc_vector[idx - 1]); + if (atomic_load(pc_ptr, memory_order_relaxed) == 0) + atomic_store(pc_ptr, pc, memory_order_relaxed); + } + + void Reset() { + internal_memset(&pc_vector[0], 0, sizeof(pc_vector[0]) * pc_vector.size()); } void Dump() { @@ -180,15 +190,31 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() { SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { __sanitizer_dump_trace_pc_guard_coverage(); } +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_reset() { + __sancov::pc_guard_controller.Reset(); +} // Default empty implementations (weak). Users should redefine them. SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {} } // extern "C" +// Weak definition for code instrumented with -fsanitize-coverage=stack-depth +// and later linked with code containing a strong definition. +// E.g., -fsanitize=fuzzer-no-link +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE uptr __sancov_lowest_stack; + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h index 7872b89c227c..42cc290502bf 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h @@ -24,8 +24,10 @@ #if SANITIZER_FREEBSD || SANITIZER_MAC # define __errno_location __error -#elif SANITIZER_ANDROID +#elif SANITIZER_ANDROID || SANITIZER_NETBSD # define __errno_location __errno +#elif SANITIZER_SOLARIS +# define __errno_location ___errno #elif SANITIZER_WINDOWS # define __errno_location _errno #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc new file mode 100644 index 000000000000..cde54bfde804 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc @@ -0,0 +1,177 @@ +//===-- sanitizer_file.cc ------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer and ThreadSanitizer +// run-time libraries. It defines filesystem-related interfaces. This +// is separate from sanitizer_common.cc so that it's simpler to disable +// all the filesystem support code for a port that doesn't use it. +// +//===---------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if !SANITIZER_FUCHSIA + +#include "sanitizer_common.h" +#include "sanitizer_file.h" + +namespace __sanitizer { + +void CatastrophicErrorWrite(const char *buffer, uptr length) { + WriteToFile(kStderrFd, buffer, length); +} + +StaticSpinMutex report_file_mu; +ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0}; + +void RawWrite(const char *buffer) { + report_file.Write(buffer, internal_strlen(buffer)); +} + +void ReportFile::ReopenIfNecessary() { + mu->CheckLocked(); + if (fd == kStdoutFd || fd == kStderrFd) return; + + uptr pid = internal_getpid(); + // If in tracer, use the parent's file. + if (pid == stoptheworld_tracer_pid) + pid = stoptheworld_tracer_ppid; + if (fd != kInvalidFd) { + // If the report file is already opened by the current process, + // do nothing. Otherwise the report file was opened by the parent + // process, close it now. + if (fd_pid == pid) + return; + else + CloseFile(fd); + } + + const char *exe_name = GetProcessName(); + if (common_flags()->log_exe_name && exe_name) { + internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix, + exe_name, pid); + } else { + internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid); + } + fd = OpenFile(full_path, WrOnly); + if (fd == kInvalidFd) { + const char *ErrorMsgPrefix = "ERROR: Can't open file: "; + WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix)); + WriteToFile(kStderrFd, full_path, internal_strlen(full_path)); + Die(); + } + fd_pid = pid; +} + +void ReportFile::SetReportPath(const char *path) { + if (!path) + return; + uptr len = internal_strlen(path); + if (len > sizeof(path_prefix) - 100) { + Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n", + path[0], path[1], path[2], path[3], + path[4], path[5], path[6], path[7]); + Die(); + } + + SpinMutexLock l(mu); + if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd) + CloseFile(fd); + fd = kInvalidFd; + if (internal_strcmp(path, "stdout") == 0) { + fd = kStdoutFd; + } else if (internal_strcmp(path, "stderr") == 0) { + fd = kStderrFd; + } else { + internal_snprintf(path_prefix, kMaxPathLength, "%s", path); + } +} + +bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr *read_len, uptr max_len, error_t *errno_p) { + uptr PageSize = GetPageSizeCached(); + uptr kMinFileLen = PageSize; + *buff = nullptr; + *buff_size = 0; + *read_len = 0; + // The files we usually open are not seekable, so try different buffer sizes. + for (uptr size = kMinFileLen; size <= max_len; size *= 2) { + fd_t fd = OpenFile(file_name, RdOnly, errno_p); + if (fd == kInvalidFd) return false; + UnmapOrDie(*buff, *buff_size); + *buff = (char*)MmapOrDie(size, __func__); + *buff_size = size; + *read_len = 0; + // Read up to one page at a time. + bool reached_eof = false; + while (*read_len + PageSize <= size) { + uptr just_read; + if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) { + UnmapOrDie(*buff, *buff_size); + return false; + } + if (just_read == 0) { + reached_eof = true; + break; + } + *read_len += just_read; + } + CloseFile(fd); + if (reached_eof) // We've read the whole file. + break; + } + return true; +} + +static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':'; + +char *FindPathToBinary(const char *name) { + if (FileExists(name)) { + return internal_strdup(name); + } + + const char *path = GetEnv("PATH"); + if (!path) + return nullptr; + uptr name_len = internal_strlen(name); + InternalScopedBuffer buffer(kMaxPathLength); + const char *beg = path; + while (true) { + const char *end = internal_strchrnul(beg, kPathSeparator); + uptr prefix_len = end - beg; + if (prefix_len + name_len + 2 <= kMaxPathLength) { + internal_memcpy(buffer.data(), beg, prefix_len); + buffer[prefix_len] = '/'; + internal_memcpy(&buffer[prefix_len + 1], name, name_len); + buffer[prefix_len + 1 + name_len] = '\0'; + if (FileExists(buffer.data())) + return internal_strdup(buffer.data()); + } + if (*end == '\0') break; + beg = end + 1; + } + return nullptr; +} + +} // namespace __sanitizer + +using namespace __sanitizer; // NOLINT + +extern "C" { +void __sanitizer_set_report_path(const char *path) { + report_file.SetReportPath(path); +} + +void __sanitizer_set_report_fd(void *fd) { + report_file.fd = (fd_t)reinterpret_cast(fd); + report_file.fd_pid = internal_getpid(); +} +} // extern "C" + +#endif // !SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h new file mode 100644 index 000000000000..9a12ab7342c7 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h @@ -0,0 +1,110 @@ +//===-- sanitizer_file.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is shared between run-time libraries of sanitizers. +// It declares filesystem-related interfaces. This is separate from +// sanitizer_common.h so that it's simpler to disable all the filesystem +// support code for a port that doesn't use it. +// +//===---------------------------------------------------------------------===// +#ifndef SANITIZER_FILE_H +#define SANITIZER_FILE_H + +#include "sanitizer_interface_internal.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_mutex.h" + +namespace __sanitizer { + +struct ReportFile { + void Write(const char *buffer, uptr length); + bool SupportsColors(); + void SetReportPath(const char *path); + + // Don't use fields directly. They are only declared public to allow + // aggregate initialization. + + // Protects fields below. + StaticSpinMutex *mu; + // Opened file descriptor. Defaults to stderr. It may be equal to + // kInvalidFd, in which case new file will be opened when necessary. + fd_t fd; + // Path prefix of report file, set via __sanitizer_set_report_path. + char path_prefix[kMaxPathLength]; + // Full path to report, obtained as .PID + char full_path[kMaxPathLength]; + // PID of the process that opened fd. If a fork() occurs, + // the PID of child will be different from fd_pid. + uptr fd_pid; + + private: + void ReopenIfNecessary(); +}; +extern ReportFile report_file; + +enum FileAccessMode { + RdOnly, + WrOnly, + RdWr +}; + +// Returns kInvalidFd on error. +fd_t OpenFile(const char *filename, FileAccessMode mode, + error_t *errno_p = nullptr); +void CloseFile(fd_t); + +// Return true on success, false on error. +bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, + uptr *bytes_read = nullptr, error_t *error_p = nullptr); +bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, + uptr *bytes_written = nullptr, error_t *error_p = nullptr); + +bool RenameFile(const char *oldpath, const char *newpath, + error_t *error_p = nullptr); + +// Scoped file handle closer. +struct FileCloser { + explicit FileCloser(fd_t fd) : fd(fd) {} + ~FileCloser() { CloseFile(fd); } + fd_t fd; +}; + +bool SupportsColoredOutput(fd_t fd); + +// OS +const char *GetPwd(); +bool FileExists(const char *filename); +char *FindPathToBinary(const char *name); +bool IsPathSeparator(const char c); +bool IsAbsolutePath(const char *path); +// Starts a subprocess and returs its pid. +// If *_fd parameters are not kInvalidFd their corresponding input/output +// streams will be redirect to the file. The files will always be closed +// in parent process even in case of an error. +// The child process will close all fds after STDERR_FILENO +// before passing control to a program. +pid_t StartSubprocess(const char *filename, const char *const argv[], + fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd, + fd_t stderr_fd = kInvalidFd); +// Checks if specified process is still running +bool IsProcessRunning(pid_t pid); +// Waits for the process to finish and returns its exit code. +// Returns -1 in case of an error. +int WaitForProcess(pid_t pid); + +// Maps given file to virtual memory, and returns pointer to it +// (or NULL if mapping fails). Stores the size of mmaped region +// in '*buff_size'. +void *MapFileToMemory(const char *file_name, uptr *buff_size); +void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset); + +} // namespace __sanitizer + +#endif // SANITIZER_FILE_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h index 4988fbb7a86d..f649f5bffea7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h @@ -75,7 +75,7 @@ inline bool FlagHandler::Parse(const char *value) { template <> inline bool FlagHandler::Parse(const char *value) { - *t_ = internal_strdup(value); + *t_ = value; return true; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc index 9f71861efcdb..445e25f9df70 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc @@ -62,6 +62,9 @@ COMMON_FLAG( COMMON_FLAG( int, verbosity, 0, "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).") +COMMON_FLAG(bool, strip_env, 1, + "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to " + "avoid passing it to children. Default is true.") COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.") COMMON_FLAG( bool, leak_check_at_exit, true, @@ -129,11 +132,11 @@ COMMON_FLAG(uptr, soft_rss_limit_mb, 0, " This limit does not affect memory allocations other than" " malloc/new.") COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only") -COMMON_FLAG(s32, allocator_release_to_os_interval_ms, kReleaseToOSIntervalNever, - "Experimental. Only affects a 64-bit allocator. If set, tries to " - "release unused memory to the OS, but not more often than this " - "interval (in milliseconds). Negative values mean do not attempt " - "to release memory to the OS.\n") +COMMON_FLAG(s32, allocator_release_to_os_interval_ms, 5000, + "Only affects a 64-bit allocator. If set, tries to release unused " + "memory to the OS, but not more often than this interval (in " + "milliseconds). Negative values mean do not attempt to release " + "memory to the OS.\n") COMMON_FLAG(bool, can_use_proc_maps_statm, true, "If false, do not attempt to read /proc/maps/statm." " Mostly useful for testing sanitizers.") @@ -229,3 +232,8 @@ COMMON_FLAG(bool, print_cmdline, false, "Print command line on crash " "(asan only).") COMMON_FLAG(bool, html_cov_report, false, "Generate html coverage report.") COMMON_FLAG(const char *, sancov_path, "sancov", "Sancov tool location.") +COMMON_FLAG(bool, dump_instruction_bytes, false, + "If true, dump 16 bytes starting at the instruction that caused SEGV") +COMMON_FLAG(bool, dump_registers, true, + "If true, dump values of CPU registers when SEGV happens. Only " + "available on OS X for now.") diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc new file mode 100644 index 000000000000..936ec794b8e8 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -0,0 +1,539 @@ +//===-- sanitizer_fuchsia.cc ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer and other sanitizer +// run-time libraries and implements Fuchsia-specific functions from +// sanitizer_common.h. +//===---------------------------------------------------------------------===// + +#include "sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "sanitizer_common.h" +#include "sanitizer_libc.h" +#include "sanitizer_mutex.h" +#include "sanitizer_stacktrace.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace __sanitizer { + +void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); } + +uptr internal_sched_yield() { + zx_status_t status = _zx_nanosleep(0); + CHECK_EQ(status, ZX_OK); + return 0; // Why doesn't this return void? +} + +static void internal_nanosleep(zx_time_t ns) { + zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns)); + CHECK_EQ(status, ZX_OK); +} + +unsigned int internal_sleep(unsigned int seconds) { + internal_nanosleep(ZX_SEC(seconds)); + return 0; +} + +u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); } + +u64 MonotonicNanoTime() { return _zx_time_get(ZX_CLOCK_MONOTONIC); } + +uptr internal_getpid() { + zx_info_handle_basic_t info; + zx_status_t status = + _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &info, + sizeof(info), NULL, NULL); + CHECK_EQ(status, ZX_OK); + uptr pid = static_cast(info.koid); + CHECK_EQ(pid, info.koid); + return pid; +} + +uptr GetThreadSelf() { return reinterpret_cast(thrd_current()); } + +uptr GetTid() { return GetThreadSelf(); } + +void Abort() { abort(); } + +int Atexit(void (*function)(void)) { return atexit(function); } + +void SleepForSeconds(int seconds) { internal_sleep(seconds); } + +void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); } + +void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) { + pthread_attr_t attr; + CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); + void *base; + size_t size; + CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0); + CHECK_EQ(pthread_attr_destroy(&attr), 0); + + *stack_bottom = reinterpret_cast(base); + *stack_top = *stack_bottom + size; +} + +void MaybeReexec() {} +void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {} +void DisableCoreDumperIfNecessary() {} +void InstallDeadlySignalHandlers(SignalHandlerType handler) {} +void StartReportDeadlySignal() {} +void ReportDeadlySignal(const SignalContext &sig, u32 tid, + UnwindSignalStackCallbackType unwind, + const void *unwind_context) {} +void SetAlternateSignalStack() {} +void UnsetAlternateSignalStack() {} +void InitTlsSize() {} + +void PrintModuleMap() {} + +bool SignalContext::IsStackOverflow() const { return false; } +void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); } +const char *SignalContext::Describe() const { UNIMPLEMENTED(); } + +struct UnwindTraceArg { + BufferedStackTrace *stack; + u32 max_depth; +}; + +_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { + UnwindTraceArg *arg = static_cast(param); + CHECK_LT(arg->stack->size, arg->max_depth); + uptr pc = _Unwind_GetIP(ctx); + if (pc < PAGE_SIZE) return _URC_NORMAL_STOP; + arg->stack->trace_buffer[arg->stack->size++] = pc; + return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP + : _URC_NO_REASON); +} + +void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { + CHECK_GE(max_depth, 2); + size = 0; + UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; + _Unwind_Backtrace(Unwind_Trace, &arg); + CHECK_GT(size, 0); + // We need to pop a few frames so that pc is on top. + uptr to_pop = LocatePcInTrace(pc); + // trace_buffer[0] belongs to the current function so we always pop it, + // unless there is only 1 frame in the stack trace (1 frame is always better + // than 0!). + PopStackFrames(Min(to_pop, static_cast(1))); + trace_buffer[0] = pc; +} + +void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, + u32 max_depth) { + CHECK_NE(context, nullptr); + UNREACHABLE("signal context doesn't exist"); +} + +enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; + +BlockingMutex::BlockingMutex() { + // NOTE! It's important that this use internal_memset, because plain + // memset might be intercepted (e.g., actually be __asan_memset). + // Defining this so the compiler initializes each field, e.g.: + // BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {} + // might result in the compiler generating a call to memset, which would + // have the same problem. + internal_memset(this, 0, sizeof(*this)); +} + +void BlockingMutex::Lock() { + CHECK_EQ(owner_, 0); + atomic_uint32_t *m = reinterpret_cast(&opaque_storage_); + if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked) + return; + while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { + zx_status_t status = _zx_futex_wait(reinterpret_cast(m), + MtxSleeping, ZX_TIME_INFINITE); + if (status != ZX_ERR_BAD_STATE) // Normal race. + CHECK_EQ(status, ZX_OK); + } +} + +void BlockingMutex::Unlock() { + atomic_uint32_t *m = reinterpret_cast(&opaque_storage_); + u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release); + CHECK_NE(v, MtxUnlocked); + if (v == MtxSleeping) { + zx_status_t status = _zx_futex_wake(reinterpret_cast(m), 1); + CHECK_EQ(status, ZX_OK); + } +} + +void BlockingMutex::CheckLocked() { + atomic_uint32_t *m = reinterpret_cast(&opaque_storage_); + CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); +} + +uptr GetPageSize() { return PAGE_SIZE; } + +uptr GetMmapGranularity() { return PAGE_SIZE; } + +sanitizer_shadow_bounds_t ShadowBounds; + +uptr GetMaxUserVirtualAddress() { + ShadowBounds = __sanitizer_shadow_bounds(); + return ShadowBounds.memory_limit - 1; +} + +uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); } + +static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, + bool raw_report, bool die_for_nomem) { + size = RoundUpTo(size, PAGE_SIZE); + + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(size, 0, &vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) + ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, + raw_report); + return nullptr; + } + _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type, + internal_strlen(mem_type)); + + // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? + uintptr_t addr; + status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + _zx_handle_close(vmo); + + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) + ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, + raw_report); + return nullptr; + } + + IncreaseTotalMmap(size); + + return reinterpret_cast(addr); +} + +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { + return DoAnonymousMmapOrDie(size, mem_type, raw_report, true); +} + +void *MmapNoReserveOrDie(uptr size, const char *mem_type) { + return MmapOrDie(size, mem_type); +} + +void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { + return DoAnonymousMmapOrDie(size, mem_type, false, false); +} + +uptr ReservedAddressRange::Init(uptr init_size, const char *name, + uptr fixed_addr) { + init_size = RoundUpTo(init_size, PAGE_SIZE); + DCHECK_EQ(os_handle_, ZX_HANDLE_INVALID); + uintptr_t base; + zx_handle_t vmar; + zx_status_t status = + _zx_vmar_allocate(_zx_vmar_root_self(), 0, init_size, + ZX_VM_FLAG_CAN_MAP_READ | ZX_VM_FLAG_CAN_MAP_WRITE | + ZX_VM_FLAG_CAN_MAP_SPECIFIC, + &vmar, &base); + if (status != ZX_OK) + ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status); + base_ = reinterpret_cast(base); + size_ = init_size; + name_ = name; + os_handle_ = vmar; + + return reinterpret_cast(base_); +} + +static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, + void *base, const char *name, bool die_for_nomem) { + uptr offset = fixed_addr - reinterpret_cast(base); + map_size = RoundUpTo(map_size, PAGE_SIZE); + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(map_size, 0, &vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) + ReportMmapFailureAndDie(map_size, name, "zx_vmo_create", status); + return 0; + } + _zx_object_set_property(vmo, ZX_PROP_NAME, name, sizeof(name) - 1); + DCHECK_GE(base + size_, map_size + offset); + uintptr_t addr; + + status = _zx_vmar_map( + vmar, offset, vmo, 0, map_size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_SPECIFIC, + &addr); + _zx_handle_close(vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY || die_for_nomem) { + ReportMmapFailureAndDie(map_size, name, "zx_vmar_map", status); + } + return 0; + } + IncreaseTotalMmap(map_size); + return addr; +} + +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size) { + return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, + name_, false); +} + +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size) { + return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, + name_, true); +} + +void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) { + if (!addr || !size) return; + size = RoundUpTo(size, PAGE_SIZE); + + zx_status_t status = + _zx_vmar_unmap(target_vmar, reinterpret_cast(addr), size); + if (status != ZX_OK) { + Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n", + SanitizerToolName, size, size, addr); + CHECK("unable to unmap" && 0); + } + + DecreaseTotalMmap(size); +} + +void ReservedAddressRange::Unmap(uptr fixed_addr, uptr size) { + uptr offset = fixed_addr - reinterpret_cast(base_); + uptr addr = reinterpret_cast(base_) + offset; + void *addr_as_void = reinterpret_cast(addr); + uptr base_as_uptr = reinterpret_cast(base_); + // Only unmap at the beginning or end of the range. + CHECK((addr_as_void == base_) || (addr + size == base_as_uptr + size_)); + CHECK_LE(size, size_); + UnmapOrDieVmar(reinterpret_cast(addr), size, + static_cast(os_handle_)); + if (addr_as_void == base_) { + base_ = reinterpret_cast(addr + size); + } + size_ = size_ - size; +} + +// This should never be called. +void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { + UNIMPLEMENTED(); +} + +void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, + const char *mem_type) { + CHECK_GE(size, PAGE_SIZE); + CHECK(IsPowerOfTwo(size)); + CHECK(IsPowerOfTwo(alignment)); + + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(size, 0, &vmo); + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY) + ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, false); + return nullptr; + } + _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type, + internal_strlen(mem_type)); + + // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? + + // Map a larger size to get a chunk of address space big enough that + // it surely contains an aligned region of the requested size. Then + // overwrite the aligned middle portion with a mapping from the + // beginning of the VMO, and unmap the excess before and after. + size_t map_size = size + alignment; + uintptr_t addr; + status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + if (status == ZX_OK) { + uintptr_t map_addr = addr; + uintptr_t map_end = map_addr + map_size; + addr = RoundUpTo(map_addr, alignment); + uintptr_t end = addr + size; + if (addr != map_addr) { + zx_info_vmar_t info; + status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info, + sizeof(info), NULL, NULL); + if (status == ZX_OK) { + uintptr_t new_addr; + status = + _zx_vmar_map(_zx_vmar_root_self(), addr - info.base, vmo, 0, size, + ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | + ZX_VM_FLAG_SPECIFIC_OVERWRITE, + &new_addr); + if (status == ZX_OK) CHECK_EQ(new_addr, addr); + } + } + if (status == ZX_OK && addr != map_addr) + status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr); + if (status == ZX_OK && end != map_end) + status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end); + } + _zx_handle_close(vmo); + + if (status != ZX_OK) { + if (status != ZX_ERR_NO_MEMORY) + ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, false); + return nullptr; + } + + IncreaseTotalMmap(size); + + return reinterpret_cast(addr); +} + +void UnmapOrDie(void *addr, uptr size) { + UnmapOrDieVmar(addr, size, _zx_vmar_root_self()); +} + +// This is used on the shadow mapping, which cannot be changed. +// Zircon doesn't have anything like MADV_DONTNEED. +void ReleaseMemoryPagesToOS(uptr beg, uptr end) {} + +void DumpProcessMap() { + // TODO(mcgrathr): write it + return; +} + +bool IsAccessibleMemoryRange(uptr beg, uptr size) { + // TODO(mcgrathr): Figure out a better way. + zx_handle_t vmo; + zx_status_t status = _zx_vmo_create(size, 0, &vmo); + if (status == ZX_OK) { + while (size > 0) { + size_t wrote; + status = _zx_vmo_write(vmo, reinterpret_cast(beg), 0, size, + &wrote); + if (status != ZX_OK) break; + CHECK_GT(wrote, 0); + CHECK_LE(wrote, size); + beg += wrote; + size -= wrote; + } + _zx_handle_close(vmo); + } + return status == ZX_OK; +} + +// FIXME implement on this platform. +void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {} + +bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr *read_len, uptr max_len, error_t *errno_p) { + zx_handle_t vmo; + zx_status_t status = __sanitizer_get_configuration(file_name, &vmo); + if (status == ZX_OK) { + uint64_t vmo_size; + status = _zx_vmo_get_size(vmo, &vmo_size); + if (status == ZX_OK) { + if (vmo_size < max_len) max_len = vmo_size; + size_t map_size = RoundUpTo(max_len, PAGE_SIZE); + uintptr_t addr; + status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size, + ZX_VM_FLAG_PERM_READ, &addr); + if (status == ZX_OK) { + *buff = reinterpret_cast(addr); + *buff_size = map_size; + *read_len = max_len; + } + } + _zx_handle_close(vmo); + } + if (status != ZX_OK && errno_p) *errno_p = status; + return status == ZX_OK; +} + +void RawWrite(const char *buffer) { + __sanitizer_log_write(buffer, internal_strlen(buffer)); +} + +void CatastrophicErrorWrite(const char *buffer, uptr length) { + __sanitizer_log_write(buffer, length); +} + +char **StoredArgv; +char **StoredEnviron; + +char **GetArgv() { return StoredArgv; } + +const char *GetEnv(const char *name) { + if (StoredEnviron) { + uptr NameLen = internal_strlen(name); + for (char **Env = StoredEnviron; *Env != 0; Env++) { + if (internal_strncmp(*Env, name, NameLen) == 0 && (*Env)[NameLen] == '=') + return (*Env) + NameLen + 1; + } + } + return nullptr; +} + +uptr ReadBinaryName(/*out*/ char *buf, uptr buf_len) { + const char *argv0 = StoredArgv[0]; + if (!argv0) argv0 = ""; + internal_strncpy(buf, argv0, buf_len); + return internal_strlen(buf); +} + +uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { + return ReadBinaryName(buf, buf_len); +} + +uptr MainThreadStackBase, MainThreadStackSize; + +bool GetRandom(void *buffer, uptr length, bool blocking) { + CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN); + size_t size; + CHECK_EQ(_zx_cprng_draw(buffer, length, &size), ZX_OK); + CHECK_EQ(size, length); + return true; +} + +u32 GetNumberOfCPUs() { + return zx_system_get_num_cpus(); +} + +uptr GetRSS() { UNIMPLEMENTED(); } + +} // namespace __sanitizer + +using namespace __sanitizer; // NOLINT + +extern "C" { +void __sanitizer_startup_hook(int argc, char **argv, char **envp, + void *stack_base, size_t stack_size) { + __sanitizer::StoredArgv = argv; + __sanitizer::StoredEnviron = envp; + __sanitizer::MainThreadStackBase = reinterpret_cast(stack_base); + __sanitizer::MainThreadStackSize = stack_size; +} + +void __sanitizer_set_report_path(const char *path) { + // Handle the initialization code in each sanitizer, but no other calls. + // This setting is never consulted on Fuchsia. + DCHECK_EQ(path, common_flags()->log_path); +} + +void __sanitizer_set_report_fd(void *fd) { + UNREACHABLE("not available on Fuchsia"); +} +} // extern "C" + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h new file mode 100644 index 000000000000..18821b4fd9f1 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h @@ -0,0 +1,31 @@ +//===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// Fuchsia-specific sanitizer support. +// +//===---------------------------------------------------------------------===// +#ifndef SANITIZER_FUCHSIA_H +#define SANITIZER_FUCHSIA_H + +#include "sanitizer_platform.h" +#if SANITIZER_FUCHSIA + +#include "sanitizer_common.h" + +#include + +namespace __sanitizer { + +extern uptr MainThreadStackBase, MainThreadStackSize; +extern sanitizer_shadow_bounds_t ShadowBounds; + +} // namespace __sanitizer + +#endif // SANITIZER_FUCHSIA +#endif // SANITIZER_FUCHSIA_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h new file mode 100644 index 000000000000..934e311a7851 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h @@ -0,0 +1,47 @@ +//===-- sanitizer_getauxval.h -----------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Common getauxval() guards and definitions. +// getauxval() is not defined until glibc version 2.16, or until API level 21 +// for Android. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_GETAUXVAL_H +#define SANITIZER_GETAUXVAL_H + +#include "sanitizer_platform.h" + +#if SANITIZER_LINUX + +# include + +# ifndef __GLIBC_PREREQ +# define __GLIBC_PREREQ(x, y) 0 +# endif + +# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) +# define SANITIZER_USE_GETAUXVAL 1 +# else +# define SANITIZER_USE_GETAUXVAL 0 +# endif + +# if SANITIZER_USE_GETAUXVAL +# include +# else +// The weak getauxval definition allows to check for the function at runtime. +// This is useful for Android, when compiled at a lower API level yet running +// on a more recent platform that offers the function. +extern "C" SANITIZER_WEAK_ATTRIBUTE +unsigned long getauxval(unsigned long type); // NOLINT +# endif + +#endif // SANITIZER_LINUX + +#endif // SANITIZER_GETAUXVAL_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h index b28d8f08e7a3..942f0b1cb586 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h @@ -46,7 +46,6 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_report_error_summary(const char *error_summary); - SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init(); SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump(); SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage( const __sanitizer::uptr *pcs, const __sanitizer::uptr len); @@ -81,6 +80,14 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp8(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp1(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp2(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp4(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_trace_const_cmp8(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_switch(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_div4(); @@ -95,6 +102,10 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*, __sanitizer::u32*); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_8bit_counters_init(); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_cov_pcs_init(); } // extern "C" #endif // SANITIZER_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index f35b095ee94e..dc480e75f98d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -35,6 +35,14 @@ # define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) #endif +// TLS is handled differently on different platforms +#if SANITIZER_LINUX || SANITIZER_NETBSD +# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \ + __attribute__((tls_model("initial-exec"))) thread_local +#else +# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE +#endif + //--------------------------- WEAK FUNCTIONS ---------------------------------// // When working with weak functions, to simplify the code and make it more // portable, when possible define a default implementation using this macro: @@ -56,11 +64,13 @@ // SANITIZER_SUPPORTS_WEAK_HOOKS means that we support real weak functions that // will evaluate to a null pointer when not defined. -#if (SANITIZER_LINUX || SANITIZER_MAC) && !SANITIZER_GO +#ifndef SANITIZER_SUPPORTS_WEAK_HOOKS +#if (SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_SOLARIS) && !SANITIZER_GO # define SANITIZER_SUPPORTS_WEAK_HOOKS 1 #else # define SANITIZER_SUPPORTS_WEAK_HOOKS 0 #endif +#endif // SANITIZER_SUPPORTS_WEAK_HOOKS // For some weak hooks that will be called very often and we want to avoid the // overhead of executing the default implementation when it is not necessary, // we can use the flag SANITIZER_SUPPORTS_WEAK_HOOKS to only define the default @@ -81,6 +91,10 @@ // FIXME: do we have anything like this on Mac? #if SANITIZER_LINUX && !SANITIZER_ANDROID && !defined(PIC) # define SANITIZER_CAN_USE_PREINIT_ARRAY 1 +// Before Solaris 11.4, .preinit_array is fully supported only with GNU ld. +// FIXME: Check for those conditions. +#elif SANITIZER_SOLARIS && !defined(PIC) +# define SANITIZER_CAN_USE_PREINIT_ARRAY 1 #else # define SANITIZER_CAN_USE_PREINIT_ARRAY 0 #endif @@ -127,14 +141,14 @@ typedef unsigned error_t; typedef int fd_t; typedef int error_t; #endif +#if SANITIZER_SOLARIS && !defined(_LP64) +typedef long pid_t; +#else typedef int pid_t; +#endif -// WARNING: OFF_T may be different from OS type off_t, depending on the value of -// _FILE_OFFSET_BITS. This definition of OFF_T matches the ABI of system calls -// like pread and mmap, as opposed to pread64 and mmap64. -// FreeBSD, Mac and Linux/x86-64 are special. -#if SANITIZER_FREEBSD || SANITIZER_MAC || \ - (SANITIZER_LINUX && defined(__x86_64__)) +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC || \ + (SANITIZER_LINUX && defined(__x86_64__)) typedef u64 OFF_T; #else typedef uptr OFF_T; @@ -257,8 +271,8 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, #define CHECK_IMPL(c1, op, c2) \ do { \ - __sanitizer::u64 v1 = (u64)(c1); \ - __sanitizer::u64 v2 = (u64)(c2); \ + __sanitizer::u64 v1 = (__sanitizer::u64)(c1); \ + __sanitizer::u64 v2 = (__sanitizer::u64)(c2); \ if (UNLIKELY(!(v1 op v2))) \ __sanitizer::CheckFailed(__FILE__, __LINE__, \ "(" #c1 ") " #op " (" #c2 ")", v1, v2); \ @@ -384,6 +398,7 @@ namespace __dfsan { using namespace __sanitizer; } // NOLINT namespace __esan { using namespace __sanitizer; } // NOLINT namespace __lsan { using namespace __sanitizer; } // NOLINT namespace __msan { using namespace __sanitizer; } // NOLINT +namespace __hwasan { using namespace __sanitizer; } // NOLINT namespace __tsan { using namespace __sanitizer; } // NOLINT namespace __scudo { using namespace __sanitizer; } // NOLINT namespace __ubsan { using namespace __sanitizer; } // NOLINT diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc index aa4fa88efffa..0df055edae97 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc @@ -9,7 +9,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD #include "sanitizer_libignore.h" #include "sanitizer_flags.h" @@ -125,4 +125,5 @@ void LibIgnore::OnLibraryUnloaded() { } // namespace __sanitizer -#endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || + // SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index 8c3c1e5d6a5d..6c83e8db42a5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -14,10 +14,12 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_getauxval.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" #include "sanitizer_linux.h" @@ -27,10 +29,14 @@ #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX #include #endif +#if SANITIZER_NETBSD +#include +#endif + // For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat' // format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To // access stat from asm/stat.h, without conflicting with definition in @@ -50,7 +56,9 @@ #include #include #include +#if !SANITIZER_SOLARIS #include +#endif #include #include #include @@ -79,24 +87,26 @@ extern "C" { extern char **environ; // provided by crt1 #endif // SANITIZER_FREEBSD +#if SANITIZER_NETBSD +#include // For NAME_MAX +#include +extern char **environ; // provided by crt1 +#include +extern struct ps_strings *__ps_strings; +#endif // SANITIZER_NETBSD + +#if SANITIZER_SOLARIS +#include +#include + +extern char **_environ; +#define environ _environ +#endif + #if !SANITIZER_ANDROID #include #endif -#ifndef __GLIBC_PREREQ -#define __GLIBC_PREREQ(x, y) 0 -#endif - -#if SANITIZER_LINUX && __GLIBC_PREREQ(2, 16) -# define SANITIZER_USE_GETAUXVAL 1 -#else -# define SANITIZER_USE_GETAUXVAL 0 -#endif - -#if SANITIZER_USE_GETAUXVAL -#include -#endif - #if SANITIZER_LINUX // struct kernel_timeval { @@ -125,21 +135,36 @@ extern void internal_sigreturn(); } #endif +#if SANITIZER_LINUX && defined(__NR_getrandom) +# if !defined(GRND_NONBLOCK) +# define GRND_NONBLOCK 1 +# endif +# define SANITIZER_USE_GETRANDOM 1 +#else +# define SANITIZER_USE_GETRANDOM 0 +#endif // SANITIZER_LINUX && defined(__NR_getrandom) + namespace __sanitizer { #if SANITIZER_LINUX && defined(__x86_64__) #include "sanitizer_syscall_linux_x86_64.inc" #elif SANITIZER_LINUX && defined(__aarch64__) #include "sanitizer_syscall_linux_aarch64.inc" +#elif SANITIZER_LINUX && defined(__arm__) +#include "sanitizer_syscall_linux_arm.inc" #else #include "sanitizer_syscall_generic.inc" #endif // --------------- sanitizer_libc.h +#if !SANITIZER_SOLARIS #if !SANITIZER_S390 uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, OFF_T offset) { -#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +#if SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(mmap), addr, length, prot, flags, fd, + (long)0, offset); +#elif SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd, offset); #else @@ -152,11 +177,11 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, #endif // !SANITIZER_S390 uptr internal_munmap(void *addr, uptr length) { - return internal_syscall(SYSCALL(munmap), (uptr)addr, length); + return internal_syscall_ptr(SYSCALL(munmap), (uptr)addr, length); } int internal_mprotect(void *addr, uptr length, int prot) { - return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot); + return internal_syscall_ptr(SYSCALL(mprotect), (uptr)addr, length, prot); } uptr internal_close(fd_t fd) { @@ -167,7 +192,7 @@ uptr internal_open(const char *filename, int flags) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags); #else - return internal_syscall(SYSCALL(open), (uptr)filename, flags); + return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags); #endif } @@ -176,32 +201,36 @@ uptr internal_open(const char *filename, int flags, u32 mode) { return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags, mode); #else - return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode); + return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags, mode); #endif } uptr internal_read(fd_t fd, void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf, + HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(read), fd, (uptr)buf, count)); return res; } uptr internal_write(fd_t fd, const void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf, + HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(write), fd, (uptr)buf, count)); return res; } uptr internal_ftruncate(fd_t fd, uptr size) { sptr res; +#if SANITIZER_NETBSD + HANDLE_EINTR(res, internal_syscall64(SYSCALL(ftruncate), fd, 0, (s64)size)); +#else HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd, (OFF_T)size)); +#endif return res; } -#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && !SANITIZER_FREEBSD +#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX static void stat64_to_stat(struct stat64 *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = in->st_dev; @@ -221,6 +250,21 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { #endif #if defined(__mips64) +// Undefine compatibility macros from +// so that they would not clash with the kernel_stat +// st_[a|m|c]time fields +#undef st_atime +#undef st_mtime +#undef st_ctime +#if defined(SANITIZER_ANDROID) +// Bionic sys/stat.h defines additional macros +// for compatibility with the old NDKs and +// they clash with the kernel_stat structure +// st_[a|m|c]time_nsec fields. +#undef st_atime_nsec +#undef st_mtime_nsec +#undef st_ctime_nsec +#endif static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = in->st_dev; @@ -233,15 +277,29 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { out->st_size = in->st_size; out->st_blksize = in->st_blksize; out->st_blocks = in->st_blocks; - out->st_atime = in->st_atime_nsec; - out->st_mtime = in->st_mtime_nsec; - out->st_ctime = in->st_ctime_nsec; +#if defined(__USE_MISC) || \ + defined(__USE_XOPEN2K8) || \ + defined(SANITIZER_ANDROID) + out->st_atim.tv_sec = in->st_atime; + out->st_atim.tv_nsec = in->st_atime_nsec; + out->st_mtim.tv_sec = in->st_mtime; + out->st_mtim.tv_nsec = in->st_mtime_nsec; + out->st_ctim.tv_sec = in->st_ctime; + out->st_ctim.tv_nsec = in->st_ctime_nsec; +#else + out->st_atime = in->st_atime; + out->st_atimensec = in->st_atime_nsec; + out->st_mtime = in->st_mtime; + out->st_mtimensec = in->st_mtime_nsec; + out->st_ctime = in->st_ctime; + out->st_atimensec = in->st_ctime_nsec; +#endif } #endif uptr internal_stat(const char *path, void *buf) { -#if SANITIZER_FREEBSD - return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, @@ -265,7 +323,9 @@ uptr internal_stat(const char *path, void *buf) { } uptr internal_lstat(const char *path, void *buf) { -#if SANITIZER_FREEBSD +#if SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(lstat), path, buf); +#elif SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS @@ -290,15 +350,15 @@ uptr internal_lstat(const char *path, void *buf) { } uptr internal_fstat(fd_t fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS -# if SANITIZER_MIPS64 +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +# if SANITIZER_MIPS64 && !SANITIZER_NETBSD // For mips64, fstat syscall fills buffer in the format of kernel_stat struct kernel_stat kbuf; int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; # else - return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); + return internal_syscall_ptr(SYSCALL(fstat), fd, (uptr)buf); # endif #else struct stat64 buf64; @@ -328,7 +388,7 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, bufsize); #else - return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize); + return internal_syscall_ptr(SYSCALL(readlink), path, buf, bufsize); #endif } @@ -336,7 +396,7 @@ uptr internal_unlink(const char *path) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0); #else - return internal_syscall(SYSCALL(unlink), (uptr)path); + return internal_syscall_ptr(SYSCALL(unlink), (uptr)path); #endif } @@ -345,7 +405,7 @@ uptr internal_rename(const char *oldpath, const char *newpath) { return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath); #else - return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); + return internal_syscall_ptr(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); #endif } @@ -354,7 +414,7 @@ uptr internal_sched_yield() { } void internal__exit(int exitcode) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD internal_syscall(SYSCALL(exit), exitcode); #else internal_syscall(SYSCALL(exit_group), exitcode); @@ -366,16 +426,17 @@ unsigned int internal_sleep(unsigned int seconds) { struct timespec ts; ts.tv_sec = 1; ts.tv_nsec = 0; - int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts); + int res = internal_syscall_ptr(SYSCALL(nanosleep), &ts, &ts); if (res) return ts.tv_sec; return 0; } uptr internal_execve(const char *filename, char *const argv[], char *const envp[]) { - return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv, + return internal_syscall_ptr(SYSCALL(execve), (uptr)filename, (uptr)argv, (uptr)envp); } +#endif // !SANITIZER_SOLARIS // ----------------- sanitizer_common.h bool FileExists(const char *filename) { @@ -393,27 +454,37 @@ bool FileExists(const char *filename) { tid_t GetTid() { #if SANITIZER_FREEBSD return (uptr)pthread_self(); +#elif SANITIZER_NETBSD + return _lwp_self(); +#elif SANITIZER_SOLARIS + return (uptr)thr_self(); #else return internal_syscall(SYSCALL(gettid)); #endif } +#if !SANITIZER_SOLARIS u64 NanoTime() { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD timeval tv; #else kernel_timeval tv; #endif internal_memset(&tv, 0, sizeof(tv)); - internal_syscall(SYSCALL(gettimeofday), (uptr)&tv, 0); + internal_syscall_ptr(SYSCALL(gettimeofday), &tv, 0); return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000; } +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { + return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp); +} +#endif // !SANITIZER_SOLARIS + // Like getenv, but reads env directly from /proc (on Linux) or parses the -// 'environ' array (on FreeBSD) and does not use libc. This function should be -// called first inside __asan_init. +// 'environ' array (on some others) and does not use libc. This function +// should be called first inside __asan_init. const char *GetEnv(const char *name) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_SOLARIS if (::environ != 0) { uptr NameLen = internal_strlen(name); for (char **Env = ::environ; *Env != 0; Env++) { @@ -451,13 +522,13 @@ const char *GetEnv(const char *name) { #endif } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD extern "C" { SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end; } #endif -#if !SANITIZER_GO && !SANITIZER_FREEBSD +#if !SANITIZER_GO && !SANITIZER_FREEBSD && !SANITIZER_NETBSD static void ReadNullSepFileToArray(const char *path, char ***arr, int arr_size) { char *buff; @@ -483,7 +554,22 @@ static void ReadNullSepFileToArray(const char *path, char ***arr, #endif static void GetArgsAndEnv(char ***argv, char ***envp) { -#if !SANITIZER_FREEBSD +#if SANITIZER_FREEBSD + // On FreeBSD, retrieving the argument and environment arrays is done via the + // kern.ps_strings sysctl, which returns a pointer to a structure containing + // this information. See also . + ps_strings *pss; + size_t sz = sizeof(pss); + if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { + Printf("sysctl kern.ps_strings failed\n"); + Die(); + } + *argv = pss->ps_argvstr; + *envp = pss->ps_envstr; +#elif SANITIZER_NETBSD + *argv = __ps_strings->ps_argvstr; + *argv = __ps_strings->ps_envstr; +#else #if !SANITIZER_GO if (&__libc_stack_end) { #endif @@ -498,18 +584,6 @@ static void GetArgsAndEnv(char ***argv, char ***envp) { ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp); } #endif -#else - // On FreeBSD, retrieving the argument and environment arrays is done via the - // kern.ps_strings sysctl, which returns a pointer to a structure containing - // this information. See also . - ps_strings *pss; - size_t sz = sizeof(pss); - if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { - Printf("sysctl kern.ps_strings failed\n"); - Die(); - } - *argv = pss->ps_argvstr; - *envp = pss->ps_envstr; #endif } @@ -521,14 +595,32 @@ char **GetArgv() { void ReExec() { char **argv, **envp; + const char *pathname = "/proc/self/exe"; + +#if SANITIZER_NETBSD + static const int name[] = { + CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME, + }; + char path[400]; + size_t len; + + len = sizeof(path); + if (sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) + pathname = path; +#elif SANITIZER_SOLARIS + pathname = getexecname(); + CHECK_NE(pathname, NULL); +#endif + GetArgsAndEnv(&argv, &envp); - uptr rv = internal_execve("/proc/self/exe", argv, envp); + uptr rv = internal_execve(pathname, argv, envp); int rverrno; CHECK_EQ(internal_iserror(rv, &rverrno), true); Printf("execve failed, errno %d\n", rverrno); Die(); } +#if !SANITIZER_SOLARIS enum MutexState { MtxUnlocked = 0, MtxLocked = 1, @@ -547,6 +639,8 @@ void BlockingMutex::Lock() { while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { #if SANITIZER_FREEBSD _umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0); +#elif SANITIZER_NETBSD + sched_yield(); /* No userspace futex-like synchronization */ #else internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT, MtxSleeping, 0, 0, 0); #endif @@ -560,6 +654,8 @@ void BlockingMutex::Unlock() { if (v == MtxSleeping) { #if SANITIZER_FREEBSD _umtx_op(m, UMTX_OP_WAKE, 1, 0, 0); +#elif SANITIZER_NETBSD + /* No userspace futex-like synchronization */ #else internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE, 1, 0, 0, 0); #endif @@ -570,11 +666,23 @@ void BlockingMutex::CheckLocked() { atomic_uint32_t *m = reinterpret_cast(&opaque_storage_); CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); } +#endif // !SANITIZER_SOLARIS // ----------------- sanitizer_linux.h // The actual size of this structure is specified by d_reclen. // Note that getdents64 uses a different structure format. We only provide the // 32-bit syscall here. +#if SANITIZER_NETBSD +// struct dirent is different for Linux and us. At this moment, we use only +// d_fileno (Linux call this d_ino), d_reclen, and d_name. +struct linux_dirent { + u64 d_ino; // d_fileno + u16 d_reclen; + u16 d_namlen; // not used + u8 d_type; // not used + char d_name[NAME_MAX + 1]; +}; +#else struct linux_dirent { #if SANITIZER_X32 || defined(__aarch64__) u64 d_ino; @@ -589,15 +697,29 @@ struct linux_dirent { #endif char d_name[256]; }; +#endif +#if !SANITIZER_SOLARIS // Syscall wrappers. uptr internal_ptrace(int request, int pid, void *addr, void *data) { +#if SANITIZER_NETBSD + // XXX We need additional work for ptrace: + // - for request, we use PT_FOO whereas Linux uses PTRACE_FOO + // - data is int for us, but void * for Linux + // - Linux sometimes uses data in the case where we use addr instead + // At this moment, this function is used only within + // "#if SANITIZER_LINUX && defined(__x86_64__)" block in + // sanitizer_stoptheworld_linux_libcdep.cc. + return internal_syscall_ptr(SYSCALL(ptrace), request, pid, (uptr)addr, + (uptr)data); +#else return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr, (uptr)data); +#endif } uptr internal_waitpid(int pid, int *status, int options) { - return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options, + return internal_syscall_ptr(SYSCALL(wait4), pid, (uptr)status, options, 0 /* rusage */); } @@ -615,12 +737,16 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) { #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count); #else - return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count); + return internal_syscall_ptr(SYSCALL(getdents), fd, (uptr)dirp, count); #endif } uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { +#if SANITIZER_NETBSD + return internal_syscall64(SYSCALL(lseek), fd, 0, offset, whence); +#else return internal_syscall(SYSCALL(lseek), fd, offset, whence); +#endif } #if SANITIZER_LINUX @@ -630,7 +756,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { #endif uptr internal_sigaltstack(const void *ss, void *oss) { - return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); + return internal_syscall_ptr(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); } int internal_fork() { @@ -711,8 +837,8 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact) { uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { -#if SANITIZER_FREEBSD - return internal_syscall(SYSCALL(sigprocmask), how, set, oldset); +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + return internal_syscall_ptr(SYSCALL(sigprocmask), how, set, oldset); #else __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; __sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset; @@ -751,6 +877,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { return k_set->sig[idx] & (1 << bit); } #endif // SANITIZER_LINUX +#endif // !SANITIZER_SOLARIS // ThreadLister implementation. ThreadLister::ThreadLister(int pid) @@ -856,7 +983,9 @@ static uptr GetKernelAreaSize() { #endif // SANITIZER_WORDSIZE == 32 uptr GetMaxVirtualAddress() { -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_NETBSD && defined(__x86_64__) + return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE) +#elif SANITIZER_WORDSIZE == 64 # if defined(__powerpc64__) || defined(__aarch64__) // On PowerPC64 we have two different address space layouts: 44- and 46-bit. // We somehow need to figure out which one we are using now and choose @@ -877,15 +1006,21 @@ uptr GetMaxVirtualAddress() { # if defined(__s390__) return (1ULL << 31) - 1; // 0x7fffffff; # else - uptr res = (1ULL << 32) - 1; // 0xffffffff; - if (!common_flags()->full_address_space) - res -= GetKernelAreaSize(); - CHECK_LT(reinterpret_cast(&res), res); - return res; + return (1ULL << 32) - 1; // 0xffffffff; # endif #endif // SANITIZER_WORDSIZE } +uptr GetMaxUserVirtualAddress() { + uptr addr = GetMaxVirtualAddress(); +#if SANITIZER_WORDSIZE == 32 && !defined(__s390__) + if (!common_flags()->full_address_space) + addr -= GetKernelAreaSize(); + CHECK_LT(reinterpret_cast(&addr), addr); +#endif + return addr; +} + uptr GetPageSize() { // Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. #if SANITIZER_ANDROID @@ -900,8 +1035,17 @@ uptr GetPageSize() { } uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { +#if SANITIZER_SOLARIS + const char *default_module_name = getexecname(); + CHECK_NE(default_module_name, NULL); + return internal_snprintf(buf, buf_len, "%s", default_module_name); +#else +#if SANITIZER_FREEBSD || SANITIZER_NETBSD #if SANITIZER_FREEBSD - const int Mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; +#else + const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME}; +#endif const char *default_module_name = "kern.proc.pathname"; size_t Size = buf_len; bool IsErr = (sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0); @@ -913,7 +1057,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { default_module_name, buf, buf_len); int readlink_error; bool IsErr = internal_iserror(module_name_len, &readlink_error); -#endif +#endif // SANITIZER_SOLARIS if (IsErr) { // We can't read binary name for some reason, assume it's unknown. Report("WARNING: reading executable name failed with errno %d, " @@ -923,6 +1067,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { CHECK_LT(module_name_len, buf_len); } return module_name_len; +#endif } uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { @@ -1539,12 +1684,17 @@ static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) { } #endif -SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { +SignalContext::WriteFlag SignalContext::GetWriteFlag() const { ucontext_t *ucontext = (ucontext_t *)context; #if defined(__x86_64__) || defined(__i386__) static const uptr PF_WRITE = 1U << 1; #if SANITIZER_FREEBSD uptr err = ucontext->uc_mcontext.mc_err; +#elif SANITIZER_NETBSD + uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR]; +#elif SANITIZER_SOLARIS && defined(__i386__) +# define ERR 13 + uptr err = ucontext->uc_mcontext.gregs[ERR]; #else uptr err = ucontext->uc_mcontext.gregs[REG_ERR]; #endif @@ -1558,6 +1708,12 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { u64 esr; if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN; return esr & ESR_ELx_WNR ? WRITE : READ; +#elif SANITIZER_SOLARIS && defined(__sparc__) + // Decode the instruction to determine the access type. + // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype). + uptr pc = ucontext->uc_mcontext.gregs[REG_PC]; + u32 instr = *(u32 *)pc; + return (instr >> 21) & 1 ? WRITE: READ; #else (void)ucontext; return UNKNOWN; // FIXME: Implement. @@ -1568,7 +1724,7 @@ void SignalContext::DumpAllRegisters(void *context) { // FIXME: Implement this. } -void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { +static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #if defined(__arm__) ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.arm_pc; @@ -1591,6 +1747,11 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.mc_rip; *bp = ucontext->uc_mcontext.mc_rbp; *sp = ucontext->uc_mcontext.mc_rsp; +#elif SANITIZER_NETBSD + ucontext_t *ucontext = (ucontext_t *)context; + *pc = ucontext->uc_mcontext.__gregs[_REG_RIP]; + *bp = ucontext->uc_mcontext.__gregs[_REG_RBP]; + *sp = ucontext->uc_mcontext.__gregs[_REG_RSP]; # else ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.gregs[REG_RIP]; @@ -1603,8 +1764,26 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.mc_eip; *bp = ucontext->uc_mcontext.mc_ebp; *sp = ucontext->uc_mcontext.mc_esp; +#elif SANITIZER_NETBSD + ucontext_t *ucontext = (ucontext_t *)context; + *pc = ucontext->uc_mcontext.__gregs[_REG_EIP]; + *bp = ucontext->uc_mcontext.__gregs[_REG_EBP]; + *sp = ucontext->uc_mcontext.__gregs[_REG_ESP]; # else ucontext_t *ucontext = (ucontext_t*)context; +# if SANITIZER_SOLARIS + /* Use the numeric values: the symbolic ones are undefined by llvm + include/llvm/Support/Solaris.h. */ +# ifndef REG_EIP +# define REG_EIP 14 // REG_PC +# endif +# ifndef REG_EBP +# define REG_EBP 6 // REG_FP +# endif +# ifndef REG_ESP +# define REG_ESP 17 // REG_SP +# endif +# endif *pc = ucontext->uc_mcontext.gregs[REG_EIP]; *bp = ucontext->uc_mcontext.gregs[REG_EBP]; *sp = ucontext->uc_mcontext.gregs[REG_ESP]; @@ -1619,7 +1798,16 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #elif defined(__sparc__) ucontext_t *ucontext = (ucontext_t*)context; uptr *stk_ptr; -# if defined (__arch64__) +# if defined (__sparcv9) +# ifndef MC_PC +# define MC_PC REG_PC +# endif +# ifndef MC_O6 +# define MC_O6 REG_O6 +# endif +# ifdef SANITIZER_SOLARIS +# define mc_gregs gregs +# endif *pc = ucontext->uc_mcontext.mc_gregs[MC_PC]; *sp = ucontext->uc_mcontext.mc_gregs[MC_O6]; stk_ptr = (uptr *) (*sp + 2047); @@ -1649,6 +1837,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #endif } +void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } + void MaybeReexec() { // No need to re-exec on Linux. } @@ -1676,25 +1866,27 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding, return 0; } -bool GetRandom(void *buffer, uptr length) { +bool GetRandom(void *buffer, uptr length, bool blocking) { if (!buffer || !length || length > 256) return false; -#if defined(__NR_getrandom) +#if SANITIZER_USE_GETRANDOM static atomic_uint8_t skip_getrandom_syscall; if (!atomic_load_relaxed(&skip_getrandom_syscall)) { // Up to 256 bytes, getrandom will not be interrupted. - uptr res = internal_syscall(SYSCALL(getrandom), buffer, length, 0); + uptr res = internal_syscall(SYSCALL(getrandom), buffer, length, + blocking ? 0 : GRND_NONBLOCK); int rverrno = 0; if (internal_iserror(res, &rverrno) && rverrno == ENOSYS) atomic_store_relaxed(&skip_getrandom_syscall, 1); else if (res == length) return true; } -#endif +#endif // SANITIZER_USE_GETRANDOM + // Up to 256 bytes, a read off /dev/urandom will not be interrupted. + // blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom. uptr fd = internal_open("/dev/urandom", O_RDONLY); if (internal_iserror(fd)) return false; - // internal_read deals with EINTR. uptr res = internal_read(fd, buffer, length); if (internal_iserror(res)) return false; @@ -1704,4 +1896,6 @@ bool GetRandom(void *buffer, uptr length) { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS + diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 11cad6b80933..2d227f82a99e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -14,11 +14,14 @@ #define SANITIZER_LINUX_H #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" -#include "sanitizer_posix.h" +#include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_posix.h" +#include "sanitizer_platform_limits_solaris.h" +#include "sanitizer_posix.h" struct link_map; // Opaque type returned by dlopen(). @@ -27,11 +30,25 @@ namespace __sanitizer { // the one in , which is used by readdir(). struct linux_dirent; +struct ProcSelfMapsBuff { + char *data; + uptr mmaped_size; + uptr len; +}; + +struct MemoryMappingLayoutData { + ProcSelfMapsBuff proc_self_maps; + const char *current; +}; + +void ReadProcMaps(ProcSelfMapsBuff *proc_maps); + // Syscall wrappers. uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count); uptr internal_sigaltstack(const void* ss, void* oss); uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp); // Linux-only syscalls. #if SANITIZER_LINUX @@ -128,5 +145,6 @@ ALWAYS_INLINE uptr *get_android_tls_ptr() { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS #endif // SANITIZER_LINUX_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 52196db12731..56fdfc8705f3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -14,11 +14,13 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_allocator_internal.h" #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_freebsd.h" #include "sanitizer_linux.h" @@ -36,19 +38,35 @@ #if SANITIZER_FREEBSD #include #include +#include #define pthread_getattr_np pthread_attr_get_np #endif +#if SANITIZER_NETBSD +#include +#include +#endif + +#if SANITIZER_SOLARIS +#include +#endif + #if SANITIZER_LINUX #include #endif #if SANITIZER_ANDROID #include +#if !defined(CPU_COUNT) && !defined(__aarch64__) +#include +#include +struct __sanitizer::linux_dirent { + long d_ino; + off_t d_off; + unsigned short d_reclen; + char d_name[]; +}; #endif - -#if SANITIZER_ANDROID && __ANDROID_API__ < 21 -#include #endif #if !SANITIZER_ANDROID @@ -102,13 +120,20 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, *stack_bottom = segment.end - stacksize; return; } + uptr stacksize = 0; + void *stackaddr = nullptr; +#if SANITIZER_SOLARIS + stack_t ss; + CHECK_EQ(thr_stksegment(&ss), 0); + stacksize = ss.ss_size; + stackaddr = (char *)ss.ss_sp - stacksize; +#else // !SANITIZER_SOLARIS pthread_attr_t attr; pthread_attr_init(&attr); CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); - uptr stacksize = 0; - void *stackaddr = nullptr; my_pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); +#endif // SANITIZER_SOLARIS *stack_top = (uptr)stackaddr + stacksize; *stack_bottom = (uptr)stackaddr; @@ -148,7 +173,8 @@ bool SanitizerGetThreadName(char *name, int max_len) { #endif } -#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \ + !SANITIZER_NETBSD && !SANITIZER_SOLARIS static uptr g_tls_size; #ifdef __i386__ @@ -176,7 +202,8 @@ void InitTlsSize() { } #else void InitTlsSize() { } -#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO +#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && + // !SANITIZER_NETBSD && !SANITIZER_SOLARIS #if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) \ || defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) \ @@ -323,7 +350,7 @@ static void **ThreadSelfSegbase() { // sysarch(AMD64_GET_FSBASE, segbase); __asm __volatile("movq %%fs:0, %0" : "=r" (segbase)); # else -# error "unsupported CPU arch for FreeBSD platform" +# error "unsupported CPU arch" # endif return segbase; } @@ -333,6 +360,35 @@ uptr ThreadSelf() { } #endif // SANITIZER_FREEBSD +#if SANITIZER_NETBSD +static struct tls_tcb * ThreadSelfTlsTcb() { + struct tls_tcb * tcb; +# ifdef __HAVE___LWP_GETTCB_FAST + tcb = (struct tls_tcb *)__lwp_gettcb_fast(); +# elif defined(__HAVE___LWP_GETPRIVATE_FAST) + tcb = (struct tls_tcb *)__lwp_getprivate_fast(); +# endif + return tcb; +} + +uptr ThreadSelf() { + return (uptr)ThreadSelfTlsTcb()->tcb_pthread; +} + +int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) { + const Elf_Phdr *hdr = info->dlpi_phdr; + const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum; + + for (; hdr != last_hdr; ++hdr) { + if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) { + *(uptr*)data = hdr->p_memsz; + break; + } + } + return 0; +} +#endif // SANITIZER_NETBSD + #if !SANITIZER_GO static void GetTls(uptr *addr, uptr *size) { #if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -362,9 +418,27 @@ static void GetTls(uptr *addr, uptr *size) { *addr = (uptr) dtv[2]; *size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]); } +#elif SANITIZER_NETBSD + struct tls_tcb * const tcb = ThreadSelfTlsTcb(); + *addr = 0; + *size = 0; + if (tcb != 0) { + // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program). + // ld.elf_so hardcodes the index 1. + dl_iterate_phdr(GetSizeFromHdr, size); + + if (*size != 0) { + // The block has been found and tcb_dtv[1] contains the base address + *addr = (uptr)tcb->tcb_dtv[1]; + } + } #elif SANITIZER_ANDROID *addr = 0; *size = 0; +#elif SANITIZER_SOLARIS + // FIXME + *addr = 0; + *size = 0; #else # error "Unknown OS" #endif @@ -373,7 +447,8 @@ static void GetTls(uptr *addr, uptr *size) { #if !SANITIZER_GO uptr GetTlsSize() { -#if SANITIZER_FREEBSD || SANITIZER_ANDROID +#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS uptr addr, size; GetTls(&addr, &size); return size; @@ -419,7 +494,7 @@ typedef ElfW(Phdr) Elf_Phdr; # endif struct DlIteratePhdrData { - InternalMmapVector *modules; + InternalMmapVectorNoCtor *modules; bool first; }; @@ -457,21 +532,41 @@ extern "C" __attribute__((weak)) int dl_iterate_phdr( int (*)(struct dl_phdr_info *, size_t, void *), void *); #endif -void ListOfModules::init() { - clear(); +static bool requiresProcmaps() { #if SANITIZER_ANDROID && __ANDROID_API__ <= 22 - u32 api_level = AndroidGetApiLevel(); // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken. // The runtime check allows the same library to work with // both K and L (and future) Android releases. - if (api_level <= ANDROID_LOLLIPOP_MR1) { // L or earlier - MemoryMappingLayout memory_mapping(false); - memory_mapping.DumpListOfModules(&modules_); - return; - } + return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1; +#else + return false; #endif - DlIteratePhdrData data = {&modules_, true}; - dl_iterate_phdr(dl_iterate_phdr_cb, &data); +} + +static void procmapsInit(InternalMmapVectorNoCtor *modules) { + MemoryMappingLayout memory_mapping(/*cache_enabled*/true); + memory_mapping.DumpListOfModules(modules); +} + +void ListOfModules::init() { + clearOrInit(); + if (requiresProcmaps()) { + procmapsInit(&modules_); + } else { + DlIteratePhdrData data = {&modules_, true}; + dl_iterate_phdr(dl_iterate_phdr_cb, &data); + } +} + +// When a custom loader is used, dl_iterate_phdr may not contain the full +// list of modules. Allow callers to fall back to using procmaps. +void ListOfModules::fallbackInit() { + if (!requiresProcmaps()) { + clearOrInit(); + procmapsInit(&modules_); + } else { + clear(); + } } // getrusage does not give us the current RSS, only the max RSS. @@ -513,12 +608,65 @@ uptr GetRSS() { return rss * GetPageSizeCached(); } -// 64-bit Android targets don't provide the deprecated __android_log_write. -// Starting with the L release, syslog() works and is preferable to -// __android_log_write. +// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as +// they allocate memory. +u32 GetNumberOfCPUs() { +#if SANITIZER_FREEBSD || SANITIZER_NETBSD + u32 ncpu; + int req[2]; + size_t len = sizeof(ncpu); + req[0] = CTL_HW; + req[1] = HW_NCPU; + CHECK_EQ(sysctl(req, 2, &ncpu, &len, NULL, 0), 0); + return ncpu; +#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__) + // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't + // exist in sched.h. That is the case for toolchains generated with older + // NDKs. + // This code doesn't work on AArch64 because internal_getdents makes use of + // the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64. + uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY); + if (internal_iserror(fd)) + return 0; + InternalScopedBuffer buffer(4096); + uptr bytes_read = buffer.size(); + uptr n_cpus = 0; + u8 *d_type; + struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read]; + while (true) { + if ((u8 *)entry >= &buffer[bytes_read]) { + bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(), + buffer.size()); + if (internal_iserror(bytes_read) || !bytes_read) + break; + entry = (struct linux_dirent *)buffer.data(); + } + d_type = (u8 *)entry + entry->d_reclen - 1; + if (d_type >= &buffer[bytes_read] || + (u8 *)&entry->d_name[3] >= &buffer[bytes_read]) + break; + if (entry->d_ino != 0 && *d_type == DT_DIR) { + if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' && + entry->d_name[2] == 'u' && + entry->d_name[3] >= '0' && entry->d_name[3] <= '9') + n_cpus++; + } + entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen); + } + internal_close(fd); + return n_cpus; +#elif SANITIZER_SOLARIS + return sysconf(_SC_NPROCESSORS_ONLN); +#else + cpu_set_t CPUs; + CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); + return CPU_COUNT(&CPUs); +#endif +} + #if SANITIZER_LINUX -#if SANITIZER_ANDROID +# if SANITIZER_ANDROID static atomic_uint8_t android_log_initialized; void AndroidLogInit() { @@ -529,34 +677,97 @@ void AndroidLogInit() { static bool ShouldLogAfterPrintf() { return atomic_load(&android_log_initialized, memory_order_acquire); } -#else + +extern "C" SANITIZER_WEAK_ATTRIBUTE +int async_safe_write_log(int pri, const char* tag, const char* msg); +extern "C" SANITIZER_WEAK_ATTRIBUTE +int __android_log_write(int prio, const char* tag, const char* msg); + +// ANDROID_LOG_INFO is 4, but can't be resolved at runtime. +#define SANITIZER_ANDROID_LOG_INFO 4 + +// async_safe_write_log is a new public version of __libc_write_log that is +// used behind syslog. It is preferable to syslog as it will not do any dynamic +// memory allocation or formatting. +// If the function is not available, syslog is preferred for L+ (it was broken +// pre-L) as __android_log_write triggers a racey behavior with the strncpy +// interceptor. Fallback to __android_log_write pre-L. +void WriteOneLineToSyslog(const char *s) { + if (&async_safe_write_log) { + async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s); + } else if (AndroidGetApiLevel() > ANDROID_KITKAT) { + syslog(LOG_INFO, "%s", s); + } else { + CHECK(&__android_log_write); + __android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s); + } +} + +extern "C" SANITIZER_WEAK_ATTRIBUTE +void android_set_abort_message(const char *); + +void SetAbortMessage(const char *str) { + if (&android_set_abort_message) + android_set_abort_message(str); +} +# else void AndroidLogInit() {} static bool ShouldLogAfterPrintf() { return true; } -#endif // SANITIZER_ANDROID -void WriteOneLineToSyslog(const char *s) { -#if SANITIZER_ANDROID &&__ANDROID_API__ < 21 - __android_log_write(ANDROID_LOG_INFO, NULL, s); -#else - syslog(LOG_INFO, "%s", s); -#endif -} +void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); } + +void SetAbortMessage(const char *str) {} +# endif // SANITIZER_ANDROID void LogMessageOnPrintf(const char *str) { if (common_flags()->log_to_syslog && ShouldLogAfterPrintf()) WriteToSyslog(str); } -#if SANITIZER_ANDROID && __ANDROID_API__ >= 21 -extern "C" void android_set_abort_message(const char *msg); -void SetAbortMessage(const char *str) { android_set_abort_message(str); } -#else -void SetAbortMessage(const char *str) {} -#endif +#endif // SANITIZER_LINUX -#endif // SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_GO +// glibc crashes when using clock_gettime from a preinit_array function as the +// vDSO function pointers haven't been initialized yet. __progname is +// initialized after the vDSO function pointers, so if it exists, is not null +// and is not empty, we can use clock_gettime. +extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; +INLINE bool CanUseVDSO() { + // Bionic is safe, it checks for the vDSO function pointers to be initialized. + if (SANITIZER_ANDROID) + return true; + if (&__progname && __progname && *__progname) + return true; + return false; +} + +// MonotonicNanoTime is a timing function that can leverage the vDSO by calling +// clock_gettime. real_clock_gettime only exists if clock_gettime is +// intercepted, so define it weakly and use it if available. +extern "C" SANITIZER_WEAK_ATTRIBUTE +int real_clock_gettime(u32 clk_id, void *tp); +u64 MonotonicNanoTime() { + timespec ts; + if (CanUseVDSO()) { + if (&real_clock_gettime) + real_clock_gettime(CLOCK_MONOTONIC, &ts); + else + clock_gettime(CLOCK_MONOTONIC, &ts); + } else { + internal_clock_gettime(CLOCK_MONOTONIC, &ts); + } + return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec; +} +#else +// Non-Linux & Go always use the syscall. +u64 MonotonicNanoTime() { + timespec ts; + internal_clock_gettime(CLOCK_MONOTONIC, &ts); + return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec; +} +#endif // SANITIZER_LINUX && !SANITIZER_GO } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc index 1edd4157fd6b..e1c51f580874 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -23,6 +23,7 @@ #include #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" @@ -184,7 +185,7 @@ uptr internal_getpid() { int internal_sigaction(int signum, const void *act, void *oldact) { return sigaction(signum, - (struct sigaction *)act, (struct sigaction *)oldact); + (const struct sigaction *)act, (struct sigaction *)oldact); } void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); } @@ -364,6 +365,10 @@ u64 NanoTime() { return 0; } +u64 MonotonicNanoTime() { + return 0; +} + uptr GetTlsSize() { return 0; } @@ -410,11 +415,13 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, } void ListOfModules::init() { - clear(); + clearOrInit(); MemoryMappingLayout memory_mapping(false); memory_mapping.DumpListOfModules(&modules_); } +void ListOfModules::fallbackInit() { clear(); } + static HandleSignalMode GetHandleSignalModeImpl(int signum) { switch (signum) { case SIGABRT: @@ -573,7 +580,7 @@ void LogFullErrorReport(const char *buffer) { #endif } -SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { +SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if defined(__x86_64__) || defined(__i386__) ucontext_t *ucontext = static_cast(context); return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ; @@ -582,7 +589,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { #endif } -void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { +static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { ucontext_t *ucontext = (ucontext_t*)context; # if defined(__aarch64__) *pc = ucontext->uc_mcontext->__ss.__pc; @@ -609,6 +616,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { # endif } +void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } + #if !SANITIZER_GO static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; @@ -736,6 +745,9 @@ void MaybeReexec() { if (!lib_is_in_env) return; + if (!common_flags()->strip_env) + return; + // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove // the dylib from the environment variable, because interceptors are installed // and we don't want our children to inherit the variable. @@ -844,7 +856,7 @@ uptr GetTaskInfoMaxAddress() { } #endif -uptr GetMaxVirtualAddress() { +uptr GetMaxUserVirtualAddress() { #if SANITIZER_WORDSIZE == 64 # if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM // Get the maximum VM address @@ -859,6 +871,10 @@ uptr GetMaxVirtualAddress() { #endif // SANITIZER_WORDSIZE } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + uptr FindAvailableMemoryRange(uptr shadow_size, uptr alignment, uptr left_padding, @@ -881,6 +897,11 @@ uptr FindAvailableMemoryRange(uptr shadow_size, mach_msg_type_number_t count = kRegionInfoSize; kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth, (vm_region_info_t)&vminfo, &count); + if (kr == KERN_INVALID_ADDRESS) { + // No more regions beyond "address", consider the gap at the end of VM. + address = GetMaxVirtualAddress() + 1; + vmsize = 0; + } if (free_begin != address) { // We found a free region [free_begin..address-1]. uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment); @@ -991,7 +1012,12 @@ void CheckNoDeepBind(const char *filename, int flag) { } // FIXME: implement on this platform. -bool GetRandom(void *buffer, uptr length) { +bool GetRandom(void *buffer, uptr length, bool blocking) { + UNIMPLEMENTED(); +} + +// FIXME: implement on this platform. +u32 GetNumberOfCPUs() { UNIMPLEMENTED(); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index 3f1c68c8610a..e022a2c0363c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -20,6 +20,17 @@ namespace __sanitizer { +struct MemoryMappingLayoutData { + int current_image; + u32 current_magic; + u32 current_filetype; + ModuleArch current_arch; + u8 current_uuid[kModuleUUIDSize]; + int current_load_cmd_count; + const char *current_load_cmd_addr; + bool current_instrumented; +}; + enum MacosVersion { MACOS_VERSION_UNINITIALIZED = 0, MACOS_VERSION_UNKNOWN, diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc index c95daa9372ad..090f17c888b4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc @@ -20,7 +20,7 @@ namespace __sanitizer { void RestrictMemoryToMaxAddress(uptr max_address) { - uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address; + uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address; void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap"); CHECK(res != MAP_FAILED); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h index 1759bf13f689..4bb878ee19dc 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h @@ -92,8 +92,10 @@ class BlockingMutex { // checks that the mutex is owned, and assumes callers to be generally // well-behaved. void CheckLocked(); + private: - uptr opaque_storage_[10]; + // Solaris mutex_t has a member that requires 64-bit alignment. + ALIGNED(8) uptr opaque_storage_[10]; uptr owner_; // for debugging }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index 396f7c9346d6..334903c26f9f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -14,7 +14,8 @@ #define SANITIZER_PLATFORM_H #if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \ - !defined(__APPLE__) && !defined(_WIN32) + !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \ + !(defined(__sun__) && defined(__svr4__)) # error "This operating system is not supported" #endif @@ -36,6 +37,12 @@ # define SANITIZER_NETBSD 0 #endif +#if defined(__sun__) && defined(__svr4__) +# define SANITIZER_SOLARIS 1 +#else +# define SANITIZER_SOLARIS 0 +#endif + #if defined(__APPLE__) # define SANITIZER_MAC 1 # include @@ -44,7 +51,7 @@ # else # define SANITIZER_IOS 0 # endif -# if TARGET_IPHONE_SIMULATOR +# if TARGET_OS_SIMULATOR # define SANITIZER_IOSSIM 1 # else # define SANITIZER_IOSSIM 0 @@ -85,8 +92,15 @@ # define SANITIZER_ANDROID 0 #endif +#if defined(__Fuchsia__) +# define SANITIZER_FUCHSIA 1 +#else +# define SANITIZER_FUCHSIA 0 +#endif + #define SANITIZER_POSIX \ - (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD) + (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \ + SANITIZER_NETBSD || SANITIZER_SOLARIS) #if __LP64__ || defined(_WIN64) # define SANITIZER_WORDSIZE 64 @@ -175,13 +189,19 @@ # define SANITIZER_ARM 0 #endif +#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32 +# define SANITIZER_SOLARIS32 1 +#else +# define SANITIZER_SOLARIS32 0 +#endif + // By default we allow to use SizeClassAllocator64 on 64-bit platform. // But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64 // does not work well and we need to fallback to SizeClassAllocator32. // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here. #ifndef SANITIZER_CAN_USE_ALLOCATOR64 -# if SANITIZER_ANDROID && defined(__aarch64__) +# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA # define SANITIZER_CAN_USE_ALLOCATOR64 1 # elif defined(__mips64) || defined(__aarch64__) # define SANITIZER_CAN_USE_ALLOCATOR64 0 @@ -276,5 +296,10 @@ # define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0 #endif +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS +# define SANITIZER_MADVISE_DONTNEED MADV_FREE +#else +# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED +#endif #endif // SANITIZER_PLATFORM_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index 0380cee92a00..b99ac4480586 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -16,21 +16,28 @@ #include "sanitizer_internal_defs.h" -#if !SANITIZER_WINDOWS -# define SI_WINDOWS 0 -# define SI_NOT_WINDOWS 1 -# include "sanitizer_platform_limits_posix.h" -#else -# define SI_WINDOWS 1 -# define SI_NOT_WINDOWS 0 -#endif - #if SANITIZER_POSIX # define SI_POSIX 1 #else # define SI_POSIX 0 #endif +#if !SANITIZER_WINDOWS +# define SI_WINDOWS 0 +#else +# define SI_WINDOWS 1 +#endif + +#if (SI_POSIX != 0) == (SI_WINDOWS != 0) && !SANITIZER_FUCHSIA +# error "Windows is not POSIX!" +#endif + +#if SI_POSIX +# include "sanitizer_platform_limits_netbsd.h" +# include "sanitizer_platform_limits_posix.h" +# include "sanitizer_platform_limits_solaris.h" +#endif + #if SANITIZER_LINUX && !SANITIZER_ANDROID # define SI_LINUX_NOT_ANDROID 1 #else @@ -75,10 +82,28 @@ # define SI_IOS 0 #endif -#if !SANITIZER_WINDOWS && !SANITIZER_MAC -# define SI_UNIX_NOT_MAC 1 +#if SANITIZER_FUCHSIA +# define SI_NOT_FUCHSIA 0 #else -# define SI_UNIX_NOT_MAC 0 +# define SI_NOT_FUCHSIA 1 +#endif + +#if SANITIZER_SOLARIS +# define SI_SOLARIS 1 +#else +# define SI_SOLARIS 0 +#endif + +#if SANITIZER_SOLARIS32 +# define SI_SOLARIS32 1 +#else +# define SI_SOLARIS32 0 +#endif + +#if SANITIZER_POSIX && !SANITIZER_MAC +# define SI_POSIX_NOT_MAC 1 +#else +# define SI_POSIX_NOT_MAC 0 #endif #if SANITIZER_LINUX && !SANITIZER_FREEBSD @@ -87,23 +112,23 @@ # define SI_LINUX_NOT_FREEBSD 0 #endif -#define SANITIZER_INTERCEPT_STRLEN 1 -#define SANITIZER_INTERCEPT_STRNLEN SI_NOT_MAC -#define SANITIZER_INTERCEPT_STRCMP 1 -#define SANITIZER_INTERCEPT_STRSTR 1 -#define SANITIZER_INTERCEPT_STRCASESTR SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_STRTOK 1 -#define SANITIZER_INTERCEPT_STRCHR 1 -#define SANITIZER_INTERCEPT_STRCHRNUL SI_UNIX_NOT_MAC -#define SANITIZER_INTERCEPT_STRRCHR 1 -#define SANITIZER_INTERCEPT_STRSPN 1 -#define SANITIZER_INTERCEPT_STRPBRK 1 -#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA) +#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRCASESTR SI_POSIX +#define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRCHR SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRCHRNUL SI_POSIX_NOT_MAC +#define SANITIZER_INTERCEPT_STRRCHR SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRSPN SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_STRPBRK SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX #define SANITIZER_INTERCEPT_MEMSET 1 #define SANITIZER_INTERCEPT_MEMMOVE 1 #define SANITIZER_INTERCEPT_MEMCPY 1 -#define SANITIZER_INTERCEPT_MEMCMP 1 +#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA #define SANITIZER_INTERCEPT_STRNDUP SI_POSIX #define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD #if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ @@ -114,24 +139,23 @@ #endif // memmem on Darwin doesn't exist on 10.6 // FIXME: enable memmem on Windows. -#define SANITIZER_INTERCEPT_MEMMEM \ - (SI_NOT_WINDOWS && !SI_MAC_DEPLOYMENT_BELOW_10_7) -#define SANITIZER_INTERCEPT_MEMCHR 1 +#define SANITIZER_INTERCEPT_MEMMEM (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_7) +#define SANITIZER_INTERCEPT_MEMCHR SI_NOT_FUCHSIA #define SANITIZER_INTERCEPT_MEMRCHR (SI_FREEBSD || SI_LINUX || SI_NETBSD) -#define SANITIZER_INTERCEPT_READ SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PREAD SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_WRITE SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PWRITE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_READ SI_POSIX +#define SANITIZER_INTERCEPT_PREAD SI_POSIX +#define SANITIZER_INTERCEPT_WRITE SI_POSIX +#define SANITIZER_INTERCEPT_PWRITE SI_POSIX -#define SANITIZER_INTERCEPT_FREAD SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_FWRITE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_FREAD SI_POSIX +#define SANITIZER_INTERCEPT_FWRITE SI_POSIX -#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 +#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 -#define SANITIZER_INTERCEPT_READV SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_WRITEV SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_READV SI_POSIX +#define SANITIZER_INTERCEPT_WRITEV SI_POSIX #define SANITIZER_INTERCEPT_PREADV \ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) @@ -141,58 +165,68 @@ #define SANITIZER_INTERCEPT_PRCTL SI_LINUX -#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_STRPTIME SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_POSIX +#define SANITIZER_INTERCEPT_STRPTIME SI_POSIX -#define SANITIZER_INTERCEPT_SCANF SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SCANF SI_POSIX #define SANITIZER_INTERCEPT_ISOC99_SCANF SI_LINUX_NOT_ANDROID #ifndef SANITIZER_INTERCEPT_PRINTF -# define SANITIZER_INTERCEPT_PRINTF SI_NOT_WINDOWS +# define SANITIZER_INTERCEPT_PRINTF SI_POSIX # define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD) # define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID #endif -#define SANITIZER_INTERCEPT_FREXP 1 -#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT___PRINTF_CHK \ + (SANITIZER_INTERCEPT_PRINTF && SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA +#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_POSIX + +#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX #define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETPWENT \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_GETPWENT_R \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SETPWENT (SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_CLOCK_GETTIME (SI_FREEBSD || SI_NETBSD || SI_LINUX) -#define SANITIZER_INTERCEPT_GETITIMER SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_TIME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_WAIT SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_INET SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETADDRINFO SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETNAMEINFO SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETSOCKNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R (SI_FREEBSD || SI_LINUX) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_SETPWENT \ + (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_CLOCK_GETTIME \ + (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS) +#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX +#define SANITIZER_INTERCEPT_TIME SI_POSIX +#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_GLOB64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WAIT SI_POSIX +#define SANITIZER_INTERCEPT_INET SI_POSIX +#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX +#define SANITIZER_INTERCEPT_GETADDRINFO SI_POSIX +#define SANITIZER_INTERCEPT_GETNAMEINFO SI_POSIX +#define SANITIZER_INTERCEPT_GETSOCKNAME SI_POSIX +#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_POSIX +#define SANITIZER_INTERCEPT_GETHOSTBYNAME2 SI_POSIX && !SI_SOLARIS +#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R \ + (SI_FREEBSD || SI_LINUX || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R \ (SI_FREEBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_GETSOCKOPT SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_ACCEPT SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R \ + (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_GETHOSTENT_R \ + (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX +#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX #define SANITIZER_INTERCEPT_ACCEPT4 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_MODF SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_RECVMSG SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SENDMSG SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETPEERNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_IOCTL SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_INET_ATON SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MODF SI_POSIX +#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX +#define SANITIZER_INTERCEPT_SENDMSG SI_POSIX +#define SANITIZER_INTERCEPT_GETPEERNAME SI_POSIX +#define SANITIZER_INTERCEPT_IOCTL SI_POSIX +#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX #define SANITIZER_INTERCEPT_SYSINFO SI_LINUX -#define SANITIZER_INTERCEPT_READDIR SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_READDIR SI_POSIX +#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 #if SI_LINUX_NOT_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ @@ -201,116 +235,127 @@ #else #define SANITIZER_INTERCEPT_PTRACE 0 #endif -#define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SETLOCALE SI_POSIX +#define SANITIZER_INTERCEPT_GETCWD SI_POSIX #define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STRTOIMAX SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_MBSTOWCS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_MBSNRTOWCS (SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_WCSTOMBS SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_STRTOIMAX SI_POSIX +#define SANITIZER_INTERCEPT_MBSTOWCS SI_POSIX +#define SANITIZER_INTERCEPT_MBSNRTOWCS \ + (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_WCSTOMBS SI_POSIX #define SANITIZER_INTERCEPT_WCSNRTOMBS \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_WCRTOMB \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_REALPATH SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_REALPATH SI_POSIX +#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \ + (SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_CONFSTR \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STRERROR SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_STRERROR_R SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_STRERROR SI_POSIX +#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX #define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SCANDIR \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_GETGROUPS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_POLL SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 +#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX +#define SANITIZER_INTERCEPT_POLL SI_POSIX +#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_WORDEXP \ - (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SIGWAIT SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID + (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \ + SI_SOLARIS) +#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX +#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_SIGSETOPS \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_SIGPENDING SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SIGPROCMASK SI_NOT_WINDOWS + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX +#define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX #define SANITIZER_INTERCEPT_BACKTRACE \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STATFS \ - (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_STATFS64 \ ((SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_STATVFS \ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_INITGROUPS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX +#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX #define SANITIZER_INTERCEPT_ETHER_HOST \ (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_SHMCTL \ - (SI_NETBSD || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \ + (SI_NETBSD || SI_SOLARIS || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \ SANITIZER_WORDSIZE == 64)) // NOLINT #define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \ + (SI_POSIX && !SI_NETBSD) +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL \ - (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING \ - (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST SI_LINUX_NOT_ANDROID + (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST \ + (SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED \ + (SI_POSIX && !SI_NETBSD) #define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_TMPNAM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_TTYNAME_R SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_TEMPNAM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SINCOS SI_LINUX -#define SANITIZER_INTERCEPT_REMQUO SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_LGAMMA SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX) -#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED \ + (SI_POSIX && !SI_NETBSD) +#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK \ + (SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \ + (SI_LINUX_NOT_ANDROID && !SI_NETBSD) +#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX +#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX +#define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX +#define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS +#define SANITIZER_INTERCEPT_REMQUO SI_POSIX +#define SANITIZER_INTERCEPT_LGAMMA SI_POSIX +#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS) +#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_RAND_R \ - (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_ICONV \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) -#define SANITIZER_INTERCEPT_TIMES SI_NOT_WINDOWS + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_TIMES SI_POSIX // FIXME: getline seems to be available on OSX 10.7 #define SANITIZER_INTERCEPT_GETLINE \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT__EXIT \ - (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC) + (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC || SI_SOLARIS) -#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP \ + (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_TLS_GET_ADDR \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX #define SANITIZER_INTERCEPT_GETXATTR SI_LINUX #define SANITIZER_INTERCEPT_GETRESID SI_LINUX #define SANITIZER_INTERCEPT_GETIFADDRS \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS) #define SANITIZER_INTERCEPT_IF_INDEXTONAME \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS) #define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID #if SI_LINUX && defined(__arm__) #define SANITIZER_INTERCEPT_AEABI_MEM 1 @@ -318,64 +363,71 @@ #define SANITIZER_INTERCEPT_AEABI_MEM 0 #endif #define SANITIZER_INTERCEPT___BZERO SI_MAC -#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_NOT_WINDOWS) -#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_POSIX) +#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_TSEARCH \ - (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD) + (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_FOPEN SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM (SI_LINUX_NOT_ANDROID || SI_NETBSD) +#define SANITIZER_INTERCEPT_FOPEN SI_POSIX +#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32 +#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM \ + (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_FFLUSH SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_FCLOSE SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_FFLUSH SI_POSIX +#define SANITIZER_INTERCEPT_FCLOSE SI_POSIX #ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE #define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \ - (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS) #endif #define SANITIZER_INTERCEPT_GETPASS \ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD) #define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_MLOCKX SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MLOCKX SI_POSIX #define SANITIZER_INTERCEPT_FOPENCOOKIE SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_SEM (SI_LINUX || SI_FREEBSD || SI_NETBSD) -#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD) +#define SANITIZER_INTERCEPT_SEM \ + (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS) +#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX +#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX #define SANITIZER_INTERCEPT_CTERMID \ - (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD) -#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD) + (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_SOLARIS) +#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD || SI_SOLARIS) #define SANITIZER_INTERCEPTOR_HOOKS (SI_LINUX || SI_MAC || SI_WINDOWS) -#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_SEND_SENDTO SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX +#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX #define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX #define SANITIZER_INTERCEPT_STAT \ - (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD) -#define SANITIZER_INTERCEPT___XSTAT \ - (!SANITIZER_INTERCEPT_STAT && SI_NOT_WINDOWS) + (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS) +#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX) #define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT #define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_UTMP (SI_NOT_WINDOWS && !SI_MAC && !SI_FREEBSD) -#define SANITIZER_INTERCEPT_UTMPX (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD) +#define SANITIZER_INTERCEPT_UTMP \ + (SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD) +#define SANITIZER_INTERCEPT_UTMPX \ + (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD) #define SANITIZER_INTERCEPT_GETLOADAVG \ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD) #define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO \ - (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) + (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA) #define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) -#define SANITIZER_INTERCEPT_PVALLOC (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) -#define SANITIZER_INTERCEPT_CFREE (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) +#define SANITIZER_INTERCEPT_PVALLOC \ + (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA) +#define SANITIZER_INTERCEPT_CFREE \ + (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA) #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_WCSCAT SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX +#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA) +#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID #endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc new file mode 100644 index 000000000000..108e196e7a05 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc @@ -0,0 +1,363 @@ +//===-- sanitizer_platform_limits_netbsd.cc -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific NetBSD data structures. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_NETBSD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Include these after system headers to avoid name clashes and ambiguities. +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_netbsd.h" + +namespace __sanitizer { +unsigned struct_utsname_sz = sizeof(struct utsname); +unsigned struct_stat_sz = sizeof(struct stat); +unsigned struct_rusage_sz = sizeof(struct rusage); +unsigned struct_tm_sz = sizeof(struct tm); +unsigned struct_passwd_sz = sizeof(struct passwd); +unsigned struct_group_sz = sizeof(struct group); +unsigned siginfo_t_sz = sizeof(siginfo_t); +unsigned struct_sigaction_sz = sizeof(struct sigaction); +unsigned struct_itimerval_sz = sizeof(struct itimerval); +unsigned pthread_t_sz = sizeof(pthread_t); +unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); +unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); +unsigned pid_t_sz = sizeof(pid_t); +unsigned timeval_sz = sizeof(timeval); +unsigned uid_t_sz = sizeof(uid_t); +unsigned gid_t_sz = sizeof(gid_t); +unsigned mbstate_t_sz = sizeof(mbstate_t); +unsigned sigset_t_sz = sizeof(sigset_t); +unsigned struct_timezone_sz = sizeof(struct timezone); +unsigned struct_tms_sz = sizeof(struct tms); +unsigned struct_sigevent_sz = sizeof(struct sigevent); +unsigned struct_sched_param_sz = sizeof(struct sched_param); +unsigned struct_sockaddr_sz = sizeof(struct sockaddr); +unsigned ucontext_t_sz = sizeof(ucontext_t); +unsigned struct_rlimit_sz = sizeof(struct rlimit); +unsigned struct_timespec_sz = sizeof(struct timespec); +unsigned struct_utimbuf_sz = sizeof(struct utimbuf); +unsigned struct_itimerspec_sz = sizeof(struct itimerspec); +unsigned struct_timex_sz = sizeof(struct timex); +unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); +unsigned struct_mq_attr_sz = sizeof(struct mq_attr); +unsigned struct_statvfs_sz = sizeof(struct statvfs); + +const uptr sig_ign = (uptr)SIG_IGN; +const uptr sig_dfl = (uptr)SIG_DFL; +const uptr sig_err = (uptr)SIG_ERR; +const uptr sa_siginfo = (uptr)SA_SIGINFO; + +int shmctl_ipc_stat = (int)IPC_STAT; + +unsigned struct_utmp_sz = sizeof(struct utmp); +unsigned struct_utmpx_sz = sizeof(struct utmpx); + +int map_fixed = MAP_FIXED; + +int af_inet = (int)AF_INET; +int af_inet6 = (int)AF_INET6; + +uptr __sanitizer_in_addr_sz(int af) { + if (af == AF_INET) + return sizeof(struct in_addr); + else if (af == AF_INET6) + return sizeof(struct in6_addr); + else + return 0; +} + +unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); + +int glob_nomatch = GLOB_NOMATCH; +int glob_altdirfunc = GLOB_ALTDIRFUNC; + +unsigned path_max = PATH_MAX; + +// ioctl arguments +unsigned struct_ifreq_sz = sizeof(struct ifreq); +unsigned struct_termios_sz = sizeof(struct termios); +unsigned struct_winsize_sz = sizeof(struct winsize); +unsigned struct_mtget_sz = sizeof(struct mtget); +unsigned struct_mtop_sz = sizeof(struct mtop); +unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); +unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); +unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); +unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + +const unsigned IOCTL_NOT_PRESENT = 0; + +unsigned IOCTL_FIOASYNC = FIOASYNC; +unsigned IOCTL_FIOCLEX = FIOCLEX; +unsigned IOCTL_FIOGETOWN = FIOGETOWN; +unsigned IOCTL_FIONBIO = FIONBIO; +unsigned IOCTL_FIONCLEX = FIONCLEX; +unsigned IOCTL_FIOSETOWN = FIOSETOWN; +unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; +unsigned IOCTL_SIOCATMARK = SIOCATMARK; +unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; +unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR; +unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR; +unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF; +unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR; +unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS; +unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC; +unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU; +unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK; +unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; +unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; +unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR; +unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR; +unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS; +unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC; +unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU; +unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; +unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; +unsigned IOCTL_TIOCCONS = TIOCCONS; +unsigned IOCTL_TIOCEXCL = TIOCEXCL; +unsigned IOCTL_TIOCGETD = TIOCGETD; +unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; +unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ; +unsigned IOCTL_TIOCMBIC = TIOCMBIC; +unsigned IOCTL_TIOCMBIS = TIOCMBIS; +unsigned IOCTL_TIOCMGET = TIOCMGET; +unsigned IOCTL_TIOCMSET = TIOCMSET; +unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; +unsigned IOCTL_TIOCNXCL = TIOCNXCL; +unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; +unsigned IOCTL_TIOCPKT = TIOCPKT; +unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; +unsigned IOCTL_TIOCSETD = TIOCSETD; +unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; +unsigned IOCTL_TIOCSTI = TIOCSTI; +unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; +unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; +unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; + +const int si_SEGV_MAPERR = SEGV_MAPERR; +const int si_SEGV_ACCERR = SEGV_ACCERR; +} // namespace __sanitizer + +using namespace __sanitizer; + +COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); + +COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); +CHECK_TYPE_SIZE(pthread_key_t); + +// There are more undocumented fields in dl_phdr_info that we are not interested +// in. +COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info)); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); + +CHECK_TYPE_SIZE(glob_t); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv); +CHECK_SIZE_AND_OFFSET(glob_t, gl_offs); +CHECK_SIZE_AND_OFFSET(glob_t, gl_flags); +CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat); +CHECK_SIZE_AND_OFFSET(glob_t, gl_stat); + +CHECK_TYPE_SIZE(addrinfo); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_family); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); + +CHECK_TYPE_SIZE(hostent); +CHECK_SIZE_AND_OFFSET(hostent, h_name); +CHECK_SIZE_AND_OFFSET(hostent, h_aliases); +CHECK_SIZE_AND_OFFSET(hostent, h_addrtype); +CHECK_SIZE_AND_OFFSET(hostent, h_length); +CHECK_SIZE_AND_OFFSET(hostent, h_addr_list); + +CHECK_TYPE_SIZE(iovec); +CHECK_SIZE_AND_OFFSET(iovec, iov_base); +CHECK_SIZE_AND_OFFSET(iovec, iov_len); + +CHECK_TYPE_SIZE(msghdr); +CHECK_SIZE_AND_OFFSET(msghdr, msg_name); +CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iov); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_control); +CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_flags); + +CHECK_TYPE_SIZE(cmsghdr); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); + +COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); +CHECK_SIZE_AND_OFFSET(dirent, d_fileno); +CHECK_SIZE_AND_OFFSET(dirent, d_reclen); + +CHECK_TYPE_SIZE(ifconf); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_len); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu); + +CHECK_TYPE_SIZE(pollfd); +CHECK_SIZE_AND_OFFSET(pollfd, fd); +CHECK_SIZE_AND_OFFSET(pollfd, events); +CHECK_SIZE_AND_OFFSET(pollfd, revents); + +CHECK_TYPE_SIZE(nfds_t); + +CHECK_TYPE_SIZE(sigset_t); + +COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); +// Can't write checks for sa_handler and sa_sigaction due to them being +// preprocessor macros. +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); + +CHECK_TYPE_SIZE(wordexp_t); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); + +CHECK_TYPE_SIZE(tm); +CHECK_SIZE_AND_OFFSET(tm, tm_sec); +CHECK_SIZE_AND_OFFSET(tm, tm_min); +CHECK_SIZE_AND_OFFSET(tm, tm_hour); +CHECK_SIZE_AND_OFFSET(tm, tm_mday); +CHECK_SIZE_AND_OFFSET(tm, tm_mon); +CHECK_SIZE_AND_OFFSET(tm, tm_year); +CHECK_SIZE_AND_OFFSET(tm, tm_wday); +CHECK_SIZE_AND_OFFSET(tm, tm_yday); +CHECK_SIZE_AND_OFFSET(tm, tm_isdst); +CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff); +CHECK_SIZE_AND_OFFSET(tm, tm_zone); + +CHECK_TYPE_SIZE(ether_addr); + +CHECK_TYPE_SIZE(ipc_perm); +CHECK_SIZE_AND_OFFSET(ipc_perm, _key); +CHECK_SIZE_AND_OFFSET(ipc_perm, _seq); +CHECK_SIZE_AND_OFFSET(ipc_perm, uid); +CHECK_SIZE_AND_OFFSET(ipc_perm, gid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); +CHECK_SIZE_AND_OFFSET(ipc_perm, mode); + +CHECK_TYPE_SIZE(shmid_ds); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch); + +CHECK_TYPE_SIZE(clock_t); + +CHECK_TYPE_SIZE(ifaddrs); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); +// Compare against the union, because we can't reach into the union in a +// compliant way. +#ifdef ifa_dstaddr +#undef ifa_dstaddr +#endif +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); + +CHECK_TYPE_SIZE(timeb); +CHECK_SIZE_AND_OFFSET(timeb, time); +CHECK_SIZE_AND_OFFSET(timeb, millitm); +CHECK_SIZE_AND_OFFSET(timeb, timezone); +CHECK_SIZE_AND_OFFSET(timeb, dstflag); + +CHECK_TYPE_SIZE(passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_name); +CHECK_SIZE_AND_OFFSET(passwd, pw_passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_uid); +CHECK_SIZE_AND_OFFSET(passwd, pw_gid); +CHECK_SIZE_AND_OFFSET(passwd, pw_dir); +CHECK_SIZE_AND_OFFSET(passwd, pw_shell); + +CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); + +CHECK_TYPE_SIZE(group); +CHECK_SIZE_AND_OFFSET(group, gr_name); +CHECK_SIZE_AND_OFFSET(group, gr_passwd); +CHECK_SIZE_AND_OFFSET(group, gr_gid); +CHECK_SIZE_AND_OFFSET(group, gr_mem); + +#endif // SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h new file mode 100644 index 000000000000..6823004d7b41 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -0,0 +1,582 @@ +//===-- sanitizer_platform_limits_netbsd.h --------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific NetBSD data structures. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_PLATFORM_LIMITS_NETBSD_H +#define SANITIZER_PLATFORM_LIMITS_NETBSD_H + +#if SANITIZER_NETBSD + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +#define _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, shift) \ + ((link_map *)((handle) == nullptr ? nullptr : ((char *)(handle) + (shift)))) + +#if defined(__x86_64__) +#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312) +#elif defined(__i386__) +#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164) +#endif + +namespace __sanitizer { +extern unsigned struct_utsname_sz; +extern unsigned struct_stat_sz; +extern unsigned struct_rusage_sz; +extern unsigned siginfo_t_sz; +extern unsigned struct_itimerval_sz; +extern unsigned pthread_t_sz; +extern unsigned pthread_mutex_t_sz; +extern unsigned pthread_cond_t_sz; +extern unsigned pid_t_sz; +extern unsigned timeval_sz; +extern unsigned uid_t_sz; +extern unsigned gid_t_sz; +extern unsigned mbstate_t_sz; +extern unsigned struct_timezone_sz; +extern unsigned struct_tms_sz; +extern unsigned struct_itimerspec_sz; +extern unsigned struct_sigevent_sz; +extern unsigned struct_sched_param_sz; +extern unsigned struct_statfs_sz; +extern unsigned struct_sockaddr_sz; +extern unsigned ucontext_t_sz; + +extern unsigned struct_rlimit_sz; +extern unsigned struct_utimbuf_sz; +extern unsigned struct_timespec_sz; + +struct __sanitizer_iocb { + u64 aio_offset; + uptr aio_buf; + long aio_nbytes; + u32 aio_fildes; + u32 aio_lio_opcode; + long aio_reqprio; +#if SANITIZER_WORDSIZE == 64 + u8 aio_sigevent[32]; +#else + u8 aio_sigevent[20]; +#endif + u32 _state; + u32 _errno; + long _retval; +}; + +struct __sanitizer___sysctl_args { + int *name; + int nlen; + void *oldval; + uptr *oldlenp; + void *newval; + uptr newlen; +}; + +struct __sanitizer_sem_t { + uptr data[5]; +}; + +struct __sanitizer_ipc_perm { + u32 uid; + u32 gid; + u32 cuid; + u32 cgid; + u32 mode; + unsigned short _seq; + long _key; +}; + +struct __sanitizer_shmid_ds { + __sanitizer_ipc_perm shm_perm; + unsigned long shm_segsz; + u32 shm_lpid; + u32 shm_cpid; + unsigned int shm_nattch; + u64 shm_atime; + u64 shm_dtime; + u64 shm_ctime; + void *_shm_internal; +}; + +extern unsigned struct_msqid_ds_sz; +extern unsigned struct_mq_attr_sz; +extern unsigned struct_timex_sz; +extern unsigned struct_statvfs_sz; + +struct __sanitizer_iovec { + void *iov_base; + uptr iov_len; +}; + +struct __sanitizer_ifaddrs { + struct __sanitizer_ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + void *ifa_addr; // (struct sockaddr *) + void *ifa_netmask; // (struct sockaddr *) + void *ifa_dstaddr; // (struct sockaddr *) + void *ifa_data; + unsigned int ifa_addrflags; +}; + +typedef unsigned __sanitizer_pthread_key_t; + +typedef long long __sanitizer_time_t; + +struct __sanitizer_passwd { + char *pw_name; + char *pw_passwd; + int pw_uid; + int pw_gid; + __sanitizer_time_t pw_change; + char *pw_class; + char *pw_gecos; + char *pw_dir; + char *pw_shell; + __sanitizer_time_t pw_expire; +}; + +struct __sanitizer_group { + char *gr_name; + char *gr_passwd; + int gr_gid; + char **gr_mem; +}; + +struct __sanitizer_timeb { + __sanitizer_time_t time; + unsigned short millitm; + short timezone; + short dstflag; +}; + +struct __sanitizer_ether_addr { + u8 octet[6]; +}; + +struct __sanitizer_tm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; + long int tm_gmtoff; + const char *tm_zone; +}; + +struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + unsigned msg_iovlen; + void *msg_control; + unsigned msg_controllen; + int msg_flags; +}; +struct __sanitizer_cmsghdr { + unsigned cmsg_len; + int cmsg_level; + int cmsg_type; +}; + +struct __sanitizer_dirent { + u64 d_fileno; + u16 d_reclen; + // more fields that we don't care about +}; + +typedef int __sanitizer_clock_t; +typedef int __sanitizer_clockid_t; + +typedef u32 __sanitizer___kernel_uid_t; +typedef u32 __sanitizer___kernel_gid_t; +typedef u64 __sanitizer___kernel_off_t; +typedef struct { + u32 fds_bits[8]; +} __sanitizer___kernel_fd_set; + +typedef struct { + unsigned int pta_magic; + int pta_flags; + void *pta_private; +} __sanitizer_pthread_attr_t; + +struct __sanitizer_sigset_t { + // uint32_t * 4 + unsigned int __bits[4]; +}; + +struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; +}; + +using __sanitizer_sighandler_ptr = void (*)(int sig); +using __sanitizer_sigactionhandler_ptr = void (*)(int sig, + __sanitizer_siginfo *siginfo, + void *uctx); + +struct __sanitizer_sigaction { + union { + __sanitizer_sighandler_ptr handler; + __sanitizer_sigactionhandler_ptr sigaction; + }; + __sanitizer_sigset_t sa_mask; + int sa_flags; +}; + +typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; + +struct __sanitizer_kernel_sigaction_t { + union { + void (*handler)(int signo); + void (*sigaction)(int signo, void *info, void *ctx); + }; + unsigned long sa_flags; + void (*sa_restorer)(void); + __sanitizer_kernel_sigset_t sa_mask; +}; + +extern const uptr sig_ign; +extern const uptr sig_dfl; +extern const uptr sig_err; +extern const uptr sa_siginfo; + +extern int af_inet; +extern int af_inet6; +uptr __sanitizer_in_addr_sz(int af); + +struct __sanitizer_dl_phdr_info { + uptr dlpi_addr; + const char *dlpi_name; + const void *dlpi_phdr; + short dlpi_phnum; +}; + +extern unsigned struct_ElfW_Phdr_sz; + +struct __sanitizer_addrinfo { + int ai_flags; + int ai_family; + int ai_socktype; + int ai_protocol; + unsigned ai_addrlen; + char *ai_canonname; + void *ai_addr; + struct __sanitizer_addrinfo *ai_next; +}; + +struct __sanitizer_hostent { + char *h_name; + char **h_aliases; + int h_addrtype; + int h_length; + char **h_addr_list; +}; + +struct __sanitizer_pollfd { + int fd; + short events; + short revents; +}; + +typedef unsigned __sanitizer_nfds_t; + +struct __sanitizer_glob_t { + uptr gl_pathc; + uptr gl_matchc; + uptr gl_offs; + int gl_flags; + char **gl_pathv; + int (*gl_errfunc)(const char *, int); + void (*gl_closedir)(void *dirp); + struct dirent *(*gl_readdir)(void *dirp); + void *(*gl_opendir)(const char *); + int (*gl_lstat)(const char *, void * /* struct stat* */); + int (*gl_stat)(const char *, void * /* struct stat* */); +}; + +extern int glob_nomatch; +extern int glob_altdirfunc; + +extern unsigned path_max; + +struct __sanitizer_wordexp_t { + uptr we_wordc; + char **we_wordv; + uptr we_offs; + char *we_strings; + uptr we_nbytes; +}; + +typedef char __sanitizer_FILE; +#define SANITIZER_HAS_STRUCT_FILE 0 + +extern int shmctl_ipc_stat; + +// This simplifies generic code +#define struct_shminfo_sz -1 +#define struct_shm_info_sz -1 +#define shmctl_shm_stat -1 +#define shmctl_ipc_info -1 +#define shmctl_shm_info -1 + +extern unsigned struct_utmp_sz; +extern unsigned struct_utmpx_sz; + +extern int map_fixed; + +// ioctl arguments +struct __sanitizer_ifconf { + int ifc_len; + union { + void *ifcu_req; + } ifc_ifcu; +}; + +#define IOC_NRBITS 8 +#define IOC_TYPEBITS 8 +#define IOC_SIZEBITS 14 +#define IOC_DIRBITS 2 +#define IOC_NONE 0U +#define IOC_WRITE 1U +#define IOC_READ 2U +#define IOC_NRMASK ((1 << IOC_NRBITS) - 1) +#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) +#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) +#undef IOC_DIRMASK +#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) +#define IOC_NRSHIFT 0 +#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) +#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) +#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) +#define EVIOC_EV_MAX 0x1f +#define EVIOC_ABS_MAX 0x3f + +#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) +#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) +#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) +#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) + +extern unsigned struct_ifreq_sz; +extern unsigned struct_termios_sz; +extern unsigned struct_winsize_sz; + +extern unsigned struct_arpreq_sz; + +extern unsigned struct_mtget_sz; +extern unsigned struct_mtop_sz; +extern unsigned struct_rtentry_sz; +extern unsigned struct_sbi_instrument_sz; +extern unsigned struct_seq_event_rec_sz; +extern unsigned struct_synth_info_sz; +extern unsigned struct_vt_mode_sz; +extern unsigned struct_audio_buf_info_sz; +extern unsigned struct_ppp_stats_sz; +extern unsigned struct_sioc_sg_req_sz; +extern unsigned struct_sioc_vif_req_sz; + +// ioctl request identifiers + +// A special value to mark ioctls that are not present on the target platform, +// when it can not be determined without including any system headers. +extern const unsigned IOCTL_NOT_PRESENT; + +extern unsigned IOCTL_FIOASYNC; +extern unsigned IOCTL_FIOCLEX; +extern unsigned IOCTL_FIOGETOWN; +extern unsigned IOCTL_FIONBIO; +extern unsigned IOCTL_FIONCLEX; +extern unsigned IOCTL_FIOSETOWN; +extern unsigned IOCTL_SIOCADDMULTI; +extern unsigned IOCTL_SIOCATMARK; +extern unsigned IOCTL_SIOCDELMULTI; +extern unsigned IOCTL_SIOCGIFADDR; +extern unsigned IOCTL_SIOCGIFBRDADDR; +extern unsigned IOCTL_SIOCGIFCONF; +extern unsigned IOCTL_SIOCGIFDSTADDR; +extern unsigned IOCTL_SIOCGIFFLAGS; +extern unsigned IOCTL_SIOCGIFMETRIC; +extern unsigned IOCTL_SIOCGIFMTU; +extern unsigned IOCTL_SIOCGIFNETMASK; +extern unsigned IOCTL_SIOCGPGRP; +extern unsigned IOCTL_SIOCSIFADDR; +extern unsigned IOCTL_SIOCSIFBRDADDR; +extern unsigned IOCTL_SIOCSIFDSTADDR; +extern unsigned IOCTL_SIOCSIFFLAGS; +extern unsigned IOCTL_SIOCSIFMETRIC; +extern unsigned IOCTL_SIOCSIFMTU; +extern unsigned IOCTL_SIOCSIFNETMASK; +extern unsigned IOCTL_SIOCSPGRP; +extern unsigned IOCTL_TIOCCONS; +extern unsigned IOCTL_TIOCEXCL; +extern unsigned IOCTL_TIOCGETD; +extern unsigned IOCTL_TIOCGPGRP; +extern unsigned IOCTL_TIOCGWINSZ; +extern unsigned IOCTL_TIOCMBIC; +extern unsigned IOCTL_TIOCMBIS; +extern unsigned IOCTL_TIOCMGET; +extern unsigned IOCTL_TIOCMSET; +extern unsigned IOCTL_TIOCNOTTY; +extern unsigned IOCTL_TIOCNXCL; +extern unsigned IOCTL_TIOCOUTQ; +extern unsigned IOCTL_TIOCPKT; +extern unsigned IOCTL_TIOCSCTTY; +extern unsigned IOCTL_TIOCSETD; +extern unsigned IOCTL_TIOCSPGRP; +extern unsigned IOCTL_TIOCSTI; +extern unsigned IOCTL_TIOCSWINSZ; +extern unsigned IOCTL_SIOCGETSGCNT; +extern unsigned IOCTL_SIOCGETVIFCNT; +extern unsigned IOCTL_MTIOCGET; +extern unsigned IOCTL_MTIOCTOP; +extern unsigned IOCTL_SIOCADDRT; +extern unsigned IOCTL_SIOCDELRT; +extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE; +extern unsigned IOCTL_SNDCTL_DSP_GETFMTS; +extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK; +extern unsigned IOCTL_SNDCTL_DSP_POST; +extern unsigned IOCTL_SNDCTL_DSP_RESET; +extern unsigned IOCTL_SNDCTL_DSP_SETFMT; +extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT; +extern unsigned IOCTL_SNDCTL_DSP_SPEED; +extern unsigned IOCTL_SNDCTL_DSP_STEREO; +extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE; +extern unsigned IOCTL_SNDCTL_DSP_SYNC; +extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE; +extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR; +extern unsigned IOCTL_SNDCTL_MIDI_INFO; +extern unsigned IOCTL_SNDCTL_MIDI_PRETIME; +extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE; +extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT; +extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT; +extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS; +extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS; +extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND; +extern unsigned IOCTL_SNDCTL_SEQ_PANIC; +extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE; +extern unsigned IOCTL_SNDCTL_SEQ_RESET; +extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES; +extern unsigned IOCTL_SNDCTL_SEQ_SYNC; +extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI; +extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD; +extern unsigned IOCTL_SNDCTL_SYNTH_INFO; +extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL; +extern unsigned IOCTL_SNDCTL_TMR_CONTINUE; +extern unsigned IOCTL_SNDCTL_TMR_METRONOME; +extern unsigned IOCTL_SNDCTL_TMR_SELECT; +extern unsigned IOCTL_SNDCTL_TMR_SOURCE; +extern unsigned IOCTL_SNDCTL_TMR_START; +extern unsigned IOCTL_SNDCTL_TMR_STOP; +extern unsigned IOCTL_SNDCTL_TMR_TEMPO; +extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE; +extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM; +extern unsigned IOCTL_SOUND_MIXER_READ_BASS; +extern unsigned IOCTL_SOUND_MIXER_READ_CAPS; +extern unsigned IOCTL_SOUND_MIXER_READ_CD; +extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK; +extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE; +extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN; +extern unsigned IOCTL_SOUND_MIXER_READ_IMIX; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE1; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE2; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE3; +extern unsigned IOCTL_SOUND_MIXER_READ_LINE; +extern unsigned IOCTL_SOUND_MIXER_READ_LOUD; +extern unsigned IOCTL_SOUND_MIXER_READ_MIC; +extern unsigned IOCTL_SOUND_MIXER_READ_MUTE; +extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN; +extern unsigned IOCTL_SOUND_MIXER_READ_PCM; +extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV; +extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK; +extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC; +extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER; +extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS; +extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH; +extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE; +extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME; +extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM; +extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS; +extern unsigned IOCTL_SOUND_MIXER_WRITE_CD; +extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN; +extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD; +extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC; +extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN; +extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM; +extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV; +extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC; +extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER; +extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH; +extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE; +extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME; +extern unsigned IOCTL_SOUND_PCM_READ_BITS; +extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS; +extern unsigned IOCTL_SOUND_PCM_READ_FILTER; +extern unsigned IOCTL_SOUND_PCM_READ_RATE; +extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS; +extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER; +extern unsigned IOCTL_VT_ACTIVATE; +extern unsigned IOCTL_VT_GETMODE; +extern unsigned IOCTL_VT_OPENQRY; +extern unsigned IOCTL_VT_RELDISP; +extern unsigned IOCTL_VT_SETMODE; +extern unsigned IOCTL_VT_WAITACTIVE; +extern unsigned IOCTL_KDDISABIO; +extern unsigned IOCTL_KDENABIO; +extern unsigned IOCTL_KDGETLED; +extern unsigned IOCTL_KDGKBMODE; +extern unsigned IOCTL_KDGKBTYPE; +extern unsigned IOCTL_KDMKTONE; +extern unsigned IOCTL_KDSETLED; +extern unsigned IOCTL_KDSETMODE; +extern unsigned IOCTL_KDSKBMODE; + +extern const int si_SEGV_MAPERR; +extern const int si_SEGV_ACCERR; +} // namespace __sanitizer + +#define CHECK_TYPE_SIZE(TYPE) \ + COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) + +#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \ + sizeof(((CLASS *)NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ + offsetof(CLASS, MEMBER)) + +// For sigaction, which is a function and struct at the same time, +// and thus requires explicit "struct" in sizeof() expression. +#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \ + sizeof(((struct CLASS *)NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ + offsetof(struct CLASS, MEMBER)) + +#define SIGACTION_SYMNAME __sigaction14 + +#endif // SANITIZER_NETBSD + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index 83f4fd22f623..f12e8206abe6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -210,6 +210,7 @@ namespace __sanitizer { unsigned struct_sigaction_sz = sizeof(struct sigaction); unsigned struct_itimerval_sz = sizeof(struct itimerval); unsigned pthread_t_sz = sizeof(pthread_t); + unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); unsigned pid_t_sz = sizeof(pid_t); unsigned timeval_sz = sizeof(timeval); @@ -264,9 +265,10 @@ namespace __sanitizer { unsigned struct_statvfs_sz = sizeof(struct statvfs); #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID - uptr sig_ign = (uptr)SIG_IGN; - uptr sig_dfl = (uptr)SIG_DFL; - uptr sa_siginfo = (uptr)SA_SIGINFO; + const uptr sig_ign = (uptr)SIG_IGN; + const uptr sig_dfl = (uptr)SIG_DFL; + const uptr sig_err = (uptr)SIG_ERR; + const uptr sa_siginfo = (uptr)SA_SIGINFO; #if SANITIZER_LINUX int e_tabsz = (int)E_TABSZ; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 808418012d75..2853a84d43ae 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -15,6 +15,8 @@ #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC + #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" @@ -44,6 +46,7 @@ namespace __sanitizer { extern unsigned siginfo_t_sz; extern unsigned struct_itimerval_sz; extern unsigned pthread_t_sz; + extern unsigned pthread_mutex_t_sz; extern unsigned pthread_cond_t_sz; extern unsigned pid_t_sz; extern unsigned timeval_sz; @@ -535,7 +538,7 @@ namespace __sanitizer { typedef long __sanitizer_clock_t; #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD typedef int __sanitizer_clockid_t; #endif @@ -596,13 +599,22 @@ namespace __sanitizer { }; #endif + struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; + }; + + using __sanitizer_sighandler_ptr = void (*)(int sig); + using __sanitizer_sigactionhandler_ptr = + void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx); + // Linux system headers define the 'sa_handler' and 'sa_sigaction' macros. #if SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 64) struct __sanitizer_sigaction { unsigned sa_flags; union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; __sanitizer_sigset_t sa_mask; void (*sa_restorer)(); @@ -611,16 +623,16 @@ namespace __sanitizer { struct __sanitizer_sigaction { unsigned sa_flags; union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; __sanitizer_sigset_t sa_mask; }; #elif SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32) struct __sanitizer_sigaction { union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; __sanitizer_sigset_t sa_mask; uptr sa_flags; @@ -632,8 +644,8 @@ namespace __sanitizer { unsigned int sa_flags; #endif union { - void (*sigaction)(int sig, void *siginfo, void *uctx); - void (*handler)(int sig); + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; }; #if SANITIZER_FREEBSD int sa_flags; @@ -692,7 +704,7 @@ namespace __sanitizer { unsigned int sa_flags; union { void (*handler)(int signo); - void (*sigaction)(int signo, void *info, void *ctx); + void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx); }; __sanitizer_kernel_sigset_t sa_mask; void (*sa_restorer)(void); @@ -701,7 +713,7 @@ namespace __sanitizer { struct __sanitizer_kernel_sigaction_t { union { void (*handler)(int signo); - void (*sigaction)(int signo, void *info, void *ctx); + void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx); }; unsigned long sa_flags; void (*sa_restorer)(void); @@ -709,9 +721,10 @@ namespace __sanitizer { }; #endif - extern uptr sig_ign; - extern uptr sig_dfl; - extern uptr sa_siginfo; + extern const uptr sig_ign; + extern const uptr sig_dfl; + extern const uptr sig_err; + extern const uptr sa_siginfo; #if SANITIZER_LINUX extern int e_tabsz; @@ -1489,4 +1502,8 @@ struct __sanitizer_cookie_io_functions_t { COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ offsetof(struct CLASS, MEMBER)) +#define SIGACTION_SYMNAME sigaction + +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC + #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc new file mode 100644 index 000000000000..15b80a81ae3b --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc @@ -0,0 +1,366 @@ +//===-- sanitizer_platform_limits_solaris.cc ------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific Solaris data structures. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_SOLARIS +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Include these after system headers to avoid name clashes and ambiguities. +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_solaris.h" + +namespace __sanitizer { + unsigned struct_utsname_sz = sizeof(struct utsname); + unsigned struct_stat_sz = sizeof(struct stat); + unsigned struct_stat64_sz = sizeof(struct stat64); + unsigned struct_rusage_sz = sizeof(struct rusage); + unsigned struct_tm_sz = sizeof(struct tm); + unsigned struct_passwd_sz = sizeof(struct passwd); + unsigned struct_group_sz = sizeof(struct group); + unsigned siginfo_t_sz = sizeof(siginfo_t); + unsigned struct_sigaction_sz = sizeof(struct sigaction); + unsigned struct_itimerval_sz = sizeof(struct itimerval); + unsigned pthread_t_sz = sizeof(pthread_t); + unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); + unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); + unsigned pid_t_sz = sizeof(pid_t); + unsigned timeval_sz = sizeof(timeval); + unsigned uid_t_sz = sizeof(uid_t); + unsigned gid_t_sz = sizeof(gid_t); + unsigned mbstate_t_sz = sizeof(mbstate_t); + unsigned sigset_t_sz = sizeof(sigset_t); + unsigned struct_timezone_sz = sizeof(struct timezone); + unsigned struct_tms_sz = sizeof(struct tms); + unsigned struct_sigevent_sz = sizeof(struct sigevent); + unsigned struct_sched_param_sz = sizeof(struct sched_param); + unsigned struct_statfs_sz = sizeof(struct statfs); + unsigned struct_sockaddr_sz = sizeof(struct sockaddr); + unsigned ucontext_t_sz = sizeof(ucontext_t); + unsigned struct_timespec_sz = sizeof(struct timespec); +#if SANITIZER_SOLARIS32 + unsigned struct_statvfs64_sz = sizeof(struct statvfs64); +#endif + unsigned struct_statvfs_sz = sizeof(struct statvfs); + + const uptr sig_ign = (uptr)SIG_IGN; + const uptr sig_dfl = (uptr)SIG_DFL; + const uptr sig_err = (uptr)SIG_ERR; + const uptr sa_siginfo = (uptr)SA_SIGINFO; + + int shmctl_ipc_stat = (int)IPC_STAT; + + unsigned struct_utmp_sz = sizeof(struct utmp); + unsigned struct_utmpx_sz = sizeof(struct utmpx); + + int map_fixed = MAP_FIXED; + + int af_inet = (int)AF_INET; + int af_inet6 = (int)AF_INET6; + + uptr __sanitizer_in_addr_sz(int af) { + if (af == AF_INET) + return sizeof(struct in_addr); + else if (af == AF_INET6) + return sizeof(struct in6_addr); + else + return 0; + } + + unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr)); + + int glob_nomatch = GLOB_NOMATCH; + + unsigned path_max = PATH_MAX; + + // ioctl arguments + unsigned struct_ifreq_sz = sizeof(struct ifreq); + unsigned struct_termios_sz = sizeof(struct termios); + unsigned struct_winsize_sz = sizeof(struct winsize); + + unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); + unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + + const unsigned IOCTL_NOT_PRESENT = 0; + + unsigned IOCTL_FIOASYNC = FIOASYNC; + unsigned IOCTL_FIOCLEX = FIOCLEX; + unsigned IOCTL_FIOGETOWN = FIOGETOWN; + unsigned IOCTL_FIONBIO = FIONBIO; + unsigned IOCTL_FIONCLEX = FIONCLEX; + unsigned IOCTL_FIOSETOWN = FIOSETOWN; + unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; + unsigned IOCTL_SIOCATMARK = SIOCATMARK; + unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; + unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR; + unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR; + unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF; + unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR; + unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS; + unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC; + unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU; + unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK; + unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; + unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; + unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR; + unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR; + unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS; + unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC; + unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU; + unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; + unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; + unsigned IOCTL_TIOCEXCL = TIOCEXCL; + unsigned IOCTL_TIOCGETD = TIOCGETD; + unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; + unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ; + unsigned IOCTL_TIOCMBIC = TIOCMBIC; + unsigned IOCTL_TIOCMBIS = TIOCMBIS; + unsigned IOCTL_TIOCMGET = TIOCMGET; + unsigned IOCTL_TIOCMSET = TIOCMSET; + unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; + unsigned IOCTL_TIOCNXCL = TIOCNXCL; + unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; + unsigned IOCTL_TIOCPKT = TIOCPKT; + unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; + unsigned IOCTL_TIOCSETD = TIOCSETD; + unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; + unsigned IOCTL_TIOCSTI = TIOCSTI; + unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; + + unsigned IOCTL_MTIOCGET = MTIOCGET; + unsigned IOCTL_MTIOCTOP = MTIOCTOP; + + const int si_SEGV_MAPERR = SEGV_MAPERR; + const int si_SEGV_ACCERR = SEGV_ACCERR; +} // namespace __sanitizer + +using namespace __sanitizer; + +COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); + +COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); +CHECK_TYPE_SIZE(pthread_key_t); + +// There are more undocumented fields in dl_phdr_info that we are not interested +// in. +COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info)); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); + +CHECK_TYPE_SIZE(glob_t); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv); +CHECK_SIZE_AND_OFFSET(glob_t, gl_offs); + +CHECK_TYPE_SIZE(addrinfo); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_family); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); + +CHECK_TYPE_SIZE(hostent); +CHECK_SIZE_AND_OFFSET(hostent, h_name); +CHECK_SIZE_AND_OFFSET(hostent, h_aliases); +CHECK_SIZE_AND_OFFSET(hostent, h_addrtype); +CHECK_SIZE_AND_OFFSET(hostent, h_length); +CHECK_SIZE_AND_OFFSET(hostent, h_addr_list); + +CHECK_TYPE_SIZE(iovec); +CHECK_SIZE_AND_OFFSET(iovec, iov_base); +CHECK_SIZE_AND_OFFSET(iovec, iov_len); + +CHECK_TYPE_SIZE(msghdr); +CHECK_SIZE_AND_OFFSET(msghdr, msg_name); +CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iov); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_control); +CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_flags); + +CHECK_TYPE_SIZE(cmsghdr); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); + +COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); +CHECK_SIZE_AND_OFFSET(dirent, d_ino); +CHECK_SIZE_AND_OFFSET(dirent, d_off); +CHECK_SIZE_AND_OFFSET(dirent, d_reclen); + +#if SANITIZER_SOLARIS32 +COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64)); +CHECK_SIZE_AND_OFFSET(dirent64, d_ino); +CHECK_SIZE_AND_OFFSET(dirent64, d_off); +CHECK_SIZE_AND_OFFSET(dirent64, d_reclen); +#endif + +CHECK_TYPE_SIZE(ifconf); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_len); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu); + +CHECK_TYPE_SIZE(pollfd); +CHECK_SIZE_AND_OFFSET(pollfd, fd); +CHECK_SIZE_AND_OFFSET(pollfd, events); +CHECK_SIZE_AND_OFFSET(pollfd, revents); + +CHECK_TYPE_SIZE(nfds_t); + +CHECK_TYPE_SIZE(sigset_t); + +COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); +// Can't write checks for sa_handler and sa_sigaction due to them being +// preprocessor macros. +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags); + +CHECK_TYPE_SIZE(wordexp_t); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); + +CHECK_TYPE_SIZE(tm); +CHECK_SIZE_AND_OFFSET(tm, tm_sec); +CHECK_SIZE_AND_OFFSET(tm, tm_min); +CHECK_SIZE_AND_OFFSET(tm, tm_hour); +CHECK_SIZE_AND_OFFSET(tm, tm_mday); +CHECK_SIZE_AND_OFFSET(tm, tm_mon); +CHECK_SIZE_AND_OFFSET(tm, tm_year); +CHECK_SIZE_AND_OFFSET(tm, tm_wday); +CHECK_SIZE_AND_OFFSET(tm, tm_yday); +CHECK_SIZE_AND_OFFSET(tm, tm_isdst); + +CHECK_TYPE_SIZE(ether_addr); + +CHECK_TYPE_SIZE(ipc_perm); +CHECK_SIZE_AND_OFFSET(ipc_perm, key); +CHECK_SIZE_AND_OFFSET(ipc_perm, seq); +CHECK_SIZE_AND_OFFSET(ipc_perm, uid); +CHECK_SIZE_AND_OFFSET(ipc_perm, gid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); +CHECK_SIZE_AND_OFFSET(ipc_perm, mode); + +CHECK_TYPE_SIZE(shmid_ds); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch); + +CHECK_TYPE_SIZE(clock_t); + +CHECK_TYPE_SIZE(ifaddrs); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); +// Compare against the union, because we can't reach into the union in a +// compliant way. +#ifdef ifa_dstaddr +#undef ifa_dstaddr +#endif +COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)nullptr)->ifa_dstaddr) == + sizeof(((ifaddrs *)nullptr)->ifa_ifu)); +COMPILER_CHECK(offsetof(__sanitizer_ifaddrs, ifa_dstaddr) == + offsetof(ifaddrs, ifa_ifu)); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); + +CHECK_TYPE_SIZE(timeb); +CHECK_SIZE_AND_OFFSET(timeb, time); +CHECK_SIZE_AND_OFFSET(timeb, millitm); +CHECK_SIZE_AND_OFFSET(timeb, timezone); +CHECK_SIZE_AND_OFFSET(timeb, dstflag); + +CHECK_TYPE_SIZE(passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_name); +CHECK_SIZE_AND_OFFSET(passwd, pw_passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_uid); +CHECK_SIZE_AND_OFFSET(passwd, pw_gid); +CHECK_SIZE_AND_OFFSET(passwd, pw_dir); +CHECK_SIZE_AND_OFFSET(passwd, pw_shell); + +CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); + +CHECK_TYPE_SIZE(group); +CHECK_SIZE_AND_OFFSET(group, gr_name); +CHECK_SIZE_AND_OFFSET(group, gr_passwd); +CHECK_SIZE_AND_OFFSET(group, gr_gid); +CHECK_SIZE_AND_OFFSET(group, gr_mem); + +CHECK_TYPE_SIZE(XDR); +CHECK_SIZE_AND_OFFSET(XDR, x_op); +CHECK_SIZE_AND_OFFSET(XDR, x_ops); +CHECK_SIZE_AND_OFFSET(XDR, x_public); +CHECK_SIZE_AND_OFFSET(XDR, x_private); +CHECK_SIZE_AND_OFFSET(XDR, x_base); +CHECK_SIZE_AND_OFFSET(XDR, x_handy); +COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE); +COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE); +COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE); + +CHECK_TYPE_SIZE(sem_t); + +#endif // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h new file mode 100644 index 000000000000..a9db71b99c81 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -0,0 +1,485 @@ +//===-- sanitizer_platform_limits_solaris.h -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific Solaris data structures. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_PLATFORM_LIMITS_SOLARIS_H +#define SANITIZER_PLATFORM_LIMITS_SOLARIS_H + +#if SANITIZER_SOLARIS + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +namespace __sanitizer { +extern unsigned struct_utsname_sz; +extern unsigned struct_stat_sz; +extern unsigned struct_stat64_sz; +extern unsigned struct_rusage_sz; +extern unsigned siginfo_t_sz; +extern unsigned struct_itimerval_sz; +extern unsigned pthread_t_sz; +extern unsigned pthread_mutex_t_sz; +extern unsigned pthread_cond_t_sz; +extern unsigned pid_t_sz; +extern unsigned timeval_sz; +extern unsigned uid_t_sz; +extern unsigned gid_t_sz; +extern unsigned mbstate_t_sz; +extern unsigned struct_timezone_sz; +extern unsigned struct_tms_sz; +extern unsigned struct_itimerspec_sz; +extern unsigned struct_sigevent_sz; +extern unsigned struct_sched_param_sz; +extern unsigned struct_statfs64_sz; +extern unsigned struct_statfs_sz; +extern unsigned struct_sockaddr_sz; +extern unsigned ucontext_t_sz; + +extern unsigned struct_timespec_sz; +extern unsigned struct_rlimit_sz; +extern unsigned struct_utimbuf_sz; + +struct __sanitizer_sem_t { + //u64 data[6]; + u32 sem_count; + u16 sem_type; + u16 sem_magic; + u64 sem_pad1[3]; + u64 sem_pad2[2]; +}; + +struct __sanitizer_ipc_perm { + unsigned int uid; // uid_t + unsigned int gid; // gid_t + unsigned int cuid; // uid_t + unsigned int cgid; // gid_t + unsigned int mode; // mode_t + unsigned int seq; // uint_t + int key; // key_t +#if !defined(_LP64) + int pad[4]; +#endif + }; + +struct __sanitizer_shmid_ds { + __sanitizer_ipc_perm shm_perm; + unsigned long shm_segsz; // size_t + unsigned long shm_flags; // uintptr_t + unsigned short shm_lkcnt; // ushort_t + int shm_lpid; // pid_t + int shm_cpid; // pid_t + unsigned long shm_nattch; // shmatt_t + unsigned long shm_cnattch; // ulong_t +#if defined(_LP64) + long shm_atime; // time_t + long shm_dtime; + long shm_ctime; + void *shm_amp; + u64 shm_gransize; // uint64_t + u64 shm_allocated; // uint64_t + u64 shm_pad4[1]; // int64_t +#else + long shm_atime; // time_t + int shm_pad1; // int32_t + long shm_dtime; // time_t + int shm_pad2; // int32_t + long shm_ctime; // time_t + void *shm_amp; + u64 shm_gransize; // uint64_t + u64 shm_allocated; // uint64_t +#endif +}; + +extern unsigned struct_statvfs_sz; +#if SANITIZER_SOLARIS32 +extern unsigned struct_statvfs64_sz; +#endif + +struct __sanitizer_iovec { + void *iov_base; + uptr iov_len; +}; + +struct __sanitizer_ifaddrs { + struct __sanitizer_ifaddrs *ifa_next; + char *ifa_name; + u64 ifa_flags; // uint64_t + void *ifa_addr; // (struct sockaddr *) + void *ifa_netmask; // (struct sockaddr *) + // This is a union on Linux. +# ifdef ifa_dstaddr +# undef ifa_dstaddr +# endif + void *ifa_dstaddr; // (struct sockaddr *) + void *ifa_data; +}; + +typedef unsigned __sanitizer_pthread_key_t; + +struct __sanitizer_XDR { + int x_op; + void *x_ops; + uptr x_public; + uptr x_private; + uptr x_base; + unsigned x_handy; +}; + +const int __sanitizer_XDR_ENCODE = 0; +const int __sanitizer_XDR_DECODE = 1; +const int __sanitizer_XDR_FREE = 2; + +struct __sanitizer_passwd { + char *pw_name; + char *pw_passwd; + unsigned int pw_uid; // uid_t + unsigned int pw_gid; // gid_t + char *pw_age; + char *pw_comment; + char *pw_gecos; + char *pw_dir; + char *pw_shell; +}; + +struct __sanitizer_group { + char *gr_name; + char *gr_passwd; + int gr_gid; + char **gr_mem; +}; + +typedef long __sanitizer_time_t; + +struct __sanitizer_timeb { + __sanitizer_time_t time; + unsigned short millitm; + short timezone; + short dstflag; +}; + +struct __sanitizer_ether_addr { + u8 octet[6]; +}; + +struct __sanitizer_tm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; +}; + +struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + unsigned msg_iovlen; + void *msg_control; + unsigned msg_controllen; + int msg_flags; +}; +struct __sanitizer_cmsghdr { + unsigned cmsg_len; + int cmsg_level; + int cmsg_type; +}; + +#if SANITIZER_SOLARIS32 && 0 +// FIXME: need to deal with large file and non-large file cases +struct __sanitizer_dirent { + unsigned long long d_ino; + long long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; +#else +struct __sanitizer_dirent { + unsigned long d_ino; + long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; +#endif + +struct __sanitizer_dirent64 { + unsigned long long d_ino; + unsigned long long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; + +typedef long __sanitizer_clock_t; +typedef int __sanitizer_clockid_t; + +// This thing depends on the platform. We are only interested in the upper +// limit. Verified with a compiler assert in .cc. +const int pthread_attr_t_max_sz = 128; +union __sanitizer_pthread_attr_t { + char size[pthread_attr_t_max_sz]; // NOLINT + void *align; +}; + +struct __sanitizer_sigset_t { + // uint32_t * 4 + unsigned int __bits[4]; +}; + +struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; +}; + +using __sanitizer_sighandler_ptr = void (*)(int sig); +using __sanitizer_sigactionhandler_ptr = + void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx); + +struct __sanitizer_sigaction { + int sa_flags; + union { + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; + }; + __sanitizer_sigset_t sa_mask; +#if !defined(_LP64) + int sa_resv[2]; +#endif +}; + +struct __sanitizer_kernel_sigset_t { + u8 sig[8]; +}; + +struct __sanitizer_kernel_sigaction_t { + union { + void (*handler)(int signo); + void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx); + }; + unsigned long sa_flags; + void (*sa_restorer)(void); + __sanitizer_kernel_sigset_t sa_mask; +}; + +extern const uptr sig_ign; +extern const uptr sig_dfl; +extern const uptr sig_err; +extern const uptr sa_siginfo; + +extern int af_inet; +extern int af_inet6; +uptr __sanitizer_in_addr_sz(int af); + +struct __sanitizer_dl_phdr_info { + uptr dlpi_addr; + const char *dlpi_name; + const void *dlpi_phdr; + short dlpi_phnum; +}; + +extern unsigned struct_ElfW_Phdr_sz; + +struct __sanitizer_addrinfo { + int ai_flags; + int ai_family; + int ai_socktype; + int ai_protocol; +#if defined(__sparcv9) + int _ai_pad; +#endif + unsigned ai_addrlen; + char *ai_canonname; + void *ai_addr; + struct __sanitizer_addrinfo *ai_next; +}; + +struct __sanitizer_hostent { + char *h_name; + char **h_aliases; + int h_addrtype; + int h_length; + char **h_addr_list; +}; + +struct __sanitizer_pollfd { + int fd; + short events; + short revents; +}; + +typedef unsigned long __sanitizer_nfds_t; + +struct __sanitizer_glob_t { + uptr gl_pathc; + char **gl_pathv; + uptr gl_offs; + char **gl_pathp; + int gl_pathn; +}; + +extern int glob_nomatch; +extern int glob_altdirfunc; + +extern unsigned path_max; + +struct __sanitizer_wordexp_t { + uptr we_wordc; + char **we_wordv; + uptr we_offs; + char **we_wordp; + int we_wordn; +}; + +typedef void __sanitizer_FILE; +#define SANITIZER_HAS_STRUCT_FILE 0 + +// This simplifies generic code +#define struct_shminfo_sz -1 +#define struct_shm_info_sz -1 +#define shmctl_shm_stat -1 +#define shmctl_ipc_info -1 +#define shmctl_shm_info -1 + +extern int shmctl_ipc_stat; + +extern unsigned struct_utmp_sz; +extern unsigned struct_utmpx_sz; + +extern int map_fixed; + +// ioctl arguments +struct __sanitizer_ifconf { + int ifc_len; + union { + void *ifcu_req; + } ifc_ifcu; +}; + +// +#define IOC_NRBITS 8 +#define IOC_TYPEBITS 8 +#define IOC_SIZEBITS 12 +#define IOC_DIRBITS 4 +#undef IOC_NONE +#define IOC_NONE 2U // IOC_VOID +#define IOC_READ 4U // IOC_OUT +#define IOC_WRITE 8U // IOC_IN + +#define IOC_NRMASK ((1 << IOC_NRBITS) - 1) +#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) +#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) +#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) +#define IOC_NRSHIFT 0 +#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) +#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) +#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) + +#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) +#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) +#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) + +#if defined(__sparc__) +// In sparc the 14 bits SIZE field overlaps with the +// least significant bit of DIR, so either IOC_READ or +// IOC_WRITE shall be 1 in order to get a non-zero SIZE. +#define IOC_SIZE(nr) \ + ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff)) +#else +#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) +#endif + +extern unsigned struct_ifreq_sz; +extern unsigned struct_termios_sz; +extern unsigned struct_winsize_sz; + +extern unsigned struct_sioc_sg_req_sz; +extern unsigned struct_sioc_vif_req_sz; + +// ioctl request identifiers + +// A special value to mark ioctls that are not present on the target platform, +// when it can not be determined without including any system headers. +extern const unsigned IOCTL_NOT_PRESENT; + +extern unsigned IOCTL_FIOASYNC; +extern unsigned IOCTL_FIOCLEX; +extern unsigned IOCTL_FIOGETOWN; +extern unsigned IOCTL_FIONBIO; +extern unsigned IOCTL_FIONCLEX; +extern unsigned IOCTL_FIOSETOWN; +extern unsigned IOCTL_SIOCADDMULTI; +extern unsigned IOCTL_SIOCATMARK; +extern unsigned IOCTL_SIOCDELMULTI; +extern unsigned IOCTL_SIOCGIFADDR; +extern unsigned IOCTL_SIOCGIFBRDADDR; +extern unsigned IOCTL_SIOCGIFCONF; +extern unsigned IOCTL_SIOCGIFDSTADDR; +extern unsigned IOCTL_SIOCGIFFLAGS; +extern unsigned IOCTL_SIOCGIFMETRIC; +extern unsigned IOCTL_SIOCGIFMTU; +extern unsigned IOCTL_SIOCGIFNETMASK; +extern unsigned IOCTL_SIOCGPGRP; +extern unsigned IOCTL_SIOCSIFADDR; +extern unsigned IOCTL_SIOCSIFBRDADDR; +extern unsigned IOCTL_SIOCSIFDSTADDR; +extern unsigned IOCTL_SIOCSIFFLAGS; +extern unsigned IOCTL_SIOCSIFMETRIC; +extern unsigned IOCTL_SIOCSIFMTU; +extern unsigned IOCTL_SIOCSIFNETMASK; +extern unsigned IOCTL_SIOCSPGRP; +extern unsigned IOCTL_TIOCEXCL; +extern unsigned IOCTL_TIOCGETD; +extern unsigned IOCTL_TIOCGPGRP; +extern unsigned IOCTL_TIOCGWINSZ; +extern unsigned IOCTL_TIOCMBIC; +extern unsigned IOCTL_TIOCMBIS; +extern unsigned IOCTL_TIOCMGET; +extern unsigned IOCTL_TIOCMSET; +extern unsigned IOCTL_TIOCNOTTY; +extern unsigned IOCTL_TIOCNXCL; +extern unsigned IOCTL_TIOCOUTQ; +extern unsigned IOCTL_TIOCPKT; +extern unsigned IOCTL_TIOCSCTTY; +extern unsigned IOCTL_TIOCSETD; +extern unsigned IOCTL_TIOCSPGRP; +extern unsigned IOCTL_TIOCSTI; +extern unsigned IOCTL_TIOCSWINSZ; +extern unsigned IOCTL_MTIOCGET; +extern unsigned IOCTL_MTIOCTOP; + +extern const int si_SEGV_MAPERR; +extern const int si_SEGV_ACCERR; +} // namespace __sanitizer + +#define CHECK_TYPE_SIZE(TYPE) \ + COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) + +#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ + offsetof(CLASS, MEMBER)) + +// For sigaction, which is a function and struct at the same time, +// and thus requires explicit "struct" in sizeof() expression. +#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((struct CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ + offsetof(struct CLASS, MEMBER)) + +#endif // SANITIZER_SOLARIS + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc index 8d3128ae199d..1fad71f3582b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc @@ -17,6 +17,7 @@ #if SANITIZER_POSIX #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_libc.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" @@ -294,18 +295,22 @@ bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) { return false; } -SignalContext SignalContext::Create(void *siginfo, void *context) { - auto si = (siginfo_t *)siginfo; - uptr addr = (uptr)si->si_addr; - uptr pc, sp, bp; - GetPcSpBp(context, &pc, &sp, &bp); - WriteFlag write_flag = GetWriteFlag(context); - bool is_memory_access = si->si_signo == SIGSEGV; - return SignalContext(context, addr, pc, sp, bp, is_memory_access, write_flag); +uptr SignalContext::GetAddress() const { + auto si = static_cast(siginfo); + return (uptr)si->si_addr; } -const char *DescribeSignalOrException(int signo) { - switch (signo) { +bool SignalContext::IsMemoryAccess() const { + auto si = static_cast(siginfo); + return si->si_signo == SIGSEGV; +} + +int SignalContext::GetType() const { + return static_cast(siginfo)->si_signo; +} + +const char *SignalContext::Describe() const { + switch (GetType()) { case SIGFPE: return "FPE"; case SIGILL: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h index e7d37cbf0882..adef08248004 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h @@ -16,7 +16,9 @@ // ----------- ATTENTION ------------- // This header should NOT include any other headers from sanitizer runtime. #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_posix.h" +#include "sanitizer_platform_limits_solaris.h" #if !SANITIZER_POSIX // Make it hard to accidentally use any of functions declared in this file: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc index e113fb1093d4..db41cad6c318 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -18,7 +18,9 @@ #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_posix.h" +#include "sanitizer_platform_limits_solaris.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" @@ -61,7 +63,11 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end) { uptr beg_aligned = RoundUpTo(beg, page_size); uptr end_aligned = RoundDownTo(end, page_size); if (beg_aligned < end_aligned) - madvise((void*)beg_aligned, end_aligned - beg_aligned, MADV_DONTNEED); + // In the default Solaris compilation environment, madvise() is declared + // to take a caddr_t arg; casting it to void * results in an invalid + // conversion error, so use char * instead. + madvise((char *)beg_aligned, end_aligned - beg_aligned, + SANITIZER_MADVISE_DONTNEED); } void NoHugePagesInRegion(uptr addr, uptr size) { @@ -213,6 +219,53 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) { MaybeInstallSigaction(SIGFPE, handler); MaybeInstallSigaction(SIGILL, handler); } + +bool SignalContext::IsStackOverflow() const { + // Access at a reasonable offset above SP, or slightly below it (to account + // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is + // probably a stack overflow. +#ifdef __s390__ + // On s390, the fault address in siginfo points to start of the page, not + // to the precise word that was accessed. Mask off the low bits of sp to + // take it into account. + bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF; +#else + bool IsStackAccess = addr + 512 > sp && addr < sp + 0xFFFF; +#endif + +#if __powerpc__ + // Large stack frames can be allocated with e.g. + // lis r0,-10000 + // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000 + // If the store faults then sp will not have been updated, so test above + // will not work, because the fault address will be more than just "slightly" + // below sp. + if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) { + u32 inst = *(unsigned *)pc; + u32 ra = (inst >> 16) & 0x1F; + u32 opcd = inst >> 26; + u32 xo = (inst >> 1) & 0x3FF; + // Check for store-with-update to sp. The instructions we accept are: + // stbu rs,d(ra) stbux rs,ra,rb + // sthu rs,d(ra) sthux rs,ra,rb + // stwu rs,d(ra) stwux rs,ra,rb + // stdu rs,ds(ra) stdux rs,ra,rb + // where ra is r1 (the stack pointer). + if (ra == 1 && + (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 || + (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181)))) + IsStackAccess = true; + } +#endif // __powerpc__ + + // We also check si_code to filter out SEGV caused by something else other + // then hitting the guard page or unmapped memory, like, for example, + // unaligned memory access. + auto si = static_cast(siginfo); + return IsStackAccess && + (si->si_code == si_SEGV_MAPERR || si->si_code == si_SEGV_ACCERR); +} + #endif // SANITIZER_GO bool IsAccessibleMemoryRange(uptr beg, uptr size) { @@ -289,6 +342,46 @@ void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { return (void *)p; } +uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { + // We don't pass `name` along because, when you enable `decorate_proc_maps` + // AND actually use a named mapping AND are using a sanitizer intercepting + // `open` (e.g. TSAN, ESAN), then you'll get a failure during initialization. + // TODO(flowerhack): Fix the implementation of GetNamedMappingFd to solve + // this problem. + if (fixed_addr) { + base_ = MmapFixedNoAccess(fixed_addr, size); + } else { + base_ = MmapNoAccess(size); + } + size_ = size; + name_ = name; + (void)os_handle_; // unsupported + return reinterpret_cast(base_); +} + +// Uses fixed_addr for now. +// Will use offset instead once we've implemented this function for real. +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { + return reinterpret_cast(MmapFixedOrDieOnFatalError(fixed_addr, size)); +} + +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { + return reinterpret_cast(MmapFixedOrDie(fixed_addr, size)); +} + +void ReservedAddressRange::Unmap(uptr addr, uptr size) { + void* addr_as_void = reinterpret_cast(addr); + uptr base_as_uptr = reinterpret_cast(base_); + // Only unmap at the beginning or end of the range. + CHECK((addr_as_void == base_) || (addr + size == base_as_uptr + size_)); + CHECK_LE(size, size_); + UnmapOrDie(reinterpret_cast(addr), size); + if (addr_as_void == base_) { + base_ = reinterpret_cast(addr + size); + } + size_ = size_ - size; +} + void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { int fd = name ? GetNamedMappingFd(name, size) : -1; unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc index 99b7ff1b55cf..5c2360043b07 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc @@ -28,8 +28,6 @@ namespace __sanitizer { -StaticSpinMutex CommonSanitizerReportMutex; - static int AppendChar(char **buff, const char *buff_end, char c) { if (*buff < buff_end) { **buff = c; @@ -229,19 +227,16 @@ static void CallPrintfAndReportCallback(const char *str) { PrintfAndReportCallback(str); } -static void SharedPrintfCode(bool append_pid, const char *format, - va_list args) { +static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid, + char *local_buffer, + int buffer_size, + const char *format, + va_list args) { va_list args2; va_copy(args2, args); const int kLen = 16 * 1024; - // |local_buffer| is small enough not to overflow the stack and/or violate - // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other - // hand, the bigger the buffer is, the more the chance the error report will - // fit into it. - char local_buffer[400]; int needed_length; char *buffer = local_buffer; - int buffer_size = ARRAY_SIZE(local_buffer); // First try to print a message using a local buffer, and then fall back to // mmaped buffer. for (int use_mmap = 0; use_mmap < 2; use_mmap++) { @@ -259,7 +254,9 @@ static void SharedPrintfCode(bool append_pid, const char *format, RAW_CHECK_MSG(needed_length < kLen, \ "Buffer in Report is too short!\n"); \ } - if (append_pid) { + // Fuchsia's logging infrastructure always keeps track of the logging + // process, thread, and timestamp, so never prepend such information. + if (!SANITIZER_FUCHSIA && append_pid) { int pid = internal_getpid(); const char *exe_name = GetProcessName(); if (common_flags()->log_exe_name && exe_name) { @@ -267,9 +264,8 @@ static void SharedPrintfCode(bool append_pid, const char *format, "==%s", exe_name); CHECK_NEEDED_LENGTH } - needed_length += internal_snprintf(buffer + needed_length, - buffer_size - needed_length, - "==%d==", pid); + needed_length += internal_snprintf( + buffer + needed_length, buffer_size - needed_length, "==%d==", pid); CHECK_NEEDED_LENGTH } needed_length += VSNPrintf(buffer + needed_length, @@ -292,6 +288,17 @@ static void SharedPrintfCode(bool append_pid, const char *format, va_end(args2); } +static void NOINLINE SharedPrintfCode(bool append_pid, const char *format, + va_list args) { + // |local_buffer| is small enough not to overflow the stack and/or violate + // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other + // hand, the bigger the buffer is, the more the chance the error report will + // fit into it. + char local_buffer[400]; + SharedPrintfCodeNoBuffer(append_pid, local_buffer, ARRAY_SIZE(local_buffer), + format, args); +} + FORMAT(1, 2) void Printf(const char *format, ...) { va_list args; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h index 06d072b4dc77..ea2cb7a5c68b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h @@ -14,22 +14,19 @@ #ifndef SANITIZER_PROCMAPS_H #define SANITIZER_PROCMAPS_H +#include "sanitizer_platform.h" + +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ + SANITIZER_MAC || SANITIZER_SOLARIS + #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_linux.h" +#include "sanitizer_mac.h" #include "sanitizer_mutex.h" namespace __sanitizer { -#if SANITIZER_FREEBSD || SANITIZER_LINUX -struct ProcSelfMapsBuff { - char *data; - uptr mmaped_size; - uptr len; -}; - -// Reads process memory map in an OS-specific way. -void ReadProcMaps(ProcSelfMapsBuff *proc_maps); -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX // Memory protection masks. static const uptr kProtectionRead = 1; @@ -37,15 +34,20 @@ static const uptr kProtectionWrite = 2; static const uptr kProtectionExecute = 4; static const uptr kProtectionShared = 8; -struct MemoryMappedSegment { +struct MemoryMappedSegmentData; + +class MemoryMappedSegment { + public: MemoryMappedSegment(char *buff = nullptr, uptr size = 0) - : filename(buff), filename_size(size) {} + : filename(buff), filename_size(size), data_(nullptr) {} ~MemoryMappedSegment() {} - bool IsReadable() { return protection & kProtectionRead; } - bool IsWritable() { return protection & kProtectionWrite; } - bool IsExecutable() { return protection & kProtectionExecute; } - bool IsShared() { return protection & kProtectionShared; } + bool IsReadable() const { return protection & kProtectionRead; } + bool IsWritable() const { return protection & kProtectionWrite; } + bool IsExecutable() const { return protection & kProtectionExecute; } + bool IsShared() const { return protection & kProtectionShared; } + + void AddAddressRanges(LoadedModule *module); uptr start; uptr end; @@ -55,6 +57,12 @@ struct MemoryMappedSegment { uptr protection; ModuleArch arch; u8 uuid[kModuleUUIDSize]; + + private: + friend class MemoryMappingLayout; + + // This field is assigned and owned by MemoryMappingLayout if needed + MemoryMappedSegmentData *data_; }; class MemoryMappingLayout { @@ -69,42 +77,14 @@ class MemoryMappingLayout { static void CacheMemoryMappings(); // Adds all mapped objects into a vector. - void DumpListOfModules(InternalMmapVector *modules); + void DumpListOfModules(InternalMmapVectorNoCtor *modules); private: void LoadFromCache(); - // FIXME: Hide implementation details for different platforms in - // platform-specific files. -# if SANITIZER_FREEBSD || SANITIZER_LINUX - ProcSelfMapsBuff proc_self_maps_; - const char *current_; - - // Static mappings cache. - static ProcSelfMapsBuff cached_proc_self_maps_; - static StaticSpinMutex cache_lock_; // protects cached_proc_self_maps_. -# elif SANITIZER_MAC - template - bool NextSegmentLoad(MemoryMappedSegment *segment); - int current_image_; - u32 current_magic_; - u32 current_filetype_; - ModuleArch current_arch_; - u8 current_uuid_[kModuleUUIDSize]; - int current_load_cmd_count_; - char *current_load_cmd_addr_; - bool current_instrumented_; -# endif + MemoryMappingLayoutData data_; }; -typedef void (*fill_profile_f)(uptr start, uptr rss, bool file, - /*out*/uptr *stats, uptr stats_size); - -// Parse the contents of /proc/self/smaps and generate a memory profile. -// |cb| is a tool-specific callback that fills the |stats| array containing -// |stats_size| elements. -void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size); - // Returns code range for the specified module. bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end); @@ -115,4 +95,6 @@ uptr ParseHex(const char **p); } // namespace __sanitizer +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || + // SANITIZER_MAC || SANITIZER_SOLARIS #endif // SANITIZER_PROCMAPS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc index b95f301a437d..0cd3e2458d87 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -12,7 +12,8 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_placement_new.h" @@ -20,9 +21,8 @@ namespace __sanitizer { -// Linker initialized. -ProcSelfMapsBuff MemoryMappingLayout::cached_proc_self_maps_; -StaticSpinMutex MemoryMappingLayout::cache_lock_; // Linker initialized. +static ProcSelfMapsBuff cached_proc_self_maps; +static StaticSpinMutex cache_lock; static int TranslateDigit(char c) { if (c >= '0' && c <= '9') @@ -64,60 +64,60 @@ uptr ParseHex(const char **p) { return ParseNumber(p, 16); } +void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) { + // data_ should be unused on this platform + CHECK(!data_); + module->addAddressRange(start, end, IsExecutable(), IsWritable()); +} + MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { - ReadProcMaps(&proc_self_maps_); - if (cache_enabled) { - if (proc_self_maps_.mmaped_size == 0) { - LoadFromCache(); - CHECK_GT(proc_self_maps_.len, 0); - } - } else { - CHECK_GT(proc_self_maps_.mmaped_size, 0); - } - Reset(); // FIXME: in the future we may want to cache the mappings on demand only. if (cache_enabled) CacheMemoryMappings(); + + // Read maps after the cache update to capture the maps/unmaps happening in + // the process of updating. + ReadProcMaps(&data_.proc_self_maps); + if (cache_enabled && data_.proc_self_maps.mmaped_size == 0) + LoadFromCache(); + CHECK_GT(data_.proc_self_maps.mmaped_size, 0); + CHECK_GT(data_.proc_self_maps.len, 0); + + Reset(); } MemoryMappingLayout::~MemoryMappingLayout() { // Only unmap the buffer if it is different from the cached one. Otherwise // it will be unmapped when the cache is refreshed. - if (proc_self_maps_.data != cached_proc_self_maps_.data) { - UnmapOrDie(proc_self_maps_.data, proc_self_maps_.mmaped_size); - } + if (data_.proc_self_maps.data != cached_proc_self_maps.data) + UnmapOrDie(data_.proc_self_maps.data, data_.proc_self_maps.mmaped_size); } void MemoryMappingLayout::Reset() { - current_ = proc_self_maps_.data; + data_.current = data_.proc_self_maps.data; } // static void MemoryMappingLayout::CacheMemoryMappings() { - SpinMutexLock l(&cache_lock_); + ProcSelfMapsBuff new_proc_self_maps; + ReadProcMaps(&new_proc_self_maps); // Don't invalidate the cache if the mappings are unavailable. - ProcSelfMapsBuff old_proc_self_maps; - old_proc_self_maps = cached_proc_self_maps_; - ReadProcMaps(&cached_proc_self_maps_); - if (cached_proc_self_maps_.mmaped_size == 0) { - cached_proc_self_maps_ = old_proc_self_maps; - } else { - if (old_proc_self_maps.mmaped_size) { - UnmapOrDie(old_proc_self_maps.data, - old_proc_self_maps.mmaped_size); - } - } + if (new_proc_self_maps.mmaped_size == 0) + return; + SpinMutexLock l(&cache_lock); + if (cached_proc_self_maps.mmaped_size) + UnmapOrDie(cached_proc_self_maps.data, cached_proc_self_maps.mmaped_size); + cached_proc_self_maps = new_proc_self_maps; } void MemoryMappingLayout::LoadFromCache() { - SpinMutexLock l(&cache_lock_); - if (cached_proc_self_maps_.data) { - proc_self_maps_ = cached_proc_self_maps_; - } + SpinMutexLock l(&cache_lock); + if (cached_proc_self_maps.data) + data_.proc_self_maps = cached_proc_self_maps; } void MemoryMappingLayout::DumpListOfModules( - InternalMmapVector *modules) { + InternalMmapVectorNoCtor *modules) { Reset(); InternalScopedString module_name(kMaxPathLength); MemoryMappedSegment segment(module_name.data(), module_name.size()); @@ -139,8 +139,7 @@ void MemoryMappingLayout::DumpListOfModules( uptr base_address = (i ? segment.start : 0) - segment.offset; LoadedModule cur_module; cur_module.set(cur_name, base_address); - cur_module.addAddressRange(segment.start, segment.end, - segment.IsExecutable(), segment.IsWritable()); + segment.AddAddressRanges(&cur_module); modules->push_back(cur_module); } } @@ -171,4 +170,5 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc index f0cdbeb4483a..c26a6d47ea75 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc @@ -7,18 +7,22 @@ // //===----------------------------------------------------------------------===// // -// Information about the process mappings (FreeBSD-specific parts). +// Information about the process mappings (FreeBSD and NetBSD-specific parts). //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD #include "sanitizer_common.h" +#if SANITIZER_FREEBSD #include "sanitizer_freebsd.h" +#endif #include "sanitizer_procmaps.h" #include #include +#if SANITIZER_FREEBSD #include +#endif // Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode. #if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32) @@ -31,16 +35,30 @@ namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - const int Mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid() }; + const int Mib[] = { +#if SANITIZER_FREEBSD + CTL_KERN, + KERN_PROC, + KERN_PROC_VMMAP, + getpid() +#else + CTL_VM, + VM_PROC, + VM_PROC_MAP, + getpid(), + sizeof(struct kinfo_vmentry) +#endif + }; + size_t Size = 0; - int Err = sysctl(Mib, 4, NULL, &Size, NULL, 0); + int Err = sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0); CHECK_EQ(Err, 0); CHECK_GT(Size, 0); size_t MmapedSize = Size * 4 / 3; void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()"); Size = MmapedSize; - Err = sysctl(Mib, 4, VmMap, &Size, NULL, 0); + Err = sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0); CHECK_EQ(Err, 0); proc_maps->data = (char*)VmMap; @@ -49,9 +67,9 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { - char *last = proc_self_maps_.data + proc_self_maps_.len; - if (current_ >= last) return false; - struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_; + char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; + if (data_.current >= last) return false; + struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry *)data_.current; segment->start = (uptr)VmEntry->kve_start; segment->end = (uptr)VmEntry->kve_end; @@ -71,11 +89,15 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { VmEntry->kve_path); } - current_ += VmEntry->kve_structsize; +#if SANITIZER_FREEBSD + data_.current += VmEntry->kve_structsize; +#else + data_.current += sizeof(*VmEntry); +#endif return true; } } // namespace __sanitizer -#endif // SANITIZER_FREEBSD +#endif // SANITIZER_FREEBSD || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc index 1bcad2bf70e6..633e9393cce0 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -18,8 +18,12 @@ namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - ReadFileToBuffer("/proc/self/maps", &proc_maps->data, &proc_maps->mmaped_size, - &proc_maps->len); + if (!ReadFileToBuffer("/proc/self/maps", &proc_maps->data, + &proc_maps->mmaped_size, &proc_maps->len)) { + proc_maps->data = nullptr; + proc_maps->mmaped_size = 0; + proc_maps->len = 0; + } } static bool IsOneOf(char c, char c1, char c2) { @@ -27,48 +31,48 @@ static bool IsOneOf(char c, char c1, char c2) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { - char *last = proc_self_maps_.data + proc_self_maps_.len; - if (current_ >= last) return false; - char *next_line = (char*)internal_memchr(current_, '\n', last - current_); + char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; + if (data_.current >= last) return false; + char *next_line = + (char *)internal_memchr(data_.current, '\n', last - data_.current); if (next_line == 0) next_line = last; // Example: 08048000-08056000 r-xp 00000000 03:0c 64593 /foo/bar - segment->start = ParseHex(¤t_); - CHECK_EQ(*current_++, '-'); - segment->end = ParseHex(¤t_); - CHECK_EQ(*current_++, ' '); - CHECK(IsOneOf(*current_, '-', 'r')); + segment->start = ParseHex(&data_.current); + CHECK_EQ(*data_.current++, '-'); + segment->end = ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ' '); + CHECK(IsOneOf(*data_.current, '-', 'r')); segment->protection = 0; - if (*current_++ == 'r') segment->protection |= kProtectionRead; - CHECK(IsOneOf(*current_, '-', 'w')); - if (*current_++ == 'w') segment->protection |= kProtectionWrite; - CHECK(IsOneOf(*current_, '-', 'x')); - if (*current_++ == 'x') segment->protection |= kProtectionExecute; - CHECK(IsOneOf(*current_, 's', 'p')); - if (*current_++ == 's') segment->protection |= kProtectionShared; - CHECK_EQ(*current_++, ' '); - segment->offset = ParseHex(¤t_); - CHECK_EQ(*current_++, ' '); - ParseHex(¤t_); - CHECK_EQ(*current_++, ':'); - ParseHex(¤t_); - CHECK_EQ(*current_++, ' '); - while (IsDecimal(*current_)) - current_++; + if (*data_.current++ == 'r') segment->protection |= kProtectionRead; + CHECK(IsOneOf(*data_.current, '-', 'w')); + if (*data_.current++ == 'w') segment->protection |= kProtectionWrite; + CHECK(IsOneOf(*data_.current, '-', 'x')); + if (*data_.current++ == 'x') segment->protection |= kProtectionExecute; + CHECK(IsOneOf(*data_.current, 's', 'p')); + if (*data_.current++ == 's') segment->protection |= kProtectionShared; + CHECK_EQ(*data_.current++, ' '); + segment->offset = ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ' '); + ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ':'); + ParseHex(&data_.current); + CHECK_EQ(*data_.current++, ' '); + while (IsDecimal(*data_.current)) data_.current++; // Qemu may lack the trailing space. // https://github.com/google/sanitizers/issues/160 - // CHECK_EQ(*current_++, ' '); + // CHECK_EQ(*data_.current++, ' '); // Skip spaces. - while (current_ < next_line && *current_ == ' ') - current_++; + while (data_.current < next_line && *data_.current == ' ') data_.current++; // Fill in the filename. if (segment->filename) { - uptr len = Min((uptr)(next_line - current_), segment->filename_size - 1); - internal_strncpy(segment->filename, current_, len); + uptr len = + Min((uptr)(next_line - data_.current), segment->filename_size - 1); + internal_strncpy(segment->filename, data_.current, len); segment->filename[len] = 0; } - current_ = next_line + 1; + data_.current = next_line + 1; return true; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 560451a16d90..0167ab18ba13 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -36,6 +36,51 @@ namespace __sanitizer { +// Contains information used to iterate through sections. +struct MemoryMappedSegmentData { + char name[kMaxSegName]; + uptr nsects; + const char *current_load_cmd_addr; + u32 lc_type; + uptr base_virt_addr; + uptr addr_mask; +}; + +template +static void NextSectionLoad(LoadedModule *module, MemoryMappedSegmentData *data, + bool isWritable) { + const Section *sc = (const Section *)data->current_load_cmd_addr; + data->current_load_cmd_addr += sizeof(Section); + + uptr sec_start = (sc->addr & data->addr_mask) + data->base_virt_addr; + uptr sec_end = sec_start + sc->size; + module->addAddressRange(sec_start, sec_end, /*executable=*/false, isWritable, + sc->sectname); +} + +void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) { + // Don't iterate over sections when the caller hasn't set up the + // data pointer, when there are no sections, or when the segment + // is executable. Avoid iterating over executable sections because + // it will confuse libignore, and because the extra granularity + // of information is not needed by any sanitizers. + if (!data_ || !data_->nsects || IsExecutable()) { + module->addAddressRange(start, end, IsExecutable(), IsWritable(), + data_ ? data_->name : nullptr); + return; + } + + do { + if (data_->lc_type == LC_SEGMENT) { + NextSectionLoad(module, data_, IsWritable()); +#ifdef MH_MAGIC_64 + } else if (data_->lc_type == LC_SEGMENT_64) { + NextSectionLoad(module, data_, IsWritable()); +#endif + } + } while (--data_->nsects); +} + MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { Reset(); } @@ -63,13 +108,13 @@ void MemoryMappingLayout::Reset() { // _dyld_image_count is thread-unsafe. We need to register callbacks for // adding and removing images which will invalidate the MemoryMappingLayout // state. - current_image_ = _dyld_image_count(); - current_load_cmd_count_ = -1; - current_load_cmd_addr_ = 0; - current_magic_ = 0; - current_filetype_ = 0; - current_arch_ = kModuleArchUnknown; - internal_memset(current_uuid_, 0, kModuleUUIDSize); + data_.current_image = _dyld_image_count(); + data_.current_load_cmd_count = -1; + data_.current_load_cmd_addr = 0; + data_.current_magic = 0; + data_.current_filetype = 0; + data_.current_arch = kModuleArchUnknown; + internal_memset(data_.current_uuid, 0, kModuleUUIDSize); } // The dyld load address should be unchanged throughout process execution, @@ -138,39 +183,57 @@ const mach_header *get_dyld_hdr() { // segment. // Note that the segment addresses are not necessarily sorted. template -bool MemoryMappingLayout::NextSegmentLoad(MemoryMappedSegment *segment) { - const char *lc = current_load_cmd_addr_; - current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize; +static bool NextSegmentLoad(MemoryMappedSegment *segment, +MemoryMappedSegmentData *seg_data, MemoryMappingLayoutData &layout_data) { + const char *lc = layout_data.current_load_cmd_addr; + layout_data.current_load_cmd_addr += ((const load_command *)lc)->cmdsize; if (((const load_command *)lc)->cmd == kLCSegment) { const SegmentCommand* sc = (const SegmentCommand *)lc; - - if (current_image_ == kDyldImageIdx) { + uptr base_virt_addr, addr_mask; + if (layout_data.current_image == kDyldImageIdx) { + base_virt_addr = (uptr)get_dyld_hdr(); // vmaddr is masked with 0xfffff because on macOS versions < 10.12, // it contains an absolute address rather than an offset for dyld. // To make matters even more complicated, this absolute address // isn't actually the absolute segment address, but the offset portion // of the address is accurate when combined with the dyld base address, // and the mask will give just this offset. - segment->start = (sc->vmaddr & 0xfffff) + (uptr)get_dyld_hdr(); - segment->end = (sc->vmaddr & 0xfffff) + sc->vmsize + (uptr)get_dyld_hdr(); + addr_mask = 0xfffff; } else { - const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_); - segment->start = sc->vmaddr + dlloff; - segment->end = sc->vmaddr + sc->vmsize + dlloff; + base_virt_addr = + (uptr)_dyld_get_image_vmaddr_slide(layout_data.current_image); + addr_mask = ~0; + } + + segment->start = (sc->vmaddr & addr_mask) + base_virt_addr; + segment->end = segment->start + sc->vmsize; + // Most callers don't need section information, so only fill this struct + // when required. + if (seg_data) { + seg_data->nsects = sc->nsects; + seg_data->current_load_cmd_addr = + (const char *)lc + sizeof(SegmentCommand); + seg_data->lc_type = kLCSegment; + seg_data->base_virt_addr = base_virt_addr; + seg_data->addr_mask = addr_mask; + internal_strncpy(seg_data->name, sc->segname, + ARRAY_SIZE(seg_data->name)); } // Return the initial protection. segment->protection = sc->initprot; - segment->offset = - (current_filetype_ == /*MH_EXECUTE*/ 0x2) ? sc->vmaddr : sc->fileoff; + segment->offset = (layout_data.current_filetype == + /*MH_EXECUTE*/ 0x2) + ? sc->vmaddr + : sc->fileoff; if (segment->filename) { - const char *src = (current_image_ == kDyldImageIdx) + const char *src = (layout_data.current_image == kDyldImageIdx) ? kDyldPath - : _dyld_get_image_name(current_image_); + : _dyld_get_image_name(layout_data.current_image); internal_strncpy(segment->filename, src, segment->filename_size); } - segment->arch = current_arch_; - internal_memcpy(segment->uuid, current_uuid_, kModuleUUIDSize); + segment->arch = layout_data.current_arch; + internal_memcpy(segment->uuid, layout_data.current_uuid, kModuleUUIDSize); return true; } return false; @@ -202,7 +265,7 @@ ModuleArch ModuleArchFromCpuType(cpu_type_t cputype, cpu_subtype_t cpusubtype) { } static const load_command *NextCommand(const load_command *lc) { - return (const load_command *)((char *)lc + lc->cmdsize); + return (const load_command *)((const char *)lc + lc->cmdsize); } static void FindUUID(const load_command *first_lc, u8 *uuid_output) { @@ -232,50 +295,53 @@ static bool IsModuleInstrumented(const load_command *first_lc) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { - for (; current_image_ >= kDyldImageIdx; current_image_--) { - const mach_header *hdr = (current_image_ == kDyldImageIdx) + for (; data_.current_image >= kDyldImageIdx; data_.current_image--) { + const mach_header *hdr = (data_.current_image == kDyldImageIdx) ? get_dyld_hdr() - : _dyld_get_image_header(current_image_); + : _dyld_get_image_header(data_.current_image); if (!hdr) continue; - if (current_load_cmd_count_ < 0) { + if (data_.current_load_cmd_count < 0) { // Set up for this image; - current_load_cmd_count_ = hdr->ncmds; - current_magic_ = hdr->magic; - current_filetype_ = hdr->filetype; - current_arch_ = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype); - switch (current_magic_) { + data_.current_load_cmd_count = hdr->ncmds; + data_.current_magic = hdr->magic; + data_.current_filetype = hdr->filetype; + data_.current_arch = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype); + switch (data_.current_magic) { #ifdef MH_MAGIC_64 case MH_MAGIC_64: { - current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header_64); + data_.current_load_cmd_addr = + (const char *)hdr + sizeof(mach_header_64); break; } #endif case MH_MAGIC: { - current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header); + data_.current_load_cmd_addr = (const char *)hdr + sizeof(mach_header); break; } default: { continue; } } - FindUUID((const load_command *)current_load_cmd_addr_, ¤t_uuid_[0]); - current_instrumented_ = - IsModuleInstrumented((const load_command *)current_load_cmd_addr_); + FindUUID((const load_command *)data_.current_load_cmd_addr, + data_.current_uuid); + data_.current_instrumented = IsModuleInstrumented( + (const load_command *)data_.current_load_cmd_addr); } - for (; current_load_cmd_count_ >= 0; current_load_cmd_count_--) { - switch (current_magic_) { - // current_magic_ may be only one of MH_MAGIC, MH_MAGIC_64. + for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) { + switch (data_.current_magic) { + // data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64. #ifdef MH_MAGIC_64 case MH_MAGIC_64: { if (NextSegmentLoad( - segment)) + segment, segment->data_, data_)) return true; break; } #endif case MH_MAGIC: { - if (NextSegmentLoad(segment)) + if (NextSegmentLoad( + segment, segment->data_, data_)) return true; break; } @@ -288,11 +354,13 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { } void MemoryMappingLayout::DumpListOfModules( - InternalMmapVector *modules) { + InternalMmapVectorNoCtor *modules) { Reset(); InternalScopedString module_name(kMaxPathLength); MemoryMappedSegment segment(module_name.data(), kMaxPathLength); - for (uptr i = 0; Next(&segment); i++) { + MemoryMappedSegmentData data; + segment.data_ = &data; + while (Next(&segment)) { if (segment.filename[0] == '\0') continue; LoadedModule *cur_module = nullptr; if (!modules->empty() && @@ -302,10 +370,9 @@ void MemoryMappingLayout::DumpListOfModules( modules->push_back(LoadedModule()); cur_module = &modules->back(); cur_module->set(segment.filename, segment.start, segment.arch, - segment.uuid, current_instrumented_); + segment.uuid, data_.current_instrumented); } - cur_module->addAddressRange(segment.start, segment.end, - segment.IsExecutable(), segment.IsWritable()); + segment.AddAddressRanges(cur_module); } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc new file mode 100644 index 000000000000..bfe83170f4e2 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc @@ -0,0 +1,57 @@ +//===-- sanitizer_procmaps_solaris.cc -------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Information about the process mappings (Solaris-specific parts). +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_SOLARIS +#include "sanitizer_common.h" +#include "sanitizer_procmaps.h" + +#include +#include + +namespace __sanitizer { + +void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { + ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, &proc_maps->mmaped_size, + &proc_maps->len); +} + +bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; + if (data_.current >= last) return false; + + prxmap_t *xmapentry = (prxmap_t*)data_.current; + + segment->start = (uptr)xmapentry->pr_vaddr; + segment->end = (uptr)(xmapentry->pr_vaddr + xmapentry->pr_size); + segment->offset = (uptr)xmapentry->pr_offset; + + segment->protection = 0; + if ((xmapentry->pr_mflags & MA_READ) != 0) + segment->protection |= kProtectionRead; + if ((xmapentry->pr_mflags & MA_WRITE) != 0) + segment->protection |= kProtectionWrite; + if ((xmapentry->pr_mflags & MA_EXEC) != 0) + segment->protection |= kProtectionExecute; + + if (segment->filename != NULL && segment->filename_size > 0) { + internal_snprintf(segment->filename, + Min(segment->filename_size, (uptr)PATH_MAX), "%s", + xmapentry->pr_mapname); + } + + data_.current += sizeof(prxmap_t); + + return true; +} + +} // namespace __sanitizer + +#endif // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h index db38867ced28..886445a27322 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h @@ -87,15 +87,14 @@ class Quarantine { // is zero (it allows us to perform just one atomic read per Put() call). CHECK((size == 0 && cache_size == 0) || cache_size != 0); - atomic_store(&max_size_, size, memory_order_relaxed); - atomic_store(&min_size_, size / 10 * 9, - memory_order_relaxed); // 90% of max size. - atomic_store(&max_cache_size_, cache_size, memory_order_relaxed); + atomic_store_relaxed(&max_size_, size); + atomic_store_relaxed(&min_size_, size / 10 * 9); // 90% of max size. + atomic_store_relaxed(&max_cache_size_, cache_size); } - uptr GetSize() const { return atomic_load(&max_size_, memory_order_relaxed); } + uptr GetSize() const { return atomic_load_relaxed(&max_size_); } uptr GetCacheSize() const { - return atomic_load(&max_cache_size_, memory_order_relaxed); + return atomic_load_relaxed(&max_cache_size_); } void Put(Cache *c, Callback cb, Node *ptr, uptr size) { @@ -117,7 +116,16 @@ class Quarantine { cache_.Transfer(c); } if (cache_.Size() > GetSize() && recycle_mutex_.TryLock()) - Recycle(cb); + Recycle(atomic_load_relaxed(&min_size_), cb); + } + + void NOINLINE DrainAndRecycle(Cache *c, Callback cb) { + { + SpinMutexLock l(&cache_mutex_); + cache_.Transfer(c); + } + recycle_mutex_.Lock(); + Recycle(0, cb); } void PrintStats() const { @@ -139,9 +147,8 @@ class Quarantine { Cache cache_; char pad2_[kCacheLineSize]; - void NOINLINE Recycle(Callback cb) { + void NOINLINE Recycle(uptr min_size, Callback cb) { Cache tmp; - uptr min_size = atomic_load(&min_size_, memory_order_relaxed); { SpinMutexLock l(&cache_mutex_); // Go over the batches and merge partially filled ones to @@ -201,7 +208,7 @@ class QuarantineCache { // Total memory used, including internal accounting. uptr Size() const { - return atomic_load(&size_, memory_order_relaxed); + return atomic_load_relaxed(&size_); } // Memory used for internal accounting. @@ -225,7 +232,7 @@ class QuarantineCache { list_.append_back(&from_cache->list_); SizeAdd(from_cache->Size()); - atomic_store(&from_cache->size_, 0, memory_order_relaxed); + atomic_store_relaxed(&from_cache->size_, 0); } void EnqueueBatch(QuarantineBatch *b) { @@ -296,10 +303,10 @@ class QuarantineCache { atomic_uintptr_t size_; void SizeAdd(uptr add) { - atomic_store(&size_, Size() + add, memory_order_relaxed); + atomic_store_relaxed(&size_, Size() + add); } void SizeSub(uptr sub) { - atomic_store(&size_, Size() - sub, memory_order_relaxed); + atomic_store_relaxed(&size_, Size() - sub); } }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h index 86536aa19658..060b58d3f2d3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h @@ -27,8 +27,9 @@ class SanitizerCommonDecorator { SanitizerCommonDecorator() : ansi_(ColorizeReports()) {} const char *Bold() const { return ansi_ ? "\033[1m" : ""; } const char *Default() const { return ansi_ ? "\033[1m\033[0m" : ""; } - const char *Warning() { return Red(); } - const char *EndWarning() { return Default(); } + const char *Warning() const { return Red(); } + const char *MemoryByte() { return Magenta(); } + protected: const char *Black() const { return ansi_ ? "\033[1m\033[30m" : ""; } const char *Red() const { return ansi_ ? "\033[1m\033[31m" : ""; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc new file mode 100644 index 000000000000..f51fc049e327 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc @@ -0,0 +1,87 @@ +//===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Signal interceptors for sanitizers. +// +//===----------------------------------------------------------------------===// + +#include "interception/interception.h" +#include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_interceptors.h" + +using namespace __sanitizer; + +#if SANITIZER_NETBSD +#define sigaction_symname __sigaction14 +#else +#define sigaction_symname sigaction +#endif + +#ifndef SIGNAL_INTERCEPTOR_SIGNAL_IMPL +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signum, handler) \ + { return REAL(func)(signum, handler); } +#endif + +#ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \ + { return REAL(sigaction_symname)(signum, act, oldact); } +#endif + +#if SANITIZER_INTERCEPT_BSD_SIGNAL +INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) { + if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0; + SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler); +} +#define INIT_BSD_SIGNAL COMMON_INTERCEPT_FUNCTION(bsd_signal) +#else // SANITIZER_INTERCEPT_BSD_SIGNAL +#define INIT_BSD_SIGNAL +#endif // SANITIZER_INTERCEPT_BSD_SIGNAL + +#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION +INTERCEPTOR(uptr, signal, int signum, uptr handler) { + if (GetHandleSignalMode(signum) == kHandleSignalExclusive) + return (uptr) nullptr; + SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler); +} +#define INIT_SIGNAL COMMON_INTERCEPT_FUNCTION(signal) + +INTERCEPTOR(int, sigaction_symname, int signum, + const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) { + if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0; + SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact); +} +#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction_symname) + +namespace __sanitizer { +int real_sigaction(int signum, const void *act, void *oldact) { + return REAL(sigaction_symname)(signum, (const __sanitizer_sigaction *)act, + (__sanitizer_sigaction *)oldact); +} +} // namespace __sanitizer +#else // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION +#define INIT_SIGNAL +#define INIT_SIGACTION +// We need to have defined REAL(sigaction) on other systems. +namespace __sanitizer { +struct __sanitizer_sigaction; +} +DEFINE_REAL(int, sigaction, int signum, const __sanitizer_sigaction *act, + __sanitizer_sigaction *oldact) +#endif // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION + +static void InitializeSignalInterceptors() { + static bool was_called_once; + CHECK(!was_called_once); + was_called_once = true; + + INIT_BSD_SIGNAL; + INIT_SIGNAL; + INIT_SIGACTION; +} diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc new file mode 100644 index 000000000000..a5db22994e0a --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc @@ -0,0 +1,219 @@ +//===-- sanitizer_solaris.cc ----------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between various sanitizers' runtime libraries and +// implements Solaris-specific functions. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_SOLARIS + +#include + +#include "sanitizer_common.h" +#include "sanitizer_flags.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_placement_new.h" +#include "sanitizer_platform_limits_posix.h" +#include "sanitizer_procmaps.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace __sanitizer { + +//#include "sanitizer_syscall_generic.inc" + +#define _REAL(func) _ ## func +#define DECLARE__REAL(ret_type, func, ...) \ + extern "C" ret_type _REAL(func)(__VA_ARGS__) +#define DECLARE__REAL_AND_INTERNAL(ret_type, func, ...) \ + DECLARE__REAL(ret_type, func, __VA_ARGS__); \ + ret_type internal_ ## func(__VA_ARGS__) + +// ---------------------- sanitizer_libc.h +DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length, + int prot, int flags, int fd, OFF_T offset) { + return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset); +} + +DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) { + return _REAL(munmap)(addr, length); +} + +DECLARE__REAL_AND_INTERNAL(int, mprotect, void *addr, uptr length, int prot) { + return _REAL(mprotect)(addr, length, prot); +} + +DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) { + return _REAL(close)(fd); +} + +extern "C" int _REAL(open)(const char *, int, ...); + +uptr internal_open(const char *filename, int flags) { + return _REAL(open)(filename, flags); +} + +uptr internal_open(const char *filename, int flags, u32 mode) { + return _REAL(open)(filename, flags, mode); +} + +uptr OpenFile(const char *filename, bool write) { + return internal_open(filename, + write ? O_WRONLY | O_CREAT : O_RDONLY, 0660); +} + +DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) { + return _REAL(read)(fd, buf, count); +} + +DECLARE__REAL_AND_INTERNAL(uptr, write, fd_t fd, const void *buf, uptr count) { + return _REAL(write)(fd, buf, count); +} + +// FIXME: There's only _ftruncate64 beginning with Solaris 11. +DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) { + return ftruncate(fd, size); +} + +DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) { + return _REAL(stat)(path, (struct stat *)buf); +} + +DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) { + return _REAL(lstat)(path, (struct stat *)buf); +} + +DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) { + return _REAL(fstat)(fd, (struct stat *)buf); +} + +uptr internal_filesize(fd_t fd) { + struct stat st; + if (internal_fstat(fd, &st)) + return -1; + return (uptr)st.st_size; +} + +DECLARE__REAL_AND_INTERNAL(uptr, dup2, int oldfd, int newfd) { + return _REAL(dup2)(oldfd, newfd); +} + +DECLARE__REAL_AND_INTERNAL(uptr, readlink, const char *path, char *buf, + uptr bufsize) { + return _REAL(readlink)(path, buf, bufsize); +} + +DECLARE__REAL_AND_INTERNAL(uptr, unlink, const char *path) { + return _REAL(unlink)(path); +} + +DECLARE__REAL_AND_INTERNAL(uptr, rename, const char *oldpath, + const char *newpath) { + return _REAL(rename)(oldpath, newpath); +} + +DECLARE__REAL_AND_INTERNAL(uptr, sched_yield, void) { + return sched_yield(); +} + +DECLARE__REAL_AND_INTERNAL(void, _exit, int exitcode) { + _exit(exitcode); +} + +DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename, + char *const argv[], char *const envp[]) { + return _REAL(execve)(filename, argv, envp); +} + +DECLARE__REAL_AND_INTERNAL(uptr, waitpid, int pid, int *status, int options) { + return _REAL(waitpid)(pid, status, options); +} + +DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) { + return _REAL(getpid)(); +} + +// FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *. +DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp, + unsigned int count) { + return _REAL(getdents)(fd, dirp, count); +} + +DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) { + return _REAL(lseek)(fd, offset, whence); +} + +// FIXME: This might be wrong: _sigfillset doesn't take a +// __sanitizer_sigset_t *. +DECLARE__REAL_AND_INTERNAL(void, sigfillset, __sanitizer_sigset_t *set) { + _REAL(sigfillset)(set); +} + +// FIXME: This might be wrong: _sigprocmask doesn't take __sanitizer_sigset_t *. +DECLARE__REAL_AND_INTERNAL(uptr, sigprocmask, int how, + __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + return _REAL(sigprocmask)(how, set, oldset); +} + +DECLARE__REAL_AND_INTERNAL(int, fork, void) { + // TODO(glider): this may call user's pthread_atfork() handlers which is bad. + return _REAL(fork)(); +} + +u64 NanoTime() { + return gethrtime(); +} + +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { + // FIXME: No internal variant. + return clock_gettime(clk_id, (timespec *)tp); +} + +// ----------------- sanitizer_common.h +BlockingMutex::BlockingMutex() { + CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_)); + internal_memset(this, 0, sizeof(*this)); + CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0); +} + +void BlockingMutex::Lock() { + CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_)); + CHECK_NE(owner_, (uptr)thr_self()); + CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0); + CHECK(!owner_); + owner_ = (uptr)thr_self(); +} + +void BlockingMutex::Unlock() { + CHECK(owner_ == (uptr)thr_self()); + owner_ = 0; + CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0); +} + +void BlockingMutex::CheckLocked() { + CHECK_EQ((uptr)thr_self(), owner_); +} + +} // namespace __sanitizer + +#endif // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index 90142dffd5e4..f15196e4b725 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -97,6 +97,11 @@ struct BufferedStackTrace : public StackTrace { void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind); + void Reset() { + *static_cast(this) = StackTrace(trace_buffer, 0); + top_frame_bp = 0; + } + private: void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, u32 max_depth); @@ -106,8 +111,8 @@ struct BufferedStackTrace : public StackTrace { void PopStackFrames(uptr count); uptr LocatePcInTrace(uptr pc); - BufferedStackTrace(const BufferedStackTrace &); - void operator=(const BufferedStackTrace &); + BufferedStackTrace(const BufferedStackTrace &) = delete; + void operator=(const BufferedStackTrace &) = delete; }; // Check if given pointer points into allocated stack area. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index 377f1ce755be..a271302708be 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -12,9 +12,14 @@ //===----------------------------------------------------------------------===// #include "sanitizer_stacktrace_printer.h" +#include "sanitizer_file.h" +#include "sanitizer_fuchsia.h" namespace __sanitizer { +// sanitizer_symbolizer_fuchsia.cc implements these differently for Fuchsia. +#if !SANITIZER_FUCHSIA + static const char *StripFunctionName(const char *function, const char *prefix) { if (!function) return nullptr; if (!prefix) return function; @@ -24,6 +29,80 @@ static const char *StripFunctionName(const char *function, const char *prefix) { return function; } +static const char *DemangleFunctionName(const char *function) { + if (!function) return nullptr; + + // NetBSD uses indirection for old threading functions for historical reasons + // The mangled names are internal implementation detail and should not be + // exposed even in backtraces. +#if SANITIZER_NETBSD + if (!internal_strcmp(function, "__libc_mutex_init")) + return "pthread_mutex_init"; + if (!internal_strcmp(function, "__libc_mutex_lock")) + return "pthread_mutex_lock"; + if (!internal_strcmp(function, "__libc_mutex_trylock")) + return "pthread_mutex_trylock"; + if (!internal_strcmp(function, "__libc_mutex_unlock")) + return "pthread_mutex_unlock"; + if (!internal_strcmp(function, "__libc_mutex_destroy")) + return "pthread_mutex_destroy"; + if (!internal_strcmp(function, "__libc_mutexattr_init")) + return "pthread_mutexattr_init"; + if (!internal_strcmp(function, "__libc_mutexattr_settype")) + return "pthread_mutexattr_settype"; + if (!internal_strcmp(function, "__libc_mutexattr_destroy")) + return "pthread_mutexattr_destroy"; + if (!internal_strcmp(function, "__libc_cond_init")) + return "pthread_cond_init"; + if (!internal_strcmp(function, "__libc_cond_signal")) + return "pthread_cond_signal"; + if (!internal_strcmp(function, "__libc_cond_broadcast")) + return "pthread_cond_broadcast"; + if (!internal_strcmp(function, "__libc_cond_wait")) + return "pthread_cond_wait"; + if (!internal_strcmp(function, "__libc_cond_timedwait")) + return "pthread_cond_timedwait"; + if (!internal_strcmp(function, "__libc_cond_destroy")) + return "pthread_cond_destroy"; + if (!internal_strcmp(function, "__libc_rwlock_init")) + return "pthread_rwlock_init"; + if (!internal_strcmp(function, "__libc_rwlock_rdlock")) + return "pthread_rwlock_rdlock"; + if (!internal_strcmp(function, "__libc_rwlock_wrlock")) + return "pthread_rwlock_wrlock"; + if (!internal_strcmp(function, "__libc_rwlock_tryrdlock")) + return "pthread_rwlock_tryrdlock"; + if (!internal_strcmp(function, "__libc_rwlock_trywrlock")) + return "pthread_rwlock_trywrlock"; + if (!internal_strcmp(function, "__libc_rwlock_unlock")) + return "pthread_rwlock_unlock"; + if (!internal_strcmp(function, "__libc_rwlock_destroy")) + return "pthread_rwlock_destroy"; + if (!internal_strcmp(function, "__libc_thr_keycreate")) + return "pthread_key_create"; + if (!internal_strcmp(function, "__libc_thr_setspecific")) + return "pthread_setspecific"; + if (!internal_strcmp(function, "__libc_thr_getspecific")) + return "pthread_getspecific"; + if (!internal_strcmp(function, "__libc_thr_keydelete")) + return "pthread_key_delete"; + if (!internal_strcmp(function, "__libc_thr_once")) + return "pthread_once"; + if (!internal_strcmp(function, "__libc_thr_self")) + return "pthread_self"; + if (!internal_strcmp(function, "__libc_thr_exit")) + return "pthread_exit"; + if (!internal_strcmp(function, "__libc_thr_setcancelstate")) + return "pthread_setcancelstate"; + if (!internal_strcmp(function, "__libc_thr_equal")) + return "pthread_equal"; + if (!internal_strcmp(function, "__libc_thr_curcpu")) + return "pthread_curcpu_np"; +#endif + + return function; +} + static const char kDefaultFormat[] = " #%n %p %F %L"; void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, @@ -55,7 +134,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, buffer->append("0x%zx", info.module_offset); break; case 'f': - buffer->append("%s", StripFunctionName(info.function, strip_func_prefix)); + buffer->append("%s", + DemangleFunctionName( + StripFunctionName(info.function, strip_func_prefix))); break; case 'q': buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown @@ -76,7 +157,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, // Function name and offset, if file is unknown. if (info.function) { buffer->append("in %s", - StripFunctionName(info.function, strip_func_prefix)); + DemangleFunctionName( + StripFunctionName(info.function, strip_func_prefix))); if (!info.file && info.function_offset != AddressInfo::kUnknown) buffer->append("+0x%zx", info.function_offset); } @@ -146,6 +228,8 @@ void RenderData(InternalScopedString *buffer, const char *format, } } +#endif // !SANITIZER_FUCHSIA + void RenderSourceLocation(InternalScopedString *buffer, const char *file, int line, int column, bool vs_style, const char *strip_path_prefix) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index d7fa5f6451d1..0f543d2d7c87 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -57,6 +57,14 @@ #include "sanitizer_mutex.h" #include "sanitizer_placement_new.h" +// Sufficiently old kernel headers don't provide this value, but we can still +// call prctl with it. If the runtime kernel is new enough, the prctl call will +// have the desired effect; if the kernel is too old, the call will error and we +// can ignore said error. +#ifndef PR_SET_PTRACER +#define PR_SET_PTRACER 0x59616d61 +#endif + // This module works by spawning a Linux task which then attaches to every // thread in the caller process with ptrace. This suspends the threads, and // PTRACE_GETREGS can then be used to obtain their register state. The callback @@ -245,8 +253,9 @@ static void TracerThreadDieCallback() { } // Signal handler to wake up suspended threads when the tracer thread dies. -static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { - SignalContext ctx = SignalContext::Create(siginfo, uctx); +static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo, + void *uctx) { + SignalContext ctx(siginfo, uctx); Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum, ctx.addr, ctx.pc, ctx.sp); ThreadSuspender *inst = thread_suspender_instance; @@ -263,7 +272,7 @@ static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { } // Size of alternative stack for signal handlers in the tracer thread. -static const int kHandlerStackSize = 4096; +static const int kHandlerStackSize = 8192; // This function will be run as a cloned task. static int TracerThread(void* argument) { @@ -433,9 +442,7 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) { ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid); // On some systems we have to explicitly declare that we want to be traced // by the tracer thread. -#ifdef PR_SET_PTRACER internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0); -#endif // Allow the tracer thread to start. tracer_thread_argument.mutex.Unlock(); // NOTE: errno is shared between this thread and the tracer thread. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc index 0c27c472f02e..d84ebef3b7f9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc @@ -55,17 +55,9 @@ void RunThread(void *arg) { struct RunThreadArgs *run_args = (struct RunThreadArgs *)arg; SuspendedThreadsListMac suspended_threads_list; - mach_port_t task; - kern_return_t err = task_for_pid(mach_task_self(), internal_getpid(), &task); - if (err != KERN_SUCCESS) { - VReport(1, "Getting task from pid failed (errno %d).\n", err); - return; - } - thread_array_t threads; mach_msg_type_number_t num_threads; - - err = task_threads(task, &threads, &num_threads); + kern_return_t err = task_threads(mach_task_self(), &threads, &num_threads); if (err != KERN_SUCCESS) { VReport(1, "Failed to get threads for task (errno %d).\n", err); return; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc index f0f2c9c725a1..89ddfddd1108 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc @@ -16,6 +16,7 @@ #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_file.h" #include "sanitizer_libc.h" #include "sanitizer_placement_new.h" @@ -50,6 +51,7 @@ void SuppressionContext::ParseFromFile(const char *filename) { if (filename[0] == '\0') return; +#if !SANITIZER_FUCHSIA // If we cannot find the file, check if its location is relative to // the location of the executable. InternalScopedString new_file_path(kMaxPathLength); @@ -58,6 +60,7 @@ void SuppressionContext::ParseFromFile(const char *filename) { new_file_path.size())) { filename = new_file_path.data(); } +#endif // !SANITIZER_FUCHSIA // Read the file. VPrintf(1, "%s: reading suppressions file at %s\n", diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc index 1cd5b6ee24c0..672c936684be 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc @@ -71,6 +71,10 @@ Symbolizer *Symbolizer::symbolizer_; StaticSpinMutex Symbolizer::init_mu_; LowLevelAllocator Symbolizer::symbolizer_allocator_; +void Symbolizer::InvalidateModuleList() { + modules_fresh_ = false; +} + void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook, Symbolizer::EndSymbolizationHook end_hook) { CHECK(start_hook_ == 0 && end_hook_ == 0); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h index 4fc7742a21e3..208362d2f478 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h @@ -119,8 +119,11 @@ class Symbolizer final { void AddHooks(StartSymbolizationHook start_hook, EndSymbolizationHook end_hook); + void RefreshModules(); const LoadedModule *FindModuleForAddress(uptr address); + void InvalidateModuleList(); + private: // GetModuleNameAndOffsetForPC has to return a string to the caller. // Since the corresponding module might get unloaded later, we should create @@ -149,6 +152,7 @@ class Symbolizer final { uptr *module_offset, ModuleArch *module_arch); ListOfModules modules_; + ListOfModules fallback_modules_; // If stale, need to reload the modules before looking up addresses. bool modules_fresh_; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc new file mode 100644 index 000000000000..3d1117d9d3ca --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc @@ -0,0 +1,107 @@ +//===-- sanitizer_symbolizer_fuchsia.cc -----------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between various sanitizers' runtime libraries. +// +// Implementation of Fuchsia-specific symbolizer. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_FUCHSIA + +#include "sanitizer_fuchsia.h" +#include "sanitizer_symbolizer.h" + +namespace __sanitizer { + +// For Fuchsia we don't do any actual symbolization per se. +// Instead, we emit text containing raw addresses and raw linkage +// symbol names, embedded in Fuchsia's symbolization markup format. +// Fuchsia's logging infrastructure emits enough information about +// process memory layout that a post-processing filter can do the +// symbolization and pretty-print the markup. See the spec at: +// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md + +// This is used by UBSan for type names, and by ASan for global variable names. +constexpr const char *kFormatDemangle = "{{{symbol:%s}}}"; +constexpr uptr kFormatDemangleMax = 1024; // Arbitrary. + +// Function name or equivalent from PC location. +constexpr const char *kFormatFunction = "{{{pc:%p}}}"; +constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex. + +// Global variable name or equivalent from data memory address. +constexpr const char *kFormatData = "{{{data:%p}}}"; + +// One frame in a backtrace (printed on a line by itself). +constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}"; + +// This is used by UBSan for type names, and by ASan for global variable names. +// It's expected to return a static buffer that will be reused on each call. +const char *Symbolizer::Demangle(const char *name) { + static char buffer[kFormatDemangleMax]; + internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name); + return buffer; +} + +// This is used mostly for suppression matching. Making it work +// would enable "interceptor_via_lib" suppressions. It's also used +// once in UBSan to say "in module ..." in a message that also +// includes an address in the module, so post-processing can already +// pretty-print that so as to indicate the module. +bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, + uptr *module_address) { + return false; +} + +// This is used in some places for suppression checking, which we +// don't really support for Fuchsia. It's also used in UBSan to +// identify a PC location to a function name, so we always fill in +// the function member with a string containing markup around the PC +// value. +// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan +// to render stack frames, but that should be changed to use +// RenderStackFrame. +SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) { + SymbolizedStack *s = SymbolizedStack::New(addr); + char buffer[kFormatFunctionMax]; + internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr); + s->info.function = internal_strdup(buffer); + return s; +} + +// Always claim we succeeded, so that RenderDataInfo will be called. +bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) { + info->Clear(); + info->start = addr; + return true; +} + +// We ignore the format argument to __sanitizer_symbolize_global. +void RenderData(InternalScopedString *buffer, const char *format, + const DataInfo *DI, const char *strip_path_prefix) { + buffer->append(kFormatData, DI->start); +} + +// We don't support the stack_trace_format flag at all. +void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, + const AddressInfo &info, bool vs_style, + const char *strip_path_prefix, const char *strip_func_prefix) { + buffer->append(kFormatFrame, frame_no, info.address); +} + +Symbolizer *Symbolizer::PlatformInit() { + return new (symbolizer_allocator_) Symbolizer({}); +} + +void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); } + +} // namespace __sanitizer + +#endif // SANITIZER_FUCHSIA diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h index 2ae42b33868b..a2ac1188241d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -15,6 +15,7 @@ #define SANITIZER_SYMBOLIZER_INTERNAL_H #include "sanitizer_symbolizer.h" +#include "sanitizer_file.h" namespace __sanitizer { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc index 57354668bfbb..e98fab01d6bb 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc @@ -95,7 +95,8 @@ struct SymbolizeCodeCallbackArg { if (frames_symbolized > 0) { SymbolizedStack *cur = SymbolizedStack::New(addr); AddressInfo *info = &cur->info; - info->FillModuleInfo(first->info.module, first->info.module_offset); + info->FillModuleInfo(first->info.module, first->info.module_offset, + first->info.module_arch); last->next = cur; last = cur; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc index 614470a633d0..a4bab668b27a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc @@ -17,6 +17,18 @@ namespace __sanitizer { +Symbolizer *Symbolizer::GetOrInit() { + SpinMutexLock l(&init_mu_); + if (symbolizer_) + return symbolizer_; + symbolizer_ = PlatformInit(); + CHECK(symbolizer_); + return symbolizer_; +} + +// See sanitizer_symbolizer_fuchsia.cc. +#if !SANITIZER_FUCHSIA + const char *ExtractToken(const char *str, const char *delims, char **result) { uptr prefix_len = internal_strcspn(str, delims); *result = (char*)InternalAlloc(prefix_len + 1); @@ -151,37 +163,47 @@ bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address, return true; } +void Symbolizer::RefreshModules() { + modules_.init(); + fallback_modules_.fallbackInit(); + RAW_CHECK(modules_.size() > 0); + modules_fresh_ = true; +} + +static const LoadedModule *SearchForModule(const ListOfModules &modules, + uptr address) { + for (uptr i = 0; i < modules.size(); i++) { + if (modules[i].containsAddress(address)) { + return &modules[i]; + } + } + return nullptr; +} + const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) { bool modules_were_reloaded = false; if (!modules_fresh_) { - modules_.init(); - RAW_CHECK(modules_.size() > 0); - modules_fresh_ = true; + RefreshModules(); modules_were_reloaded = true; } - for (uptr i = 0; i < modules_.size(); i++) { - if (modules_[i].containsAddress(address)) { - return &modules_[i]; - } - } - // Reload the modules and look up again, if we haven't tried it yet. - if (!modules_were_reloaded) { - // FIXME: set modules_fresh_ from dlopen()/dlclose() interceptors. - // It's too aggressive to reload the list of modules each time we fail - // to find a module for a given address. - modules_fresh_ = false; - return FindModuleForAddress(address); - } - return 0; -} + const LoadedModule *module = SearchForModule(modules_, address); + if (module) return module; -Symbolizer *Symbolizer::GetOrInit() { - SpinMutexLock l(&init_mu_); - if (symbolizer_) - return symbolizer_; - symbolizer_ = PlatformInit(); - CHECK(symbolizer_); - return symbolizer_; + // dlopen/dlclose interceptors invalidate the module list, but when + // interception is disabled, we need to retry if the lookup fails in + // case the module list changed. +#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE + if (!modules_were_reloaded) { + RefreshModules(); + module = SearchForModule(modules_, address); + if (module) return module; + } +#endif + + if (fallback_modules_.size()) { + module = SearchForModule(fallback_modules_, address); + } + return module; } // For now we assume the following protocol: @@ -451,7 +473,7 @@ bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) { if (ReachedEndOfOutput(buffer, read_len)) break; if (read_len + 1 == max_length) { - Report("WARNING: Symbolizer buffer too small"); + Report("WARNING: Symbolizer buffer too small\n"); read_len = 0; break; } @@ -472,4 +494,6 @@ bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) { return true; } +#endif // !SANITIZER_FUCHSIA + } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index 60ec7506c312..71d748d0515f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -16,6 +16,7 @@ #if SANITIZER_POSIX #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" +#include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" #include "sanitizer_linux.h" @@ -76,6 +77,7 @@ static swift_demangle_ft swift_demangle_f; // symbolication. static void InitializeSwiftDemangler() { swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle"); + (void)dlerror(); // Cleanup error message in case of failure } // Attempts to demangle a Swift name. The demangler will return nullptr if a @@ -271,6 +273,10 @@ class Addr2LineProcess : public SymbolizerProcess { bool ReadFromSymbolizer(char *buffer, uptr max_length) override { if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length)) return false; + // The returned buffer is empty when output is valid, but exceeds + // max_length. + if (*buffer == '\0') + return true; // We should cut out output_terminator_ at the end of given buffer, // appended by addr2line to mark the end of its meaningful output. // We cannot scan buffer from it's beginning, because it is legal for it @@ -470,16 +476,16 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) { // Otherwise symbolizer program is unknown, let's search $PATH CHECK(path == nullptr); - if (const char *found_path = FindPathToBinary("llvm-symbolizer")) { - VReport(2, "Using llvm-symbolizer found at: %s\n", found_path); - return new(*allocator) LLVMSymbolizer(found_path, allocator); - } #if SANITIZER_MAC if (const char *found_path = FindPathToBinary("atos")) { VReport(2, "Using atos found at: %s\n", found_path); return new(*allocator) AtosSymbolizer(found_path, allocator); } #endif // SANITIZER_MAC + if (const char *found_path = FindPathToBinary("llvm-symbolizer")) { + VReport(2, "Using llvm-symbolizer found at: %s\n", found_path); + return new(*allocator) LLVMSymbolizer(found_path, allocator); + } if (common_flags()->allow_addr2line) { if (const char *found_path = FindPathToBinary("addr2line")) { VReport(2, "Using addr2line found at: %s\n", found_path); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc index 15cf05f06087..3c8e77867ce5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -7,20 +7,49 @@ // //===----------------------------------------------------------------------===// // -// Generic implementations of internal_syscall and internal_iserror. +// Generic implementations of internal_syscall* and internal_iserror. // //===----------------------------------------------------------------------===// -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS # define SYSCALL(name) SYS_ ## name #else # define SYSCALL(name) __NR_ ## name #endif -#if (SANITIZER_FREEBSD || SANITIZER_MAC) && defined(__x86_64__) +#if SANITIZER_NETBSD +// We use 3 kinds of internal_syscall's for different types of retval in order +// to address differences in calling conventions (e.g. registers to place the +// return value in). +// - internal_syscall for 32-bit length (int, pid_t) +// - internal_syscall64 for 64-bit length (off_t) +// - internal_syscall_ptr for pointer and (s)size_t +# define internal_syscall syscall +# define internal_syscall64 __syscall +// Handle syscall renames manually +# define SYS_stat SYS___stat50 +# define SYS_lstat SYS___lstat50 +# define SYS_fstat SYS___fstat50 +# define SYS_gettimeofday SYS___gettimeofday50 +# define SYS_wait4 SYS___wait450 +# define SYS_getdents SYS___getdents30 +# define SYS_sigaltstack SYS___sigaltstack14 +# define SYS_sigprocmask SYS___sigprocmask14 +# define SYS_nanosleep SYS___nanosleep50 +# define SYS_clock_gettime SYS___clock_gettime50 +# if SANITIZER_WORDSIZE == 64 +# define internal_syscall_ptr __syscall +# else +# define internal_syscall_ptr syscall +# endif +#elif defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) # define internal_syscall __syscall +# define internal_syscall64 __syscall +# define internal_syscall_ptr __syscall # else # define internal_syscall syscall +# define internal_syscall64 syscall +# define internal_syscall_ptr syscall #endif bool internal_iserror(uptr retval, int *rverrno) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc index 7ab1d7641449..1f05ed9b6c1c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -127,6 +127,9 @@ static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, #define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) +#define internal_syscall_ptr internal_syscall +#define internal_syscall64 internal_syscall + // Helper function used to avoid cobbler errno. bool internal_iserror(uptr retval, int *rverrno) { if (retval >= (uptr)-4095) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc new file mode 100644 index 000000000000..a3fdb9e60d5d --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc @@ -0,0 +1,141 @@ +//===-- sanitizer_syscall_linux_arm.inc -------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementations of internal_syscall and internal_iserror for Linux/arm. +// +//===----------------------------------------------------------------------===// + +#define SYSCALL(name) __NR_ ## name + +static uptr __internal_syscall(u32 nr) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0"); + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8) + : "memory", "cc"); + return r0; +} +#define __internal_syscall0(n) \ + (__internal_syscall)(n) + +static uptr __internal_syscall(u32 nr, u32 arg1) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0) + : "memory", "cc"); + return r0; +} +#define __internal_syscall1(n, a1) \ + (__internal_syscall)(n, (u32)(a1)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1) + : "memory", "cc"); + return r0; +} +#define __internal_syscall2(n, a1, a2) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2) + : "memory", "cc"); + return r0; +} +#define __internal_syscall3(n, a1, a2, a3) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, + u32 arg4) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + register u32 r3 asm("r3") = arg4; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3) + : "memory", "cc"); + return r0; +} +#define __internal_syscall4(n, a1, a2, a3, a4) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, + u32 arg4, long arg5) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + register u32 r3 asm("r3") = arg4; + register u32 r4 asm("r4") = arg5; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4) + : "memory", "cc"); + return r0; +} +#define __internal_syscall5(n, a1, a2, a3, a4, a5) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u32)(a5)) + +static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, + u32 arg4, long arg5, long arg6) { + register u32 r8 asm("r7") = nr; + register u32 r0 asm("r0") = arg1; + register u32 r1 asm("r1") = arg2; + register u32 r2 asm("r2") = arg3; + register u32 r3 asm("r3") = arg4; + register u32 r4 asm("r4") = arg5; + register u32 r5 asm("r5") = arg6; + asm volatile("swi #0" + : "=r"(r0) + : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5) + : "memory", "cc"); + return r0; +} +#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \ + (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u32)(a5), (long)(a6)) + +#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) + +#define internal_syscall_ptr internal_syscall +#define internal_syscall64 internal_syscall + +// Helper function used to avoid cobbler errno. +bool internal_iserror(uptr retval, int *rverrno) { + if (retval >= (uptr)-4095) { + if (rverrno) + *rverrno = -retval; + return true; + } + return false; +} diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc index 9853a6a675d3..327aaa80a674 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc @@ -20,6 +20,9 @@ static uptr internal_syscall(u64 nr) { return retval; } +#define internal_syscall_ptr internal_syscall +#define internal_syscall64 internal_syscall + template static uptr internal_syscall(u64 nr, T1 arg1) { u64 retval; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc index 439e33a08d01..d9fd6549ba48 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc @@ -21,6 +21,7 @@ ThreadContextBase::ThreadContextBase(u32 tid) status(ThreadStatusInvalid), detached(false), workerthread(false), parent_tid(0), next(0) { name[0] = '\0'; + atomic_store(&thread_destroyed, 0, memory_order_release); } ThreadContextBase::~ThreadContextBase() { @@ -44,6 +45,14 @@ void ThreadContextBase::SetDead() { OnDead(); } +void ThreadContextBase::SetDestroyed() { + atomic_store(&thread_destroyed, 1, memory_order_release); +} + +bool ThreadContextBase::GetDestroyed() { + return !!atomic_load(&thread_destroyed, memory_order_acquire); +} + void ThreadContextBase::SetJoined(void *arg) { // FIXME(dvyukov): print message and continue (it's user error). CHECK_EQ(false, detached); @@ -54,8 +63,11 @@ void ThreadContextBase::SetJoined(void *arg) { } void ThreadContextBase::SetFinished() { - if (!detached) - status = ThreadStatusFinished; + // ThreadRegistry::FinishThread calls here in ThreadStatusCreated state + // for a thread that never actually started. In that case the thread + // should go to ThreadStatusFinished regardless of whether it was created + // as detached. + if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished; OnFinished(); } @@ -82,6 +94,7 @@ void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id, void ThreadContextBase::Reset() { status = ThreadStatusInvalid; SetName(0); + atomic_store(&thread_destroyed, 0, memory_order_release); OnReset(); } @@ -204,7 +217,8 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) { CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); - CHECK_EQ(ThreadStatusRunning, tctx->status); + CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning, + tctx->status); tctx->SetName(name); } @@ -239,33 +253,54 @@ void ThreadRegistry::DetachThread(u32 tid, void *arg) { } void ThreadRegistry::JoinThread(u32 tid, void *arg) { - BlockingMutexLock l(&mtx_); - CHECK_LT(tid, n_contexts_); - ThreadContextBase *tctx = threads_[tid]; - CHECK_NE(tctx, 0); - if (tctx->status == ThreadStatusInvalid) { - Report("%s: Join of non-existent thread\n", SanitizerToolName); - return; - } - tctx->SetJoined(arg); - QuarantinePush(tctx); + bool destroyed = false; + do { + { + BlockingMutexLock l(&mtx_); + CHECK_LT(tid, n_contexts_); + ThreadContextBase *tctx = threads_[tid]; + CHECK_NE(tctx, 0); + if (tctx->status == ThreadStatusInvalid) { + Report("%s: Join of non-existent thread\n", SanitizerToolName); + return; + } + if ((destroyed = tctx->GetDestroyed())) { + tctx->SetJoined(arg); + QuarantinePush(tctx); + } + } + if (!destroyed) + internal_sched_yield(); + } while (!destroyed); } +// Normally this is called when the thread is about to exit. If +// called in ThreadStatusCreated state, then this thread was never +// really started. We just did CreateThread for a prospective new +// thread before trying to create it, and then failed to actually +// create it, and so never called StartThread. void ThreadRegistry::FinishThread(u32 tid) { BlockingMutexLock l(&mtx_); CHECK_GT(alive_threads_, 0); alive_threads_--; - CHECK_GT(running_threads_, 0); - running_threads_--; CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); - CHECK_EQ(ThreadStatusRunning, tctx->status); + bool dead = tctx->detached; + if (tctx->status == ThreadStatusRunning) { + CHECK_GT(running_threads_, 0); + running_threads_--; + } else { + // The thread never really existed. + CHECK_EQ(tctx->status, ThreadStatusCreated); + dead = true; + } tctx->SetFinished(); - if (tctx->detached) { + if (dead) { tctx->SetDead(); QuarantinePush(tctx); } + tctx->SetDestroyed(); } void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h index 9aae875c7360..b203be2f4dbc 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h @@ -50,6 +50,8 @@ class ThreadContextBase { u32 parent_tid; ThreadContextBase *next; // For storing thread contexts in a list. + atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished + void SetName(const char *new_name); void SetDead(); @@ -60,6 +62,9 @@ class ThreadContextBase { u32 _parent_tid, void *arg); void Reset(); + void SetDestroyed(); + bool GetDestroyed(); + // The following methods may be overriden by subclasses. // Some of them take opaque arg that may be optionally be used // by subclasses. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc index 29db37b8a464..6fa0e9298b31 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc @@ -45,7 +45,7 @@ static const uptr kDestroyedThread = -1; static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) { if (!size) return; - VPrintf(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size); + VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size); UnmapOrDie(dtv, size * sizeof(DTLS::DTV)); atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed); } @@ -58,7 +58,7 @@ static inline void DTLS_Resize(uptr new_size) { (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize"); uptr num_live_dtls = atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed); - VPrintf(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls); + VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls); CHECK_LT(num_live_dtls, 1 << 20); uptr old_dtv_size = dtls.dtv_size; DTLS::DTV *old_dtv = dtls.dtv; @@ -72,7 +72,7 @@ static inline void DTLS_Resize(uptr new_size) { void DTLS_Destroy() { if (!common_flags()->intercept_tls_get_addr) return; - VPrintf(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size); + VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size); uptr s = dtls.dtv_size; dtls.dtv_size = kDestroyedThread; // Do this before unmap for AS-safety. DTLS_Deallocate(dtls.dtv, s); @@ -97,28 +97,28 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, if (dtls.dtv[dso_id].beg) return 0; uptr tls_size = 0; uptr tls_beg = reinterpret_cast(res) - arg->offset - kDtvOffset; - VPrintf(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p " + VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p " "num_live_dtls %zd\n", arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg, atomic_load(&number_of_live_dtls, memory_order_relaxed)); if (dtls.last_memalign_ptr == tls_beg) { tls_size = dtls.last_memalign_size; - VPrintf(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n", + VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n", tls_beg, tls_size); } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) { // This is the static TLS block which was initialized / unpoisoned at thread // creation. - VPrintf(2, "__tls_get_addr: static tls: %p\n", tls_beg); + VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg); tls_size = 0; } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) { // We may want to check gnu_get_libc_version(). Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1; tls_size = header->size; tls_beg = header->start; - VPrintf(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n", + VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n", tls_beg, tls_size); } else { - VPrintf(2, "__tls_get_addr: Can't guess glibc version\n"); + VReport(2, "__tls_get_addr: Can't guess glibc version\n"); // This may happen inside the DTOR of main thread, so just ignore it. tls_size = 0; } @@ -129,7 +129,7 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, void DTLS_on_libc_memalign(void *ptr, uptr size) { if (!common_flags()->intercept_tls_get_addr) return; - VPrintf(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size); + VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size); dtls.last_memalign_ptr = reinterpret_cast(ptr); dtls.last_memalign_size = size; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index 5943125c898c..24c94700173d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -8,11 +8,12 @@ //===----------------------------------------------------------------------===// // // This file contains the unwind.h-based (aka "slow") stack unwinding routines -// available to the tools on Linux, Android, and FreeBSD. +// available to the tools on Linux, Android, NetBSD, FreeBSD, and Solaris. //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_stacktrace.h" @@ -78,7 +79,8 @@ void SanitizerInitializeUnwinder() { } #endif -#ifdef __arm__ +#if defined(__arm__) && !SANITIZER_NETBSD +// NetBSD uses dwarf EH #define UNWIND_STOP _URC_END_OF_STACK #define UNWIND_CONTINUE _URC_NO_REASON #else @@ -165,4 +167,5 @@ void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || + // SANITIZER_SOLARIS diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h similarity index 78% rename from contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h rename to contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h index a7fb3fa58d54..25cfeed35f22 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h @@ -1,4 +1,4 @@ -//===-- tsan_vector.h -------------------------------------------*- C++ -*-===// +//===-- sanitizer_vector.h -------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,38 +7,37 @@ // //===----------------------------------------------------------------------===// // -// This file is a part of ThreadSanitizer (TSan), a race detector. +// This file is shared between sanitizers run-time libraries. // //===----------------------------------------------------------------------===// // Low-fat STL-like vector container. -#ifndef TSAN_VECTOR_H -#define TSAN_VECTOR_H +#ifndef SANITIZER_VECTOR_H +#define SANITIZER_VECTOR_H -#include "tsan_defs.h" -#include "tsan_mman.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_libc.h" -namespace __tsan { +namespace __sanitizer { template class Vector { public: - explicit Vector(MBlockType typ) - : typ_(typ) - , begin_() + explicit Vector() + : begin_() , end_() , last_() { } ~Vector() { if (begin_) - internal_free(begin_); + InternalFree(begin_); } void Reset() { if (begin_) - internal_free(begin_); + InternalFree(begin_); begin_ = 0; end_ = 0; last_ = 0; @@ -91,7 +90,6 @@ class Vector { } private: - const MBlockType typ_; T *begin_; T *end_; T *last_; @@ -109,10 +107,10 @@ class Vector { cap = 16; if (cap < size) cap = size; - T *p = (T*)internal_alloc(typ_, cap * sizeof(T)); + T *p = (T*)InternalAlloc(cap * sizeof(T)); if (cap0) { internal_memcpy(p, begin_, cap0 * sizeof(T)); - internal_free(begin_); + InternalFree(begin_); } begin_ = p; end_ = begin_ + size; @@ -122,6 +120,6 @@ class Vector { Vector(const Vector&); void operator=(const Vector&); }; -} // namespace __tsan +} // namespace __sanitizer -#endif // #ifndef TSAN_VECTOR_H +#endif // #ifndef SANITIZER_VECTOR_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc index de01e8d119a1..34bf1812d605 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -24,10 +24,10 @@ #include "sanitizer_common.h" #include "sanitizer_dbghelp.h" +#include "sanitizer_file.h" #include "sanitizer_libc.h" #include "sanitizer_mutex.h" #include "sanitizer_placement_new.h" -#include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" #include "sanitizer_win_defs.h" @@ -64,12 +64,16 @@ uptr GetMmapGranularity() { return si.dwAllocationGranularity; } -uptr GetMaxVirtualAddress() { +uptr GetMaxUserVirtualAddress() { SYSTEM_INFO si; GetSystemInfo(&si); return (uptr)si.lpMaximumApplicationAddress; } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + bool FileExists(const char *filename) { return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES; } @@ -235,6 +239,28 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) { return p; } +// Uses fixed_addr for now. +// Will use offset instead once we've implemented this function for real. +uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) { + return reinterpret_cast(MmapFixedOrDieOnFatalError(fixed_addr, size)); +} + +uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) { + return reinterpret_cast(MmapFixedOrDie(fixed_addr, size)); +} + +void ReservedAddressRange::Unmap(uptr addr, uptr size) { + void* addr_as_void = reinterpret_cast(addr); + uptr base_as_uptr = reinterpret_cast(base_); + // Only unmap if it covers the entire range. + CHECK((addr == base_as_uptr) && (size == size_)); + UnmapOrDie(addr_as_void, size); + if (addr_as_void == base_) { + base_ = reinterpret_cast(addr + size); + } + size_ = size_ - size; +} + void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) { void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_COMMIT, PAGE_READWRITE); @@ -252,6 +278,19 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) { return MmapOrDie(size, mem_type); } +uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { + if (fixed_addr) { + base_ = MmapFixedNoAccess(fixed_addr, size, name); + } else { + base_ = MmapNoAccess(size); + } + size_ = size; + name_ = name; + (void)os_handle_; // unsupported + return reinterpret_cast(base_); +} + + void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { (void)name; // unsupported void *res = VirtualAlloc((LPVOID)fixed_addr, size, @@ -379,7 +418,7 @@ struct ModuleInfo { #if !SANITIZER_GO int CompareModulesBase(const void *pl, const void *pr) { - const ModuleInfo *l = (ModuleInfo *)pl, *r = (ModuleInfo *)pr; + const ModuleInfo *l = (const ModuleInfo *)pl, *r = (const ModuleInfo *)pr; if (l->base_address < r->base_address) return -1; return l->base_address > r->base_address; @@ -466,6 +505,10 @@ u64 NanoTime() { return 0; } +u64 MonotonicNanoTime() { + return 0; +} + void Abort() { internal__exit(3); } @@ -524,7 +567,7 @@ static uptr GetPreferredBase(const char *modname) { } void ListOfModules::init() { - clear(); + clearOrInit(); HANDLE cur_process = GetCurrentProcess(); // Query the list of modules. Start by assuming there are no more than 256 @@ -583,7 +626,9 @@ void ListOfModules::init() { modules_.push_back(cur_module); } UnmapOrDie(hmodules, modules_buffer_size); -}; +} + +void ListOfModules::fallbackInit() { clear(); } // We can't use atexit() directly at __asan_init time as the CRT is not fully // initialized at this point. Place the functions into a vector and use @@ -792,7 +837,7 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { // FIXME: Compare with StackWalk64. // FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax), - (void**)trace, 0); + (void **)&trace_buffer[0], 0); if (size == 0) return; @@ -889,32 +934,6 @@ bool IsHandledDeadlyException(DWORD exceptionCode) { return false; } -const char *DescribeSignalOrException(int signo) { - unsigned code = signo; - // Get the string description of the exception if this is a known deadly - // exception. - switch (code) { - case EXCEPTION_ACCESS_VIOLATION: return "access-violation"; - case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: return "array-bounds-exceeded"; - case EXCEPTION_STACK_OVERFLOW: return "stack-overflow"; - case EXCEPTION_DATATYPE_MISALIGNMENT: return "datatype-misalignment"; - case EXCEPTION_IN_PAGE_ERROR: return "in-page-error"; - case EXCEPTION_ILLEGAL_INSTRUCTION: return "illegal-instruction"; - case EXCEPTION_PRIV_INSTRUCTION: return "priv-instruction"; - case EXCEPTION_BREAKPOINT: return "breakpoint"; - case EXCEPTION_FLT_DENORMAL_OPERAND: return "flt-denormal-operand"; - case EXCEPTION_FLT_DIVIDE_BY_ZERO: return "flt-divide-by-zero"; - case EXCEPTION_FLT_INEXACT_RESULT: return "flt-inexact-result"; - case EXCEPTION_FLT_INVALID_OPERATION: return "flt-invalid-operation"; - case EXCEPTION_FLT_OVERFLOW: return "flt-overflow"; - case EXCEPTION_FLT_STACK_CHECK: return "flt-stack-check"; - case EXCEPTION_FLT_UNDERFLOW: return "flt-underflow"; - case EXCEPTION_INT_DIVIDE_BY_ZERO: return "int-divide-by-zero"; - case EXCEPTION_INT_OVERFLOW: return "int-overflow"; - } - return "unknown exception"; -} - bool IsAccessibleMemoryRange(uptr beg, uptr size) { SYSTEM_INFO si; GetNativeSystemInfo(&si); @@ -940,39 +959,101 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { return true; } -SignalContext SignalContext::Create(void *siginfo, void *context) { +bool SignalContext::IsStackOverflow() const { + return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW; +} + +void SignalContext::InitPcSpBp() { EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; CONTEXT *context_record = (CONTEXT *)context; - uptr pc = (uptr)exception_record->ExceptionAddress; + pc = (uptr)exception_record->ExceptionAddress; #ifdef _WIN64 - uptr bp = (uptr)context_record->Rbp; - uptr sp = (uptr)context_record->Rsp; + bp = (uptr)context_record->Rbp; + sp = (uptr)context_record->Rsp; #else - uptr bp = (uptr)context_record->Ebp; - uptr sp = (uptr)context_record->Esp; + bp = (uptr)context_record->Ebp; + sp = (uptr)context_record->Esp; #endif - uptr access_addr = exception_record->ExceptionInformation[1]; +} +uptr SignalContext::GetAddress() const { + EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; + return exception_record->ExceptionInformation[1]; +} + +bool SignalContext::IsMemoryAccess() const { + return GetWriteFlag() != SignalContext::UNKNOWN; +} + +SignalContext::WriteFlag SignalContext::GetWriteFlag() const { + EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; // The contents of this array are documented at // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx // The first element indicates read as 0, write as 1, or execute as 8. The // second element is the faulting address. - WriteFlag write_flag = SignalContext::UNKNOWN; switch (exception_record->ExceptionInformation[0]) { - case 0: write_flag = SignalContext::READ; break; - case 1: write_flag = SignalContext::WRITE; break; - case 8: write_flag = SignalContext::UNKNOWN; break; + case 0: + return SignalContext::READ; + case 1: + return SignalContext::WRITE; + case 8: + return SignalContext::UNKNOWN; } - bool is_memory_access = write_flag != SignalContext::UNKNOWN; - return SignalContext(context, access_addr, pc, sp, bp, is_memory_access, - write_flag); + return SignalContext::UNKNOWN; } void SignalContext::DumpAllRegisters(void *context) { // FIXME: Implement this. } +int SignalContext::GetType() const { + return static_cast(siginfo)->ExceptionCode; +} + +const char *SignalContext::Describe() const { + unsigned code = GetType(); + // Get the string description of the exception if this is a known deadly + // exception. + switch (code) { + case EXCEPTION_ACCESS_VIOLATION: + return "access-violation"; + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + return "array-bounds-exceeded"; + case EXCEPTION_STACK_OVERFLOW: + return "stack-overflow"; + case EXCEPTION_DATATYPE_MISALIGNMENT: + return "datatype-misalignment"; + case EXCEPTION_IN_PAGE_ERROR: + return "in-page-error"; + case EXCEPTION_ILLEGAL_INSTRUCTION: + return "illegal-instruction"; + case EXCEPTION_PRIV_INSTRUCTION: + return "priv-instruction"; + case EXCEPTION_BREAKPOINT: + return "breakpoint"; + case EXCEPTION_FLT_DENORMAL_OPERAND: + return "flt-denormal-operand"; + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + return "flt-divide-by-zero"; + case EXCEPTION_FLT_INEXACT_RESULT: + return "flt-inexact-result"; + case EXCEPTION_FLT_INVALID_OPERATION: + return "flt-invalid-operation"; + case EXCEPTION_FLT_OVERFLOW: + return "flt-overflow"; + case EXCEPTION_FLT_STACK_CHECK: + return "flt-stack-check"; + case EXCEPTION_FLT_UNDERFLOW: + return "flt-underflow"; + case EXCEPTION_INT_DIVIDE_BY_ZERO: + return "int-divide-by-zero"; + case EXCEPTION_INT_OVERFLOW: + return "int-overflow"; + } + return "unknown exception"; +} + uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { // FIXME: Actually implement this function. CHECK_GT(buf_len, 0); @@ -1021,7 +1102,12 @@ void CheckNoDeepBind(const char *filename, int flag) { } // FIXME: implement on this platform. -bool GetRandom(void *buffer, uptr length) { +bool GetRandom(void *buffer, uptr length, bool blocking) { + UNIMPLEMENTED(); +} + +// FIXME: implement on this platform. +u32 GetNumberOfCPUs() { UNIMPLEMENTED(); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc index 364319398198..5711f5dc8b6a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc @@ -12,7 +12,7 @@ // definition is provided. //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" +#include "sanitizer_platform.h" #if SANITIZER_WINDOWS && SANITIZER_DYNAMIC #include "sanitizer_win_weak_interception.h" #include "sanitizer_allocator_interface.h" diff --git a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh index c5865ecfee6c..0559a2e7eb05 100755 --- a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh +++ b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh @@ -110,12 +110,12 @@ cd ${LIBCXX_BUILD} ninja cxx cxxabi FLAGS="${FLAGS} -fno-rtti -fno-exceptions" +LLVM_FLAGS="${FLAGS} -nostdinc++ -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1" # Build LLVM. if [[ ! -d ${LLVM_BUILD} ]]; then mkdir -p ${LLVM_BUILD} cd ${LLVM_BUILD} - LLVM_FLAGS="${FLAGS} -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1" cmake -GNinja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=$CC \ @@ -137,7 +137,7 @@ mkdir ${SYMBOLIZER_BUILD} cd ${SYMBOLIZER_BUILD} echo "Compiling..." -SYMBOLIZER_FLAGS="$FLAGS -std=c++11 -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -I${LIBCXX_BUILD}/include/c++/v1" +SYMBOLIZER_FLAGS="$LLVM_FLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std=c++11" $CXX $SYMBOLIZER_FLAGS ${SRC_DIR}/sanitizer_symbolize.cc ${SRC_DIR}/sanitizer_wrappers.cc -c $AR rc symbolizer.a sanitizer_symbolize.o sanitizer_wrappers.o diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp index 6f30ee987513..e5a4d714c66e 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -16,26 +16,27 @@ #include "scudo_allocator.h" #include "scudo_crc32.h" -#include "scudo_tls.h" +#include "scudo_flags.h" +#include "scudo_tsd.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_quarantine.h" +#include #include namespace __scudo { // Global static cookie, initialized at start-up. -static uptr Cookie; +static u32 Cookie; // We default to software CRC32 if the alternatives are not supported, either // at compilation or at runtime. static atomic_uint8_t HashAlgorithm = { CRC32Software }; -INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) { +INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) { // If the hardware CRC32 feature is defined here, it was enabled everywhere, // as opposed to only for scudo_crc32.cpp. This means that other hardware // specific instructions were likely emitted at other places, and as a @@ -61,46 +62,60 @@ INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) { static ScudoBackendAllocator &getBackendAllocator(); -struct ScudoChunk : UnpackedHeader { +namespace Chunk { // We can't use the offset member of the chunk itself, as we would double // fetch it without any warranty that it wouldn't have been tampered. To // prevent this, we work with a local copy of the header. - void *getAllocBeg(UnpackedHeader *Header) { - return reinterpret_cast( - reinterpret_cast(this) - (Header->Offset << MinAlignmentLog)); + static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) { + return reinterpret_cast(reinterpret_cast(Ptr) - + AlignedChunkHeaderSize - + (Header->Offset << MinAlignmentLog)); + } + + static INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) { + return reinterpret_cast(reinterpret_cast(Ptr) - + AlignedChunkHeaderSize); + } + static INLINE + const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { + return reinterpret_cast( + reinterpret_cast(Ptr) - AlignedChunkHeaderSize); + } + + static INLINE bool isAligned(const void *Ptr) { + return IsAligned(reinterpret_cast(Ptr), MinAlignment); } // Returns the usable size for a chunk, meaning the amount of bytes from the // beginning of the user data to the end of the backend allocated chunk. - uptr getUsableSize(UnpackedHeader *Header) { - uptr Size = - getBackendAllocator().getActuallyAllocatedSize(getAllocBeg(Header), - Header->FromPrimary); + static INLINE uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) { + const uptr Size = getBackendAllocator().getActuallyAllocatedSize( + getBackendPtr(Ptr, Header), Header->ClassId); if (Size == 0) return 0; return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog); } - // Compute the checksum of the Chunk pointer and its ChunkHeader. - u16 computeChecksum(UnpackedHeader *Header) const { + // Compute the checksum of the chunk pointer and its header. + static INLINE u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) { UnpackedHeader ZeroChecksumHeader = *Header; ZeroChecksumHeader.Checksum = 0; uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); - u32 Crc = computeCRC32(Cookie, reinterpret_cast(this), HeaderHolder, - ARRAY_SIZE(HeaderHolder)); + const u32 Crc = computeCRC32(Cookie, reinterpret_cast(Ptr), + HeaderHolder, ARRAY_SIZE(HeaderHolder)); return static_cast(Crc); } // Checks the validity of a chunk by verifying its checksum. It doesn't // incur termination in the event of an invalid chunk. - bool isValid() { - UnpackedHeader NewUnpackedHeader; - const AtomicPackedHeader *AtomicHeader = - reinterpret_cast(this); - PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader); - NewUnpackedHeader = bit_cast(NewPackedHeader); - return (NewUnpackedHeader.Checksum == computeChecksum(&NewUnpackedHeader)); + static INLINE bool isValid(const void *Ptr) { + PackedHeader NewPackedHeader = + atomic_load_relaxed(getConstAtomicHeader(Ptr)); + UnpackedHeader NewUnpackedHeader = + bit_cast(NewPackedHeader); + return (NewUnpackedHeader.Checksum == + computeChecksum(Ptr, &NewUnpackedHeader)); } // Nulls out a chunk header. When returning the chunk to the backend, there @@ -109,114 +124,46 @@ struct ScudoChunk : UnpackedHeader { // the header invalid. In the extremely rare event where 0 would be a valid // checksum for the chunk, the state of the chunk is ChunkAvailable anyway. COMPILER_CHECK(ChunkAvailable == 0); - void eraseHeader() { - PackedHeader NullPackedHeader = 0; - AtomicPackedHeader *AtomicHeader = - reinterpret_cast(this); - atomic_store_relaxed(AtomicHeader, NullPackedHeader); + static INLINE void eraseHeader(void *Ptr) { + const PackedHeader NullPackedHeader = 0; + atomic_store_relaxed(getAtomicHeader(Ptr), NullPackedHeader); } // Loads and unpacks the header, verifying the checksum in the process. - void loadHeader(UnpackedHeader *NewUnpackedHeader) const { - const AtomicPackedHeader *AtomicHeader = - reinterpret_cast(this); - PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader); + static INLINE + void loadHeader(const void *Ptr, UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = + atomic_load_relaxed(getConstAtomicHeader(Ptr)); *NewUnpackedHeader = bit_cast(NewPackedHeader); if (UNLIKELY(NewUnpackedHeader->Checksum != - computeChecksum(NewUnpackedHeader))) { - dieWithMessage("ERROR: corrupted chunk header at address %p\n", this); + computeChecksum(Ptr, NewUnpackedHeader))) { + dieWithMessage("ERROR: corrupted chunk header at address %p\n", Ptr); } } // Packs and stores the header, computing the checksum in the process. - void storeHeader(UnpackedHeader *NewUnpackedHeader) { - NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader); + static INLINE void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) { + NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); - AtomicPackedHeader *AtomicHeader = - reinterpret_cast(this); - atomic_store_relaxed(AtomicHeader, NewPackedHeader); + atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); } // Packs and stores the header, computing the checksum in the process. We // compare the current header with the expected provided one to ensure that // we are not being raced by a corruption occurring in another thread. - void compareExchangeHeader(UnpackedHeader *NewUnpackedHeader, - UnpackedHeader *OldUnpackedHeader) { - NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader); + static INLINE void compareExchangeHeader(void *Ptr, + UnpackedHeader *NewUnpackedHeader, + UnpackedHeader *OldUnpackedHeader) { + NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); - AtomicPackedHeader *AtomicHeader = - reinterpret_cast(this); - if (UNLIKELY(!atomic_compare_exchange_strong(AtomicHeader, - &OldPackedHeader, - NewPackedHeader, - memory_order_relaxed))) { - dieWithMessage("ERROR: race on chunk header at address %p\n", this); + if (UNLIKELY(!atomic_compare_exchange_strong( + getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, + memory_order_relaxed))) { + dieWithMessage("ERROR: race on chunk header at address %p\n", Ptr); } } -}; - -ScudoChunk *getScudoChunk(uptr UserBeg) { - return reinterpret_cast(UserBeg - AlignedChunkHeaderSize); -} - -struct AllocatorOptions { - u32 QuarantineSizeMb; - u32 ThreadLocalQuarantineSizeKb; - bool MayReturnNull; - s32 ReleaseToOSIntervalMs; - bool DeallocationTypeMismatch; - bool DeleteSizeMismatch; - bool ZeroContents; - - void setFrom(const Flags *f, const CommonFlags *cf); - void copyTo(Flags *f, CommonFlags *cf) const; -}; - -void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) { - MayReturnNull = cf->allocator_may_return_null; - ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms; - QuarantineSizeMb = f->QuarantineSizeMb; - ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb; - DeallocationTypeMismatch = f->DeallocationTypeMismatch; - DeleteSizeMismatch = f->DeleteSizeMismatch; - ZeroContents = f->ZeroContents; -} - -void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const { - cf->allocator_may_return_null = MayReturnNull; - cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs; - f->QuarantineSizeMb = QuarantineSizeMb; - f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb; - f->DeallocationTypeMismatch = DeallocationTypeMismatch; - f->DeleteSizeMismatch = DeleteSizeMismatch; - f->ZeroContents = ZeroContents; -} - -static void initScudoInternal(const AllocatorOptions &Options); - -static bool ScudoInitIsRunning = false; - -void initScudo() { - SanitizerToolName = "Scudo"; - CHECK(!ScudoInitIsRunning && "Scudo init calls itself!"); - ScudoInitIsRunning = true; - - // Check if hardware CRC32 is supported in the binary and by the platform, if - // so, opt for the CRC32 hardware version of the checksum. - if (computeHardwareCRC32 && testCPUFeature(CRC32CPUFeature)) - atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); - - initFlags(); - - AllocatorOptions Options; - Options.setFrom(getFlags(), common_flags()); - initScudoInternal(Options); - - // TODO(kostyak): determine if MaybeStartBackgroudThread could be of some use. - - ScudoInitIsRunning = false; -} +} // namespace Chunk struct QuarantineCallback { explicit QuarantineCallback(AllocatorCache *Cache) @@ -224,52 +171,46 @@ struct QuarantineCallback { // Chunk recycling function, returns a quarantined chunk to the backend, // first making sure it hasn't been tampered with. - void Recycle(ScudoChunk *Chunk) { + void Recycle(void *Ptr) { UnpackedHeader Header; - Chunk->loadHeader(&Header); + Chunk::loadHeader(Ptr, &Header); if (UNLIKELY(Header.State != ChunkQuarantine)) { dieWithMessage("ERROR: invalid chunk state when recycling address %p\n", - Chunk); + Ptr); } - Chunk->eraseHeader(); - void *Ptr = Chunk->getAllocBeg(&Header); - if (Header.FromPrimary) - getBackendAllocator().deallocatePrimary(Cache_, Ptr); + Chunk::eraseHeader(Ptr); + void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header); + if (Header.ClassId) + getBackendAllocator().deallocatePrimary(Cache_, BackendPtr, + Header.ClassId); else - getBackendAllocator().deallocateSecondary(Ptr); + getBackendAllocator().deallocateSecondary(BackendPtr); } // Internal quarantine allocation and deallocation functions. We first check // that the batches are indeed serviced by the Primary. // TODO(kostyak): figure out the best way to protect the batches. - COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize); void *Allocate(uptr Size) { - return getBackendAllocator().allocatePrimary(Cache_, Size); + return getBackendAllocator().allocatePrimary(Cache_, BatchClassId); } void Deallocate(void *Ptr) { - getBackendAllocator().deallocatePrimary(Cache_, Ptr); + getBackendAllocator().deallocatePrimary(Cache_, Ptr, BatchClassId); } AllocatorCache *Cache_; + COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize); + const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch)); }; -typedef Quarantine ScudoQuarantine; +typedef Quarantine ScudoQuarantine; typedef ScudoQuarantine::Cache ScudoQuarantineCache; COMPILER_CHECK(sizeof(ScudoQuarantineCache) <= - sizeof(ScudoThreadContext::QuarantineCachePlaceHolder)); + sizeof(ScudoTSD::QuarantineCachePlaceHolder)); -AllocatorCache *getAllocatorCache(ScudoThreadContext *ThreadContext) { - return &ThreadContext->Cache; -} - -ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) { - return reinterpret_cast< - ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder); -} - -ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) { - return &ThreadContext->Prng; +ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) { + return reinterpret_cast( + TSD->QuarantineCachePlaceHolder); } struct ScudoAllocator { @@ -281,26 +222,22 @@ struct ScudoAllocator { ScudoBackendAllocator BackendAllocator; ScudoQuarantine AllocatorQuarantine; - StaticSpinMutex GlobalPrngMutex; - ScudoPrng GlobalPrng; - - // The fallback caches are used when the thread local caches have been - // 'detroyed' on thread tear-down. They are protected by a Mutex as they can - // be accessed by different threads. - StaticSpinMutex FallbackMutex; - AllocatorCache FallbackAllocatorCache; - ScudoQuarantineCache FallbackQuarantineCache; - ScudoPrng FallbackPrng; + u32 QuarantineChunksUpToSize; bool DeallocationTypeMismatch; bool ZeroContents; bool DeleteSizeMismatch; - explicit ScudoAllocator(LinkerInitialized) - : AllocatorQuarantine(LINKER_INITIALIZED), - FallbackQuarantineCache(LINKER_INITIALIZED) {} + bool CheckRssLimit; + uptr HardRssLimitMb; + uptr SoftRssLimitMb; + atomic_uint8_t RssLimitExceeded; + atomic_uint64_t RssLastCheckedAtNS; - void init(const AllocatorOptions &Options) { + explicit ScudoAllocator(LinkerInitialized) + : AllocatorQuarantine(LINKER_INITIALIZED) {} + + void performSanityChecks() { // Verify that the header offset field can hold the maximum offset. In the // case of the Secondary allocator, it takes care of alignment and the // offset will always be 0. In the case of the Primary, the worst case @@ -310,9 +247,9 @@ struct ScudoAllocator { // result, the maximum offset will be at most the maximum alignment for the // last size class minus the header size, in multiples of MinAlignment. UnpackedHeader Header = {}; - uptr MaxPrimaryAlignment = + const uptr MaxPrimaryAlignment = 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment); - uptr MaxOffset = + const uptr MaxOffset = (MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog; Header.Offset = MaxOffset; if (Header.Offset != MaxOffset) { @@ -324,36 +261,97 @@ struct ScudoAllocator { // case scenario happens in the Primary. It will depend on the second to // last and last class sizes, as well as the dynamic base for the Primary. // The following is an over-approximation that works for our needs. - uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1; + const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1; Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes; if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) { dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in " "the header\n"); } - DeallocationTypeMismatch = Options.DeallocationTypeMismatch; - DeleteSizeMismatch = Options.DeleteSizeMismatch; - ZeroContents = Options.ZeroContents; - SetAllocatorMayReturnNull(Options.MayReturnNull); - BackendAllocator.init(Options.ReleaseToOSIntervalMs); + const uptr LargestClassId = SizeClassMap::kLargestClassID; + Header.ClassId = LargestClassId; + if (Header.ClassId != LargestClassId) { + dieWithMessage("ERROR: the largest class ID doesn't fit in the header\n"); + } + } + + void init() { + SanitizerToolName = "Scudo"; + initFlags(); + + performSanityChecks(); + + // Check if hardware CRC32 is supported in the binary and by the platform, + // if so, opt for the CRC32 hardware version of the checksum. + if (&computeHardwareCRC32 && hasHardwareCRC32()) + atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); + + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms); + HardRssLimitMb = common_flags()->hard_rss_limit_mb; + SoftRssLimitMb = common_flags()->soft_rss_limit_mb; AllocatorQuarantine.Init( - static_cast(Options.QuarantineSizeMb) << 20, - static_cast(Options.ThreadLocalQuarantineSizeKb) << 10); - GlobalPrng.init(); - Cookie = GlobalPrng.getU64(); - BackendAllocator.initCache(&FallbackAllocatorCache); - FallbackPrng.init(); + static_cast(getFlags()->QuarantineSizeKb) << 10, + static_cast(getFlags()->ThreadLocalQuarantineSizeKb) << 10); + QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize; + DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch; + DeleteSizeMismatch = getFlags()->DeleteSizeMismatch; + ZeroContents = getFlags()->ZeroContents; + + if (UNLIKELY(!GetRandom(reinterpret_cast(&Cookie), sizeof(Cookie), + /*blocking=*/false))) { + Cookie = static_cast((NanoTime() >> 12) ^ + (reinterpret_cast(this) >> 4)); + } + + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + if (CheckRssLimit) + atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime()); } // Helper function that checks for a valid Scudo chunk. nullptr isn't. - bool isValidPointer(const void *UserPtr) { + bool isValidPointer(const void *Ptr) { initThreadMaybe(); - if (UNLIKELY(!UserPtr)) + if (UNLIKELY(!Ptr)) return false; - uptr UserBeg = reinterpret_cast(UserPtr); - if (!IsAligned(UserBeg, MinAlignment)) + if (!Chunk::isAligned(Ptr)) return false; - return getScudoChunk(UserBeg)->isValid(); + return Chunk::isValid(Ptr); + } + + // Opportunistic RSS limit check. This will update the RSS limit status, if + // it can, every 100ms, otherwise it will just return the current one. + bool isRssLimitExceeded() { + u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); + const u64 CurrentCheck = MonotonicNanoTime(); + if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL))) + return atomic_load_relaxed(&RssLimitExceeded); + if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, + CurrentCheck, memory_order_relaxed)) + return atomic_load_relaxed(&RssLimitExceeded); + // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the + // RSS from /proc/self/statm by default. We might want to + // call getrusage directly, even if it's less accurate. + const uptr CurrentRssMb = GetRSS() >> 20; + if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) { + Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, HardRssLimitMb, CurrentRssMb); + DumpProcessMap(); + Die(); + } + if (SoftRssLimitMb) { + if (atomic_load_relaxed(&RssLimitExceeded)) { + if (CurrentRssMb <= SoftRssLimitMb) + atomic_store_relaxed(&RssLimitExceeded, false); + } else { + if (CurrentRssMb > SoftRssLimitMb) { + atomic_store_relaxed(&RssLimitExceeded, true); + Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, SoftRssLimitMb, CurrentRssMb); + } + } + } + return atomic_load_relaxed(&RssLimitExceeded); } // Allocates a chunk. @@ -375,207 +373,185 @@ struct ScudoAllocator { if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) return FailureHandler::OnBadRequest(); + if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) + return FailureHandler::OnOOM(); + // Primary and Secondary backed allocations have a different treatment. We // deal with alignment requirements of Primary serviced allocations here, // but the Secondary will take care of its own alignment needs. - bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment); - - void *Ptr; - u8 Salt; - uptr AllocSize; - if (FromPrimary) { - AllocSize = AlignedSize; - ScudoThreadContext *ThreadContext = getThreadContextAndLock(); - if (LIKELY(ThreadContext)) { - Salt = getPrng(ThreadContext)->getU8(); - Ptr = BackendAllocator.allocatePrimary(getAllocatorCache(ThreadContext), - AllocSize); - ThreadContext->unlock(); - } else { - SpinMutexLock l(&FallbackMutex); - Salt = FallbackPrng.getU8(); - Ptr = BackendAllocator.allocatePrimary(&FallbackAllocatorCache, - AllocSize); - } + void *BackendPtr; + uptr BackendSize; + u8 ClassId; + if (PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment)) { + BackendSize = AlignedSize; + ClassId = SizeClassMap::ClassID(BackendSize); + ScudoTSD *TSD = getTSDAndLock(); + BackendPtr = BackendAllocator.allocatePrimary(&TSD->Cache, ClassId); + TSD->unlock(); } else { - { - SpinMutexLock l(&GlobalPrngMutex); - Salt = GlobalPrng.getU8(); - } - AllocSize = NeededSize; - Ptr = BackendAllocator.allocateSecondary(AllocSize, Alignment); + BackendSize = NeededSize; + ClassId = 0; + BackendPtr = BackendAllocator.allocateSecondary(BackendSize, Alignment); } - if (UNLIKELY(!Ptr)) + if (UNLIKELY(!BackendPtr)) return FailureHandler::OnOOM(); // If requested, we will zero out the entire contents of the returned chunk. - if ((ForceZeroContents || ZeroContents) && FromPrimary) - memset(Ptr, 0, BackendAllocator.getActuallyAllocatedSize( - Ptr, /*FromPrimary=*/true)); + if ((ForceZeroContents || ZeroContents) && ClassId) + memset(BackendPtr, 0, + BackendAllocator.getActuallyAllocatedSize(BackendPtr, ClassId)); UnpackedHeader Header = {}; - uptr AllocBeg = reinterpret_cast(Ptr); - uptr UserBeg = AllocBeg + AlignedChunkHeaderSize; - if (UNLIKELY(!IsAligned(UserBeg, Alignment))) { + uptr UserPtr = reinterpret_cast(BackendPtr) + AlignedChunkHeaderSize; + if (UNLIKELY(!IsAligned(UserPtr, Alignment))) { // Since the Secondary takes care of alignment, a non-aligned pointer // means it is from the Primary. It is also the only case where the offset // field of the header would be non-zero. - CHECK(FromPrimary); - UserBeg = RoundUpTo(UserBeg, Alignment); - uptr Offset = UserBeg - AlignedChunkHeaderSize - AllocBeg; - Header.Offset = Offset >> MinAlignmentLog; + DCHECK(ClassId); + const uptr AlignedUserPtr = RoundUpTo(UserPtr, Alignment); + Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog; + UserPtr = AlignedUserPtr; } - CHECK_LE(UserBeg + Size, AllocBeg + AllocSize); + CHECK_LE(UserPtr + Size, reinterpret_cast(BackendPtr) + BackendSize); Header.State = ChunkAllocated; Header.AllocType = Type; - if (FromPrimary) { - Header.FromPrimary = 1; + if (ClassId) { + Header.ClassId = ClassId; Header.SizeOrUnusedBytes = Size; } else { // The secondary fits the allocations to a page, so the amount of unused // bytes is the difference between the end of the user allocation and the // next page boundary. - uptr PageSize = GetPageSizeCached(); - uptr TrailingBytes = (UserBeg + Size) & (PageSize - 1); + const uptr PageSize = GetPageSizeCached(); + const uptr TrailingBytes = (UserPtr + Size) & (PageSize - 1); if (TrailingBytes) Header.SizeOrUnusedBytes = PageSize - TrailingBytes; } - Header.Salt = Salt; - getScudoChunk(UserBeg)->storeHeader(&Header); - void *UserPtr = reinterpret_cast(UserBeg); - // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(UserPtr, Size); - return UserPtr; + void *Ptr = reinterpret_cast(UserPtr); + Chunk::storeHeader(Ptr, &Header); + // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(Ptr, Size); + return Ptr; } - // Place a chunk in the quarantine. In the event of a zero-sized quarantine, - // we directly deallocate the chunk, otherwise the flow would lead to the - // chunk being loaded (and checked) twice, and stored (and checksummed) once, - // with no additional security value. - void quarantineOrDeallocateChunk(ScudoChunk *Chunk, UnpackedHeader *Header, + // Place a chunk in the quarantine or directly deallocate it in the event of + // a zero-sized quarantine, or if the size of the chunk is greater than the + // quarantine chunk size threshold. + void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header, uptr Size) { - bool FromPrimary = Header->FromPrimary; - bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0); + const bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0) || + (Size > QuarantineChunksUpToSize); if (BypassQuarantine) { - Chunk->eraseHeader(); - void *Ptr = Chunk->getAllocBeg(Header); - if (FromPrimary) { - ScudoThreadContext *ThreadContext = getThreadContextAndLock(); - if (LIKELY(ThreadContext)) { - getBackendAllocator().deallocatePrimary( - getAllocatorCache(ThreadContext), Ptr); - ThreadContext->unlock(); - } else { - SpinMutexLock Lock(&FallbackMutex); - getBackendAllocator().deallocatePrimary(&FallbackAllocatorCache, Ptr); - } + Chunk::eraseHeader(Ptr); + void *BackendPtr = Chunk::getBackendPtr(Ptr, Header); + if (Header->ClassId) { + ScudoTSD *TSD = getTSDAndLock(); + getBackendAllocator().deallocatePrimary(&TSD->Cache, BackendPtr, + Header->ClassId); + TSD->unlock(); } else { - getBackendAllocator().deallocateSecondary(Ptr); + getBackendAllocator().deallocateSecondary(BackendPtr); } } else { + // If a small memory amount was allocated with a larger alignment, we want + // to take that into account. Otherwise the Quarantine would be filled + // with tiny chunks, taking a lot of VA memory. This is an approximation + // of the usable size, that allows us to not call + // GetActuallyAllocatedSize. + uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog); UnpackedHeader NewHeader = *Header; NewHeader.State = ChunkQuarantine; - Chunk->compareExchangeHeader(&NewHeader, Header); - ScudoThreadContext *ThreadContext = getThreadContextAndLock(); - if (LIKELY(ThreadContext)) { - AllocatorQuarantine.Put(getQuarantineCache(ThreadContext), - QuarantineCallback( - getAllocatorCache(ThreadContext)), - Chunk, Size); - ThreadContext->unlock(); - } else { - SpinMutexLock l(&FallbackMutex); - AllocatorQuarantine.Put(&FallbackQuarantineCache, - QuarantineCallback(&FallbackAllocatorCache), - Chunk, Size); - } + Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); + ScudoTSD *TSD = getTSDAndLock(); + AllocatorQuarantine.Put(getQuarantineCache(TSD), + QuarantineCallback(&TSD->Cache), Ptr, + EstimatedSize); + TSD->unlock(); } } - // Deallocates a Chunk, which means adding it to the delayed free list (or - // Quarantine). - void deallocate(void *UserPtr, uptr DeleteSize, AllocType Type) { - initThreadMaybe(); - // if (&__sanitizer_free_hook) __sanitizer_free_hook(UserPtr); - if (UNLIKELY(!UserPtr)) + // Deallocates a Chunk, which means either adding it to the quarantine or + // directly returning it to the backend if criteria are met. + void deallocate(void *Ptr, uptr DeleteSize, AllocType Type) { + // For a deallocation, we only ensure minimal initialization, meaning thread + // local data will be left uninitialized for now (when using ELF TLS). The + // fallback cache will be used instead. This is a workaround for a situation + // where the only heap operation performed in a thread would be a free past + // the TLS destructors, ending up in initialized thread specific data never + // being destroyed properly. Any other heap operation will do a full init. + initThreadMaybe(/*MinimalInit=*/true); + // if (&__sanitizer_free_hook) __sanitizer_free_hook(Ptr); + if (UNLIKELY(!Ptr)) return; - uptr UserBeg = reinterpret_cast(UserPtr); - if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) { + if (UNLIKELY(!Chunk::isAligned(Ptr))) { dieWithMessage("ERROR: attempted to deallocate a chunk not properly " - "aligned at address %p\n", UserPtr); + "aligned at address %p\n", Ptr); } - ScudoChunk *Chunk = getScudoChunk(UserBeg); - UnpackedHeader OldHeader; - Chunk->loadHeader(&OldHeader); - if (UNLIKELY(OldHeader.State != ChunkAllocated)) { + UnpackedHeader Header; + Chunk::loadHeader(Ptr, &Header); + if (UNLIKELY(Header.State != ChunkAllocated)) { dieWithMessage("ERROR: invalid chunk state when deallocating address " - "%p\n", UserPtr); + "%p\n", Ptr); } if (DeallocationTypeMismatch) { // The deallocation type has to match the allocation one. - if (OldHeader.AllocType != Type) { + if (Header.AllocType != Type) { // With the exception of memalign'd Chunks, that can be still be free'd. - if (OldHeader.AllocType != FromMemalign || Type != FromMalloc) { - dieWithMessage("ERROR: allocation type mismatch on address %p\n", - UserPtr); + if (Header.AllocType != FromMemalign || Type != FromMalloc) { + dieWithMessage("ERROR: allocation type mismatch when deallocating " + "address %p\n", Ptr); } } } - uptr Size = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes : - Chunk->getUsableSize(&OldHeader) - OldHeader.SizeOrUnusedBytes; + uptr Size = Header.ClassId ? Header.SizeOrUnusedBytes : + Chunk::getUsableSize(Ptr, &Header) - Header.SizeOrUnusedBytes; if (DeleteSizeMismatch) { if (DeleteSize && DeleteSize != Size) { dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n", - UserPtr); + Ptr); } } - - // If a small memory amount was allocated with a larger alignment, we want - // to take that into account. Otherwise the Quarantine would be filled with - // tiny chunks, taking a lot of VA memory. This is an approximation of the - // usable size, that allows us to not call GetActuallyAllocatedSize. - uptr LiableSize = Size + (OldHeader.Offset << MinAlignment); - quarantineOrDeallocateChunk(Chunk, &OldHeader, LiableSize); + quarantineOrDeallocateChunk(Ptr, &Header, Size); } // Reallocates a chunk. We can save on a new allocation if the new requested // size still fits in the chunk. void *reallocate(void *OldPtr, uptr NewSize) { initThreadMaybe(); - uptr UserBeg = reinterpret_cast(OldPtr); - if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) { + if (UNLIKELY(!Chunk::isAligned(OldPtr))) { dieWithMessage("ERROR: attempted to reallocate a chunk not properly " "aligned at address %p\n", OldPtr); } - ScudoChunk *Chunk = getScudoChunk(UserBeg); UnpackedHeader OldHeader; - Chunk->loadHeader(&OldHeader); + Chunk::loadHeader(OldPtr, &OldHeader); if (UNLIKELY(OldHeader.State != ChunkAllocated)) { dieWithMessage("ERROR: invalid chunk state when reallocating address " "%p\n", OldPtr); } - if (UNLIKELY(OldHeader.AllocType != FromMalloc)) { - dieWithMessage("ERROR: invalid chunk type when reallocating address %p\n", - OldPtr); + if (DeallocationTypeMismatch) { + if (UNLIKELY(OldHeader.AllocType != FromMalloc)) { + dieWithMessage("ERROR: allocation type mismatch when reallocating " + "address %p\n", OldPtr); + } } - uptr UsableSize = Chunk->getUsableSize(&OldHeader); + const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader); // The new size still fits in the current chunk, and the size difference // is reasonable. if (NewSize <= UsableSize && (UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) { UnpackedHeader NewHeader = OldHeader; NewHeader.SizeOrUnusedBytes = - OldHeader.FromPrimary ? NewSize : UsableSize - NewSize; - Chunk->compareExchangeHeader(&NewHeader, &OldHeader); + OldHeader.ClassId ? NewSize : UsableSize - NewSize; + Chunk::compareExchangeHeader(OldPtr, &NewHeader, &OldHeader); return OldPtr; } // Otherwise, we have to allocate a new chunk and copy the contents of the // old one. void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); if (NewPtr) { - uptr OldSize = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes : + uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes : UsableSize - OldHeader.SizeOrUnusedBytes; - memcpy(NewPtr, OldPtr, Min(NewSize, OldSize)); - quarantineOrDeallocateChunk(Chunk, &OldHeader, UsableSize); + memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize)); + quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize); } return NewPtr; } @@ -585,16 +561,14 @@ struct ScudoAllocator { initThreadMaybe(); if (UNLIKELY(!Ptr)) return 0; - uptr UserBeg = reinterpret_cast(Ptr); - ScudoChunk *Chunk = getScudoChunk(UserBeg); UnpackedHeader Header; - Chunk->loadHeader(&Header); + Chunk::loadHeader(Ptr, &Header); // Getting the usable size of a chunk only makes sense if it's allocated. if (UNLIKELY(Header.State != ChunkAllocated)) { dieWithMessage("ERROR: invalid chunk state when sizing address %p\n", Ptr); } - return Chunk->getUsableSize(&Header); + return Chunk::getUsableSize(Ptr, &Header); } void *calloc(uptr NMemB, uptr Size) { @@ -604,11 +578,10 @@ struct ScudoAllocator { return allocate(NMemB * Size, MinAlignment, FromMalloc, true); } - void commitBack(ScudoThreadContext *ThreadContext) { - AllocatorCache *Cache = getAllocatorCache(ThreadContext); - AllocatorQuarantine.Drain(getQuarantineCache(ThreadContext), - QuarantineCallback(Cache)); - BackendAllocator.destroyCache(Cache); + void commitBack(ScudoTSD *TSD) { + AllocatorQuarantine.Drain(getQuarantineCache(TSD), + QuarantineCallback(&TSD->Cache)); + BackendAllocator.destroyCache(&TSD->Cache); } uptr getStats(AllocatorStat StatType) { @@ -617,6 +590,19 @@ struct ScudoAllocator { BackendAllocator.getStats(stats); return stats[StatType]; } + + void *handleBadRequest() { + initThreadMaybe(); + return FailureHandler::OnBadRequest(); + } + + void setRssLimit(uptr LimitMb, bool HardLimit) { + if (HardLimit) + HardRssLimitMb = LimitMb; + else + SoftRssLimitMb = LimitMb; + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + } }; static ScudoAllocator Instance(LINKER_INITIALIZED); @@ -625,17 +611,17 @@ static ScudoBackendAllocator &getBackendAllocator() { return Instance.BackendAllocator; } -static void initScudoInternal(const AllocatorOptions &Options) { - Instance.init(Options); +void initScudo() { + Instance.init(); } -void ScudoThreadContext::init() { +void ScudoTSD::init(bool Shared) { + UnlockRequired = Shared; getBackendAllocator().initCache(&Cache); - Prng.init(); memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder)); } -void ScudoThreadContext::commitBack() { +void ScudoTSD::commitBack() { Instance.commitBack(this); } @@ -672,6 +658,10 @@ void *scudoValloc(uptr Size) { void *scudoPvalloc(uptr Size) { uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) { + errno = ENOMEM; + return Instance.handleBadRequest(); + } // pvalloc(0) should allocate one page. Size = Size ? RoundUpTo(Size, PageSize) : PageSize; return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign)); @@ -679,28 +669,28 @@ void *scudoPvalloc(uptr Size) { void *scudoMemalign(uptr Alignment, uptr Size) { if (UNLIKELY(!IsPowerOfTwo(Alignment))) { - errno = errno_EINVAL; - return ScudoAllocator::FailureHandler::OnBadRequest(); + errno = EINVAL; + return Instance.handleBadRequest(); } return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMemalign)); } int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) { if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) { - ScudoAllocator::FailureHandler::OnBadRequest(); - return errno_EINVAL; + Instance.handleBadRequest(); + return EINVAL; } void *Ptr = Instance.allocate(Size, Alignment, FromMemalign); if (UNLIKELY(!Ptr)) - return errno_ENOMEM; + return ENOMEM; *MemPtr = Ptr; return 0; } void *scudoAlignedAlloc(uptr Alignment, uptr Size) { if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) { - errno = errno_EINVAL; - return ScudoAllocator::FailureHandler::OnBadRequest(); + errno = EINVAL; + return Instance.handleBadRequest(); } return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc)); } @@ -742,3 +732,13 @@ int __sanitizer_get_ownership(const void *Ptr) { uptr __sanitizer_get_allocated_size(const void *Ptr) { return Instance.getUsableSize(Ptr); } + +// Interface functions + +extern "C" { +void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit) { // NOLINT + if (!SCUDO_CAN_USE_PUBLIC_INTERFACE) + return; + Instance.setRssLimit(LimitMb, !!HardLimit); +} +} // extern "C" diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.h b/contrib/compiler-rt/lib/scudo/scudo_allocator.h index 29d85995a3ee..a561247def9c 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.h @@ -14,21 +14,15 @@ #ifndef SCUDO_ALLOCATOR_H_ #define SCUDO_ALLOCATOR_H_ -#include "scudo_flags.h" - -#include "sanitizer_common/sanitizer_allocator.h" - -#if !SANITIZER_LINUX -# error "The Scudo hardened allocator is currently only supported on Linux." -#endif +#include "scudo_platform.h" namespace __scudo { enum AllocType : u8 { - FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc. - FromNew = 1, // Memory block came from operator new. - FromNewArray = 2, // Memory block came from operator new []. - FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc. + FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc. + FromNew = 1, // Memory block came from operator new. + FromNewArray = 2, // Memory block came from operator new []. + FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc. }; enum ChunkState : u8 { @@ -45,16 +39,15 @@ enum ChunkState : u8 { typedef u64 PackedHeader; struct UnpackedHeader { u64 Checksum : 16; - u64 SizeOrUnusedBytes : 19; // Size for Primary backed allocations, amount of - // unused bytes in the chunk for Secondary ones. - u64 FromPrimary : 1; - u64 State : 2; // available, allocated, or quarantined - u64 AllocType : 2; // malloc, new, new[], or memalign - u64 Offset : 16; // Offset from the beginning of the backend - // allocation to the beginning of the chunk - // itself, in multiples of MinAlignment. See - // comment about its maximum value and in init(). - u64 Salt : 8; + u64 ClassId : 8; + u64 SizeOrUnusedBytes : 20; // Size for Primary backed allocations, amount of + // unused bytes in the chunk for Secondary ones. + u64 State : 2; // available, allocated, or quarantined + u64 AllocType : 2; // malloc, new, new[], or memalign + u64 Offset : 16; // Offset from the beginning of the backend + // allocation to the beginning of the chunk + // itself, in multiples of MinAlignment. See + // comment about its maximum value and in init(). }; typedef atomic_uint64_t AtomicPackedHeader; @@ -72,14 +65,6 @@ const uptr AlignedChunkHeaderSize = #if SANITIZER_CAN_USE_ALLOCATOR64 const uptr AllocatorSpace = ~0ULL; -# if defined(__aarch64__) && SANITIZER_ANDROID -const uptr AllocatorSize = 0x4000000000ULL; // 256G. -# elif defined(__aarch64__) -const uptr AllocatorSize = 0x10000000000ULL; // 1T. -# else -const uptr AllocatorSize = 0x40000000000ULL; // 4T. -# endif -typedef DefaultSizeClassMap SizeClassMap; struct AP64 { static const uptr kSpaceBeg = AllocatorSpace; static const uptr kSpaceSize = AllocatorSize; @@ -91,17 +76,12 @@ struct AP64 { }; typedef SizeClassAllocator64 PrimaryAllocator; #else -// Currently, the 32-bit Sanitizer allocator has not yet benefited from all the -// security improvements brought to the 64-bit one. This makes the 32-bit -// version of Scudo slightly less toughened. -static const uptr RegionSizeLog = 20; static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog; # if SANITIZER_WORDSIZE == 32 typedef FlatByteMap ByteMap; # elif SANITIZER_WORDSIZE == 64 typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap; # endif // SANITIZER_WORDSIZE -typedef DefaultSizeClassMap SizeClassMap; struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; @@ -111,7 +91,8 @@ struct AP32 { typedef __scudo::ByteMap ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = - SizeClassAllocator32FlagMasks::kRandomShuffleChunks; + SizeClassAllocator32FlagMasks::kRandomShuffleChunks | + SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; }; typedef SizeClassAllocator32 PrimaryAllocator; #endif // SANITIZER_CAN_USE_ALLOCATOR64 diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h b/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h index 7599c12abb6d..25e273114c23 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h @@ -31,8 +31,8 @@ class ScudoCombinedAllocator { // Primary allocations are always MinAlignment aligned, and as such do not // require an Alignment parameter. - void *allocatePrimary(AllocatorCache *Cache, uptr Size) { - return Cache->Allocate(&Primary, Primary.ClassID(Size)); + void *allocatePrimary(AllocatorCache *Cache, uptr ClassId) { + return Cache->Allocate(&Primary, ClassId); } // Secondary allocations do not require a Cache, but do require an Alignment @@ -41,17 +41,17 @@ class ScudoCombinedAllocator { return Secondary.Allocate(&Stats, Size, Alignment); } - void deallocatePrimary(AllocatorCache *Cache, void *Ptr) { - Cache->Deallocate(&Primary, Primary.GetSizeClass(Ptr), Ptr); + void deallocatePrimary(AllocatorCache *Cache, void *Ptr, uptr ClassId) { + Cache->Deallocate(&Primary, ClassId, Ptr); } void deallocateSecondary(void *Ptr) { Secondary.Deallocate(&Stats, Ptr); } - uptr getActuallyAllocatedSize(void *Ptr, bool FromPrimary) { - if (FromPrimary) - return PrimaryAllocator::ClassIdToSize(Primary.GetSizeClass(Ptr)); + uptr getActuallyAllocatedSize(void *Ptr, uptr ClassId) { + if (ClassId) + return PrimaryAllocator::ClassIdToSize(ClassId); return Secondary.GetActuallyAllocatedSize(Ptr); } diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h b/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h index dbfb22565f9c..f2002ed986c3 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h @@ -23,23 +23,24 @@ class ScudoLargeMmapAllocator { public: - void Init() { - PageSize = GetPageSizeCached(); + PageSizeCached = GetPageSizeCached(); } void *Allocate(AllocatorStats *Stats, uptr Size, uptr Alignment) { - uptr UserSize = Size - AlignedChunkHeaderSize; + const uptr UserSize = Size - AlignedChunkHeaderSize; // The Scudo frontend prevents us from allocating more than // MaxAllowedMallocSize, so integer overflow checks would be superfluous. - uptr MapSize = Size + SecondaryHeaderSize; + uptr MapSize = Size + AlignedReservedAddressRangeSize; if (Alignment > MinAlignment) MapSize += Alignment; + const uptr PageSize = PageSizeCached; MapSize = RoundUpTo(MapSize, PageSize); // Account for 2 guard pages, one before and one after the chunk. MapSize += 2 * PageSize; - uptr MapBeg = reinterpret_cast(MmapNoAccess(MapSize)); + ReservedAddressRange AddressRange; + uptr MapBeg = AddressRange.Init(MapSize); if (MapBeg == ~static_cast(0)) return ReturnNullOrDieOnFailure::OnOOM(); // A page-aligned pointer is assumed after that, so check it now. @@ -62,27 +63,27 @@ class ScudoLargeMmapAllocator { PageSize; CHECK_GE(NewMapBeg, MapBeg); if (NewMapBeg != MapBeg) { - UnmapOrDie(reinterpret_cast(MapBeg), NewMapBeg - MapBeg); + AddressRange.Unmap(MapBeg, NewMapBeg - MapBeg); MapBeg = NewMapBeg; } UserEnd = UserBeg + UserSize; } uptr NewMapEnd = RoundUpTo(UserEnd, PageSize) + PageSize; if (NewMapEnd != MapEnd) { - UnmapOrDie(reinterpret_cast(NewMapEnd), MapEnd - NewMapEnd); + AddressRange.Unmap(NewMapEnd, MapEnd - NewMapEnd); MapEnd = NewMapEnd; } MapSize = MapEnd - MapBeg; } CHECK_LE(UserEnd, MapEnd - PageSize); - // Actually mmap the memory, preserving the guard pages on either side. - CHECK_EQ(MapBeg + PageSize, reinterpret_cast( - MmapFixedOrDie(MapBeg + PageSize, MapSize - 2 * PageSize))); - uptr Ptr = UserBeg - AlignedChunkHeaderSize; - SecondaryHeader *Header = getHeader(Ptr); - Header->MapBeg = MapBeg; - Header->MapSize = MapSize; + // Actually mmap the memory, preserving the guard pages on either side + CHECK_EQ(MapBeg + PageSize, + AddressRange.Map(MapBeg + PageSize, MapSize - 2 * PageSize)); + const uptr Ptr = UserBeg - AlignedChunkHeaderSize; + ReservedAddressRange *StoredRange = getReservedAddressRange(Ptr); + *StoredRange = AddressRange; + // The primary adds the whole class size to the stats when allocating a // chunk, so we will do something similar here. But we will not account for // the guard pages. @@ -96,42 +97,43 @@ class ScudoLargeMmapAllocator { } void Deallocate(AllocatorStats *Stats, void *Ptr) { - SecondaryHeader *Header = getHeader(Ptr); + // Since we're unmapping the entirety of where the ReservedAddressRange + // actually is, copy onto the stack. + const uptr PageSize = PageSizeCached; + ReservedAddressRange AddressRange = *getReservedAddressRange(Ptr); { SpinMutexLock l(&StatsMutex); - Stats->Sub(AllocatorStatAllocated, Header->MapSize - 2 * PageSize); - Stats->Sub(AllocatorStatMapped, Header->MapSize - 2 * PageSize); + Stats->Sub(AllocatorStatAllocated, AddressRange.size() - 2 * PageSize); + Stats->Sub(AllocatorStatMapped, AddressRange.size() - 2 * PageSize); } - UnmapOrDie(reinterpret_cast(Header->MapBeg), Header->MapSize); + AddressRange.Unmap(reinterpret_cast(AddressRange.base()), + AddressRange.size()); } uptr GetActuallyAllocatedSize(void *Ptr) { - SecondaryHeader *Header = getHeader(Ptr); - // Deduct PageSize as MapSize includes the trailing guard page. - uptr MapEnd = Header->MapBeg + Header->MapSize - PageSize; + ReservedAddressRange *StoredRange = getReservedAddressRange(Ptr); + // Deduct PageSize as ReservedAddressRange size includes the trailing guard + // page. + uptr MapEnd = reinterpret_cast(StoredRange->base()) + + StoredRange->size() - PageSizeCached; return MapEnd - reinterpret_cast(Ptr); } private: - // A Secondary allocated chunk header contains the base of the mapping and - // its size, which comprises the guard pages. - struct SecondaryHeader { - uptr MapBeg; - uptr MapSize; - }; - // Check that sizeof(SecondaryHeader) is a multiple of MinAlignment. - COMPILER_CHECK((sizeof(SecondaryHeader) & (MinAlignment - 1)) == 0); - - SecondaryHeader *getHeader(uptr Ptr) { - return reinterpret_cast(Ptr - sizeof(SecondaryHeader)); + ReservedAddressRange *getReservedAddressRange(uptr Ptr) { + return reinterpret_cast( + Ptr - sizeof(ReservedAddressRange)); } - SecondaryHeader *getHeader(const void *Ptr) { - return getHeader(reinterpret_cast(Ptr)); + ReservedAddressRange *getReservedAddressRange(const void *Ptr) { + return getReservedAddressRange(reinterpret_cast(Ptr)); } - const uptr SecondaryHeaderSize = sizeof(SecondaryHeader); - const uptr HeadersSize = SecondaryHeaderSize + AlignedChunkHeaderSize; - uptr PageSize; + static constexpr uptr AlignedReservedAddressRangeSize = + (sizeof(ReservedAddressRange) + MinAlignment - 1) & ~(MinAlignment - 1); + static constexpr uptr HeadersSize = + AlignedReservedAddressRangeSize + AlignedChunkHeaderSize; + + uptr PageSizeCached; SpinMutex StatsMutex; }; diff --git a/contrib/compiler-rt/lib/scudo/scudo_crc32.h b/contrib/compiler-rt/lib/scudo/scudo_crc32.h index 5ffcc62658cc..e89e430f4085 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_crc32.h +++ b/contrib/compiler-rt/lib/scudo/scudo_crc32.h @@ -40,7 +40,7 @@ enum : u8 { CRC32Hardware = 1, }; -const static u32 CRC32Table[] = { +static const u32 CRC32Table[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, diff --git a/contrib/compiler-rt/lib/scudo/scudo_flags.cpp b/contrib/compiler-rt/lib/scudo/scudo_flags.cpp index 90f0cbf4bb86..2aff3ef1e8fa 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_flags.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_flags.cpp @@ -17,12 +17,11 @@ #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" -extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -const char* __scudo_default_options(); +SANITIZER_INTERFACE_WEAK_DEF(const char*, __scudo_default_options, void); namespace __scudo { -Flags ScudoFlags; // Use via getFlags(). +static Flags ScudoFlags; // Use via getFlags(). void Flags::setDefaults() { #define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; @@ -37,7 +36,7 @@ static void RegisterScudoFlags(FlagParser *parser, Flags *f) { #undef SCUDO_FLAG } -static const char *callGetScudoDefaultOptions() { +static const char *getScudoDefaultOptions() { return (&__scudo_default_options) ? __scudo_default_options() : ""; } @@ -57,8 +56,7 @@ void initFlags() { RegisterCommonFlags(&ScudoParser); // Override from user-specified string. - const char *ScudoDefaultOptions = callGetScudoDefaultOptions(); - ScudoParser.ParseString(ScudoDefaultOptions); + ScudoParser.ParseString(getScudoDefaultOptions()); // Override from environment. ScudoParser.ParseString(GetEnv("SCUDO_OPTIONS")); @@ -67,27 +65,52 @@ void initFlags() { // Sanity checks and default settings for the Quarantine parameters. - if (f->QuarantineSizeMb < 0) { - const int DefaultQuarantineSizeMb = FIRST_32_SECOND_64(4, 16); - f->QuarantineSizeMb = DefaultQuarantineSizeMb; + if (f->QuarantineSizeMb >= 0) { + // Backward compatible logic if QuarantineSizeMb is set. + if (f->QuarantineSizeKb >= 0) { + dieWithMessage("ERROR: please use either QuarantineSizeMb (deprecated) " + "or QuarantineSizeKb, but not both\n"); + } + if (f->QuarantineChunksUpToSize >= 0) { + dieWithMessage("ERROR: QuarantineChunksUpToSize cannot be used in " + " conjunction with the deprecated QuarantineSizeMb option\n"); + } + // If everything is in order, update QuarantineSizeKb accordingly. + f->QuarantineSizeKb = f->QuarantineSizeMb * 1024; + } else { + // Otherwise proceed with the new options. + if (f->QuarantineSizeKb < 0) { + const int DefaultQuarantineSizeKb = FIRST_32_SECOND_64(64, 256); + f->QuarantineSizeKb = DefaultQuarantineSizeKb; + } + if (f->QuarantineChunksUpToSize < 0) { + const int DefaultQuarantineChunksUpToSize = FIRST_32_SECOND_64(512, 2048); + f->QuarantineChunksUpToSize = DefaultQuarantineChunksUpToSize; + } } - // We enforce an upper limit for the quarantine size of 4Gb. - if (f->QuarantineSizeMb > (4 * 1024)) { + + // We enforce an upper limit for the chunk quarantine threshold of 4Mb. + if (f->QuarantineChunksUpToSize > (4 * 1024 * 1024)) { + dieWithMessage("ERROR: the chunk quarantine threshold is too large\n"); + } + + // We enforce an upper limit for the quarantine size of 32Mb. + if (f->QuarantineSizeKb > (32 * 1024)) { dieWithMessage("ERROR: the quarantine size is too large\n"); } + if (f->ThreadLocalQuarantineSizeKb < 0) { - const int DefaultThreadLocalQuarantineSizeKb = - FIRST_32_SECOND_64(64, 256); + const int DefaultThreadLocalQuarantineSizeKb = FIRST_32_SECOND_64(16, 64); f->ThreadLocalQuarantineSizeKb = DefaultThreadLocalQuarantineSizeKb; } - // And an upper limit of 128Mb for the thread quarantine cache. - if (f->ThreadLocalQuarantineSizeKb > (128 * 1024)) { + // And an upper limit of 8Mb for the thread quarantine cache. + if (f->ThreadLocalQuarantineSizeKb > (8 * 1024)) { dieWithMessage("ERROR: the per thread quarantine cache size is too " - "large\n"); + "large\n"); } - if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeMb > 0) { + if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeKb > 0) { dieWithMessage("ERROR: ThreadLocalQuarantineSizeKb can be set to 0 only " - "when QuarantineSizeMb is set to 0\n"); + "when QuarantineSizeKb is set to 0\n"); } } diff --git a/contrib/compiler-rt/lib/scudo/scudo_flags.inc b/contrib/compiler-rt/lib/scudo/scudo_flags.inc index 45f9ea846e1a..f180478fdac3 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_flags.inc +++ b/contrib/compiler-rt/lib/scudo/scudo_flags.inc @@ -15,17 +15,27 @@ # error "Define SCUDO_FLAG prior to including this file!" #endif -// Default value is set in scudo_flags.cpp based on architecture. SCUDO_FLAG(int, QuarantineSizeMb, -1, - "Size (in Mb) of quarantine used to delay the actual deallocation " - "of chunks. Lower value may reduce memory usage but decrease the " - "effectiveness of the mitigation.") + "Deprecated. Please use QuarantineSizeKb.") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, QuarantineSizeKb, -1, + "Size in KB of quarantine used to delay the actual deallocation of " + "chunks. Lower value may reduce memory usage but decrease the " + "effectiveness of the mitigation. Defaults to 64KB (32-bit) or " + "256KB (64-bit)") // Default value is set in scudo_flags.cpp based on architecture. SCUDO_FLAG(int, ThreadLocalQuarantineSizeKb, -1, - "Size (in Kb) of per-thread cache used to offload the global " + "Size in KB of per-thread cache used to offload the global " "quarantine. Lower value may reduce memory usage but might increase " - "the contention on the global quarantine.") + "the contention on the global quarantine. Defaults to 16KB (32-bit) " + "or 64KB (64-bit)") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, QuarantineChunksUpToSize, -1, + "Size in bytes up to which chunks will be quarantined (if lower than" + "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)") SCUDO_FLAG(bool, DeallocationTypeMismatch, true, "Report errors on malloc/delete, new/free, new/delete[], etc.") diff --git a/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h b/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h new file mode 100644 index 000000000000..3f39e0c4ee0b --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h @@ -0,0 +1,22 @@ +//===-- scudo_interface_internal.h ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Private Scudo interface header. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERFACE_INTERNAL_H_ +#define SCUDO_INTERFACE_INTERNAL_H_ + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit); // NOLINT +} // extern "C" + +#endif // SCUDO_INTERFACE_INTERNAL_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp b/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp index cdefb127b965..c5a1abbed82b 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp @@ -15,7 +15,7 @@ #include "interception/interception.h" -#include +#include using namespace __scudo; diff --git a/contrib/compiler-rt/lib/scudo/scudo_platform.h b/contrib/compiler-rt/lib/scudo/scudo_platform.h new file mode 100644 index 000000000000..e1c9c32e9a62 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_platform.h @@ -0,0 +1,80 @@ +//===-- scudo_platform.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo platform specific definitions. +/// TODO(kostyak): add tests for the compile time defines. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PLATFORM_H_ +#define SCUDO_PLATFORM_H_ + +#include "sanitizer_common/sanitizer_allocator.h" + +#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA +# error "The Scudo hardened allocator is not supported on this platform." +#endif + +#define SCUDO_TSD_EXCLUSIVE_SUPPORTED (!SANITIZER_ANDROID && !SANITIZER_FUCHSIA) + +#ifndef SCUDO_TSD_EXCLUSIVE +// SCUDO_TSD_EXCLUSIVE wasn't defined, use a default TSD model for the platform. +# if SANITIZER_ANDROID || SANITIZER_FUCHSIA +// Android and Fuchsia use a pool of TSDs shared between threads. +# define SCUDO_TSD_EXCLUSIVE 0 +# elif SANITIZER_LINUX && !SANITIZER_ANDROID +// Non-Android Linux use an exclusive TSD per thread. +# define SCUDO_TSD_EXCLUSIVE 1 +# else +# error "No default TSD model defined for this platform." +# endif // SANITIZER_ANDROID || SANITIZER_FUCHSIA +#endif // SCUDO_TSD_EXCLUSIVE + +// If the exclusive TSD model is chosen, make sure the platform supports it. +#if SCUDO_TSD_EXCLUSIVE && !SCUDO_TSD_EXCLUSIVE_SUPPORTED +# error "The exclusive TSD model is not supported on this platform." +#endif + +// Maximum number of TSDs that can be created for the Shared model. +#ifndef SCUDO_SHARED_TSD_POOL_SIZE +# define SCUDO_SHARED_TSD_POOL_SIZE 32U +#endif // SCUDO_SHARED_TSD_POOL_SIZE + +// The following allows the public interface functions to be disabled. +#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE +# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1 +#endif + +namespace __scudo { + +#if SANITIZER_CAN_USE_ALLOCATOR64 +# if defined(__aarch64__) && SANITIZER_ANDROID +const uptr AllocatorSize = 0x4000000000ULL; // 256G. +# elif defined(__aarch64__) +const uptr AllocatorSize = 0x10000000000ULL; // 1T. +# else +const uptr AllocatorSize = 0x40000000000ULL; // 4T. +# endif +#else +const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20; +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + +#if !defined(SCUDO_SIZE_CLASS_MAP) +# define SCUDO_SIZE_CLASS_MAP Default +#endif + +#define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP) +#define SIZE_CLASS_MAP_TYPE_(T) SIZE_CLASS_MAP_TYPE__(T) +#define SIZE_CLASS_MAP_TYPE__(T) T##SizeClassMap + +typedef SIZE_CLASS_MAP_TYPE SizeClassMap; + +} // namespace __scudo + +#endif // SCUDO_PLATFORM_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls.h b/contrib/compiler-rt/lib/scudo/scudo_tls.h deleted file mode 100644 index 20c49204cf13..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls.h +++ /dev/null @@ -1,47 +0,0 @@ -//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure definition. -/// Implementation will differ based on the thread local storage primitives -/// offered by the underlying platform. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_H_ -#define SCUDO_TLS_H_ - -#include "scudo_allocator.h" -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_platform.h" - -namespace __scudo { - -// Platform specific base thread context definitions. -#include "scudo_tls_context_android.inc" -#include "scudo_tls_context_linux.inc" - -struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform { - AllocatorCache Cache; - ScudoPrng Prng; - uptr QuarantineCachePlaceHolder[4]; - void init(); - void commitBack(); -}; - -void initThread(); - -// Platform specific dastpath functions definitions. -#include "scudo_tls_android.inc" -#include "scudo_tls_linux.inc" - -} // namespace __scudo - -#endif // SCUDO_TLS_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp b/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp deleted file mode 100644 index ec74e37c8dbc..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp +++ /dev/null @@ -1,95 +0,0 @@ -//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure implementation for Android. -/// -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -#include "scudo_tls.h" - -#include - -namespace __scudo { - -static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; -static pthread_key_t PThreadKey; - -static atomic_uint32_t ThreadContextCurrentIndex; -static ScudoThreadContext *ThreadContexts; -static uptr NumberOfContexts; - -// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used as they allocate memory. -static uptr getNumberOfCPUs() { - cpu_set_t CPUs; - CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); - return CPU_COUNT(&CPUs); -} - -static void initOnce() { - // Hack: TLS_SLOT_TSAN was introduced in N. To be able to use it on M for - // testing, we create an unused key. Since the key_data array follows the tls - // array, it basically gives us the extra entry we need. - // TODO(kostyak): remove and restrict to N and above. - CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); - initScudo(); - NumberOfContexts = getNumberOfCPUs(); - ThreadContexts = reinterpret_cast( - MmapOrDie(sizeof(ScudoThreadContext) * NumberOfContexts, __func__)); - for (uptr i = 0; i < NumberOfContexts; i++) - ThreadContexts[i].init(); -} - -void initThread() { - pthread_once(&GlobalInitialized, initOnce); - // Initial context assignment is done in a plain round-robin fashion. - u32 Index = atomic_fetch_add(&ThreadContextCurrentIndex, 1, - memory_order_relaxed); - ScudoThreadContext *ThreadContext = - &ThreadContexts[Index % NumberOfContexts]; - *get_android_tls_ptr() = reinterpret_cast(ThreadContext); -} - -ScudoThreadContext *getThreadContextAndLockSlow() { - ScudoThreadContext *ThreadContext; - // Go through all the contexts and find the first unlocked one. - for (u32 i = 0; i < NumberOfContexts; i++) { - ThreadContext = &ThreadContexts[i]; - if (ThreadContext->tryLock()) { - *get_android_tls_ptr() = reinterpret_cast(ThreadContext); - return ThreadContext; - } - } - // No luck, find the one with the lowest precedence, and slow lock it. - u64 Precedence = UINT64_MAX; - for (u32 i = 0; i < NumberOfContexts; i++) { - u64 SlowLockPrecedence = ThreadContexts[i].getSlowLockPrecedence(); - if (SlowLockPrecedence && SlowLockPrecedence < Precedence) { - ThreadContext = &ThreadContexts[i]; - Precedence = SlowLockPrecedence; - } - } - if (LIKELY(Precedence != UINT64_MAX)) { - ThreadContext->lock(); - *get_android_tls_ptr() = reinterpret_cast(ThreadContext); - return ThreadContext; - } - // Last resort (can this happen?), stick with the current one. - ThreadContext = - reinterpret_cast(*get_android_tls_ptr()); - ThreadContext->lock(); - return ThreadContext; -} - -} // namespace __scudo - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc deleted file mode 100644 index 8ecad7a30a6c..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc +++ /dev/null @@ -1,44 +0,0 @@ -//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure fastpath functions implementation for Android. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_ANDROID_H_ -#define SCUDO_TLS_ANDROID_H_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -ALWAYS_INLINE void initThreadMaybe() { - if (LIKELY(*get_android_tls_ptr())) - return; - initThread(); -} - -ScudoThreadContext *getThreadContextAndLockSlow(); - -ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() { - ScudoThreadContext *ThreadContext = - reinterpret_cast(*get_android_tls_ptr()); - CHECK(ThreadContext); - // Try to lock the currently associated context. - if (ThreadContext->tryLock()) - return ThreadContext; - // If it failed, go the slow path. - return getThreadContextAndLockSlow(); -} - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID - -#endif // SCUDO_TLS_ANDROID_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc deleted file mode 100644 index f1951319d487..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc +++ /dev/null @@ -1,54 +0,0 @@ -//===-- scudo_tls_context_android.inc ---------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Android specific base thread context definition. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_CONTEXT_ANDROID_INC_ -#define SCUDO_TLS_CONTEXT_ANDROID_INC_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -struct ScudoThreadContextPlatform { - INLINE bool tryLock() { - if (Mutex.TryLock()) { - atomic_store_relaxed(&SlowLockPrecedence, 0); - return true; - } - if (atomic_load_relaxed(&SlowLockPrecedence) == 0) - atomic_store_relaxed(&SlowLockPrecedence, NanoTime()); - return false; - } - - INLINE void lock() { - Mutex.Lock(); - atomic_store_relaxed(&SlowLockPrecedence, 0); - } - - INLINE void unlock() { - Mutex.Unlock(); - } - - INLINE u64 getSlowLockPrecedence() { - return atomic_load_relaxed(&SlowLockPrecedence); - } - - private: - StaticSpinMutex Mutex; - atomic_uint64_t SlowLockPrecedence; -}; - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID - -#endif // SCUDO_TLS_CONTEXT_ANDROID_INC_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc deleted file mode 100644 index 8d292bdbc932..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc +++ /dev/null @@ -1,29 +0,0 @@ -//===-- scudo_tls_context_linux.inc -----------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Linux specific base thread context definition. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_CONTEXT_LINUX_INC_ -#define SCUDO_TLS_CONTEXT_LINUX_INC_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -struct ScudoThreadContextPlatform { - ALWAYS_INLINE void unlock() {} -}; - -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID - -#endif // SCUDO_TLS_CONTEXT_LINUX_INC_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc deleted file mode 100644 index 242ee3329ea8..000000000000 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc +++ /dev/null @@ -1,48 +0,0 @@ -//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure fastpath functions implementation for platforms -/// supporting thread_local. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_LINUX_H_ -#define SCUDO_TLS_LINUX_H_ - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -enum ThreadState : u8 { - ThreadNotInitialized = 0, - ThreadInitialized, - ThreadTornDown, -}; -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ThreadState ScudoThreadState; -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ScudoThreadContext ThreadLocalContext; - -ALWAYS_INLINE void initThreadMaybe() { - if (LIKELY(ScudoThreadState != ThreadNotInitialized)) - return; - initThread(); -} - -ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() { - if (UNLIKELY(ScudoThreadState == ThreadTornDown)) - return nullptr; - return &ThreadLocalContext; -} - -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID - -#endif // SCUDO_TLS_LINUX_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd.h b/contrib/compiler-rt/lib/scudo/scudo_tsd.h new file mode 100644 index 000000000000..80464b5ea1e4 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd.h @@ -0,0 +1,72 @@ +//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo thread specific data definition. +/// Implementation will differ based on the thread local storage primitives +/// offered by the underlying platform. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +#define SCUDO_TSD_H_ + +#include "scudo_allocator.h" +#include "scudo_utils.h" + +#include + +namespace __scudo { + +struct ALIGNED(64) ScudoTSD { + AllocatorCache Cache; + uptr QuarantineCachePlaceHolder[4]; + + void init(bool Shared); + void commitBack(); + + INLINE bool tryLock() { + if (Mutex.TryLock()) { + atomic_store_relaxed(&Precedence, 0); + return true; + } + if (atomic_load_relaxed(&Precedence) == 0) + atomic_store_relaxed(&Precedence, MonotonicNanoTime()); + return false; + } + + INLINE void lock() { + Mutex.Lock(); + atomic_store_relaxed(&Precedence, 0); + } + + INLINE void unlock() { + if (!UnlockRequired) + return; + Mutex.Unlock(); + } + + INLINE u64 getPrecedence() { + return atomic_load_relaxed(&Precedence); + } + + private: + bool UnlockRequired; + StaticSpinMutex Mutex; + atomic_uint64_t Precedence; +}; + +void initThread(bool MinimalInit); + +// TSD model specific fastpath functions definitions. +#include "scudo_tsd_exclusive.inc" +#include "scudo_tsd_shared.inc" + +} // namespace __scudo + +#endif // SCUDO_TSD_H_ diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp similarity index 75% rename from contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp rename to contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp index 1e38233f339c..1084dfac91e1 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp @@ -1,4 +1,4 @@ -//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===// +//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,18 +7,13 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure implementation for platforms supporting -/// thread_local. +/// Scudo exclusive TSD implementation. /// //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" +#include "scudo_tsd.h" -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -#include "scudo_tls.h" - -#include +#if SCUDO_TSD_EXCLUSIVE namespace __scudo { @@ -28,7 +23,11 @@ static pthread_key_t PThreadKey; __attribute__((tls_model("initial-exec"))) THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized; __attribute__((tls_model("initial-exec"))) -THREADLOCAL ScudoThreadContext ThreadLocalContext; +THREADLOCAL ScudoTSD TSD; + +// Fallback TSD for when the thread isn't initialized yet or is torn down. It +// can be shared between multiple threads and as such must be locked. +ScudoTSD FallbackTSD; static void teardownThread(void *Ptr) { uptr I = reinterpret_cast(Ptr); @@ -43,7 +42,7 @@ static void teardownThread(void *Ptr) { reinterpret_cast(I - 1)) == 0)) return; } - ThreadLocalContext.commitBack(); + TSD.commitBack(); ScudoThreadState = ThreadTornDown; } @@ -51,16 +50,19 @@ static void teardownThread(void *Ptr) { static void initOnce() { CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0); initScudo(); + FallbackTSD.init(/*Shared=*/true); } -void initThread() { +void initThread(bool MinimalInit) { CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0); + if (UNLIKELY(MinimalInit)) + return; CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast( GetPthreadDestructorIterations())), 0); - ThreadLocalContext.init(); + TSD.init(/*Shared=*/false); ScudoThreadState = ThreadInitialized; } } // namespace __scudo -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc new file mode 100644 index 000000000000..567b6a1edd12 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc @@ -0,0 +1,46 @@ +//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo exclusive TSD fastpath functions implementation. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ + +#if SCUDO_TSD_EXCLUSIVE + +enum ThreadState : u8 { + ThreadNotInitialized = 0, + ThreadInitialized, + ThreadTornDown, +}; +__attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ThreadState ScudoThreadState; +__attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ScudoTSD TSD; + +extern ScudoTSD FallbackTSD; + +ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + if (LIKELY(ScudoThreadState != ThreadNotInitialized)) + return; + initThread(MinimalInit); +} + +ALWAYS_INLINE ScudoTSD *getTSDAndLock() { + if (UNLIKELY(ScudoThreadState != ThreadInitialized)) { + FallbackTSD.lock(); + return &FallbackTSD; + } + return &TSD; +} + +#endif // SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp new file mode 100644 index 000000000000..3e13e5d3a109 --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp @@ -0,0 +1,87 @@ +//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo shared TSD implementation. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_tsd.h" + +#if !SCUDO_TSD_EXCLUSIVE + +namespace __scudo { + +static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; +pthread_key_t PThreadKey; + +static atomic_uint32_t CurrentIndex; +static ScudoTSD *TSDs; +static u32 NumberOfTSDs; + +static void initOnce() { + CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); + initScudo(); + NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()), + static_cast(SCUDO_SHARED_TSD_POOL_SIZE)); + TSDs = reinterpret_cast( + MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs")); + for (u32 i = 0; i < NumberOfTSDs; i++) + TSDs[i].init(/*Shared=*/true); +} + +ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) { +#if SANITIZER_ANDROID + *get_android_tls_ptr() = reinterpret_cast(TSD); +#else + CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast(TSD)), 0); +#endif // SANITIZER_ANDROID +} + +void initThread(bool MinimalInit) { + pthread_once(&GlobalInitialized, initOnce); + // Initial context assignment is done in a plain round-robin fashion. + u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed); + setCurrentTSD(&TSDs[Index % NumberOfTSDs]); +} + +ScudoTSD *getTSDAndLockSlow() { + ScudoTSD *TSD; + if (NumberOfTSDs > 1) { + // Go through all the contexts and find the first unlocked one. + for (u32 i = 0; i < NumberOfTSDs; i++) { + TSD = &TSDs[i]; + if (TSD->tryLock()) { + setCurrentTSD(TSD); + return TSD; + } + } + // No luck, find the one with the lowest Precedence, and slow lock it. + u64 LowestPrecedence = UINT64_MAX; + for (u32 i = 0; i < NumberOfTSDs; i++) { + u64 Precedence = TSDs[i].getPrecedence(); + if (Precedence && Precedence < LowestPrecedence) { + TSD = &TSDs[i]; + LowestPrecedence = Precedence; + } + } + if (LIKELY(LowestPrecedence != UINT64_MAX)) { + TSD->lock(); + setCurrentTSD(TSD); + return TSD; + } + } + // Last resort, stick with the current one. + TSD = getCurrentTSD(); + TSD->lock(); + return TSD; +} + +} // namespace __scudo + +#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc new file mode 100644 index 000000000000..79fcd651ed2d --- /dev/null +++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc @@ -0,0 +1,48 @@ +//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo shared TSD fastpath functions implementation. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ + +#if !SCUDO_TSD_EXCLUSIVE + +extern pthread_key_t PThreadKey; + +ALWAYS_INLINE ScudoTSD* getCurrentTSD() { +#if SANITIZER_ANDROID + return reinterpret_cast(*get_android_tls_ptr()); +#else + return reinterpret_cast(pthread_getspecific(PThreadKey)); +#endif // SANITIZER_ANDROID +} + +ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + if (LIKELY(getCurrentTSD())) + return; + initThread(MinimalInit); +} + +ScudoTSD *getTSDAndLockSlow(); + +ALWAYS_INLINE ScudoTSD *getTSDAndLock() { + ScudoTSD *TSD = getCurrentTSD(); + CHECK(TSD && "No TSD associated with the current thread!"); + // Try to lock the currently associated context. + if (TSD->tryLock()) + return TSD; + // If it failed, go the slow path. + return getTSDAndLockSlow(); +} + +#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp index f7903ff34c73..2f936bf9e780 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp @@ -13,16 +13,17 @@ #include "scudo_utils.h" -#include -#include -#include -#include #if defined(__x86_64__) || defined(__i386__) # include +#elif defined(__arm__) || defined(__aarch64__) +# include "sanitizer_common/sanitizer_getauxval.h" +# if SANITIZER_POSIX +# include "sanitizer_common/sanitizer_posix.h" +# include +# endif #endif -#if defined(__arm__) || defined(__aarch64__) -# include -#endif + +#include // TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less // complicated string formatting code. The following is a @@ -36,13 +37,12 @@ extern int VSNPrintf(char *buff, int buff_length, const char *format, namespace __scudo { -FORMAT(1, 2) -void NORETURN dieWithMessage(const char *Format, ...) { +FORMAT(1, 2) void NORETURN dieWithMessage(const char *Format, ...) { // Our messages are tiny, 256 characters is more than enough. char Message[256]; va_list Args; va_start(Args, Format); - __sanitizer::VSNPrintf(Message, sizeof(Message), Format, Args); + VSNPrintf(Message, sizeof(Message), Format, Args); va_end(Args); RawWrite(Message); Die(); @@ -51,76 +51,68 @@ void NORETURN dieWithMessage(const char *Format, ...) { #if defined(__x86_64__) || defined(__i386__) // i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. // CRC32 requires the SSE 4.2 instruction set. -typedef struct { - u32 Eax; - u32 Ebx; - u32 Ecx; - u32 Edx; -} CPUIDRegs; - -static void getCPUID(CPUIDRegs *Regs, u32 Level) -{ - __get_cpuid(Level, &Regs->Eax, &Regs->Ebx, &Regs->Ecx, &Regs->Edx); -} - -CPUIDRegs getCPUFeatures() { - CPUIDRegs VendorRegs = {}; - getCPUID(&VendorRegs, 0); - bool IsIntel = - (VendorRegs.Ebx == signature_INTEL_ebx) && - (VendorRegs.Edx == signature_INTEL_edx) && - (VendorRegs.Ecx == signature_INTEL_ecx); - bool IsAMD = - (VendorRegs.Ebx == signature_AMD_ebx) && - (VendorRegs.Edx == signature_AMD_edx) && - (VendorRegs.Ecx == signature_AMD_ecx); - // Default to an empty feature set if not on a supported CPU. - CPUIDRegs FeaturesRegs = {}; - if (IsIntel || IsAMD) { - getCPUID(&FeaturesRegs, 1); - } - return FeaturesRegs; -} - -#ifndef bit_SSE4_2 -# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. -#endif - -bool testCPUFeature(CPUFeature Feature) -{ - CPUIDRegs FeaturesRegs = getCPUFeatures(); - - switch (Feature) { - case CRC32CPUFeature: // CRC32 is provided by SSE 4.2. - return !!(FeaturesRegs.Ecx & bit_SSE4_2); - default: - break; - } - return false; +# ifndef bit_SSE4_2 +# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. +# endif +bool hasHardwareCRC32() { + u32 Eax, Ebx, Ecx, Edx; + __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); + const bool IsIntel = (Ebx == signature_INTEL_ebx) && + (Edx == signature_INTEL_edx) && + (Ecx == signature_INTEL_ecx); + const bool IsAMD = (Ebx == signature_AMD_ebx) && + (Edx == signature_AMD_edx) && + (Ecx == signature_AMD_ecx); + if (!IsIntel && !IsAMD) + return false; + __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); + return !!(Ecx & bit_SSE4_2); } #elif defined(__arm__) || defined(__aarch64__) -// For ARM and AArch64, hardware CRC32 support is indicated in the -// AT_HWVAL auxiliary vector. - -#ifndef HWCAP_CRC32 -# define HWCAP_CRC32 (1<<7) // HWCAP_CRC32 is missing on older platforms. -#endif - -bool testCPUFeature(CPUFeature Feature) { - uptr HWCap = getauxval(AT_HWCAP); - - switch (Feature) { - case CRC32CPUFeature: - return !!(HWCap & HWCAP_CRC32); - default: +// For ARM and AArch64, hardware CRC32 support is indicated in the AT_HWCAP +// auxiliary vector. +# ifndef AT_HWCAP +# define AT_HWCAP 16 +# endif +# ifndef HWCAP_CRC32 +# define HWCAP_CRC32 (1 << 7) // HWCAP_CRC32 is missing on older platforms. +# endif +# if SANITIZER_POSIX +bool hasHardwareCRC32ARMPosix() { + uptr F = internal_open("/proc/self/auxv", O_RDONLY); + if (internal_iserror(F)) + return false; + struct { uptr Tag; uptr Value; } Entry = { 0, 0 }; + for (;;) { + uptr N = internal_read(F, &Entry, sizeof(Entry)); + if (internal_iserror(N) || N != sizeof(Entry) || + (Entry.Tag == 0 && Entry.Value == 0) || Entry.Tag == AT_HWCAP) break; } - return false; + internal_close(F); + return (Entry.Tag == AT_HWCAP && (Entry.Value & HWCAP_CRC32) != 0); +} +# else +bool hasHardwareCRC32ARMPosix() { return false; } +# endif // SANITIZER_POSIX + +// Bionic doesn't initialize its globals early enough. This causes issues when +// trying to access them from a preinit_array (b/25751302) or from another +// constructor called before the libc one (b/68046352). __progname is +// initialized after the other globals, so we can check its value to know if +// calling getauxval is safe. +extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; +INLINE bool areBionicGlobalsInitialized() { + return !SANITIZER_ANDROID || (&__progname && __progname); +} + +bool hasHardwareCRC32() { + if (&getauxval && areBionicGlobalsInitialized()) + return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); + return hasHardwareCRC32ARMPosix(); } #else -bool testCPUFeature(CPUFeature Feature) { - return false; -} +bool hasHardwareCRC32() { return false; } #endif // defined(__x86_64__) || defined(__i386__) } // namespace __scudo diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.h b/contrib/compiler-rt/lib/scudo/scudo_utils.h index 6c6c9d893404..43448e0831e8 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_utils.h +++ b/contrib/compiler-rt/lib/scudo/scudo_utils.h @@ -14,14 +14,14 @@ #ifndef SCUDO_UTILS_H_ #define SCUDO_UTILS_H_ -#include - #include "sanitizer_common/sanitizer_common.h" +#include + namespace __scudo { template -inline Dest bit_cast(const Source& source) { +INLINE Dest bit_cast(const Source& source) { static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!"); Dest dest; memcpy(&dest, &source, sizeof(dest)); @@ -30,63 +30,7 @@ inline Dest bit_cast(const Source& source) { void NORETURN dieWithMessage(const char *Format, ...); -enum CPUFeature { - CRC32CPUFeature = 0, - MaxCPUFeature, -}; -bool testCPUFeature(CPUFeature feature); - -INLINE u64 rotl(const u64 X, int K) { - return (X << K) | (X >> (64 - K)); -} - -// XoRoShiRo128+ PRNG (http://xoroshiro.di.unimi.it/). -struct XoRoShiRo128Plus { - public: - void init() { - if (UNLIKELY(!GetRandom(reinterpret_cast(State), sizeof(State)))) { - // Early processes (eg: init) do not have /dev/urandom yet, but we still - // have to provide them with some degree of entropy. Not having a secure - // seed is not as problematic for them, as they are less likely to be - // the target of heap based vulnerabilities exploitation attempts. - State[0] = NanoTime(); - State[1] = 0; - } - fillCache(); - } - u8 getU8() { - if (UNLIKELY(isCacheEmpty())) - fillCache(); - const u8 Result = static_cast(CachedBytes & 0xff); - CachedBytes >>= 8; - CachedBytesAvailable--; - return Result; - } - u64 getU64() { return next(); } - - private: - u8 CachedBytesAvailable; - u64 CachedBytes; - u64 State[2]; - u64 next() { - const u64 S0 = State[0]; - u64 S1 = State[1]; - const u64 Result = S0 + S1; - S1 ^= S0; - State[0] = rotl(S0, 55) ^ S1 ^ (S1 << 14); - State[1] = rotl(S1, 36); - return Result; - } - bool isCacheEmpty() { - return CachedBytesAvailable == 0; - } - void fillCache() { - CachedBytes = next(); - CachedBytesAvailable = sizeof(CachedBytes); - } -}; - -typedef XoRoShiRo128Plus ScudoPrng; +bool hasHardwareCRC32(); } // namespace __scudo diff --git a/contrib/compiler-rt/lib/stats/stats.cc b/contrib/compiler-rt/lib/stats/stats.cc index df9845a38de9..6a6eb3a3cdbc 100644 --- a/contrib/compiler-rt/lib/stats/stats.cc +++ b/contrib/compiler-rt/lib/stats/stats.cc @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_internal_defs.h" #if SANITIZER_POSIX #include "sanitizer_common/sanitizer_posix.h" diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc index d84df4a6413c..f13a7432ec95 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc @@ -48,8 +48,8 @@ static bool bogusfd(int fd) { } static FdSync *allocsync(ThreadState *thr, uptr pc) { - FdSync *s = (FdSync*)user_alloc(thr, pc, sizeof(FdSync), kDefaultAlignment, - false); + FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync), + kDefaultAlignment, false); atomic_store(&s->rc, 1, memory_order_relaxed); return s; } @@ -79,7 +79,7 @@ static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) { if (l1 == 0) { uptr size = kTableSizeL2 * sizeof(FdDesc); // We need this to reside in user memory to properly catch races on it. - void *p = user_alloc(thr, pc, size, kDefaultAlignment, false); + void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false); internal_memset(p, 0, size); MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size); if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel)) diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 001123f4941e..d92976ad6e9e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_posix.h" @@ -39,6 +40,21 @@ using namespace __tsan; // NOLINT #define stderr __stderrp #endif +#if SANITIZER_NETBSD +#define dirfd(dirp) (*(int *)(dirp)) +#define fileno_unlocked fileno + +#if _LP64 +#define __sF_size 152 +#else +#define __sF_size 88 +#endif + +#define stdout ((char*)&__sF + (__sF_size * 1)) +#define stderr ((char*)&__sF + (__sF_size * 2)) + +#endif + #if SANITIZER_ANDROID #define mallopt(a, b) #endif @@ -49,11 +65,6 @@ const int kSigCount = 129; const int kSigCount = 65; #endif -struct my_siginfo_t { - // The size is determined by looking at sizeof of real siginfo_t on linux. - u64 opaque[128 / sizeof(u64)]; -}; - #ifdef __mips__ struct ucontext_t { u64 opaque[768 / sizeof(u64) + 1]; @@ -84,19 +95,25 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int fileno_unlocked(void *stream); +#if !SANITIZER_NETBSD extern "C" int dirfd(void *dirp); -#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID +#endif +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD extern "C" int mallopt(int param, int value); #endif +#if SANITIZER_NETBSD +extern __sanitizer_FILE __sF[]; +#else extern __sanitizer_FILE *stdout, *stderr; -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#endif +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; #else const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD const int EPOLL_CTL_ADD = 1; #endif const int SIGILL = 4; @@ -105,7 +122,7 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC +#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; #else @@ -113,7 +130,9 @@ const int SIGBUS = 7; const int SIGSYS = 31; #endif void *const MAP_FAILED = (void*)-1; -#if !SANITIZER_MAC +#if SANITIZER_NETBSD +const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; +#elif !SANITIZER_MAC const int PTHREAD_BARRIER_SERIAL_THREAD = -1; #endif const int MAP_FIXED = 0x10; @@ -125,48 +144,7 @@ typedef long long_t; // NOLINT # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ -typedef void (*sighandler_t)(int sig); -typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); - -#if SANITIZER_ANDROID -struct sigaction_t { - u32 sa_flags; - union { - sighandler_t sa_handler; - sigactionhandler_t sa_sigaction; - }; - __sanitizer_sigset_t sa_mask; - void (*sa_restorer)(); -}; -#else -struct sigaction_t { -#ifdef __mips__ - u32 sa_flags; -#endif - union { - sighandler_t sa_handler; - sigactionhandler_t sa_sigaction; - }; -#if SANITIZER_FREEBSD - int sa_flags; - __sanitizer_sigset_t sa_mask; -#elif SANITIZER_MAC - __sanitizer_sigset_t sa_mask; - int sa_flags; -#else - __sanitizer_sigset_t sa_mask; -#ifndef __mips__ - int sa_flags; -#endif - void (*sa_restorer)(); -#endif -}; -#endif - -const sighandler_t SIG_DFL = (sighandler_t)0; -const sighandler_t SIG_IGN = (sighandler_t)1; -const sighandler_t SIG_ERR = (sighandler_t)-1; -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) @@ -180,13 +158,11 @@ const int SIG_SETMASK = 2; #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ (!cur_thread()->is_inited) -static sigaction_t sigactions[kSigCount]; - namespace __tsan { struct SignalDesc { bool armed; bool sigaction; - my_siginfo_t siginfo; + __sanitizer_siginfo siginfo; ucontext_t ctx; }; @@ -200,11 +176,41 @@ struct ThreadSignalContext { __sanitizer_sigset_t oldset; }; -// The object is 64-byte aligned, because we want hot data to be located in -// a single cache line if possible (it's accessed in every interceptor). -static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)]; +// The sole reason tsan wraps atexit callbacks is to establish synchronization +// between callback setup and callback execution. +struct AtExitCtx { + void (*f)(); + void *arg; +}; + +// InterceptorContext holds all global data required for interceptors. +// It's explicitly constructed in InitializeInterceptors with placement new +// and is never destroyed. This allows usage of members with non-trivial +// constructors and destructors. +struct InterceptorContext { + // The object is 64-byte aligned, because we want hot data to be located + // in a single cache line if possible (it's accessed in every interceptor). + ALIGNED(64) LibIgnore libignore; + __sanitizer_sigaction sigactions[kSigCount]; +#if !SANITIZER_MAC && !SANITIZER_NETBSD + unsigned finalize_key; +#endif + + BlockingMutex atexit_mu; + Vector AtExitStack; + + InterceptorContext() + : libignore(LINKER_INITIALIZED), AtExitStack() { + } +}; + +static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; +InterceptorContext *interceptor_ctx() { + return reinterpret_cast(&interceptor_placeholder[0]); +} + LibIgnore *libignore() { - return reinterpret_cast(&libignore_placeholder[0]); + return &interceptor_ctx()->libignore; } void InitializeLibIgnore() { @@ -232,10 +238,6 @@ static ThreadSignalContext *SigCtx(ThreadState *thr) { return ctx; } -#if !SANITIZER_MAC -static unsigned g_thread_finalize_key; -#endif - ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) { @@ -284,8 +286,18 @@ void ScopedInterceptor::DisableIgnores() { #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) #if SANITIZER_FREEBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) +#elif SANITIZER_NETBSD +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ + INTERCEPT_FUNCTION(__libc_##func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ + INTERCEPT_FUNCTION(__libc_thr_##func) #else # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) #endif #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ @@ -346,17 +358,30 @@ TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { return res; } -// The sole reason tsan wraps atexit callbacks is to establish synchronization -// between callback setup and callback execution. -struct AtExitCtx { - void (*f)(); - void *arg; -}; +TSAN_INTERCEPTOR(int, pause, int fake) { + SCOPED_TSAN_INTERCEPTOR(pause, fake); + return BLOCK_REAL(pause)(fake); +} -static void at_exit_wrapper(void *arg) { - ThreadState *thr = cur_thread(); - uptr pc = 0; - Acquire(thr, pc, (uptr)arg); +static void at_exit_wrapper() { + AtExitCtx *ctx; + { + // Ensure thread-safety. + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + // Pop AtExitCtx from the top of the stack of callback functions + uptr element = interceptor_ctx()->AtExitStack.Size() - 1; + ctx = interceptor_ctx()->AtExitStack[element]; + interceptor_ctx()->AtExitStack.PopBack(); + } + + Acquire(cur_thread(), (uptr)0, (uptr)ctx); + ((void(*)())ctx->f)(); + InternalFree(ctx); +} + +static void cxa_at_exit_wrapper(void *arg) { + Acquire(cur_thread(), 0, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(void *arg))ctx->f)(ctx->arg); InternalFree(ctx); @@ -367,7 +392,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. @@ -377,7 +402,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); @@ -392,12 +417,27 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); - int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso); + int res; + if (!dso) { + // NetBSD does not preserve the 2nd argument if dso is equal to 0 + // Store ctx in a local stack-like structure + + // Ensure thread-safety. + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0); + // Push AtExitCtx on the top of the stack of callback functions + if (!res) { + interceptor_ctx()->AtExitStack.PushBack(ctx); + } + } else { + res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso); + } ThreadIgnoreEnd(thr, pc); return res; } -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_NETBSD static void on_exit_wrapper(int status, void *arg) { ThreadState *thr = cur_thread(); uptr pc = 0; @@ -408,7 +448,7 @@ static void on_exit_wrapper(int status, void *arg) { } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); @@ -422,6 +462,9 @@ TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { ThreadIgnoreEnd(thr, pc); return res; } +#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit) +#else +#define TSAN_MAYBE_INTERCEPT_ON_EXIT #endif // Cleanup old bufs. @@ -461,13 +504,15 @@ static void LongJmp(ThreadState *thr, uptr *env) { uptr mangled_sp = env[0]; #elif SANITIZER_FREEBSD uptr mangled_sp = env[2]; +#elif SANITIZER_NETBSD + uptr mangled_sp = env[6]; #elif SANITIZER_MAC # ifdef __aarch64__ uptr mangled_sp = env[13]; # else uptr mangled_sp = env[2]; # endif -#elif defined(SANITIZER_LINUX) +#elif SANITIZER_LINUX # ifdef __aarch64__ uptr mangled_sp = env[13]; # elif defined(__mips64) @@ -510,10 +555,29 @@ TSAN_INTERCEPTOR(int, setjmp, void *env); TSAN_INTERCEPTOR(int, _setjmp, void *env); TSAN_INTERCEPTOR(int, sigsetjmp, void *env); #else // SANITIZER_MAC + +#if SANITIZER_NETBSD +#define setjmp_symname __setjmp14 +#define sigsetjmp_symname __sigsetjmp14 +#else +#define setjmp_symname setjmp +#define sigsetjmp_symname sigsetjmp +#endif + +#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x +#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) +#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) +#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) + +#define TSAN_STRING_SETJMP_(x) # x +#define TSAN_STRING_SETJMP__(x) TSAN_STRING_SETJMP_(x) +#define TSAN_STRING_SETJMP TSAN_STRING_SETJMP__(setjmp_symname) +#define TSAN_STRING_SIGSETJMP TSAN_STRING_SETJMP__(sigsetjmp_symname) + // Not called. Merely to satisfy TSAN_INTERCEPT(). extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __interceptor_setjmp(void *env); -extern "C" int __interceptor_setjmp(void *env) { +int TSAN_INTERCEPTOR_SETJMP(void *env); +extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { CHECK(0); return 0; } @@ -527,51 +591,75 @@ extern "C" int __interceptor__setjmp(void *env) { } extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __interceptor_sigsetjmp(void *env); -extern "C" int __interceptor_sigsetjmp(void *env) { +int TSAN_INTERCEPTOR_SIGSETJMP(void *env); +extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { CHECK(0); return 0; } +#if !SANITIEZER_NETBSD extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor___sigsetjmp(void *env); extern "C" int __interceptor___sigsetjmp(void *env) { CHECK(0); return 0; } +#endif -extern "C" int setjmp(void *env); +extern "C" int setjmp_symname(void *env); extern "C" int _setjmp(void *env); -extern "C" int sigsetjmp(void *env); +extern "C" int sigsetjmp_symname(void *env); +#if !SANITIEZER_NETBSD extern "C" int __sigsetjmp(void *env); -DEFINE_REAL(int, setjmp, void *env) +#endif +DEFINE_REAL(int, setjmp_symname, void *env) DEFINE_REAL(int, _setjmp, void *env) -DEFINE_REAL(int, sigsetjmp, void *env) +DEFINE_REAL(int, sigsetjmp_symname, void *env) +#if !SANITIEZER_NETBSD DEFINE_REAL(int, __sigsetjmp, void *env) +#endif #endif // SANITIZER_MAC -TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { +#if SANITIZER_NETBSD +#define longjmp_symname __longjmp14 +#define siglongjmp_symname __siglongjmp14 +#else +#define longjmp_symname longjmp +#define siglongjmp_symname siglongjmp +#endif + +TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, // bad things will happen. We will jump over ScopedInterceptor dtor and can // leave thr->in_ignored_lib set. { - SCOPED_INTERCEPTOR_RAW(longjmp, env, val); + SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val); } LongJmp(cur_thread(), env); - REAL(longjmp)(env, val); + REAL(longjmp_symname)(env, val); } -TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { +TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { { - SCOPED_INTERCEPTOR_RAW(siglongjmp, env, val); + SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val); } LongJmp(cur_thread(), env); - REAL(siglongjmp)(env, val); + REAL(siglongjmp_symname)(env, val); } +#if SANITIZER_NETBSD +TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { + { + SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); + } + LongJmp(cur_thread(), env); + REAL(_longjmp)(env, val); +} +#endif + #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalAlloc(size); void *p = 0; { @@ -584,11 +672,11 @@ TSAN_INTERCEPTOR(void*, malloc, uptr size) { TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); - return user_alloc(thr, pc, sz, align); + return user_memalign(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalCalloc(size, n); void *p = 0; { @@ -600,7 +688,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalRealloc(p, size); if (p) invoke_free_hook(p); @@ -615,7 +703,7 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); @@ -625,7 +713,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); @@ -730,7 +818,7 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); - return user_alloc(thr, pc, sz, align); + return user_memalign(thr, pc, align, sz); } #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) #else @@ -739,21 +827,29 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { - SCOPED_INTERCEPTOR_RAW(memalign, align, sz); - return user_alloc(thr, pc, sz, align); + if (UNLIKELY(cur_thread()->in_symbolizer)) + return InternalAlloc(sz, nullptr, align); + SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); + return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { + if (UNLIKELY(cur_thread()->in_symbolizer)) + return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); - return user_alloc(thr, pc, sz, GetPageSizeCached()); + return user_valloc(thr, pc, sz); } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { + if (UNLIKELY(cur_thread()->in_symbolizer)) { + uptr PageSize = GetPageSizeCached(); + sz = sz ? RoundUpTo(sz, PageSize) : PageSize; + return InternalAlloc(sz, nullptr, PageSize); + } SCOPED_INTERCEPTOR_RAW(pvalloc, sz); - sz = RoundUp(sz, GetPageSizeCached()); - return user_alloc(thr, pc, sz, GetPageSizeCached()); + return user_pvalloc(thr, pc, sz); } #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) #else @@ -762,9 +858,15 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { + if (UNLIKELY(cur_thread()->in_symbolizer)) { + void *p = InternalAlloc(sz, nullptr, align); + if (!p) + return errno_ENOMEM; + *memptr = p; + return 0; + } SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); - *memptr = user_alloc(thr, pc, sz, align); - return 0; + return user_posix_memalign(thr, pc, memptr, align, sz); } #endif @@ -830,11 +932,12 @@ void DestroyThreadState() { } } // namespace __tsan -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_NETBSD static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { - if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { + if (pthread_setspecific(interceptor_ctx()->finalize_key, + (void*)(iter - 1))) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } @@ -860,9 +963,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_NETBSD ThreadIgnoreBegin(thr, 0); - if (pthread_setspecific(g_thread_finalize_key, + if (pthread_setspecific(interceptor_ctx()->finalize_key, (void *)GetPthreadDestructorIterations())) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); @@ -887,6 +990,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) { TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); + + MaybeSpawnBackgroundThread(); + if (ctx->after_multithreaded_fork) { if (flags()->die_after_fork) { Report("ThreadSanitizer: starting new threads after multi-threaded " @@ -1312,10 +1418,15 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { if (o == 0 || f == 0) return errno_EINVAL; atomic_uint32_t *a; - if (!SANITIZER_MAC) - a = static_cast(o); - else // On OS X, pthread_once_t has a header with a long-sized signature. + + if (SANITIZER_MAC) a = static_cast((void *)((char *)o + sizeof(long_t))); + else if (SANITIZER_NETBSD) + a = static_cast + ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz)); + else + a = static_cast(o); + u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { @@ -1347,7 +1458,7 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); @@ -1762,7 +1873,9 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, namespace __tsan { static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, - bool sigact, int sig, my_siginfo_t *info, void *uctx) { + bool sigact, int sig, + __sanitizer_siginfo *info, void *uctx) { + __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; if (acquire) Acquire(thr, 0, (uptr)&sigactions[sig]); // Signals are generally asynchronous, so if we receive a signals when @@ -1784,14 +1897,13 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, // This code races with sigaction. Be careful to not read sa_sigaction twice. // Also need to remember pc for reporting before the call, // because the handler can reset it. - volatile uptr pc = sigact ? - (uptr)sigactions[sig].sa_sigaction : - (uptr)sigactions[sig].sa_handler; - if (pc != (uptr)SIG_DFL && pc != (uptr)SIG_IGN) { + volatile uptr pc = + sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler; + if (pc != sig_dfl && pc != sig_ign) { if (sigact) - ((sigactionhandler_t)pc)(sig, info, uctx); + ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); else - ((sighandler_t)pc)(sig); + ((__sanitizer_sighandler_ptr)pc)(sig); } if (!ctx->after_multithreaded_fork) { thr->ignore_reads_and_writes = ignore_reads_and_writes; @@ -1855,7 +1967,8 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { } void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, - my_siginfo_t *info, void *ctx) { + __sanitizer_siginfo *info, + void *ctx) { ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { @@ -1905,58 +2018,10 @@ static void rtl_sighandler(int sig) { rtl_generic_sighandler(false, sig, 0, 0); } -static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { +static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) { rtl_generic_sighandler(true, sig, info, ctx); } -TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { - // Note: if we call REAL(sigaction) directly for any reason without proxying - // the signal handler through rtl_sigaction, very bad things will happen. - // The handler will run synchronously and corrupt tsan per-thread state. - SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); - if (old) - internal_memcpy(old, &sigactions[sig], sizeof(*old)); - if (act == 0) - return 0; - // Copy act into sigactions[sig]. - // Can't use struct copy, because compiler can emit call to memcpy. - // Can't use internal_memcpy, because it copies byte-by-byte, - // and signal handler reads the sa_handler concurrently. It it can read - // some bytes from old value and some bytes from new value. - // Use volatile to prevent insertion of memcpy. - sigactions[sig].sa_handler = *(volatile sighandler_t*)&act->sa_handler; - sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags; - internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, - sizeof(sigactions[sig].sa_mask)); -#if !SANITIZER_FREEBSD && !SANITIZER_MAC - sigactions[sig].sa_restorer = act->sa_restorer; -#endif - sigaction_t newact; - internal_memcpy(&newact, act, sizeof(newact)); - internal_sigfillset(&newact.sa_mask); - if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { - if (newact.sa_flags & SA_SIGINFO) - newact.sa_sigaction = rtl_sigaction; - else - newact.sa_handler = rtl_sighandler; - } - ReleaseStore(thr, pc, (uptr)&sigactions[sig]); - int res = REAL(sigaction)(sig, &newact, 0); - return res; -} - -TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { - sigaction_t act; - act.sa_handler = h; - internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); - act.sa_flags = 0; - sigaction_t old; - int res = sigaction(sig, &act, &old); - if (res) - return SIG_ERR; - return old.sa_handler; -} - TSAN_INTERCEPTOR(int, raise, int sig) { SCOPED_TSAN_INTERCEPTOR(raise, sig); ThreadSignalContext *sctx = SigCtx(thr); @@ -2020,7 +2085,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, } TSAN_INTERCEPTOR(int, fork, int fake) { - if (cur_thread()->in_symbolizer) + if (UNLIKELY(cur_thread()->in_symbolizer)) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); @@ -2270,6 +2335,77 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #include "sanitizer_common/sanitizer_common_interceptors.inc" +static int sigaction_impl(int sig, const __sanitizer_sigaction *act, + __sanitizer_sigaction *old); +static __sanitizer_sighandler_ptr signal_impl(int sig, + __sanitizer_sighandler_ptr h); + +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ + { return sigaction_impl(signo, act, oldact); } + +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ + { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } + +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +int sigaction_impl(int sig, const __sanitizer_sigaction *act, + __sanitizer_sigaction *old) { + // Note: if we call REAL(sigaction) directly for any reason without proxying + // the signal handler through rtl_sigaction, very bad things will happen. + // The handler will run synchronously and corrupt tsan per-thread state. + SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); + __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; + __sanitizer_sigaction old_stored; + if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored)); + __sanitizer_sigaction newact; + if (act) { + // Copy act into sigactions[sig]. + // Can't use struct copy, because compiler can emit call to memcpy. + // Can't use internal_memcpy, because it copies byte-by-byte, + // and signal handler reads the handler concurrently. It it can read + // some bytes from old value and some bytes from new value. + // Use volatile to prevent insertion of memcpy. + sigactions[sig].handler = + *(volatile __sanitizer_sighandler_ptr const *)&act->handler; + sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; + internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, + sizeof(sigactions[sig].sa_mask)); +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD + sigactions[sig].sa_restorer = act->sa_restorer; +#endif + internal_memcpy(&newact, act, sizeof(newact)); + internal_sigfillset(&newact.sa_mask); + if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) { + if (newact.sa_flags & SA_SIGINFO) + newact.sigaction = rtl_sigaction; + else + newact.handler = rtl_sighandler; + } + ReleaseStore(thr, pc, (uptr)&sigactions[sig]); + act = &newact; + } + int res = REAL(sigaction)(sig, act, old); + if (res == 0 && old) { + uptr cb = (uptr)old->sigaction; + if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) { + internal_memcpy(old, &old_stored, sizeof(*old)); + } + } + return res; +} + +static __sanitizer_sighandler_ptr signal_impl(int sig, + __sanitizer_sighandler_ptr h) { + __sanitizer_sigaction act; + act.handler = h; + internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); + act.sa_flags = 0; + __sanitizer_sigaction old; + int res = sigaction_symname(sig, &act, &old); + if (res) return (__sanitizer_sighandler_ptr)sig_err; + return old.handler; +} + #define TSAN_SYSCALL() \ ThreadState *thr = cur_thread(); \ if (thr->ignore_interceptors) \ @@ -2290,7 +2426,7 @@ struct ScopedSyscall { } }; -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); @@ -2409,6 +2545,34 @@ TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { } #endif +#if SANITIZER_NETBSD +TSAN_INTERCEPTOR(void, _lwp_exit) { + SCOPED_TSAN_INTERCEPTOR(_lwp_exit); + DestroyThreadState(); + REAL(_lwp_exit)(); +} +#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit) +#else +#define TSAN_MAYBE_INTERCEPT__LWP_EXIT +#endif + +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m); +TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()); + namespace __tsan { static void finalize(void *arg) { @@ -2440,20 +2604,30 @@ void InitializeInterceptors() { mallopt(-3, 32*1024); // M_MMAP_THRESHOLD #endif + new(interceptor_ctx()) InterceptorContext(); + InitializeCommonInterceptors(); + InitializeSignalInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. using __interception::GetRealFunctionAddress; - GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0); + GetRealFunctionAddress(TSAN_STRING_SETJMP, + (uptr*)&REAL(setjmp_symname), 0, 0); GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0); + GetRealFunctionAddress(TSAN_STRING_SIGSETJMP, + (uptr*)&REAL(sigsetjmp_symname), 0, 0); +#if !SANITIZER_NETBSD GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); +#endif #endif - TSAN_INTERCEPT(longjmp); - TSAN_INTERCEPT(siglongjmp); + TSAN_INTERCEPT(longjmp_symname); + TSAN_INTERCEPT(siglongjmp_symname); +#if SANITIZER_NETBSD + TSAN_INTERCEPT(_longjmp); +#endif TSAN_INTERCEPT(malloc); TSAN_INTERCEPT(__libc_memalign); @@ -2548,8 +2722,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(rmdir); TSAN_INTERCEPT(closedir); - TSAN_INTERCEPT(sigaction); - TSAN_INTERCEPT(signal); TSAN_INTERCEPT(sigsuspend); TSAN_INTERCEPT(sigblock); TSAN_INTERCEPT(sigsetmask); @@ -2560,6 +2732,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(sleep); TSAN_INTERCEPT(usleep); TSAN_INTERCEPT(nanosleep); + TSAN_INTERCEPT(pause); TSAN_INTERCEPT(gettimeofday); TSAN_INTERCEPT(getaddrinfo); @@ -2568,7 +2741,7 @@ void InitializeInterceptors() { #if !SANITIZER_ANDROID TSAN_INTERCEPT(dl_iterate_phdr); #endif - TSAN_INTERCEPT(on_exit); + TSAN_MAYBE_INTERCEPT_ON_EXIT; TSAN_INTERCEPT(__cxa_atexit); TSAN_INTERCEPT(_exit); @@ -2576,6 +2749,8 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__tls_get_addr); #endif + TSAN_MAYBE_INTERCEPT__LWP_EXIT; + #if !SANITIZER_MAC && !SANITIZER_ANDROID // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. @@ -2587,13 +2762,30 @@ void InitializeInterceptors() { Die(); } -#if !SANITIZER_MAC - if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { +#if !SANITIZER_MAC && !SANITIZER_NETBSD + if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } #endif + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); + FdInit(); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index de47466501da..959a39465e33 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -49,4 +49,16 @@ LibIgnore *libignore(); #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) +#if SANITIZER_NETBSD +# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +#else +# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) +#endif + #endif // TSAN_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc index 4f1079467331..3b3a4a2815c3 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -100,7 +100,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ return tsan_atomic_f##_compare_exchange_strong( \ - (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ kMacOrderNonBarrier, kMacOrderNonBarrier); \ } \ \ @@ -108,7 +108,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, t volatile *ptr) { \ SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ return tsan_atomic_f##_compare_exchange_strong( \ - (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ kMacOrderBarrier, kMacOrderNonBarrier); \ } @@ -122,14 +122,14 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, int64_t) -#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ - TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ - char *byte_ptr = ((char *)ptr) + (n >> 3); \ - char bit = 0x80u >> (n & 7); \ - char mask = clear ? ~bit : bit; \ - char orig_byte = op((a8 *)byte_ptr, mask, mo); \ - return orig_byte & bit; \ +#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ + TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ + volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ + char bit = 0x80u >> (n & 7); \ + char mask = clear ? ~bit : bit; \ + char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ + return orig_byte & bit; \ } #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc index f68a0468de53..a7922a42e42c 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc @@ -14,6 +14,7 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_vector.h" #include "tsan_interface_ann.h" #include "tsan_mutex.h" #include "tsan_report.h" @@ -21,7 +22,6 @@ #include "tsan_mman.h" #include "tsan_flags.h" #include "tsan_platform.h" -#include "tsan_vector.h" #define CALLERPC ((uptr)__builtin_return_address(0)) @@ -185,10 +185,10 @@ void PrintMatchedBenignRaces() { int unique_count = 0; int hit_count = 0; int add_count = 0; - Vector hit_matched(MBlockScopedBuf); + Vector hit_matched; CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count, &ExpectRace::hitcount); - Vector add_matched(MBlockScopedBuf); + Vector add_matched; CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count, &ExpectRace::addcount); if (hit_matched.Size()) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc index 8c759a3be4e1..d6c1ca6622f4 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc @@ -25,6 +25,11 @@ #include #include +// DISPATCH_NOESCAPE is not defined prior to XCode 8. +#ifndef DISPATCH_NOESCAPE +#define DISPATCH_NOESCAPE +#endif + typedef long long_t; // NOLINT namespace __tsan { @@ -86,7 +91,8 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, void *orig_context, dispatch_function_t orig_work) { tsan_block_context_t *new_context = - (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t)); + (tsan_block_context_t *)user_alloc_internal(thr, pc, + sizeof(tsan_block_context_t)); new_context->queue = queue; new_context->orig_context = orig_context; new_context->orig_work = orig_work; @@ -176,7 +182,8 @@ static void invoke_and_release_block(void *param) { } #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ - TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ + DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ dispatch_block_t heap_block = Block_copy(block); \ @@ -266,7 +273,7 @@ TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, // need to undefine the macro. #undef dispatch_once TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, - dispatch_block_t block) { + DISPATCH_NOESCAPE dispatch_block_t block) { SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); atomic_uint32_t *a = reinterpret_cast(predicate); u32 v = atomic_load(a, memory_order_acquire); @@ -476,7 +483,8 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, } TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, - dispatch_queue_t queue, void (^block)(size_t)) { + dispatch_queue_t queue, + DISPATCH_NOESCAPE void (^block)(size_t)) { SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); void *parent_to_child_sync = nullptr; @@ -519,9 +527,9 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, return REAL(dispatch_data_create)(buffer, size, q, destructor); if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) - destructor = ^(void) { WRAP(free)((void *)buffer); }; + destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) - destructor = ^(void) { WRAP(munmap)((void *)buffer, size); }; + destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); dispatch_block_t heap_block = Block_copy(destructor); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc index 8d31ccbcaaca..455c95df6c50 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc @@ -26,7 +26,7 @@ using namespace __tsan; #define COMMON_MALLOC_FORCE_UNLOCK() #define COMMON_MALLOC_MEMALIGN(alignment, size) \ void *p = \ - user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment) + user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ SCOPED_INTERCEPTOR_RAW(malloc, size); \ @@ -43,7 +43,7 @@ using namespace __tsan; if (cur_thread()->in_symbolizer) \ return InternalAlloc(size, nullptr, GetPageSizeCached()); \ SCOPED_INTERCEPTOR_RAW(valloc, size); \ - void *p = user_alloc(thr, pc, size, GetPageSizeCached()) + void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc index f79dccddba9f..19680238bf76 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc @@ -13,6 +13,7 @@ #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "tsan_mman.h" #include "tsan_rtl.h" @@ -149,11 +150,12 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { OutputReport(thr, rep); } -void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { +void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align, + bool signal) { if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) return Allocator::FailureHandler::OnBadRequest(); void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align); - if (p == 0) + if (UNLIKELY(p == 0)) return 0; if (ctx && ctx->initialized) OnUserAlloc(thr, pc, (uptr)p, sz, true); @@ -162,15 +164,6 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { return p; } -void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { - if (CheckForCallocOverflow(size, n)) - return Allocator::FailureHandler::OnBadRequest(); - void *p = user_alloc(thr, pc, n * size); - if (p) - internal_memset(p, 0, n * size); - return p; -} - void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { ScopedGlobalProcessor sgp; if (ctx && ctx->initialized) @@ -180,6 +173,19 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { SignalUnsafeCall(thr, pc); } +void *user_alloc(ThreadState *thr, uptr pc, uptr sz) { + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment)); +} + +void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) + return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest()); + void *p = user_alloc_internal(thr, pc, n * size); + if (p) + internal_memset(p, 0, n * size); + return SetErrnoOnNull(p); +} + void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); ctx->metamap.AllocBlock(thr, pc, p, sz); @@ -200,15 +206,64 @@ void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { // FIXME: Handle "shrinking" more efficiently, // it seems that some software actually does this. - void *p2 = user_alloc(thr, pc, sz); - if (p2 == 0) - return 0; - if (p) { - uptr oldsz = user_alloc_usable_size(p); - internal_memcpy(p2, p, min(oldsz, sz)); + if (!p) + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz)); + if (!sz) { + user_free(thr, pc, p); + return nullptr; + } + void *new_p = user_alloc_internal(thr, pc, sz); + if (new_p) { + uptr old_sz = user_alloc_usable_size(p); + internal_memcpy(new_p, p, min(old_sz, sz)); user_free(thr, pc, p); } - return p2; + return SetErrnoOnNull(new_p); +} + +void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) { + if (UNLIKELY(!IsPowerOfTwo(align))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align)); +} + +int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, + uptr sz) { + if (UNLIKELY(!CheckPosixMemalignAlignment(align))) { + Allocator::FailureHandler::OnBadRequest(); + return errno_EINVAL; + } + void *ptr = user_alloc_internal(thr, pc, sz, align); + if (UNLIKELY(!ptr)) + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, align)); + *memptr = ptr; + return 0; +} + +void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) { + errno = errno_EINVAL; + return Allocator::FailureHandler::OnBadRequest(); + } + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align)); +} + +void *user_valloc(ThreadState *thr, uptr pc, uptr sz) { + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached())); +} + +void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) { + errno = errno_ENOMEM; + return Allocator::FailureHandler::OnBadRequest(); + } + // pvalloc(0) should allocate one page. + sz = sz ? RoundUpTo(sz, PageSize) : PageSize; + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize)); } uptr user_alloc_usable_size(const void *p) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h index 8cdeeb35aa6b..6042c5c5d96d 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h @@ -27,13 +27,20 @@ void AllocatorProcFinish(Processor *proc); void AllocatorPrintStats(); // For user allocations. -void *user_alloc(ThreadState *thr, uptr pc, uptr sz, - uptr align = kDefaultAlignment, bool signal = true); -void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); +void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, + uptr align = kDefaultAlignment, bool signal = true); // Does not accept NULL. void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); +// Interceptor implementations. +void *user_alloc(ThreadState *thr, uptr pc, uptr sz); +void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); -void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align); +void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); +int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, + uptr sz); +void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz); +void *user_valloc(ThreadState *thr, uptr pc, uptr sz); +void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz); uptr user_alloc_usable_size(const void *p); // Invoking malloc/free hooks that may be installed by the user. diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc index 4d03145c16ad..a1bb22690e0f 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc @@ -20,6 +20,7 @@ using namespace __tsan; // NOLINT namespace std { struct nothrow_t {}; +enum class align_val_t: __sanitizer::uptr {}; } // namespace std DECLARE_REAL(void *, malloc, uptr size) @@ -38,6 +39,18 @@ DECLARE_REAL(void, free, void *ptr) invoke_malloc_hook(p, size); \ return p; +#define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \ + if (cur_thread()->in_symbolizer) \ + return InternalAlloc(size, nullptr, (uptr)align); \ + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ + p = user_memalign(thr, pc, (uptr)align, size); \ + if (!nothrow && UNLIKELY(!p)) DieOnFailure::OnOOM(); \ + } \ + invoke_malloc_hook(p, size); \ + return p; + SANITIZER_INTERFACE_ATTRIBUTE void *operator new(__sanitizer::uptr size); void *operator new(__sanitizer::uptr size) { @@ -62,6 +75,36 @@ void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /*nothrow*/); } +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size, std::align_val_t align); +void *operator new(__sanitizer::uptr size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_t, false /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size, std::align_val_t align); +void *operator new[](__sanitizer::uptr size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_t, false /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&); +void *operator new(__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&) { + OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_tRKSt9nothrow_t, + true /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&); +void *operator new[](__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&) { + OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_tRKSt9nothrow_t, + true /*nothrow*/); +} + #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ if (cur_thread()->in_symbolizer) \ @@ -93,3 +136,57 @@ void operator delete[](void *ptr, std::nothrow_t const&); void operator delete[](void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); } + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT; +void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPvm); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT; +void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPvm); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT; +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT; +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&); +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_tRKSt9nothrow_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const&); +void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_tRKSt9nothrow_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT; +void operator delete(void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPvmSt11align_val_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT; +void operator delete[](void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPvmSt11align_val_t); +} diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h index bea1daba3952..bfac70df12d9 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -42,6 +42,19 @@ C/C++ on linux/x86_64 and freebsd/x86_64 7b00 0000 0000 - 7c00 0000 0000: heap 7c00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack + +C/C++ on netbsd/amd64 can reuse the same mapping: + * The address space starts from 0x1000 (option with 0x0) and ends with + 0x7f7ffffff000. + * LoAppMem-kHeapMemEnd can be reused as it is. + * No VDSO support. + * No MidAppMem region. + * No additional HeapMem region. + * HiAppMem contains the stack, loader, shared libraries and heap. + * Stack on NetBSD/amd64 has prereserved 128MB. + * Heap grows downwards (top-down). + * ASLR must be disabled per-process or globally. + */ struct Mapping { static const uptr kMetaShadowBeg = 0x300000000000ull; @@ -111,7 +124,8 @@ C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) 0c00 0000 00 - 0d00 0000 00: - (4 GB) 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) 0e00 0000 00 - 0f00 0000 00: - (4 GB) -0f00 0000 00 - 1000 0000 00: traces (4 GB) +0f00 0000 00 - 0fc0 0000 00: traces (3 GB) +0fc0 0000 00 - 1000 0000 00: - */ struct Mapping { static const uptr kLoAppMemBeg = 0x0100000000ull; @@ -123,9 +137,9 @@ struct Mapping { static const uptr kMetaShadowBeg = 0x0d00000000ull; static const uptr kMetaShadowEnd = 0x0e00000000ull; static const uptr kTraceMemBeg = 0x0f00000000ull; - static const uptr kTraceMemEnd = 0x1000000000ull; - static const uptr kHiAppMemBeg = 0x1000000000ull; - static const uptr kHiAppMemEnd = 0x1000000000ull; + static const uptr kTraceMemEnd = 0x0fc0000000ull; + static const uptr kHiAppMemBeg = 0x0fc0000000ull; + static const uptr kHiAppMemEnd = 0x0fc0000000ull; static const uptr kAppMemMsk = 0x0ull; static const uptr kAppMemXor = 0x0ull; static const uptr kVdsoBeg = 0x7000000000000000ull; @@ -303,6 +317,38 @@ struct Mapping46 { static const uptr kVdsoBeg = 0x7800000000000000ull; }; +/* +C/C++ on linux/powerpc64 (47-bit VMA) +0000 0000 1000 - 0100 0000 0000: main binary +0100 0000 0000 - 0200 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) +2000 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2200 0000 0000: traces +2200 0000 0000 - 7d00 0000 0000: - +7d00 0000 0000 - 7e00 0000 0000: heap +7e00 0000 0000 - 7e80 0000 0000: - +7e80 0000 0000 - 8000 0000 0000: modules and main thread stack +*/ +struct Mapping47 { + static const uptr kMetaShadowBeg = 0x100000000000ull; + static const uptr kMetaShadowEnd = 0x200000000000ull; + static const uptr kTraceMemBeg = 0x200000000000ull; + static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x7d0000000000ull; + static const uptr kHeapMemEnd = 0x7e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits + static const uptr kAppMemMsk = 0x7c0000000000ull; + static const uptr kAppMemXor = 0x020000000000ull; + static const uptr kVdsoBeg = 0x7800000000000000ull; +}; + // Indicates the runtime will define the memory regions at runtime. #define TSAN_RUNTIME_VMA 1 #endif @@ -429,11 +475,13 @@ uptr MappingArchImpl(void) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return MappingImpl(); - else - return MappingImpl(); + switch (vmaSize) { + case 44: return MappingImpl(); + case 46: return MappingImpl(); + case 47: return MappingImpl(); + } DCHECK(0); + return 0; #else return MappingImpl(); #endif @@ -582,11 +630,13 @@ bool IsAppMem(uptr mem) { DCHECK(0); return false; #elif defined(__powerpc64__) - if (vmaSize == 44) - return IsAppMemImpl(mem); - else - return IsAppMemImpl(mem); + switch (vmaSize) { + case 44: return IsAppMemImpl(mem); + case 46: return IsAppMemImpl(mem); + case 47: return IsAppMemImpl(mem); + } DCHECK(0); + return false; #else return IsAppMemImpl(mem); #endif @@ -609,11 +659,13 @@ bool IsShadowMem(uptr mem) { DCHECK(0); return false; #elif defined(__powerpc64__) - if (vmaSize == 44) - return IsShadowMemImpl(mem); - else - return IsShadowMemImpl(mem); + switch (vmaSize) { + case 44: return IsShadowMemImpl(mem); + case 46: return IsShadowMemImpl(mem); + case 47: return IsShadowMemImpl(mem); + } DCHECK(0); + return false; #else return IsShadowMemImpl(mem); #endif @@ -636,11 +688,13 @@ bool IsMetaMem(uptr mem) { DCHECK(0); return false; #elif defined(__powerpc64__) - if (vmaSize == 44) - return IsMetaMemImpl(mem); - else - return IsMetaMemImpl(mem); + switch (vmaSize) { + case 44: return IsMetaMemImpl(mem); + case 46: return IsMetaMemImpl(mem); + case 47: return IsMetaMemImpl(mem); + } DCHECK(0); + return false; #else return IsMetaMemImpl(mem); #endif @@ -673,11 +727,13 @@ uptr MemToShadow(uptr x) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return MemToShadowImpl(x); - else - return MemToShadowImpl(x); + switch (vmaSize) { + case 44: return MemToShadowImpl(x); + case 46: return MemToShadowImpl(x); + case 47: return MemToShadowImpl(x); + } DCHECK(0); + return 0; #else return MemToShadowImpl(x); #endif @@ -712,11 +768,13 @@ u32 *MemToMeta(uptr x) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return MemToMetaImpl(x); - else - return MemToMetaImpl(x); + switch (vmaSize) { + case 44: return MemToMetaImpl(x); + case 46: return MemToMetaImpl(x); + case 47: return MemToMetaImpl(x); + } DCHECK(0); + return 0; #else return MemToMetaImpl(x); #endif @@ -764,11 +822,13 @@ uptr ShadowToMem(uptr s) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return ShadowToMemImpl(s); - else - return ShadowToMemImpl(s); + switch (vmaSize) { + case 44: return ShadowToMemImpl(s); + case 46: return ShadowToMemImpl(s); + case 47: return ShadowToMemImpl(s); + } DCHECK(0); + return 0; #else return ShadowToMemImpl(s); #endif @@ -799,11 +859,13 @@ uptr GetThreadTrace(int tid) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return GetThreadTraceImpl(tid); - else - return GetThreadTraceImpl(tid); + switch (vmaSize) { + case 44: return GetThreadTraceImpl(tid); + case 46: return GetThreadTraceImpl(tid); + case 47: return GetThreadTraceImpl(tid); + } DCHECK(0); + return 0; #else return GetThreadTraceImpl(tid); #endif @@ -829,11 +891,13 @@ uptr GetThreadTraceHeader(int tid) { DCHECK(0); return 0; #elif defined(__powerpc64__) - if (vmaSize == 44) - return GetThreadTraceHeaderImpl(tid); - else - return GetThreadTraceHeaderImpl(tid); + switch (vmaSize) { + case 44: return GetThreadTraceHeaderImpl(tid); + case 46: return GetThreadTraceHeaderImpl(tid); + case 47: return GetThreadTraceHeaderImpl(tid); + } DCHECK(0); + return 0; #else return GetThreadTraceHeaderImpl(tid); #endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc index ead1e5704989..e14d5f575a2e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc @@ -14,11 +14,12 @@ #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_procmaps.h" @@ -216,9 +217,9 @@ void InitializePlatformEarly() { Die(); } #elif defined(__powerpc64__) - if (vmaSize != 44 && vmaSize != 46) { + if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); - Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize); + Printf("FATAL: Found %d - Supported 44, 46, and 47\n", vmaSize); Die(); } #endif @@ -401,4 +402,4 @@ void cur_thread_finalize() { } // namespace __tsan -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc index 73a656ffca5e..f8d7324b763c 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc @@ -167,8 +167,8 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { #else // !SANITIZER_GO "app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" #endif - "stacks: %ld unique IDs, %ld kB allocated\n" - "threads: %ld total, %ld live\n" + "stacks: %zd unique IDs, %zd kB allocated\n" + "threads: %zd total, %zd live\n" "------------------------------\n", ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024, MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024, @@ -231,7 +231,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, void InitializePlatformEarly() { #if defined(__aarch64__) - uptr max_vm = GetMaxVirtualAddress() + 1; + uptr max_vm = GetMaxUserVirtualAddress() + 1; if (max_vm != Mapping::kHiAppMemEnd) { Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n", max_vm, Mapping::kHiAppMemEnd); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc index 32cc3325fe40..af47076968d3 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc @@ -13,6 +13,7 @@ #include "tsan_report.h" #include "tsan_platform.h" #include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stacktrace_printer.h" @@ -38,34 +39,27 @@ ReportLocation *ReportLocation::New(ReportLocationType type) { class Decorator: public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() { } - const char *Warning() { return Red(); } - const char *EndWarning() { return Default(); } const char *Access() { return Blue(); } - const char *EndAccess() { return Default(); } const char *ThreadDescription() { return Cyan(); } - const char *EndThreadDescription() { return Default(); } const char *Location() { return Green(); } - const char *EndLocation() { return Default(); } const char *Sleep() { return Yellow(); } - const char *EndSleep() { return Default(); } const char *Mutex() { return Magenta(); } - const char *EndMutex() { return Default(); } }; ReportDesc::ReportDesc() : tag(kExternalTagNone) - , stacks(MBlockReportStack) - , mops(MBlockReportMop) - , locs(MBlockReportLoc) - , mutexes(MBlockReportMutex) - , threads(MBlockReportThread) - , unique_tids(MBlockReportThread) + , stacks() + , mops() + , locs() + , mutexes() + , threads() + , unique_tids() , sleep() , count() { } ReportMop::ReportMop() - : mset(MBlockReportMutex) { + : mset() { } ReportDesc::~ReportDesc() { @@ -180,7 +174,7 @@ static void PrintMop(const ReportMop *mop, bool first) { } PrintMutexSet(mop->mset); Printf(":\n"); - Printf("%s", d.EndAccess()); + Printf("%s", d.Default()); PrintStack(mop->stack); } @@ -221,20 +215,20 @@ static void PrintLocation(const ReportLocation *loc) { loc->fd, thread_name(thrbuf, loc->tid)); print_stack = true; } - Printf("%s", d.EndLocation()); + Printf("%s", d.Default()); if (print_stack) PrintStack(loc->stack); } static void PrintMutexShort(const ReportMutex *rm, const char *after) { Decorator d; - Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.EndMutex(), after); + Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after); } static void PrintMutexShortWithAddress(const ReportMutex *rm, const char *after) { Decorator d; - Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.EndMutex(), after); + Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after); } static void PrintMutex(const ReportMutex *rm) { @@ -242,11 +236,11 @@ static void PrintMutex(const ReportMutex *rm) { if (rm->destroyed) { Printf("%s", d.Mutex()); Printf(" Mutex M%llu is already destroyed.\n\n", rm->id); - Printf("%s", d.EndMutex()); + Printf("%s", d.Default()); } else { Printf("%s", d.Mutex()); Printf(" Mutex M%llu (%p) created at:\n", rm->id, rm->addr); - Printf("%s", d.EndMutex()); + Printf("%s", d.Default()); PrintStack(rm->stack); } } @@ -264,7 +258,7 @@ static void PrintThread(const ReportThread *rt) { if (rt->workerthread) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); - Printf("%s", d.EndThreadDescription()); + Printf("%s", d.Default()); return; } Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status, @@ -272,7 +266,7 @@ static void PrintThread(const ReportThread *rt) { if (rt->stack) Printf(" at:"); Printf("\n"); - Printf("%s", d.EndThreadDescription()); + Printf("%s", d.Default()); PrintStack(rt->stack); } @@ -280,7 +274,7 @@ static void PrintSleep(const ReportStack *s) { Decorator d; Printf("%s", d.Sleep()); Printf(" As if synchronized via sleep:\n"); - Printf("%s", d.EndSleep()); + Printf("%s", d.Default()); PrintStack(s); } @@ -324,7 +318,7 @@ void PrintReport(const ReportDesc *rep) { Printf("%s", d.Warning()); Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str, (int)internal_getpid()); - Printf("%s", d.EndWarning()); + Printf("%s", d.Default()); if (rep->typ == ReportTypeDeadlock) { char thrbuf[kThreadBufSize]; @@ -342,7 +336,7 @@ void PrintReport(const ReportDesc *rep) { PrintMutexShort(rep->mutexes[i], " in "); Printf("%s", d.ThreadDescription()); Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i])); - Printf("%s", d.EndThreadDescription()); + Printf("%s", d.Default()); if (flags()->second_deadlock_stack) { PrintStack(rep->stacks[2*i]); Printf(" Mutex "); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h index bc1582f90fe4..cdc999c6af1d 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h @@ -14,8 +14,8 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" -#include "tsan_vector.h" namespace __tsan { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index a01525302b02..17b820977c26 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -14,6 +14,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_placement_new.h" @@ -101,8 +102,8 @@ Context::Context() , thread_registry(new(thread_registry_placeholder) ThreadRegistry( CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)) , racy_mtx(MutexTypeRacy, StatMtxRacy) - , racy_stacks(MBlockRacyStacks) - , racy_addresses(MBlockRacyAddresses) + , racy_stacks() + , racy_addresses() , fired_suppressions_mtx(MutexTypeFired, StatMtxFired) , fired_suppressions(8) , clock_alloc("clock allocator") { @@ -120,7 +121,7 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, // , ignore_interceptors() , clock(tid, reuse_count) #if !SANITIZER_GO - , jmp_bufs(MBlockJmpBuf) + , jmp_bufs() #endif , tid(tid) , unique_id(unique_id) @@ -213,7 +214,7 @@ static void BackgroundThread(void *arg) { memory_order_relaxed); if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { Lock l(&ctx->report_mtx); - SpinMutexLock l2(&CommonSanitizerReportMutex); + ScopedErrorReportLock l2; SymbolizeFlush(); atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); } @@ -323,6 +324,21 @@ static void CheckShadowMapping() { } } +#if !SANITIZER_GO +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + uptr top = 0; + uptr bottom = 0; + bool fast = common_flags()->fast_unwind_on_fatal; + if (fast) GetThreadStackTopAndBottom(false, &top, &bottom); + stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast); +} + +static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} +#endif + void Initialize(ThreadState *thr) { // Thread safe because done before all threads exist. static bool is_initialized = false; @@ -360,6 +376,7 @@ void Initialize(ThreadState *thr) { #if !SANITIZER_GO InitializeShadowMemory(); InitializeAllocatorLate(); + InstallDeadlySignalHandlers(TsanOnDeadlySignal); #endif // Setup correct file descriptor for error reports. __sanitizer_set_report_path(common_flags()->log_path); @@ -367,13 +384,6 @@ void Initialize(ThreadState *thr) { #if !SANITIZER_GO InitializeLibIgnore(); Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); - // On MIPS, TSan initialization is run before - // __pthread_initialize_minimal_internal() is finished, so we can not spawn - // new threads. -#ifndef __mips__ - StartBackgroundThread(); - SetSandboxingCallback(StopBackgroundThread); -#endif #endif VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", @@ -402,6 +412,21 @@ void Initialize(ThreadState *thr) { OnInitialize(); } +void MaybeSpawnBackgroundThread() { + // On MIPS, TSan initialization is run before + // __pthread_initialize_minimal_internal() is finished, so we can not spawn + // new threads. +#if !SANITIZER_GO && !defined(__mips__) + static atomic_uint32_t bg_thread = {}; + if (atomic_load(&bg_thread, memory_order_relaxed) == 0 && + atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) { + StartBackgroundThread(); + SetSandboxingCallback(StopBackgroundThread); + } +#endif +} + + int Finalize(ThreadState *thr) { bool failed = false; @@ -412,8 +437,7 @@ int Finalize(ThreadState *thr) { // Wait for pending reports. ctx->report_mtx.Lock(); - CommonSanitizerReportMutex.Lock(); - CommonSanitizerReportMutex.Unlock(); + { ScopedErrorReportLock l; } ctx->report_mtx.Unlock(); #if !SANITIZER_GO diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h index 2cf2e168454d..7d44ccae9893 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -34,12 +34,13 @@ #include "sanitizer_common/sanitizer_libignore.h" #include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_thread_registry.h" +#include "sanitizer_common/sanitizer_vector.h" #include "tsan_clock.h" #include "tsan_defs.h" #include "tsan_flags.h" +#include "tsan_mman.h" #include "tsan_sync.h" #include "tsan_trace.h" -#include "tsan_vector.h" #include "tsan_report.h" #include "tsan_platform.h" #include "tsan_mutexset.h" @@ -574,11 +575,8 @@ const char *GetObjectTypeFromTag(uptr tag); const char *GetReportHeaderFromTag(uptr tag); uptr TagFromShadowStackFrame(uptr pc); -class ScopedReport { +class ScopedReportBase { public: - explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); - ~ScopedReport(); - void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack, const MutexSet *mset); void AddStack(StackTrace stack, bool suppressable = false); @@ -593,6 +591,10 @@ class ScopedReport { const ReportDesc *GetReport() const; + protected: + ScopedReportBase(ReportType typ, uptr tag); + ~ScopedReportBase(); + private: ReportDesc *rep_; // Symbolizer makes lots of intercepted calls. If we try to process them, @@ -601,8 +603,17 @@ class ScopedReport { void AddDeadMutex(u64 id); - ScopedReport(const ScopedReport&); - void operator = (const ScopedReport&); + ScopedReportBase(const ScopedReportBase &) = delete; + void operator=(const ScopedReportBase &) = delete; +}; + +class ScopedReport : public ScopedReportBase { + public: + explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); + ~ScopedReport(); + + private: + ScopedErrorReportLock lock_; }; ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack); @@ -690,6 +701,7 @@ void PrintCurrentStack(ThreadState *thr, uptr pc); void PrintCurrentStackSlow(uptr pc); // uses libunwind void Initialize(ThreadState *thr); +void MaybeSpawnBackgroundThread(); int Finalize(ThreadState *thr); void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); @@ -825,7 +837,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, return; DCHECK_GE((int)typ, 0); DCHECK_LE((int)typ, 7); - DCHECK_EQ(GetLsb(addr, 61), addr); + DCHECK_EQ(GetLsb(addr, kEventPCBits), addr); StatInc(thr, StatEvents); u64 pos = fs.GetTracePos(); if (UNLIKELY((pos % kTracePartSize) == 0)) { @@ -837,7 +849,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, } Event *trace = (Event*)GetThreadTrace(fs.tid()); Event *evp = &trace[pos]; - Event ev = (u64)addr | ((u64)typ << 61); + Event ev = (u64)addr | ((u64)typ << kEventPCBits); *evp = ev; } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S index f8c6453a5da4..f1e4dbb50fbb 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -6,7 +6,7 @@ #if !defined(__APPLE__) .section .bss .type __tsan_pointer_chk_guard, %object -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) __tsan_pointer_chk_guard: .zero 8 #endif @@ -51,7 +51,7 @@ _sigsetjmp$non_lazy_ptr: // original ones. ASM_HIDDEN(_Z18InitializeGuardPtrv) .global _Z18InitializeGuardPtrv -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) _Z18InitializeGuardPtrv: CFI_STARTPROC // Allocates a jmp_buf for the setjmp call. @@ -88,14 +88,14 @@ _Z18InitializeGuardPtrv: CFI_DEF_CFA (31, 0) ret CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) #endif ASM_HIDDEN(__tsan_setjmp) .comm _ZN14__interception11real_setjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC // save env parameters for function call @@ -125,7 +125,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) // restore env parameter mov x0, x19 @@ -148,12 +148,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): br x1 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) .comm _ZN14__interception12real__setjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameters for function call @@ -183,7 +183,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) // Restore jmp_buf parameter mov x0, x19 @@ -206,12 +206,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): br x1 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -243,7 +243,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) // restore env parameter mov w1, w20 @@ -268,13 +268,13 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): #endif br x2 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) #if !defined(__APPLE__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -303,7 +303,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): #endif // call tsan interceptor - bl ASM_TSAN_SYMBOL(__tsan_setjmp) + bl ASM_SYMBOL(__tsan_setjmp) mov w1, w20 mov x0, x19 @@ -321,12 +321,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE] ldr x2, [x2] #else - adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page - add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff + adrp x2, ASM_SYMBOL(__sigsetjmp)@page + add x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff #endif br x2 CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) #endif #if defined(__FreeBSD__) || defined(__linux__) diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S index 98947fd2a1ba..8af61bf0e892 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S @@ -10,8 +10,8 @@ #endif ASM_HIDDEN(__tsan_trace_switch) -.globl ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk) -ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): +.globl ASM_SYMBOL(__tsan_trace_switch_thunk) +ASM_SYMBOL(__tsan_trace_switch_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -50,7 +50,7 @@ ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call ASM_TSAN_SYMBOL(__tsan_trace_switch) + call ASM_SYMBOL(__tsan_trace_switch) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -90,8 +90,8 @@ ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): CFI_ENDPROC ASM_HIDDEN(__tsan_report_race) -.globl ASM_TSAN_SYMBOL(__tsan_report_race_thunk) -ASM_TSAN_SYMBOL(__tsan_report_race_thunk): +.globl ASM_SYMBOL(__tsan_report_race_thunk) +ASM_SYMBOL(__tsan_report_race_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -130,7 +130,7 @@ ASM_TSAN_SYMBOL(__tsan_report_race_thunk): shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call ASM_TSAN_SYMBOL(__tsan_report_race) + call ASM_SYMBOL(__tsan_report_race) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -170,19 +170,27 @@ ASM_TSAN_SYMBOL(__tsan_report_race_thunk): CFI_ENDPROC ASM_HIDDEN(__tsan_setjmp) -#if !defined(__APPLE__) +#if defined(__NetBSD__) +.comm _ZN14__interception15real___setjmp14E,8,8 +#elif !defined(__APPLE__) .comm _ZN14__interception11real_setjmpE,8,8 #endif -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): +#if defined(__NetBSD__) +.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) +ASM_SYMBOL_INTERCEPTOR(__setjmp14): +#else +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): +#endif CFI_STARTPROC // save env parameter push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) // obtain %rsp -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi #elif defined(__APPLE__) @@ -197,33 +205,40 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): # error "Unknown platform" #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) CFI_RESTORE(%rdi) // tail jump to libc setjmp movl $0, %eax -#if !defined(__APPLE__) +#if defined(__NetBSD__) + movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#elif !defined(__APPLE__) movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) #else - jmp ASM_TSAN_SYMBOL(setjmp) + jmp ASM_SYMBOL(setjmp) #endif CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +#if defined(__NetBSD__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) +#else +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) +#endif .comm _ZN14__interception12real__setjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameter push %rdi CFI_ADJUST_CFA_OFFSET(8) CFI_REL_OFFSET(%rdi, 0) // obtain %rsp -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi #elif defined(__APPLE__) @@ -238,7 +253,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): # error "Unknown platform" #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) @@ -249,15 +264,22 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) #else - jmp ASM_TSAN_SYMBOL(_setjmp) + jmp ASM_SYMBOL(_setjmp) #endif CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +#if defined(__NetBSD__) +.comm _ZN14__interception18real___sigsetjmp14E,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14): +#else .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): +#endif CFI_STARTPROC // save env parameter push %rdi @@ -271,7 +293,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): sub $8, %rsp CFI_ADJUST_CFA_OFFSET(8) // obtain %rsp -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) lea 24(%rsp), %rdi mov %rdi, %rsi #elif defined(__APPLE__) @@ -286,7 +308,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): # error "Unknown platform" #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -300,20 +322,27 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_RESTORE(%rdi) // tail jump to libc sigsetjmp movl $0, %eax -#if !defined(__APPLE__) +#if defined(__NetBSD__) + movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#elif !defined(__APPLE__) movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) #else - jmp ASM_TSAN_SYMBOL(sigsetjmp) + jmp ASM_SYMBOL(sigsetjmp) #endif CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +#if defined(__NetBSD__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) +#else +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +#endif -#if !defined(__APPLE__) +#if !defined(__APPLE__) && !defined(__NetBSD__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) -ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) -ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameter push %rdi @@ -337,7 +366,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): rol $0x11, %rsi #endif // call tsan interceptor - call ASM_TSAN_SYMBOL(__tsan_setjmp) + call ASM_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -354,11 +383,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) CFI_ENDPROC -ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) -#endif // !defined(__APPLE__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif // !defined(__APPLE__) && !defined(__NetBSD__) #if defined(__FreeBSD__) || defined(__linux__) /* We do not need executable stack. */ +/* This note is not needed on NetBSD. */ .section .note.GNU-stack,"",@progbits #endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc index 2f85811620f1..152b965ad535 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -84,7 +84,9 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true); if (s == 0) return; - if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit)) { + if ((flagz & MutexFlagLinkerInit) + || s->IsFlagSet(MutexFlagLinkerInit) + || ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) { // Destroy is no-op for linker-initialized mutexes. s->mtx.Unlock(); return; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc index 85a982941ed7..cc582ab50190 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc @@ -143,30 +143,28 @@ static ReportStack *SymbolizeStack(StackTrace trace) { return stack; } -ScopedReport::ScopedReport(ReportType typ, uptr tag) { +ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) { ctx->thread_registry->CheckLocked(); void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc)); rep_ = new(mem) ReportDesc; rep_->typ = typ; rep_->tag = tag; ctx->report_mtx.Lock(); - CommonSanitizerReportMutex.Lock(); } -ScopedReport::~ScopedReport() { - CommonSanitizerReportMutex.Unlock(); +ScopedReportBase::~ScopedReportBase() { ctx->report_mtx.Unlock(); DestroyAndFree(rep_); } -void ScopedReport::AddStack(StackTrace stack, bool suppressable) { +void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) { ReportStack **rs = rep_->stacks.PushBack(); *rs = SymbolizeStack(stack); (*rs)->suppressable = suppressable; } -void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, - StackTrace stack, const MutexSet *mset) { +void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, + StackTrace stack, const MutexSet *mset) { void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop)); ReportMop *mop = new(mem) ReportMop; rep_->mops.PushBack(mop); @@ -187,11 +185,11 @@ void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, } } -void ScopedReport::AddUniqueTid(int unique_tid) { +void ScopedReportBase::AddUniqueTid(int unique_tid) { rep_->unique_tids.PushBack(unique_tid); } -void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) { +void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { for (uptr i = 0; i < rep_->threads.Size(); i++) { if ((u32)rep_->threads[i]->id == tctx->tid) return; @@ -255,14 +253,14 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { } #endif -void ScopedReport::AddThread(int unique_tid, bool suppressable) { +void ScopedReportBase::AddThread(int unique_tid, bool suppressable) { #if !SANITIZER_GO if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid)) AddThread(tctx, suppressable); #endif } -void ScopedReport::AddMutex(const SyncVar *s) { +void ScopedReportBase::AddMutex(const SyncVar *s) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { if (rep_->mutexes[i]->id == s->uid) return; @@ -276,7 +274,7 @@ void ScopedReport::AddMutex(const SyncVar *s) { rm->stack = SymbolizeStackId(s->creation_stack_id); } -u64 ScopedReport::AddMutex(u64 id) { +u64 ScopedReportBase::AddMutex(u64 id) { u64 uid = 0; u64 mid = id; uptr addr = SyncVar::SplitId(id, &uid); @@ -295,7 +293,7 @@ u64 ScopedReport::AddMutex(u64 id) { return mid; } -void ScopedReport::AddDeadMutex(u64 id) { +void ScopedReportBase::AddDeadMutex(u64 id) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { if (rep_->mutexes[i]->id == id) return; @@ -309,7 +307,7 @@ void ScopedReport::AddDeadMutex(u64 id) { rm->stack = 0; } -void ScopedReport::AddLocation(uptr addr, uptr size) { +void ScopedReportBase::AddLocation(uptr addr, uptr size) { if (addr == 0) return; #if !SANITIZER_GO @@ -364,18 +362,19 @@ void ScopedReport::AddLocation(uptr addr, uptr size) { } #if !SANITIZER_GO -void ScopedReport::AddSleep(u32 stack_id) { +void ScopedReportBase::AddSleep(u32 stack_id) { rep_->sleep = SymbolizeStackId(stack_id); } #endif -void ScopedReport::SetCount(int count) { - rep_->count = count; -} +void ScopedReportBase::SetCount(int count) { rep_->count = count; } -const ReportDesc *ScopedReport::GetReport() const { - return rep_; -} +const ReportDesc *ScopedReportBase::GetReport() const { return rep_; } + +ScopedReport::ScopedReport(ReportType typ, uptr tag) + : ScopedReportBase(typ, tag) {} + +ScopedReport::~ScopedReport() {} void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, MutexSet *mset, uptr *tag) { @@ -394,7 +393,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, const u64 ebegin = RoundDown(eend, kTracePartSize); DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n", tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx); - Vector stack(MBlockReportStack); + Vector stack; stack.Resize(hdr->stack0.size + 64); for (uptr i = 0; i < hdr->stack0.size; i++) { stack[i] = hdr->stack0.trace[i]; @@ -406,8 +405,8 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, Event *events = (Event*)GetThreadTrace(tid); for (uptr i = ebegin; i <= eend; i++) { Event ev = events[i]; - EventType typ = (EventType)(ev >> 61); - uptr pc = (uptr)(ev & ((1ull << 61) - 1)); + EventType typ = (EventType)(ev >> kEventPCBits); + uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1)); DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc); if (typ == EventTypeMop) { stack[pos] = pc; @@ -634,13 +633,33 @@ void ReportRace(ThreadState *thr) { const uptr kMop = 2; VarSizeStackTrace traces[kMop]; uptr tags[kMop] = {kExternalTagNone}; - const uptr toppc = TraceTopPC(thr); + uptr toppc = TraceTopPC(thr); + if (toppc >> kEventPCBits) { + // This is a work-around for a known issue. + // The scenario where this happens is rather elaborate and requires + // an instrumented __sanitizer_report_error_summary callback and + // a __tsan_symbolize_external callback and a race during a range memory + // access larger than 8 bytes. MemoryAccessRange adds the current PC to + // the trace and starts processing memory accesses. A first memory access + // triggers a race, we report it and call the instrumented + // __sanitizer_report_error_summary, which adds more stuff to the trace + // since it is intrumented. Then a second memory access in MemoryAccessRange + // also triggers a race and we get here and call TraceTopPC to get the + // current PC, however now it contains some unrelated events from the + // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit + // event. Later we subtract -1 from it (in GetPreviousInstructionPc) + // and the resulting PC has kExternalPCBit set, so we pass it to + // __tsan_symbolize_external. __tsan_symbolize_external is within its rights + // to crash since the PC is completely bogus. + // test/tsan/double_race.cc contains a test case for this. + toppc = 0; + } ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]); if (IsFiredSuppression(ctx, typ, traces[0])) return; // MutexSet is too large to live on stack. - Vector mset_buffer(MBlockScopedBuf); + Vector mset_buffer; mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1); MutexSet *mset2 = new(&mset_buffer[0]) MutexSet(); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc index 83fab082afe3..e4d65b9a909b 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc @@ -211,7 +211,7 @@ void ThreadFinalize(ThreadState *thr) { if (!flags()->report_thread_leaks) return; ThreadRegistryLock l(ctx->thread_registry); - Vector leaks(MBlockScopedBuf); + Vector leaks; ctx->thread_registry->RunCallbackForEachThreadLocked( MaybeReportThreadLeak, &leaks); for (uptr i = 0; i < leaks.Size(); i++) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h index b83c09ff784e..9039970bcf86 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h @@ -34,6 +34,7 @@ enum MutexFlags { MutexFlagTryLockFailed = 1 << 5, // __tsan_mutex_try_lock_failed MutexFlagRecursiveLock = 1 << 6, // __tsan_mutex_recursive_lock MutexFlagRecursiveUnlock = 1 << 7, // __tsan_mutex_recursive_unlock + MutexFlagNotStatic = 1 << 8, // __tsan_mutex_not_static // The following flags are runtime private. // Mutex API misuse was detected, so don't report any more. @@ -43,7 +44,8 @@ enum MutexFlags { // Must list all mutex creation flags. MutexCreationFlagMask = MutexFlagLinkerInit | MutexFlagWriteReentrant | - MutexFlagReadReentrant, + MutexFlagReadReentrant | + MutexFlagNotStatic, }; struct SyncVar { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h index 96a18ac4101a..9aef375cbfe6 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h @@ -41,6 +41,8 @@ enum EventType { // u64 addr : 61; // Associated pc. typedef u64 Event; +const uptr kEventPCBits = 61; + struct TraceHeader { #if !SANITIZER_GO BufferedStackTrace stack0; // Start stack for the trace. diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc index 0a87e6e8e3a4..73c69363215a 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc @@ -29,6 +29,7 @@ UBSAN_CHECK(UnsignedIntegerOverflow, "unsigned-integer-overflow", UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero", "integer-divide-by-zero") UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero") +UBSAN_CHECK(InvalidBuiltin, "invalid-builtin-use", "invalid-builtin-use") UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base") UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent") UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds") diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc index 742802b8f341..d5b68407bb37 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc @@ -26,21 +26,24 @@ using namespace __ubsan; +void __ubsan::GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, + uptr bp, void *context, bool fast) { + uptr top = 0; + uptr bottom = 0; + if (fast) + GetThreadStackTopAndBottom(false, &top, &bottom); + stack->Unwind(max_depth, pc, bp, context, top, bottom, fast); +} + static void MaybePrintStackTrace(uptr pc, uptr bp) { // We assume that flags are already parsed, as UBSan runtime // will definitely be called when we print the first diagnostics message. if (!flags()->print_stacktrace) return; - uptr top = 0; - uptr bottom = 0; - bool request_fast_unwind = common_flags()->fast_unwind_on_fatal; - if (request_fast_unwind) - __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom); - BufferedStackTrace stack; - stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, - request_fast_unwind); + GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, + common_flags()->fast_unwind_on_fatal); stack.Print(); } @@ -97,9 +100,7 @@ class Decorator : public SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() {} const char *Highlight() const { return Green(); } - const char *EndHighlight() const { return Default(); } const char *Note() const { return Black(); } - const char *EndNote() const { return Default(); } }; } @@ -295,7 +296,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, Buffer.append("%c", P == Loc ? '^' : Byte); Buffer.append("%c", Byte); } - Buffer.append("%s\n", Decor.EndHighlight()); + Buffer.append("%s\n", Decor.Default()); // Go over the line again, and print names for the ranges. InRange = 0; @@ -335,7 +336,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, Diag::~Diag() { // All diagnostics should be printed under report mutex. - CommonSanitizerReportMutex.CheckLocked(); + ScopedReport::CheckLocked(); Decorator Decor; InternalScopedString Buffer(1024); @@ -345,12 +346,12 @@ Diag::~Diag() { switch (Level) { case DL_Error: - Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.EndWarning(), + Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.Default(), Decor.Bold()); break; case DL_Note: - Buffer.append("%s note: %s", Decor.Note(), Decor.EndNote()); + Buffer.append("%s note: %s", Decor.Note(), Decor.Default()); break; } @@ -363,17 +364,15 @@ Diag::~Diag() { PrintMemorySnippet(Decor, Loc.getMemoryLocation(), Ranges, NumRanges, Args); } +ScopedReport::Initializer::Initializer() { InitAsStandaloneIfNecessary(); } + ScopedReport::ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type) - : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) { - InitAsStandaloneIfNecessary(); - CommonSanitizerReportMutex.Lock(); -} + : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) {} ScopedReport::~ScopedReport() { MaybePrintStackTrace(Opts.pc, Opts.bp); MaybeReportErrorSummary(SummaryLoc, Type); - CommonSanitizerReportMutex.Unlock(); if (flags()->halt_on_error) Die(); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h index 3edb67a03c1f..7370b1b62421 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h @@ -231,10 +231,19 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET); GET_CALLER_PC_BP; \ ReportOptions Opts = {unrecoverable_handler, pc, bp} +void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp, + void *context, bool fast); + /// \brief Instantiate this class before printing diagnostics in the error /// report. This class ensures that reports from different threads and from /// different sanitizers won't be mixed. class ScopedReport { + struct Initializer { + Initializer(); + }; + Initializer initializer_; + ScopedErrorReportLock report_lock_; + ReportOptions Opts; Location SummaryLoc; ErrorType Type; @@ -242,6 +251,8 @@ class ScopedReport { public: ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type); ~ScopedReport(); + + static void CheckLocked() { ScopedErrorReportLock::CheckLocked(); } }; void InitializeSuppressions(); diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc index df8ed5fcdf6d..1f4a5bd4062b 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc @@ -26,9 +26,10 @@ void __sanitizer_print_stack_trace() { if (request_fast_unwind) __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom); - GET_REPORT_OPTIONS(false); + GET_CURRENT_PC_BP_SP; + (void)sp; BufferedStackTrace stack; - stack.Unwind(kStackTraceMax, Opts.pc, Opts.bp, nullptr, top, bottom, + stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, request_fast_unwind); stack.Print(); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc index 8e1f40885a58..c4fb278c8367 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc @@ -18,6 +18,8 @@ #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" +#include + namespace __ubsan { const char *MaybeCallUbsanDefaultOptions() { @@ -45,7 +47,7 @@ void InitializeFlags() { CommonFlags cf; cf.CopyFrom(*common_flags()); cf.print_summary = false; - cf.external_symbolizer_path = GetEnv("UBSAN_SYMBOLIZER_PATH"); + cf.external_symbolizer_path = getenv("UBSAN_SYMBOLIZER_PATH"); OverrideCommonFlags(cf); } @@ -59,7 +61,7 @@ void InitializeFlags() { // Override from user-specified string. parser.ParseString(MaybeCallUbsanDefaultOptions()); // Override from environment variable. - parser.ParseString(GetEnv("UBSAN_OPTIONS")); + parser.ParseString(getenv("UBSAN_OPTIONS")); InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc index 75a4490a1843..bcda746a1783 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc @@ -38,7 +38,8 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET) { const char *TypeCheckKinds[] = { "load of", "store to", "reference binding to", "member access within", "member call on", "constructor call on", "downcast of", "downcast of", - "upcast of", "cast to virtual base of", "_Nonnull binding to"}; + "upcast of", "cast to virtual base of", "_Nonnull binding to", + "dynamic operation on"}; } static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer, @@ -297,7 +298,7 @@ void __ubsan::__ubsan_handle_out_of_bounds_abort(OutOfBoundsData *Data, static void handleBuiltinUnreachableImpl(UnreachableData *Data, ReportOptions Opts) { ScopedReport R(Opts, Data->Loc, ErrorType::UnreachableCall); - Diag(Data->Loc, DL_Error, "execution reached a __builtin_unreachable() call"); + Diag(Data->Loc, DL_Error, "execution reached an unreachable program point"); } void __ubsan::__ubsan_handle_builtin_unreachable(UnreachableData *Data) { @@ -437,6 +438,30 @@ void __ubsan::__ubsan_handle_load_invalid_value_abort(InvalidValueData *Data, Die(); } +static void handleInvalidBuiltin(InvalidBuiltinData *Data, ReportOptions Opts) { + SourceLocation Loc = Data->Loc.acquire(); + ErrorType ET = ErrorType::InvalidBuiltin; + + if (ignoreReport(Loc, Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + Diag(Loc, DL_Error, + "passing zero to %0, which is not a valid argument") + << ((Data->Kind == BCK_CTZPassedZero) ? "ctz()" : "clz()"); +} + +void __ubsan::__ubsan_handle_invalid_builtin(InvalidBuiltinData *Data) { + GET_REPORT_OPTIONS(true); + handleInvalidBuiltin(Data, Opts); +} +void __ubsan::__ubsan_handle_invalid_builtin_abort(InvalidBuiltinData *Data) { + GET_REPORT_OPTIONS(true); + handleInvalidBuiltin(Data, Opts); + Die(); +} + static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, ValueHandle Function, ReportOptions Opts) { @@ -628,16 +653,32 @@ static void handleCFIBadIcall(CFICheckFailData *Data, ValueHandle Function, } namespace __ubsan { + #ifdef UBSAN_CAN_USE_CXXABI -SANITIZER_WEAK_ATTRIBUTE -void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable, - bool ValidVtable, ReportOptions Opts); + +#ifdef _WIN32 + +extern "C" void __ubsan_handle_cfi_bad_type_default(CFICheckFailData *Data, + ValueHandle Vtable, + bool ValidVtable, + ReportOptions Opts) { + Die(); +} + +WIN_WEAK_ALIAS(__ubsan_handle_cfi_bad_type, __ubsan_handle_cfi_bad_type_default) #else -static void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable, - bool ValidVtable, ReportOptions Opts) { +SANITIZER_WEAK_ATTRIBUTE +#endif +void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, + bool ValidVtable, ReportOptions Opts); + +#else +void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, + bool ValidVtable, ReportOptions Opts) { Die(); } #endif + } // namespace __ubsan void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, @@ -647,7 +688,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, if (Data->CheckKind == CFITCK_ICall) handleCFIBadIcall(Data, Value, Opts); else - HandleCFIBadType(Data, Value, ValidVtable, Opts); + __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts); } void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data, @@ -657,7 +698,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data, if (Data->CheckKind == CFITCK_ICall) handleCFIBadIcall(Data, Value, Opts); else - HandleCFIBadType(Data, Value, ValidVtable, Opts); + __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts); Die(); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h index 796321b81889..311776b9f22c 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h @@ -122,6 +122,21 @@ struct InvalidValueData { /// \brief Handle a load of an invalid value for the type. RECOVERABLE(load_invalid_value, InvalidValueData *Data, ValueHandle Val) +/// Known builtin check kinds. +/// Keep in sync with the enum of the same name in CodeGenFunction.h +enum BuiltinCheckKind : unsigned char { + BCK_CTZPassedZero, + BCK_CLZPassedZero, +}; + +struct InvalidBuiltinData { + SourceLocation Loc; + unsigned char Kind; +}; + +/// Handle a builtin called in an invalid way. +RECOVERABLE(invalid_builtin, InvalidBuiltinData *Data) + struct FunctionTypeMismatchData { SourceLocation Loc; const TypeDescriptor &Type; @@ -177,6 +192,13 @@ struct CFICheckFailData { /// \brief Handle control flow integrity failures. RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function, uptr VtableIsValid) + +struct ReportOptions; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type( + CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable, + ReportOptions Opts); + } #endif // UBSAN_HANDLERS_H diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc index d97ec4813ccd..e15abc64ecca 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc @@ -95,8 +95,8 @@ void __ubsan::__ubsan_handle_dynamic_type_cache_miss_abort( } namespace __ubsan { -void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable, - bool ValidVtable, ReportOptions Opts) { +void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, + bool ValidVtable, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); ErrorType ET = ErrorType::CFIBadType; diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc index 307bca37680e..32fc434ad2bc 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc @@ -27,11 +27,7 @@ const char *__ubsan::GetSanititizerToolName() { return "UndefinedBehaviorSanitizer"; } -static enum { - UBSAN_MODE_UNKNOWN = 0, - UBSAN_MODE_STANDALONE, - UBSAN_MODE_PLUGIN -} ubsan_mode; +static bool ubsan_initialized; static StaticSpinMutex ubsan_init_mu; static void CommonInit() { @@ -40,44 +36,30 @@ static void CommonInit() { static void CommonStandaloneInit() { SanitizerToolName = GetSanititizerToolName(); - InitializeFlags(); CacheBinaryName(); + InitializeFlags(); __sanitizer_set_report_path(common_flags()->log_path); AndroidLogInit(); InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); CommonInit(); - ubsan_mode = UBSAN_MODE_STANDALONE; } void __ubsan::InitAsStandalone() { - if (SANITIZER_CAN_USE_PREINIT_ARRAY) { - CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode); - CommonStandaloneInit(); - return; - } SpinMutexLock l(&ubsan_init_mu); - CHECK_NE(UBSAN_MODE_PLUGIN, ubsan_mode); - if (ubsan_mode == UBSAN_MODE_UNKNOWN) + if (!ubsan_initialized) { CommonStandaloneInit(); + ubsan_initialized = true; + } } -void __ubsan::InitAsStandaloneIfNecessary() { - if (SANITIZER_CAN_USE_PREINIT_ARRAY) { - CHECK_NE(UBSAN_MODE_UNKNOWN, ubsan_mode); - return; - } - SpinMutexLock l(&ubsan_init_mu); - if (ubsan_mode == UBSAN_MODE_UNKNOWN) - CommonStandaloneInit(); -} +void __ubsan::InitAsStandaloneIfNecessary() { return InitAsStandalone(); } void __ubsan::InitAsPlugin() { -#if !SANITIZER_CAN_USE_PREINIT_ARRAY SpinMutexLock l(&ubsan_init_mu); -#endif - CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode); - CommonInit(); - ubsan_mode = UBSAN_MODE_PLUGIN; + if (!ubsan_initialized) { + CommonInit(); + ubsan_initialized = true; + } } #endif // CAN_SANITIZE_UB diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc index ff1a20efea3d..8bd500025cbc 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc @@ -18,18 +18,17 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "ubsan_init.h" +#include "ubsan_signals_standalone.h" + +namespace __ubsan { -#if SANITIZER_CAN_USE_PREINIT_ARRAY -__attribute__((section(".preinit_array"), used)) -void (*__local_ubsan_preinit)(void) = __ubsan::InitAsStandalone; -#else -// Use a dynamic initializer. class UbsanStandaloneInitializer { public: UbsanStandaloneInitializer() { - __ubsan::InitAsStandalone(); + InitAsStandalone(); + InitializeDeadlySignals(); } }; static UbsanStandaloneInitializer ubsan_standalone_initializer; -#endif // SANITIZER_CAN_USE_PREINIT_ARRAY +} // namespace __ubsan diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc index a69ca57cd7af..9beb3e2ff953 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc @@ -11,14 +11,19 @@ INTERFACE_FUNCTION(__ubsan_handle_add_overflow) INTERFACE_FUNCTION(__ubsan_handle_add_overflow_abort) INTERFACE_FUNCTION(__ubsan_handle_builtin_unreachable) +INTERFACE_FUNCTION(__ubsan_handle_cfi_bad_type) INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail) INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail_abort) INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow) INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow_abort) +INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss) +INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort) INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch) INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort) +INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin) +INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin_abort) INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value) INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value_abort) INTERFACE_FUNCTION(__ubsan_handle_missing_return) diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_platform.h b/contrib/compiler-rt/lib/ubsan/ubsan_platform.h index 1a3bfd6afb81..72eb4199960e 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_platform.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_platform.h @@ -14,12 +14,13 @@ #define UBSAN_PLATFORM_H // Other platforms should be easy to add, and probably work as-is. -#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \ - defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \ - defined(__s390__)) +#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \ + defined(__NetBSD__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \ + defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \ + defined(__s390__)) || (defined(__sun__) && defined(__svr4__)) # define CAN_SANITIZE_UB 1 -#elif defined(_WIN32) +#elif defined(_WIN32) || defined(__Fuchsia__) # define CAN_SANITIZE_UB 1 #else # define CAN_SANITIZE_UB 0 diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc new file mode 100644 index 000000000000..4c6646dd8b0b --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc @@ -0,0 +1,53 @@ +//=-- ubsan_signals_standalone.cc +//------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Installs signal handlers and related interceptors for UBSan standalone. +// +//===----------------------------------------------------------------------===// + +#include "ubsan_platform.h" +#if CAN_SANITIZE_UB +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "ubsan_diag.h" +#include "ubsan_init.h" + +#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +namespace __ubsan { + +#if SANITIZER_FUCHSIA +void InitializeDeadlySignals() {} +#else +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void UBsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} + +static bool is_initialized = false; + +void InitializeDeadlySignals() { + if (is_initialized) + return; + is_initialized = true; + InitializeSignalInterceptors(); + InstallDeadlySignalHandlers(&UBsanOnDeadlySignal); +} +#endif + +} // namespace __ubsan + +#endif // CAN_SANITIZE_UB diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h new file mode 100644 index 000000000000..b29c29482ec8 --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h @@ -0,0 +1,25 @@ +//=-- ubsan_signals_standalone.h +//------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Installs signal handlers and related interceptors for UBSan standalone. +// +//===----------------------------------------------------------------------===// + +#ifndef UBSAN_SIGNALS_STANDALONE_H +#define UBSAN_SIGNALS_STANDALONE_H + +namespace __ubsan { + +// Initializes signal handlers and interceptors. +void InitializeDeadlySignals(); + +} // namespace __ubsan + +#endif // UBSAN_SIGNALS_STANDALONE_H diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra b/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra new file mode 100644 index 000000000000..7f8be694401a --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra @@ -0,0 +1 @@ +__ubsan_* diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc new file mode 100644 index 000000000000..5a5675c983fe --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc @@ -0,0 +1,104 @@ +#include "sanitizer_common/sanitizer_atomic.h" + +#include +#include +#include +#include + +#ifdef KERNEL_USE +extern "C" void ubsan_message(const char *msg); +static void message(const char *msg) { ubsan_message(msg); } +#else +static void message(const char *msg) { + write(2, msg, strlen(msg)); +} +#endif + +static const int kMaxCallerPcs = 20; +static __sanitizer::atomic_uintptr_t caller_pcs[kMaxCallerPcs]; +// Number of elements in caller_pcs. A special value of kMaxCallerPcs + 1 means +// that "too many errors" has already been reported. +static __sanitizer::atomic_uint32_t caller_pcs_sz; + +__attribute__((noinline)) static bool report_this_error(void *caller_p) { + uintptr_t caller = reinterpret_cast(caller_p); + if (caller == 0) return false; + while (true) { + unsigned sz = __sanitizer::atomic_load_relaxed(&caller_pcs_sz); + if (sz > kMaxCallerPcs) return false; // early exit + // when sz==kMaxCallerPcs print "too many errors", but only when cmpxchg + // succeeds in order to not print it multiple times. + if (sz > 0 && sz < kMaxCallerPcs) { + uintptr_t p; + for (unsigned i = 0; i < sz; ++i) { + p = __sanitizer::atomic_load_relaxed(&caller_pcs[i]); + if (p == 0) break; // Concurrent update. + if (p == caller) return false; + } + if (p == 0) continue; // FIXME: yield? + } + + if (!__sanitizer::atomic_compare_exchange_strong( + &caller_pcs_sz, &sz, sz + 1, __sanitizer::memory_order_seq_cst)) + continue; // Concurrent update! Try again from the start. + + if (sz == kMaxCallerPcs) { + message("ubsan: too many errors\n"); + return false; + } + __sanitizer::atomic_store_relaxed(&caller_pcs[sz], caller); + return true; + } +} + +#if defined(__ANDROID__) +extern "C" __attribute__((weak)) void android_set_abort_message(const char *); +static void abort_with_message(const char *msg) { + if (&android_set_abort_message) android_set_abort_message(msg); + abort(); +} +#else +static void abort_with_message(const char *) { abort(); } +#endif + +#define INTERFACE extern "C" __attribute__((visibility("default"))) + +// FIXME: add caller pc to the error message (possibly as "ubsan: error-type +// @1234ABCD"). +#define HANDLER_RECOVER(name, msg) \ + INTERFACE void __ubsan_handle_##name##_minimal() { \ + if (!report_this_error(__builtin_return_address(0))) return; \ + message("ubsan: " msg "\n"); \ + } + +#define HANDLER_NORECOVER(name, msg) \ + INTERFACE void __ubsan_handle_##name##_minimal_abort() { \ + message("ubsan: " msg "\n"); \ + abort_with_message("ubsan: " msg); \ + } + +#define HANDLER(name, msg) \ + HANDLER_RECOVER(name, msg) \ + HANDLER_NORECOVER(name, msg) + +HANDLER(type_mismatch, "type-mismatch") +HANDLER(add_overflow, "add-overflow") +HANDLER(sub_overflow, "sub-overflow") +HANDLER(mul_overflow, "mul-overflow") +HANDLER(negate_overflow, "negate-overflow") +HANDLER(divrem_overflow, "divrem-overflow") +HANDLER(shift_out_of_bounds, "shift-out-of-bounds") +HANDLER(out_of_bounds, "out-of-bounds") +HANDLER_RECOVER(builtin_unreachable, "builtin-unreachable") +HANDLER_RECOVER(missing_return, "missing-return") +HANDLER(vla_bound_not_positive, "vla-bound-not-positive") +HANDLER(float_cast_overflow, "float-cast-overflow") +HANDLER(load_invalid_value, "load-invalid-value") +HANDLER(invalid_builtin, "invalid-builtin") +HANDLER(function_type_mismatch, "function-type-mismatch") +HANDLER(nonnull_arg, "nonnull-arg") +HANDLER(nonnull_return, "nonnull-return") +HANDLER(nullability_arg, "nullability-arg") +HANDLER(nullability_return, "nullability-return") +HANDLER(pointer_overflow, "pointer-overflow") +HANDLER(cfi_check_fail, "cfi-check-fail") diff --git a/contrib/compiler-rt/lib/xray/weak_symbols.txt b/contrib/compiler-rt/lib/xray/weak_symbols.txt new file mode 100644 index 000000000000..963fff2d697e --- /dev/null +++ b/contrib/compiler-rt/lib/xray/weak_symbols.txt @@ -0,0 +1,4 @@ +___start_xray_fn_idx +___start_xray_instr_map +___stop_xray_fn_idx +___stop_xray_instr_map diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc index 7ba755ac3069..a0018f6b0cba 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc @@ -13,28 +13,34 @@ // //===----------------------------------------------------------------------===// #include "xray_buffer_queue.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" -#include -#include - using namespace __xray; using namespace __sanitizer; -BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success) - : BufferSize(B), Buffers(N), Mutex(), OwnedBuffers(), Finalizing{0} { - for (auto &T : Buffers) { - void *Tmp = malloc(BufferSize); +BufferQueue::BufferQueue(size_t B, size_t N, bool &Success) + : BufferSize(B), Buffers(new BufferRep[N]()), BufferCount(N), Finalizing{0}, + OwnedBuffers(new void *[N]()), Next(Buffers), First(Buffers), + LiveBuffers(0) { + for (size_t i = 0; i < N; ++i) { + auto &T = Buffers[i]; + void *Tmp = InternalAlloc(BufferSize, nullptr, 64); if (Tmp == nullptr) { Success = false; return; } - - auto &Buf = std::get<0>(T); + void *Extents = InternalAlloc(sizeof(BufferExtents), nullptr, 64); + if (Extents == nullptr) { + Success = false; + return; + } + auto &Buf = T.Buff; Buf.Buffer = Tmp; Buf.Size = B; - OwnedBuffers.emplace(Tmp); + Buf.Extents = reinterpret_cast(Extents); + OwnedBuffers[i] = Tmp; } Success = true; } @@ -42,27 +48,50 @@ BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success) BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) { if (__sanitizer::atomic_load(&Finalizing, __sanitizer::memory_order_acquire)) return ErrorCode::QueueFinalizing; - __sanitizer::BlockingMutexLock Guard(&Mutex); - if (Buffers.empty()) + __sanitizer::SpinMutexLock Guard(&Mutex); + if (LiveBuffers == BufferCount) return ErrorCode::NotEnoughMemory; - auto &T = Buffers.front(); - auto &B = std::get<0>(T); + + auto &T = *Next; + auto &B = T.Buff; Buf = B; - B.Buffer = nullptr; - B.Size = 0; - Buffers.pop_front(); + T.Used = true; + ++LiveBuffers; + + if (++Next == (Buffers + BufferCount)) + Next = Buffers; + return ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) { - if (OwnedBuffers.count(Buf.Buffer) == 0) + // Blitz through the buffers array to find the buffer. + bool Found = false; + for (auto I = OwnedBuffers, E = OwnedBuffers + BufferCount; I != E; ++I) { + if (*I == Buf.Buffer) { + Found = true; + break; + } + } + if (!Found) return ErrorCode::UnrecognizedBuffer; - __sanitizer::BlockingMutexLock Guard(&Mutex); + + __sanitizer::SpinMutexLock Guard(&Mutex); + + // This points to a semantic bug, we really ought to not be releasing more + // buffers than we actually get. + if (LiveBuffers == 0) + return ErrorCode::NotEnoughMemory; // Now that the buffer has been released, we mark it as "used". - Buffers.emplace(Buffers.end(), Buf, true /* used */); + First->Buff = Buf; + First->Used = true; Buf.Buffer = nullptr; Buf.Size = 0; + --LiveBuffers; + if (++First == (Buffers + BufferCount)) + First = Buffers; + return ErrorCode::Ok; } @@ -74,8 +103,12 @@ BufferQueue::ErrorCode BufferQueue::finalize() { } BufferQueue::~BufferQueue() { - for (auto &T : Buffers) { - auto &Buf = std::get<0>(T); - free(Buf.Buffer); + for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { + auto &T = *I; + auto &Buf = T.Buff; + InternalFree(Buf.Buffer); + InternalFree(Buf.Extents); } + delete[] Buffers; + delete[] OwnedBuffers; } diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h index e051695a297b..1ceb58274616 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h @@ -15,11 +15,9 @@ #ifndef XRAY_BUFFER_QUEUE_H #define XRAY_BUFFER_QUEUE_H +#include #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_mutex.h" -#include -#include -#include namespace __xray { @@ -29,23 +27,50 @@ namespace __xray { /// the "flight data recorder" (FDR) mode to support ongoing XRay function call /// trace collection. class BufferQueue { -public: + public: + struct alignas(64) BufferExtents { + __sanitizer::atomic_uint64_t Size; + }; + struct Buffer { void *Buffer = nullptr; size_t Size = 0; + BufferExtents* Extents; }; -private: + private: + struct BufferRep { + // The managed buffer. + Buffer Buff; + + // This is true if the buffer has been returned to the available queue, and + // is considered "used" by another thread. + bool Used = false; + }; + + // Size of each individual Buffer. size_t BufferSize; - // We use a bool to indicate whether the Buffer has been used in this - // freelist implementation. - std::deque> Buffers; - __sanitizer::BlockingMutex Mutex; - std::unordered_set OwnedBuffers; + BufferRep *Buffers; + size_t BufferCount; + + __sanitizer::SpinMutex Mutex; __sanitizer::atomic_uint8_t Finalizing; -public: + // Pointers to buffers managed/owned by the BufferQueue. + void **OwnedBuffers; + + // Pointer to the next buffer to be handed out. + BufferRep *Next; + + // Pointer to the entry in the array where the next released buffer will be + // placed. + BufferRep *First; + + // Count of buffers that have been handed out through 'getBuffer'. + size_t LiveBuffers; + + public: enum class ErrorCode : unsigned { Ok, NotEnoughMemory, @@ -56,16 +81,16 @@ class BufferQueue { static const char *getErrorString(ErrorCode E) { switch (E) { - case ErrorCode::Ok: - return "(none)"; - case ErrorCode::NotEnoughMemory: - return "no available buffers in the queue"; - case ErrorCode::QueueFinalizing: - return "queue already finalizing"; - case ErrorCode::UnrecognizedBuffer: - return "buffer being returned not owned by buffer queue"; - case ErrorCode::AlreadyFinalized: - return "queue already finalized"; + case ErrorCode::Ok: + return "(none)"; + case ErrorCode::NotEnoughMemory: + return "no available buffers in the queue"; + case ErrorCode::QueueFinalizing: + return "queue already finalizing"; + case ErrorCode::UnrecognizedBuffer: + return "buffer being returned not owned by buffer queue"; + case ErrorCode::AlreadyFinalized: + return "queue already finalized"; } return "unknown error"; } @@ -82,15 +107,18 @@ class BufferQueue { /// - BufferQueue is not finalising. /// /// Returns: - /// - std::errc::not_enough_memory on exceeding MaxSize. - /// - no error when we find a Buffer. - /// - std::errc::state_not_recoverable on finalising BufferQueue. + /// - ErrorCode::NotEnoughMemory on exceeding MaxSize. + /// - ErrorCode::Ok when we find a Buffer. + /// - ErrorCode::QueueFinalizing or ErrorCode::AlreadyFinalized on + /// a finalizing/finalized BufferQueue. ErrorCode getBuffer(Buffer &Buf); /// Updates |Buf| to point to nullptr, with size 0. /// /// Returns: - /// - ... + /// - ErrorCode::Ok when we successfully release the buffer. + /// - ErrorCode::UnrecognizedBuffer for when this BufferQueue does not own + /// the buffer being released. ErrorCode releaseBuffer(Buffer &Buf); bool finalizing() const { @@ -107,17 +135,18 @@ class BufferQueue { /// - All releaseBuffer operations will not fail. /// /// After a call to finalize succeeds, all subsequent calls to finalize will - /// fail with std::errc::state_not_recoverable. + /// fail with ErrorCode::QueueFinalizing. ErrorCode finalize(); /// Applies the provided function F to each Buffer in the queue, only if the /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a - /// releaseBuffer(...) operation. - template void apply(F Fn) { - __sanitizer::BlockingMutexLock G(&Mutex); - for (const auto &T : Buffers) { - if (std::get<1>(T)) - Fn(std::get<0>(T)); + /// releaseBuffer(...) operation). + template + void apply(F Fn) { + __sanitizer::SpinMutexLock G(&Mutex); + for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { + const auto &T = *I; + if (T.Used) Fn(T.Buff); } } @@ -125,6 +154,6 @@ class BufferQueue { ~BufferQueue(); }; -} // namespace __xray +} // namespace __xray -#endif // XRAY_BUFFER_QUEUE_H +#endif // XRAY_BUFFER_QUEUE_H diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h index 3d6d38892c76..324208db82ca 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h @@ -30,7 +30,10 @@ struct alignas(16) MetadataRecord { TSCWrap, WalltimeMarker, CustomEventMarker, + CallArgument, + BufferExtents, }; + // Use 7 bits to identify this record type. /* RecordKinds */ uint8_t RecordKind : 7; char Data[15]; diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc index a7e1382c3865..1bfa10c21f5c 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc @@ -15,15 +15,11 @@ // //===----------------------------------------------------------------------===// #include "xray_fdr_logging.h" -#include -#include -#include -#include +#include #include #include #include #include -#include #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" @@ -39,7 +35,7 @@ namespace __xray { // Global BufferQueue. -std::shared_ptr BQ; +BufferQueue *BQ = nullptr; __sanitizer::atomic_sint32_t LogFlushStatus = { XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; @@ -52,19 +48,31 @@ __sanitizer::SpinMutex FDROptionsMutex; XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { if (__sanitizer::atomic_load(&LoggingStatus, __sanitizer::memory_order_acquire) != - XRayLogInitStatus::XRAY_LOG_FINALIZED) + XRayLogInitStatus::XRAY_LOG_FINALIZED) { + if (__sanitizer::Verbosity()) + Report("Not flushing log, implementation is not finalized.\n"); return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; if (!__sanitizer::atomic_compare_exchange_strong( &LogFlushStatus, &Result, XRayLogFlushStatus::XRAY_LOG_FLUSHING, - __sanitizer::memory_order_release)) - return static_cast(Result); + __sanitizer::memory_order_release)) { - // Make a copy of the BufferQueue pointer to prevent other threads that may be - // resetting it from blowing away the queue prematurely while we're dealing - // with it. - auto LocalBQ = BQ; + if (__sanitizer::Verbosity()) + Report("Not flushing log, implementation is still finalizing.\n"); + return static_cast(Result); + } + + if (BQ == nullptr) { + if (__sanitizer::Verbosity()) + Report("Cannot flush when global buffer queue is null.\n"); + return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } + + // We wait a number of milliseconds to allow threads to see that we've + // finalised before attempting to flush the log. + __sanitizer::SleepForMillis(flags()->xray_fdr_log_grace_period_ms); // We write out the file in the following format: // @@ -95,24 +103,44 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { TSCSupported ? getTSCFrequency() : __xray::NanosecondsPerSecond; XRayFileHeader Header; - Header.Version = 1; + + // Version 2 of the log writes the extents of the buffer, instead of relying + // on an end-of-buffer record. + Header.Version = 2; Header.Type = FileTypes::FDR_LOG; Header.CycleFrequency = CycleFrequency; + // FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc' // before setting the values in the header. Header.ConstantTSC = 1; Header.NonstopTSC = 1; - Header.FdrData = FdrAdditionalHeaderData{LocalBQ->ConfiguredBufferSize()}; + Header.FdrData = FdrAdditionalHeaderData{BQ->ConfiguredBufferSize()}; retryingWriteAll(Fd, reinterpret_cast(&Header), reinterpret_cast(&Header) + sizeof(Header)); - LocalBQ->apply([&](const BufferQueue::Buffer &B) { - uint64_t BufferSize = B.Size; - if (BufferSize > 0) { + BQ->apply([&](const BufferQueue::Buffer &B) { + // Starting at version 2 of the FDR logging implementation, we only write + // the records identified by the extents of the buffer. We use the Extents + // from the Buffer and write that out as the first record in the buffer. + // We still use a Metadata record, but fill in the extents instead for the + // data. + MetadataRecord ExtentsRecord; + auto BufferExtents = __sanitizer::atomic_load( + &B.Extents->Size, __sanitizer::memory_order_acquire); + assert(BufferExtents <= B.Size); + ExtentsRecord.Type = uint8_t(RecordType::Metadata); + ExtentsRecord.RecordKind = + uint8_t(MetadataRecord::RecordKinds::BufferExtents); + std::memcpy(ExtentsRecord.Data, &BufferExtents, sizeof(BufferExtents)); + if (BufferExtents > 0) { + retryingWriteAll(Fd, reinterpret_cast(&ExtentsRecord), + reinterpret_cast(&ExtentsRecord) + + sizeof(MetadataRecord)); retryingWriteAll(Fd, reinterpret_cast(B.Buffer), - reinterpret_cast(B.Buffer) + B.Size); + reinterpret_cast(B.Buffer) + BufferExtents); } }); + __sanitizer::atomic_store(&LogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, __sanitizer::memory_order_release); @@ -124,8 +152,11 @@ XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT { if (!__sanitizer::atomic_compare_exchange_strong( &LoggingStatus, &CurrentStatus, XRayLogInitStatus::XRAY_LOG_FINALIZING, - __sanitizer::memory_order_release)) + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report("Cannot finalize log, implementation not initialized.\n"); return static_cast(CurrentStatus); + } // Do special things to make the log finalize itself, and not allow any more // operations to be performed until re-initialized. @@ -146,7 +177,8 @@ XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT { return static_cast(CurrentStatus); // Release the in-memory buffer queue. - BQ.reset(); + delete BQ; + BQ = nullptr; // Spin until the flushing status is flushed. s32 CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED; @@ -163,19 +195,22 @@ XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT { return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } -static std::tuple -getTimestamp() XRAY_NEVER_INSTRUMENT { +struct TSCAndCPU { + uint64_t TSC = 0; + unsigned char CPU = 0; +}; + +static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT { // We want to get the TSC as early as possible, so that we can check whether // we've seen this CPU before. We also do it before we load anything else, to // allow for forward progress with the scheduling. - unsigned char CPU; - uint64_t TSC; + TSCAndCPU Result; // Test once for required CPU features static bool TSCSupported = probeRequiredCPUFeatures(); if (TSCSupported) { - TSC = __xray::readTSC(CPU); + Result.TSC = __xray::readTSC(Result.CPU); } else { // FIXME: This code needs refactoring as it appears in multiple locations timespec TS; @@ -184,32 +219,35 @@ getTimestamp() XRAY_NEVER_INSTRUMENT { Report("clock_gettime(2) return %d, errno=%d", result, int(errno)); TS = {0, 0}; } - CPU = 0; - TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec; + Result.CPU = 0; + Result.TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec; } - return std::make_tuple(TSC, CPU); + return Result; } void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { - auto TSC_CPU = getTimestamp(); - __xray_fdr_internal::processFunctionHook(FuncId, Entry, std::get<0>(TSC_CPU), - std::get<1>(TSC_CPU), clock_gettime, - LoggingStatus, BQ); + auto TC = getTimestamp(); + __xray_fdr_internal::processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, 0, + clock_gettime, BQ); +} + +void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, + uint64_t Arg) XRAY_NEVER_INSTRUMENT { + auto TC = getTimestamp(); + __xray_fdr_internal::processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, Arg, + clock_gettime, BQ); } void fdrLoggingHandleCustomEvent(void *Event, std::size_t EventSize) XRAY_NEVER_INSTRUMENT { using namespace __xray_fdr_internal; - auto TSC_CPU = getTimestamp(); - auto &TSC = std::get<0>(TSC_CPU); - auto &CPU = std::get<1>(TSC_CPU); - thread_local bool Running = false; + auto TC = getTimestamp(); + auto &TSC = TC.TSC; + auto &CPU = TC.CPU; RecursionGuard Guard{Running}; - if (!Guard) { - assert(Running && "RecursionGuard is buggy!"); + if (!Guard) return; - } if (EventSize > std::numeric_limits::max()) { using Empty = struct {}; static Empty Once = [&] { @@ -220,15 +258,16 @@ void fdrLoggingHandleCustomEvent(void *Event, (void)Once; } int32_t ReducedEventSize = static_cast(EventSize); - if (!isLogInitializedAndReady(LocalBQ, TSC, CPU, clock_gettime)) + auto &TLD = getThreadLocalData(); + if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime)) return; // Here we need to prepare the log to handle: // - The metadata record we're going to write. (16 bytes) // - The additional data we're going to write. Currently, that's the size of // the event we're going to dump into the log as free-form bytes. - if (!prepareBuffer(clock_gettime, MetadataRecSize + EventSize)) { - LocalBQ = nullptr; + if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) { + TLD.BQ = nullptr; return; } @@ -240,27 +279,35 @@ void fdrLoggingHandleCustomEvent(void *Event, CustomEvent.Type = uint8_t(RecordType::Metadata); CustomEvent.RecordKind = uint8_t(MetadataRecord::RecordKinds::CustomEventMarker); - constexpr auto TSCSize = sizeof(std::get<0>(TSC_CPU)); + constexpr auto TSCSize = sizeof(TC.TSC); std::memcpy(&CustomEvent.Data, &ReducedEventSize, sizeof(int32_t)); std::memcpy(&CustomEvent.Data[sizeof(int32_t)], &TSC, TSCSize); - std::memcpy(RecordPtr, &CustomEvent, sizeof(CustomEvent)); - RecordPtr += sizeof(CustomEvent); - std::memcpy(RecordPtr, Event, ReducedEventSize); + std::memcpy(TLD.RecordPtr, &CustomEvent, sizeof(CustomEvent)); + TLD.RecordPtr += sizeof(CustomEvent); + std::memcpy(TLD.RecordPtr, Event, ReducedEventSize); + incrementExtents(MetadataRecSize + EventSize); endBufferIfFull(); } XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { - if (OptionsSize != sizeof(FDRLoggingOptions)) + if (OptionsSize != sizeof(FDRLoggingOptions)) { + if (__sanitizer::Verbosity()) + Report("Cannot initialize FDR logging; wrong size for options: %d\n", + OptionsSize); return static_cast(__sanitizer::atomic_load( &LoggingStatus, __sanitizer::memory_order_acquire)); + } s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; if (!__sanitizer::atomic_compare_exchange_strong( &LoggingStatus, &CurrentStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZING, - __sanitizer::memory_order_release)) + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report("Cannot initialize already initialized implementation.\n"); return static_cast(CurrentStatus); + } { __sanitizer::SpinMutexLock Guard(&FDROptionsMutex); @@ -268,12 +315,40 @@ XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax, } bool Success = false; - BQ = std::make_shared(BufferSize, BufferMax, Success); + + if (BQ != nullptr) { + delete BQ; + BQ = nullptr; + } + + if (BQ == nullptr) + BQ = new BufferQueue(BufferSize, BufferMax, Success); + if (!Success) { Report("BufferQueue init failed.\n"); + if (BQ != nullptr) { + delete BQ; + BQ = nullptr; + } return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } + static bool UNUSED Once = [] { + pthread_key_create(&__xray_fdr_internal::Key, +[](void *) { + auto &TLD = __xray_fdr_internal::getThreadLocalData(); + if (TLD.BQ == nullptr) + return; + auto EC = TLD.BQ->releaseBuffer(TLD.Buffer); + if (EC != BufferQueue::ErrorCode::Ok) + Report("At thread exit, failed to release buffer at %p; error=%s\n", + TLD.Buffer.Buffer, BufferQueue::getErrorString(EC)); + }); + return false; + }(); + + // Arg1 handler should go in first to avoid concurrent code accidentally + // falling back to arg0 when it should have ran arg1. + __xray_set_handler_arg1(fdrLoggingHandleArg1); // Install the actual handleArg0 handler after initialising the buffers. __xray_set_handler(fdrLoggingHandleArg0); __xray_set_customevent_handler(fdrLoggingHandleCustomEvent); @@ -281,20 +356,31 @@ XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax, __sanitizer::atomic_store(&LoggingStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZED, __sanitizer::memory_order_release); - Report("XRay FDR init successful.\n"); + + if (__sanitizer::Verbosity()) + Report("XRay FDR init successful.\n"); return XRayLogInitStatus::XRAY_LOG_INITIALIZED; } +bool fdrLogDynamicInitializer() XRAY_NEVER_INSTRUMENT { + using namespace __xray; + XRayLogImpl Impl{ + fdrLoggingInit, + fdrLoggingFinalize, + fdrLoggingHandleArg0, + fdrLoggingFlush, + }; + auto RegistrationResult = __xray_log_register_mode("xray-fdr", Impl); + if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + __sanitizer::Verbosity()) + Report("Cannot register XRay FDR mode to 'xray-fdr'; error = %d\n", + RegistrationResult); + if (flags()->xray_fdr_log || + !__sanitizer::internal_strcmp(flags()->xray_mode, "xray-fdr")) + __xray_set_log_impl(Impl); + return true; +} + } // namespace __xray -static auto UNUSED Unused = [] { - using namespace __xray; - if (flags()->xray_fdr_log) { - XRayLogImpl Impl{ - fdrLoggingInit, fdrLoggingFinalize, fdrLoggingHandleArg0, - fdrLoggingFlush, - }; - __xray_set_log_impl(Impl); - } - return true; -}(); +static auto UNUSED Unused = __xray::fdrLogDynamicInitializer(); diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.h b/contrib/compiler-rt/lib/xray/xray_fdr_logging.h index 426b54dc7884..1639d550a44c 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.h @@ -30,6 +30,7 @@ XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, size_t OptionsSize); XRayLogInitStatus fdrLoggingFinalize(); void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry); +void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, uint64_t Arg1); XRayLogFlushStatus fdrLoggingFlush(); XRayLogInitStatus fdrLoggingReset(); diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h b/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h index 4a1d80fd0eba..59eab55b2573 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h @@ -18,13 +18,13 @@ #define XRAY_XRAY_FDR_LOGGING_IMPL_H #include -#include +#include #include #include -#include -#include +#include #include #include +#include #include #include "sanitizer_common/sanitizer_common.h" @@ -52,57 +52,104 @@ __sanitizer::atomic_sint32_t LoggingStatus = { /// cooperation with xray_fdr_logging class, so be careful and think twice. namespace __xray_fdr_internal { -/// Writes the new buffer record and wallclock time that begin a buffer for a -/// thread to MemPtr and increments MemPtr. Bypasses the thread local state -/// machine and writes directly to memory without checks. -static void writeNewBufferPreamble(pid_t Tid, timespec TS, char *&MemPtr); +/// Writes the new buffer record and wallclock time that begin a buffer for the +/// current thread. +static void writeNewBufferPreamble(pid_t Tid, timespec TS); -/// Write a metadata record to switch to a new CPU to MemPtr and increments -/// MemPtr. Bypasses the thread local state machine and writes directly to -/// memory without checks. -static void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, char *&MemPtr); - -/// Writes an EOB metadata record to MemPtr and increments MemPtr. Bypasses the -/// thread local state machine and writes directly to memory without checks. -static void writeEOBMetadata(char *&MemPtr); - -/// Writes a TSC Wrap metadata record to MemPtr and increments MemPtr. Bypasses -/// the thread local state machine and directly writes to memory without checks. -static void writeTSCWrapMetadata(uint64_t TSC, char *&MemPtr); - -/// Writes a Function Record to MemPtr and increments MemPtr. Bypasses the -/// thread local state machine and writes the function record directly to -/// memory. +/// Writes a Function Record to the buffer associated with the current thread. static void writeFunctionRecord(int FuncId, uint32_t TSCDelta, - XRayEntryType EntryType, char *&MemPtr); + XRayEntryType EntryType); /// Sets up a new buffer in thread_local storage and writes a preamble. The /// wall_clock_reader function is used to populate the WallTimeRecord entry. static void setupNewBuffer(int (*wall_clock_reader)(clockid_t, struct timespec *)); -/// Called to record CPU time for a new CPU within the current thread. -static void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC); - -/// Called to close the buffer when the thread exhausts the buffer or when the -/// thread exits (via a thread local variable destructor). -static void writeEOBMetadata(); - /// TSC Wrap records are written when a TSC delta encoding scheme overflows. static void writeTSCWrapMetadata(uint64_t TSC); -/// Here's where the meat of the processing happens. The writer captures -/// function entry, exit and tail exit points with a time and will create -/// TSCWrap, NewCPUId and Function records as necessary. The writer might -/// walk backward through its buffer and erase trivial functions to avoid -/// polluting the log and may use the buffer queue to obtain or release a -/// buffer. -static void processFunctionHook(int32_t FuncId, XRayEntryType Entry, - uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, - struct timespec *), - __sanitizer::atomic_sint32_t &LoggingStatus, - const std::shared_ptr &BQ); +// Group together thread-local-data in a struct, then hide it behind a function +// call so that it can be initialized on first use instead of as a global. We +// force the alignment to 64-bytes for x86 cache line alignment, as this +// structure is used in the hot path of implementation. +struct alignas(64) ThreadLocalData { + BufferQueue::Buffer Buffer; + char *RecordPtr = nullptr; + // The number of FunctionEntry records immediately preceding RecordPtr. + uint8_t NumConsecutiveFnEnters = 0; + + // The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit + // records preceding RecordPtr. + uint8_t NumTailCalls = 0; + + // We use a thread_local variable to keep track of which CPUs we've already + // run, and the TSC times for these CPUs. This allows us to stop repeating the + // CPU field in the function records. + // + // We assume that we'll support only 65536 CPUs for x86_64. + uint16_t CurrentCPU = std::numeric_limits::max(); + uint64_t LastTSC = 0; + uint64_t LastFunctionEntryTSC = 0; + + // Make sure a thread that's ever called handleArg0 has a thread-local + // live reference to the buffer queue for this particular instance of + // FDRLogging, and that we're going to clean it up when the thread exits. + BufferQueue *BQ = nullptr; +}; + +static_assert(std::is_trivially_destructible::value, + "ThreadLocalData must be trivially destructible"); + +static constexpr auto MetadataRecSize = sizeof(MetadataRecord); +static constexpr auto FunctionRecSize = sizeof(FunctionRecord); + +// Use a global pthread key to identify thread-local data for logging. +static pthread_key_t Key; + +// This function will initialize the thread-local data structure used by the FDR +// logging implementation and return a reference to it. The implementation +// details require a bit of care to maintain. +// +// First, some requirements on the implementation in general: +// +// - XRay handlers should not call any memory allocation routines that may +// delegate to an instrumented implementation. This means functions like +// malloc() and free() should not be called while instrumenting. +// +// - We would like to use some thread-local data initialized on first-use of +// the XRay instrumentation. These allow us to implement unsynchronized +// routines that access resources associated with the thread. +// +// The implementation here uses a few mechanisms that allow us to provide both +// the requirements listed above. We do this by: +// +// 1. Using a thread-local aligned storage buffer for representing the +// ThreadLocalData struct. This data will be uninitialized memory by +// design. +// +// 2. Not requiring a thread exit handler/implementation, keeping the +// thread-local as purely a collection of references/data that do not +// require cleanup. +// +// We're doing this to avoid using a `thread_local` object that has a +// non-trivial destructor, because the C++ runtime might call std::malloc(...) +// to register calls to destructors. Deadlocks may arise when, for example, an +// externally provided malloc implementation is XRay instrumented, and +// initializing the thread-locals involves calling into malloc. A malloc +// implementation that does global synchronization might be holding a lock for a +// critical section, calling a function that might be XRay instrumented (and +// thus in turn calling into malloc by virtue of registration of the +// thread_local's destructor). +static ThreadLocalData &getThreadLocalData() { + static_assert(alignof(ThreadLocalData) >= 64, + "ThreadLocalData must be cache line aligned."); + thread_local ThreadLocalData TLD; + thread_local bool UNUSED ThreadOnce = [] { + pthread_setspecific(Key, &TLD); + return false; + }(); + return TLD; +} //-----------------------------------------------------------------------------| // The rest of the file is implementation. | @@ -113,71 +160,12 @@ static void processFunctionHook(int32_t FuncId, XRayEntryType Entry, namespace { -thread_local BufferQueue::Buffer Buffer; -thread_local char *RecordPtr = nullptr; - -// The number of FunctionEntry records immediately preceding RecordPtr. -thread_local uint8_t NumConsecutiveFnEnters = 0; - -// The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit -// records preceding RecordPtr. -thread_local uint8_t NumTailCalls = 0; - -constexpr auto MetadataRecSize = sizeof(MetadataRecord); -constexpr auto FunctionRecSize = sizeof(FunctionRecord); - -// We use a thread_local variable to keep track of which CPUs we've already -// run, and the TSC times for these CPUs. This allows us to stop repeating the -// CPU field in the function records. -// -// We assume that we'll support only 65536 CPUs for x86_64. -thread_local uint16_t CurrentCPU = std::numeric_limits::max(); -thread_local uint64_t LastTSC = 0; -thread_local uint64_t LastFunctionEntryTSC = 0; - -class ThreadExitBufferCleanup { - std::shared_ptr &Buffers; - BufferQueue::Buffer &Buffer; - -public: - explicit ThreadExitBufferCleanup(std::shared_ptr &BQ, - BufferQueue::Buffer &Buffer) - XRAY_NEVER_INSTRUMENT : Buffers(BQ), - Buffer(Buffer) {} - - ~ThreadExitBufferCleanup() noexcept XRAY_NEVER_INSTRUMENT { - if (RecordPtr == nullptr) - return; - - // We make sure that upon exit, a thread will write out the EOB - // MetadataRecord in the thread-local log, and also release the buffer to - // the queue. - assert((RecordPtr + MetadataRecSize) - static_cast(Buffer.Buffer) >= - static_cast(MetadataRecSize)); - if (Buffers) { - writeEOBMetadata(); - auto EC = Buffers->releaseBuffer(Buffer); - if (EC != BufferQueue::ErrorCode::Ok) - Report("Failed to release buffer at %p; error=%s\n", Buffer.Buffer, - BufferQueue::getErrorString(EC)); - Buffers = nullptr; - return; - } - } -}; - -// Make sure a thread that's ever called handleArg0 has a thread-local -// live reference to the buffer queue for this particular instance of -// FDRLogging, and that we're going to clean it up when the thread exits. -thread_local std::shared_ptr LocalBQ = nullptr; -thread_local ThreadExitBufferCleanup Cleanup(LocalBQ, Buffer); - class RecursionGuard { - bool &Running; + volatile bool &Running; const bool Valid; public: - explicit RecursionGuard(bool &R) : Running(R), Valid(!R) { + explicit RecursionGuard(volatile bool &R) : Running(R), Valid(!R) { if (Valid) Running = true; } @@ -195,34 +183,29 @@ class RecursionGuard { } }; -inline bool loggingInitialized( - const __sanitizer::atomic_sint32_t &LoggingStatus) XRAY_NEVER_INSTRUMENT { - return __sanitizer::atomic_load(&LoggingStatus, - __sanitizer::memory_order_acquire) == - XRayLogInitStatus::XRAY_LOG_INITIALIZED; -} - } // namespace -inline void writeNewBufferPreamble(pid_t Tid, timespec TS, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { +static void writeNewBufferPreamble(pid_t Tid, + timespec TS) XRAY_NEVER_INSTRUMENT { static constexpr int InitRecordsCount = 2; - std::aligned_storage::type Records[InitRecordsCount]; + auto &TLD = getThreadLocalData(); + MetadataRecord Metadata[InitRecordsCount]; { // Write out a MetadataRecord to signify that this is the start of a new // buffer, associated with a particular thread, with a new CPU. For the // data, we have 15 bytes to squeeze as much information as we can. At this // point we only write down the following bytes: // - Thread ID (pid_t, 4 bytes) - auto &NewBuffer = *reinterpret_cast(&Records[0]); + auto &NewBuffer = Metadata[0]; NewBuffer.Type = uint8_t(RecordType::Metadata); NewBuffer.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewBuffer); std::memcpy(&NewBuffer.Data, &Tid, sizeof(pid_t)); } + // Also write the WalltimeMarker record. { static_assert(sizeof(time_t) <= 8, "time_t needs to be at most 8 bytes"); - auto &WalltimeMarker = *reinterpret_cast(&Records[1]); + auto &WalltimeMarker = Metadata[1]; WalltimeMarker.Type = uint8_t(RecordType::Metadata); WalltimeMarker.RecordKind = uint8_t(MetadataRecord::RecordKinds::WalltimeMarker); @@ -235,26 +218,48 @@ inline void writeNewBufferPreamble(pid_t Tid, timespec TS, std::memcpy(WalltimeMarker.Data, &Seconds, sizeof(Seconds)); std::memcpy(WalltimeMarker.Data + sizeof(Seconds), &Micros, sizeof(Micros)); } - std::memcpy(MemPtr, Records, sizeof(MetadataRecord) * InitRecordsCount); - MemPtr += sizeof(MetadataRecord) * InitRecordsCount; - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + if (TLD.BQ == nullptr || TLD.BQ->finalizing()) + return; + std::memcpy(TLD.RecordPtr, Metadata, sizeof(Metadata)); + TLD.RecordPtr += sizeof(Metadata); + // Since we write out the extents as the first metadata record of the + // buffer, we need to write out the extents including the extents record. + __sanitizer::atomic_store(&TLD.Buffer.Extents->Size, sizeof(Metadata), + __sanitizer::memory_order_release); } inline void setupNewBuffer(int (*wall_clock_reader)( clockid_t, struct timespec *)) XRAY_NEVER_INSTRUMENT { - RecordPtr = static_cast(Buffer.Buffer); + auto &TLD = getThreadLocalData(); + auto &B = TLD.Buffer; + TLD.RecordPtr = static_cast(B.Buffer); pid_t Tid = syscall(SYS_gettid); timespec TS{0, 0}; // This is typically clock_gettime, but callers have injection ability. wall_clock_reader(CLOCK_MONOTONIC, &TS); - writeNewBufferPreamble(Tid, TS, RecordPtr); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + writeNewBufferPreamble(Tid, TS); + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; } -inline void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { +static void incrementExtents(size_t Add) { + auto &TLD = getThreadLocalData(); + __sanitizer::atomic_fetch_add(&TLD.Buffer.Extents->Size, Add, + __sanitizer::memory_order_acq_rel); +} + +static void decrementExtents(size_t Subtract) { + auto &TLD = getThreadLocalData(); + __sanitizer::atomic_fetch_sub(&TLD.Buffer.Extents->Size, Subtract, + __sanitizer::memory_order_acq_rel); +} + +inline void writeNewCPUIdMetadata(uint16_t CPU, + uint64_t TSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); MetadataRecord NewCPUId; NewCPUId.Type = uint8_t(RecordType::Metadata); NewCPUId.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewCPUId); @@ -265,34 +270,15 @@ inline void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, // Total = 10 bytes. std::memcpy(&NewCPUId.Data, &CPU, sizeof(CPU)); std::memcpy(&NewCPUId.Data[sizeof(CPU)], &TSC, sizeof(TSC)); - std::memcpy(MemPtr, &NewCPUId, sizeof(MetadataRecord)); - MemPtr += sizeof(MetadataRecord); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + std::memcpy(TLD.RecordPtr, &NewCPUId, sizeof(MetadataRecord)); + TLD.RecordPtr += sizeof(MetadataRecord); + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + incrementExtents(sizeof(MetadataRecord)); } -inline void writeNewCPUIdMetadata(uint16_t CPU, - uint64_t TSC) XRAY_NEVER_INSTRUMENT { - writeNewCPUIdMetadata(CPU, TSC, RecordPtr); -} - -inline void writeEOBMetadata(char *&MemPtr) XRAY_NEVER_INSTRUMENT { - MetadataRecord EOBMeta; - EOBMeta.Type = uint8_t(RecordType::Metadata); - EOBMeta.RecordKind = uint8_t(MetadataRecord::RecordKinds::EndOfBuffer); - // For now we don't write any bytes into the Data field. - std::memcpy(MemPtr, &EOBMeta, sizeof(MetadataRecord)); - MemPtr += sizeof(MetadataRecord); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; -} - -inline void writeEOBMetadata() XRAY_NEVER_INSTRUMENT { - writeEOBMetadata(RecordPtr); -} - -inline void writeTSCWrapMetadata(uint64_t TSC, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { +inline void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); MetadataRecord TSCWrap; TSCWrap.Type = uint8_t(RecordType::Metadata); TSCWrap.RecordKind = uint8_t(MetadataRecord::RecordKinds::TSCWrap); @@ -301,58 +287,67 @@ inline void writeTSCWrapMetadata(uint64_t TSC, // - Full TSC (uint64_t, 8 bytes) // Total = 8 bytes. std::memcpy(&TSCWrap.Data, &TSC, sizeof(TSC)); - std::memcpy(MemPtr, &TSCWrap, sizeof(MetadataRecord)); - MemPtr += sizeof(MetadataRecord); - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + std::memcpy(TLD.RecordPtr, &TSCWrap, sizeof(MetadataRecord)); + TLD.RecordPtr += sizeof(MetadataRecord); + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + incrementExtents(sizeof(MetadataRecord)); } -inline void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT { - writeTSCWrapMetadata(TSC, RecordPtr); +// Call Argument metadata records store the arguments to a function in the +// order of their appearance; holes are not supported by the buffer format. +static inline void writeCallArgumentMetadata(uint64_t A) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); + MetadataRecord CallArg; + CallArg.Type = uint8_t(RecordType::Metadata); + CallArg.RecordKind = uint8_t(MetadataRecord::RecordKinds::CallArgument); + + std::memcpy(CallArg.Data, &A, sizeof(A)); + std::memcpy(TLD.RecordPtr, &CallArg, sizeof(MetadataRecord)); + TLD.RecordPtr += sizeof(MetadataRecord); + incrementExtents(sizeof(MetadataRecord)); } -inline void writeFunctionRecord(int FuncId, uint32_t TSCDelta, - XRayEntryType EntryType, - char *&MemPtr) XRAY_NEVER_INSTRUMENT { - std::aligned_storage::type - AlignedFuncRecordBuffer; - auto &FuncRecord = - *reinterpret_cast(&AlignedFuncRecordBuffer); +static inline void +writeFunctionRecord(int FuncId, uint32_t TSCDelta, + XRayEntryType EntryType) XRAY_NEVER_INSTRUMENT { + FunctionRecord FuncRecord; FuncRecord.Type = uint8_t(RecordType::Function); // Only take 28 bits of the function id. FuncRecord.FuncId = FuncId & ~(0x0F << 28); FuncRecord.TSCDelta = TSCDelta; + auto &TLD = getThreadLocalData(); switch (EntryType) { case XRayEntryType::ENTRY: - ++NumConsecutiveFnEnters; + ++TLD.NumConsecutiveFnEnters; FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); break; case XRayEntryType::LOG_ARGS_ENTRY: // We should not rewind functions with logged args. - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); break; case XRayEntryType::EXIT: // If we've decided to log the function exit, we will never erase the log // before it. - NumConsecutiveFnEnters = 0; - NumTailCalls = 0; + TLD.NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionExit); break; case XRayEntryType::TAIL: // If we just entered the function we're tail exiting from or erased every // invocation since then, this function entry tail pair is a candidate to // be erased when the child function exits. - if (NumConsecutiveFnEnters > 0) { - ++NumTailCalls; - NumConsecutiveFnEnters = 0; + if (TLD.NumConsecutiveFnEnters > 0) { + ++TLD.NumTailCalls; + TLD.NumConsecutiveFnEnters = 0; } else { // We will never be able to erase this tail call since we have logged // something in between the function entry and tail exit. - NumTailCalls = 0; - NumConsecutiveFnEnters = 0; + TLD.NumTailCalls = 0; + TLD.NumConsecutiveFnEnters = 0; } FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionTailExit); @@ -370,8 +365,9 @@ inline void writeFunctionRecord(int FuncId, uint32_t TSCDelta, } } - std::memcpy(MemPtr, &AlignedFuncRecordBuffer, sizeof(FunctionRecord)); - MemPtr += sizeof(FunctionRecord); + std::memcpy(TLD.RecordPtr, &FuncRecord, sizeof(FunctionRecord)); + TLD.RecordPtr += sizeof(FunctionRecord); + incrementExtents(sizeof(FunctionRecord)); } static uint64_t thresholdTicks() { @@ -387,23 +383,21 @@ static uint64_t thresholdTicks() { // "Function Entry" record and any "Tail Call Exit" records after that. static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, uint64_t &LastFunctionEntryTSC, int32_t FuncId) { - using AlignedFuncStorage = - std::aligned_storage::type; - RecordPtr -= FunctionRecSize; - AlignedFuncStorage AlignedFuncRecordBuffer; - const auto &FuncRecord = *reinterpret_cast( - std::memcpy(&AlignedFuncRecordBuffer, RecordPtr, FunctionRecSize)); + auto &TLD = getThreadLocalData(); + TLD.RecordPtr -= FunctionRecSize; + decrementExtents(FunctionRecSize); + FunctionRecord FuncRecord; + std::memcpy(&FuncRecord, TLD.RecordPtr, FunctionRecSize); assert(FuncRecord.RecordKind == uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && "Expected to find function entry recording when rewinding."); assert(FuncRecord.FuncId == (FuncId & ~(0x0F << 28)) && "Expected matching function id when rewinding Exit"); - --NumConsecutiveFnEnters; + --TLD.NumConsecutiveFnEnters; LastTSC -= FuncRecord.TSCDelta; // We unwound one call. Update the state and return without writing a log. - if (NumConsecutiveFnEnters != 0) { + if (TLD.NumConsecutiveFnEnters != 0) { LastFunctionEntryTSC -= FuncRecord.TSCDelta; return; } @@ -413,22 +407,19 @@ static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, // exited from via this exit. LastFunctionEntryTSC = 0; auto RewindingTSC = LastTSC; - auto RewindingRecordPtr = RecordPtr - FunctionRecSize; - while (NumTailCalls > 0) { - AlignedFuncStorage TailExitRecordBuffer; + auto RewindingRecordPtr = TLD.RecordPtr - FunctionRecSize; + while (TLD.NumTailCalls > 0) { // Rewind the TSC back over the TAIL EXIT record. - const auto &ExpectedTailExit = - *reinterpret_cast(std::memcpy( - &TailExitRecordBuffer, RewindingRecordPtr, FunctionRecSize)); + FunctionRecord ExpectedTailExit; + std::memcpy(&ExpectedTailExit, RewindingRecordPtr, FunctionRecSize); assert(ExpectedTailExit.RecordKind == uint8_t(FunctionRecord::RecordKinds::FunctionTailExit) && "Expected to find tail exit when rewinding."); RewindingRecordPtr -= FunctionRecSize; RewindingTSC -= ExpectedTailExit.TSCDelta; - AlignedFuncStorage FunctionEntryBuffer; - const auto &ExpectedFunctionEntry = *reinterpret_cast( - std::memcpy(&FunctionEntryBuffer, RewindingRecordPtr, FunctionRecSize)); + FunctionRecord ExpectedFunctionEntry; + std::memcpy(&ExpectedFunctionEntry, RewindingRecordPtr, FunctionRecSize); assert(ExpectedFunctionEntry.RecordKind == uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && "Expected to find function entry when rewinding tail call."); @@ -437,80 +428,87 @@ static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, // This tail call exceeded the threshold duration. It will not be erased. if ((TSC - RewindingTSC) >= thresholdTicks()) { - NumTailCalls = 0; + TLD.NumTailCalls = 0; return; } // We can erase a tail exit pair that we're exiting through since // its duration is under threshold. - --NumTailCalls; + --TLD.NumTailCalls; RewindingRecordPtr -= FunctionRecSize; RewindingTSC -= ExpectedFunctionEntry.TSCDelta; - RecordPtr -= 2 * FunctionRecSize; + TLD.RecordPtr -= 2 * FunctionRecSize; LastTSC = RewindingTSC; + decrementExtents(2 * FunctionRecSize); } } -inline bool releaseThreadLocalBuffer(BufferQueue *BQ) { - auto EC = BQ->releaseBuffer(Buffer); +inline bool releaseThreadLocalBuffer(BufferQueue &BQArg) { + auto &TLD = getThreadLocalData(); + auto EC = BQArg.releaseBuffer(TLD.Buffer); if (EC != BufferQueue::ErrorCode::Ok) { - Report("Failed to release buffer at %p; error=%s\n", Buffer.Buffer, + Report("Failed to release buffer at %p; error=%s\n", TLD.Buffer.Buffer, BufferQueue::getErrorString(EC)); return false; } return true; } -inline bool prepareBuffer(int (*wall_clock_reader)(clockid_t, +inline bool prepareBuffer(uint64_t TSC, unsigned char CPU, + int (*wall_clock_reader)(clockid_t, struct timespec *), size_t MaxSize) XRAY_NEVER_INSTRUMENT { - char *BufferStart = static_cast(Buffer.Buffer); - if ((RecordPtr + MaxSize) > (BufferStart + Buffer.Size - MetadataRecSize)) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + auto &TLD = getThreadLocalData(); + char *BufferStart = static_cast(TLD.Buffer.Buffer); + if ((TLD.RecordPtr + MaxSize) > (BufferStart + TLD.Buffer.Size)) { + if (!releaseThreadLocalBuffer(*TLD.BQ)) return false; - auto EC = LocalBQ->getBuffer(Buffer); + auto EC = TLD.BQ->getBuffer(TLD.Buffer); if (EC != BufferQueue::ErrorCode::Ok) { Report("Failed to acquire a buffer; error=%s\n", BufferQueue::getErrorString(EC)); return false; } setupNewBuffer(wall_clock_reader); + + // Always write the CPU metadata as the first record in the buffer. + writeNewCPUIdMetadata(CPU, TSC); } return true; } -inline bool isLogInitializedAndReady( - std::shared_ptr &LocalBQ, uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, - struct timespec *)) XRAY_NEVER_INSTRUMENT { +inline bool +isLogInitializedAndReady(BufferQueue *LBQ, uint64_t TSC, unsigned char CPU, + int (*wall_clock_reader)(clockid_t, struct timespec *)) + XRAY_NEVER_INSTRUMENT { // Bail out right away if logging is not initialized yet. // We should take the opportunity to release the buffer though. auto Status = __sanitizer::atomic_load(&LoggingStatus, __sanitizer::memory_order_acquire); + auto &TLD = getThreadLocalData(); if (Status != XRayLogInitStatus::XRAY_LOG_INITIALIZED) { - if (RecordPtr != nullptr && + if (TLD.RecordPtr != nullptr && (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || Status == XRayLogInitStatus::XRAY_LOG_FINALIZED)) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + if (!releaseThreadLocalBuffer(*LBQ)) return false; - RecordPtr = nullptr; - LocalBQ = nullptr; + TLD.RecordPtr = nullptr; return false; } return false; } - if (!loggingInitialized(LoggingStatus) || LocalBQ->finalizing()) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + if (__sanitizer::atomic_load(&LoggingStatus, + __sanitizer::memory_order_acquire) != + XRayLogInitStatus::XRAY_LOG_INITIALIZED || + LBQ->finalizing()) { + if (!releaseThreadLocalBuffer(*LBQ)) return false; - RecordPtr = nullptr; + TLD.RecordPtr = nullptr; } - if (Buffer.Buffer == nullptr) { - auto EC = LocalBQ->getBuffer(Buffer); + if (TLD.Buffer.Buffer == nullptr) { + auto EC = LBQ->getBuffer(TLD.Buffer); if (EC != BufferQueue::ErrorCode::Ok) { auto LS = __sanitizer::atomic_load(&LoggingStatus, __sanitizer::memory_order_acquire); @@ -522,51 +520,100 @@ inline bool isLogInitializedAndReady( } setupNewBuffer(wall_clock_reader); + + // Always write the CPU metadata as the first record in the buffer. + writeNewCPUIdMetadata(CPU, TSC); } - if (CurrentCPU == std::numeric_limits::max()) { + if (TLD.CurrentCPU == std::numeric_limits::max()) { // This means this is the first CPU this thread has ever run on. We set // the current CPU and record this as the first TSC we've seen. - CurrentCPU = CPU; + TLD.CurrentCPU = CPU; writeNewCPUIdMetadata(CPU, TSC); } return true; } // namespace __xray_fdr_internal +// Compute the TSC difference between the time of measurement and the previous +// event. There are a few interesting situations we need to account for: +// +// - The thread has migrated to a different CPU. If this is the case, then +// we write down the following records: +// +// 1. A 'NewCPUId' Metadata record. +// 2. A FunctionRecord with a 0 for the TSCDelta field. +// +// - The TSC delta is greater than the 32 bits we can store in a +// FunctionRecord. In this case we write down the following records: +// +// 1. A 'TSCWrap' Metadata record. +// 2. A FunctionRecord with a 0 for the TSCDelta field. +// +// - The TSC delta is representable within the 32 bits we can store in a +// FunctionRecord. In this case we write down just a FunctionRecord with +// the correct TSC delta. +inline uint32_t writeCurrentCPUTSC(ThreadLocalData &TLD, uint64_t TSC, + uint8_t CPU) { + if (CPU != TLD.CurrentCPU) { + // We've moved to a new CPU. + writeNewCPUIdMetadata(CPU, TSC); + return 0; + } + // If the delta is greater than the range for a uint32_t, then we write out + // the TSC wrap metadata entry with the full TSC, and the TSC for the + // function record be 0. + uint64_t Delta = TSC - TLD.LastTSC; + if (Delta <= std::numeric_limits::max()) + return Delta; + + writeTSCWrapMetadata(TSC); + return 0; +} + inline void endBufferIfFull() XRAY_NEVER_INSTRUMENT { - auto BufferStart = static_cast(Buffer.Buffer); - if ((RecordPtr + MetadataRecSize) - BufferStart == MetadataRecSize) { - writeEOBMetadata(); - if (!releaseThreadLocalBuffer(LocalBQ.get())) + auto &TLD = getThreadLocalData(); + auto BufferStart = static_cast(TLD.Buffer.Buffer); + if ((TLD.RecordPtr + MetadataRecSize) - BufferStart <= + ptrdiff_t{MetadataRecSize}) { + if (!releaseThreadLocalBuffer(*TLD.BQ)) return; - RecordPtr = nullptr; + TLD.RecordPtr = nullptr; } } -inline void processFunctionHook( - int32_t FuncId, XRayEntryType Entry, uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, struct timespec *), - __sanitizer::atomic_sint32_t &LoggingStatus, - const std::shared_ptr &BQ) XRAY_NEVER_INSTRUMENT { +thread_local volatile bool Running = false; + +/// Here's where the meat of the processing happens. The writer captures +/// function entry, exit and tail exit points with a time and will create +/// TSCWrap, NewCPUId and Function records as necessary. The writer might +/// walk backward through its buffer and erase trivial functions to avoid +/// polluting the log and may use the buffer queue to obtain or release a +/// buffer. +inline void processFunctionHook(int32_t FuncId, XRayEntryType Entry, + uint64_t TSC, unsigned char CPU, uint64_t Arg1, + int (*wall_clock_reader)(clockid_t, + struct timespec *), + BufferQueue *BQ) XRAY_NEVER_INSTRUMENT { // Prevent signal handler recursion, so in case we're already in a log writing // mode and the signal handler comes in (and is also instrumented) then we // don't want to be clobbering potentially partial writes already happening in // the thread. We use a simple thread_local latch to only allow one on-going // handleArg0 to happen at any given time. - thread_local bool Running = false; RecursionGuard Guard{Running}; if (!Guard) { assert(Running == true && "RecursionGuard is buggy!"); return; } + auto &TLD = getThreadLocalData(); + // In case the reference has been cleaned up before, we make sure we // initialize it to the provided BufferQueue. - if (LocalBQ == nullptr) - LocalBQ = BQ; + if (TLD.BQ == nullptr) + TLD.BQ = BQ; - if (!isLogInitializedAndReady(LocalBQ, TSC, CPU, wall_clock_reader)) + if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, wall_clock_reader)) return; // Before we go setting up writing new function entries, we need to be really @@ -579,10 +626,10 @@ inline void processFunctionHook( // - The least number of bytes we will ever write is 8 // (sizeof(FunctionRecord)) only if the delta between the previous entry // and this entry is within 32 bits. - // - The most number of bytes we will ever write is 8 + 16 = 24. This is - // computed by: + // - The most number of bytes we will ever write is 8 + 16 + 16 = 40. + // This is computed by: // - // sizeof(FunctionRecord) + sizeof(MetadataRecord) + // MaxSize = sizeof(FunctionRecord) + 2 * sizeof(MetadataRecord) // // These arise in the following cases: // @@ -596,77 +643,39 @@ inline void processFunctionHook( // FunctionRecord. // 3. When we learn about a new CPU ID, we need to write down a "new cpu // id" MetadataRecord before writing out the actual FunctionRecord. + // 4. The second MetadataRecord is the optional function call argument. // - // - An End-of-Buffer (EOB) MetadataRecord is 16 bytes. - // - // So the math we need to do is to determine whether writing 24 bytes past the - // current pointer leaves us with enough bytes to write the EOB - // MetadataRecord. If we don't have enough space after writing as much as 24 - // bytes in the end of the buffer, we need to write out the EOB, get a new - // Buffer, set it up properly before doing any further writing. - // - if (!prepareBuffer(wall_clock_reader, FunctionRecSize + MetadataRecSize)) { - LocalBQ = nullptr; + // So the math we need to do is to determine whether writing 40 bytes past the + // current pointer exceeds the buffer's maximum size. If we don't have enough + // space to write 40 bytes in the buffer, we need get a new Buffer, set it up + // properly before doing any further writing. + size_t MaxSize = FunctionRecSize + 2 * MetadataRecSize; + if (!prepareBuffer(TSC, CPU, wall_clock_reader, MaxSize)) { + TLD.BQ = nullptr; return; } - // By this point, we are now ready to write at most 24 bytes (one metadata - // record and one function record). - assert((RecordPtr + (MetadataRecSize + FunctionRecSize)) - - static_cast(Buffer.Buffer) >= + // By this point, we are now ready to write up to 40 bytes (explained above). + assert((TLD.RecordPtr + MaxSize) - static_cast(TLD.Buffer.Buffer) >= static_cast(MetadataRecSize) && "Misconfigured BufferQueue provided; Buffer size not large enough."); - // Here we compute the TSC Delta. There are a few interesting situations we - // need to account for: - // - // - The thread has migrated to a different CPU. If this is the case, then - // we write down the following records: - // - // 1. A 'NewCPUId' Metadata record. - // 2. A FunctionRecord with a 0 for the TSCDelta field. - // - // - The TSC delta is greater than the 32 bits we can store in a - // FunctionRecord. In this case we write down the following records: - // - // 1. A 'TSCWrap' Metadata record. - // 2. A FunctionRecord with a 0 for the TSCDelta field. - // - // - The TSC delta is representable within the 32 bits we can store in a - // FunctionRecord. In this case we write down just a FunctionRecord with - // the correct TSC delta. - // - uint32_t RecordTSCDelta = 0; - if (CPU != CurrentCPU) { - // We've moved to a new CPU. - writeNewCPUIdMetadata(CPU, TSC); - } else { - // If the delta is greater than the range for a uint32_t, then we write out - // the TSC wrap metadata entry with the full TSC, and the TSC for the - // function record be 0. - auto Delta = TSC - LastTSC; - if (Delta > (1ULL << 32) - 1) - writeTSCWrapMetadata(TSC); - else - RecordTSCDelta = Delta; - } - - LastTSC = TSC; - CurrentCPU = CPU; + auto RecordTSCDelta = writeCurrentCPUTSC(TLD, TSC, CPU); + TLD.LastTSC = TSC; + TLD.CurrentCPU = CPU; switch (Entry) { case XRayEntryType::ENTRY: case XRayEntryType::LOG_ARGS_ENTRY: // Update the thread local state for the next invocation. - LastFunctionEntryTSC = TSC; + TLD.LastFunctionEntryTSC = TSC; break; case XRayEntryType::TAIL: - break; case XRayEntryType::EXIT: // Break out and write the exit record if we can't erase any functions. - if (NumConsecutiveFnEnters == 0 || - (TSC - LastFunctionEntryTSC) >= thresholdTicks()) + if (TLD.NumConsecutiveFnEnters == 0 || + (TSC - TLD.LastFunctionEntryTSC) >= thresholdTicks()) break; - rewindRecentCall(TSC, LastTSC, LastFunctionEntryTSC, FuncId); + rewindRecentCall(TSC, TLD.LastTSC, TLD.LastFunctionEntryTSC, FuncId); return; // without writing log. case XRayEntryType::CUSTOM_EVENT: { // This is a bug in patching, so we'll report it once and move on. @@ -681,7 +690,9 @@ inline void processFunctionHook( } } - writeFunctionRecord(FuncId, RecordTSCDelta, Entry, RecordPtr); + writeFunctionRecord(FuncId, RecordTSCDelta, Entry); + if (Entry == XRayEntryType::LOG_ARGS_ENTRY) + writeCallArgumentMetadata(Arg1); // If we've exhausted the buffer by this time, we then release the buffer to // make sure that other threads may start using this buffer. diff --git a/contrib/compiler-rt/lib/xray/xray_flags.h b/contrib/compiler-rt/lib/xray/xray_flags.h index f4e30283b8de..3ed5b8844cb4 100644 --- a/contrib/compiler-rt/lib/xray/xray_flags.h +++ b/contrib/compiler-rt/lib/xray/xray_flags.h @@ -16,6 +16,7 @@ #define XRAY_FLAGS_H #include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_internal_defs.h" namespace __xray { diff --git a/contrib/compiler-rt/lib/xray/xray_flags.inc b/contrib/compiler-rt/lib/xray/xray_flags.inc index 7ddce78eb413..29f1fce7d7f4 100644 --- a/contrib/compiler-rt/lib/xray/xray_flags.inc +++ b/contrib/compiler-rt/lib/xray/xray_flags.inc @@ -16,12 +16,34 @@ XRAY_FLAG(bool, patch_premain, false, "Whether to patch instrumentation points before main.") -XRAY_FLAG(bool, xray_naive_log, true, - "Whether to install the naive log implementation.") XRAY_FLAG(const char *, xray_logfile_base, "xray-log.", "Filename base for the xray logfile.") +XRAY_FLAG(const char *, xray_mode, "", "Mode to install by default.") +XRAY_FLAG(uptr, xray_page_size_override, 0, + "Override the default page size for the system, in bytes. The size " + "should be a power-of-two.") + +// Basic (Naive) Mode logging options. +XRAY_FLAG(bool, xray_naive_log, false, + "DEPRECATED: Use xray_mode=xray-basic instead.") +XRAY_FLAG(int, xray_naive_log_func_duration_threshold_us, 5, + "Naive logging will try to skip functions that execute for fewer " + "microseconds than this threshold.") +XRAY_FLAG(int, xray_naive_log_max_stack_depth, 64, + "Naive logging will keep track of at most this deep a call stack, " + "any more and the recordings will be droppped.") +XRAY_FLAG(int, xray_naive_log_thread_buffer_size, 1024, + "The number of entries to keep on a per-thread buffer.") + +// FDR (Flight Data Recorder) Mode logging options. XRAY_FLAG(bool, xray_fdr_log, false, - "Whether to install the flight data recorder logging implementation.") + "DEPRECATED: Use xray_mode=xray-fdr instead.") XRAY_FLAG(int, xray_fdr_log_func_duration_threshold_us, 5, "FDR logging will try to skip functions that execute for fewer " "microseconds than this threshold.") +XRAY_FLAG(int, xray_fdr_log_grace_period_us, 0, + "DEPRECATED: use xray_fdr_log_grace_period_ms instead.") +XRAY_FLAG(int, xray_fdr_log_grace_period_ms, 100, + "FDR logging will wait this much time in microseconds before " + "actually flushing the log; this gives a chance for threads to " + "notice that the log has been finalized and clean up.") diff --git a/contrib/compiler-rt/lib/xray/xray_init.cc b/contrib/compiler-rt/lib/xray/xray_init.cc index aa660baa9920..11892cb8b7a3 100644 --- a/contrib/compiler-rt/lib/xray/xray_init.cc +++ b/contrib/compiler-rt/lib/xray/xray_init.cc @@ -44,12 +44,31 @@ __sanitizer::atomic_uint8_t XRayInitialized{0}; __sanitizer::SpinMutex XRayInstrMapMutex; XRaySledMap XRayInstrMap; +// Global flag to determine whether the flags have been initialized. +__sanitizer::atomic_uint8_t XRayFlagsInitialized{0}; + +// A mutex to allow only one thread to initialize the XRay data structures. +__sanitizer::SpinMutex XRayInitMutex; + // __xray_init() will do the actual loading of the current process' memory map // and then proceed to look for the .xray_instr_map section/segment. void __xray_init() XRAY_NEVER_INSTRUMENT { - initializeFlags(); + __sanitizer::SpinMutexLock Guard(&XRayInitMutex); + // Short-circuit if we've already initialized XRay before. + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) + return; + + if (!__sanitizer::atomic_load(&XRayFlagsInitialized, + __sanitizer::memory_order_acquire)) { + initializeFlags(); + __sanitizer::atomic_store(&XRayFlagsInitialized, true, + __sanitizer::memory_order_release); + } + if (__start_xray_instr_map == nullptr) { - Report("XRay instrumentation map missing. Not initializing XRay.\n"); + if (Verbosity()) + Report("XRay instrumentation map missing. Not initializing XRay.\n"); return; } @@ -63,9 +82,21 @@ void __xray_init() XRAY_NEVER_INSTRUMENT { __sanitizer::atomic_store(&XRayInitialized, true, __sanitizer::memory_order_release); +#ifndef XRAY_NO_PREINIT if (flags()->patch_premain) __xray_patch(); +#endif } +#if !defined(XRAY_NO_PREINIT) && SANITIZER_CAN_USE_PREINIT_ARRAY +// Only add the preinit array initialization if the sanitizers can. __attribute__((section(".preinit_array"), used)) void (*__local_xray_preinit)(void) = __xray_init; +#else +// If we cannot use the .preinit_array section, we should instead use dynamic +// initialisation. +static bool UNUSED __local_xray_dyninit = [] { + __xray_init(); + return true; +}(); +#endif diff --git a/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc b/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc index 83aecfaf7700..a27ffbcbd12e 100644 --- a/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc +++ b/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc @@ -16,80 +16,85 @@ //===----------------------------------------------------------------------===// #include +#include +#include #include -#include +#include #include #include #include -#include +#include #include +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_libc.h" #include "xray/xray_records.h" #include "xray_defs.h" #include "xray_flags.h" +#include "xray_inmemory_log.h" #include "xray_interface_internal.h" #include "xray_tsc.h" #include "xray_utils.h" -// __xray_InMemoryRawLog will use a thread-local aligned buffer capped to a -// certain size (32kb by default) and use it as if it were a circular buffer for -// events. We store simple fixed-sized entries in the log for external analysis. - -extern "C" { -void __xray_InMemoryRawLog(int32_t FuncId, - XRayEntryType Type) XRAY_NEVER_INSTRUMENT; -} - namespace __xray { -std::mutex LogMutex; +__sanitizer::SpinMutex LogMutex; -class ThreadExitFlusher { - int Fd; - XRayRecord *Start; - size_t &Offset; - -public: - explicit ThreadExitFlusher(int Fd, XRayRecord *Start, - size_t &Offset) XRAY_NEVER_INSTRUMENT - : Fd(Fd), - Start(Start), - Offset(Offset) {} - - ~ThreadExitFlusher() XRAY_NEVER_INSTRUMENT { - std::lock_guard L(LogMutex); - if (Fd > 0 && Start != nullptr) { - retryingWriteAll(Fd, reinterpret_cast(Start), - reinterpret_cast(Start + Offset)); - // Because this thread's exit could be the last one trying to write to the - // file and that we're not able to close out the file properly, we sync - // instead and hope that the pending writes are flushed as the thread - // exits. - fsync(Fd); - } - } +// We use elements of this type to record the entry TSC of every function ID we +// see as we're tracing a particular thread's execution. +struct alignas(16) StackEntry { + int32_t FuncId; + uint16_t Type; + uint8_t CPU; + uint8_t Padding; + uint64_t TSC; }; -} // namespace __xray +static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry"); -using namespace __xray; +struct alignas(64) ThreadLocalData { + void *InMemoryBuffer = nullptr; + size_t BufferSize = 0; + size_t BufferOffset = 0; + void *ShadowStack = nullptr; + size_t StackSize = 0; + size_t StackEntries = 0; + int Fd = -1; + pid_t TID = 0; +}; -static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT { +static pthread_key_t PThreadKey; + +static __sanitizer::atomic_uint8_t BasicInitialized{0}; + +BasicLoggingOptions GlobalOptions; + +thread_local volatile bool RecursionGuard = false; + +static uint64_t thresholdTicks() XRAY_NEVER_INSTRUMENT { + static uint64_t TicksPerSec = probeRequiredCPUFeatures() + ? getTSCFrequency() + : __xray::NanosecondsPerSecond; + static const uint64_t ThresholdTicks = + TicksPerSec * GlobalOptions.DurationFilterMicros / 1000000; + return ThresholdTicks; +} + +static int openLogFile() XRAY_NEVER_INSTRUMENT { int F = getLogFD(); if (F == -1) return -1; // Test for required CPU features and cache the cycle frequency static bool TSCSupported = probeRequiredCPUFeatures(); - static uint64_t CycleFrequency = TSCSupported ? getTSCFrequency() - : __xray::NanosecondsPerSecond; + static uint64_t CycleFrequency = + TSCSupported ? getTSCFrequency() : __xray::NanosecondsPerSecond; // Since we're here, we get to write the header. We set it up so that the // header will only be written once, at the start, and let the threads // logging do writes which just append. XRayFileHeader Header; - Header.Version = 1; + Header.Version = 2; // Version 2 includes tail exit records. Header.Type = FileTypes::NAIVE_LOG; Header.CycleFrequency = CycleFrequency; @@ -102,47 +107,210 @@ static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT { return F; } +int getGlobalFd() XRAY_NEVER_INSTRUMENT { + static int Fd = openLogFile(); + return Fd; +} + +ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { + thread_local ThreadLocalData TLD; + thread_local bool UNUSED TOnce = [] { + if (GlobalOptions.ThreadBufferSize == 0) { + if (__sanitizer::Verbosity()) + Report("Not initializing TLD since ThreadBufferSize == 0.\n"); + return false; + } + TLD.TID = __sanitizer::GetTid(); + pthread_setspecific(PThreadKey, &TLD); + TLD.Fd = getGlobalFd(); + TLD.InMemoryBuffer = reinterpret_cast( + InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize, + nullptr, alignof(XRayRecord))); + TLD.BufferSize = GlobalOptions.ThreadBufferSize; + TLD.BufferOffset = 0; + if (GlobalOptions.MaxStackDepth == 0) { + if (__sanitizer::Verbosity()) + Report("Not initializing the ShadowStack since MaxStackDepth == 0.\n"); + TLD.StackSize = 0; + TLD.StackEntries = 0; + TLD.ShadowStack = nullptr; + return false; + } + TLD.ShadowStack = reinterpret_cast( + InternalAlloc(sizeof(StackEntry) * GlobalOptions.MaxStackDepth, nullptr, + alignof(StackEntry))); + TLD.StackSize = GlobalOptions.MaxStackDepth; + TLD.StackEntries = 0; + if (__sanitizer::Verbosity() >= 2) { + static auto UNUSED Once = [] { + auto ticks = thresholdTicks(); + Report("Ticks threshold: %d\n", ticks); + return false; + }(); + } + return false; + }(); + return TLD; +} + template -void __xray_InMemoryRawLog(int32_t FuncId, XRayEntryType Type, - RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { - using Buffer = - std::aligned_storage::type; - static constexpr size_t BuffLen = 1024; - thread_local static Buffer InMemoryBuffer[BuffLen] = {}; - thread_local static size_t Offset = 0; - static int Fd = __xray_OpenLogFile(); +void InMemoryRawLog(int32_t FuncId, XRayEntryType Type, + RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); + auto &InMemoryBuffer = TLD.InMemoryBuffer; + int Fd = getGlobalFd(); if (Fd == -1) return; - thread_local __xray::ThreadExitFlusher Flusher( - Fd, reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer), Offset); - thread_local pid_t TId = syscall(SYS_gettid); - // First we get the useful data, and stuff it into the already aligned buffer - // through a pointer offset. - auto &R = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer)[Offset]; + // Use a simple recursion guard, to handle cases where we're already logging + // and for one reason or another, this function gets called again in the same + // thread. + if (RecursionGuard) + return; + RecursionGuard = true; + auto ExitGuard = __sanitizer::at_scope_exit([] { RecursionGuard = false; }); + + uint8_t CPU = 0; + uint64_t TSC = ReadTSC(CPU); + + switch (Type) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: { + // Short circuit if we've reached the maximum depth of the stack. + if (TLD.StackEntries++ >= TLD.StackSize) + return; + + // When we encounter an entry event, we keep track of the TSC and the CPU, + // and put it in the stack. + StackEntry E; + E.FuncId = FuncId; + E.CPU = CPU; + E.Type = Type; + E.TSC = TSC; + auto StackEntryPtr = static_cast(TLD.ShadowStack) + + (sizeof(StackEntry) * (TLD.StackEntries - 1)); + __sanitizer::internal_memcpy(StackEntryPtr, &E, sizeof(StackEntry)); + break; + } + case XRayEntryType::EXIT: + case XRayEntryType::TAIL: { + if (TLD.StackEntries == 0) + break; + + if (--TLD.StackEntries >= TLD.StackSize) + return; + + // When we encounter an exit event, we check whether all the following are + // true: + // + // - The Function ID is the same as the most recent entry in the stack. + // - The CPU is the same as the most recent entry in the stack. + // - The Delta of the TSCs is less than the threshold amount of time we're + // looking to record. + // + // If all of these conditions are true, we pop the stack and don't write a + // record and move the record offset back. + StackEntry StackTop; + auto StackEntryPtr = static_cast(TLD.ShadowStack) + + (sizeof(StackEntry) * TLD.StackEntries); + __sanitizer::internal_memcpy(&StackTop, StackEntryPtr, sizeof(StackEntry)); + if (StackTop.FuncId == FuncId && StackTop.CPU == CPU && + StackTop.TSC < TSC) { + auto Delta = TSC - StackTop.TSC; + if (Delta < thresholdTicks()) { + assert(TLD.BufferOffset > 0); + TLD.BufferOffset -= StackTop.Type == XRayEntryType::ENTRY ? 1 : 2; + return; + } + } + break; + } + default: + // Should be unreachable. + assert(false && "Unsupported XRayEntryType encountered."); + break; + } + + // First determine whether the delta between the function's enter record and + // the exit record is higher than the threshold. + __xray::XRayRecord R; R.RecordType = RecordTypes::NORMAL; - R.TSC = ReadTSC(R.CPU); - R.TId = TId; + R.CPU = CPU; + R.TSC = TSC; + R.TId = TLD.TID; R.Type = Type; R.FuncId = FuncId; - ++Offset; - if (Offset == BuffLen) { - std::lock_guard L(LogMutex); + auto EntryPtr = static_cast(InMemoryBuffer) + + (sizeof(__xray::XRayRecord) * TLD.BufferOffset); + __sanitizer::internal_memcpy(EntryPtr, &R, sizeof(R)); + if (++TLD.BufferOffset == TLD.BufferSize) { + __sanitizer::SpinMutexLock L(&LogMutex); + auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer); + retryingWriteAll(Fd, reinterpret_cast(RecordBuffer), + reinterpret_cast(RecordBuffer + TLD.BufferOffset)); + TLD.BufferOffset = 0; + TLD.StackEntries = 0; + } +} + +template +void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1, + RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { + auto &TLD = getThreadLocalData(); + auto &InMemoryBuffer = TLD.InMemoryBuffer; + auto &Offset = TLD.BufferOffset; + const auto &BuffLen = TLD.BufferSize; + int Fd = getGlobalFd(); + if (Fd == -1) + return; + + // First we check whether there's enough space to write the data consecutively + // in the thread-local buffer. If not, we first flush the buffer before + // attempting to write the two records that must be consecutive. + if (Offset + 2 > BuffLen) { + __sanitizer::SpinMutexLock L(&LogMutex); auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer); retryingWriteAll(Fd, reinterpret_cast(RecordBuffer), reinterpret_cast(RecordBuffer + Offset)); Offset = 0; + TLD.StackEntries = 0; + } + + // Then we write the "we have an argument" record. + InMemoryRawLog(FuncId, Type, ReadTSC); + + if (RecursionGuard) + return; + RecursionGuard = true; + auto ExitGuard = __sanitizer::at_scope_exit([] { RecursionGuard = false; }); + + // And from here on write the arg payload. + __xray::XRayArgPayload R; + R.RecordType = RecordTypes::ARG_PAYLOAD; + R.FuncId = FuncId; + R.TId = TLD.TID; + R.Arg = Arg1; + auto EntryPtr = + &reinterpret_cast<__xray::XRayArgPayload *>(&InMemoryBuffer)[Offset]; + std::memcpy(EntryPtr, &R, sizeof(R)); + if (++Offset == BuffLen) { + __sanitizer::SpinMutexLock L(&LogMutex); + auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer); + retryingWriteAll(Fd, reinterpret_cast(RecordBuffer), + reinterpret_cast(RecordBuffer + Offset)); + Offset = 0; + TLD.StackEntries = 0; } } -void __xray_InMemoryRawLogRealTSC(int32_t FuncId, - XRayEntryType Type) XRAY_NEVER_INSTRUMENT { - __xray_InMemoryRawLog(FuncId, Type, __xray::readTSC); +void basicLoggingHandleArg0RealTSC(int32_t FuncId, + XRayEntryType Type) XRAY_NEVER_INSTRUMENT { + InMemoryRawLog(FuncId, Type, __xray::readTSC); } -void __xray_InMemoryEmulateTSC(int32_t FuncId, - XRayEntryType Type) XRAY_NEVER_INSTRUMENT { - __xray_InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT { +void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Type) + XRAY_NEVER_INSTRUMENT { + InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT { timespec TS; int result = clock_gettime(CLOCK_REALTIME, &TS); if (result != 0) { @@ -154,13 +322,150 @@ void __xray_InMemoryEmulateTSC(int32_t FuncId, }); } -static auto UNUSED Unused = [] { - auto UseRealTSC = probeRequiredCPUFeatures(); - if (!UseRealTSC) +void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Type, + uint64_t Arg1) XRAY_NEVER_INSTRUMENT { + InMemoryRawLogWithArg(FuncId, Type, Arg1, __xray::readTSC); +} + +void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Type, + uint64_t Arg1) XRAY_NEVER_INSTRUMENT { + InMemoryRawLogWithArg( + FuncId, Type, Arg1, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT { + timespec TS; + int result = clock_gettime(CLOCK_REALTIME, &TS); + if (result != 0) { + Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno)); + TS = {0, 0}; + } + CPU = 0; + return TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec; + }); +} + +static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT { + ThreadLocalData &TLD = *reinterpret_cast(P); + auto ExitGuard = __sanitizer::at_scope_exit([&TLD] { + // Clean up dynamic resources. + if (TLD.InMemoryBuffer) + InternalFree(TLD.InMemoryBuffer); + if (TLD.ShadowStack) + InternalFree(TLD.ShadowStack); + if (__sanitizer::Verbosity()) + Report("Cleaned up log for TID: %d\n", TLD.TID); + }); + + if (TLD.Fd == -1 || TLD.BufferOffset == 0) { + if (__sanitizer::Verbosity()) + Report("Skipping buffer for TID: %d; Fd = %d; Offset = %llu\n", TLD.TID, + TLD.Fd, TLD.BufferOffset); + return; + } + + { + __sanitizer::SpinMutexLock L(&LogMutex); + retryingWriteAll(TLD.Fd, reinterpret_cast(TLD.InMemoryBuffer), + reinterpret_cast(TLD.InMemoryBuffer) + + (sizeof(__xray::XRayRecord) * TLD.BufferOffset)); + } + + // Because this thread's exit could be the last one trying to write to + // the file and that we're not able to close out the file properly, we + // sync instead and hope that the pending writes are flushed as the + // thread exits. + fsync(TLD.Fd); +} + +XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, + void *Options, + size_t OptionsSize) XRAY_NEVER_INSTRUMENT { + static bool UNUSED Once = [] { + pthread_key_create(&PThreadKey, TLDDestructor); + return false; + }(); + + uint8_t Expected = 0; + if (!__sanitizer::atomic_compare_exchange_strong( + &BasicInitialized, &Expected, 1, __sanitizer::memory_order_acq_rel)) { + if (__sanitizer::Verbosity()) + Report("Basic logging already initialized.\n"); + return XRayLogInitStatus::XRAY_LOG_INITIALIZED; + } + + if (OptionsSize != sizeof(BasicLoggingOptions)) { + Report("Invalid options size, potential ABI mismatch; expected %d got %d", + sizeof(BasicLoggingOptions), OptionsSize); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + + static auto UseRealTSC = probeRequiredCPUFeatures(); + if (!UseRealTSC && __sanitizer::Verbosity()) Report("WARNING: Required CPU features missing for XRay instrumentation, " "using emulation instead.\n"); - if (flags()->xray_naive_log) - __xray_set_handler(UseRealTSC ? __xray_InMemoryRawLogRealTSC - : __xray_InMemoryEmulateTSC); + + GlobalOptions = *reinterpret_cast(Options); + __xray_set_handler_arg1(UseRealTSC ? basicLoggingHandleArg1RealTSC + : basicLoggingHandleArg1EmulateTSC); + __xray_set_handler(UseRealTSC ? basicLoggingHandleArg0RealTSC + : basicLoggingHandleArg0EmulateTSC); + __xray_remove_customevent_handler(); + + return XRayLogInitStatus::XRAY_LOG_INITIALIZED; +} + +XRayLogInitStatus basicLoggingFinalize() XRAY_NEVER_INSTRUMENT { + uint8_t Expected = 0; + if (!__sanitizer::atomic_compare_exchange_strong( + &BasicInitialized, &Expected, 0, __sanitizer::memory_order_acq_rel) && + __sanitizer::Verbosity()) + Report("Basic logging already finalized.\n"); + + // Nothing really to do aside from marking state of the global to be + // uninitialized. + + return XRayLogInitStatus::XRAY_LOG_FINALIZED; +} + +XRayLogFlushStatus basicLoggingFlush() XRAY_NEVER_INSTRUMENT { + // This really does nothing, since flushing the logs happen at the end of a + // thread's lifetime, or when the buffers are full. + return XRayLogFlushStatus::XRAY_LOG_FLUSHED; +} + +// This is a handler that, effectively, does nothing. +void basicLoggingHandleArg0Empty(int32_t, XRayEntryType) XRAY_NEVER_INSTRUMENT { +} + +bool basicLogDynamicInitializer() XRAY_NEVER_INSTRUMENT { + XRayLogImpl Impl{ + basicLoggingInit, + basicLoggingFinalize, + basicLoggingHandleArg0Empty, + basicLoggingFlush, + }; + auto RegistrationResult = __xray_log_register_mode("xray-basic", Impl); + if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + __sanitizer::Verbosity()) + Report("Cannot register XRay Basic Mode to 'xray-basic'; error = %d\n", + RegistrationResult); + if (flags()->xray_naive_log || + !__sanitizer::internal_strcmp(flags()->xray_mode, "xray-basic")) { + __xray_set_log_impl(Impl); + BasicLoggingOptions Options; + Options.DurationFilterMicros = + flags()->xray_naive_log_func_duration_threshold_us; + Options.MaxStackDepth = flags()->xray_naive_log_max_stack_depth; + Options.ThreadBufferSize = flags()->xray_naive_log_thread_buffer_size; + __xray_log_init(flags()->xray_naive_log_thread_buffer_size, 0, &Options, + sizeof(BasicLoggingOptions)); + static auto UNUSED Once = [] { + static auto UNUSED &TLD = getThreadLocalData(); + __sanitizer::Atexit(+[] { TLDDestructor(&TLD); }); + return false; + }(); + } return true; -}(); +} + +} // namespace __xray + +static auto UNUSED Unused = __xray::basicLogDynamicInitializer(); diff --git a/contrib/compiler-rt/lib/xray/xray_inmemory_log.h b/contrib/compiler-rt/lib/xray/xray_inmemory_log.h new file mode 100644 index 000000000000..e4fcb8ca5ffd --- /dev/null +++ b/contrib/compiler-rt/lib/xray/xray_inmemory_log.h @@ -0,0 +1,44 @@ +//===-- xray_inmemory_log.h +//------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a function call tracing system. +// +//===----------------------------------------------------------------------===// +#ifndef XRAY_XRAY_INMEMORY_LOG_H +#define XRAY_XRAY_INMEMORY_LOG_H + +#include "xray/xray_log_interface.h" + +/// Basic (Naive) Mode +/// ================== +/// +/// This implementation hooks in through the XRay logging implementation +/// framework. The Basic Mode implementation will keep appending to a file as +/// soon as the thread-local buffers are full. It keeps minimal in-memory state +/// and does the minimum filtering required to keep log files smaller. + +namespace __xray { + +XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, + void *Options, size_t OptionsSize); +XRayLogInitStatus basicLoggingFinalize(); + +void basicLoggingHandleArg0RealTSC(int32_t FuncId, XRayEntryType Entry); +void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Entry); +void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Entry, + uint64_t Arg1); +void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Entry, + uint64_t Arg1); +XRayLogFlushStatus basicLoggingFlush(); +XRayLogInitStatus basicLoggingReset(); + +} // namespace __xray + +#endif // XRAY_XRAY_INMEMORY_LOG_H diff --git a/contrib/compiler-rt/lib/xray/xray_interface.cc b/contrib/compiler-rt/lib/xray/xray_interface.cc index 694d34c0102b..766313e85c58 100644 --- a/contrib/compiler-rt/lib/xray/xray_interface.cc +++ b/contrib/compiler-rt/lib/xray/xray_interface.cc @@ -23,12 +23,15 @@ #include "sanitizer_common/sanitizer_common.h" #include "xray_defs.h" +#include "xray_flags.h" + +extern __sanitizer::SpinMutex XRayInstrMapMutex; +extern __sanitizer::atomic_uint8_t XRayInitialized; +extern __xray::XRaySledMap XRayInstrMap; namespace __xray { #if defined(__x86_64__) -// FIXME: The actual length is 11 bytes. Why was length 12 passed to mprotect() -// ? static const int16_t cSledLength = 12; #elif defined(__aarch64__) static const int16_t cSledLength = 32; @@ -53,6 +56,10 @@ __sanitizer::atomic_uintptr_t XRayArgLogger{0}; // This is the function to call when we encounter a custom event log call. __sanitizer::atomic_uintptr_t XRayPatchedCustomEvent{0}; +// This is the global status to determine whether we are currently +// patching/unpatching. +__sanitizer::atomic_uint8_t XRayPatching{0}; + // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will undo // any successful mprotect(...) changes. This is used to make a page writeable // and executable, and upon destruction if it was successful in doing so returns @@ -88,85 +95,10 @@ public: } }; -} // namespace __xray - -extern __sanitizer::SpinMutex XRayInstrMapMutex; -extern __sanitizer::atomic_uint8_t XRayInitialized; -extern __xray::XRaySledMap XRayInstrMap; - -int __xray_set_handler(void (*entry)(int32_t, - XRayEntryType)) XRAY_NEVER_INSTRUMENT { - if (__sanitizer::atomic_load(&XRayInitialized, - __sanitizer::memory_order_acquire)) { - - __sanitizer::atomic_store(&__xray::XRayPatchedFunction, - reinterpret_cast(entry), - __sanitizer::memory_order_release); - return 1; - } - return 0; -} - -int __xray_set_customevent_handler(void (*entry)(void *, size_t)) - XRAY_NEVER_INSTRUMENT { - if (__sanitizer::atomic_load(&XRayInitialized, - __sanitizer::memory_order_acquire)) { - __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent, - reinterpret_cast(entry), - __sanitizer::memory_order_release); - return 1; - } - return 0; -} - - -int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { - return __xray_set_handler(nullptr); -} - -int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { - return __xray_set_customevent_handler(nullptr); -} - -__sanitizer::atomic_uint8_t XRayPatching{0}; - -using namespace __xray; - -// FIXME: Figure out whether we can move this class to sanitizer_common instead -// as a generic "scope guard". -template class CleanupInvoker { - Function Fn; - -public: - explicit CleanupInvoker(Function Fn) XRAY_NEVER_INSTRUMENT : Fn(Fn) {} - CleanupInvoker(const CleanupInvoker &) XRAY_NEVER_INSTRUMENT = default; - CleanupInvoker(CleanupInvoker &&) XRAY_NEVER_INSTRUMENT = default; - CleanupInvoker & - operator=(const CleanupInvoker &) XRAY_NEVER_INSTRUMENT = delete; - CleanupInvoker &operator=(CleanupInvoker &&) XRAY_NEVER_INSTRUMENT = delete; - ~CleanupInvoker() XRAY_NEVER_INSTRUMENT { Fn(); } -}; - -template -CleanupInvoker scopeCleanup(Function Fn) XRAY_NEVER_INSTRUMENT { - return CleanupInvoker{Fn}; -} - -inline bool patchSled(const XRaySledEntry &Sled, bool Enable, - int32_t FuncId) XRAY_NEVER_INSTRUMENT { - // While we're here, we should patch the nop sled. To do that we mprotect - // the page containing the function to be writeable. - const uint64_t PageSize = GetPageSizeCached(); - void *PageAlignedAddr = - reinterpret_cast(Sled.Address & ~(PageSize - 1)); - std::size_t MProtectLen = (Sled.Address + cSledLength) - - reinterpret_cast(PageAlignedAddr); - MProtectHelper Protector(PageAlignedAddr, MProtectLen); - if (Protector.MakeWriteable() == -1) { - printf("Failed mprotect: %d\n", errno); - return XRayPatchingStatus::FAILED; - } +namespace { +bool patchSled(const XRaySledEntry &Sled, bool Enable, + int32_t FuncId) XRAY_NEVER_INSTRUMENT { bool Success = false; switch (Sled.Kind) { case XRayEntryType::ENTRY: @@ -191,69 +123,6 @@ inline bool patchSled(const XRaySledEntry &Sled, bool Enable, return Success; } -// controlPatching implements the common internals of the patching/unpatching -// implementation. |Enable| defines whether we're enabling or disabling the -// runtime XRay instrumentation. -XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { - if (!__sanitizer::atomic_load(&XRayInitialized, - __sanitizer::memory_order_acquire)) - return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. - - uint8_t NotPatching = false; - if (!__sanitizer::atomic_compare_exchange_strong( - &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) - return XRayPatchingStatus::ONGOING; // Already patching. - - uint8_t PatchingSuccess = false; - auto XRayPatchingStatusResetter = scopeCleanup([&PatchingSuccess] { - if (!PatchingSuccess) - __sanitizer::atomic_store(&XRayPatching, false, - __sanitizer::memory_order_release); - }); - - // Step 1: Compute the function id, as a unique identifier per function in the - // instrumentation map. - XRaySledMap InstrMap; - { - __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); - InstrMap = XRayInstrMap; - } - if (InstrMap.Entries == 0) - return XRayPatchingStatus::NOT_INITIALIZED; - - const uint64_t PageSize = GetPageSizeCached(); - if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { - Report("System page size is not a power of two: %lld\n", PageSize); - return XRayPatchingStatus::FAILED; - } - - uint32_t FuncId = 1; - uint64_t CurFun = 0; - for (std::size_t I = 0; I < InstrMap.Entries; I++) { - auto Sled = InstrMap.Sleds[I]; - auto F = Sled.Function; - if (CurFun == 0) - CurFun = F; - if (F != CurFun) { - ++FuncId; - CurFun = F; - } - patchSled(Sled, Enable, FuncId); - } - __sanitizer::atomic_store(&XRayPatching, false, - __sanitizer::memory_order_release); - PatchingSuccess = true; - return XRayPatchingStatus::SUCCESS; -} - -XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { - return controlPatching(true); -} - -XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { - return controlPatching(false); -} - XRayPatchingStatus patchFunction(int32_t FuncId, bool Enable) XRAY_NEVER_INSTRUMENT { if (!__sanitizer::atomic_load(&XRayInitialized, @@ -303,13 +172,198 @@ XRayPatchingStatus patchFunction(int32_t FuncId, return XRayPatchingStatus::SUCCESS; } +// controlPatching implements the common internals of the patching/unpatching +// implementation. |Enable| defines whether we're enabling or disabling the +// runtime XRay instrumentation. +XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { + if (!__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) + return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. + + uint8_t NotPatching = false; + if (!__sanitizer::atomic_compare_exchange_strong( + &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) + return XRayPatchingStatus::ONGOING; // Already patching. + + uint8_t PatchingSuccess = false; + auto XRayPatchingStatusResetter = + __sanitizer::at_scope_exit([&PatchingSuccess] { + if (!PatchingSuccess) + __sanitizer::atomic_store(&XRayPatching, false, + __sanitizer::memory_order_release); + }); + + XRaySledMap InstrMap; + { + __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); + InstrMap = XRayInstrMap; + } + if (InstrMap.Entries == 0) + return XRayPatchingStatus::NOT_INITIALIZED; + + uint32_t FuncId = 1; + uint64_t CurFun = 0; + + // First we want to find the bounds for which we have instrumentation points, + // and try to get as few calls to mprotect(...) as possible. We're assuming + // that all the sleds for the instrumentation map are contiguous as a single + // set of pages. When we do support dynamic shared object instrumentation, + // we'll need to do this for each set of page load offsets per DSO loaded. For + // now we're assuming we can mprotect the whole section of text between the + // minimum sled address and the maximum sled address (+ the largest sled + // size). + auto MinSled = InstrMap.Sleds[0]; + auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1]; + for (std::size_t I = 0; I < InstrMap.Entries; I++) { + const auto &Sled = InstrMap.Sleds[I]; + if (Sled.Address < MinSled.Address) + MinSled = Sled; + if (Sled.Address > MaxSled.Address) + MaxSled = Sled; + } + + const size_t PageSize = flags()->xray_page_size_override > 0 + ? flags()->xray_page_size_override + : GetPageSizeCached(); + if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { + Report("System page size is not a power of two: %lld\n", PageSize); + return XRayPatchingStatus::FAILED; + } + + void *PageAlignedAddr = + reinterpret_cast(MinSled.Address & ~(PageSize - 1)); + size_t MProtectLen = + (MaxSled.Address - reinterpret_cast(PageAlignedAddr)) + cSledLength; + MProtectHelper Protector(PageAlignedAddr, MProtectLen); + if (Protector.MakeWriteable() == -1) { + Report("Failed mprotect: %d\n", errno); + return XRayPatchingStatus::FAILED; + } + + for (std::size_t I = 0; I < InstrMap.Entries; ++I) { + auto &Sled = InstrMap.Sleds[I]; + auto F = Sled.Function; + if (CurFun == 0) + CurFun = F; + if (F != CurFun) { + ++FuncId; + CurFun = F; + } + patchSled(Sled, Enable, FuncId); + } + __sanitizer::atomic_store(&XRayPatching, false, + __sanitizer::memory_order_release); + PatchingSuccess = true; + return XRayPatchingStatus::SUCCESS; +} + +XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, + bool Enable) XRAY_NEVER_INSTRUMENT { + XRaySledMap InstrMap; + { + __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); + InstrMap = XRayInstrMap; + } + + // FuncId must be a positive number, less than the number of functions + // instrumented. + if (FuncId <= 0 || static_cast(FuncId) > InstrMap.Functions) { + Report("Invalid function id provided: %d\n", FuncId); + return XRayPatchingStatus::FAILED; + } + + const size_t PageSize = flags()->xray_page_size_override > 0 + ? flags()->xray_page_size_override + : GetPageSizeCached(); + if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { + Report("Provided page size is not a power of two: %lld\n", PageSize); + return XRayPatchingStatus::FAILED; + } + + // Here we compute the minumum sled and maximum sled associated with a + // particular function ID. + auto SledRange = InstrMap.SledsIndex[FuncId - 1]; + auto *f = SledRange.Begin; + auto *e = SledRange.End; + auto MinSled = *f; + auto MaxSled = *(SledRange.End - 1); + while (f != e) { + if (f->Address < MinSled.Address) + MinSled = *f; + if (f->Address > MaxSled.Address) + MaxSled = *f; + ++f; + } + + void *PageAlignedAddr = + reinterpret_cast(MinSled.Address & ~(PageSize - 1)); + size_t MProtectLen = + (MaxSled.Address - reinterpret_cast(PageAlignedAddr)) + cSledLength; + MProtectHelper Protector(PageAlignedAddr, MProtectLen); + if (Protector.MakeWriteable() == -1) { + Report("Failed mprotect: %d\n", errno); + return XRayPatchingStatus::FAILED; + } + return patchFunction(FuncId, Enable); +} + +} // namespace + +} // namespace __xray + +using namespace __xray; + +// The following functions are declared `extern "C" {...}` in the header, hence +// they're defined in the global namespace. + +int __xray_set_handler(void (*entry)(int32_t, + XRayEntryType)) XRAY_NEVER_INSTRUMENT { + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) { + + __sanitizer::atomic_store(&__xray::XRayPatchedFunction, + reinterpret_cast(entry), + __sanitizer::memory_order_release); + return 1; + } + return 0; +} + +int __xray_set_customevent_handler(void (*entry)(void *, size_t)) + XRAY_NEVER_INSTRUMENT { + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) { + __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent, + reinterpret_cast(entry), + __sanitizer::memory_order_release); + return 1; + } + return 0; +} + +int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { + return __xray_set_handler(nullptr); +} + +int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { + return __xray_set_customevent_handler(nullptr); +} + +XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { + return controlPatching(true); +} + +XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { + return controlPatching(false); +} + XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { - return patchFunction(FuncId, true); + return mprotectAndPatchFunction(FuncId, true); } XRayPatchingStatus __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { - return patchFunction(FuncId, false); + return mprotectAndPatchFunction(FuncId, false); } int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) { @@ -331,7 +385,7 @@ uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT { __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); if (FuncId <= 0 || static_cast(FuncId) > XRayInstrMap.Functions) return 0; - return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Address + return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function // On PPC, function entries are always aligned to 16 bytes. The beginning of a // sled might be a local entry, which is always +8 based on the global entry. // Always return the global entry. diff --git a/contrib/compiler-rt/lib/xray/xray_interface_internal.h b/contrib/compiler-rt/lib/xray/xray_interface_internal.h index 4a2784612fcb..5811e2b7300a 100644 --- a/contrib/compiler-rt/lib/xray/xray_interface_internal.h +++ b/contrib/compiler-rt/lib/xray/xray_interface_internal.h @@ -28,13 +28,15 @@ struct XRaySledEntry { uint64_t Function; unsigned char Kind; unsigned char AlwaysInstrument; - unsigned char Padding[14]; // Need 32 bytes + unsigned char Version; + unsigned char Padding[13]; // Need 32 bytes #elif SANITIZER_WORDSIZE == 32 uint32_t Address; uint32_t Function; unsigned char Kind; unsigned char AlwaysInstrument; - unsigned char Padding[6]; // Need 16 bytes + unsigned char Version; + unsigned char Padding[5]; // Need 16 bytes #else #error "Unsupported word size." #endif diff --git a/contrib/compiler-rt/lib/xray/xray_log_interface.cc b/contrib/compiler-rt/lib/xray/xray_log_interface.cc index ee14ae4b1b62..783f004d292a 100644 --- a/contrib/compiler-rt/lib/xray/xray_log_interface.cc +++ b/contrib/compiler-rt/lib/xray/xray_log_interface.cc @@ -12,35 +12,84 @@ //===----------------------------------------------------------------------===// #include "xray/xray_log_interface.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_mutex.h" #include "xray/xray_interface.h" #include "xray_defs.h" -#include - __sanitizer::SpinMutex XRayImplMutex; -std::unique_ptr GlobalXRayImpl; +XRayLogImpl CurrentXRayImpl{nullptr, nullptr, nullptr, nullptr}; +XRayLogImpl *GlobalXRayImpl = nullptr; + +// We use a linked list of Mode to XRayLogImpl mappings. This is a linked list +// when it should be a map because we're avoiding having to depend on C++ +// standard library data structures at this level of the implementation. +struct ModeImpl { + ModeImpl *Next; + const char *Mode; + XRayLogImpl Impl; +}; + +ModeImpl SentinelModeImpl{ + nullptr, nullptr, {nullptr, nullptr, nullptr, nullptr}}; +ModeImpl *ModeImpls = &SentinelModeImpl; + +XRayLogRegisterStatus +__xray_log_register_mode(const char *Mode, + XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT { + if (Impl.flush_log == nullptr || Impl.handle_arg0 == nullptr || + Impl.log_finalize == nullptr || Impl.log_init == nullptr) + return XRayLogRegisterStatus::XRAY_INCOMPLETE_IMPL; + + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); + // First, look for whether the mode already has a registered implementation. + for (ModeImpl *it = ModeImpls; it != &SentinelModeImpl; it = it->Next) { + if (!__sanitizer::internal_strcmp(Mode, it->Mode)) + return XRayLogRegisterStatus::XRAY_DUPLICATE_MODE; + } + auto *NewModeImpl = + static_cast(__sanitizer::InternalAlloc(sizeof(ModeImpl))); + NewModeImpl->Next = ModeImpls; + NewModeImpl->Mode = __sanitizer::internal_strdup(Mode); + NewModeImpl->Impl = Impl; + ModeImpls = NewModeImpl; + return XRayLogRegisterStatus::XRAY_REGISTRATION_OK; +} + +XRayLogRegisterStatus +__xray_log_select_mode(const char *Mode) XRAY_NEVER_INSTRUMENT { + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); + for (ModeImpl *it = ModeImpls; it != &SentinelModeImpl; it = it->Next) { + if (!__sanitizer::internal_strcmp(Mode, it->Mode)) { + CurrentXRayImpl = it->Impl; + GlobalXRayImpl = &CurrentXRayImpl; + __xray_set_handler(it->Impl.handle_arg0); + return XRayLogRegisterStatus::XRAY_REGISTRATION_OK; + } + } + return XRayLogRegisterStatus::XRAY_MODE_NOT_FOUND; +} void __xray_set_log_impl(XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT { if (Impl.log_init == nullptr || Impl.log_finalize == nullptr || Impl.handle_arg0 == nullptr || Impl.flush_log == nullptr) { __sanitizer::SpinMutexLock Guard(&XRayImplMutex); - GlobalXRayImpl.reset(); + GlobalXRayImpl = nullptr; __xray_remove_handler(); __xray_remove_handler_arg1(); return; } __sanitizer::SpinMutexLock Guard(&XRayImplMutex); - GlobalXRayImpl.reset(new XRayLogImpl); - *GlobalXRayImpl = Impl; + CurrentXRayImpl = Impl; + GlobalXRayImpl = &CurrentXRayImpl; __xray_set_handler(Impl.handle_arg0); } void __xray_remove_log_impl() XRAY_NEVER_INSTRUMENT { __sanitizer::SpinMutexLock Guard(&XRayImplMutex); - GlobalXRayImpl.reset(); + GlobalXRayImpl = nullptr; __xray_remove_handler(); __xray_remove_handler_arg1(); } diff --git a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S index b59eedc4bb1b..350afd9265fd 100644 --- a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S +++ b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S @@ -14,10 +14,13 @@ //===----------------------------------------------------------------------===// #include "../builtins/assembly.h" +#include "../sanitizer_common/sanitizer_asm.h" + + .macro SAVE_REGISTERS subq $192, %rsp - .cfi_def_cfa_offset 200 + CFI_DEF_CFA_OFFSET(200) // At this point, the stack pointer should be aligned to an 8-byte boundary, // because any call instructions that come after this will add another 8 // bytes and therefore align it to 16-bytes. @@ -57,63 +60,82 @@ movq 8(%rsp), %r8 movq 0(%rsp), %r9 addq $192, %rsp - .cfi_def_cfa_offset 8 + CFI_DEF_CFA_OFFSET(8) +.endm + +.macro ALIGNED_CALL_RAX + // Call the logging handler, after aligning the stack to a 16-byte boundary. + // The approach we're taking here uses additional stack space to stash the + // stack pointer twice before aligning the pointer to 16-bytes. If the stack + // was 8-byte aligned, it will become 16-byte aligned -- when restoring the + // pointer, we can always look -8 bytes from the current position to get + // either of the values we've stashed in the first place. + pushq %rsp + pushq (%rsp) + andq $-0x10, %rsp + callq *%rax + movq 8(%rsp), %rsp .endm .text +#if !defined(__APPLE__) + .section .text +#else + .section __TEXT,__text +#endif .file "xray_trampoline_x86.S" //===----------------------------------------------------------------------===// - .globl __xray_FunctionEntry + .globl ASM_SYMBOL(__xray_FunctionEntry) .align 16, 0x90 - .type __xray_FunctionEntry,@function - -__xray_FunctionEntry: - .cfi_startproc + ASM_TYPE_FUNCTION(__xray_FunctionEntry) +ASM_SYMBOL(__xray_FunctionEntry): + CFI_STARTPROC SAVE_REGISTERS // This load has to be atomic, it's concurrent with __xray_patch(). // On x86/amd64, a simple (type-aligned) MOV instruction is enough. - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax, %rax je .Ltmp0 // The patched function prolog puts its xray_instr_map index into %r10d. movl %r10d, %edi xor %esi,%esi - callq *%rax + ALIGNED_CALL_RAX + .Ltmp0: RESTORE_REGISTERS retq -.Ltmp1: - .size __xray_FunctionEntry, .Ltmp1-__xray_FunctionEntry - .cfi_endproc + ASM_SIZE(__xray_FunctionEntry) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .globl __xray_FunctionExit + .globl ASM_SYMBOL(__xray_FunctionExit) .align 16, 0x90 - .type __xray_FunctionExit,@function -__xray_FunctionExit: - .cfi_startproc + ASM_TYPE_FUNCTION(__xray_FunctionExit) +ASM_SYMBOL(__xray_FunctionExit): + CFI_STARTPROC // Save the important registers first. Since we're assuming that this // function is only jumped into, we only preserve the registers for // returning. subq $56, %rsp - .cfi_def_cfa_offset 64 + CFI_DEF_CFA_OFFSET(64) movq %rbp, 48(%rsp) movupd %xmm0, 32(%rsp) movupd %xmm1, 16(%rsp) movq %rax, 8(%rsp) movq %rdx, 0(%rsp) - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax,%rax je .Ltmp2 movl %r10d, %edi movl $1, %esi - callq *%rax + ALIGNED_CALL_RAX + .Ltmp2: // Restore the important registers. movq 48(%rsp), %rbp @@ -122,111 +144,94 @@ __xray_FunctionExit: movq 8(%rsp), %rax movq 0(%rsp), %rdx addq $56, %rsp - .cfi_def_cfa_offset 8 + CFI_DEF_CFA_OFFSET(8) retq -.Ltmp3: - .size __xray_FunctionExit, .Ltmp3-__xray_FunctionExit - .cfi_endproc + ASM_SIZE(__xray_FunctionExit) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .global __xray_FunctionTailExit + .globl ASM_SYMBOL(__xray_FunctionTailExit) .align 16, 0x90 - .type __xray_FunctionTailExit,@function -__xray_FunctionTailExit: - .cfi_startproc - // Save the important registers as in the entry trampoline, but indicate that - // this is an exit. In the future, we will introduce a new entry type that - // differentiates between a normal exit and a tail exit, but we'd have to do - // this and increment the version number for the header. + ASM_TYPE_FUNCTION(__xray_FunctionTailExit) +ASM_SYMBOL(__xray_FunctionTailExit): + CFI_STARTPROC SAVE_REGISTERS - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax,%rax je .Ltmp4 movl %r10d, %edi - movl $1, %esi - callq *%rax + movl $2, %esi + + ALIGNED_CALL_RAX .Ltmp4: RESTORE_REGISTERS retq -.Ltmp5: - .size __xray_FunctionTailExit, .Ltmp5-__xray_FunctionTailExit - .cfi_endproc + ASM_SIZE(__xray_FunctionTailExit) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .globl __xray_ArgLoggerEntry + .globl ASM_SYMBOL(__xray_ArgLoggerEntry) .align 16, 0x90 - .type __xray_ArgLoggerEntry,@function -__xray_ArgLoggerEntry: - .cfi_startproc + ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry) +ASM_SYMBOL(__xray_ArgLoggerEntry): + CFI_STARTPROC SAVE_REGISTERS // Again, these function pointer loads must be atomic; MOV is fine. - movq _ZN6__xray13XRayArgLoggerE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax testq %rax, %rax jne .Larg1entryLog // If [arg1 logging handler] not set, defer to no-arg logging. - movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax testq %rax, %rax je .Larg1entryFail .Larg1entryLog: - // First argument will become the third + // First argument will become the third movq %rdi, %rdx - // XRayEntryType::ENTRY into the second - xorq %rsi, %rsi + // XRayEntryType::LOG_ARGS_ENTRY into the second + mov $0x3, %esi // 32-bit function ID becomes the first movl %r10d, %edi - callq *%rax + ALIGNED_CALL_RAX .Larg1entryFail: RESTORE_REGISTERS retq - -.Larg1entryEnd: - .size __xray_ArgLoggerEntry, .Larg1entryEnd-__xray_ArgLoggerEntry - .cfi_endproc + ASM_SIZE(__xray_ArgLoggerEntry) + CFI_ENDPROC //===----------------------------------------------------------------------===// - .global __xray_CustomEvent + .global ASM_SYMBOL(__xray_CustomEvent) .align 16, 0x90 - .type __xray_CustomEvent,@function -__xray_CustomEvent: - .cfi_startproc - subq $16, %rsp - .cfi_def_cfa_offset 24 - movq %rbp, 8(%rsp) - movq %rax, 0(%rsp) + ASM_TYPE_FUNCTION(__xray_CustomEvent) +ASM_SYMBOL(__xray_CustomEvent): + CFI_STARTPROC + SAVE_REGISTERS // We take two arguments to this trampoline, which should be in rdi and rsi // already. We also make sure that we stash %rax because we use that register // to call the logging handler. - movq _ZN6__xray22XRayPatchedCustomEventE(%rip), %rax + movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax testq %rax,%rax je .LcustomEventCleanup - // At this point we know that rcx and rdx already has the data, so we just - // call the logging handler. - callq *%rax + ALIGNED_CALL_RAX .LcustomEventCleanup: - movq 0(%rsp), %rax - movq 8(%rsp), %rbp - addq $16, %rsp - .cfi_def_cfa_offset 8 + RESTORE_REGISTERS retq - -.Ltmp8: - .size __xray_CustomEvent, .Ltmp8-__xray_CustomEvent - .cfi_endproc + ASM_SIZE(__xray_CustomEvent) + CFI_ENDPROC NO_EXEC_STACK_DIRECTIVE diff --git a/contrib/compiler-rt/lib/xray/xray_utils.cc b/contrib/compiler-rt/lib/xray/xray_utils.cc index b9a38d1b98eb..cf800d3aeaf8 100644 --- a/contrib/compiler-rt/lib/xray/xray_utils.cc +++ b/contrib/compiler-rt/lib/xray/xray_utils.cc @@ -117,7 +117,8 @@ int getLogFD() XRAY_NEVER_INSTRUMENT { TmpFilename); return -1; } - Report("XRay: Log file in '%s'\n", TmpFilename); + if (__sanitizer::Verbosity()) + Report("XRay: Log file in '%s'\n", TmpFilename); return Fd; } diff --git a/contrib/compiler-rt/lib/xray/xray_x86_64.cc b/contrib/compiler-rt/lib/xray/xray_x86_64.cc index e34806fa1cea..e17f00ac3a62 100644 --- a/contrib/compiler-rt/lib/xray/xray_x86_64.cc +++ b/contrib/compiler-rt/lib/xray/xray_x86_64.cc @@ -76,6 +76,7 @@ static constexpr uint8_t CallOpCode = 0xe8; static constexpr uint16_t MovR10Seq = 0xba41; static constexpr uint16_t Jmp9Seq = 0x09eb; static constexpr uint16_t Jmp20Seq = 0x14eb; +static constexpr uint16_t Jmp15Seq = 0x0feb; static constexpr uint8_t JmpOpCode = 0xe9; static constexpr uint8_t RetOpCode = 0xc3; static constexpr uint16_t NopwSeq = 0x9066; @@ -207,8 +208,10 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { // Here we do the dance of replacing the following sled: // + // In Version 0: + // // xray_sled_n: - // jmp +19 // 2 bytes + // jmp +20 // 2 bytes // ... // // With the following: @@ -216,24 +219,35 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId, // nopw // 2 bytes* // ... // - // We need to do this in the following order: // - // 1. Overwrite the 5-byte nop with the call (relative), where (relative) is - // the relative offset to the __xray_CustomEvent trampoline. - // 2. Do a two-byte atomic write over the 'jmp +24' to turn it into a 'nopw'. - // This allows us to "enable" this code once the changes have committed. + // The "unpatch" should just turn the 'nopw' back to a 'jmp +20'. // - // The "unpatch" should just turn the 'nopw' back to a 'jmp +24'. + // --- + // + // In Version 1: + // + // The jump offset is now 15 bytes (0x0f), so when restoring the nopw back + // to a jmp, use 15 bytes instead. // if (Enable) { std::atomic_store_explicit( reinterpret_cast *>(Sled.Address), NopwSeq, std::memory_order_release); } else { - std::atomic_store_explicit( - reinterpret_cast *>(Sled.Address), Jmp20Seq, - std::memory_order_release); - } + switch (Sled.Version) { + case 1: + std::atomic_store_explicit( + reinterpret_cast *>(Sled.Address), Jmp15Seq, + std::memory_order_release); + break; + case 0: + default: + std::atomic_store_explicit( + reinterpret_cast *>(Sled.Address), Jmp20Seq, + std::memory_order_release); + break; + } + } return false; } @@ -244,9 +258,9 @@ bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { // We check whether rdtscp support is enabled. According to the x86_64 manual, // level should be set at 0x80000001, and we should have a look at bit 27 in - // EDX. That's 0x8000000 (or 1u << 26). + // EDX. That's 0x8000000 (or 1u << 27). __get_cpuid(0x80000001, &EAX, &EBX, &ECX, &EDX); - if (!(EDX & (1u << 26))) { + if (!(EDX & (1u << 27))) { Report("Missing rdtscp support.\n"); return false; } diff --git a/contrib/libc++/CREDITS.TXT b/contrib/libc++/CREDITS.TXT index 92123d736bf0..46a06c6eab08 100644 --- a/contrib/libc++/CREDITS.TXT +++ b/contrib/libc++/CREDITS.TXT @@ -41,6 +41,10 @@ N: Jonathan B Coe E: jbcoe@me.com D: Implementation of propagate_const. +N: Glen Joseph Fernandes +E: glenjofe@gmail.com +D: Implementation of to_address. + N: Eric Fiselier E: eric@efcs.ca D: LFTS support, patches and bug fixes. @@ -97,7 +101,7 @@ E: nico.rieck@gmail.com D: Windows fixes N: Jon Roelofs -E: jonathan@codesourcery.com +E: jroelofS@jroelofs.com D: Remote testing, Newlib port, baremetal/single-threaded support. N: Jonathan Sauer diff --git a/contrib/libc++/include/__config b/contrib/libc++/include/__config index f15d2d06e564..a4dfc41736c9 100644 --- a/contrib/libc++/include/__config +++ b/contrib/libc++/include/__config @@ -33,7 +33,7 @@ #define _GNUC_VER_NEW 0 #endif -#define _LIBCPP_VERSION 5000 +#define _LIBCPP_VERSION 6000 #ifndef _LIBCPP_ABI_VERSION #define _LIBCPP_ABI_VERSION 1 @@ -45,6 +45,8 @@ #define _LIBCPP_OBJECT_FORMAT_MACHO 1 #elif defined(_WIN32) #define _LIBCPP_OBJECT_FORMAT_COFF 1 +#elif defined(__wasm__) +#define _LIBCPP_OBJECT_FORMAT_WASM 1 #else #error Unknown object file format #endif @@ -55,11 +57,11 @@ #define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT // Fix deque iterator type in order to support incomplete types. #define _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE -// Fix undefined behavior in how std::list stores it's linked nodes. +// Fix undefined behavior in how std::list stores its linked nodes. #define _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB // Fix undefined behavior in how __tree stores its end and parent nodes. #define _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB -// Fix undefined behavior in how __hash_table stores it's pointer types +// Fix undefined behavior in how __hash_table stores its pointer types. #define _LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB #define _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB #define _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE @@ -76,9 +78,11 @@ // its vtable and typeinfo to libc++ rather than having all other libraries // using that class define their own copies. #define _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION - // Enable optimized version of __do_get_(un)signed which avoids redundant copies. #define _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET +// Use the smallest possible integer type to represent the index of the variant. +// Previously libc++ used "unsigned int" exclusivly. +#define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION #elif _LIBCPP_ABI_VERSION == 1 #if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support @@ -121,6 +125,9 @@ #ifndef __has_feature #define __has_feature(__x) 0 #endif +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(__x) 0 +#endif // '__is_identifier' returns '0' if '__x' is a reserved identifier provided by // the compiler and '1' otherwise. #ifndef __is_identifier @@ -157,11 +164,21 @@ // FIXME: ABI detection should be done via compiler builtin macros. This // is just a placeholder until Clang implements such macros. For now assume -// that Windows compilers pretending to be MSVC++ target the microsoft ABI. -#if defined(_WIN32) && defined(_MSC_VER) +// that Windows compilers pretending to be MSVC++ target the Microsoft ABI, +// and allow the user to explicitly specify the ABI to handle cases where this +// heuristic falls short. +#if defined(_LIBCPP_ABI_FORCE_ITANIUM) && defined(_LIBCPP_ABI_FORCE_MICROSOFT) +# error "Only one of _LIBCPP_ABI_FORCE_ITANIUM and _LIBCPP_ABI_FORCE_MICROSOFT can be defined" +#elif defined(_LIBCPP_ABI_FORCE_ITANIUM) +# define _LIBCPP_ABI_ITANIUM +#elif defined(_LIBCPP_ABI_FORCE_MICROSOFT) # define _LIBCPP_ABI_MICROSOFT #else -# define _LIBCPP_ABI_ITANIUM +# if defined(_WIN32) && defined(_MSC_VER) +# define _LIBCPP_ABI_MICROSOFT +# else +# define _LIBCPP_ABI_ITANIUM +# endif #endif // Need to detect which libc we're using if we're on Linux. @@ -174,36 +191,30 @@ #ifdef __LITTLE_ENDIAN__ #if __LITTLE_ENDIAN__ -#define _LIBCPP_LITTLE_ENDIAN 1 -#define _LIBCPP_BIG_ENDIAN 0 +#define _LIBCPP_LITTLE_ENDIAN #endif // __LITTLE_ENDIAN__ #endif // __LITTLE_ENDIAN__ #ifdef __BIG_ENDIAN__ #if __BIG_ENDIAN__ -#define _LIBCPP_LITTLE_ENDIAN 0 -#define _LIBCPP_BIG_ENDIAN 1 +#define _LIBCPP_BIG_ENDIAN #endif // __BIG_ENDIAN__ #endif // __BIG_ENDIAN__ #ifdef __BYTE_ORDER__ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define _LIBCPP_LITTLE_ENDIAN 1 -#define _LIBCPP_BIG_ENDIAN 0 +#define _LIBCPP_LITTLE_ENDIAN #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define _LIBCPP_LITTLE_ENDIAN 0 -#define _LIBCPP_BIG_ENDIAN 1 +#define _LIBCPP_BIG_ENDIAN #endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #endif // __BYTE_ORDER__ #ifdef __FreeBSD__ # include # if _BYTE_ORDER == _LITTLE_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 1 -# define _LIBCPP_BIG_ENDIAN 0 +# define _LIBCPP_LITTLE_ENDIAN # else // _BYTE_ORDER == _LITTLE_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 0 -# define _LIBCPP_BIG_ENDIAN 1 +# define _LIBCPP_BIG_ENDIAN # endif // _BYTE_ORDER == _LITTLE_ENDIAN # ifndef __LONG_LONG_SUPPORTED # define _LIBCPP_HAS_NO_LONG_LONG @@ -213,19 +224,16 @@ #ifdef __NetBSD__ # include # if _BYTE_ORDER == _LITTLE_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 1 -# define _LIBCPP_BIG_ENDIAN 0 +# define _LIBCPP_LITTLE_ENDIAN # else // _BYTE_ORDER == _LITTLE_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 0 -# define _LIBCPP_BIG_ENDIAN 1 +# define _LIBCPP_BIG_ENDIAN # endif // _BYTE_ORDER == _LITTLE_ENDIAN # define _LIBCPP_HAS_QUICK_EXIT #endif // __NetBSD__ #if defined(_WIN32) # define _LIBCPP_WIN32API -# define _LIBCPP_LITTLE_ENDIAN 1 -# define _LIBCPP_BIG_ENDIAN 0 +# define _LIBCPP_LITTLE_ENDIAN # define _LIBCPP_SHORT_WCHAR 1 // Both MinGW and native MSVC provide a "MSVC"-like enviroment # define _LIBCPP_MSVCRT_LIKE @@ -255,11 +263,9 @@ #ifdef __sun__ # include # ifdef _LITTLE_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 1 -# define _LIBCPP_BIG_ENDIAN 0 +# define _LIBCPP_LITTLE_ENDIAN # else -# define _LIBCPP_LITTLE_ENDIAN 0 -# define _LIBCPP_BIG_ENDIAN 1 +# define _LIBCPP_BIG_ENDIAN # endif #endif // __sun__ @@ -269,6 +275,8 @@ // random data even when using sandboxing mechanisms such as chroots, // Capsicum, etc. # define _LIBCPP_USING_ARC4_RANDOM +#elif defined(__Fuchsia__) +# define _LIBCPP_USING_GETENTROPY #elif defined(__native_client__) // NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access, // including accesses to the special files under /dev. C++11's @@ -280,18 +288,16 @@ # define _LIBCPP_USING_DEV_RANDOM #endif -#if !defined(_LIBCPP_LITTLE_ENDIAN) || !defined(_LIBCPP_BIG_ENDIAN) +#if !defined(_LIBCPP_LITTLE_ENDIAN) && !defined(_LIBCPP_BIG_ENDIAN) # include # if __BYTE_ORDER == __LITTLE_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 1 -# define _LIBCPP_BIG_ENDIAN 0 +# define _LIBCPP_LITTLE_ENDIAN # elif __BYTE_ORDER == __BIG_ENDIAN -# define _LIBCPP_LITTLE_ENDIAN 0 -# define _LIBCPP_BIG_ENDIAN 1 +# define _LIBCPP_BIG_ENDIAN # else // __BYTE_ORDER == __BIG_ENDIAN # error unable to determine endian # endif -#endif // !defined(_LIBCPP_LITTLE_ENDIAN) || !defined(_LIBCPP_BIG_ENDIAN) +#endif // !defined(_LIBCPP_LITTLE_ENDIAN) && !defined(_LIBCPP_BIG_ENDIAN) #if __has_attribute(__no_sanitize__) && !defined(_LIBCPP_COMPILER_GCC) #define _LIBCPP_NO_CFI __attribute__((__no_sanitize__("cfi"))) @@ -454,6 +460,14 @@ namespace std { #define _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK __attribute__((__no_sanitize__("unsigned-integer-overflow"))) #endif +#if __has_builtin(__builtin_launder) +#define _LIBCPP_COMPILER_HAS_BUILTIN_LAUNDER +#endif + +#if !__is_identifier(__has_unique_object_representations) +#define _LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS +#endif + #elif defined(_LIBCPP_COMPILER_GCC) #define _ALIGNAS(x) __attribute__((__aligned__(x))) @@ -536,6 +550,14 @@ namespace std { #define _LIBCPP_HAS_NO_ASAN #endif +#if _GNUC_VER >= 700 +#define _LIBCPP_COMPILER_HAS_BUILTIN_LAUNDER +#endif + +#if _GNUC_VER >= 700 +#define _LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS +#endif + #elif defined(_LIBCPP_COMPILER_MSVC) #define _LIBCPP_TOSTRING2(x) #x @@ -880,7 +902,7 @@ template struct __static_assert_check {}; #define _LIBCPP_NONUNIQUE_RTTI_BIT (1ULL << 63) #endif -#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_LIBCPP_MSVCRT) || \ +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_LIBCPP_MSVCRT_LIKE) || \ defined(__sun__) || defined(__NetBSD__) || defined(__CloudABI__) #define _LIBCPP_LOCALE__L_EXTENSIONS 1 #endif @@ -954,9 +976,22 @@ template struct __static_assert_check {}; #define _LIBCPP_CONSTEXPR_AFTER_CXX14 #endif -// FIXME: Remove all usages of this macro once compilers catch up. -#if !defined(__cpp_inline_variables) || (__cpp_inline_variables < 201606L) -# define _LIBCPP_HAS_NO_INLINE_VARIABLES +#if _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_HAS_NO_CXX14_CONSTEXPR) +#define _LIBCPP_CONSTEXPR_AFTER_CXX17 constexpr +#else +#define _LIBCPP_CONSTEXPR_AFTER_CXX17 +#endif + +#if __has_cpp_attribute(nodiscard) && _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_NODISCARD_AFTER_CXX17) +#define _LIBCPP_NODISCARD_AFTER_CXX17 [[nodiscard]] +#else +#define _LIBCPP_NODISCARD_AFTER_CXX17 +#endif + +#if _LIBCPP_STD_VER > 14 && defined(__cpp_inline_variables) && (__cpp_inline_variables >= 201606L) +# define _LIBCPP_INLINE_VAR inline +#else +# define _LIBCPP_INLINE_VAR #endif #ifdef _LIBCPP_HAS_NO_RVALUE_REFERENCES @@ -1241,6 +1276,13 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( # endif #endif // defined(_LIBCPP_HAS_NO_PRAGMA_PUSH_POP_MACRO) +#if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_BUILDING_LIBRARY) +# if defined(_DLL) +# pragma(lib, "c++.lib") +# else +# pragma(lib, "libc++.lib") +# endif +#endif // defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_BUILDING_LIBRARY) #endif // __cplusplus diff --git a/contrib/libc++/include/__functional_base b/contrib/libc++/include/__functional_base index 79017fe759b4..12af4dc96751 100644 --- a/contrib/libc++/include/__functional_base +++ b/contrib/libc++/include/__functional_base @@ -564,7 +564,7 @@ struct _LIBCPP_TEMPLATE_VIS allocator_arg_t { }; #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_MEMORY) extern const allocator_arg_t allocator_arg; #else -constexpr allocator_arg_t allocator_arg = allocator_arg_t(); +/* _LIBCPP_INLINE_VAR */ constexpr allocator_arg_t allocator_arg = allocator_arg_t(); #endif // uses_allocator @@ -601,7 +601,7 @@ struct _LIBCPP_TEMPLATE_VIS uses_allocator #if _LIBCPP_STD_VER > 14 template -constexpr size_t uses_allocator_v = uses_allocator<_Tp, _Alloc>::value; +_LIBCPP_INLINE_VAR constexpr size_t uses_allocator_v = uses_allocator<_Tp, _Alloc>::value; #endif #ifndef _LIBCPP_CXX03_LANG diff --git a/contrib/libc++/include/__libcpp_version b/contrib/libc++/include/__libcpp_version index e9c02dad1826..a77fd92cf17b 100644 --- a/contrib/libc++/include/__libcpp_version +++ b/contrib/libc++/include/__libcpp_version @@ -1 +1 @@ -5000 +6000 diff --git a/contrib/libc++/include/__locale b/contrib/libc++/include/__locale index 91ed9e709ee3..601f0d1ec325 100644 --- a/contrib/libc++/include/__locale +++ b/contrib/libc++/include/__locale @@ -49,7 +49,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_LOCALE__L_EXTENSIONS) || defined(_LIBCPP_MSVCRT) +#if !defined(_LIBCPP_LOCALE__L_EXTENSIONS) struct __libcpp_locale_guard { _LIBCPP_INLINE_VISIBILITY __libcpp_locale_guard(locale_t& __loc) : __old_loc_(uselocale(__loc)) {} @@ -65,6 +65,32 @@ private: __libcpp_locale_guard(__libcpp_locale_guard const&); __libcpp_locale_guard& operator=(__libcpp_locale_guard const&); }; +#elif defined(_LIBCPP_MSVCRT_LIKE) +struct __libcpp_locale_guard { + __libcpp_locale_guard(locale_t __l) : + __status(_configthreadlocale(_ENABLE_PER_THREAD_LOCALE)), + __locale_collate(setlocale(LC_COLLATE, __l.__get_locale())), + __locale_ctype(setlocale(LC_CTYPE, __l.__get_locale())), + __locale_monetary(setlocale(LC_MONETARY, __l.__get_locale())), + __locale_numeric(setlocale(LC_NUMERIC, __l.__get_locale())), + __locale_time(setlocale(LC_TIME, __l.__get_locale())) + // LC_MESSAGES is not supported on Windows. + {} + ~__libcpp_locale_guard() { + setlocale(LC_COLLATE, __locale_collate); + setlocale(LC_CTYPE, __locale_ctype); + setlocale(LC_MONETARY, __locale_monetary); + setlocale(LC_NUMERIC, __locale_numeric); + setlocale(LC_TIME, __locale_time); + _configthreadlocale(__status); + } + int __status; + char* __locale_collate; + char* __locale_ctype; + char* __locale_monetary; + char* __locale_numeric; + char* __locale_time; +}; #endif diff --git a/contrib/libc++/include/__mutex_base b/contrib/libc++/include/__mutex_base index 3b2453f1ba1a..402a52d945e1 100644 --- a/contrib/libc++/include/__mutex_base +++ b/contrib/libc++/include/__mutex_base @@ -82,9 +82,9 @@ extern const adopt_lock_t adopt_lock; #else -constexpr defer_lock_t defer_lock = defer_lock_t(); -constexpr try_to_lock_t try_to_lock = try_to_lock_t(); -constexpr adopt_lock_t adopt_lock = adopt_lock_t(); +/* _LIBCPP_INLINE_VAR */ constexpr defer_lock_t defer_lock = defer_lock_t(); +/* _LIBCPP_INLINE_VAR */ constexpr try_to_lock_t try_to_lock = try_to_lock_t(); +/* _LIBCPP_INLINE_VAR */ constexpr adopt_lock_t adopt_lock = adopt_lock_t(); #endif diff --git a/contrib/libc++/include/__tree b/contrib/libc++/include/__tree index 792870aa6ebc..3ccfcb700310 100644 --- a/contrib/libc++/include/__tree +++ b/contrib/libc++/include/__tree @@ -84,7 +84,7 @@ __tree_is_left_child(_NodePtr __x) _NOEXCEPT return __x == __x->__parent_->__left_; } -// Determintes if the subtree rooted at __x is a proper red black subtree. If +// Determines if the subtree rooted at __x is a proper red black subtree. If // __x is a proper subtree, returns the black height (null counts as 1). If // __x is an improper subtree, returns 0. template @@ -119,7 +119,7 @@ __tree_sub_invariant(_NodePtr __x) return __h + __x->__is_black_; // return black height of this node } -// Determintes if the red black tree rooted at __root is a proper red black tree. +// Determines if the red black tree rooted at __root is a proper red black tree. // __root == nullptr is a proper tree. Returns true is __root is a proper // red black tree, else returns false. template diff --git a/contrib/libc++/include/algorithm b/contrib/libc++/include/algorithm index 00db6d7e7c87..35c6129ea508 100644 --- a/contrib/libc++/include/algorithm +++ b/contrib/libc++/include/algorithm @@ -734,15 +734,15 @@ struct __less<_T1, const _T1> }; template -class __negate +class __invert // invert the sense of a comparison { private: _Predicate __p_; public: - _LIBCPP_INLINE_VISIBILITY __negate() {} + _LIBCPP_INLINE_VISIBILITY __invert() {} _LIBCPP_INLINE_VISIBILITY - explicit __negate(_Predicate __p) : __p_(__p) {} + explicit __invert(_Predicate __p) : __p_(__p) {} template _LIBCPP_INLINE_VISIBILITY @@ -750,7 +750,7 @@ public: template _LIBCPP_INLINE_VISIBILITY - bool operator()(const _T1& __x, const _T2& __y) {return !__p_(__x, __y);} + bool operator()(const _T1& __x, const _T2& __y) {return __p_(__y, __x);} }; #ifdef _LIBCPP_DEBUG @@ -797,7 +797,7 @@ unsigned __ctz(unsigned __x) { unsigned long where; // Search from LSB to MSB for first set bit. // Returns zero if no set bit is found. - if (_BitScanForward(&where, mask)) + if (_BitScanForward(&where, __x)) return where; return 32; #endif @@ -823,15 +823,15 @@ unsigned long long __ctz(unsigned long long __x) { // Returns zero if no set bit is found. #if defined(_LIBCPP_HAS_BITSCAN64) (defined(_M_AMD64) || defined(__x86_64__)) - if (_BitScanForward64(&where, mask)) + if (_BitScanForward64(&where, __x)) return static_cast(where); #else // Win32 doesn't have _BitScanForward64 so emulate it with two 32 bit calls. // Scan the Low Word. - if (_BitScanForward(&where, static_cast(mask))) + if (_BitScanForward(&where, static_cast(__x))) return where; // Scan the High Word. - if (_BitScanForward(&where, static_cast(mask >> 32))) + if (_BitScanForward(&where, static_cast(__x >> 32))) return where + 32; // Create a bit offset from the LSB. #endif return 64; @@ -849,7 +849,7 @@ unsigned __clz(unsigned __x) { unsigned long where; // Search from LSB to MSB for first set bit. // Returns zero if no set bit is found. - if (_BitScanReverse(&where, mask)) + if (_BitScanReverse(&where, __x)) return 31 - where; return 32; // Undefined Behavior. #endif @@ -874,14 +874,14 @@ unsigned long long __clz(unsigned long long __x) { // BitScanReverse scans from MSB to LSB for first set bit. // Returns 0 if no set bit is found. #if defined(_LIBCPP_HAS_BITSCAN64) - if (_BitScanReverse64(&where, mask)) + if (_BitScanReverse64(&where, __x)) return static_cast(63 - where); #else // Scan the high 32 bits. - if (_BitScanReverse(&where, static_cast(mask >> 32))) + if (_BitScanReverse(&where, static_cast(__x >> 32))) return 63 - (where + 32); // Create a bit offset from the MSB. // Scan the low 32 bits. - if (_BitScanReverse(&where, static_cast(mask))) + if (_BitScanReverse(&where, static_cast(__x))) return 63 - where; #endif return 64; // Undefined Behavior. @@ -3210,28 +3210,28 @@ template _LIBCPP_INLINE_VISIBILITY _SampleIterator __sample(_PopulationIterator __first, - _PopulationIterator __last, _SampleIterator __output, + _PopulationIterator __last, _SampleIterator __output_iter, _Distance __n, _UniformRandomNumberGenerator & __g, input_iterator_tag) { _Distance __k = 0; for (; __first != __last && __k < __n; ++__first, (void)++__k) - __output[__k] = *__first; + __output_iter[__k] = *__first; _Distance __sz = __k; for (; __first != __last; ++__first, (void)++__k) { _Distance __r = _VSTD::uniform_int_distribution<_Distance>(0, __k)(__g); if (__r < __sz) - __output[__r] = *__first; + __output_iter[__r] = *__first; } - return __output + _VSTD::min(__n, __k); + return __output_iter + _VSTD::min(__n, __k); } template _LIBCPP_INLINE_VISIBILITY _SampleIterator __sample(_PopulationIterator __first, - _PopulationIterator __last, _SampleIterator __output, + _PopulationIterator __last, _SampleIterator __output_iter, _Distance __n, _UniformRandomNumberGenerator& __g, forward_iterator_tag) { @@ -3240,18 +3240,18 @@ _SampleIterator __sample(_PopulationIterator __first, _Distance __r = _VSTD::uniform_int_distribution<_Distance>(0, --__unsampled_sz)(__g); if (__r < __n) { - *__output++ = *__first; + *__output_iter++ = *__first; --__n; } } - return __output; + return __output_iter; } template _LIBCPP_INLINE_VISIBILITY _SampleIterator __sample(_PopulationIterator __first, - _PopulationIterator __last, _SampleIterator __output, + _PopulationIterator __last, _SampleIterator __output_iter, _Distance __n, _UniformRandomNumberGenerator& __g) { typedef typename iterator_traits<_PopulationIterator>::iterator_category _PopCategory; @@ -3263,7 +3263,7 @@ _SampleIterator __sample(_PopulationIterator __first, typedef typename common_type<_Distance, _Difference>::type _CommonType; _LIBCPP_ASSERT(__n >= 0, "N must be a positive number."); return _VSTD::__sample( - __first, __last, __output, _CommonType(__n), + __first, __last, __output_iter, _CommonType(__n), __g, _PopCategory()); } @@ -3272,9 +3272,9 @@ template inline _LIBCPP_INLINE_VISIBILITY _SampleIterator sample(_PopulationIterator __first, - _PopulationIterator __last, _SampleIterator __output, + _PopulationIterator __last, _SampleIterator __output_iter, _Distance __n, _UniformRandomNumberGenerator&& __g) { - return _VSTD::__sample(__first, __last, __output, __n, __g); + return _VSTD::__sample(__first, __last, __output_iter, __n, __g); } #endif // _LIBCPP_STD_VER > 14 @@ -4568,7 +4568,7 @@ __buffered_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator typedef reverse_iterator _Rv; __half_inplace_merge(_Rv(__p), _Rv(__buff), _RBi(__middle), _RBi(__first), - _RBi(__last), __negate<_Compare>(__comp)); + _RBi(__last), __invert<_Compare>(__comp)); } } @@ -5547,9 +5547,9 @@ __set_union(_InputIterator1 __first1, _InputIterator1 __last1, } else { - *__result = *__first1; if (!__comp(*__first1, *__first2)) ++__first2; + *__result = *__first1; ++__first1; } } diff --git a/contrib/libc++/include/array b/contrib/libc++/include/array index 12780992e5c4..4eb2fe6fc624 100644 --- a/contrib/libc++/include/array +++ b/contrib/libc++/include/array @@ -183,7 +183,7 @@ struct _LIBCPP_TEMPLATE_VIS array _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT {return _Size;} _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT {return _Size;} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT {return _Size == 0;} // element access: diff --git a/contrib/libc++/include/bitset b/contrib/libc++/include/bitset index 583122fbfdab..cd6c289b4c48 100644 --- a/contrib/libc++/include/bitset +++ b/contrib/libc++/include/bitset @@ -235,8 +235,13 @@ void __bitset<_N_words, _Size>::__init(unsigned long long __v, false_type) _NOEXCEPT { __storage_type __t[sizeof(unsigned long long) / sizeof(__storage_type)]; - for (size_t __i = 0; __i < sizeof(__t)/sizeof(__t[0]); ++__i, __v >>= __bits_per_word) - __t[__i] = static_cast<__storage_type>(__v); + size_t __sz = _Size; + for (size_t __i = 0; __i < sizeof(__t)/sizeof(__t[0]); ++__i, __v >>= __bits_per_word, __sz -= __bits_per_word ) + if ( __sz < __bits_per_word) + __t[__i] = static_cast<__storage_type>(__v) & ( 1ULL << __sz ) - 1; + else + __t[__i] = static_cast<__storage_type>(__v); + _VSTD::copy(__t, __t + sizeof(__t)/sizeof(__t[0]), __first_); _VSTD::fill(__first_ + sizeof(__t)/sizeof(__t[0]), __first_ + sizeof(__first_)/sizeof(__first_[0]), __storage_type(0)); @@ -248,6 +253,9 @@ void __bitset<_N_words, _Size>::__init(unsigned long long __v, true_type) _NOEXCEPT { __first_[0] = __v; + if (_Size < __bits_per_word) + __first_[0] &= ( 1ULL << _Size ) - 1; + _VSTD::fill(__first_ + 1, __first_ + sizeof(__first_)/sizeof(__first_[0]), __storage_type(0)); } @@ -261,7 +269,9 @@ __bitset<_N_words, _Size>::__bitset(unsigned long long __v) _NOEXCEPT #if __SIZEOF_SIZE_T__ == 8 : __first_{__v} #elif __SIZEOF_SIZE_T__ == 4 - : __first_{static_cast<__storage_type>(__v), static_cast<__storage_type>(__v >> __bits_per_word)} + : __first_{static_cast<__storage_type>(__v), + _Size >= 2 * __bits_per_word ? static_cast<__storage_type>(__v >> __bits_per_word) + : static_cast<__storage_type>((__v >> __bits_per_word) & (__storage_type(1) << (_Size - __bits_per_word)) - 1)} #else #error This constructor has not been ported to this platform #endif @@ -503,7 +513,10 @@ template inline _LIBCPP_CONSTEXPR __bitset<1, _Size>::__bitset(unsigned long long __v) _NOEXCEPT - : __first_(static_cast<__storage_type>(__v)) + : __first_( + _Size == __bits_per_word ? static_cast<__storage_type>(__v) + : static_cast<__storage_type>(__v) & ((__storage_type(1) << _Size) - 1) + ) { } diff --git a/contrib/libc++/include/chrono b/contrib/libc++/include/chrono index 4b793c760559..809f773b6c6b 100644 --- a/contrib/libc++/include/chrono +++ b/contrib/libc++/include/chrono @@ -26,7 +26,7 @@ duration_cast(const duration& fd); template struct treat_as_floating_point : is_floating_point {}; -template constexpr bool treat_as_floating_point_v +template inline constexpr bool treat_as_floating_point_v = treat_as_floating_point::value; // C++17 template @@ -283,18 +283,18 @@ typedef steady_clock high_resolution_clock; } // chrono -constexpr chrono::hours operator "" h(unsigned long long); // C++14 -constexpr chrono::duration> operator "" h(long double); // C++14 -constexpr chrono::minutes operator "" min(unsigned long long); // C++14 -constexpr chrono::duration> operator "" min(long double); // C++14 -constexpr chrono::seconds operator "" s(unsigned long long); // C++14 -constexpr chrono::duration operator "" s(long double); // C++14 -constexpr chrono::milliseconds operator "" ms(unsigned long long); // C++14 -constexpr chrono::duration operator "" ms(long double); // C++14 -constexpr chrono::microseconds operator "" us(unsigned long long); // C++14 -constexpr chrono::duration operator "" us(long double); // C++14 -constexpr chrono::nanoseconds operator "" ns(unsigned long long); // C++14 -constexpr chrono::duration operator "" ns(long double); // C++14 +constexpr chrono::hours operator ""h(unsigned long long); // C++14 +constexpr chrono::duration> operator ""h(long double); // C++14 +constexpr chrono::minutes operator ""min(unsigned long long); // C++14 +constexpr chrono::duration> operator ""min(long double); // C++14 +constexpr chrono::seconds operator ""s(unsigned long long); // C++14 +constexpr chrono::duration operator ""s(long double); // C++14 +constexpr chrono::milliseconds operator ""ms(unsigned long long); // C++14 +constexpr chrono::duration operator ""ms(long double); // C++14 +constexpr chrono::microseconds operator ""us(unsigned long long); // C++14 +constexpr chrono::duration operator ""us(long double); // C++14 +constexpr chrono::nanoseconds operator ""ns(unsigned long long); // C++14 +constexpr chrono::duration operator ""ns(long double); // C++14 } // std */ @@ -419,7 +419,8 @@ template struct _LIBCPP_TEMPLATE_VIS treat_as_floating_point : is_floating_point<_Rep> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool treat_as_floating_point_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool treat_as_floating_point_v = treat_as_floating_point<_Rep>::value; #endif @@ -1087,67 +1088,67 @@ inline namespace literals inline namespace chrono_literals { - constexpr chrono::hours operator"" h(unsigned long long __h) + constexpr chrono::hours operator""h(unsigned long long __h) { return chrono::hours(static_cast(__h)); } - constexpr chrono::duration> operator"" h(long double __h) + constexpr chrono::duration> operator""h(long double __h) { return chrono::duration>(__h); } - constexpr chrono::minutes operator"" min(unsigned long long __m) + constexpr chrono::minutes operator""min(unsigned long long __m) { return chrono::minutes(static_cast(__m)); } - constexpr chrono::duration> operator"" min(long double __m) + constexpr chrono::duration> operator""min(long double __m) { return chrono::duration> (__m); } - constexpr chrono::seconds operator"" s(unsigned long long __s) + constexpr chrono::seconds operator""s(unsigned long long __s) { return chrono::seconds(static_cast(__s)); } - constexpr chrono::duration operator"" s(long double __s) + constexpr chrono::duration operator""s(long double __s) { return chrono::duration (__s); } - constexpr chrono::milliseconds operator"" ms(unsigned long long __ms) + constexpr chrono::milliseconds operator""ms(unsigned long long __ms) { return chrono::milliseconds(static_cast(__ms)); } - constexpr chrono::duration operator"" ms(long double __ms) + constexpr chrono::duration operator""ms(long double __ms) { return chrono::duration(__ms); } - constexpr chrono::microseconds operator"" us(unsigned long long __us) + constexpr chrono::microseconds operator""us(unsigned long long __us) { return chrono::microseconds(static_cast(__us)); } - constexpr chrono::duration operator"" us(long double __us) + constexpr chrono::duration operator""us(long double __us) { return chrono::duration (__us); } - constexpr chrono::nanoseconds operator"" ns(unsigned long long __ns) + constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) { return chrono::nanoseconds(static_cast(__ns)); } - constexpr chrono::duration operator"" ns(long double __ns) + constexpr chrono::duration operator""ns(long double __ns) { return chrono::duration (__ns); } diff --git a/contrib/libc++/include/cstddef b/contrib/libc++/include/cstddef index 62584494d97c..adeefdac9be6 100644 --- a/contrib/libc++/include/cstddef +++ b/contrib/libc++/include/cstddef @@ -64,23 +64,46 @@ namespace std // purposefully not versioned { enum class byte : unsigned char {}; -constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept -{ return __lhs = byte(static_cast(__lhs) | static_cast(__rhs)); } constexpr byte operator| (byte __lhs, byte __rhs) noexcept -{ return byte(static_cast(__lhs) | static_cast(__rhs)); } +{ + return static_cast( + static_cast( + static_cast(__lhs) | static_cast(__rhs) + )); +} + +constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept +{ return __lhs = __lhs | __rhs; } + +constexpr byte operator& (byte __lhs, byte __rhs) noexcept +{ + return static_cast( + static_cast( + static_cast(__lhs) & static_cast(__rhs) + )); +} constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept -{ return __lhs = byte(static_cast(__lhs) & static_cast(__rhs)); } -constexpr byte operator& (byte __lhs, byte __rhs) noexcept -{ return byte(static_cast(__lhs) & static_cast(__rhs)); } +{ return __lhs = __lhs & __rhs; } + +constexpr byte operator^ (byte __lhs, byte __rhs) noexcept +{ + return static_cast( + static_cast( + static_cast(__lhs) ^ static_cast(__rhs) + )); +} constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept -{ return __lhs = byte(static_cast(__lhs) ^ static_cast(__rhs)); } -constexpr byte operator^ (byte __lhs, byte __rhs) noexcept -{ return byte(static_cast(__lhs) ^ static_cast(__rhs)); } +{ return __lhs = __lhs ^ __rhs; } constexpr byte operator~ (byte __b) noexcept -{ return byte(~static_cast(__b)); } +{ + return static_cast( + static_cast( + ~static_cast(__b) + )); +} } diff --git a/contrib/libc++/include/deque b/contrib/libc++/include/deque index fee75614b97f..08cb295408be 100644 --- a/contrib/libc++/include/deque +++ b/contrib/libc++/include/deque @@ -1314,7 +1314,7 @@ public: void resize(size_type __n); void resize(size_type __n, const value_type& __v); void shrink_to_fit() _NOEXCEPT; - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __base::size() == 0;} // element access: diff --git a/contrib/libc++/include/exception b/contrib/libc++/include/exception index ca2eaf5c6a04..79bd6ac2ae35 100644 --- a/contrib/libc++/include/exception +++ b/contrib/libc++/include/exception @@ -82,7 +82,7 @@ template void rethrow_if_nested(const E& e); #include #include -#if defined(_LIBCPP_ABI_MICROSOFT) +#if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_NO_VCRUNTIME) #include #endif @@ -93,7 +93,7 @@ template void rethrow_if_nested(const E& e); namespace std // purposefully not using versioning namespace { -#if !defined(_LIBCPP_ABI_MICROSOFT) +#if !defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME) class _LIBCPP_EXCEPTION_ABI exception { public: @@ -110,7 +110,7 @@ public: virtual ~bad_exception() _NOEXCEPT; virtual const char* what() const _NOEXCEPT; }; -#endif // !_LIBCPP_ABI_MICROSOFT +#endif // !_LIBCPP_ABI_MICROSOFT || _LIBCPP_NO_VCRUNTIME #if _LIBCPP_STD_VER <= 14 \ || defined(_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS) \ diff --git a/contrib/libc++/include/experimental/algorithm b/contrib/libc++/include/experimental/algorithm index 3801cae4f16e..a6a28f07181b 100644 --- a/contrib/libc++/include/experimental/algorithm +++ b/contrib/libc++/include/experimental/algorithm @@ -61,9 +61,9 @@ template inline _LIBCPP_INLINE_VISIBILITY _SampleIterator sample(_PopulationIterator __first, _PopulationIterator __last, - _SampleIterator __output, _Distance __n, + _SampleIterator __output_iter, _Distance __n, _UniformRandomNumberGenerator &&__g) { - return _VSTD::__sample(__first, __last, __output, __n, __g); + return _VSTD::__sample(__first, __last, __output_iter, __n, __g); } _LIBCPP_END_NAMESPACE_LFTS diff --git a/contrib/libc++/include/experimental/filesystem b/contrib/libc++/include/experimental/filesystem index 42157ba309bc..674490f60381 100644 --- a/contrib/libc++/include/experimental/filesystem +++ b/contrib/libc++/include/experimental/filesystem @@ -81,10 +81,10 @@ path canonical(const path& p, const path& base, error_code& ec); void copy(const path& from, const path& to); - void copy(const path& from, const path& to, error_code& ec) _NOEXCEPT; + void copy(const path& from, const path& to, error_code& ec); void copy(const path& from, const path& to, copy_options options); void copy(const path& from, const path& to, copy_options options, - error_code& ec) _NOEXCEPT; + error_code& ec); bool copy_file(const path& from, const path& to); bool copy_file(const path& from, const path& to, error_code& ec) _NOEXCEPT; @@ -972,7 +972,8 @@ public: _LIBCPP_INLINE_VISIBILITY path extension() const { return string_type(__extension()); } // query - _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT { return __pn_.empty(); } + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY + bool empty() const _NOEXCEPT { return __pn_.empty(); } _LIBCPP_INLINE_VISIBILITY bool has_root_name() const { return !__root_name().empty(); } _LIBCPP_INLINE_VISIBILITY bool has_root_directory() const { return !__root_directory().empty(); } @@ -1351,7 +1352,7 @@ void copy(const path& __from, const path& __to) { } inline _LIBCPP_INLINE_VISIBILITY -void copy(const path& __from, const path& __to, error_code& __ec) _NOEXCEPT { +void copy(const path& __from, const path& __to, error_code& __ec) { __copy(__from, __to, copy_options::none, &__ec); } @@ -1362,7 +1363,7 @@ void copy(const path& __from, const path& __to, copy_options __opt) { inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to, - copy_options __opt, error_code& __ec) _NOEXCEPT { + copy_options __opt, error_code& __ec) { __copy(__from, __to, __opt, &__ec); } @@ -1561,7 +1562,7 @@ bool is_empty(const path& __p) { } inline _LIBCPP_INLINE_VISIBILITY -bool is_empty(const path& __p, error_code& __ec) _NOEXCEPT { +bool is_empty(const path& __p, error_code& __ec) { return __fs_is_empty(__p, &__ec); } @@ -1903,12 +1904,12 @@ public: : directory_iterator(__p, nullptr, __opts) { } - directory_iterator(const path& __p, error_code& __ec) _NOEXCEPT + directory_iterator(const path& __p, error_code& __ec) : directory_iterator(__p, &__ec) { } directory_iterator(const path& __p, directory_options __opts, - error_code& __ec) _NOEXCEPT + error_code& __ec) : directory_iterator(__p, &__ec, __opts) { } @@ -1943,7 +1944,7 @@ public: return __p; } - directory_iterator& increment(error_code& __ec) _NOEXCEPT + directory_iterator& increment(error_code& __ec) { return __increment(&__ec); } private: @@ -2013,12 +2014,12 @@ public: _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator(const path& __p, - directory_options __xoptions, error_code& __ec) _NOEXCEPT + directory_options __xoptions, error_code& __ec) : recursive_directory_iterator(__p, __xoptions, &__ec) { } _LIBCPP_INLINE_VISIBILITY - recursive_directory_iterator(const path& __p, error_code& __ec) _NOEXCEPT + recursive_directory_iterator(const path& __p, error_code& __ec) : recursive_directory_iterator(__p, directory_options::none, &__ec) { } @@ -2060,7 +2061,7 @@ public: } _LIBCPP_INLINE_VISIBILITY - recursive_directory_iterator& increment(error_code& __ec) _NOEXCEPT + recursive_directory_iterator& increment(error_code& __ec) { return __increment(&__ec); } _LIBCPP_FUNC_VIS directory_options options() const; diff --git a/contrib/libc++/include/experimental/iterator b/contrib/libc++/include/experimental/iterator index 37186b3d0b30..ea672e966315 100644 --- a/contrib/libc++/include/experimental/iterator +++ b/contrib/libc++/include/experimental/iterator @@ -75,19 +75,19 @@ public: typedef void reference; ostream_joiner(ostream_type& __os, _Delim&& __d) - : __output(_VSTD::addressof(__os)), __delim(_VSTD::move(__d)), __first(true) {} + : __output_iter(_VSTD::addressof(__os)), __delim(_VSTD::move(__d)), __first(true) {} ostream_joiner(ostream_type& __os, const _Delim& __d) - : __output(_VSTD::addressof(__os)), __delim(__d), __first(true) {} + : __output_iter(_VSTD::addressof(__os)), __delim(__d), __first(true) {} template ostream_joiner& operator=(const _Tp& __v) { if (!__first) - *__output << __delim; + *__output_iter << __delim; __first = false; - *__output << __v; + *__output_iter << __v; return *this; } @@ -96,7 +96,7 @@ public: ostream_joiner& operator++(int) _NOEXCEPT { return *this; } private: - ostream_type* __output; + ostream_type* __output_iter; _Delim __delim; bool __first; }; diff --git a/contrib/libc++/include/forward_list b/contrib/libc++/include/forward_list index 8bfa9a084338..7b8204137201 100644 --- a/contrib/libc++/include/forward_list +++ b/contrib/libc++/include/forward_list @@ -728,7 +728,7 @@ public: const_iterator cbefore_begin() const _NOEXCEPT {return const_iterator(base::__before_begin());} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return base::__before_begin()->__next_ == nullptr;} _LIBCPP_INLINE_VISIBILITY diff --git a/contrib/libc++/include/fstream b/contrib/libc++/include/fstream index ffd569839bd4..f57908c8dfaf 100644 --- a/contrib/libc++/include/fstream +++ b/contrib/libc++/include/fstream @@ -315,7 +315,7 @@ basic_filebuf<_CharT, _Traits>::basic_filebuf(basic_filebuf&& __rhs) else this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__rhs. epptr() - __rhs.pbase())); - this->pbump(__rhs. pptr() - __rhs.pbase()); + this->__pbump(__rhs. pptr() - __rhs.pbase()); } else if (__rhs.eback()) { @@ -434,7 +434,7 @@ basic_filebuf<_CharT, _Traits>::swap(basic_filebuf& __rhs) ptrdiff_t __e = this->epptr() - this->pbase(); this->setp((char_type*)__extbuf_min_, (char_type*)__extbuf_min_ + __e); - this->pbump(__n); + this->__pbump(__n); } if (__rhs.eback() == (char_type*)__extbuf_min_) { @@ -450,7 +450,7 @@ basic_filebuf<_CharT, _Traits>::swap(basic_filebuf& __rhs) ptrdiff_t __e = __rhs.epptr() - __rhs.pbase(); __rhs.setp((char_type*)__rhs.__extbuf_min_, (char_type*)__rhs.__extbuf_min_ + __e); - __rhs.pbump(__n); + __rhs.__pbump(__n); } } @@ -724,7 +724,7 @@ basic_filebuf<_CharT, _Traits>::overflow(int_type __c) if (__r == codecvt_base::partial) { this->setp(const_cast(__e), this->pptr()); - this->pbump(this->epptr() - this->pbase()); + this->__pbump(this->epptr() - this->pbase()); } } else diff --git a/contrib/libc++/include/functional b/contrib/libc++/include/functional index f73c3ca56a8e..3e5215b137b5 100644 --- a/contrib/libc++/include/functional +++ b/contrib/libc++/include/functional @@ -213,9 +213,9 @@ template struct is_bind_expression; template struct is_placeholder; // See C++14 20.9.9, Function object binders -template constexpr bool is_bind_expression_v +template inline constexpr bool is_bind_expression_v = is_bind_expression::value; // C++17 -template constexpr int is_placeholder_v +template inline constexpr int is_placeholder_v = is_placeholder::value; // C++17 @@ -1991,7 +1991,7 @@ template struct _LIBCPP_TEMPLATE_VIS is_bind_expression #if _LIBCPP_STD_VER > 14 template -constexpr size_t is_bind_expression_v = is_bind_expression<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr size_t is_bind_expression_v = is_bind_expression<_Tp>::value; #endif template struct __is_placeholder : public integral_constant {}; @@ -2000,7 +2000,7 @@ template struct _LIBCPP_TEMPLATE_VIS is_placeholder #if _LIBCPP_STD_VER > 14 template -constexpr size_t is_placeholder_v = is_placeholder<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr size_t is_placeholder_v = is_placeholder<_Tp>::value; #endif namespace placeholders @@ -2020,16 +2020,16 @@ _LIBCPP_FUNC_VIS extern const __ph<8> _8; _LIBCPP_FUNC_VIS extern const __ph<9> _9; _LIBCPP_FUNC_VIS extern const __ph<10> _10; #else -constexpr __ph<1> _1{}; -constexpr __ph<2> _2{}; -constexpr __ph<3> _3{}; -constexpr __ph<4> _4{}; -constexpr __ph<5> _5{}; -constexpr __ph<6> _6{}; -constexpr __ph<7> _7{}; -constexpr __ph<8> _8{}; -constexpr __ph<9> _9{}; -constexpr __ph<10> _10{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<1> _1{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<2> _2{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<3> _3{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<4> _4{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<5> _5{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<6> _6{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<7> _7{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<8> _8{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<9> _9{}; +/* _LIBCPP_INLINE_VAR */ constexpr __ph<10> _10{}; #endif // defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_BIND) } // placeholders diff --git a/contrib/libc++/include/future b/contrib/libc++/include/future index e38876758e13..a7c28a4746cf 100644 --- a/contrib/libc++/include/future +++ b/contrib/libc++/include/future @@ -2335,6 +2335,7 @@ inline _LIBCPP_INLINE_VISIBILITY bool __does_policy_contain(launch __policy, lau { return (int(__policy) & int(__value)) != 0; } template +_LIBCPP_NODISCARD_AFTER_CXX17 future::type, typename decay<_Args>::type...>::type> async(launch __policy, _Fp&& __f, _Args&&... __args) { @@ -2360,7 +2361,7 @@ async(launch __policy, _Fp&& __f, _Args&&... __args) } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY future::type, typename decay<_Args>::type...>::type> async(_Fp&& __f, _Args&&... __args) { diff --git a/contrib/libc++/include/istream b/contrib/libc++/include/istream index 0b8e05d95bd8..5c73df38f650 100644 --- a/contrib/libc++/include/istream +++ b/contrib/libc++/include/istream @@ -1069,16 +1069,18 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ this->rdbuf()->sbumpc(); ++__gc_; } - if (__n > 0) - *__s = char_type(); if (__gc_ == 0) __err |= ios_base::failbit; this->setstate(__err); } + if (__n > 0) + *__s = char_type(); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { + if (__n > 0) + *__s = char_type(); this->__set_badbit_and_consider_rethrow(); } #endif // _LIBCPP_NO_EXCEPTIONS diff --git a/contrib/libc++/include/iterator b/contrib/libc++/include/iterator index d163ab1b0910..8b887db037b9 100644 --- a/contrib/libc++/include/iterator +++ b/contrib/libc++/include/iterator @@ -37,16 +37,6 @@ struct iterator_traits typedef random_access_iterator_tag iterator_category; }; -template -struct iterator_traits -{ - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef const T* pointer; - typedef const T& reference; - typedef random_access_iterator_tag iterator_category; -}; - template struct iterator @@ -494,7 +484,7 @@ template struct _LIBCPP_TEMPLATE_VIS iterator_traits<_Tp*> { typedef ptrdiff_t difference_type; - typedef typename remove_const<_Tp>::type value_type; + typedef typename remove_cv<_Tp>::type value_type; typedef _Tp* pointer; typedef _Tp& reference; typedef random_access_iterator_tag iterator_category; @@ -604,21 +594,27 @@ distance(_InputIter __first, _InputIter __last) template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 -_InputIter +typename enable_if +< + __is_input_iterator<_InputIter>::value, + _InputIter +>::type next(_InputIter __x, - typename iterator_traits<_InputIter>::difference_type __n = 1, - typename enable_if<__is_input_iterator<_InputIter>::value>::type* = 0) + typename iterator_traits<_InputIter>::difference_type __n = 1) { _VSTD::advance(__x, __n); return __x; } -template +template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 -_BidiretionalIter -prev(_BidiretionalIter __x, - typename iterator_traits<_BidiretionalIter>::difference_type __n = 1, - typename enable_if<__is_bidirectional_iterator<_BidiretionalIter>::value>::type* = 0) +typename enable_if +< + __is_bidirectional_iterator<_BidirectionalIter>::value, + _BidirectionalIter +>::type +prev(_BidirectionalIter __x, + typename iterator_traits<_BidirectionalIter>::difference_type __n = 1) { _VSTD::advance(__x, -__n); return __x; @@ -908,15 +904,37 @@ public: _LIBCPP_INLINE_VISIBILITY istream_iterator operator++(int) {istream_iterator __t(*this); ++(*this); return __t;} + template friend _LIBCPP_INLINE_VISIBILITY - bool operator==(const istream_iterator& __x, const istream_iterator& __y) - {return __x.__in_stream_ == __y.__in_stream_;} + bool + operator==(const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __x, + const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __y); + template friend _LIBCPP_INLINE_VISIBILITY - bool operator!=(const istream_iterator& __x, const istream_iterator& __y) - {return !(__x == __y);} + bool + operator==(const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __x, + const istream_iterator<_Up, _CharU, _TraitsU, _DistanceU>& __y); }; +template +inline _LIBCPP_INLINE_VISIBILITY +bool +operator==(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x, + const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) +{ + return __x.__in_stream_ == __y.__in_stream_; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +bool +operator!=(const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __x, + const istream_iterator<_Tp, _CharT, _Traits, _Distance>& __y) +{ + return !(__x == __y); +} + template > class _LIBCPP_TEMPLATE_VIS ostream_iterator : public iterator @@ -1794,31 +1812,67 @@ end(const _Cp& __c) #endif // !defined(_LIBCPP_CXX03_LANG) #if _LIBCPP_STD_VER > 14 + +// #if _LIBCPP_STD_VER > 11 +// template <> +// struct _LIBCPP_TEMPLATE_VIS plus +// { +// template +// _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY +// auto operator()(_T1&& __t, _T2&& __u) const +// _NOEXCEPT_(noexcept(_VSTD::forward<_T1>(__t) + _VSTD::forward<_T2>(__u))) +// -> decltype (_VSTD::forward<_T1>(__t) + _VSTD::forward<_T2>(__u)) +// { return _VSTD::forward<_T1>(__t) + _VSTD::forward<_T2>(__u); } +// typedef void is_transparent; +// }; +// #endif + template -constexpr auto size(const _Cont& __c) -> decltype(__c.size()) { return __c.size(); } +inline _LIBCPP_INLINE_VISIBILITY +constexpr auto size(const _Cont& __c) +_NOEXCEPT_(noexcept(__c.size())) +-> decltype (__c.size()) +{ return __c.size(); } template +inline _LIBCPP_INLINE_VISIBILITY constexpr size_t size(const _Tp (&)[_Sz]) noexcept { return _Sz; } template -constexpr auto empty(const _Cont& __c) -> decltype(__c.empty()) { return __c.empty(); } +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY +constexpr auto empty(const _Cont& __c) +_NOEXCEPT_(noexcept(__c.empty())) +-> decltype (__c.empty()) +{ return __c.empty(); } template +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY constexpr bool empty(const _Tp (&)[_Sz]) noexcept { return false; } template +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY constexpr bool empty(initializer_list<_Ep> __il) noexcept { return __il.size() == 0; } template constexpr -auto data(_Cont& __c) -> decltype(__c.data()) { return __c.data(); } +inline _LIBCPP_INLINE_VISIBILITY +auto data(_Cont& __c) +_NOEXCEPT_(noexcept(__c.data())) +-> decltype (__c.data()) +{ return __c.data(); } template constexpr -auto data(const _Cont& __c) -> decltype(__c.data()) { return __c.data(); } +inline _LIBCPP_INLINE_VISIBILITY +auto data(const _Cont& __c) +_NOEXCEPT_(noexcept(__c.data())) +-> decltype (__c.data()) +{ return __c.data(); } template +inline _LIBCPP_INLINE_VISIBILITY constexpr _Tp* data(_Tp (&__array)[_Sz]) noexcept { return __array; } template +inline _LIBCPP_INLINE_VISIBILITY constexpr const _Ep* data(initializer_list<_Ep> __il) noexcept { return __il.begin(); } #endif diff --git a/contrib/libc++/include/list b/contrib/libc++/include/list index 9c70fff946c7..32e9a27bd2a4 100644 --- a/contrib/libc++/include/list +++ b/contrib/libc++/include/list @@ -481,7 +481,7 @@ public: { #if _LIBCPP_DEBUG_LEVEL >= 2 _LIBCPP_ASSERT(__get_const_db()->__dereferenceable(this), - "Attempted to dereference a non-dereferenceable list::iterator"); + "Attempted to dereference a non-dereferenceable list::const_iterator"); #endif return pointer_traits::pointer_to(__ptr_->__as_node()->__value_); } @@ -896,7 +896,7 @@ public: _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return base::__sz();} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return base::empty();} _LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT @@ -1071,6 +1071,16 @@ public: bool __invariants() const; + typedef __allocator_destructor<__node_allocator> __node_destructor; + typedef unique_ptr<__node, __node_destructor> __hold_pointer; + + _LIBCPP_INLINE_VISIBILITY + __hold_pointer __allocate_node(__node_allocator& __na) { + __node_pointer __p = __node_alloc_traits::allocate(__na, 1); + __p->__prev_ = nullptr; + return __hold_pointer(__p, __node_destructor(__na, 1)); + } + #if _LIBCPP_DEBUG_LEVEL >= 2 bool __dereferenceable(const const_iterator* __i) const; @@ -1397,9 +1407,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, const value_type& __x) " referring to this list"); #endif __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), __x); __link_nodes(__p.__ptr_, __hold->__as_link(), __hold->__as_link()); ++base::__sz(); @@ -1426,9 +1434,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _ { size_type __ds = 0; __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), __x); ++__ds; #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1494,9 +1500,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, _InpIter __f, _InpIter __l, { size_type __ds = 0; __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), *__f); ++__ds; #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1549,8 +1553,7 @@ void list<_Tp, _Alloc>::push_front(const value_type& __x) { __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), __x); __link_pointer __nl = __hold->__as_link(); __link_nodes_at_front(__nl, __nl); @@ -1563,8 +1566,7 @@ void list<_Tp, _Alloc>::push_back(const value_type& __x) { __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), __x); __link_nodes_at_back(__hold.get()->__as_link(), __hold.get()->__as_link()); ++base::__sz(); @@ -1578,8 +1580,7 @@ void list<_Tp, _Alloc>::push_front(value_type&& __x) { __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), _VSTD::move(__x)); __link_nodes_at_front(__hold.get()->__as_link(), __hold.get()->__as_link()); ++base::__sz(); @@ -1591,8 +1592,7 @@ void list<_Tp, _Alloc>::push_back(value_type&& __x) { __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), _VSTD::move(__x)); __link_nodes_at_back(__hold.get()->__as_link(), __hold.get()->__as_link()); ++base::__sz(); @@ -1609,8 +1609,7 @@ void list<_Tp, _Alloc>::emplace_front(_Args&&... __args) { __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), _VSTD::forward<_Args>(__args)...); __link_nodes_at_front(__hold.get()->__as_link(), __hold.get()->__as_link()); ++base::__sz(); @@ -1631,8 +1630,7 @@ void list<_Tp, _Alloc>::emplace_back(_Args&&... __args) { __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), _VSTD::forward<_Args>(__args)...); __link_pointer __nl = __hold->__as_link(); __link_nodes_at_back(__nl, __nl); @@ -1655,9 +1653,7 @@ list<_Tp, _Alloc>::emplace(const_iterator __p, _Args&&... __args) " referring to this list"); #endif __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), _VSTD::forward<_Args>(__args)...); __link_pointer __nl = __hold.get()->__as_link(); __link_nodes(__p.__ptr_, __nl, __nl); @@ -1680,9 +1676,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, value_type&& __x) " referring to this list"); #endif __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), _VSTD::move(__x)); __link_pointer __nl = __hold->__as_link(); __link_nodes(__p.__ptr_, __nl, __nl); @@ -1855,9 +1849,7 @@ list<_Tp, _Alloc>::resize(size_type __n) __n -= base::__sz(); size_type __ds = 0; __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_)); ++__ds; #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1914,9 +1906,7 @@ list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) __n -= base::__sz(); size_type __ds = 0; __node_allocator& __na = base::__node_alloc(); - typedef __allocator_destructor<__node_allocator> _Dp; - unique_ptr<__node, _Dp> __hold(__node_alloc_traits::allocate(__na, 1), _Dp(__na, 1)); - __hold->__prev_ = 0; + __hold_pointer __hold = __allocate_node(__na); __node_alloc_traits::construct(__na, _VSTD::addressof(__hold->__value_), __x); ++__ds; __link_pointer __nl = __hold.release()->__as_link(); diff --git a/contrib/libc++/include/locale b/contrib/libc++/include/locale index d30d950c7f85..a86645d2cc4b 100644 --- a/contrib/libc++/include/locale +++ b/contrib/libc++/include/locale @@ -4110,7 +4110,7 @@ wbuffer_convert<_Codecvt, _Elem, _Tr>::overflow(int_type __c) if (__r == codecvt_base::partial) { this->setp(const_cast(__e), this->pptr()); - this->pbump(this->epptr() - this->pbase()); + this->__pbump(this->epptr() - this->pbase()); } } else diff --git a/contrib/libc++/include/map b/contrib/libc++/include/map index 71f18693f6d1..952149389814 100644 --- a/contrib/libc++/include/map +++ b/contrib/libc++/include/map @@ -1012,7 +1012,7 @@ public: _LIBCPP_INLINE_VISIBILITY const_reverse_iterator crend() const _NOEXCEPT {return rend();} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __tree_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __tree_.size();} @@ -1669,7 +1669,7 @@ public: _LIBCPP_INLINE_VISIBILITY const_reverse_iterator crend() const _NOEXCEPT {return rend();} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __tree_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __tree_.size();} diff --git a/contrib/libc++/include/memory b/contrib/libc++/include/memory index 22706d029d5f..d6b427b941b5 100644 --- a/contrib/libc++/include/memory +++ b/contrib/libc++/include/memory @@ -18,7 +18,7 @@ namespace std { struct allocator_arg_t { }; -constexpr allocator_arg_t allocator_arg = allocator_arg_t(); +inline constexpr allocator_arg_t allocator_arg = allocator_arg_t(); template struct uses_allocator; @@ -46,6 +46,9 @@ struct pointer_traits static pointer pointer_to(
) noexcept; }; +template constexpr T* to_address(T* p) noexcept; // C++20 +template auto to_address(const Ptr& p) noexcept; // C++20 + template struct allocator_traits { @@ -81,8 +84,8 @@ struct allocator_traits template using rebind_alloc = Alloc::rebind::other | Alloc; template using rebind_traits = allocator_traits>; - static pointer allocate(allocator_type& a, size_type n); - static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint); + static pointer allocate(allocator_type& a, size_type n); // [[nodiscard]] in C++20 + static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint); // [[nodiscard]] in C++20 static void deallocate(allocator_type& a, pointer p, size_type n) noexcept; @@ -384,6 +387,9 @@ template unique_ptr make_unique(Args&&... args); template unique_ptr make_unique(size_t n); // C++14 template unspecified make_unique(Args&&...) = delete; // C++14, T == U[N] +template + basic_ostream& operator<< (basic_ostream& os, unique_ptr const& p); + template class shared_ptr { @@ -625,6 +631,9 @@ template struct hash; template struct hash >; template struct hash >; +template + inline constexpr bool uses_allocator_v = uses_allocator::value; + // Pointer safety enum class pointer_safety { relaxed, preferred, strict }; void declare_reachable(void *p); @@ -724,7 +733,7 @@ template struct __has_element_type : false_type {}; template -struct __has_element_type<_Tp, +struct __has_element_type<_Tp, typename __void_t::type> : true_type {}; template ::value> @@ -808,7 +817,7 @@ template struct __has_difference_type : false_type {}; template -struct __has_difference_type<_Tp, +struct __has_difference_type<_Tp, typename __void_t::type> : true_type {}; template ::value> @@ -994,7 +1003,7 @@ template struct __has_pointer_type : false_type {}; template -struct __has_pointer_type<_Tp, +struct __has_pointer_type<_Tp, typename __void_t::type> : true_type {}; namespace __pointer_type_imp @@ -1024,7 +1033,7 @@ template struct __has_const_pointer : false_type {}; template -struct __has_const_pointer<_Tp, +struct __has_const_pointer<_Tp, typename __void_t::type> : true_type {}; template ::value> @@ -1047,7 +1056,7 @@ template struct __has_void_pointer : false_type {}; template -struct __has_void_pointer<_Tp, +struct __has_void_pointer<_Tp, typename __void_t::type> : true_type {}; template ::value> @@ -1070,7 +1079,7 @@ template struct __has_const_void_pointer : false_type {}; template -struct __has_const_void_pointer<_Tp, +struct __has_const_void_pointer<_Tp, typename __void_t::type> : true_type {}; template ::value> @@ -1090,13 +1099,14 @@ struct __const_void_pointer<_Ptr, _Alloc, false> }; template -inline _LIBCPP_INLINE_VISIBILITY +inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR _Tp* __to_raw_pointer(_Tp* __p) _NOEXCEPT { return __p; } +#if _LIBCPP_STD_VER <= 17 template inline _LIBCPP_INLINE_VISIBILITY typename pointer_traits<_Pointer>::element_type* @@ -1104,6 +1114,41 @@ __to_raw_pointer(_Pointer __p) _NOEXCEPT { return _VSTD::__to_raw_pointer(__p.operator->()); } +#else +template +inline _LIBCPP_INLINE_VISIBILITY +auto +__to_raw_pointer(const _Pointer& __p) _NOEXCEPT +-> decltype(pointer_traits<_Pointer>::to_address(__p)) +{ + return pointer_traits<_Pointer>::to_address(__p); +} + +template +inline _LIBCPP_INLINE_VISIBILITY +auto +__to_raw_pointer(const _Pointer& __p, _None...) _NOEXCEPT +{ + return _VSTD::__to_raw_pointer(__p.operator->()); +} + +template +inline _LIBCPP_INLINE_VISIBILITY constexpr +_Tp* +to_address(_Tp* __p) _NOEXCEPT +{ + static_assert(!is_function_v<_Tp>, "_Tp is a function type"); + return __p; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +auto +to_address(const _Pointer& __p) _NOEXCEPT +{ + return _VSTD::__to_raw_pointer(__p); +} +#endif template struct __has_size_type : false_type {}; @@ -1148,7 +1193,7 @@ template struct __has_propagate_on_container_move_assignment : false_type {}; template -struct __has_propagate_on_container_move_assignment<_Tp, +struct __has_propagate_on_container_move_assignment<_Tp, typename __void_t::type> : true_type {}; @@ -1168,7 +1213,7 @@ template struct __has_propagate_on_container_swap : false_type {}; template -struct __has_propagate_on_container_swap<_Tp, +struct __has_propagate_on_container_swap<_Tp, typename __void_t::type> : true_type {}; @@ -1188,7 +1233,7 @@ template struct __has_is_always_equal : false_type {}; template -struct __has_is_always_equal<_Tp, +struct __has_is_always_equal<_Tp, typename __void_t::type> : true_type {}; @@ -1302,7 +1347,7 @@ struct __allocator_traits_rebind<_Alloc<_Tp, _A0, _A1, _A2>, _Up, false> template auto __has_allocate_hint_test(_Alloc&& __a, _SizeType&& __sz, _ConstVoidPtr&& __p) - -> decltype(__a.allocate(__sz, __p), true_type()); + -> decltype((void)__a.allocate(__sz, __p), true_type()); template auto @@ -1313,7 +1358,7 @@ template struct __has_allocate_hint : integral_constant(), + decltype(_VSTD::__has_allocate_hint_test(declval<_Alloc>(), declval<_SizeType>(), declval<_ConstVoidPtr>())), true_type>::value> @@ -1346,7 +1391,7 @@ template struct __has_construct : integral_constant(), + decltype(_VSTD::__has_construct_test(declval<_Alloc>(), declval<_Pointer>(), declval<_Args>()...)), true_type>::value> @@ -1367,7 +1412,7 @@ template struct __has_destroy : integral_constant(), + decltype(_VSTD::__has_destroy_test(declval<_Alloc>(), declval<_Pointer>())), true_type>::value> { @@ -1387,7 +1432,7 @@ template struct __has_max_size : integral_constant())), + decltype(_VSTD::__has_max_size_test(declval<_Alloc&>())), true_type>::value> { }; @@ -1406,7 +1451,7 @@ template struct __has_select_on_container_copy_construction : integral_constant())), + decltype(_VSTD::__has_select_on_container_copy_construction_test(declval<_Alloc&>())), true_type>::value> { }; @@ -1497,10 +1542,10 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits {typedef allocator_traits::other> other;}; #endif // _LIBCPP_CXX03_LANG - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY static pointer allocate(allocator_type& __a, size_type __n) {return __a.allocate(__n);} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY static pointer allocate(allocator_type& __a, size_type __n, const_void_pointer __hint) {return __allocate(__a, __n, __hint, __has_allocate_hint());} @@ -1688,10 +1733,10 @@ private: } _LIBCPP_INLINE_VISIBILITY - static size_type __max_size(true_type, const allocator_type& __a) + static size_type __max_size(true_type, const allocator_type& __a) _NOEXCEPT {return __a.max_size();} _LIBCPP_INLINE_VISIBILITY - static size_type __max_size(false_type, const allocator_type&) + static size_type __max_size(false_type, const allocator_type&) _NOEXCEPT {return numeric_limits::max() / sizeof(value_type);} _LIBCPP_INLINE_VISIBILITY @@ -1739,7 +1784,8 @@ public: {return _VSTD::addressof(__x);} _LIBCPP_INLINE_VISIBILITY const_pointer address(const_reference __x) const _NOEXCEPT {return _VSTD::addressof(__x);} - _LIBCPP_INLINE_VISIBILITY pointer allocate(size_type __n, allocator::const_pointer = 0) + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY + pointer allocate(size_type __n, allocator::const_pointer = 0) { if (__n > max_size()) __throw_length_error("allocator::allocate(size_t n)" @@ -1941,7 +1987,7 @@ public: _LIBCPP_INLINE_VISIBILITY raw_storage_iterator operator++(int) {raw_storage_iterator __t(*this); ++__x_; return __t;} #if _LIBCPP_STD_VER >= 14 - _LIBCPP_INLINE_VISIBILITY _OutputIterator base() const { return __x_; } + _LIBCPP_INLINE_VISIBILITY _OutputIterator base() const { return __x_; } #endif }; @@ -2040,11 +2086,12 @@ struct __compressed_pair_elem { typedef const _Tp& const_reference; #ifndef _LIBCPP_CXX03_LANG - constexpr __compressed_pair_elem() : __value_() {} + _LIBCPP_INLINE_VISIBILITY constexpr __compressed_pair_elem() : __value_() {} template ::type>::value >::type> + _LIBCPP_INLINE_VISIBILITY constexpr explicit __compressed_pair_elem(_Up&& __u) : __value_(_VSTD::forward<_Up>(__u)){}; @@ -2055,11 +2102,13 @@ struct __compressed_pair_elem { __tuple_indices<_Indexes...>) : __value_(_VSTD::forward<_Args>(_VSTD::get<_Indexes>(__args))...) {} #else - __compressed_pair_elem() : __value_() {} + _LIBCPP_INLINE_VISIBILITY __compressed_pair_elem() : __value_() {} + _LIBCPP_INLINE_VISIBILITY __compressed_pair_elem(_ParamT __p) : __value_(std::forward<_ParamT>(__p)) {} #endif - reference __get() _NOEXCEPT { return __value_; } + _LIBCPP_INLINE_VISIBILITY reference __get() _NOEXCEPT { return __value_; } + _LIBCPP_INLINE_VISIBILITY const_reference __get() const _NOEXCEPT { return __value_; } private: @@ -2074,11 +2123,12 @@ struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp { typedef _Tp __value_type; #ifndef _LIBCPP_CXX03_LANG - constexpr __compressed_pair_elem() = default; + _LIBCPP_INLINE_VISIBILITY constexpr __compressed_pair_elem() = default; template ::type>::value >::type> + _LIBCPP_INLINE_VISIBILITY constexpr explicit __compressed_pair_elem(_Up&& __u) : __value_type(_VSTD::forward<_Up>(__u)){}; @@ -2089,12 +2139,14 @@ struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp { __tuple_indices<_Indexes...>) : __value_type(_VSTD::forward<_Args>(_VSTD::get<_Indexes>(__args))...) {} #else - __compressed_pair_elem() : __value_type() {} + _LIBCPP_INLINE_VISIBILITY __compressed_pair_elem() : __value_type() {} + _LIBCPP_INLINE_VISIBILITY __compressed_pair_elem(_ParamT __p) : __value_type(std::forward<_ParamT>(__p)) {} #endif - reference __get() _NOEXCEPT { return *this; } + _LIBCPP_INLINE_VISIBILITY reference __get() _NOEXCEPT { return *this; } + _LIBCPP_INLINE_VISIBILITY const_reference __get() const _NOEXCEPT { return *this; } }; @@ -3850,7 +3902,7 @@ public: _LIBCPP_INLINE_VISIBILITY _Dp* __get_deleter() const _NOEXCEPT {return static_cast<_Dp*>(__cntrl_ - ? const_cast(__cntrl_->__get_deleter(typeid(_Dp))) + ? const_cast(__cntrl_->__get_deleter(typeid(_Dp))) : nullptr);} #endif // _LIBCPP_NO_RTTI @@ -4216,6 +4268,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::make_shared(_Args&& ...__args) { + static_assert( is_constructible<_Tp, _Args...>::value, "Can't construct object in make_shared" ); typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk; typedef allocator<_CntrlBlk> _A2; typedef __allocator_destructor<_A2> _D2; @@ -4234,6 +4287,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _Args&& ...__args) { + static_assert( is_constructible<_Tp, _Args...>::value, "Can't construct object in allocate_shared" ); typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk; typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2; typedef __allocator_destructor<_A2> _D2; @@ -4254,6 +4308,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::make_shared() { + static_assert((is_constructible<_Tp>::value), "Can't construct object in make_shared" ); typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk; typedef allocator<_CntrlBlk> _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4272,6 +4327,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::make_shared(_A0& __a0) { + static_assert((is_constructible<_Tp, _A0>::value), "Can't construct object in make_shared" ); typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk; typedef allocator<_CntrlBlk> _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4290,6 +4346,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::make_shared(_A0& __a0, _A1& __a1) { + static_assert((is_constructible<_Tp, _A0, _A1>::value), "Can't construct object in make_shared" ); typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk; typedef allocator<_CntrlBlk> _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4308,6 +4365,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::make_shared(_A0& __a0, _A1& __a1, _A2& __a2) { + static_assert((is_constructible<_Tp, _A0, _A1, _A2>::value), "Can't construct object in make_shared" ); typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk; typedef allocator<_CntrlBlk> _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4326,6 +4384,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::allocate_shared(const _Alloc& __a) { + static_assert((is_constructible<_Tp>::value), "Can't construct object in allocate_shared" ); typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk; typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4345,6 +4404,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _A0& __a0) { + static_assert((is_constructible<_Tp, _A0>::value), "Can't construct object in allocate_shared" ); typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk; typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4364,6 +4424,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _A0& __a0, _A1& __a1) { + static_assert((is_constructible<_Tp, _A0, _A1>::value), "Can't construct object in allocate_shared" ); typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk; typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4383,6 +4444,7 @@ template shared_ptr<_Tp> shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _A0& __a0, _A1& __a1, _A2& __a2) { + static_assert((is_constructible<_Tp, _A0, _A1, _A2>::value), "Can't construct object in allocate_shared" ); typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk; typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2; typedef __allocator_destructor<_Alloc2> _D2; @@ -4477,7 +4539,7 @@ inline typename enable_if < !is_array<_Yp>::value && - is_convertible::pointer, + is_convertible::pointer, typename shared_ptr<_Tp>::element_type*>::value, shared_ptr<_Tp>& >::type @@ -4512,7 +4574,7 @@ inline _LIBCPP_INLINE_VISIBILITY typename enable_if < !is_array<_Yp>::value && - is_convertible::pointer, + is_convertible::pointer, typename shared_ptr<_Tp>::element_type*>::value, shared_ptr<_Tp>& >::type @@ -5311,7 +5373,7 @@ atomic_load(const shared_ptr<_Tp>* __p) __m.unlock(); return __q; } - + template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_ATOMIC_SHARED_PTR @@ -5352,7 +5414,7 @@ atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) __m.unlock(); return __r; } - + template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_ATOMIC_SHARED_PTR @@ -5484,7 +5546,7 @@ void __swap_allocator(_Alloc & __a1, _Alloc & __a2) _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) #endif { - __swap_allocator(__a1, __a2, + __swap_allocator(__a1, __a2, integral_constant::propagate_on_container_swap::value>()); } @@ -5506,7 +5568,7 @@ inline _LIBCPP_INLINE_VISIBILITY void __swap_allocator(_Alloc &, _Alloc &, false_type) _NOEXCEPT {} template > -struct __noexcept_move_assign_container : public integral_constant 14 || _Traits::is_always_equal::value @@ -5520,17 +5582,17 @@ struct __noexcept_move_assign_container : public integral_constant struct __temp_value { typedef allocator_traits<_Alloc> _Traits; - + typename aligned_storage::type __v; _Alloc &__a; _Tp *__addr() { return reinterpret_cast<_Tp *>(addressof(__v)); } _Tp & get() { return *__addr(); } - + template __temp_value(_Alloc &__alloc, _Args&& ... __args) : __a(__alloc) { _Traits::construct(__a, __addr(), _VSTD::forward<_Args>(__args)...); } - + ~__temp_value() { _Traits::destroy(__a, __addr()); } }; #endif diff --git a/contrib/libc++/include/mutex b/contrib/libc++/include/mutex index fbcc0989fa98..52e39b0fb1f8 100644 --- a/contrib/libc++/include/mutex +++ b/contrib/libc++/include/mutex @@ -91,9 +91,9 @@ struct defer_lock_t {}; struct try_to_lock_t {}; struct adopt_lock_t {}; -constexpr defer_lock_t defer_lock{}; -constexpr try_to_lock_t try_to_lock{}; -constexpr adopt_lock_t adopt_lock{}; +inline constexpr defer_lock_t defer_lock{}; +inline constexpr try_to_lock_t try_to_lock{}; +inline constexpr adopt_lock_t adopt_lock{}; template class lock_guard diff --git a/contrib/libc++/include/new b/contrib/libc++/include/new index 34df2efee09e..4e527501b1e2 100644 --- a/contrib/libc++/include/new +++ b/contrib/libc++/include/new @@ -46,13 +46,15 @@ typedef void (*new_handler)(); new_handler set_new_handler(new_handler new_p) noexcept; new_handler get_new_handler() noexcept; +// 21.6.4, pointer optimization barrier +template constexpr T* launder(T* p) noexcept; // C++17 } // std -void* operator new(std::size_t size); // replaceable -void* operator new(std::size_t size, std::align_val_t alignment); // replaceable, C++17 -void* operator new(std::size_t size, const std::nothrow_t&) noexcept; // replaceable +void* operator new(std::size_t size); // replaceable, nodiscard in C++2a +void* operator new(std::size_t size, std::align_val_t alignment); // replaceable, C++17, nodiscard in C++2a +void* operator new(std::size_t size, const std::nothrow_t&) noexcept; // replaceable, nodiscard in C++2a void* operator new(std::size_t size, std::align_val_t alignment, - const std::nothrow_t&) noexcept; // replaceable, C++17 + const std::nothrow_t&) noexcept; // replaceable, C++17, nodiscard in C++2a void operator delete(void* ptr) noexcept; // replaceable void operator delete(void* ptr, std::size_t size) noexcept; // replaceable, C++14 void operator delete(void* ptr, std::align_val_t alignment) noexcept; // replaceable, C++17 @@ -62,12 +64,12 @@ void operator delete(void* ptr, const std::nothrow_t&) noexcept; // repla void operator delete(void* ptr, std:align_val_t alignment, const std::nothrow_t&) noexcept; // replaceable, C++17 -void* operator new[](std::size_t size); // replaceable +void* operator new[](std::size_t size); // replaceable, nodiscard in C++2a void* operator new[](std::size_t size, - std::align_val_t alignment) noexcept; // replaceable, C++17 -void* operator new[](std::size_t size, const std::nothrow_t&) noexcept; // replaceable + std::align_val_t alignment) noexcept; // replaceable, C++17, nodiscard in C++2a +void* operator new[](std::size_t size, const std::nothrow_t&) noexcept; // replaceable, nodiscard in C++2a void* operator new[](std::size_t size, std::align_val_t alignment, - const std::nothrow_t&) noexcept; // replaceable, C++17 + const std::nothrow_t&) noexcept; // replaceable, C++17, nodiscard in C++2a void operator delete[](void* ptr) noexcept; // replaceable void operator delete[](void* ptr, std::size_t size) noexcept; // replaceable, C++14 void operator delete[](void* ptr, @@ -78,8 +80,8 @@ void operator delete[](void* ptr, const std::nothrow_t&) noexcept; // repla void operator delete[](void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept; // replaceable, C++17 -void* operator new (std::size_t size, void* ptr) noexcept; -void* operator new[](std::size_t size, void* ptr) noexcept; +void* operator new (std::size_t size, void* ptr) noexcept; // nodiscard in C++2a +void* operator new[](std::size_t size, void* ptr) noexcept; // nodiscard in C++2a void operator delete (void* ptr, void*) noexcept; void operator delete[](void* ptr, void*) noexcept; @@ -92,7 +94,7 @@ void operator delete[](void* ptr, void*) noexcept; #include #endif -#if defined(_LIBCPP_ABI_MICROSOFT) +#if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_NO_VCRUNTIME) #include #endif @@ -114,7 +116,7 @@ void operator delete[](void* ptr, void*) noexcept; namespace std // purposefully not using versioning namespace { -#if !defined(_LIBCPP_ABI_MICROSOFT) +#if !defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME) struct _LIBCPP_TYPE_VIS nothrow_t {}; extern _LIBCPP_FUNC_VIS const nothrow_t nothrow; @@ -140,7 +142,7 @@ typedef void (*new_handler)(); _LIBCPP_FUNC_VIS new_handler set_new_handler(new_handler) _NOEXCEPT; _LIBCPP_FUNC_VIS new_handler get_new_handler() _NOEXCEPT; -#endif // !_LIBCPP_ABI_MICROSOFT +#endif // !_LIBCPP_ABI_MICROSOFT || _LIBCPP_NO_VCRUNTIME _LIBCPP_NORETURN _LIBCPP_FUNC_VIS void __throw_bad_alloc(); // not in C++ spec @@ -174,18 +176,18 @@ enum align_val_t { __zero = 0, __max = (size_t)-1 }; #define _THROW_BAD_ALLOC #endif -#if !defined(_LIBCPP_ABI_MICROSOFT) +#if !defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME) -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz) _THROW_BAD_ALLOC; -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t&) _NOEXCEPT _NOALIAS; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz) _THROW_BAD_ALLOC; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t&) _NOEXCEPT; #ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operator delete(void* __p, std::size_t __sz) _NOEXCEPT; #endif -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz) _THROW_BAD_ALLOC; -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t&) _NOEXCEPT _NOALIAS; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz) _THROW_BAD_ALLOC; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t&) _NOEXCEPT; #ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION @@ -193,16 +195,16 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operato #endif #ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC; -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _NOALIAS; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT; #ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operator delete(void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT; #endif -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC; -_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _NOALIAS; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC; +_LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _NOALIAS; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t) _NOEXCEPT; _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT; #ifndef _LIBCPP_HAS_NO_SIZED_DEALLOCATION @@ -210,12 +212,12 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE void operato #endif #endif -inline _LIBCPP_INLINE_VISIBILITY void* operator new (std::size_t, void* __p) _NOEXCEPT {return __p;} -inline _LIBCPP_INLINE_VISIBILITY void* operator new[](std::size_t, void* __p) _NOEXCEPT {return __p;} +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY void* operator new (std::size_t, void* __p) _NOEXCEPT {return __p;} +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY void* operator new[](std::size_t, void* __p) _NOEXCEPT {return __p;} inline _LIBCPP_INLINE_VISIBILITY void operator delete (void*, void*) _NOEXCEPT {} inline _LIBCPP_INLINE_VISIBILITY void operator delete[](void*, void*) _NOEXCEPT {} -#endif // !_LIBCPP_ABI_MICROSOFT +#endif // !_LIBCPP_ABI_MICROSOFT || _LIBCPP_NO_VCRUNTIME _LIBCPP_BEGIN_NAMESPACE_STD @@ -250,6 +252,29 @@ void __throw_bad_array_length() } #endif +template +_LIBCPP_NODISCARD_AFTER_CXX17 inline +_LIBCPP_CONSTEXPR _Tp* __launder(_Tp* __p) _NOEXCEPT +{ + static_assert (!(is_function<_Tp>::value), "can't launder functions" ); + static_assert (!(is_same::type>::value), "can't launder cv-void" ); +#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_LAUNDER + return __builtin_launder(__p); +#else + return __p; +#endif +} + + +#if _LIBCPP_STD_VER > 14 +template +_LIBCPP_NODISCARD_AFTER_CXX17 inline _LIBCPP_INLINE_VISIBILITY +constexpr _Tp* launder(_Tp* __p) noexcept +{ + return _VSTD::__launder(__p); +} +#endif + _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP_NEW diff --git a/contrib/libc++/include/optional b/contrib/libc++/include/optional index 1fb953bab743..88fd6b5aba2d 100644 --- a/contrib/libc++/include/optional +++ b/contrib/libc++/include/optional @@ -22,7 +22,7 @@ namespace std { // 23.6.4, no-value state indicator struct nullopt_t{see below }; - constexpr nullopt_t nullopt(unspecified ); + inline constexpr nullopt_t nullopt(unspecified ); // 23.6.5, class bad_optional_access class bad_optional_access; @@ -57,17 +57,17 @@ namespace std { // 23.6.8, comparison with T template constexpr bool operator==(const optional&, const U&); - template constexpr bool operator==(const U&, const optional&); + template constexpr bool operator==(const T&, const optional&); template constexpr bool operator!=(const optional&, const U&); - template constexpr bool operator!=(const U&, const optional&); + template constexpr bool operator!=(const T&, const optional&); template constexpr bool operator<(const optional&, const U&); - template constexpr bool operator<(const U&, const optional&); + template constexpr bool operator<(const T&, const optional&); template constexpr bool operator<=(const optional&, const U&); - template constexpr bool operator<=(const U&, const optional&); + template constexpr bool operator<=(const T&, const optional&); template constexpr bool operator>(const optional&, const U&); - template constexpr bool operator>(const U&, const optional&); + template constexpr bool operator>(const T&, const optional&); template constexpr bool operator>=(const optional&, const U&); - template constexpr bool operator>=(const U&, const optional&); + template constexpr bool operator>=(const T&, const optional&); // 23.6.9, specialized algorithms template void swap(optional&, optional&) noexcept(see below ); @@ -195,7 +195,7 @@ struct nullopt_t _LIBCPP_INLINE_VISIBILITY constexpr explicit nullopt_t(__secret_tag, __secret_tag) noexcept {} }; -/* inline */ constexpr nullopt_t nullopt{nullopt_t::__secret_tag{}, nullopt_t::__secret_tag{}}; +_LIBCPP_INLINE_VAR constexpr nullopt_t nullopt{nullopt_t::__secret_tag{}, nullopt_t::__secret_tag{}}; template ::value> struct __optional_destruct_base; diff --git a/contrib/libc++/include/ostream b/contrib/libc++/include/ostream index 9bf8d3cdcfb5..f3250a7080dd 100644 --- a/contrib/libc++/include/ostream +++ b/contrib/libc++/include/ostream @@ -1071,6 +1071,20 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, shared_ptr<_Yp> const& __p) return __os << __p.get(); } +#ifndef _LIBCPP_HAS_NO_DECLTYPE +template +inline _LIBCPP_INLINE_VISIBILITY +typename enable_if +< + is_same&>() << declval<_Yp>())>::type>::value, + basic_ostream<_CharT, _Traits>& +>::type +operator<<(basic_ostream<_CharT, _Traits>& __os, unique_ptr<_Yp, _Dp> const& __p) +{ + return __os << __p.get(); +} +#endif + template basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, const bitset<_Size>& __x) diff --git a/contrib/libc++/include/queue b/contrib/libc++/include/queue index feaae8920463..670fbb722ee2 100644 --- a/contrib/libc++/include/queue +++ b/contrib/libc++/include/queue @@ -268,7 +268,7 @@ public: #endif // _LIBCPP_CXX03_LANG - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const {return c.empty();} _LIBCPP_INLINE_VISIBILITY size_type size() const {return c.size();} @@ -490,7 +490,7 @@ public: _Alloc>::value>::type* = 0); #endif // _LIBCPP_CXX03_LANG - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const {return c.empty();} _LIBCPP_INLINE_VISIBILITY size_type size() const {return c.size();} diff --git a/contrib/libc++/include/random b/contrib/libc++/include/random index ba8a088dfcb1..9073a5285b1a 100644 --- a/contrib/libc++/include/random +++ b/contrib/libc++/include/random @@ -2308,6 +2308,7 @@ template ::seed(result_type __sd) + _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK { // __w >= 2 __x_[0] = __sd & _Max; for (size_t __i = 1; __i < __n; ++__i) diff --git a/contrib/libc++/include/ratio b/contrib/libc++/include/ratio index 27aa0ded32e6..7ee5ec245174 100644 --- a/contrib/libc++/include/ratio +++ b/contrib/libc++/include/ratio @@ -63,17 +63,17 @@ typedef ratio< 1000000000000000000000, 1> zetta; // not supported typedef ratio<1000000000000000000000000, 1> yotta; // not supported // 20.11.5, ratio comparison - template constexpr bool ratio_equal_v + template inline constexpr bool ratio_equal_v = ratio_equal::value; // C++17 - template constexpr bool ratio_not_equal_v + template inline constexpr bool ratio_not_equal_v = ratio_not_equal::value; // C++17 - template constexpr bool ratio_less_v + template inline constexpr bool ratio_less_v = ratio_less::value; // C++17 - template constexpr bool ratio_less_equal_v + template inline constexpr bool ratio_less_equal_v = ratio_less_equal::value; // C++17 - template constexpr bool ratio_greater_v + template inline constexpr bool ratio_greater_v = ratio_greater::value; // C++17 - template constexpr bool ratio_greater_equal_v + template inline constexpr bool ratio_greater_equal_v = ratio_greater_equal::value; // C++17 } */ @@ -501,22 +501,28 @@ struct __ratio_gcd }; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool ratio_equal_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool ratio_equal_v = ratio_equal<_R1, _R2>::value; -template _LIBCPP_CONSTEXPR bool ratio_not_equal_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool ratio_not_equal_v = ratio_not_equal<_R1, _R2>::value; -template _LIBCPP_CONSTEXPR bool ratio_less_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool ratio_less_v = ratio_less<_R1, _R2>::value; -template _LIBCPP_CONSTEXPR bool ratio_less_equal_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool ratio_less_equal_v = ratio_less_equal<_R1, _R2>::value; -template _LIBCPP_CONSTEXPR bool ratio_greater_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool ratio_greater_v = ratio_greater<_R1, _R2>::value; -template _LIBCPP_CONSTEXPR bool ratio_greater_equal_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool ratio_greater_equal_v = ratio_greater_equal<_R1, _R2>::value; #endif diff --git a/contrib/libc++/include/regex b/contrib/libc++/include/regex index 77ca648109b2..ff84b2738b78 100644 --- a/contrib/libc++/include/regex +++ b/contrib/libc++/include/regex @@ -773,6 +773,8 @@ _LIBCPP_PUSH_MACROS #include <__undef_macros> +#define _LIBCPP_REGEX_COMPLEXITY_FACTOR 4096 + _LIBCPP_BEGIN_NAMESPACE_STD namespace regex_constants @@ -2407,17 +2409,28 @@ __bracket_expression<_CharT, _Traits>::__exec(__state& __s) const goto __exit; } } - if (!__neg_chars_.empty()) + // set of "__found" chars = + // union(complement(union(__neg_chars_, __neg_mask_)), + // other cases...) + // + // __neg_chars_ and __neg_mask_'d better be handled together, as there + // are no short circuit opportunities. + // + // In addition, when __neg_mask_/__neg_chars_ is empty, they should be + // treated as all ones/all chars. { - for (size_t __i = 0; __i < __neg_chars_.size(); ++__i) - { - if (__ch == __neg_chars_[__i]) - goto __is_neg_char; - } + const bool __in_neg_mask = (__neg_mask_ == 0) || + __traits_.isctype(__ch, __neg_mask_); + const bool __in_neg_chars = + __neg_chars_.empty() || + std::find(__neg_chars_.begin(), __neg_chars_.end(), __ch) != + __neg_chars_.end(); + if (!(__in_neg_mask || __in_neg_chars)) + { __found = true; goto __exit; + } } -__is_neg_char: if (!__ranges_.empty()) { string_type __s2 = __collate_ ? @@ -2449,11 +2462,6 @@ __is_neg_char: __found = true; goto __exit; } - if (__neg_mask_ && !__traits_.isctype(__ch, __neg_mask_)) - { - __found = true; - goto __exit; - } } else __found = __negate_; // force reject @@ -4056,6 +4064,8 @@ basic_regex<_CharT, _Traits>::__parse_DUP_COUNT(_ForwardIterator __first, __first != __last && ( __val = __traits_.value(*__first, 10)) != -1; ++__first) { + if (__c >= std::numeric_limits::max() / 10) + __throw_regex_error(); __c *= 10; __c += __val; } @@ -4317,8 +4327,12 @@ basic_regex<_CharT, _Traits>::__parse_decimal_escape(_ForwardIterator __first, unsigned __v = *__first - '0'; for (++__first; __first != __last && '0' <= *__first && *__first <= '9'; ++__first) + { + if (__v >= std::numeric_limits::max() / 10) + __throw_regex_error(); __v = 10 * __v + *__first - '0'; - if (__v > mark_count()) + } + if (__v == 0 || __v > mark_count()) __throw_regex_error(); __push_back_ref(__v); } @@ -5226,11 +5240,11 @@ public: // size: _LIBCPP_INLINE_VISIBILITY - size_type size() const {return __matches_.size();} + size_type size() const _NOEXCEPT {return __matches_.size();} _LIBCPP_INLINE_VISIBILITY - size_type max_size() const {return __matches_.max_size();} - _LIBCPP_INLINE_VISIBILITY - bool empty() const {return size() == 0;} + size_type max_size() const _NOEXCEPT {return __matches_.max_size();} + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY + bool empty() const _NOEXCEPT {return size() == 0;} // element access: _LIBCPP_INLINE_VISIBILITY @@ -5263,15 +5277,15 @@ public: // format: template _OutputIter - format(_OutputIter __output, const char_type* __fmt_first, + format(_OutputIter __output_iter, const char_type* __fmt_first, const char_type* __fmt_last, regex_constants::match_flag_type __flags = regex_constants::format_default) const; template _LIBCPP_INLINE_VISIBILITY _OutputIter - format(_OutputIter __output, const basic_string& __fmt, + format(_OutputIter __output_iter, const basic_string& __fmt, regex_constants::match_flag_type __flags = regex_constants::format_default) const - {return format(__output, __fmt.data(), __fmt.data() + __fmt.size(), __flags);} + {return format(__output_iter, __fmt.data(), __fmt.data() + __fmt.size(), __flags);} template _LIBCPP_INLINE_VISIBILITY basic_string @@ -5383,7 +5397,7 @@ match_results<_BidirectionalIterator, _Allocator>::__init(unsigned __s, template template _OutputIter -match_results<_BidirectionalIterator, _Allocator>::format(_OutputIter __output, +match_results<_BidirectionalIterator, _Allocator>::format(_OutputIter __output_iter, const char_type* __fmt_first, const char_type* __fmt_last, regex_constants::match_flag_type __flags) const { @@ -5392,27 +5406,27 @@ match_results<_BidirectionalIterator, _Allocator>::format(_OutputIter __output, for (; __fmt_first != __fmt_last; ++__fmt_first) { if (*__fmt_first == '&') - __output = _VSTD::copy(__matches_[0].first, __matches_[0].second, - __output); + __output_iter = _VSTD::copy(__matches_[0].first, __matches_[0].second, + __output_iter); else if (*__fmt_first == '\\' && __fmt_first + 1 != __fmt_last) { ++__fmt_first; if ('0' <= *__fmt_first && *__fmt_first <= '9') { size_t __i = *__fmt_first - '0'; - __output = _VSTD::copy((*this)[__i].first, - (*this)[__i].second, __output); + __output_iter = _VSTD::copy((*this)[__i].first, + (*this)[__i].second, __output_iter); } else { - *__output = *__fmt_first; - ++__output; + *__output_iter = *__fmt_first; + ++__output_iter; } } else { - *__output = *__fmt_first; - ++__output; + *__output_iter = *__fmt_first; + ++__output_iter; } } } @@ -5425,52 +5439,54 @@ match_results<_BidirectionalIterator, _Allocator>::format(_OutputIter __output, switch (__fmt_first[1]) { case '$': - *__output = *++__fmt_first; - ++__output; + *__output_iter = *++__fmt_first; + ++__output_iter; break; case '&': ++__fmt_first; - __output = _VSTD::copy(__matches_[0].first, __matches_[0].second, - __output); + __output_iter = _VSTD::copy(__matches_[0].first, __matches_[0].second, + __output_iter); break; case '`': ++__fmt_first; - __output = _VSTD::copy(__prefix_.first, __prefix_.second, __output); + __output_iter = _VSTD::copy(__prefix_.first, __prefix_.second, __output_iter); break; case '\'': ++__fmt_first; - __output = _VSTD::copy(__suffix_.first, __suffix_.second, __output); + __output_iter = _VSTD::copy(__suffix_.first, __suffix_.second, __output_iter); break; default: if ('0' <= __fmt_first[1] && __fmt_first[1] <= '9') { ++__fmt_first; - size_t __i = *__fmt_first - '0'; + size_t __idx = *__fmt_first - '0'; if (__fmt_first + 1 != __fmt_last && '0' <= __fmt_first[1] && __fmt_first[1] <= '9') { ++__fmt_first; - __i = 10 * __i + *__fmt_first - '0'; + if (__idx >= std::numeric_limits::max() / 10) + __throw_regex_error(); + __idx = 10 * __idx + *__fmt_first - '0'; } - __output = _VSTD::copy((*this)[__i].first, - (*this)[__i].second, __output); + __output_iter = _VSTD::copy((*this)[__idx].first, + (*this)[__idx].second, __output_iter); } else { - *__output = *__fmt_first; - ++__output; + *__output_iter = *__fmt_first; + ++__output_iter; } break; } } else { - *__output = *__fmt_first; - ++__output; + *__output_iter = *__fmt_first; + ++__output_iter; } } } - return __output; + return __output_iter; } template @@ -5552,8 +5568,14 @@ basic_regex<_CharT, _Traits>::__match_at_start_ecma( __states.back().__node_ = __st; __states.back().__flags_ = __flags; __states.back().__at_first_ = __at_first; + int __counter = 0; + int __length = __last - __first; do { + ++__counter; + if (__counter % _LIBCPP_REGEX_COMPLEXITY_FACTOR == 0 && + __counter / _LIBCPP_REGEX_COMPLEXITY_FACTOR >= __length) + __throw_regex_error(); __state& __s = __states.back(); if (__s.__node_) __s.__node_->__exec(__s); @@ -5627,8 +5649,14 @@ basic_regex<_CharT, _Traits>::__match_at_start_posix_nosubs( __states.back().__flags_ = __flags; __states.back().__at_first_ = __at_first; bool __matched = false; + int __counter = 0; + int __length = __last - __first; do { + ++__counter; + if (__counter % _LIBCPP_REGEX_COMPLEXITY_FACTOR == 0 && + __counter / _LIBCPP_REGEX_COMPLEXITY_FACTOR >= __length) + __throw_regex_error(); __state& __s = __states.back(); if (__s.__node_) __s.__node_->__exec(__s); @@ -5724,8 +5752,14 @@ basic_regex<_CharT, _Traits>::__match_at_start_posix_subs( __states.back().__at_first_ = __at_first; const _CharT* __current = __first; bool __matched = false; + int __counter = 0; + int __length = __last - __first; do { + ++__counter; + if (__counter % _LIBCPP_REGEX_COMPLEXITY_FACTOR == 0 && + __counter / _LIBCPP_REGEX_COMPLEXITY_FACTOR >= __length) + __throw_regex_error(); __state& __s = __states.back(); if (__s.__node_) __s.__node_->__exec(__s); @@ -6460,7 +6494,7 @@ typedef regex_token_iterator wsregex_token_iterator; template _OutputIterator -regex_replace(_OutputIterator __output, +regex_replace(_OutputIterator __output_iter, _BidirectionalIterator __first, _BidirectionalIterator __last, const basic_regex<_CharT, _Traits>& __e, const _CharT* __fmt, regex_constants::match_flag_type __flags = regex_constants::match_default) @@ -6471,7 +6505,7 @@ regex_replace(_OutputIterator __output, if (__i == __eof) { if (!(__flags & regex_constants::format_no_copy)) - __output = _VSTD::copy(__first, __last, __output); + __output_iter = _VSTD::copy(__first, __last, __output_iter); } else { @@ -6479,29 +6513,29 @@ regex_replace(_OutputIterator __output, for (size_t __len = char_traits<_CharT>::length(__fmt); __i != __eof; ++__i) { if (!(__flags & regex_constants::format_no_copy)) - __output = _VSTD::copy(__i->prefix().first, __i->prefix().second, __output); - __output = __i->format(__output, __fmt, __fmt + __len, __flags); + __output_iter = _VSTD::copy(__i->prefix().first, __i->prefix().second, __output_iter); + __output_iter = __i->format(__output_iter, __fmt, __fmt + __len, __flags); __lm = __i->suffix(); if (__flags & regex_constants::format_first_only) break; } if (!(__flags & regex_constants::format_no_copy)) - __output = _VSTD::copy(__lm.first, __lm.second, __output); + __output_iter = _VSTD::copy(__lm.first, __lm.second, __output_iter); } - return __output; + return __output_iter; } template inline _LIBCPP_INLINE_VISIBILITY _OutputIterator -regex_replace(_OutputIterator __output, +regex_replace(_OutputIterator __output_iter, _BidirectionalIterator __first, _BidirectionalIterator __last, const basic_regex<_CharT, _Traits>& __e, const basic_string<_CharT, _ST, _SA>& __fmt, regex_constants::match_flag_type __flags = regex_constants::match_default) { - return _VSTD::regex_replace(__output, __first, __last, __e, __fmt.c_str(), __flags); + return _VSTD::regex_replace(__output_iter, __first, __last, __e, __fmt.c_str(), __flags); } template :: allocate(outer_allocator(), __n);} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY pointer allocate(size_type __n, const_void_pointer __hint) {return allocator_traits:: allocate(outer_allocator(), __n, __hint);} diff --git a/contrib/libc++/include/set b/contrib/libc++/include/set index 30f2fa88f256..b4c6b2ef8c53 100644 --- a/contrib/libc++/include/set +++ b/contrib/libc++/include/set @@ -575,7 +575,7 @@ public: _LIBCPP_INLINE_VISIBILITY const_reverse_iterator crend() const _NOEXCEPT {return rend();} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __tree_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __tree_.size();} @@ -984,7 +984,7 @@ public: _LIBCPP_INLINE_VISIBILITY const_reverse_iterator crend() const _NOEXCEPT {return rend();} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __tree_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __tree_.size();} diff --git a/contrib/libc++/include/sstream b/contrib/libc++/include/sstream index fe65fd7db53d..b01f47b6872c 100644 --- a/contrib/libc++/include/sstream +++ b/contrib/libc++/include/sstream @@ -243,7 +243,6 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::basic_stringbuf(ios_base::openmode : __hm_(0), __mode_(__wch) { - str(string_type()); } template @@ -289,7 +288,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::basic_stringbuf(basic_stringbuf&& if (__bout != -1) { this->setp(__p + __bout, __p + __eout); - this->pbump(__nout); + this->__pbump(__nout); } __hm_ = __hm == -1 ? nullptr : __p + __hm; __p = const_cast(__rhs.__str_.data()); @@ -332,7 +331,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::operator=(basic_stringbuf&& __rhs) if (__bout != -1) { this->setp(__p + __bout, __p + __eout); - this->pbump(__nout); + this->__pbump(__nout); } else this->setp(nullptr, nullptr); @@ -403,7 +402,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::swap(basic_stringbuf& __rhs) if (__rbout != -1) { this->setp(__p + __rbout, __p + __reout); - this->pbump(__rnout); + this->__pbump(__rnout); } else this->setp(nullptr, nullptr); @@ -416,7 +415,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::swap(basic_stringbuf& __rhs) if (__lbout != -1) { __rhs.setp(__p + __lbout, __p + __leout); - __rhs.pbump(__lnout); + __rhs.__pbump(__lnout); } else __rhs.setp(nullptr, nullptr); @@ -471,7 +470,15 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::str(const string_type& __s) this->setp(const_cast(__str_.data()), const_cast(__str_.data()) + __str_.size()); if (__mode_ & (ios_base::app | ios_base::ate)) - this->pbump(__sz); + { + while (__sz > INT_MAX) + { + this->pbump(INT_MAX); + __sz -= INT_MAX; + } + if (__sz > 0) + this->pbump(__sz); + } } } @@ -536,7 +543,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::overflow(int_type __c) __str_.resize(__str_.capacity()); char_type* __p = const_cast(__str_.data()); this->setp(__p, __p + __str_.size()); - this->pbump(__nout); + this->__pbump(__nout); __hm_ = this->pbase() + __hm; #ifndef _LIBCPP_NO_EXCEPTIONS } @@ -570,6 +577,7 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::seekoff(off_type __off, if ((__wch & (ios_base::in | ios_base::out)) == (ios_base::in | ios_base::out) && __way == ios_base::cur) return pos_type(-1); + const ptrdiff_t __hm = __hm_ == nullptr ? 0 : __hm_ - __str_.data(); off_type __noff; switch (__way) { @@ -583,13 +591,13 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::seekoff(off_type __off, __noff = this->pptr() - this->pbase(); break; case ios_base::end: - __noff = __hm_ - __str_.data(); + __noff = __hm; break; default: return pos_type(-1); } __noff += __off; - if (__noff < 0 || __hm_ - __str_.data() < __noff) + if (__noff < 0 || __hm < __noff) return pos_type(-1); if (__noff != 0) { diff --git a/contrib/libc++/include/stack b/contrib/libc++/include/stack index b2d4e2395017..0cb4c6ca925e 100644 --- a/contrib/libc++/include/stack +++ b/contrib/libc++/include/stack @@ -181,7 +181,7 @@ public: : c(_VSTD::move(__s.c), __a) {} #endif // _LIBCPP_CXX03_LANG - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const {return c.empty();} _LIBCPP_INLINE_VISIBILITY size_type size() const {return c.size();} diff --git a/contrib/libc++/include/streambuf b/contrib/libc++/include/streambuf index a10ce1bf5636..ea64f5780459 100644 --- a/contrib/libc++/include/streambuf +++ b/contrib/libc++/include/streambuf @@ -255,6 +255,9 @@ protected: inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY void pbump(int __n) { __nout_ += __n; } + _LIBCPP_ALWAYS_INLINE + void __pbump(streamsize __n) { __nout_ += __n; } + inline _LIBCPP_EXTERN_TEMPLATE_INLINE_VISIBILITY void setp(char_type* __pbeg, char_type* __pend) { __bout_ = __nout_ = __pbeg; @@ -404,7 +407,8 @@ basic_streambuf<_CharT, _Traits>::xsgetn(char_type* __s, streamsize __n) { if (__ninp_ < __einp_) { - const streamsize __len = _VSTD::min(__einp_ - __ninp_, __n - __i); + const streamsize __len = _VSTD::min(static_cast(INT_MAX), + _VSTD::min(__einp_ - __ninp_, __n - __i)); traits_type::copy(__s, __ninp_, __len); __s += __len; __i += __len; diff --git a/contrib/libc++/include/string b/contrib/libc++/include/string index 7775587a42d9..f5d548965aae 100644 --- a/contrib/libc++/include/string +++ b/contrib/libc++/include/string @@ -301,6 +301,13 @@ public: int compare(size_type pos1, size_type n1, const value_type* s) const; int compare(size_type pos1, size_type n1, const value_type* s, size_type n2) const; + bool starts_with(basic_string_view sv) const noexcept; // C++2a + bool starts_with(charT c) const noexcept; // C++2a + bool starts_with(const charT* s) const; // C++2a + bool ends_with(basic_string_view sv) const noexcept; // C++2a + bool ends_with(charT c) const noexcept; // C++2a + bool ends_with(const charT* s) const; // C++2a + bool __invariants() const; }; @@ -608,7 +615,7 @@ struct __libcpp_string_gets_noexcept_iterator template struct __can_be_converted_to_string_view : public _LIBCPP_BOOL_CONSTANT( - ( is_convertible >::value && + ( is_convertible >::value && !is_convertible::value)) {}; #ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT @@ -670,7 +677,7 @@ private: size_type __cap_; }; -#if _LIBCPP_BIG_ENDIAN +#ifdef _LIBCPP_BIG_ENDIAN static const size_type __short_mask = 0x01; static const size_type __long_mask = 0x1ul; #else // _LIBCPP_BIG_ENDIAN @@ -700,7 +707,7 @@ private: pointer __data_; }; -#if _LIBCPP_BIG_ENDIAN +#ifdef _LIBCPP_BIG_ENDIAN static const size_type __short_mask = 0x80; static const size_type __long_mask = ~(size_type(~0) >> 1); #else // _LIBCPP_BIG_ENDIAN @@ -901,7 +908,8 @@ public: void shrink_to_fit() _NOEXCEPT {reserve();} _LIBCPP_INLINE_VISIBILITY void clear() _NOEXCEPT; - _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return size() == 0;} + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY + bool empty() const _NOEXCEPT {return size() == 0;} _LIBCPP_INLINE_VISIBILITY const_reference operator[](size_type __pos) const _NOEXCEPT; _LIBCPP_INLINE_VISIBILITY reference operator[](size_type __pos) _NOEXCEPT; @@ -1127,7 +1135,7 @@ public: const value_type* c_str() const _NOEXCEPT {return data();} _LIBCPP_INLINE_VISIBILITY const value_type* data() const _NOEXCEPT {return _VSTD::__to_raw_pointer(__get_pointer());} -#if _LIBCPP_STD_VER > 14 +#if _LIBCPP_STD_VER > 14 || defined(_LIBCPP_BUILDING_LIBRARY) _LIBCPP_INLINE_VISIBILITY value_type* data() _NOEXCEPT {return _VSTD::__to_raw_pointer(__get_pointer());} #endif @@ -1214,6 +1222,32 @@ public: int compare(size_type __pos1, size_type __n1, const value_type* __s) const; int compare(size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const; +#if _LIBCPP_STD_VER > 17 + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool starts_with(__self_view __sv) const _NOEXCEPT + { return __self_view(data(), size()).starts_with(__sv); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool starts_with(value_type __c) const _NOEXCEPT + { return !empty() && _Traits::eq(front(), __c); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool starts_with(const value_type* __s) const _NOEXCEPT + { return starts_with(__self_view(__s)); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool ends_with(__self_view __sv) const _NOEXCEPT + { return __self_view(data(), size()).ends_with( __sv); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool ends_with(value_type __c) const _NOEXCEPT + { return !empty() && _Traits::eq(back(), __c); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool ends_with(const value_type* __s) const _NOEXCEPT + { return ends_with(__self_view(__s)); } +#endif + _LIBCPP_INLINE_VISIBILITY bool __invariants() const; _LIBCPP_INLINE_VISIBILITY @@ -1241,7 +1275,7 @@ private: _LIBCPP_INLINE_VISIBILITY void __set_short_size(size_type __s) _NOEXCEPT -# if _LIBCPP_BIG_ENDIAN +# ifdef _LIBCPP_BIG_ENDIAN {__r_.first().__s.__size_ = (unsigned char)(__s << 1);} # else {__r_.first().__s.__size_ = (unsigned char)(__s);} @@ -1249,7 +1283,7 @@ private: _LIBCPP_INLINE_VISIBILITY size_type __get_short_size() const _NOEXCEPT -# if _LIBCPP_BIG_ENDIAN +# ifdef _LIBCPP_BIG_ENDIAN {return __r_.first().__s.__size_ >> 1;} # else {return __r_.first().__s.__size_;} @@ -1259,7 +1293,7 @@ private: _LIBCPP_INLINE_VISIBILITY void __set_short_size(size_type __s) _NOEXCEPT -# if _LIBCPP_BIG_ENDIAN +# ifdef _LIBCPP_BIG_ENDIAN {__r_.first().__s.__size_ = (unsigned char)(__s);} # else {__r_.first().__s.__size_ = (unsigned char)(__s << 1);} @@ -1267,7 +1301,7 @@ private: _LIBCPP_INLINE_VISIBILITY size_type __get_short_size() const _NOEXCEPT -# if _LIBCPP_BIG_ENDIAN +# ifdef _LIBCPP_BIG_ENDIAN {return __r_.first().__s.__size_;} # else {return __r_.first().__s.__size_ >> 1;} @@ -1756,10 +1790,10 @@ template template basic_string<_CharT, _Traits, _Allocator>::basic_string( const _Tp& __t, size_type __pos, size_type __n, const allocator_type& __a, - typename enable_if<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, void>::type *) + typename enable_if<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, void>::type *) : __r_(__second_tag(), __a) { - __self_view __sv = __self_view(__t).substr(__pos, __n); + __self_view __sv = __self_view(__t).substr(__pos, __n); __init(__sv.data(), __sv.size()); #if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->__insert_c(this); @@ -2145,7 +2179,7 @@ template typename enable_if < __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, - basic_string<_CharT, _Traits, _Allocator>& + basic_string<_CharT, _Traits, _Allocator>& >::type basic_string<_CharT, _Traits, _Allocator>::assign(const _Tp & __t, size_type __pos, size_type __n) { @@ -2493,7 +2527,7 @@ template typename enable_if < __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, - basic_string<_CharT, _Traits, _Allocator>& + basic_string<_CharT, _Traits, _Allocator>& >::type basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos1, const _Tp& __t, size_type __pos2, size_type __n) @@ -2682,8 +2716,8 @@ template template typename enable_if < - __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, - basic_string<_CharT, _Traits, _Allocator>& + __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, + basic_string<_CharT, _Traits, _Allocator>& >::type basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2) @@ -2870,7 +2904,7 @@ typename basic_string<_CharT, _Traits, _Allocator>::size_type basic_string<_CharT, _Traits, _Allocator>::max_size() const _NOEXCEPT { size_type __m = __alloc_traits::max_size(__alloc()); -#if _LIBCPP_BIG_ENDIAN +#ifdef _LIBCPP_BIG_ENDIAN return (__m <= ~__long_mask ? __m : __m/2) - __alignment; #else return __m - __alignment; @@ -3462,8 +3496,8 @@ template template typename enable_if < - __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, - int + __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, + int >::type basic_string<_CharT, _Traits, _Allocator>::compare(size_type __pos1, size_type __n1, diff --git a/contrib/libc++/include/string_view b/contrib/libc++/include/string_view index 4c759ab5daea..72cf816e8d52 100644 --- a/contrib/libc++/include/string_view +++ b/contrib/libc++/include/string_view @@ -143,6 +143,13 @@ namespace std { constexpr size_type find_last_not_of(const charT* s, size_type pos, size_type n) const; constexpr size_type find_last_not_of(const charT* s, size_type pos = npos) const; + constexpr bool starts_with(basic_string_view s) const noexcept; // C++2a + constexpr bool starts_with(charT c) const noexcept; // C++2a + constexpr bool starts_with(const charT* s) const; // C++2a + constexpr bool ends_with(basic_string_view s) const noexcept; // C++2a + constexpr bool ends_with(charT c) const noexcept; // C++2a + constexpr bool ends_with(const charT* s) const; // C++2a + private: const_pointer data_; // exposition only size_type size_; // exposition only @@ -155,10 +162,10 @@ namespace std { template <> struct hash; template <> struct hash; - constexpr basic_string operator "" s( const char *str, size_t len ); // C++17 - constexpr basic_string operator "" s( const wchar_t *str, size_t len ); // C++17 - constexpr basic_string operator "" s( const char16_t *str, size_t len ); // C++17 - constexpr basic_string operator "" s( const char32_t *str, size_t len ); // C++17 + constexpr basic_string_view operator "" sv( const char *str, size_t len ) noexcept; + constexpr basic_string_view operator "" sv( const wchar_t *str, size_t len ) noexcept; + constexpr basic_string_view operator "" sv( const char16_t *str, size_t len ) noexcept; + constexpr basic_string_view operator "" sv( const char32_t *str, size_t len ) noexcept; } // namespace std @@ -186,388 +193,414 @@ _LIBCPP_BEGIN_NAMESPACE_STD template > class _LIBCPP_TEMPLATE_VIS basic_string_view { public: - // types - typedef _Traits traits_type; - typedef _CharT value_type; - typedef const _CharT* pointer; - typedef const _CharT* const_pointer; - typedef const _CharT& reference; - typedef const _CharT& const_reference; - typedef const_pointer const_iterator; // See [string.view.iterators] - typedef const_iterator iterator; - typedef _VSTD::reverse_iterator const_reverse_iterator; - typedef const_reverse_iterator reverse_iterator; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - static _LIBCPP_CONSTEXPR const size_type npos = -1; // size_type(-1); + // types + typedef _Traits traits_type; + typedef _CharT value_type; + typedef _CharT* pointer; + typedef const _CharT* const_pointer; + typedef _CharT& reference; + typedef const _CharT& const_reference; + typedef const_pointer const_iterator; // See [string.view.iterators] + typedef const_iterator iterator; + typedef _VSTD::reverse_iterator const_reverse_iterator; + typedef const_reverse_iterator reverse_iterator; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + static _LIBCPP_CONSTEXPR const size_type npos = -1; // size_type(-1); static_assert(is_pod::value, "Character type of basic_string_view must be a POD"); static_assert((is_same<_CharT, typename traits_type::char_type>::value), "traits_type::char_type must be the same type as CharT"); - // [string.view.cons], construct/copy - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - basic_string_view() _NOEXCEPT : __data (nullptr), __size(0) {} + // [string.view.cons], construct/copy + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + basic_string_view() _NOEXCEPT : __data (nullptr), __size(0) {} - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - basic_string_view(const basic_string_view&) _NOEXCEPT = default; + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + basic_string_view(const basic_string_view&) _NOEXCEPT = default; - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - basic_string_view& operator=(const basic_string_view&) _NOEXCEPT = default; + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + basic_string_view& operator=(const basic_string_view&) _NOEXCEPT = default; - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - basic_string_view(const _CharT* __s, size_type __len) - : __data(__s), __size(__len) - { + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + basic_string_view(const _CharT* __s, size_type __len) _NOEXCEPT + : __data(__s), __size(__len) + { // #if _LIBCPP_STD_VER > 11 // _LIBCPP_ASSERT(__len == 0 || __s != nullptr, "string_view::string_view(_CharT *, size_t): received nullptr"); // #endif - } + } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - basic_string_view(const _CharT* __s) - : __data(__s), __size(_Traits::length(__s)) {} + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + basic_string_view(const _CharT* __s) + : __data(__s), __size(_Traits::length(__s)) {} - // [string.view.iterators], iterators - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_iterator begin() const _NOEXCEPT { return cbegin(); } + // [string.view.iterators], iterators + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_iterator begin() const _NOEXCEPT { return cbegin(); } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_iterator end() const _NOEXCEPT { return cend(); } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_iterator end() const _NOEXCEPT { return cend(); } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_iterator cbegin() const _NOEXCEPT { return __data; } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_iterator cbegin() const _NOEXCEPT { return __data; } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_iterator cend() const _NOEXCEPT { return __data + __size; } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_iterator cend() const _NOEXCEPT { return __data + __size; } - _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY - const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(cend()); } + _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY + const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(cend()); } - _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY - const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(cbegin()); } + _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY + const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(cbegin()); } - _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY - const_reverse_iterator crbegin() const _NOEXCEPT { return const_reverse_iterator(cend()); } + _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY + const_reverse_iterator crbegin() const _NOEXCEPT { return const_reverse_iterator(cend()); } - _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY - const_reverse_iterator crend() const _NOEXCEPT { return const_reverse_iterator(cbegin()); } + _LIBCPP_CONSTEXPR_AFTER_CXX14 _LIBCPP_INLINE_VISIBILITY + const_reverse_iterator crend() const _NOEXCEPT { return const_reverse_iterator(cbegin()); } - // [string.view.capacity], capacity - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - size_type size() const _NOEXCEPT { return __size; } + // [string.view.capacity], capacity + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + size_type size() const _NOEXCEPT { return __size; } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - size_type length() const _NOEXCEPT { return __size; } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + size_type length() const _NOEXCEPT { return __size; } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - size_type max_size() const _NOEXCEPT { return numeric_limits::max(); } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + size_type max_size() const _NOEXCEPT { return numeric_limits::max(); } - _LIBCPP_CONSTEXPR bool _LIBCPP_INLINE_VISIBILITY - empty() const _NOEXCEPT { return __size == 0; } + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR + bool empty() const _NOEXCEPT { return __size == 0; } - // [string.view.access], element access - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_reference operator[](size_type __pos) const _NOEXCEPT { return __data[__pos]; } + // [string.view.access], element access + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_reference operator[](size_type __pos) const _NOEXCEPT { return __data[__pos]; } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_reference at(size_type __pos) const - { - return __pos >= size() - ? (__throw_out_of_range("string_view::at"), __data[0]) - : __data[__pos]; - } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_reference at(size_type __pos) const + { + return __pos >= size() + ? (__throw_out_of_range("string_view::at"), __data[0]) + : __data[__pos]; + } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_reference front() const - { - return _LIBCPP_ASSERT(!empty(), "string_view::front(): string is empty"), __data[0]; - } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_reference front() const + { + return _LIBCPP_ASSERT(!empty(), "string_view::front(): string is empty"), __data[0]; + } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_reference back() const - { - return _LIBCPP_ASSERT(!empty(), "string_view::back(): string is empty"), __data[__size-1]; - } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_reference back() const + { + return _LIBCPP_ASSERT(!empty(), "string_view::back(): string is empty"), __data[__size-1]; + } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - const_pointer data() const _NOEXCEPT { return __data; } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + const_pointer data() const _NOEXCEPT { return __data; } - // [string.view.modifiers], modifiers: - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - void remove_prefix(size_type __n) _NOEXCEPT - { - _LIBCPP_ASSERT(__n <= size(), "remove_prefix() can't remove more than size()"); - __data += __n; - __size -= __n; - } + // [string.view.modifiers], modifiers: + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + void remove_prefix(size_type __n) _NOEXCEPT + { + _LIBCPP_ASSERT(__n <= size(), "remove_prefix() can't remove more than size()"); + __data += __n; + __size -= __n; + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - void remove_suffix(size_type __n) _NOEXCEPT - { - _LIBCPP_ASSERT(__n <= size(), "remove_suffix() can't remove more than size()"); - __size -= __n; - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + void remove_suffix(size_type __n) _NOEXCEPT + { + _LIBCPP_ASSERT(__n <= size(), "remove_suffix() can't remove more than size()"); + __size -= __n; + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - void swap(basic_string_view& __other) _NOEXCEPT - { - const value_type *__p = __data; - __data = __other.__data; - __other.__data = __p; + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + void swap(basic_string_view& __other) _NOEXCEPT + { + const value_type *__p = __data; + __data = __other.__data; + __other.__data = __p; - size_type __sz = __size; - __size = __other.__size; - __other.__size = __sz; - } + size_type __sz = __size; + __size = __other.__size; + __other.__size = __sz; + } - _LIBCPP_INLINE_VISIBILITY - size_type copy(_CharT* __s, size_type __n, size_type __pos = 0) const - { - if (__pos > size()) - __throw_out_of_range("string_view::copy"); - size_type __rlen = _VSTD::min(__n, size() - __pos); - _Traits::copy(__s, data() + __pos, __rlen); - return __rlen; - } + _LIBCPP_INLINE_VISIBILITY + size_type copy(_CharT* __s, size_type __n, size_type __pos = 0) const + { + if (__pos > size()) + __throw_out_of_range("string_view::copy"); + size_type __rlen = _VSTD::min(__n, size() - __pos); + _Traits::copy(__s, data() + __pos, __rlen); + return __rlen; + } - _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY - basic_string_view substr(size_type __pos = 0, size_type __n = npos) const - { - return __pos > size() - ? (__throw_out_of_range("string_view::substr"), basic_string_view()) - : basic_string_view(data() + __pos, _VSTD::min(__n, size() - __pos)); - } + _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY + basic_string_view substr(size_type __pos = 0, size_type __n = npos) const + { + return __pos > size() + ? (__throw_out_of_range("string_view::substr"), basic_string_view()) + : basic_string_view(data() + __pos, _VSTD::min(__n, size() - __pos)); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 int compare(basic_string_view __sv) const _NOEXCEPT - { - size_type __rlen = _VSTD::min( size(), __sv.size()); - int __retval = _Traits::compare(data(), __sv.data(), __rlen); - if ( __retval == 0 ) // first __rlen chars matched - __retval = size() == __sv.size() ? 0 : ( size() < __sv.size() ? -1 : 1 ); - return __retval; - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 int compare(basic_string_view __sv) const _NOEXCEPT + { + size_type __rlen = _VSTD::min( size(), __sv.size()); + int __retval = _Traits::compare(data(), __sv.data(), __rlen); + if ( __retval == 0 ) // first __rlen chars matched + __retval = size() == __sv.size() ? 0 : ( size() < __sv.size() ? -1 : 1 ); + return __retval; + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - int compare(size_type __pos1, size_type __n1, basic_string_view __sv) const - { - return substr(__pos1, __n1).compare(__sv); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + int compare(size_type __pos1, size_type __n1, basic_string_view __sv) const + { + return substr(__pos1, __n1).compare(__sv); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - int compare( size_type __pos1, size_type __n1, - basic_string_view __sv, size_type __pos2, size_type __n2) const - { - return substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + int compare( size_type __pos1, size_type __n1, + basic_string_view __sv, size_type __pos2, size_type __n2) const + { + return substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2)); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - int compare(const _CharT* __s) const _NOEXCEPT - { - return compare(basic_string_view(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + int compare(const _CharT* __s) const _NOEXCEPT + { + return compare(basic_string_view(__s)); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - int compare(size_type __pos1, size_type __n1, const _CharT* __s) const - { - return substr(__pos1, __n1).compare(basic_string_view(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + int compare(size_type __pos1, size_type __n1, const _CharT* __s) const + { + return substr(__pos1, __n1).compare(basic_string_view(__s)); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - int compare(size_type __pos1, size_type __n1, const _CharT* __s, size_type __n2) const - { - return substr(__pos1, __n1).compare(basic_string_view(__s, __n2)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + int compare(size_type __pos1, size_type __n1, const _CharT* __s, size_type __n2) const + { + return substr(__pos1, __n1).compare(basic_string_view(__s, __n2)); + } - // find - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT - { - _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr"); - return __str_find - (data(), size(), __s.data(), __pos, __s.size()); - } + // find + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT + { + _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr"); + return __str_find + (data(), size(), __s.data(), __pos, __s.size()); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find(_CharT __c, size_type __pos = 0) const _NOEXCEPT - { - return __str_find - (data(), size(), __c, __pos); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find(_CharT __c, size_type __pos = 0) const _NOEXCEPT + { + return __str_find + (data(), size(), __c, __pos); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find(const _CharT* __s, size_type __pos, size_type __n) const - { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find(): received nullptr"); - return __str_find - (data(), size(), __s, __pos, __n); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find(const _CharT* __s, size_type __pos, size_type __n) const + { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find(): received nullptr"); + return __str_find + (data(), size(), __s, __pos, __n); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find(const _CharT* __s, size_type __pos = 0) const - { - _LIBCPP_ASSERT(__s != nullptr, "string_view::find(): received nullptr"); - return __str_find - (data(), size(), __s, __pos, traits_type::length(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find(const _CharT* __s, size_type __pos = 0) const + { + _LIBCPP_ASSERT(__s != nullptr, "string_view::find(): received nullptr"); + return __str_find + (data(), size(), __s, __pos, traits_type::length(__s)); + } - // rfind - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type rfind(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT - { - _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr"); - return __str_rfind - (data(), size(), __s.data(), __pos, __s.size()); - } + // rfind + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type rfind(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT + { + _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr"); + return __str_rfind + (data(), size(), __s.data(), __pos, __s.size()); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type rfind(_CharT __c, size_type __pos = npos) const _NOEXCEPT - { - return __str_rfind - (data(), size(), __c, __pos); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type rfind(_CharT __c, size_type __pos = npos) const _NOEXCEPT + { + return __str_rfind + (data(), size(), __c, __pos); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type rfind(const _CharT* __s, size_type __pos, size_type __n) const - { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::rfind(): received nullptr"); - return __str_rfind - (data(), size(), __s, __pos, __n); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type rfind(const _CharT* __s, size_type __pos, size_type __n) const + { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::rfind(): received nullptr"); + return __str_rfind + (data(), size(), __s, __pos, __n); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type rfind(const _CharT* __s, size_type __pos=npos) const - { - _LIBCPP_ASSERT(__s != nullptr, "string_view::rfind(): received nullptr"); - return __str_rfind - (data(), size(), __s, __pos, traits_type::length(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type rfind(const _CharT* __s, size_type __pos=npos) const + { + _LIBCPP_ASSERT(__s != nullptr, "string_view::rfind(): received nullptr"); + return __str_rfind + (data(), size(), __s, __pos, traits_type::length(__s)); + } - // find_first_of - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT - { - _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_first_of(): received nullptr"); - return __str_find_first_of - (data(), size(), __s.data(), __pos, __s.size()); - } + // find_first_of + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT + { + _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_first_of(): received nullptr"); + return __str_find_first_of + (data(), size(), __s.data(), __pos, __s.size()); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_of(_CharT __c, size_type __pos = 0) const _NOEXCEPT - { return find(__c, __pos); } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_of(_CharT __c, size_type __pos = 0) const _NOEXCEPT + { return find(__c, __pos); } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_of(const _CharT* __s, size_type __pos, size_type __n) const - { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_first_of(): received nullptr"); - return __str_find_first_of - (data(), size(), __s, __pos, __n); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_of(const _CharT* __s, size_type __pos, size_type __n) const + { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_first_of(): received nullptr"); + return __str_find_first_of + (data(), size(), __s, __pos, __n); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_of(const _CharT* __s, size_type __pos=0) const - { - _LIBCPP_ASSERT(__s != nullptr, "string_view::find_first_of(): received nullptr"); - return __str_find_first_of - (data(), size(), __s, __pos, traits_type::length(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_of(const _CharT* __s, size_type __pos=0) const + { + _LIBCPP_ASSERT(__s != nullptr, "string_view::find_first_of(): received nullptr"); + return __str_find_first_of + (data(), size(), __s, __pos, traits_type::length(__s)); + } - // find_last_of - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_of(basic_string_view __s, size_type __pos=npos) const _NOEXCEPT - { - _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_last_of(): received nullptr"); - return __str_find_last_of - (data(), size(), __s.data(), __pos, __s.size()); - } + // find_last_of + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_of(basic_string_view __s, size_type __pos=npos) const _NOEXCEPT + { + _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_last_of(): received nullptr"); + return __str_find_last_of + (data(), size(), __s.data(), __pos, __s.size()); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_of(_CharT __c, size_type __pos = npos) const _NOEXCEPT - { return rfind(__c, __pos); } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_of(_CharT __c, size_type __pos = npos) const _NOEXCEPT + { return rfind(__c, __pos); } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_of(const _CharT* __s, size_type __pos, size_type __n) const - { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_last_of(): received nullptr"); - return __str_find_last_of - (data(), size(), __s, __pos, __n); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_of(const _CharT* __s, size_type __pos, size_type __n) const + { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_last_of(): received nullptr"); + return __str_find_last_of + (data(), size(), __s, __pos, __n); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_of(const _CharT* __s, size_type __pos=npos) const - { - _LIBCPP_ASSERT(__s != nullptr, "string_view::find_last_of(): received nullptr"); - return __str_find_last_of - (data(), size(), __s, __pos, traits_type::length(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_of(const _CharT* __s, size_type __pos=npos) const + { + _LIBCPP_ASSERT(__s != nullptr, "string_view::find_last_of(): received nullptr"); + return __str_find_last_of + (data(), size(), __s, __pos, traits_type::length(__s)); + } - // find_first_not_of - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_not_of(basic_string_view __s, size_type __pos=0) const _NOEXCEPT - { - _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_first_not_of(): received nullptr"); - return __str_find_first_not_of - (data(), size(), __s.data(), __pos, __s.size()); - } + // find_first_not_of + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_not_of(basic_string_view __s, size_type __pos=0) const _NOEXCEPT + { + _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_first_not_of(): received nullptr"); + return __str_find_first_not_of + (data(), size(), __s.data(), __pos, __s.size()); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_not_of(_CharT __c, size_type __pos=0) const _NOEXCEPT - { - return __str_find_first_not_of - (data(), size(), __c, __pos); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_not_of(_CharT __c, size_type __pos=0) const _NOEXCEPT + { + return __str_find_first_not_of + (data(), size(), __c, __pos); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_not_of(const _CharT* __s, size_type __pos, size_type __n) const - { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_first_not_of(): received nullptr"); - return __str_find_first_not_of - (data(), size(), __s, __pos, __n); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_not_of(const _CharT* __s, size_type __pos, size_type __n) const + { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_first_not_of(): received nullptr"); + return __str_find_first_not_of + (data(), size(), __s, __pos, __n); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_first_not_of(const _CharT* __s, size_type __pos=0) const - { - _LIBCPP_ASSERT(__s != nullptr, "string_view::find_first_not_of(): received nullptr"); - return __str_find_first_not_of - (data(), size(), __s, __pos, traits_type::length(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_first_not_of(const _CharT* __s, size_type __pos=0) const + { + _LIBCPP_ASSERT(__s != nullptr, "string_view::find_first_not_of(): received nullptr"); + return __str_find_first_not_of + (data(), size(), __s, __pos, traits_type::length(__s)); + } - // find_last_not_of - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_not_of(basic_string_view __s, size_type __pos=npos) const _NOEXCEPT - { - _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_last_not_of(): received nullptr"); - return __str_find_last_not_of - (data(), size(), __s.data(), __pos, __s.size()); - } + // find_last_not_of + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_not_of(basic_string_view __s, size_type __pos=npos) const _NOEXCEPT + { + _LIBCPP_ASSERT(__s.size() == 0 || __s.data() != nullptr, "string_view::find_last_not_of(): received nullptr"); + return __str_find_last_not_of + (data(), size(), __s.data(), __pos, __s.size()); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_not_of(_CharT __c, size_type __pos=npos) const _NOEXCEPT - { - return __str_find_last_not_of - (data(), size(), __c, __pos); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_not_of(_CharT __c, size_type __pos=npos) const _NOEXCEPT + { + return __str_find_last_not_of + (data(), size(), __c, __pos); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_not_of(const _CharT* __s, size_type __pos, size_type __n) const - { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_last_not_of(): received nullptr"); - return __str_find_last_not_of - (data(), size(), __s, __pos, __n); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_not_of(const _CharT* __s, size_type __pos, size_type __n) const + { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string_view::find_last_not_of(): received nullptr"); + return __str_find_last_not_of + (data(), size(), __s, __pos, __n); + } - _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY - size_type find_last_not_of(const _CharT* __s, size_type __pos=npos) const - { - _LIBCPP_ASSERT(__s != nullptr, "string_view::find_last_not_of(): received nullptr"); - return __str_find_last_not_of - (data(), size(), __s, __pos, traits_type::length(__s)); - } + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + size_type find_last_not_of(const _CharT* __s, size_type __pos=npos) const + { + _LIBCPP_ASSERT(__s != nullptr, "string_view::find_last_not_of(): received nullptr"); + return __str_find_last_not_of + (data(), size(), __s, __pos, traits_type::length(__s)); + } + +#if _LIBCPP_STD_VER > 17 + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool starts_with(basic_string_view __s) const _NOEXCEPT + { return size() >= __s.size() && compare(0, __s.size(), __s) == 0; } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool starts_with(value_type __c) const _NOEXCEPT + { return !empty() && _Traits::eq(front(), __c); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool starts_with(const value_type* __s) const _NOEXCEPT + { return starts_with(basic_string_view(__s)); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool ends_with(basic_string_view __s) const _NOEXCEPT + { return size() >= __s.size() && compare(size() - __s.size(), npos, __s) == 0; } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool ends_with(value_type __c) const _NOEXCEPT + { return !empty() && _Traits::eq(back(), __c); } + + _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY + bool ends_with(const value_type* __s) const _NOEXCEPT + { return ends_with(basic_string_view(__s)); } +#endif private: - const value_type* __data; - size_type __size; + const value_type* __data; + size_type __size; }; @@ -576,28 +609,28 @@ private: template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator==(basic_string_view<_CharT, _Traits> __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - if ( __lhs.size() != __rhs.size()) return false; - return __lhs.compare(__rhs) == 0; + if ( __lhs.size() != __rhs.size()) return false; + return __lhs.compare(__rhs) == 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator==(basic_string_view<_CharT, _Traits> __lhs, - typename common_type >::type __rhs) _NOEXCEPT + typename common_type >::type __rhs) _NOEXCEPT { - if ( __lhs.size() != __rhs.size()) return false; - return __lhs.compare(__rhs) == 0; + if ( __lhs.size() != __rhs.size()) return false; + return __lhs.compare(__rhs) == 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator==(typename common_type >::type __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - if ( __lhs.size() != __rhs.size()) return false; - return __lhs.compare(__rhs) == 0; + if ( __lhs.size() != __rhs.size()) return false; + return __lhs.compare(__rhs) == 0; } @@ -606,29 +639,29 @@ template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator!=(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - if ( __lhs.size() != __rhs.size()) - return true; - return __lhs.compare(__rhs) != 0; + if ( __lhs.size() != __rhs.size()) + return true; + return __lhs.compare(__rhs) != 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator!=(basic_string_view<_CharT, _Traits> __lhs, - typename common_type >::type __rhs) _NOEXCEPT + typename common_type >::type __rhs) _NOEXCEPT { - if ( __lhs.size() != __rhs.size()) - return true; - return __lhs.compare(__rhs) != 0; + if ( __lhs.size() != __rhs.size()) + return true; + return __lhs.compare(__rhs) != 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator!=(typename common_type >::type __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - if ( __lhs.size() != __rhs.size()) - return true; - return __lhs.compare(__rhs) != 0; + if ( __lhs.size() != __rhs.size()) + return true; + return __lhs.compare(__rhs) != 0; } @@ -637,23 +670,23 @@ template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator<(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) < 0; + return __lhs.compare(__rhs) < 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator<(basic_string_view<_CharT, _Traits> __lhs, - typename common_type >::type __rhs) _NOEXCEPT + typename common_type >::type __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) < 0; + return __lhs.compare(__rhs) < 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator<(typename common_type >::type __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) < 0; + return __lhs.compare(__rhs) < 0; } @@ -662,23 +695,23 @@ template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator> (basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) > 0; + return __lhs.compare(__rhs) > 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator>(basic_string_view<_CharT, _Traits> __lhs, - typename common_type >::type __rhs) _NOEXCEPT + typename common_type >::type __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) > 0; + return __lhs.compare(__rhs) > 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator>(typename common_type >::type __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) > 0; + return __lhs.compare(__rhs) > 0; } @@ -687,23 +720,23 @@ template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator<=(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) <= 0; + return __lhs.compare(__rhs) <= 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator<=(basic_string_view<_CharT, _Traits> __lhs, - typename common_type >::type __rhs) _NOEXCEPT + typename common_type >::type __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) <= 0; + return __lhs.compare(__rhs) <= 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator<=(typename common_type >::type __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) <= 0; + return __lhs.compare(__rhs) <= 0; } @@ -712,24 +745,24 @@ template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator>=(basic_string_view<_CharT, _Traits> __lhs, basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) >= 0; + return __lhs.compare(__rhs) >= 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator>=(basic_string_view<_CharT, _Traits> __lhs, - typename common_type >::type __rhs) _NOEXCEPT + typename common_type >::type __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) >= 0; + return __lhs.compare(__rhs) >= 0; } template _LIBCPP_CONSTEXPR_AFTER_CXX11 _LIBCPP_INLINE_VISIBILITY bool operator>=(typename common_type >::type __lhs, - basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT + basic_string_view<_CharT, _Traits> __rhs) _NOEXCEPT { - return __lhs.compare(__rhs) >= 0; + return __lhs.compare(__rhs) >= 0; } typedef basic_string_view string_view; @@ -760,25 +793,25 @@ inline namespace literals inline namespace string_view_literals { inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR - basic_string_view operator "" sv(const char *__str, size_t __len) + basic_string_view operator "" sv(const char *__str, size_t __len) _NOEXCEPT { return basic_string_view (__str, __len); } inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR - basic_string_view operator "" sv(const wchar_t *__str, size_t __len) + basic_string_view operator "" sv(const wchar_t *__str, size_t __len) _NOEXCEPT { return basic_string_view (__str, __len); } inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR - basic_string_view operator "" sv(const char16_t *__str, size_t __len) + basic_string_view operator "" sv(const char16_t *__str, size_t __len) _NOEXCEPT { return basic_string_view (__str, __len); } inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR - basic_string_view operator "" sv(const char32_t *__str, size_t __len) + basic_string_view operator "" sv(const char32_t *__str, size_t __len) _NOEXCEPT { return basic_string_view (__str, __len); } diff --git a/contrib/libc++/include/system_error b/contrib/libc++/include/system_error index a29807db0d65..c577edceee82 100644 --- a/contrib/libc++/include/system_error +++ b/contrib/libc++/include/system_error @@ -47,10 +47,10 @@ template struct is_error_condition_enum : public false_type {}; template -constexpr size_t is_error_condition_enum_v = is_error_condition_enum<_Tp>::value; // C++17 +inline constexpr size_t is_error_condition_enum_v = is_error_condition_enum<_Tp>::value; // C++17 template -constexpr size_t is_error_code_enum_v = is_error_code_enum<_Tp>::value; // C++17 +inline constexpr size_t is_error_code_enum_v = is_error_code_enum<_Tp>::value; // C++17 class error_code { @@ -230,6 +230,7 @@ template <> struct hash; #include #include #include <__functional_base> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header @@ -245,7 +246,7 @@ struct _LIBCPP_TEMPLATE_VIS is_error_code_enum #if _LIBCPP_STD_VER > 14 template -constexpr size_t is_error_code_enum_v = is_error_code_enum<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr size_t is_error_code_enum_v = is_error_code_enum<_Tp>::value; #endif // is_error_condition_enum @@ -256,7 +257,7 @@ struct _LIBCPP_TEMPLATE_VIS is_error_condition_enum #if _LIBCPP_STD_VER > 14 template -constexpr size_t is_error_condition_enum_v = is_error_condition_enum<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr size_t is_error_condition_enum_v = is_error_condition_enum<_Tp>::value; #endif // Some error codes are not present on all platforms, so we provide equivalents diff --git a/contrib/libc++/include/tuple b/contrib/libc++/include/tuple index a52b934aaca7..31578d1557a9 100644 --- a/contrib/libc++/include/tuple +++ b/contrib/libc++/include/tuple @@ -70,7 +70,7 @@ public: void swap(tuple&) noexcept(AND(swap(declval(), declval())...)); }; -const unspecified ignore; +inline constexpr unspecified ignore; template tuple make_tuple(T&&...); // constexpr in C++14 template tuple forward_as_tuple(T&&...) noexcept; // constexpr in C++14 @@ -87,7 +87,7 @@ template template class tuple_size; // undefined template class tuple_size>; template - constexpr size_t tuple_size_v = tuple_size::value; // C++17 + inline constexpr size_t tuple_size_v = tuple_size::value; // C++17 template class tuple_element; // undefined template class tuple_element>; template @@ -929,7 +929,7 @@ public: void swap(tuple&) _NOEXCEPT {} }; -#ifdef __cpp_deduction_guides +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES // NOTE: These are not yet standardized, but are required to simulate the // implicit deduction guide that should be generated had libc++ declared the // tuple-like constructors "correctly" @@ -1012,10 +1012,10 @@ constexpr size_t __find_idx(size_t __i, const bool (&__matches)[_Nx]) { template struct __find_exactly_one_checked { - static constexpr bool __matches[] = {is_same<_T1, _Args>::value...}; + static constexpr bool __matches[sizeof...(_Args)] = {is_same<_T1, _Args>::value...}; static constexpr size_t value = __find_detail::__find_idx(0, __matches); - static_assert (value != __not_found, "type not found in type list" ); - static_assert(value != __ambiguous,"type occurs more than once in type list"); + static_assert(value != __not_found, "type not found in type list" ); + static_assert(value != __ambiguous, "type occurs more than once in type list"); }; template @@ -1079,7 +1079,7 @@ struct __ignore_t }; namespace { - constexpr __ignore_t ignore = __ignore_t(); + _LIBCPP_INLINE_VAR constexpr __ignore_t ignore = __ignore_t(); } template @@ -1368,7 +1368,7 @@ pair<_T1, _T2>::pair(piecewise_construct_t, #if _LIBCPP_STD_VER > 14 template -constexpr size_t tuple_size_v = tuple_size<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr size_t tuple_size_v = tuple_size<_Tp>::value; #define _LIBCPP_NOEXCEPT_RETURN(...) noexcept(noexcept(__VA_ARGS__)) { return __VA_ARGS__; } diff --git a/contrib/libc++/include/type_traits b/contrib/libc++/include/type_traits index 6c111abfd1e8..eb443ee0abd6 100644 --- a/contrib/libc++/include/type_traits +++ b/contrib/libc++/include/type_traits @@ -132,30 +132,32 @@ namespace std template struct has_virtual_destructor; + template struct has_unique_object_representations; // C++17 + // Relationships between types: template struct is_same; template struct is_base_of; template struct is_convertible; - template struct is_callable; // not defined - template - struct is_callable; + template struct is_invocable; + template struct is_invocable_r; - template struct is_nothrow_callable; // not defined - template - struct is_nothrow_callable; + template struct is_nothrow_invocable; + template struct is_nothrow_invocable_r; // Alignment properties and transformations: template struct alignment_of; template struct aligned_storage; template struct aligned_union; + template struct remove_cvref; // C++20 template struct decay; template struct common_type; template struct underlying_type; template class result_of; // undefined template class result_of; + template struct invoke_result; // C++17 // const-volatile modifications: template @@ -202,6 +204,8 @@ namespace std using aligned_storage_t = typename aligned_storage::type; // C++14 template using aligned_union_t = typename aligned_union::type; // C++14 + template + using remove_cvref_t = typename remove_cvref::type; // C++20 template using decay_t = typename decay::type; // C++14 template @@ -212,176 +216,184 @@ namespace std using common_type_t = typename common_type::type; // C++14 template using underlying_type_t = typename underlying_type::type; // C++14 - template - using result_of_t = typename result_of::type; // C++14 + template + using result_of_t = typename result_of::type; // C++14 + template + using invoke_result_t = typename invoke_result::type; // C++17 template using void_t = void; // C++17 // See C++14 20.10.4.1, primary type categories - template constexpr bool is_void_v + template inline constexpr bool is_void_v = is_void::value; // C++17 - template constexpr bool is_null_pointer_v + template inline constexpr bool is_null_pointer_v = is_null_pointer::value; // C++17 - template constexpr bool is_integral_v + template inline constexpr bool is_integral_v = is_integral::value; // C++17 - template constexpr bool is_floating_point_v + template inline constexpr bool is_floating_point_v = is_floating_point::value; // C++17 - template constexpr bool is_array_v + template inline constexpr bool is_array_v = is_array::value; // C++17 - template constexpr bool is_pointer_v + template inline constexpr bool is_pointer_v = is_pointer::value; // C++17 - template constexpr bool is_lvalue_reference_v + template inline constexpr bool is_lvalue_reference_v = is_lvalue_reference::value; // C++17 - template constexpr bool is_rvalue_reference_v + template inline constexpr bool is_rvalue_reference_v = is_rvalue_reference::value; // C++17 - template constexpr bool is_member_object_pointer_v + template inline constexpr bool is_member_object_pointer_v = is_member_object_pointer::value; // C++17 - template constexpr bool is_member_function_pointer_v + template inline constexpr bool is_member_function_pointer_v = is_member_function_pointer::value; // C++17 - template constexpr bool is_enum_v + template inline constexpr bool is_enum_v = is_enum::value; // C++17 - template constexpr bool is_union_v + template inline constexpr bool is_union_v = is_union::value; // C++17 - template constexpr bool is_class_v + template inline constexpr bool is_class_v = is_class::value; // C++17 - template constexpr bool is_function_v + template inline constexpr bool is_function_v = is_function::value; // C++17 // See C++14 20.10.4.2, composite type categories - template constexpr bool is_reference_v + template inline constexpr bool is_reference_v = is_reference::value; // C++17 - template constexpr bool is_arithmetic_v + template inline constexpr bool is_arithmetic_v = is_arithmetic::value; // C++17 - template constexpr bool is_fundamental_v + template inline constexpr bool is_fundamental_v = is_fundamental::value; // C++17 - template constexpr bool is_object_v + template inline constexpr bool is_object_v = is_object::value; // C++17 - template constexpr bool is_scalar_v + template inline constexpr bool is_scalar_v = is_scalar::value; // C++17 - template constexpr bool is_compound_v + template inline constexpr bool is_compound_v = is_compound::value; // C++17 - template constexpr bool is_member_pointer_v + template inline constexpr bool is_member_pointer_v = is_member_pointer::value; // C++17 // See C++14 20.10.4.3, type properties - template constexpr bool is_const_v + template inline constexpr bool is_const_v = is_const::value; // C++17 - template constexpr bool is_volatile_v + template inline constexpr bool is_volatile_v = is_volatile::value; // C++17 - template constexpr bool is_trivial_v + template inline constexpr bool is_trivial_v = is_trivial::value; // C++17 - template constexpr bool is_trivially_copyable_v + template inline constexpr bool is_trivially_copyable_v = is_trivially_copyable::value; // C++17 - template constexpr bool is_standard_layout_v + template inline constexpr bool is_standard_layout_v = is_standard_layout::value; // C++17 - template constexpr bool is_pod_v + template inline constexpr bool is_pod_v = is_pod::value; // C++17 - template constexpr bool is_literal_type_v + template inline constexpr bool is_literal_type_v = is_literal_type::value; // C++17 - template constexpr bool is_empty_v + template inline constexpr bool is_empty_v = is_empty::value; // C++17 - template constexpr bool is_polymorphic_v + template inline constexpr bool is_polymorphic_v = is_polymorphic::value; // C++17 - template constexpr bool is_abstract_v + template inline constexpr bool is_abstract_v = is_abstract::value; // C++17 - template constexpr bool is_final_v + template inline constexpr bool is_final_v = is_final::value; // C++17 - template constexpr bool is_aggregate_v + template inline constexpr bool is_aggregate_v = is_aggregate::value; // C++17 - template constexpr bool is_signed_v + template inline constexpr bool is_signed_v = is_signed::value; // C++17 - template constexpr bool is_unsigned_v + template inline constexpr bool is_unsigned_v = is_unsigned::value; // C++17 - template constexpr bool is_constructible_v + template inline constexpr bool is_constructible_v = is_constructible::value; // C++17 - template constexpr bool is_default_constructible_v + template inline constexpr bool is_default_constructible_v = is_default_constructible::value; // C++17 - template constexpr bool is_copy_constructible_v + template inline constexpr bool is_copy_constructible_v = is_copy_constructible::value; // C++17 - template constexpr bool is_move_constructible_v + template inline constexpr bool is_move_constructible_v = is_move_constructible::value; // C++17 - template constexpr bool is_assignable_v + template inline constexpr bool is_assignable_v = is_assignable::value; // C++17 - template constexpr bool is_copy_assignable_v + template inline constexpr bool is_copy_assignable_v = is_copy_assignable::value; // C++17 - template constexpr bool is_move_assignable_v + template inline constexpr bool is_move_assignable_v = is_move_assignable::value; // C++17 - template constexpr bool is_swappable_with_v + template inline constexpr bool is_swappable_with_v = is_swappable_with::value; // C++17 - template constexpr bool is_swappable_v + template inline constexpr bool is_swappable_v = is_swappable::value; // C++17 - template constexpr bool is_destructible_v + template inline constexpr bool is_destructible_v = is_destructible::value; // C++17 - template constexpr bool is_trivially_constructible_v + template inline constexpr bool is_trivially_constructible_v = is_trivially_constructible::value; // C++17 - template constexpr bool is_trivially_default_constructible_v + template inline constexpr bool is_trivially_default_constructible_v = is_trivially_default_constructible::value; // C++17 - template constexpr bool is_trivially_copy_constructible_v + template inline constexpr bool is_trivially_copy_constructible_v = is_trivially_copy_constructible::value; // C++17 - template constexpr bool is_trivially_move_constructible_v + template inline constexpr bool is_trivially_move_constructible_v = is_trivially_move_constructible::value; // C++17 - template constexpr bool is_trivially_assignable_v + template inline constexpr bool is_trivially_assignable_v = is_trivially_assignable::value; // C++17 - template constexpr bool is_trivially_copy_assignable_v + template inline constexpr bool is_trivially_copy_assignable_v = is_trivially_copy_assignable::value; // C++17 - template constexpr bool is_trivially_move_assignable_v + template inline constexpr bool is_trivially_move_assignable_v = is_trivially_move_assignable::value; // C++17 - template constexpr bool is_trivially_destructible_v + template inline constexpr bool is_trivially_destructible_v = is_trivially_destructible::value; // C++17 - template constexpr bool is_nothrow_constructible_v + template inline constexpr bool is_nothrow_constructible_v = is_nothrow_constructible::value; // C++17 - template constexpr bool is_nothrow_default_constructible_v + template inline constexpr bool is_nothrow_default_constructible_v = is_nothrow_default_constructible::value; // C++17 - template constexpr bool is_nothrow_copy_constructible_v + template inline constexpr bool is_nothrow_copy_constructible_v = is_nothrow_copy_constructible::value; // C++17 - template constexpr bool is_nothrow_move_constructible_v + template inline constexpr bool is_nothrow_move_constructible_v = is_nothrow_move_constructible::value; // C++17 - template constexpr bool is_nothrow_assignable_v + template inline constexpr bool is_nothrow_assignable_v = is_nothrow_assignable::value; // C++17 - template constexpr bool is_nothrow_copy_assignable_v + template inline constexpr bool is_nothrow_copy_assignable_v = is_nothrow_copy_assignable::value; // C++17 - template constexpr bool is_nothrow_move_assignable_v + template inline constexpr bool is_nothrow_move_assignable_v = is_nothrow_move_assignable::value; // C++17 - template constexpr bool is_nothrow_swappable_with_v + template inline constexpr bool is_nothrow_swappable_with_v = is_nothrow_swappable_with::value; // C++17 - template constexpr bool is_nothrow_swappable_v + template inline constexpr bool is_nothrow_swappable_v = is_nothrow_swappable::value; // C++17 - template constexpr bool is_nothrow_destructible_v + template inline constexpr bool is_nothrow_destructible_v = is_nothrow_destructible::value; // C++17 - template constexpr bool has_virtual_destructor_v + template inline constexpr bool has_virtual_destructor_v = has_virtual_destructor::value; // C++17 + template inline constexpr bool has_unique_object_representations_v // C++17 + = has_unique_object_representations::value; // See C++14 20.10.5, type property queries - template constexpr size_t alignment_of_v + template inline constexpr size_t alignment_of_v = alignment_of::value; // C++17 - template constexpr size_t rank_v + template inline constexpr size_t rank_v = rank::value; // C++17 - template constexpr size_t extent_v + template inline constexpr size_t extent_v = extent::value; // C++17 // See C++14 20.10.6, type relations - template constexpr bool is_same_v + template inline constexpr bool is_same_v = is_same::value; // C++17 - template constexpr bool is_base_of_v + template inline constexpr bool is_base_of_v = is_base_of::value; // C++17 - template constexpr bool is_convertible_v + template inline constexpr bool is_convertible_v = is_convertible::value; // C++17 - template constexpr bool is_callable_v - = is_callable::value; // C++17 - template constexpr bool is_nothrow_callable_v - = is_nothrow_callable::value; // C++17 + template inline constexpr bool is_invocable_v + = is_invocable::value; // C++17 + template inline constexpr bool is_invocable_r_v + = is_invocable_r::value; // C++17 + template inline constexpr bool is_nothrow_invocable_v + = is_nothrow_invocable::value; // C++17 + template inline constexpr bool is_nothrow_invocable_r_v + = is_nothrow_invocable_r::value; // C++17 // [meta.logical], logical operator traits: template struct conjunction; // C++17 template - constexpr bool conjunction_v = conjunction::value; // C++17 + inline constexpr bool conjunction_v = conjunction::value; // C++17 template struct disjunction; // C++17 template - constexpr bool disjunction_v = disjunction::value; // C++17 + inline constexpr bool disjunction_v = disjunction::value; // C++17 template struct negation; // C++17 template - constexpr bool negation_v = negation::value; // C++17 + inline constexpr bool negation_v = negation::value; // C++17 } @@ -611,7 +623,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_const : public fa template struct _LIBCPP_TEMPLATE_VIS is_const<_Tp const> : public true_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_const_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_const_v = is_const<_Tp>::value; #endif @@ -621,7 +634,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_volatile : pub template struct _LIBCPP_TEMPLATE_VIS is_volatile<_Tp volatile> : public true_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_volatile_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_volatile_v = is_volatile<_Tp>::value; #endif @@ -658,7 +672,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_void : public __libcpp_is_void::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_void_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_void_v = is_void<_Tp>::value; #endif @@ -675,7 +690,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_null_pointer : public __is_nullptr_t_impl::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_null_pointer_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_null_pointer_v = is_null_pointer<_Tp>::value; #endif #endif @@ -709,7 +725,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_integral : public __libcpp_is_integral::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_integral_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_integral_v = is_integral<_Tp>::value; #endif @@ -724,7 +741,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_floating_point : public __libcpp_is_floating_point::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_floating_point_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_floating_point_v = is_floating_point<_Tp>::value; #endif @@ -738,7 +756,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[_Np]> : public true_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_array_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_array_v = is_array<_Tp>::value; #endif @@ -751,7 +770,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_pointer : public __libcpp_is_pointer::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_pointer_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_pointer_v = is_pointer<_Tp>::value; #endif @@ -772,13 +792,16 @@ template struct _LIBCPP_TEMPLATE_VIS is_reference<_Tp&&> : public tr #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_reference_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_reference_v = is_reference<_Tp>::value; -template _LIBCPP_CONSTEXPR bool is_lvalue_reference_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_lvalue_reference_v = is_lvalue_reference<_Tp>::value; -template _LIBCPP_CONSTEXPR bool is_rvalue_reference_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_rvalue_reference_v = is_rvalue_reference<_Tp>::value; #endif // is_union @@ -797,7 +820,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_union #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_union_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_union_v = is_union<_Tp>::value; #endif @@ -822,7 +846,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_class #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_class_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_class_v = is_class<_Tp>::value; #endif @@ -832,7 +857,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_same : template struct _LIBCPP_TEMPLATE_VIS is_same<_Tp, _Tp> : public true_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_same_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_same_v = is_same<_Tp, _Up>::value; #endif @@ -862,7 +888,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_function : public __libcpp_is_function<_Tp> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_function_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_function_v = is_function<_Tp>::value; #endif @@ -889,7 +916,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_member_function_pointer : public __libcpp_is_member_function_pointer::type>::type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_member_function_pointer_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_member_function_pointer_v = is_member_function_pointer<_Tp>::value; #endif @@ -902,7 +930,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_member_pointer : public __libcpp_is_member_pointer::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_member_pointer_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_member_pointer_v = is_member_pointer<_Tp>::value; #endif @@ -913,7 +942,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_member_object_pointer !is_member_function_pointer<_Tp>::value> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_member_object_pointer_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_member_object_pointer_v = is_member_object_pointer<_Tp>::value; #endif @@ -941,7 +971,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_enum #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_enum_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_enum_v = is_enum<_Tp>::value; #endif @@ -952,7 +983,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_arithmetic is_floating_point<_Tp>::value> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_arithmetic_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_arithmetic_v = is_arithmetic<_Tp>::value; #endif @@ -964,7 +996,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_fundamental is_arithmetic<_Tp>::value> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_fundamental_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_fundamental_v = is_fundamental<_Tp>::value; #endif @@ -980,7 +1013,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_scalar template <> struct _LIBCPP_TEMPLATE_VIS is_scalar : public true_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_scalar_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_scalar_v = is_scalar<_Tp>::value; #endif @@ -993,7 +1027,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_object is_class<_Tp>::value > {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_object_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_object_v = is_object<_Tp>::value; #endif @@ -1003,7 +1038,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_compound : public integral_constant::value> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_compound_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_compound_v = is_compound<_Tp>::value; #endif @@ -1141,6 +1177,17 @@ template struct __is_same_uncvref : is_same::type, typename __uncvref<_Up>::type> {}; +#if _LIBCPP_STD_VER > 17 +// aligned_union - same as __uncvref +template +struct remove_cvref { + using type = remove_cv_t>; +}; + +template using remove_cvref_t = typename remove_cvref<_Tp>::type; +#endif + + struct __any { __any(...); @@ -1191,7 +1238,8 @@ template struct __libcpp_is_signed<_Tp, false> : public false_type { template struct _LIBCPP_TEMPLATE_VIS is_signed : public __libcpp_is_signed<_Tp> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_signed_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_signed_v = is_signed<_Tp>::value; #endif @@ -1211,7 +1259,8 @@ template struct __libcpp_is_unsigned<_Tp, false> : public false_type template struct _LIBCPP_TEMPLATE_VIS is_unsigned : public __libcpp_is_unsigned<_Tp> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_unsigned_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_unsigned_v = is_unsigned<_Tp>::value; #endif @@ -1225,7 +1274,8 @@ template struct _LIBCPP_TEMPLATE_VIS rank<_Tp[_Np]> : public integral_constant::value + 1> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR size_t rank_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR size_t rank_v = rank<_Tp>::value; #endif @@ -1243,7 +1293,8 @@ template struct _LIBCPP_TEMPLATE_VIS exten : public integral_constant::value> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR size_t extent_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR size_t extent_v = extent<_Tp, _Ip>::value; #endif @@ -1315,7 +1366,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_abstract : public integral_constant {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_abstract_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_abstract_v = is_abstract<_Tp>::value; #endif @@ -1335,7 +1387,8 @@ is_final : public integral_constant {}; #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_final_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_final_v = is_final<_Tp>::value; #endif @@ -1347,7 +1400,8 @@ is_aggregate : public integral_constant {}; #if !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) template -constexpr bool is_aggregate_v = is_aggregate<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr bool is_aggregate_v + = is_aggregate<_Tp>::value; #endif #endif // _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_IS_AGGREGATE) @@ -1388,7 +1442,8 @@ struct _LIBCPP_TEMPLATE_VIS is_base_of #endif // _LIBCPP_HAS_IS_BASE_OF #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_base_of_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_base_of_v = is_base_of<_Bp, _Dp>::value; #endif @@ -1478,7 +1533,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_convertible #endif // __has_feature(is_convertible_to) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_convertible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_convertible_v = is_convertible<_From, _To>::value; #endif @@ -1514,7 +1570,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_empty : public __libcpp_empt #endif // __has_feature(is_empty) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_empty_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_empty_v = is_empty<_Tp>::value; #endif @@ -1539,7 +1596,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_polymorphic #endif // __has_feature(is_polymorphic) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_polymorphic_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_polymorphic_v = is_polymorphic<_Tp>::value; #endif @@ -1558,17 +1616,35 @@ template struct _LIBCPP_TEMPLATE_VIS has_virtual_destructor #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool has_virtual_destructor_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool has_virtual_destructor_v = has_virtual_destructor<_Tp>::value; #endif +// has_unique_object_representations + +#if _LIBCPP_STD_VER > 14 && defined(_LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS) + +template struct _LIBCPP_TEMPLATE_VIS has_unique_object_representations + : public integral_constant>)> {}; + +#if !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool has_unique_object_representations_v + = has_unique_object_representations<_Tp>::value; +#endif + +#endif + // alignment_of template struct _LIBCPP_TEMPLATE_VIS alignment_of : public integral_constant {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR size_t alignment_of_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR size_t alignment_of_v = alignment_of<_Tp>::value; #endif @@ -2107,7 +2183,8 @@ struct is_assignable : public __is_assignable_imp<_Tp, _Arg> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_assignable_v = is_assignable<_Tp, _Arg>::value; #endif @@ -2118,7 +2195,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_copy_assignable typename add_lvalue_reference::type>::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_copy_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_copy_assignable_v = is_copy_assignable<_Tp>::value; #endif @@ -2133,7 +2211,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_move_assignable #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_move_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_move_assignable_v = is_move_assignable<_Tp>::value; #endif @@ -2196,7 +2275,8 @@ struct is_destructible : public _VSTD::false_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_destructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_destructible_v = is_destructible<_Tp>::value; #endif @@ -3092,6 +3172,14 @@ template false_type __is_constructible2_test(__any, _A0&, _A1&); +template +decltype((_Tp(_VSTD::declval<_A0>(), _VSTD::declval<_A1>(), _VSTD::declval<_A2>()), true_type())) +__is_constructible3_test(_Tp&, _A0&, _A1&, _A2&); + +template +false_type +__is_constructible3_test(__any, _A0&, _A1&, _A2&); + template struct __is_constructible0_imp // false, _Tp is not a scalar : public common_type @@ -3116,6 +3204,14 @@ struct __is_constructible2_imp // false, _Tp is not a scalar >::type {}; +template +struct __is_constructible3_imp // false, _Tp is not a scalar + : public common_type + < + decltype(__is_constructible3_test(declval<_Tp&>(), declval<_A0>(), declval<_A1>(), declval<_A2>())) + >::type + {}; + // handle scalars and reference types // Scalars are default constructible, references are not @@ -3135,6 +3231,11 @@ struct __is_constructible2_imp : public false_type {}; +template +struct __is_constructible3_imp + : public false_type + {}; + // Treat scalars and reference types separately template @@ -3155,6 +3256,12 @@ struct __is_constructible2_void_check _Tp, _A0, _A1> {}; +template +struct __is_constructible3_void_check + : public __is_constructible3_imp::value || is_reference<_Tp>::value, + _Tp, _A0, _A1, _A2> + {}; + // If any of T or Args is void, is_constructible should be false template @@ -3172,17 +3279,24 @@ struct __is_constructible2_void_check : public false_type {}; +template +struct __is_constructible3_void_check + : public false_type + {}; + // is_constructible entry point template + class _A1 = __is_construct::__nat, + class _A2 = __is_construct::__nat> struct _LIBCPP_TEMPLATE_VIS is_constructible - : public __is_constructible2_void_check::value + : public __is_constructible3_void_check::value || is_abstract<_Tp>::value || is_function<_Tp>::value || is_void<_A0>::value - || is_void<_A1>::value, - _Tp, _A0, _A1> + || is_void<_A1>::value + || is_void<_A2>::value, + _Tp, _A0, _A1, _A2> {}; template @@ -3202,6 +3316,16 @@ struct _LIBCPP_TEMPLATE_VIS is_constructible<_Tp, _A0, __is_construct::__nat> _Tp, _A0> {}; +template +struct _LIBCPP_TEMPLATE_VIS is_constructible<_Tp, _A0, _A1, __is_construct::__nat> + : public __is_constructible2_void_check::value + || is_abstract<_Tp>::value + || is_function<_Tp>::value + || is_void<_A0>::value + || is_void<_A1>::value, + _Tp, _A0, _A1> + {}; + // Array types are default constructible if their element type // is default constructible @@ -3220,6 +3344,11 @@ struct __is_constructible2_imp : public false_type {}; +template +struct __is_constructible3_imp + : public false_type + {}; + // Incomplete array types are not constructible template @@ -3237,11 +3366,17 @@ struct __is_constructible2_imp : public false_type {}; +template +struct __is_constructible3_imp + : public false_type + {}; + #endif // __has_feature(is_constructible) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) && !defined(_LIBCPP_HAS_NO_VARIADICS) -template _LIBCPP_CONSTEXPR bool is_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_constructible_v = is_constructible<_Tp, _Args...>::value; #endif @@ -3253,7 +3388,8 @@ struct _LIBCPP_TEMPLATE_VIS is_default_constructible {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_default_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_default_constructible_v = is_default_constructible<_Tp>::value; #endif @@ -3265,7 +3401,8 @@ struct _LIBCPP_TEMPLATE_VIS is_copy_constructible typename add_lvalue_reference::type>::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_copy_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_copy_constructible_v = is_copy_constructible<_Tp>::value; #endif @@ -3281,7 +3418,8 @@ struct _LIBCPP_TEMPLATE_VIS is_move_constructible {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_move_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_move_constructible_v = is_move_constructible<_Tp>::value; #endif @@ -3413,7 +3551,8 @@ struct _LIBCPP_TEMPLATE_VIS is_trivially_constructible<_Tp, _Tp&, #endif // _LIBCPP_HAS_NO_VARIADICS #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) && !defined(_LIBCPP_HAS_NO_VARIADICS) -template _LIBCPP_CONSTEXPR bool is_trivially_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_constructible_v = is_trivially_constructible<_Tp, _Args...>::value; #endif @@ -3424,7 +3563,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_default_constructi {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_default_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_default_constructible_v = is_trivially_default_constructible<_Tp>::value; #endif @@ -3435,7 +3575,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_constructible {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_copy_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_copy_constructible_v = is_trivially_copy_constructible<_Tp>::value; #endif @@ -3450,7 +3591,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_move_constructible {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_move_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_move_constructible_v = is_trivially_move_constructible<_Tp>::value; #endif @@ -3493,7 +3635,8 @@ struct is_trivially_assignable<_Tp&, _Tp&&> #endif // !__has_feature(is_trivially_assignable) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_assignable_v = is_trivially_assignable<_Tp, _Arg>::value; #endif @@ -3504,7 +3647,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_assignable typename add_lvalue_reference::type>::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_copy_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_copy_assignable_v = is_trivially_copy_assignable<_Tp>::value; #endif @@ -3520,7 +3664,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_move_assignable {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_move_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_move_assignable_v = is_trivially_move_assignable<_Tp>::value; #endif @@ -3546,7 +3691,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible<_Tp[] #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_destructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_destructible_v = is_trivially_destructible<_Tp>::value; #endif @@ -3711,7 +3857,8 @@ struct _LIBCPP_TEMPLATE_VIS is_nothrow_constructible<_Tp, _Tp&, #endif // __has_feature(is_nothrow_constructible) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) && !defined(_LIBCPP_HAS_NO_VARIADICS) -template _LIBCPP_CONSTEXPR bool is_nothrow_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_constructible_v = is_nothrow_constructible<_Tp, _Args...>::value; #endif @@ -3722,7 +3869,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_nothrow_default_constructibl {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_default_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_default_constructible_v = is_nothrow_default_constructible<_Tp>::value; #endif @@ -3733,7 +3881,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_constructible typename add_lvalue_reference::type>::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_copy_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_copy_constructible_v = is_nothrow_copy_constructible<_Tp>::value; #endif @@ -3748,7 +3897,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_constructible {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_move_constructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_move_constructible_v = is_nothrow_move_constructible<_Tp>::value; #endif @@ -3821,7 +3971,8 @@ struct is_nothrow_assignable<_Tp&, _Tp&&> #endif // __has_feature(cxx_noexcept) #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_assignable_v = is_nothrow_assignable<_Tp, _Arg>::value; #endif @@ -3832,7 +3983,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_assignable typename add_lvalue_reference::type>::type> {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_copy_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_copy_assignable_v = is_nothrow_copy_assignable<_Tp>::value; #endif @@ -3848,7 +4000,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_assignable {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_move_assignable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_move_assignable_v = is_nothrow_move_assignable<_Tp>::value; #endif @@ -3914,7 +4067,8 @@ struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp[]> #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_nothrow_destructible_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_nothrow_destructible_v = is_nothrow_destructible<_Tp>::value; #endif @@ -3936,7 +4090,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_pod #endif #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_pod_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_pod_v = is_pod<_Tp>::value; #endif @@ -3952,7 +4107,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_literal_type {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_literal_type_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_literal_type_v = is_literal_type<_Tp>::value; #endif @@ -3967,7 +4123,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_standard_layout {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_standard_layout_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_standard_layout_v = is_standard_layout<_Tp>::value; #endif @@ -3984,7 +4141,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivially_copyable {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivially_copyable_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivially_copyable_v = is_trivially_copyable<_Tp>::value; #endif @@ -4000,7 +4158,8 @@ template struct _LIBCPP_TEMPLATE_VIS is_trivial {}; #if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES) -template _LIBCPP_CONSTEXPR bool is_trivial_v +template +_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_trivial_v = is_trivial<_Tp>::value; #endif @@ -4387,6 +4546,13 @@ using __nothrow_invokable_r = _Ret, _Fp, _Args... >; +template +using __nothrow_invokable = + __nothrow_invokable_r_imp< + __invokable<_Fp, _Args...>::value, + true, void, _Fp, _Args... + >; + template struct __invoke_of : public enable_if< @@ -4409,30 +4575,52 @@ template using result_of_t = typename result_of<_Tp>::type; #if _LIBCPP_STD_VER > 14 -// is_callable +// invoke_result -template -struct _LIBCPP_TEMPLATE_VIS is_callable; +template +struct _LIBCPP_TEMPLATE_VIS invoke_result + : __invoke_of<_Fn, _Args...> +{ +}; -template -struct _LIBCPP_TEMPLATE_VIS is_callable<_Fn(_Args...), _Ret> +template +using invoke_result_t = typename invoke_result<_Fn, _Args...>::type; + +// is_invocable + +template +struct _LIBCPP_TEMPLATE_VIS is_invocable + : integral_constant::value> {}; + +template +struct _LIBCPP_TEMPLATE_VIS is_invocable_r : integral_constant::value> {}; -template -constexpr bool is_callable_v = is_callable<_Fn, _Ret>::value; +template +_LIBCPP_INLINE_VAR constexpr bool is_invocable_v + = is_invocable<_Fn, _Args...>::value; -// is_nothrow_callable +template +_LIBCPP_INLINE_VAR constexpr bool is_invocable_r_v + = is_invocable_r<_Ret, _Fn, _Args...>::value; -template -struct _LIBCPP_TEMPLATE_VIS is_nothrow_callable; +// is_nothrow_invocable -template -struct _LIBCPP_TEMPLATE_VIS is_nothrow_callable<_Fn(_Args...), _Ret> - : integral_constant::value> -{}; +template +struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable + : integral_constant::value> {}; -template -constexpr bool is_nothrow_callable_v = is_nothrow_callable<_Fn, _Ret>::value; +template +struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable_r + : integral_constant::value> {}; + +template +_LIBCPP_INLINE_VAR constexpr bool is_nothrow_invocable_v + = is_nothrow_invocable<_Fn, _Args...>::value; + +template +_LIBCPP_INLINE_VAR constexpr bool is_nothrow_invocable_r_v + = is_nothrow_invocable_r<_Ret, _Fn, _Args...>::value; #endif // _LIBCPP_STD_VER > 14 @@ -4572,16 +4760,20 @@ struct _LIBCPP_TEMPLATE_VIS is_nothrow_swappable }; template -constexpr bool is_swappable_with_v = is_swappable_with<_Tp, _Up>::value; +_LIBCPP_INLINE_VAR constexpr bool is_swappable_with_v + = is_swappable_with<_Tp, _Up>::value; template -constexpr bool is_swappable_v = is_swappable<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr bool is_swappable_v + = is_swappable<_Tp>::value; template -constexpr bool is_nothrow_swappable_with_v = is_nothrow_swappable_with<_Tp, _Up>::value; +_LIBCPP_INLINE_VAR constexpr bool is_nothrow_swappable_with_v + = is_nothrow_swappable_with<_Tp, _Up>::value; template -constexpr bool is_nothrow_swappable_v = is_nothrow_swappable<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr bool is_nothrow_swappable_v + = is_nothrow_swappable<_Tp>::value; #endif // _LIBCPP_STD_VER > 14 @@ -4638,6 +4830,11 @@ long long __convert_to_integral(long long __val) { return __val; } inline _LIBCPP_INLINE_VISIBILITY unsigned long long __convert_to_integral(unsigned long long __val) {return __val; } +template +inline _LIBCPP_INLINE_VISIBILITY +typename enable_if::value, long long>::type + __convert_to_integral(_Fp __val) { return __val; } + #ifndef _LIBCPP_HAS_NO_INT128 inline _LIBCPP_INLINE_VISIBILITY __int128_t __convert_to_integral(__int128_t __val) { return __val; } @@ -4693,15 +4890,21 @@ template using void_t = void; # ifndef _LIBCPP_HAS_NO_VARIADICS template struct conjunction : __and_<_Args...> {}; -template constexpr bool conjunction_v = conjunction<_Args...>::value; +template +_LIBCPP_INLINE_VAR constexpr bool conjunction_v + = conjunction<_Args...>::value; template struct disjunction : __or_<_Args...> {}; -template constexpr bool disjunction_v = disjunction<_Args...>::value; +template +_LIBCPP_INLINE_VAR constexpr bool disjunction_v + = disjunction<_Args...>::value; template struct negation : __not_<_Tp> {}; -template constexpr bool negation_v = negation<_Tp>::value; +template +_LIBCPP_INLINE_VAR constexpr bool negation_v + = negation<_Tp>::value; # endif // _LIBCPP_HAS_NO_VARIADICS #endif // _LIBCPP_STD_VER > 14 @@ -4747,26 +4950,26 @@ namespace std // purposefully not versioned template constexpr typename enable_if, byte>::type & operator<<=(byte& __lhs, _Integer __shift) noexcept - { return __lhs = byte(static_cast(__lhs) << __shift); } + { return __lhs = __lhs << __shift; } template constexpr typename enable_if, byte>::type operator<< (byte __lhs, _Integer __shift) noexcept - { return byte(static_cast(__lhs) << __shift); } + { return static_cast(static_cast(static_cast(__lhs) << __shift)); } template constexpr typename enable_if, byte>::type & operator>>=(byte& __lhs, _Integer __shift) noexcept - { return __lhs = byte(static_cast(__lhs) >> __shift); } + { return __lhs = __lhs >> __shift; } template constexpr typename enable_if, byte>::type operator>> (byte __lhs, _Integer __shift) noexcept - { return byte(static_cast(__lhs) >> __shift); } + { return static_cast(static_cast(static_cast(__lhs) >> __shift)); } template constexpr typename enable_if, _Integer>::type - to_integer(byte __b) noexcept { return _Integer(__b); } + to_integer(byte __b) noexcept { return static_cast<_Integer>(__b); } } #endif diff --git a/contrib/libc++/include/typeinfo b/contrib/libc++/include/typeinfo index 8624b349764a..7e8d3990ed98 100644 --- a/contrib/libc++/include/typeinfo +++ b/contrib/libc++/include/typeinfo @@ -69,18 +69,17 @@ public: #pragma GCC system_header #endif -#if defined(_LIBCPP_ABI_MICROSOFT) -#include -#elif defined(_LIBCPP_NONUNIQUE_RTTI_BIT) +#if !defined(_LIBCPP_ABI_MICROSOFT) +#if defined(_LIBCPP_NONUNIQUE_RTTI_BIT) #define _LIBCPP_HAS_NONUNIQUE_TYPEINFO #else #define _LIBCPP_HAS_UNIQUE_TYPEINFO #endif +#endif namespace std // purposefully not using versioning namespace { -#if !defined(_LIBCPP_ABI_MICROSOFT) class _LIBCPP_EXCEPTION_ABI type_info { type_info& operator=(const type_info&); @@ -92,7 +91,17 @@ class _LIBCPP_EXCEPTION_ABI type_info { return __builtin_strcmp(name(), __arg.name()); } #endif +#if defined(_LIBCPP_ABI_MICROSOFT) + mutable struct { + const char *__undecorated_name; + const char __decorated_name[1]; + } __data; + + int __compare(const type_info &__rhs) const _NOEXCEPT; +#endif // _LIBCPP_ABI_MICROSOFT + protected: +#if !defined(_LIBCPP_ABI_MICROSOFT) #if defined(_LIBCPP_HAS_NONUNIQUE_TYPEINFO) // A const char* with the non-unique RTTI bit possibly set. uintptr_t __type_name; @@ -106,11 +115,27 @@ protected: _LIBCPP_INLINE_VISIBILITY explicit type_info(const char* __n) : __type_name(__n) {} #endif +#endif // ! _LIBCPP_ABI_MICROSOFT public: _LIBCPP_AVAILABILITY_TYPEINFO_VTABLE virtual ~type_info(); +#if defined(_LIBCPP_ABI_MICROSOFT) + const char *name() const _NOEXCEPT; + + _LIBCPP_INLINE_VISIBILITY + bool before(const type_info& __arg) const _NOEXCEPT { + return __compare(__arg) < 0; + } + + size_t hash_code() const _NOEXCEPT; + + _LIBCPP_INLINE_VISIBILITY + bool operator==(const type_info& __arg) const _NOEXCEPT { + return __compare(__arg) == 0; + } +#else #if defined(_LIBCPP_HAS_NONUNIQUE_TYPEINFO) _LIBCPP_INLINE_VISIBILITY const char* name() const _NOEXCEPT @@ -167,6 +192,7 @@ public: bool operator==(const type_info& __arg) const _NOEXCEPT { return __type_name == __arg.__type_name; } #endif +#endif // _LIBCPP_ABI_MICROSOFT _LIBCPP_INLINE_VISIBILITY bool operator!=(const type_info& __arg) const _NOEXCEPT @@ -191,8 +217,6 @@ public: virtual const char* what() const _NOEXCEPT; }; -#endif // !_LIBCPP_ABI_MICROSOFT - } // std _LIBCPP_BEGIN_NAMESPACE_STD diff --git a/contrib/libc++/include/unordered_map b/contrib/libc++/include/unordered_map index 0546c0e2f9f2..725cb6af2759 100644 --- a/contrib/libc++/include/unordered_map +++ b/contrib/libc++/include/unordered_map @@ -885,7 +885,7 @@ public: allocator_type get_allocator() const _NOEXCEPT {return allocator_type(__table_.__node_alloc());} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __table_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __table_.size();} @@ -1634,7 +1634,7 @@ public: allocator_type get_allocator() const _NOEXCEPT {return allocator_type(__table_.__node_alloc());} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __table_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __table_.size();} diff --git a/contrib/libc++/include/unordered_set b/contrib/libc++/include/unordered_set index a14fb0004922..3ae024a45eed 100644 --- a/contrib/libc++/include/unordered_set +++ b/contrib/libc++/include/unordered_set @@ -450,7 +450,7 @@ public: allocator_type get_allocator() const _NOEXCEPT {return allocator_type(__table_.__node_alloc());} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __table_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __table_.size();} @@ -968,7 +968,7 @@ public: allocator_type get_allocator() const _NOEXCEPT {return allocator_type(__table_.__node_alloc());} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __table_.size() == 0;} _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __table_.size();} diff --git a/contrib/libc++/include/utility b/contrib/libc++/include/utility index 00e3cd208ecb..6fa2e15d1448 100644 --- a/contrib/libc++/include/utility +++ b/contrib/libc++/include/utility @@ -99,7 +99,7 @@ void swap(pair& x, pair& y) noexcept(noexcept(x.swap(y))); struct piecewise_construct_t { }; -constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t(); +inline constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t(); template class tuple_size; template class tuple_element; @@ -296,7 +296,7 @@ struct _LIBCPP_TEMPLATE_VIS piecewise_construct_t { }; #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_UTILITY) extern const piecewise_construct_t piecewise_construct;// = piecewise_construct_t(); #else -constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t(); +/* _LIBCPP_INLINE_VAR */ constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t(); #endif #if defined(_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR) @@ -545,6 +545,11 @@ private: #endif }; +#ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES +template +pair(_T1, _T2) -> pair<_T1, _T2>; +#endif // _LIBCPP_HAS_NO_DEDUCTION_GUIDES + template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 bool @@ -899,30 +904,21 @@ _T1 exchange(_T1& __obj, _T2 && __new_value) struct _LIBCPP_TYPE_VIS in_place_t { explicit in_place_t() = default; }; -#ifndef _LIBCPP_HAS_NO_INLINE_VARIABLES -inline -#endif -constexpr in_place_t in_place{}; +_LIBCPP_INLINE_VAR constexpr in_place_t in_place{}; template struct _LIBCPP_TEMPLATE_VIS in_place_type_t { explicit in_place_type_t() = default; }; template -#ifndef _LIBCPP_HAS_NO_INLINE_VARIABLES -inline -#endif -constexpr in_place_type_t<_Tp> in_place_type{}; +_LIBCPP_INLINE_VAR constexpr in_place_type_t<_Tp> in_place_type{}; template struct _LIBCPP_TYPE_VIS in_place_index_t { explicit in_place_index_t() = default; }; template -#ifndef _LIBCPP_HAS_NO_INLINE_VARIABLES -inline -#endif -constexpr in_place_index_t<_Idx> in_place_index{}; +_LIBCPP_INLINE_VAR constexpr in_place_index_t<_Idx> in_place_index{}; template struct __is_inplace_type_imp : false_type {}; template struct __is_inplace_type_imp> : true_type {}; diff --git a/contrib/libc++/include/variant b/contrib/libc++/include/variant index f8d3e28bae45..987b8a7982df 100644 --- a/contrib/libc++/include/variant +++ b/contrib/libc++/include/variant @@ -76,7 +76,7 @@ namespace std { template struct variant_size; // undefined template - constexpr size_t variant_size_v = variant_size::value; + inline constexpr size_t variant_size_v = variant_size::value; template struct variant_size; template struct variant_size; @@ -97,7 +97,7 @@ namespace std { template struct variant_alternative>; - constexpr size_t variant_npos = -1; + inline constexpr size_t variant_npos = -1; // 20.7.4, value access template @@ -207,11 +207,15 @@ namespace std { #include #include #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + namespace std { // explicitly not using versioning namespace class _LIBCPP_EXCEPTION_ABI bad_variant_access : public exception { @@ -242,7 +246,7 @@ template struct _LIBCPP_TEMPLATE_VIS variant_size; template -constexpr size_t variant_size_v = variant_size<_Tp>::value; +_LIBCPP_INLINE_VAR constexpr size_t variant_size_v = variant_size<_Tp>::value; template struct _LIBCPP_TEMPLATE_VIS variant_size : variant_size<_Tp> {}; @@ -282,8 +286,29 @@ struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, variant<_Types...>> { using type = __type_pack_element<_Ip, _Types...>; }; -constexpr size_t variant_npos = static_cast(-1); -constexpr unsigned int __variant_npos = static_cast(-1); +_LIBCPP_INLINE_VAR constexpr size_t variant_npos = static_cast(-1); + +constexpr int __choose_index_type(unsigned int __num_elem) { + if (__num_elem < std::numeric_limits::max()) + return 0; + if (__num_elem < std::numeric_limits::max()) + return 1; + return 2; +} + +template +using __variant_index_t = +#ifndef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION + unsigned int; +#else + std::tuple_element_t< + __choose_index_type(_NumAlts), + std::tuple + >; +#endif + +template +constexpr _IndexType __variant_npos = static_cast<_IndexType>(-1); namespace __find_detail { @@ -557,7 +582,7 @@ struct __variant { private: template static constexpr void __std_visit_exhaustive_visitor_check() { - static_assert(is_callable_v<_Visitor(_Values...)>, + static_assert(is_invocable_v<_Visitor, _Values...>, "`std::visit` requires the visitor to be exhaustive."); } @@ -647,9 +672,11 @@ _LIBCPP_VARIANT_UNION(_Trait::_Unavailable, ~__union() = delete;); template <_Trait _DestructibleTrait, class... _Types> class _LIBCPP_TEMPLATE_VIS __base { public: + using __index_t = __variant_index_t; + inline _LIBCPP_INLINE_VISIBILITY explicit constexpr __base(__valueless_t tag) noexcept - : __data(tag), __index(__variant_npos) {} + : __data(tag), __index(__variant_npos<__index_t>) {} template inline _LIBCPP_INLINE_VISIBILITY @@ -665,7 +692,7 @@ public: inline _LIBCPP_INLINE_VISIBILITY constexpr size_t index() const noexcept { - return __index == __variant_npos ? variant_npos : __index; + return __index == __variant_npos<__index_t> ? variant_npos : __index; } protected: @@ -685,7 +712,7 @@ protected: static constexpr size_t __size() { return sizeof...(_Types); } __union<_DestructibleTrait, 0, _Types...> __data; - unsigned int __index; + __index_t __index; friend struct __access::__base; friend struct __visitation::__base; @@ -696,10 +723,11 @@ class _LIBCPP_TEMPLATE_VIS __destructor; #define _LIBCPP_VARIANT_DESTRUCTOR(destructible_trait, destructor, destroy) \ template \ - class _LIBCPP_TEMPLATE_VIS __destructor<__traits<_Types...>, \ + class _LIBCPP_TEMPLATE_VIS __destructor<__traits<_Types...>, \ destructible_trait> \ : public __base { \ using __base_type = __base; \ + using __index_t = typename __base_type::__index_t; \ \ public: \ using __base_type::__base_type; \ @@ -719,7 +747,7 @@ class _LIBCPP_TEMPLATE_VIS __destructor; _LIBCPP_VARIANT_DESTRUCTOR( _Trait::_TriviallyAvailable, ~__destructor() = default;, - void __destroy() noexcept { this->__index = __variant_npos; }); + void __destroy() noexcept { this->__index = __variant_npos<__index_t>; }); _LIBCPP_VARIANT_DESTRUCTOR( _Trait::_Available, @@ -733,7 +761,7 @@ _LIBCPP_VARIANT_DESTRUCTOR( }, *this); } - this->__index = __variant_npos; + this->__index = __variant_npos<__index_t>; }); _LIBCPP_VARIANT_DESTRUCTOR( @@ -1561,4 +1589,6 @@ struct _LIBCPP_TEMPLATE_VIS hash { _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP_VARIANT diff --git a/contrib/libc++/include/vector b/contrib/libc++/include/vector index b2f8f092c63d..54b1e8831d5b 100644 --- a/contrib/libc++/include/vector +++ b/contrib/libc++/include/vector @@ -634,7 +634,7 @@ public: _LIBCPP_INLINE_VISIBILITY size_type capacity() const _NOEXCEPT {return __base::capacity();} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return this->__begin_ == this->__end_;} size_type max_size() const _NOEXCEPT; @@ -2290,7 +2290,7 @@ public: _LIBCPP_INLINE_VISIBILITY size_type size() const _NOEXCEPT {return __size_;} - _LIBCPP_INLINE_VISIBILITY + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool empty() const _NOEXCEPT {return __size_ == 0;} void reserve(size_type __n); diff --git a/contrib/libc++/include/wchar.h b/contrib/libc++/include/wchar.h index a5666e193a5b..f74fe6ddcff4 100644 --- a/contrib/libc++/include/wchar.h +++ b/contrib/libc++/include/wchar.h @@ -125,6 +125,10 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len, # if __GLIBC_PREREQ(2, 10) # define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1 # endif +#elif defined(_LIBCPP_MSVCRT) +# if defined(_CRT_CONST_CORRECT_OVERLOADS) +# define _LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS 1 +# endif #endif #if defined(__cplusplus) && !defined(_LIBCPP_WCHAR_H_HAS_CONST_OVERLOADS) && defined(_LIBCPP_PREFERRED_OVERLOAD) diff --git a/contrib/libc++/src/exception.cpp b/contrib/libc++/src/exception.cpp index 4359d126173a..3d2dcfd5b10e 100644 --- a/contrib/libc++/src/exception.cpp +++ b/contrib/libc++/src/exception.cpp @@ -31,6 +31,7 @@ #include "support/runtime/exception_glibcxx.ipp" #include "support/runtime/exception_pointer_glibcxx.ipp" #else +#include "include/atomic_support.h" #include "support/runtime/exception_fallback.ipp" #include "support/runtime/exception_pointer_unimplemented.ipp" #endif diff --git a/contrib/libc++/src/experimental/filesystem/directory_iterator.cpp b/contrib/libc++/src/experimental/filesystem/directory_iterator.cpp index 25135857d40c..a552fdc4461d 100644 --- a/contrib/libc++/src/experimental/filesystem/directory_iterator.cpp +++ b/contrib/libc++/src/experimental/filesystem/directory_iterator.cpp @@ -296,24 +296,43 @@ void recursive_directory_iterator::__advance(error_code* ec) { } bool recursive_directory_iterator::__try_recursion(error_code *ec) { - bool rec_sym = bool(options() & directory_options::follow_directory_symlink); + auto& curr_it = __imp_->__stack_.top(); - if (is_directory(curr_it.__entry_.status()) && - (!is_symlink(curr_it.__entry_.symlink_status()) || rec_sym)) - { - std::error_code m_ec; + bool skip_rec = false; + std::error_code m_ec; + if (!rec_sym) { + file_status st = curr_it.__entry_.symlink_status(m_ec); + if (m_ec && status_known(st)) + m_ec.clear(); + if (m_ec || is_symlink(st) || !is_directory(st)) + skip_rec = true; + } else { + file_status st = curr_it.__entry_.status(m_ec); + if (m_ec && status_known(st)) + m_ec.clear(); + if (m_ec || !is_directory(st)) + skip_rec = true; + } + + if (!skip_rec) { __dir_stream new_it(curr_it.__entry_.path(), __imp_->__options_, m_ec); if (new_it.good()) { __imp_->__stack_.push(_VSTD::move(new_it)); return true; } - if (m_ec) { - __imp_.reset(); - set_or_throw(m_ec, ec, - "recursive_directory_iterator::operator++()"); + } + if (m_ec) { + const bool allow_eacess = bool(__imp_->__options_ + & directory_options::skip_permission_denied); + if (m_ec.value() == EACCES && allow_eacess) { + if (ec) ec->clear(); + } else { + __imp_.reset(); + set_or_throw(m_ec, ec, + "recursive_directory_iterator::operator++()"); } } return false; diff --git a/contrib/libc++/src/experimental/filesystem/operations.cpp b/contrib/libc++/src/experimental/filesystem/operations.cpp index 641a3c53636d..662fa7b8626a 100644 --- a/contrib/libc++/src/experimental/filesystem/operations.cpp +++ b/contrib/libc++/src/experimental/filesystem/operations.cpp @@ -182,20 +182,20 @@ void __copy(const path& from, const path& to, copy_options options, const bool sym_status2 = bool(options & copy_options::copy_symlinks); - std::error_code m_ec; + std::error_code m_ec1; struct ::stat f_st = {}; const file_status f = sym_status || sym_status2 - ? detail::posix_lstat(from, f_st, &m_ec) - : detail::posix_stat(from, f_st, &m_ec); - if (m_ec) - return set_or_throw(m_ec, ec, "copy", from, to); + ? detail::posix_lstat(from, f_st, &m_ec1) + : detail::posix_stat(from, f_st, &m_ec1); + if (m_ec1) + return set_or_throw(m_ec1, ec, "copy", from, to); struct ::stat t_st = {}; - const file_status t = sym_status ? detail::posix_lstat(to, t_st, &m_ec) - : detail::posix_stat(to, t_st, &m_ec); + const file_status t = sym_status ? detail::posix_lstat(to, t_st, &m_ec1) + : detail::posix_stat(to, t_st, &m_ec1); if (not status_known(t)) - return set_or_throw(m_ec, ec, "copy", from, to); + return set_or_throw(m_ec1, ec, "copy", from, to); if (!exists(f) || is_other(f) || is_other(t) || (is_directory(f) && is_regular_file(t)) @@ -249,9 +249,9 @@ void __copy(const path& from, const path& to, copy_options options, directory_iterator it = ec ? directory_iterator(from, *ec) : directory_iterator(from); if (ec && *ec) { return; } - std::error_code m_ec; - for (; it != directory_iterator(); it.increment(m_ec)) { - if (m_ec) return set_or_throw(m_ec, ec, "copy", from, to); + std::error_code m_ec2; + for (; it != directory_iterator(); it.increment(m_ec2)) { + if (m_ec2) return set_or_throw(m_ec2, ec, "copy", from, to); __copy(it->path(), to / it->path().filename(), options | copy_options::__in_recursive_copy, ec); if (ec && *ec) { return; } @@ -512,6 +512,7 @@ using StatT = struct stat; #if defined(__APPLE__) TimeSpec extract_mtime(StatT const& st) { return st.st_mtimespec; } +__attribute__((unused)) // Suppress warning TimeSpec extract_atime(StatT const& st) { return st.st_atimespec; } #else TimeSpec extract_mtime(StatT const& st) { return st.st_mtim; } diff --git a/contrib/libc++/src/include/atomic_support.h b/contrib/libc++/src/include/atomic_support.h index 08847e630705..ccd8d78d3d71 100644 --- a/contrib/libc++/src/include/atomic_support.h +++ b/contrib/libc++/src/include/atomic_support.h @@ -16,6 +16,7 @@ #if defined(__clang__) && __has_builtin(__atomic_load_n) \ && __has_builtin(__atomic_store_n) \ && __has_builtin(__atomic_add_fetch) \ + && __has_builtin(__atomic_exchange_n) \ && __has_builtin(__atomic_compare_exchange_n) \ && defined(__ATOMIC_RELAXED) \ && defined(__ATOMIC_CONSUME) \ @@ -82,6 +83,14 @@ _ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a, return __atomic_add_fetch(__val, __a, __order); } +template +inline _LIBCPP_INLINE_VISIBILITY +_ValueType __libcpp_atomic_exchange(_ValueType* __target, + _ValueType __value, int __order = _AO_Seq) +{ + return __atomic_exchange_n(__target, __value, __order); +} + template inline _LIBCPP_INLINE_VISIBILITY bool __libcpp_atomic_compare_exchange(_ValueType* __val, @@ -135,6 +144,16 @@ _ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a, return *__val += __a; } +template +inline _LIBCPP_INLINE_VISIBILITY +_ValueType __libcpp_atomic_exchange(_ValueType* __target, + _ValueType __value, int __order = _AO_Seq) +{ + _ValueType old = *__target; + *__target = __value; + return old; +} + template inline _LIBCPP_INLINE_VISIBILITY bool __libcpp_atomic_compare_exchange(_ValueType* __val, diff --git a/contrib/libc++/src/include/config_elast.h b/contrib/libc++/src/include/config_elast.h index 8328978ef953..4c4d853c2a68 100644 --- a/contrib/libc++/src/include/config_elast.h +++ b/contrib/libc++/src/include/config_elast.h @@ -24,7 +24,7 @@ #define _LIBCPP_ELAST __ELASTERROR #elif defined(__Fuchsia__) // No _LIBCPP_ELAST needed on Fuchsia -#elif defined(__linux__) +#elif defined(__linux__) || defined(_LIBCPP_HAS_MUSL_LIBC) #define _LIBCPP_ELAST 4095 #elif defined(__APPLE__) // No _LIBCPP_ELAST needed on Apple diff --git a/contrib/libc++/src/include/refstring.h b/contrib/libc++/src/include/refstring.h index f0d5b4456da5..702f2b7388da 100644 --- a/contrib/libc++/src/include/refstring.h +++ b/contrib/libc++/src/include/refstring.h @@ -18,6 +18,7 @@ #include #include #endif +#include "atomic_support.h" _LIBCPP_BEGIN_NAMESPACE_STD @@ -83,7 +84,7 @@ __libcpp_refstring::__libcpp_refstring(const __libcpp_refstring &s) _NOEXCEPT : __imp_(s.__imp_) { if (__uses_refcount()) - __sync_add_and_fetch(&rep_from_data(__imp_)->count, 1); + __libcpp_atomic_add(&rep_from_data(__imp_)->count, 1); } inline @@ -92,10 +93,10 @@ __libcpp_refstring& __libcpp_refstring::operator=(__libcpp_refstring const& s) _ struct _Rep_base *old_rep = rep_from_data(__imp_); __imp_ = s.__imp_; if (__uses_refcount()) - __sync_add_and_fetch(&rep_from_data(__imp_)->count, 1); + __libcpp_atomic_add(&rep_from_data(__imp_)->count, 1); if (adjust_old_count) { - if (__sync_add_and_fetch(&old_rep->count, count_t(-1)) < 0) + if (__libcpp_atomic_add(&old_rep->count, count_t(-1)) < 0) { ::operator delete(old_rep); } @@ -107,7 +108,7 @@ inline __libcpp_refstring::~__libcpp_refstring() { if (__uses_refcount()) { _Rep_base* rep = rep_from_data(__imp_); - if (__sync_add_and_fetch(&rep->count, count_t(-1)) < 0) { + if (__libcpp_atomic_add(&rep->count, count_t(-1)) < 0) { ::operator delete(rep); } } diff --git a/contrib/libc++/src/iostream.cpp b/contrib/libc++/src/iostream.cpp index 534b47a917d5..2b47cf25b640 100644 --- a/contrib/libc++/src/iostream.cpp +++ b/contrib/libc++/src/iostream.cpp @@ -11,19 +11,23 @@ #include "string" #include "new" +#define _str(s) #s +#define str(s) _str(s) +#define _LIBCPP_NAMESPACE_STR str(_LIBCPP_NAMESPACE) + _LIBCPP_BEGIN_NAMESPACE_STD #ifndef _LIBCPP_HAS_NO_STDIN _ALIGNAS_TYPE (istream) _LIBCPP_FUNC_VIS char cin[sizeof(istream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?cin@__1@std@@3V?$basic_istream@DU?$char_traits@D@__1@std@@@12@A") +__asm__("?cin@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_istream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdinbuf ) static char __cin[sizeof(__stdinbuf )]; static mbstate_t mb_cin; _ALIGNAS_TYPE (wistream) _LIBCPP_FUNC_VIS char wcin[sizeof(wistream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wcin@__1@std@@3V?$basic_istream@_WU?$char_traits@_W@__1@std@@@12@A") +__asm__("?wcin@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_istream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdinbuf ) static char __wcin[sizeof(__stdinbuf )]; @@ -33,14 +37,14 @@ static mbstate_t mb_wcin; #ifndef _LIBCPP_HAS_NO_STDOUT _ALIGNAS_TYPE (ostream) _LIBCPP_FUNC_VIS char cout[sizeof(ostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?cout@__1@std@@3V?$basic_ostream@DU?$char_traits@D@__1@std@@@12@A") +__asm__("?cout@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __cout[sizeof(__stdoutbuf)]; static mbstate_t mb_cout; _ALIGNAS_TYPE (wostream) _LIBCPP_FUNC_VIS char wcout[sizeof(wostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wcout@__1@std@@3V?$basic_ostream@_WU?$char_traits@_W@__1@std@@@12@A") +__asm__("?wcout@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __wcout[sizeof(__stdoutbuf)]; @@ -49,14 +53,14 @@ static mbstate_t mb_wcout; _ALIGNAS_TYPE (ostream) _LIBCPP_FUNC_VIS char cerr[sizeof(ostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?cerr@__1@std@@3V?$basic_ostream@DU?$char_traits@D@__1@std@@@12@A") +__asm__("?cerr@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __cerr[sizeof(__stdoutbuf)]; static mbstate_t mb_cerr; _ALIGNAS_TYPE (wostream) _LIBCPP_FUNC_VIS char wcerr[sizeof(wostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wcerr@__1@std@@3V?$basic_ostream@_WU?$char_traits@_W@__1@std@@@12@A") +__asm__("?wcerr@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (__stdoutbuf) static char __wcerr[sizeof(__stdoutbuf)]; @@ -64,12 +68,12 @@ static mbstate_t mb_wcerr; _ALIGNAS_TYPE (ostream) _LIBCPP_FUNC_VIS char clog[sizeof(ostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?clog@__1@std@@3V?$basic_ostream@DU?$char_traits@D@__1@std@@@12@A") +__asm__("?clog@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@DU?$char_traits@D@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; _ALIGNAS_TYPE (wostream) _LIBCPP_FUNC_VIS char wclog[sizeof(wostream)] #if defined(_LIBCPP_ABI_MICROSOFT) && defined(__clang__) -__asm__("?wclog@__1@std@@3V?$basic_ostream@_WU?$char_traits@_W@__1@std@@@12@A") +__asm__("?wclog@" _LIBCPP_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_traits@_W@" _LIBCPP_NAMESPACE_STR "@std@@@12@A") #endif ; diff --git a/contrib/libc++/src/locale.cpp b/contrib/libc++/src/locale.cpp index 3b4c83a09007..a6cb0e97d128 100644 --- a/contrib/libc++/src/locale.cpp +++ b/contrib/libc++/src/locale.cpp @@ -36,6 +36,7 @@ #endif #include #include +#include "include/atomic_support.h" #include "__undef_macros" // On Linux, wint_t and wchar_t have different signed-ness, and this causes @@ -667,7 +668,7 @@ locale::id::__get() void locale::id::__init() { - __id_ = __sync_add_and_fetch(&__next_id, 1); + __id_ = __libcpp_atomic_add(&__next_id, 1); } // template <> class collate_byname diff --git a/contrib/libc++/src/new.cpp b/contrib/libc++/src/new.cpp index 2b2682fa6617..e228a0d83d8e 100644 --- a/contrib/libc++/src/new.cpp +++ b/contrib/libc++/src/new.cpp @@ -12,9 +12,12 @@ #include #include "new" +#include "include/atomic_support.h" #if defined(_LIBCPP_ABI_MICROSOFT) -// nothing todo +#if defined(_LIBCPP_NO_VCRUNTIME) +#include "support/runtime/new_handler_fallback.ipp" +#endif #elif defined(LIBCXX_BUILDING_LIBCXXABI) #include #elif defined(LIBCXXRT) @@ -53,7 +56,8 @@ __throw_bad_alloc() } // std -#if !defined(__GLIBCXX__) && !defined(_LIBCPP_ABI_MICROSOFT) && \ +#if !defined(__GLIBCXX__) && \ + (!defined(_LIBCPP_ABI_MICROSOFT) || defined(_LIBCPP_NO_VCRUNTIME)) && \ !defined(_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS) // Implement all new and delete operators as weak definitions @@ -299,4 +303,4 @@ operator delete[] (void* ptr, size_t, std::align_val_t alignment) _NOEXCEPT } #endif // !_LIBCPP_HAS_NO_ALIGNED_ALLOCATION -#endif // !__GLIBCXX__ && !_LIBCPP_ABI_MICROSOFT && !_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS +#endif // !__GLIBCXX__ && (!_LIBCPP_ABI_MICROSOFT || _LIBCPP_NO_VCRUNTIME) && !_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS diff --git a/contrib/libc++/src/random.cpp b/contrib/libc++/src/random.cpp index eb2510a48c85..4a2468368d06 100644 --- a/contrib/libc++/src/random.cpp +++ b/contrib/libc++/src/random.cpp @@ -25,7 +25,9 @@ #include #include -#if defined(_LIBCPP_USING_DEV_RANDOM) +#if defined(_LIBCPP_USING_GETENTROPY) +#include +#elif defined(_LIBCPP_USING_DEV_RANDOM) #include #include #elif defined(_LIBCPP_USING_NACL_RANDOM) @@ -35,7 +37,30 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if defined(_LIBCPP_USING_ARC4_RANDOM) +#if defined(_LIBCPP_USING_GETENTROPY) + +random_device::random_device(const string& __token) +{ + if (__token != "/dev/urandom") + __throw_system_error(ENOENT, ("random device not supported " + __token).c_str()); +} + +random_device::~random_device() +{ +} + +unsigned +random_device::operator()() +{ + unsigned r; + size_t n = sizeof(r); + int err = getentropy(&r, n); + if (err) + __throw_system_error(errno, "random_device getentropy failed"); + return r; +} + +#elif defined(_LIBCPP_USING_ARC4_RANDOM) random_device::random_device(const string& __token) { diff --git a/contrib/libc++/src/strstream.cpp b/contrib/libc++/src/strstream.cpp index 01523cf45462..8b8521f76af1 100644 --- a/contrib/libc++/src/strstream.cpp +++ b/contrib/libc++/src/strstream.cpp @@ -186,7 +186,7 @@ strstreambuf::overflow(int_type __c) } setg(buf, buf + ninp, buf + einp); setp(buf + einp, buf + new_size); - pbump(static_cast(nout)); + __pbump(nout); __strmode_ |= __allocated; } *pptr() = static_cast(__c); @@ -282,7 +282,7 @@ strstreambuf::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmod // min(pbase, newpos), newpos, epptr() __off = epptr() - newpos; setp(min(pbase(), newpos), epptr()); - pbump(static_cast((epptr() - pbase()) - __off)); + __pbump((epptr() - pbase()) - __off); } __p = newoff; } @@ -312,7 +312,7 @@ strstreambuf::seekpos(pos_type __sp, ios_base::openmode __which) // min(pbase, newpos), newpos, epptr() off_type temp = epptr() - newpos; setp(min(pbase(), newpos), epptr()); - pbump(static_cast((epptr() - pbase()) - temp)); + __pbump((epptr() - pbase()) - temp); } __p = newoff; } diff --git a/contrib/libc++/src/support/runtime/exception_fallback.ipp b/contrib/libc++/src/support/runtime/exception_fallback.ipp index 69c06a9ce3a8..664e7f48c09a 100644 --- a/contrib/libc++/src/support/runtime/exception_fallback.ipp +++ b/contrib/libc++/src/support/runtime/exception_fallback.ipp @@ -20,13 +20,13 @@ _LIBCPP_SAFE_STATIC static std::unexpected_handler __unexpected_handler; unexpected_handler set_unexpected(unexpected_handler func) _NOEXCEPT { - return __sync_lock_test_and_set(&__unexpected_handler, func); + return __libcpp_atomic_exchange(&__unexpected_handler, func); } unexpected_handler get_unexpected() _NOEXCEPT { - return __sync_fetch_and_add(&__unexpected_handler, (unexpected_handler)0); + return __libcpp_atomic_load(&__unexpected_handler); } @@ -41,14 +41,13 @@ void unexpected() terminate_handler set_terminate(terminate_handler func) _NOEXCEPT { - return __sync_lock_test_and_set(&__terminate_handler, func); + return __libcpp_atomic_exchange(&__terminate_handler, func); } terminate_handler get_terminate() _NOEXCEPT { - return __sync_fetch_and_add(&__terminate_handler, (terminate_handler)0); - + return __libcpp_atomic_load(&__terminate_handler); } #ifndef __EMSCRIPTEN__ // We provide this in JS diff --git a/contrib/libc++/src/support/runtime/exception_msvc.ipp b/contrib/libc++/src/support/runtime/exception_msvc.ipp index 950ec0cebfe7..d5bf5b726ea5 100644 --- a/contrib/libc++/src/support/runtime/exception_msvc.ipp +++ b/contrib/libc++/src/support/runtime/exception_msvc.ipp @@ -14,12 +14,35 @@ #include #include -#include -#include + +#if !defined(_ACRTIMP) +#define _ACRTIMP __declspec(dllimport) +#endif + +#if !defined(_VCRTIMP) +#define _VCRTIMP __declspec(dllimport) +#endif + +#if !defined(__CRTDECL) +#define __CRTDECL __cdecl +#endif + +extern "C" { +typedef void (__CRTDECL* terminate_handler)(); +_ACRTIMP terminate_handler __cdecl set_terminate( + terminate_handler _NewTerminateHandler) throw(); +_ACRTIMP terminate_handler __cdecl _get_terminate(); + +typedef void (__CRTDECL* unexpected_handler)(); +_VCRTIMP unexpected_handler __cdecl set_unexpected( + unexpected_handler _NewUnexpectedHandler) throw(); +_VCRTIMP unexpected_handler __cdecl _get_unexpected(); + +_VCRTIMP int __cdecl __uncaught_exceptions(); +} namespace std { -// libcxxrt provides implementations of these functions itself. unexpected_handler set_unexpected(unexpected_handler func) _NOEXCEPT { return ::set_unexpected(func); @@ -86,4 +109,82 @@ bad_array_length::what() const _NOEXCEPT return "bad_array_length"; } +bad_cast::bad_cast() _NOEXCEPT +{ +} + +bad_cast::~bad_cast() _NOEXCEPT +{ +} + +const char * +bad_cast::what() const _NOEXCEPT +{ + return "std::bad_cast"; +} + +bad_typeid::bad_typeid() _NOEXCEPT +{ +} + +bad_typeid::~bad_typeid() _NOEXCEPT +{ +} + +const char * +bad_typeid::what() const _NOEXCEPT +{ + return "std::bad_typeid"; +} + +#if defined(_LIBCPP_NO_VCRUNTIME) +exception::~exception() _NOEXCEPT +{ +} + +const char* exception::what() const _NOEXCEPT +{ + return "std::exception"; +} + + +bad_exception::~bad_exception() _NOEXCEPT +{ +} + +const char* bad_exception::what() const _NOEXCEPT +{ + return "std::bad_exception"; +} + + +bad_alloc::bad_alloc() _NOEXCEPT +{ +} + +bad_alloc::~bad_alloc() _NOEXCEPT +{ +} + +const char* +bad_alloc::what() const _NOEXCEPT +{ + return "std::bad_alloc"; +} + +bad_array_new_length::bad_array_new_length() _NOEXCEPT +{ +} + +bad_array_new_length::~bad_array_new_length() _NOEXCEPT +{ +} + +const char* +bad_array_new_length::what() const _NOEXCEPT +{ + return "bad_array_new_length"; +} +#endif // _LIBCPP_NO_VCRUNTIME + } // namespace std diff --git a/contrib/libc++/src/support/runtime/exception_pointer_msvc.ipp b/contrib/libc++/src/support/runtime/exception_pointer_msvc.ipp index eab5d30a9487..5ca7519a6001 100644 --- a/contrib/libc++/src/support/runtime/exception_pointer_msvc.ipp +++ b/contrib/libc++/src/support/runtime/exception_pointer_msvc.ipp @@ -10,26 +10,32 @@ #include #include -#include // for _CRTIMP2_PURE -_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrCreate(_Out_ void*); -_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrDestroy(_Inout_ void*); -_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrCopy(_Out_ void*, - _In_ const void*); +#if !defined(_CRTIMP2_PURE) +#define _CRTIMP2_PURE __declspec(dllimport) +#endif + +#if !defined(__CLRCALL_PURE_OR_CDECL) +#define __CLRCALL_PURE_OR_CDECL __cdecl +#endif + +_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrCreate(void*); +_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrDestroy(void*); +_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrCopy(void*, + const void*); _CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL -__ExceptionPtrAssign(_Inout_ void*, _In_ const void*); +__ExceptionPtrAssign(void*, const void*); _CRTIMP2_PURE bool __CLRCALL_PURE_OR_CDECL -__ExceptionPtrCompare(_In_ const void*, _In_ const void*); +__ExceptionPtrCompare(const void*, const void*); _CRTIMP2_PURE bool __CLRCALL_PURE_OR_CDECL -__ExceptionPtrToBool(_In_ const void*); -_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrSwap(_Inout_ void*, - _Inout_ void*); +__ExceptionPtrToBool(const void*); +_CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL __ExceptionPtrSwap(void*, void*); _CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL -__ExceptionPtrCurrentException(_Out_ void*); +__ExceptionPtrCurrentException(void*); [[noreturn]] _CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL -__ExceptionPtrRethrow(_In_ const void*); +__ExceptionPtrRethrow(const void*); _CRTIMP2_PURE void __CLRCALL_PURE_OR_CDECL -__ExceptionPtrCopyException(_Inout_ void*, _In_ const void*, _In_ const void*); +__ExceptionPtrCopyException(void*, const void*, const void*); namespace std { diff --git a/contrib/libc++/src/support/runtime/new_handler_fallback.ipp b/contrib/libc++/src/support/runtime/new_handler_fallback.ipp index b7092d542d97..ec3f52354ca8 100644 --- a/contrib/libc++/src/support/runtime/new_handler_fallback.ipp +++ b/contrib/libc++/src/support/runtime/new_handler_fallback.ipp @@ -15,13 +15,13 @@ _LIBCPP_SAFE_STATIC static std::new_handler __new_handler; new_handler set_new_handler(new_handler handler) _NOEXCEPT { - return __sync_lock_test_and_set(&__new_handler, handler); + return __libcpp_atomic_exchange(&__new_handler, handler); } new_handler get_new_handler() _NOEXCEPT { - return __sync_fetch_and_add(&__new_handler, nullptr); + return __libcpp_atomic_load(&__new_handler); } } // namespace std diff --git a/contrib/libc++/src/system_error.cpp b/contrib/libc++/src/system_error.cpp index 17f2c9a5ba79..72623ea6bc81 100644 --- a/contrib/libc++/src/system_error.cpp +++ b/contrib/libc++/src/system_error.cpp @@ -73,39 +73,59 @@ string do_strerror_r(int ev) { std::snprintf(buffer, strerror_buff_size, "unknown error %d", ev); return string(buffer); } -#elif defined(__linux__) && !defined(_LIBCPP_HAS_MUSL_LIBC) && \ - (!defined(__ANDROID__) || __ANDROID_API__ >= 23) -// GNU Extended version -string do_strerror_r(int ev) { - char buffer[strerror_buff_size]; - char* ret = ::strerror_r(ev, buffer, strerror_buff_size); - return string(ret); -} #else -// POSIX version + +// Only one of the two following functions will be used, depending on +// the return type of strerror_r: + +// For the GNU variant, a char* return value: +__attribute__((unused)) const char * +handle_strerror_r_return(char *strerror_return, char *buffer) { + // GNU always returns a string pointer in its return value. The + // string might point to either the input buffer, or a static + // buffer, but we don't care which. + return strerror_return; +} + +// For the POSIX variant: an int return value. +__attribute__((unused)) const char * +handle_strerror_r_return(int strerror_return, char *buffer) { + // The POSIX variant either: + // - fills in the provided buffer and returns 0 + // - returns a positive error value, or + // - returns -1 and fills in errno with an error value. + if (strerror_return == 0) + return buffer; + + // Only handle EINVAL. Other errors abort. + int new_errno = strerror_return == -1 ? errno : strerror_return; + if (new_errno == EINVAL) + return ""; + + _LIBCPP_ASSERT(new_errno == ERANGE, "unexpected error from ::strerror_r"); + // FIXME maybe? 'strerror_buff_size' is likely to exceed the + // maximum error size so ERANGE shouldn't be returned. + std::abort(); +} + +// This function handles both GNU and POSIX variants, dispatching to +// one of the two above functions. string do_strerror_r(int ev) { char buffer[strerror_buff_size]; + // Preserve errno around the call. (The C++ standard requires that + // system_error functions not modify errno). const int old_errno = errno; - int ret; - if ((ret = ::strerror_r(ev, buffer, strerror_buff_size)) != 0) { - // If `ret == -1` then the error is specified using `errno`, otherwise - // `ret` represents the error. - const int new_errno = ret == -1 ? errno : ret; - errno = old_errno; - if (new_errno == EINVAL) { - std::snprintf(buffer, strerror_buff_size, "Unknown error %d", ev); - return string(buffer); - } else { - _LIBCPP_ASSERT(new_errno == ERANGE, "unexpected error from ::strerr_r"); - // FIXME maybe? 'strerror_buff_size' is likely to exceed the - // maximum error size so ERANGE shouldn't be returned. - std::abort(); - } + const char *error_message = handle_strerror_r_return( + ::strerror_r(ev, buffer, strerror_buff_size), buffer); + // If we didn't get any message, print one now. + if (!error_message[0]) { + std::snprintf(buffer, strerror_buff_size, "Unknown error %d", ev); + error_message = buffer; } - return string(buffer); + errno = old_errno; + return string(error_message); } #endif - } // end namespace #endif diff --git a/contrib/libc++/src/typeinfo.cpp b/contrib/libc++/src/typeinfo.cpp index 02778f368704..0cb193b77d9e 100644 --- a/contrib/libc++/src/typeinfo.cpp +++ b/contrib/libc++/src/typeinfo.cpp @@ -9,11 +9,48 @@ #include "typeinfo" +#if defined(_LIBCPP_ABI_MICROSOFT) +#include + +int std::type_info::__compare(const type_info &__rhs) const _NOEXCEPT { + if (&__data == &__rhs.__data) + return 0; + return strcmp(&__data.__decorated_name[1], &__rhs.__data.__decorated_name[1]); +} + +const char *std::type_info::name() const _NOEXCEPT { + // TODO(compnerd) cache demangled &__data.__decorated_name[1] + return &__data.__decorated_name[1]; +} + +size_t std::type_info::hash_code() const _NOEXCEPT { +#if defined(_WIN64) + constexpr size_t fnv_offset_basis = 14695981039346656037ull; + constexpr size_t fnv_prime = 10995116282110ull; +#else + constexpr size_t fnv_offset_basis = 2166136261ull; + constexpr size_t fnv_prime = 16777619ull; +#endif + + size_t value = fnv_offset_basis; + for (const char* c = &__data.__decorated_name[1]; *c; ++c) { + value ^= static_cast(static_cast(*c)); + value *= fnv_prime; + } + +#if defined(_WIN64) + value ^= value >> 32; +#endif + + return value; +} +#endif // _LIBCPP_ABI_MICROSOFT + // FIXME: Remove __APPLE__ default here once buildit is gone. -#if (!defined(_LIBCPP_ABI_MICROSOFT) && !defined(LIBCXX_BUILDING_LIBCXXABI) && \ - !defined(LIBCXXRT) && !defined(__GLIBCXX__) && \ - !defined(__APPLE__)) || \ - defined(_LIBCPP_BUILDING_HAS_NO_ABI_LIBRARY) // FIXME: remove this configuration. +// FIXME: Remove the _LIBCPP_BUILDING_HAS_NO_ABI_LIBRARY configuration. +#if (!defined(LIBCXX_BUILDING_LIBCXXABI) && !defined(LIBCXXRT) && \ + !defined(__GLIBCXX__) && !defined(__APPLE__)) || \ + defined(_LIBCPP_BUILDING_HAS_NO_ABI_LIBRARY) std::type_info::~type_info() { } diff --git a/contrib/llvm/include/llvm-c/Core.h b/contrib/llvm/include/llvm-c/Core.h index 22cef23007c3..8238c09f9dd0 100644 --- a/contrib/llvm/include/llvm-c/Core.h +++ b/contrib/llvm/include/llvm-c/Core.h @@ -1136,6 +1136,16 @@ LLVMTypeRef LLVMLabelTypeInContext(LLVMContextRef C); */ LLVMTypeRef LLVMX86MMXTypeInContext(LLVMContextRef C); +/** + * Create a token type in a context. + */ +LLVMTypeRef LLVMTokenTypeInContext(LLVMContextRef C); + +/** + * Create a metadata type in a context. + */ +LLVMTypeRef LLVMMetadataTypeInContext(LLVMContextRef C); + /** * These are similar to the above functions except they operate on the * global context. diff --git a/contrib/llvm/include/llvm-c/DebugInfo.h b/contrib/llvm/include/llvm-c/DebugInfo.h new file mode 100644 index 000000000000..d17c690be4da --- /dev/null +++ b/contrib/llvm/include/llvm-c/DebugInfo.h @@ -0,0 +1,232 @@ +//===------------ DebugInfo.h - LLVM C API Debug Info API -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// This file declares the C API endpoints for generating DWARF Debug Info +/// +/// Note: This interface is experimental. It is *NOT* stable, and may be +/// changed without warning. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_C_DEBUGINFO_H +#define LLVM_C_DEBUGINFO_H + +#include "llvm-c/Core.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Debug info flags. + */ +typedef enum { + LLVMDIFlagZero = 0, + LLVMDIFlagPrivate = 1, + LLVMDIFlagProtected = 2, + LLVMDIFlagPublic = 3, + LLVMDIFlagFwdDecl = 1 << 2, + LLVMDIFlagAppleBlock = 1 << 3, + LLVMDIFlagBlockByrefStruct = 1 << 4, + LLVMDIFlagVirtual = 1 << 5, + LLVMDIFlagArtificial = 1 << 6, + LLVMDIFlagExplicit = 1 << 7, + LLVMDIFlagPrototyped = 1 << 8, + LLVMDIFlagObjcClassComplete = 1 << 9, + LLVMDIFlagObjectPointer = 1 << 10, + LLVMDIFlagVector = 1 << 11, + LLVMDIFlagStaticMember = 1 << 12, + LLVMDIFlagLValueReference = 1 << 13, + LLVMDIFlagRValueReference = 1 << 14, + LLVMDIFlagReserved = 1 << 15, + LLVMDIFlagSingleInheritance = 1 << 16, + LLVMDIFlagMultipleInheritance = 2 << 16, + LLVMDIFlagVirtualInheritance = 3 << 16, + LLVMDIFlagIntroducedVirtual = 1 << 18, + LLVMDIFlagBitField = 1 << 19, + LLVMDIFlagNoReturn = 1 << 20, + LLVMDIFlagMainSubprogram = 1 << 21, + LLVMDIFlagIndirectVirtualBase = (1 << 2) | (1 << 5), + LLVMDIFlagAccessibility = LLVMDIFlagPrivate | LLVMDIFlagProtected | + LLVMDIFlagPublic, + LLVMDIFlagPtrToMemberRep = LLVMDIFlagSingleInheritance | + LLVMDIFlagMultipleInheritance | + LLVMDIFlagVirtualInheritance +} LLVMDIFlags; + +/** + * Source languages known by DWARF. + */ +typedef enum { + LLVMDWARFSourceLanguageC89, + LLVMDWARFSourceLanguageC, + LLVMDWARFSourceLanguageAda83, + LLVMDWARFSourceLanguageC_plus_plus, + LLVMDWARFSourceLanguageCobol74, + LLVMDWARFSourceLanguageCobol85, + LLVMDWARFSourceLanguageFortran77, + LLVMDWARFSourceLanguageFortran90, + LLVMDWARFSourceLanguagePascal83, + LLVMDWARFSourceLanguageModula2, + // New in DWARF v3: + LLVMDWARFSourceLanguageJava, + LLVMDWARFSourceLanguageC99, + LLVMDWARFSourceLanguageAda95, + LLVMDWARFSourceLanguageFortran95, + LLVMDWARFSourceLanguagePLI, + LLVMDWARFSourceLanguageObjC, + LLVMDWARFSourceLanguageObjC_plus_plus, + LLVMDWARFSourceLanguageUPC, + LLVMDWARFSourceLanguageD, + // New in DWARF v4: + LLVMDWARFSourceLanguagePython, + // New in DWARF v5: + LLVMDWARFSourceLanguageOpenCL, + LLVMDWARFSourceLanguageGo, + LLVMDWARFSourceLanguageModula3, + LLVMDWARFSourceLanguageHaskell, + LLVMDWARFSourceLanguageC_plus_plus_03, + LLVMDWARFSourceLanguageC_plus_plus_11, + LLVMDWARFSourceLanguageOCaml, + LLVMDWARFSourceLanguageRust, + LLVMDWARFSourceLanguageC11, + LLVMDWARFSourceLanguageSwift, + LLVMDWARFSourceLanguageJulia, + LLVMDWARFSourceLanguageDylan, + LLVMDWARFSourceLanguageC_plus_plus_14, + LLVMDWARFSourceLanguageFortran03, + LLVMDWARFSourceLanguageFortran08, + LLVMDWARFSourceLanguageRenderScript, + LLVMDWARFSourceLanguageBLISS, + // Vendor extensions: + LLVMDWARFSourceLanguageMips_Assembler, + LLVMDWARFSourceLanguageGOOGLE_RenderScript, + LLVMDWARFSourceLanguageBORLAND_Delphi +} LLVMDWARFSourceLanguage; + +/** + * The amount of debug information to emit. + */ +typedef enum { + LLVMDWARFEmissionNone = 0, + LLVMDWARFEmissionFull, + LLVMDWARFEmissionLineTablesOnly +} LLVMDWARFEmissionKind; + +/** + * The current debug metadata version number. + */ +unsigned LLVMDebugMetadataVersion(void); + +/** + * The version of debug metadata that's present in the provided \c Module. + */ +unsigned LLVMGetModuleDebugMetadataVersion(LLVMModuleRef Module); + +/** + * Strip debug info in the module if it exists. + * To do this, we remove all calls to the debugger intrinsics and any named + * metadata for debugging. We also remove debug locations for instructions. + * Return true if module is modified. + */ +LLVMBool LLVMStripModuleDebugInfo(LLVMModuleRef Module); + +/** + * Construct a builder for a module, and do not allow for unresolved nodes + * attached to the module. + */ +LLVMDIBuilderRef LLVMCreateDIBuilderDisallowUnresolved(LLVMModuleRef M); + +/** + * Construct a builder for a module and collect unresolved nodes attached + * to the module in order to resolve cycles during a call to + * \c LLVMDIBuilderFinalize. + */ +LLVMDIBuilderRef LLVMCreateDIBuilder(LLVMModuleRef M); + +/** + * Deallocates the \c DIBuilder and everything it owns. + * @note You must call \c LLVMDIBuilderFinalize before this + */ +void LLVMDisposeDIBuilder(LLVMDIBuilderRef Builder); + +/** + * Construct any deferred debug info descriptors. + */ +void LLVMDIBuilderFinalize(LLVMDIBuilderRef Builder); + +/** + * A CompileUnit provides an anchor for all debugging + * information generated during this instance of compilation. + * \param Lang Source programming language, eg. + * \c LLVMDWARFSourceLanguageC99 + * \param FileRef File info. + * \param Producer Identify the producer of debugging information + * and code. Usually this is a compiler + * version string. + * \param ProducerLen The length of the C string passed to \c Producer. + * \param isOptimized A boolean flag which indicates whether optimization + * is enabled or not. + * \param Flags This string lists command line options. This + * string is directly embedded in debug info + * output which may be used by a tool + * analyzing generated debugging information. + * \param FlagsLen The length of the C string passed to \c Flags. + * \param RuntimeVer This indicates runtime version for languages like + * Objective-C. + * \param SplitName The name of the file that we'll split debug info + * out into. + * \param SplitNameLen The length of the C string passed to \c SplitName. + * \param Kind The kind of debug information to generate. + * \param DWOId The DWOId if this is a split skeleton compile unit. + * \param SplitDebugInlining Whether to emit inline debug info. + * \param DebugInfoForProfiling Whether to emit extra debug info for + * profile collection. + */ +LLVMMetadataRef LLVMDIBuilderCreateCompileUnit( + LLVMDIBuilderRef Builder, LLVMDWARFSourceLanguage Lang, + LLVMMetadataRef FileRef, const char *Producer, size_t ProducerLen, + LLVMBool isOptimized, const char *Flags, size_t FlagsLen, + unsigned RuntimeVer, const char *SplitName, size_t SplitNameLen, + LLVMDWARFEmissionKind Kind, unsigned DWOId, LLVMBool SplitDebugInlining, + LLVMBool DebugInfoForProfiling); + +/** + * Create a file descriptor to hold debugging information for a file. + * \param Builder The \c DIBuilder. + * \param Filename File name. + * \param FilenameLen The length of the C string passed to \c Filename. + * \param Directory Directory. + * \param DirectoryLen The length of the C string passed to \c Directory. + */ +LLVMMetadataRef +LLVMDIBuilderCreateFile(LLVMDIBuilderRef Builder, const char *Filename, + size_t FilenameLen, const char *Directory, + size_t DirectoryLen); + +/** + * Creates a new DebugLocation that describes a source location. + * \param Line The line in the source file. + * \param Column The column in the source file. + * \param Scope The scope in which the location resides. + * \param InlinedAt The scope where this location was inlined, if at all. + * (optional). + * \note If the item to which this location is attached cannot be + * attributed to a source line, pass 0 for the line and column. + */ +LLVMMetadataRef +LLVMDIBuilderCreateDebugLocation(LLVMContextRef Ctx, unsigned Line, + unsigned Column, LLVMMetadataRef Scope, + LLVMMetadataRef InlinedAt); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif diff --git a/contrib/llvm/include/llvm-c/OrcBindings.h b/contrib/llvm/include/llvm-c/OrcBindings.h index 7ee395431358..abb3ac6a7f03 100644 --- a/contrib/llvm/include/llvm-c/OrcBindings.h +++ b/contrib/llvm/include/llvm-c/OrcBindings.h @@ -30,7 +30,6 @@ extern "C" { #endif typedef struct LLVMOpaqueSharedModule *LLVMSharedModuleRef; -typedef struct LLVMOpaqueSharedObjectBuffer *LLVMSharedObjectBufferRef; typedef struct LLVMOrcOpaqueJITStack *LLVMOrcJITStackRef; typedef uint32_t LLVMOrcModuleHandle; typedef uint64_t LLVMOrcTargetAddress; @@ -67,18 +66,6 @@ LLVMSharedModuleRef LLVMOrcMakeSharedModule(LLVMModuleRef Mod); void LLVMOrcDisposeSharedModuleRef(LLVMSharedModuleRef SharedMod); -/** - * Get an LLVMSharedObjectBufferRef from an LLVMMemoryBufferRef. - */ -LLVMSharedObjectBufferRef -LLVMOrcMakeSharedObjectBuffer(LLVMMemoryBufferRef ObjBuffer); - -/** - * Dispose of a shared object buffer. - */ -void -LLVMOrcDisposeSharedObjectBufferRef(LLVMSharedObjectBufferRef SharedObjBuffer); - /** * Create an ORC JIT stack. * @@ -155,10 +142,15 @@ LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, /** * Add an object file. + * + * This method takes ownership of the given memory buffer and attempts to add + * it to the JIT as an object file. + * Clients should *not* dispose of the 'Obj' argument: the JIT will manage it + * from this call onwards. */ LLVMOrcErrorCode LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, LLVMOrcModuleHandle *RetHandle, - LLVMSharedObjectBufferRef Obj, + LLVMMemoryBufferRef Obj, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx); diff --git a/contrib/llvm/include/llvm-c/Transforms/IPO.h b/contrib/llvm/include/llvm-c/Transforms/IPO.h index 3af7425dd268..7705b1864dc3 100644 --- a/contrib/llvm/include/llvm-c/Transforms/IPO.h +++ b/contrib/llvm/include/llvm-c/Transforms/IPO.h @@ -34,6 +34,9 @@ void LLVMAddArgumentPromotionPass(LLVMPassManagerRef PM); /** See llvm::createConstantMergePass function. */ void LLVMAddConstantMergePass(LLVMPassManagerRef PM); +/** See llvm::createCalledValuePropagationPass function. */ +void LLVMAddCalledValuePropagationPass(LLVMPassManagerRef PM); + /** See llvm::createDeadArgEliminationPass function. */ void LLVMAddDeadArgEliminationPass(LLVMPassManagerRef PM); diff --git a/contrib/llvm/include/llvm-c/Transforms/Scalar.h b/contrib/llvm/include/llvm-c/Transforms/Scalar.h index b9612b9cec04..8991e0904849 100644 --- a/contrib/llvm/include/llvm-c/Transforms/Scalar.h +++ b/contrib/llvm/include/llvm-c/Transforms/Scalar.h @@ -44,9 +44,6 @@ void LLVMAddAlignmentFromAssumptionsPass(LLVMPassManagerRef PM); /** See llvm::createCFGSimplificationPass function. */ void LLVMAddCFGSimplificationPass(LLVMPassManagerRef PM); -/** See llvm::createLateCFGSimplificationPass function. */ -void LLVMAddLateCFGSimplificationPass(LLVMPassManagerRef PM); - /** See llvm::createDeadStoreEliminationPass function. */ void LLVMAddDeadStoreEliminationPass(LLVMPassManagerRef PM); diff --git a/contrib/llvm/include/llvm-c/lto.h b/contrib/llvm/include/llvm-c/lto.h index 8d45b7832041..55f3e46c45ed 100644 --- a/contrib/llvm/include/llvm-c/lto.h +++ b/contrib/llvm/include/llvm-c/lto.h @@ -757,17 +757,17 @@ extern void thinlto_codegen_add_cross_referenced_symbol(thinlto_code_gen_t cg, * @ingroup LLVMCTLTO * * These entry points control the ThinLTO cache. The cache is intended to - * support incremental build, and thus needs to be persistent accross build. - * The client enabled the cache by supplying a path to an existing directory. + * support incremental builds, and thus needs to be persistent across builds. + * The client enables the cache by supplying a path to an existing directory. * The code generator will use this to store objects files that may be reused * during a subsequent build. * To avoid filling the disk space, a few knobs are provided: - * - The pruning interval limit the frequency at which the garbage collector - * will try to scan the cache directory to prune it from expired entries. - * Setting to -1 disable the pruning (default). + * - The pruning interval limits the frequency at which the garbage collector + * will try to scan the cache directory to prune expired entries. + * Setting to a negative number disables the pruning. * - The pruning expiration time indicates to the garbage collector how old an * entry needs to be to be removed. - * - Finally, the garbage collector can be instructed to prune the cache till + * - Finally, the garbage collector can be instructed to prune the cache until * the occupied space goes below a threshold. * @{ */ @@ -782,7 +782,7 @@ extern void thinlto_codegen_set_cache_dir(thinlto_code_gen_t cg, const char *cache_dir); /** - * Sets the cache pruning interval (in seconds). A negative value disable the + * Sets the cache pruning interval (in seconds). A negative value disables the * pruning. An unspecified default value will be applied, and a value of 0 will * be ignored. * diff --git a/contrib/llvm/include/llvm/ADT/APFloat.h b/contrib/llvm/include/llvm/ADT/APFloat.h index 9c5e392c4808..6c0b6ae78ae3 100644 --- a/contrib/llvm/include/llvm/ADT/APFloat.h +++ b/contrib/llvm/include/llvm/ADT/APFloat.h @@ -1119,6 +1119,21 @@ class APFloat : public APFloatBase { llvm_unreachable("Unexpected semantics"); } + /// We don't rely on operator== working on double values, as + /// it returns true for things that are clearly not equal, like -0.0 and 0.0. + /// As such, this method can be used to do an exact bit-for-bit comparison of + /// two floating point values. + /// + /// We leave the version with the double argument here because it's just so + /// convenient to write "2.0" and the like. Without this function we'd + /// have to duplicate its logic everywhere it's called. + bool isExactlyValue(double V) const { + bool ignored; + APFloat Tmp(V); + Tmp.convert(getSemantics(), APFloat::rmNearestTiesToEven, &ignored); + return bitwiseIsEqual(Tmp); + } + unsigned int convertToHexString(char *DST, unsigned int HexDigits, bool UpperCase, roundingMode RM) const { APFLOAT_DISPATCH_ON_SEMANTICS( diff --git a/contrib/llvm/include/llvm/ADT/APInt.h b/contrib/llvm/include/llvm/ADT/APInt.h index a1cce6e5fe17..c81363cc16b7 100644 --- a/contrib/llvm/include/llvm/ADT/APInt.h +++ b/contrib/llvm/include/llvm/ADT/APInt.h @@ -1724,13 +1724,13 @@ class LLVM_NODISCARD APInt { /// @{ /// \returns the floor log base 2 of this APInt. - unsigned logBase2() const { return BitWidth - 1 - countLeadingZeros(); } + unsigned logBase2() const { return getActiveBits() - 1; } /// \returns the ceil log base 2 of this APInt. unsigned ceilLogBase2() const { APInt temp(*this); --temp; - return BitWidth - temp.countLeadingZeros(); + return temp.getActiveBits(); } /// \returns the nearest log base 2 of this APInt. Ties round up. diff --git a/contrib/llvm/include/llvm/ADT/ArrayRef.h b/contrib/llvm/include/llvm/ADT/ArrayRef.h index 925ebafc3fed..5f7a769ddac4 100644 --- a/contrib/llvm/include/llvm/ADT/ArrayRef.h +++ b/contrib/llvm/include/llvm/ADT/ArrayRef.h @@ -294,7 +294,7 @@ namespace llvm { using reverse_iterator = std::reverse_iterator; /// Construct an empty MutableArrayRef. - /*implicit*/ MutableArrayRef() : ArrayRef() {} + /*implicit*/ MutableArrayRef() = default; /// Construct an empty MutableArrayRef from None. /*implicit*/ MutableArrayRef(NoneType) : ArrayRef() {} diff --git a/contrib/llvm/include/llvm/ADT/BitVector.h b/contrib/llvm/include/llvm/ADT/BitVector.h index e68ef5f53d10..99147fec4d4c 100644 --- a/contrib/llvm/include/llvm/ADT/BitVector.h +++ b/contrib/llvm/include/llvm/ADT/BitVector.h @@ -911,7 +911,7 @@ class BitVector { size_t getBitCapacity() const { return Bits.size() * BITWORD_SIZE; } }; -static inline size_t capacity_in_bytes(const BitVector &X) { +inline size_t capacity_in_bytes(const BitVector &X) { return X.getMemorySize(); } diff --git a/contrib/llvm/include/llvm/ADT/DenseMap.h b/contrib/llvm/include/llvm/ADT/DenseMap.h index b311e69ec9d3..ba60b7972a8f 100644 --- a/contrib/llvm/include/llvm/ADT/DenseMap.h +++ b/contrib/llvm/include/llvm/ADT/DenseMap.h @@ -19,6 +19,7 @@ #include "llvm/Support/AlignOf.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/ReverseIteration.h" #include "llvm/Support/type_traits.h" #include #include @@ -67,18 +68,26 @@ class DenseMapBase : public DebugEpochBase { DenseMapIterator; inline iterator begin() { - // When the map is empty, avoid the overhead of AdvancePastEmptyBuckets(). - return empty() ? end() : iterator(getBuckets(), getBucketsEnd(), *this); + // When the map is empty, avoid the overhead of advancing/retreating past + // empty buckets. + if (empty()) + return end(); + if (shouldReverseIterate()) + return makeIterator(getBucketsEnd() - 1, getBuckets(), *this); + return makeIterator(getBuckets(), getBucketsEnd(), *this); } inline iterator end() { - return iterator(getBucketsEnd(), getBucketsEnd(), *this, true); + return makeIterator(getBucketsEnd(), getBucketsEnd(), *this, true); } inline const_iterator begin() const { - return empty() ? end() - : const_iterator(getBuckets(), getBucketsEnd(), *this); + if (empty()) + return end(); + if (shouldReverseIterate()) + return makeConstIterator(getBucketsEnd() - 1, getBuckets(), *this); + return makeConstIterator(getBuckets(), getBucketsEnd(), *this); } inline const_iterator end() const { - return const_iterator(getBucketsEnd(), getBucketsEnd(), *this, true); + return makeConstIterator(getBucketsEnd(), getBucketsEnd(), *this, true); } LLVM_NODISCARD bool empty() const { @@ -107,17 +116,23 @@ class DenseMapBase : public DebugEpochBase { } const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); - unsigned NumEntries = getNumEntries(); - for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { - if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { - if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { - P->getSecond().~ValueT(); - --NumEntries; - } + if (isPodLike::value && isPodLike::value) { + // Use a simpler loop when these are trivial types. + for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) P->getFirst() = EmptyKey; + } else { + unsigned NumEntries = getNumEntries(); + for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { + if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { + if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { + P->getSecond().~ValueT(); + --NumEntries; + } + P->getFirst() = EmptyKey; + } } + assert(NumEntries == 0 && "Node count imbalance!"); } - assert(NumEntries == 0 && "Node count imbalance!"); setNumEntries(0); setNumTombstones(0); } @@ -131,13 +146,13 @@ class DenseMapBase : public DebugEpochBase { iterator find(const_arg_type_t Val) { BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return iterator(TheBucket, getBucketsEnd(), *this, true); + return makeIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } const_iterator find(const_arg_type_t Val) const { const BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return const_iterator(TheBucket, getBucketsEnd(), *this, true); + return makeConstIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } @@ -150,14 +165,14 @@ class DenseMapBase : public DebugEpochBase { iterator find_as(const LookupKeyT &Val) { BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return iterator(TheBucket, getBucketsEnd(), *this, true); + return makeIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } template const_iterator find_as(const LookupKeyT &Val) const { const BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return const_iterator(TheBucket, getBucketsEnd(), *this, true); + return makeConstIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } @@ -191,14 +206,16 @@ class DenseMapBase : public DebugEpochBase { std::pair try_emplace(KeyT &&Key, Ts &&... Args) { BucketT *TheBucket; if (LookupBucketFor(Key, TheBucket)) - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - false); // Already in map. + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + false); // Already in map. // Otherwise, insert the new element. TheBucket = InsertIntoBucket(TheBucket, std::move(Key), std::forward(Args)...); - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - true); + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + true); } // Inserts key,value pair into the map if the key isn't already in the map. @@ -208,13 +225,15 @@ class DenseMapBase : public DebugEpochBase { std::pair try_emplace(const KeyT &Key, Ts &&... Args) { BucketT *TheBucket; if (LookupBucketFor(Key, TheBucket)) - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - false); // Already in map. + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + false); // Already in map. // Otherwise, insert the new element. TheBucket = InsertIntoBucket(TheBucket, Key, std::forward(Args)...); - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - true); + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + true); } /// Alternate version of insert() which allows a different, and possibly @@ -227,14 +246,16 @@ class DenseMapBase : public DebugEpochBase { const LookupKeyT &Val) { BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - false); // Already in map. + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + false); // Already in map. // Otherwise, insert the new element. TheBucket = InsertIntoBucketWithLookup(TheBucket, std::move(KV.first), std::move(KV.second), Val); - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - true); + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + true); } /// insert - Range insertion of pairs. @@ -405,6 +426,26 @@ class DenseMapBase : public DebugEpochBase { } private: + iterator makeIterator(BucketT *P, BucketT *E, + DebugEpochBase &Epoch, + bool NoAdvance=false) { + if (shouldReverseIterate()) { + BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; + return iterator(B, E, Epoch, NoAdvance); + } + return iterator(P, E, Epoch, NoAdvance); + } + + const_iterator makeConstIterator(const BucketT *P, const BucketT *E, + const DebugEpochBase &Epoch, + const bool NoAdvance=false) const { + if (shouldReverseIterate()) { + const BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; + return const_iterator(B, E, Epoch, NoAdvance); + } + return const_iterator(P, E, Epoch, NoAdvance); + } + unsigned getNumEntries() const { return static_cast(this)->getNumEntries(); } @@ -1089,7 +1130,13 @@ class DenseMapIterator : DebugEpochBase::HandleBase { bool NoAdvance = false) : DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E) { assert(isHandleInSync() && "invalid construction!"); - if (!NoAdvance) AdvancePastEmptyBuckets(); + + if (NoAdvance) return; + if (shouldReverseIterate()) { + RetreatPastEmptyBuckets(); + return; + } + AdvancePastEmptyBuckets(); } // Converting ctor from non-const iterators to const iterators. SFINAE'd out @@ -1103,10 +1150,14 @@ class DenseMapIterator : DebugEpochBase::HandleBase { reference operator*() const { assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate()) + return Ptr[-1]; return *Ptr; } pointer operator->() const { assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate()) + return &(Ptr[-1]); return Ptr; } @@ -1127,6 +1178,11 @@ class DenseMapIterator : DebugEpochBase::HandleBase { inline DenseMapIterator& operator++() { // Preincrement assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate()) { + --Ptr; + RetreatPastEmptyBuckets(); + return *this; + } ++Ptr; AdvancePastEmptyBuckets(); return *this; @@ -1138,6 +1194,7 @@ class DenseMapIterator : DebugEpochBase::HandleBase { private: void AdvancePastEmptyBuckets() { + assert(Ptr <= End); const KeyT Empty = KeyInfoT::getEmptyKey(); const KeyT Tombstone = KeyInfoT::getTombstoneKey(); @@ -1145,11 +1202,20 @@ class DenseMapIterator : DebugEpochBase::HandleBase { KeyInfoT::isEqual(Ptr->getFirst(), Tombstone))) ++Ptr; } + + void RetreatPastEmptyBuckets() { + assert(Ptr >= End); + const KeyT Empty = KeyInfoT::getEmptyKey(); + const KeyT Tombstone = KeyInfoT::getTombstoneKey(); + + while (Ptr != End && (KeyInfoT::isEqual(Ptr[-1].getFirst(), Empty) || + KeyInfoT::isEqual(Ptr[-1].getFirst(), Tombstone))) + --Ptr; + } }; -template -static inline size_t -capacity_in_bytes(const DenseMap &X) { +template +inline size_t capacity_in_bytes(const DenseMap &X) { return X.getMemorySize(); } diff --git a/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h b/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h index af293d4c1422..e3f48433c69f 100644 --- a/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h +++ b/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h @@ -239,6 +239,16 @@ class EquivalenceClasses { return L1; } + // isEquivalent - Return true if V1 is equivalent to V2. This can happen if + // V1 is equal to V2 or if they belong to one equivalence class. + bool isEquivalent(const ElemTy &V1, const ElemTy &V2) const { + // Fast path: any element is equivalent to itself. + if (V1 == V2) + return true; + auto It = findLeader(V1); + return It != member_end() && It == findLeader(V2); + } + class member_iterator : public std::iterator { friend class EquivalenceClasses; diff --git a/contrib/llvm/include/llvm/ADT/FoldingSet.h b/contrib/llvm/include/llvm/ADT/FoldingSet.h index c5987a947e18..e363e69d032a 100644 --- a/contrib/llvm/include/llvm/ADT/FoldingSet.h +++ b/contrib/llvm/include/llvm/ADT/FoldingSet.h @@ -1,4 +1,4 @@ -//===-- llvm/ADT/FoldingSet.h - Uniquing Hash Set ---------------*- C++ -*-===// +//===- llvm/ADT/FoldingSet.h - Uniquing Hash Set ----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -115,11 +115,9 @@ class FoldingSetBase { protected: /// Buckets - Array of bucket chains. - /// void **Buckets; /// NumBuckets - Length of the Buckets array. Always a power of 2. - /// unsigned NumBuckets; /// NumNodes - Number of nodes in the folding set. Growth occurs when NumNodes @@ -135,14 +133,13 @@ class FoldingSetBase { //===--------------------------------------------------------------------===// /// Node - This class is used to maintain the singly linked bucket list in /// a folding set. - /// class Node { private: // NextInFoldingSetBucket - next link in the bucket list. - void *NextInFoldingSetBucket; + void *NextInFoldingSetBucket = nullptr; public: - Node() : NextInFoldingSetBucket(nullptr) {} + Node() = default; // Accessors void *getNextInBucket() const { return NextInFoldingSetBucket; } @@ -221,7 +218,6 @@ class FoldingSetBase { /// DefaultFoldingSetTrait - This class provides default implementations /// for FoldingSetTrait implementations. -/// template struct DefaultFoldingSetTrait { static void Profile(const T &X, FoldingSetNodeID &ID) { X.Profile(ID); @@ -307,7 +303,6 @@ class FoldingSetNodeIDRef { /// FoldingSetNodeID - This class is used to gather all the unique data bits of /// a node. When all the bits are gathered this class is used to produce a /// hash value for the node. -/// class FoldingSetNodeID { /// Bits - Vector of all the data bits that make the node unique. /// Use a SmallVector to avoid a heap allocation in the common case. @@ -320,7 +315,6 @@ class FoldingSetNodeID { : Bits(Ref.getData(), Ref.getData() + Ref.getSize()) {} /// Add* - Add various data types to Bit data. - /// void AddPointer(const void *Ptr); void AddInteger(signed I); void AddInteger(unsigned I); @@ -344,7 +338,6 @@ class FoldingSetNodeID { unsigned ComputeHash() const; /// operator== - Used to compare two nodes to each other. - /// bool operator==(const FoldingSetNodeID &RHS) const; bool operator==(const FoldingSetNodeIDRef RHS) const; @@ -363,7 +356,7 @@ class FoldingSetNodeID { }; // Convenience type to hide the implementation of the folding set. -typedef FoldingSetBase::Node FoldingSetNode; +using FoldingSetNode = FoldingSetBase::Node; template class FoldingSetIterator; template class FoldingSetBucketIterator; @@ -415,15 +408,17 @@ template class FoldingSetImpl : public FoldingSetBase { ~FoldingSetImpl() = default; public: - typedef FoldingSetIterator iterator; + using iterator = FoldingSetIterator; + iterator begin() { return iterator(Buckets); } iterator end() { return iterator(Buckets+NumBuckets); } - typedef FoldingSetIterator const_iterator; + using const_iterator = FoldingSetIterator; + const_iterator begin() const { return const_iterator(Buckets); } const_iterator end() const { return const_iterator(Buckets+NumBuckets); } - typedef FoldingSetBucketIterator bucket_iterator; + using bucket_iterator = FoldingSetBucketIterator; bucket_iterator bucket_begin(unsigned hash) { return bucket_iterator(Buckets + (hash & (NumBuckets-1))); @@ -503,9 +498,7 @@ template class FoldingSet final : public FoldingSetImpl { } public: - explicit FoldingSet(unsigned Log2InitSize = 6) - : Super(Log2InitSize) {} - + explicit FoldingSet(unsigned Log2InitSize = 6) : Super(Log2InitSize) {} FoldingSet(FoldingSet &&Arg) = default; FoldingSet &operator=(FoldingSet &&RHS) = default; }; @@ -552,8 +545,7 @@ class ContextualFoldingSet final : public FoldingSetImpl { public: explicit ContextualFoldingSet(Ctx Context, unsigned Log2InitSize = 6) - : Super(Log2InitSize), Context(Context) - {} + : Super(Log2InitSize), Context(Context) {} Ctx getContext() const { return Context; } }; @@ -569,15 +561,15 @@ class FoldingSetVector { VectorT Vector; public: - explicit FoldingSetVector(unsigned Log2InitSize = 6) - : Set(Log2InitSize) { - } + explicit FoldingSetVector(unsigned Log2InitSize = 6) : Set(Log2InitSize) {} + + using iterator = pointee_iterator; - typedef pointee_iterator iterator; iterator begin() { return Vector.begin(); } iterator end() { return Vector.end(); } - typedef pointee_iterator const_iterator; + using const_iterator = pointee_iterator; + const_iterator begin() const { return Vector.begin(); } const_iterator end() const { return Vector.end(); } @@ -667,15 +659,13 @@ template class FoldingSetIterator : public FoldingSetIteratorImpl { /// FoldingSetBucketIteratorImpl - This is the common bucket iterator support /// shared by all folding sets, which knows how to walk a particular bucket /// of a folding set hash table. - class FoldingSetBucketIteratorImpl { protected: void *Ptr; explicit FoldingSetBucketIteratorImpl(void **Bucket); - FoldingSetBucketIteratorImpl(void **Bucket, bool) - : Ptr(Bucket) {} + FoldingSetBucketIteratorImpl(void **Bucket, bool) : Ptr(Bucket) {} void advance() { void *Probe = static_cast(Ptr)->getNextInBucket(); diff --git a/contrib/llvm/include/llvm/ADT/MapVector.h b/contrib/llvm/include/llvm/ADT/MapVector.h index 26a555ee1d3b..3d78f4b203c8 100644 --- a/contrib/llvm/include/llvm/ADT/MapVector.h +++ b/contrib/llvm/include/llvm/ADT/MapVector.h @@ -56,6 +56,13 @@ class MapVector { size_type size() const { return Vector.size(); } + /// Grow the MapVector so that it can contain at least \p NumEntries items + /// before resizing again. + void reserve(size_type NumEntries) { + Map.reserve(NumEntries); + Vector.reserve(NumEntries); + } + iterator begin() { return Vector.begin(); } const_iterator begin() const { return Vector.begin(); } iterator end() { return Vector.end(); } diff --git a/contrib/llvm/include/llvm/ADT/Optional.h b/contrib/llvm/include/llvm/ADT/Optional.h index b782d9da17ac..2811d5c1e21b 100644 --- a/contrib/llvm/include/llvm/ADT/Optional.h +++ b/contrib/llvm/include/llvm/ADT/Optional.h @@ -27,8 +27,7 @@ namespace llvm { -template -class Optional { +template class Optional { AlignedCharArrayUnion storage; bool hasVal = false; @@ -38,18 +37,14 @@ class Optional { Optional(NoneType) {} explicit Optional() {} - Optional(const T &y) : hasVal(true) { - new (storage.buffer) T(y); - } + Optional(const T &y) : hasVal(true) { new (storage.buffer) T(y); } Optional(const Optional &O) : hasVal(O.hasVal) { if (hasVal) new (storage.buffer) T(*O); } - Optional(T &&y) : hasVal(true) { - new (storage.buffer) T(std::forward(y)); - } + Optional(T &&y) : hasVal(true) { new (storage.buffer) T(std::forward(y)); } Optional(Optional &&O) : hasVal(O) { if (O) { @@ -58,9 +53,7 @@ class Optional { } } - ~Optional() { - reset(); - } + ~Optional() { reset(); } Optional &operator=(T &&y) { if (hasVal) @@ -83,14 +76,13 @@ class Optional { } /// Create a new object by constructing it in place with the given arguments. - template - void emplace(ArgTypes &&...Args) { + template void emplace(ArgTypes &&... Args) { reset(); hasVal = true; new (storage.buffer) T(std::forward(Args)...); } - static inline Optional create(const T* y) { + static inline Optional create(const T *y) { return y ? Optional(*y) : Optional(); } @@ -124,17 +116,35 @@ class Optional { } } - const T* getPointer() const { assert(hasVal); return reinterpret_cast(storage.buffer); } - T* getPointer() { assert(hasVal); return reinterpret_cast(storage.buffer); } - const T& getValue() const LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } - T& getValue() LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } + const T *getPointer() const { + assert(hasVal); + return reinterpret_cast(storage.buffer); + } + T *getPointer() { + assert(hasVal); + return reinterpret_cast(storage.buffer); + } + const T &getValue() const LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } + T &getValue() LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } explicit operator bool() const { return hasVal; } bool hasValue() const { return hasVal; } - const T* operator->() const { return getPointer(); } - T* operator->() { return getPointer(); } - const T& operator*() const LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } - T& operator*() LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } + const T *operator->() const { return getPointer(); } + T *operator->() { return getPointer(); } + const T &operator*() const LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } + T &operator*() LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } template constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION { @@ -142,8 +152,14 @@ class Optional { } #if LLVM_HAS_RVALUE_REFERENCE_THIS - T&& getValue() && { assert(hasVal); return std::move(*getPointer()); } - T&& operator*() && { assert(hasVal); return std::move(*getPointer()); } + T &&getValue() && { + assert(hasVal); + return std::move(*getPointer()); + } + T &&operator*() && { + assert(hasVal); + return std::move(*getPointer()); + } template T getValueOr(U &&value) && { diff --git a/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h b/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h index 34323b5b8af4..ab4e1048a5bc 100644 --- a/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h +++ b/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h @@ -52,7 +52,7 @@ class PointerEmbeddedInt { explicit RawValueTag() = default; }; - friend class PointerLikeTypeTraits; + friend struct PointerLikeTypeTraits; explicit PointerEmbeddedInt(uintptr_t Value, RawValueTag) : Value(Value) {} @@ -80,10 +80,9 @@ class PointerEmbeddedInt { // Provide pointer like traits to support use with pointer unions and sum // types. template -class PointerLikeTypeTraits> { +struct PointerLikeTypeTraits> { using T = PointerEmbeddedInt; -public: static inline void *getAsVoidPointer(const T &P) { return reinterpret_cast(P.Value); } diff --git a/contrib/llvm/include/llvm/ADT/PointerIntPair.h b/contrib/llvm/include/llvm/ADT/PointerIntPair.h index 83fbf127e6da..884d05155bff 100644 --- a/contrib/llvm/include/llvm/ADT/PointerIntPair.h +++ b/contrib/llvm/include/llvm/ADT/PointerIntPair.h @@ -14,15 +14,14 @@ #ifndef LLVM_ADT_POINTERINTPAIR_H #define LLVM_ADT_POINTERINTPAIR_H -#include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include +#include #include namespace llvm { template struct DenseMapInfo; - template struct PointerIntPairInfo; @@ -39,25 +38,24 @@ struct PointerIntPairInfo; /// for something else. For example, this allows: /// PointerIntPair, 1, bool> /// ... and the two bools will land in different bits. -/// template , typename Info = PointerIntPairInfo> class PointerIntPair { - intptr_t Value; + intptr_t Value = 0; public: - PointerIntPair() : Value(0) {} + constexpr PointerIntPair() = default; + PointerIntPair(PointerTy PtrVal, IntType IntVal) { setPointerAndInt(PtrVal, IntVal); } + explicit PointerIntPair(PointerTy PtrVal) { initWithPointer(PtrVal); } PointerTy getPointer() const { return Info::getPointer(Value); } - IntType getInt() const { - return (IntType)Info::getInt(Value); - } + IntType getInt() const { return (IntType)Info::getInt(Value); } void setPointer(PointerTy PtrVal) { Value = Info::updatePointer(Value, PtrVal); @@ -88,6 +86,7 @@ class PointerIntPair { } void *getOpaqueValue() const { return reinterpret_cast(Value); } + void setFromOpaqueValue(void *Val) { Value = reinterpret_cast(Val); } @@ -108,14 +107,18 @@ class PointerIntPair { bool operator==(const PointerIntPair &RHS) const { return Value == RHS.Value; } + bool operator!=(const PointerIntPair &RHS) const { return Value != RHS.Value; } + bool operator<(const PointerIntPair &RHS) const { return Value < RHS.Value; } bool operator>(const PointerIntPair &RHS) const { return Value > RHS.Value; } + bool operator<=(const PointerIntPair &RHS) const { return Value <= RHS.Value; } + bool operator>=(const PointerIntPair &RHS) const { return Value >= RHS.Value; } @@ -180,44 +183,51 @@ struct isPodLike> { // Provide specialization of DenseMapInfo for PointerIntPair. template struct DenseMapInfo> { - typedef PointerIntPair Ty; + using Ty = PointerIntPair; + static Ty getEmptyKey() { uintptr_t Val = static_cast(-1); Val <<= PointerLikeTypeTraits::NumLowBitsAvailable; return Ty::getFromOpaqueValue(reinterpret_cast(Val)); } + static Ty getTombstoneKey() { uintptr_t Val = static_cast(-2); Val <<= PointerLikeTypeTraits::NumLowBitsAvailable; return Ty::getFromOpaqueValue(reinterpret_cast(Val)); } + static unsigned getHashValue(Ty V) { uintptr_t IV = reinterpret_cast(V.getOpaqueValue()); return unsigned(IV) ^ unsigned(IV >> 9); } + static bool isEqual(const Ty &LHS, const Ty &RHS) { return LHS == RHS; } }; // Teach SmallPtrSet that PointerIntPair is "basically a pointer". template -class PointerLikeTypeTraits< +struct PointerLikeTypeTraits< PointerIntPair> { -public: static inline void * getAsVoidPointer(const PointerIntPair &P) { return P.getOpaqueValue(); } + static inline PointerIntPair getFromVoidPointer(void *P) { return PointerIntPair::getFromOpaqueValue(P); } + static inline PointerIntPair getFromVoidPointer(const void *P) { return PointerIntPair::getFromOpaqueValue(P); } + enum { NumLowBitsAvailable = PtrTraits::NumLowBitsAvailable - IntBits }; }; } // end namespace llvm -#endif + +#endif // LLVM_ADT_POINTERINTPAIR_H diff --git a/contrib/llvm/include/llvm/ADT/PointerSumType.h b/contrib/llvm/include/llvm/ADT/PointerSumType.h index 062544eedf84..e37957160d98 100644 --- a/contrib/llvm/include/llvm/ADT/PointerSumType.h +++ b/contrib/llvm/include/llvm/ADT/PointerSumType.h @@ -11,8 +11,10 @@ #define LLVM_ADT_POINTERSUMTYPE_H #include "llvm/ADT/DenseMapInfo.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include +#include +#include namespace llvm { @@ -24,16 +26,15 @@ template > struct PointerSumTypeMember { enum { Tag = N }; - typedef PointerArgT PointerT; - typedef TraitsArgT TraitsT; + using PointerT = PointerArgT; + using TraitsT = TraitsArgT; }; namespace detail { -template -struct PointerSumTypeHelper; +template struct PointerSumTypeHelper; -} +} // end namespace detail /// A sum type over pointer-like types. /// @@ -60,12 +61,12 @@ struct PointerSumTypeHelper; /// There is no support for constructing or accessing with a dynamic tag as /// that would fundamentally violate the type safety provided by the sum type. template class PointerSumType { - uintptr_t Value; + uintptr_t Value = 0; - typedef detail::PointerSumTypeHelper HelperT; + using HelperT = detail::PointerSumTypeHelper; public: - PointerSumType() : Value(0) {} + constexpr PointerSumType() = default; /// A typed constructor for a specific tagged member of the sum type. template @@ -128,14 +129,14 @@ struct PointerSumTypeHelper : MemberTs... { template static void LookupOverload(...); template struct Lookup { // Compute a particular member type by resolving the lookup helper ovorload. - typedef decltype(LookupOverload( - static_cast(nullptr))) MemberT; + using MemberT = decltype( + LookupOverload(static_cast(nullptr))); /// The Nth member's pointer type. - typedef typename MemberT::PointerT PointerT; + using PointerT = typename MemberT::PointerT; /// The Nth member's traits type. - typedef typename MemberT::TraitsT TraitsT; + using TraitsT = typename MemberT::TraitsT; }; // Next we need to compute the number of bits available for the discriminant @@ -171,35 +172,36 @@ struct PointerSumTypeHelper : MemberTs... { "Each member must pass the checker."); }; -} +} // end namespace detail // Teach DenseMap how to use PointerSumTypes as keys. template struct DenseMapInfo> { - typedef PointerSumType SumType; - - typedef detail::PointerSumTypeHelper HelperT; + using SumType = PointerSumType; + using HelperT = detail::PointerSumTypeHelper; enum { SomeTag = HelperT::MinTag }; - typedef typename HelperT::template Lookup::PointerT - SomePointerT; - typedef DenseMapInfo SomePointerInfo; + using SomePointerT = + typename HelperT::template Lookup::PointerT; + using SomePointerInfo = DenseMapInfo; static inline SumType getEmptyKey() { return SumType::create(SomePointerInfo::getEmptyKey()); } + static inline SumType getTombstoneKey() { - return SumType::create( - SomePointerInfo::getTombstoneKey()); + return SumType::create(SomePointerInfo::getTombstoneKey()); } + static unsigned getHashValue(const SumType &Arg) { uintptr_t OpaqueValue = Arg.getOpaqueValue(); return DenseMapInfo::getHashValue(OpaqueValue); } + static bool isEqual(const SumType &LHS, const SumType &RHS) { return LHS == RHS; } }; -} +} // end namespace llvm -#endif +#endif // LLVM_ADT_POINTERSUMTYPE_H diff --git a/contrib/llvm/include/llvm/ADT/PointerUnion.h b/contrib/llvm/include/llvm/ADT/PointerUnion.h index aeab641f5715..4276859e9254 100644 --- a/contrib/llvm/include/llvm/ADT/PointerUnion.h +++ b/contrib/llvm/include/llvm/ADT/PointerUnion.h @@ -25,7 +25,7 @@ namespace llvm { template struct PointerUnionTypeSelectorReturn { - typedef T Return; + using Return = T; }; /// Get a type based on whether two types are the same or not. @@ -33,25 +33,25 @@ template struct PointerUnionTypeSelectorReturn { /// For: /// /// \code -/// typedef typename PointerUnionTypeSelector::Return Ret; +/// using Ret = typename PointerUnionTypeSelector::Return; /// \endcode /// /// Ret will be EQ type if T1 is same as T2 or NE type otherwise. template struct PointerUnionTypeSelector { - typedef typename PointerUnionTypeSelectorReturn::Return Return; + using Return = typename PointerUnionTypeSelectorReturn::Return; }; template struct PointerUnionTypeSelector { - typedef typename PointerUnionTypeSelectorReturn::Return Return; + using Return = typename PointerUnionTypeSelectorReturn::Return; }; template struct PointerUnionTypeSelectorReturn< PointerUnionTypeSelector> { - typedef - typename PointerUnionTypeSelector::Return Return; + using Return = + typename PointerUnionTypeSelector::Return; }; /// Provide PointerLikeTypeTraits for void* that is used by PointerUnion @@ -86,8 +86,8 @@ template class PointerUnionUIntTraits { /// X = P.get(); // runtime assertion failure. template class PointerUnion { public: - typedef PointerIntPair> - ValTy; + using ValTy = + PointerIntPair>; private: ValTy Val; @@ -102,7 +102,6 @@ template class PointerUnion { public: PointerUnion() = default; - PointerUnion(PT1 V) : Val(const_cast( PointerLikeTypeTraits::getAsVoidPointer(V))) {} @@ -117,14 +116,15 @@ template class PointerUnion { // we recursively strip off low bits if we have a nested PointerUnion. return !PointerLikeTypeTraits::getFromVoidPointer(Val.getPointer()); } + explicit operator bool() const { return !isNull(); } /// Test if the Union currently holds the type matching T. template int is() const { - typedef typename ::llvm::PointerUnionTypeSelector< - PT1, T, IsPT1, ::llvm::PointerUnionTypeSelector< - PT2, T, IsPT2, UNION_DOESNT_CONTAIN_TYPE>>::Return - Ty; + using Ty = typename ::llvm::PointerUnionTypeSelector< + PT1, T, IsPT1, + ::llvm::PointerUnionTypeSelector>>::Return; int TyNo = Ty::Num; return static_cast(Val.getInt()) == TyNo; } @@ -158,7 +158,8 @@ template class PointerUnion { assert( get() == Val.getPointer() && "Can't get the address because PointerLikeTypeTraits changes the ptr"); - return const_cast(reinterpret_cast(Val.getAddrOfPointer())); + return const_cast( + reinterpret_cast(Val.getAddrOfPointer())); } /// Assignment from nullptr which just clears the union. @@ -207,8 +208,7 @@ bool operator<(PointerUnion lhs, PointerUnion rhs) { // Teach SmallPtrSet that PointerUnion is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits)-1. template -class PointerLikeTypeTraits> { -public: +struct PointerLikeTypeTraits> { static inline void *getAsVoidPointer(const PointerUnion &P) { return P.getOpaqueValue(); } @@ -228,19 +228,22 @@ class PointerLikeTypeTraits> { /// for usage. template class PointerUnion3 { public: - typedef PointerUnion InnerUnion; - typedef PointerUnion ValTy; + using InnerUnion = PointerUnion; + using ValTy = PointerUnion; private: ValTy Val; struct IsInnerUnion { ValTy Val; + IsInnerUnion(ValTy val) : Val(val) {} + template int is() const { return Val.template is() && Val.template get().template is(); } + template T get() const { return Val.template get().template get(); } @@ -248,14 +251,15 @@ template class PointerUnion3 { struct IsPT3 { ValTy Val; + IsPT3(ValTy val) : Val(val) {} + template int is() const { return Val.template is(); } template T get() const { return Val.template get(); } }; public: PointerUnion3() = default; - PointerUnion3(PT1 V) { Val = InnerUnion(V); } PointerUnion3(PT2 V) { Val = InnerUnion(V); } PointerUnion3(PT3 V) { Val = V; } @@ -268,10 +272,9 @@ template class PointerUnion3 { /// Test if the Union currently holds the type matching T. template int is() const { // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - typedef typename ::llvm::PointerUnionTypeSelector< + using Ty = typename ::llvm::PointerUnionTypeSelector< PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector>::Return - Ty; + ::llvm::PointerUnionTypeSelector>::Return; return Ty(Val).template is(); } @@ -281,10 +284,9 @@ template class PointerUnion3 { template T get() const { assert(is() && "Invalid accessor called"); // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - typedef typename ::llvm::PointerUnionTypeSelector< + using Ty = typename ::llvm::PointerUnionTypeSelector< PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector>::Return - Ty; + ::llvm::PointerUnionTypeSelector>::Return; return Ty(Val).template get(); } @@ -328,8 +330,7 @@ template class PointerUnion3 { // Teach SmallPtrSet that PointerUnion3 is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits,PT2bits)-2. template -class PointerLikeTypeTraits> { -public: +struct PointerLikeTypeTraits> { static inline void *getAsVoidPointer(const PointerUnion3 &P) { return P.getOpaqueValue(); } @@ -350,16 +351,15 @@ class PointerLikeTypeTraits> { template class PointerUnion4 { public: - typedef PointerUnion InnerUnion1; - typedef PointerUnion InnerUnion2; - typedef PointerUnion ValTy; + using InnerUnion1 = PointerUnion; + using InnerUnion2 = PointerUnion; + using ValTy = PointerUnion; private: ValTy Val; public: PointerUnion4() = default; - PointerUnion4(PT1 V) { Val = InnerUnion1(V); } PointerUnion4(PT2 V) { Val = InnerUnion1(V); } PointerUnion4(PT3 V) { Val = InnerUnion2(V); } @@ -373,9 +373,10 @@ class PointerUnion4 { /// Test if the Union currently holds the type matching T. template int is() const { // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - typedef typename ::llvm::PointerUnionTypeSelector< - PT1, T, InnerUnion1, ::llvm::PointerUnionTypeSelector< - PT2, T, InnerUnion1, InnerUnion2>>::Return Ty; + using Ty = typename ::llvm::PointerUnionTypeSelector< + PT1, T, InnerUnion1, + ::llvm::PointerUnionTypeSelector>::Return; return Val.template is() && Val.template get().template is(); } @@ -385,9 +386,10 @@ class PointerUnion4 { template T get() const { assert(is() && "Invalid accessor called"); // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - typedef typename ::llvm::PointerUnionTypeSelector< - PT1, T, InnerUnion1, ::llvm::PointerUnionTypeSelector< - PT2, T, InnerUnion1, InnerUnion2>>::Return Ty; + using Ty = typename ::llvm::PointerUnionTypeSelector< + PT1, T, InnerUnion1, + ::llvm::PointerUnionTypeSelector>::Return; return Val.template get().template get(); } @@ -435,8 +437,7 @@ class PointerUnion4 { // Teach SmallPtrSet that PointerUnion4 is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits,PT2bits)-2. template -class PointerLikeTypeTraits> { -public: +struct PointerLikeTypeTraits> { static inline void * getAsVoidPointer(const PointerUnion4 &P) { return P.getOpaqueValue(); @@ -455,18 +456,21 @@ class PointerLikeTypeTraits> { // Teach DenseMap how to use PointerUnions as keys. template struct DenseMapInfo> { - typedef PointerUnion Pair; - typedef DenseMapInfo FirstInfo; - typedef DenseMapInfo SecondInfo; + using Pair = PointerUnion; + using FirstInfo = DenseMapInfo; + using SecondInfo = DenseMapInfo; static inline Pair getEmptyKey() { return Pair(FirstInfo::getEmptyKey()); } + static inline Pair getTombstoneKey() { return Pair(FirstInfo::getTombstoneKey()); } + static unsigned getHashValue(const Pair &PairVal) { intptr_t key = (intptr_t)PairVal.getOpaqueValue(); return DenseMapInfo::getHashValue(key); } + static bool isEqual(const Pair &LHS, const Pair &RHS) { return LHS.template is() == RHS.template is() && (LHS.template is() ? FirstInfo::isEqual(LHS.template get(), diff --git a/contrib/llvm/include/llvm/ADT/STLExtras.h b/contrib/llvm/include/llvm/ADT/STLExtras.h index 83f289c42a23..bcd992b4a716 100644 --- a/contrib/llvm/include/llvm/ADT/STLExtras.h +++ b/contrib/llvm/include/llvm/ADT/STLExtras.h @@ -17,23 +17,24 @@ #ifndef LLVM_ADT_STLEXTRAS_H #define LLVM_ADT_STLEXTRAS_H -#include // for std::all_of -#include -#include // for std::size_t -#include // for qsort -#include -#include -#include -#include -#include -#include // for std::pair - #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace llvm { @@ -50,14 +51,15 @@ template using ValueOfRange = typename std::remove_reference()))>::type; -} // End detail namespace +} // end namespace detail //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// -template -struct identity : public std::unary_function { +template struct identity { + using argument_type = Ty; + Ty &operator()(Ty &self) const { return self; } @@ -66,15 +68,13 @@ struct identity : public std::unary_function { } }; -template -struct less_ptr : public std::binary_function { +template struct less_ptr { bool operator()(const Ty* left, const Ty* right) const { return *left < *right; } }; -template -struct greater_ptr : public std::binary_function { +template struct greater_ptr { bool operator()(const Ty* left, const Ty* right) const { return *right < *left; } @@ -90,7 +90,7 @@ template class function_ref; template class function_ref { - Ret (*callback)(intptr_t callable, Params ...params); + Ret (*callback)(intptr_t callable, Params ...params) = nullptr; intptr_t callable; template @@ -100,7 +100,7 @@ class function_ref { } public: - function_ref() : callback(nullptr) {} + function_ref() = default; template function_ref(Callable &&callable, @@ -109,6 +109,7 @@ class function_ref { function_ref>::value>::type * = nullptr) : callback(callback_fn::type>), callable(reinterpret_cast(&callable)) {} + Ret operator()(Params ...params) const { return callback(callable, std::forward(params)...); } @@ -120,114 +121,95 @@ class function_ref { // delete on something. It is used like this: // // for_each(V.begin(), B.end(), deleter); -// template inline void deleter(T *Ptr) { delete Ptr; } - - //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// -// mapped_iterator - This is a simple iterator adapter that causes a function to -// be applied whenever operator* is invoked on the iterator. -// -template -class mapped_iterator { - RootIt current; - UnaryFunc Fn; -public: - typedef typename std::iterator_traits::iterator_category - iterator_category; - typedef typename std::iterator_traits::difference_type - difference_type; - typedef decltype(std::declval()(*std::declval())) - value_type; +namespace adl_detail { - typedef void pointer; - //typedef typename UnaryFunc::result_type *pointer; - typedef void reference; // Can't modify value returned by fn +using std::begin; - typedef RootIt iterator_type; - - inline const RootIt &getCurrent() const { return current; } - inline const UnaryFunc &getFunc() const { return Fn; } - - inline explicit mapped_iterator(const RootIt &I, UnaryFunc F) - : current(I), Fn(F) {} - - inline value_type operator*() const { // All this work to do this - return Fn(*current); // little change - } - - mapped_iterator &operator++() { - ++current; - return *this; - } - mapped_iterator &operator--() { - --current; - return *this; - } - mapped_iterator operator++(int) { - mapped_iterator __tmp = *this; - ++current; - return __tmp; - } - mapped_iterator operator--(int) { - mapped_iterator __tmp = *this; - --current; - return __tmp; - } - mapped_iterator operator+(difference_type n) const { - return mapped_iterator(current + n, Fn); - } - mapped_iterator &operator+=(difference_type n) { - current += n; - return *this; - } - mapped_iterator operator-(difference_type n) const { - return mapped_iterator(current - n, Fn); - } - mapped_iterator &operator-=(difference_type n) { - current -= n; - return *this; - } - reference operator[](difference_type n) const { return *(*this + n); } - - bool operator!=(const mapped_iterator &X) const { return !operator==(X); } - bool operator==(const mapped_iterator &X) const { - return current == X.current; - } - bool operator<(const mapped_iterator &X) const { return current < X.current; } - - difference_type operator-(const mapped_iterator &X) const { - return current - X.current; - } -}; - -template -inline mapped_iterator -operator+(typename mapped_iterator::difference_type N, - const mapped_iterator &X) { - return mapped_iterator(X.getCurrent() - N, X.getFunc()); +template +auto adl_begin(ContainerTy &&container) + -> decltype(begin(std::forward(container))) { + return begin(std::forward(container)); } +using std::end; + +template +auto adl_end(ContainerTy &&container) + -> decltype(end(std::forward(container))) { + return end(std::forward(container)); +} + +using std::swap; + +template +void adl_swap(T &&lhs, T &&rhs) noexcept(noexcept(swap(std::declval(), + std::declval()))) { + swap(std::forward(lhs), std::forward(rhs)); +} + +} // end namespace adl_detail + +template +auto adl_begin(ContainerTy &&container) + -> decltype(adl_detail::adl_begin(std::forward(container))) { + return adl_detail::adl_begin(std::forward(container)); +} + +template +auto adl_end(ContainerTy &&container) + -> decltype(adl_detail::adl_end(std::forward(container))) { + return adl_detail::adl_end(std::forward(container)); +} + +template +void adl_swap(T &&lhs, T &&rhs) noexcept( + noexcept(adl_detail::adl_swap(std::declval(), std::declval()))) { + adl_detail::adl_swap(std::forward(lhs), std::forward(rhs)); +} + +// mapped_iterator - This is a simple iterator adapter that causes a function to +// be applied whenever operator* is invoked on the iterator. + +template ()(*std::declval()))> +class mapped_iterator + : public iterator_adaptor_base< + mapped_iterator, ItTy, + typename std::iterator_traits::iterator_category, + typename std::remove_reference::type> { +public: + mapped_iterator(ItTy U, FuncTy F) + : mapped_iterator::iterator_adaptor_base(std::move(U)), F(std::move(F)) {} + + ItTy getCurrent() { return this->I; } + + FuncReturnTy operator*() { return F(*this->I); } + +private: + FuncTy F; +}; // map_iterator - Provide a convenient way to create mapped_iterators, just like // make_pair is useful for creating pairs... -// template -inline mapped_iterator map_iterator(const ItTy &I, FuncTy F) { - return mapped_iterator(I, F); +inline mapped_iterator map_iterator(ItTy I, FuncTy F) { + return mapped_iterator(std::move(I), std::move(F)); } /// Helper to determine if type T has a member called rbegin(). template class has_rbegin_impl { - typedef char yes[1]; - typedef char no[2]; + using yes = char[1]; + using no = char[2]; template static yes& test(Inner *I, decltype(I->rbegin()) * = nullptr); @@ -365,12 +347,13 @@ template struct index_sequence; template struct index_sequence_for; namespace detail { + using std::declval; // We have to alias this since inlining the actual type at the usage site // in the parameter list of iterator_facade_base<> below ICEs MSVC 2017. template struct ZipTupleType { - typedef std::tuple())...> type; + using type = std::tuple())...>; }; template @@ -456,11 +439,11 @@ class zip_shortest : public zip_common, Iters...> { public: using Base = zip_common, Iters...>; + zip_shortest(Iters &&... ts) : Base(std::forward(ts)...) {} + bool operator==(const zip_shortest &other) const { return !test(other, index_sequence_for{}); } - - zip_shortest(Iters &&... ts) : Base(std::forward(ts)...) {} }; template