diff --git a/Makefile.inc1 b/Makefile.inc1 index 56582b8fdd80..ee530735c31c 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1240,8 +1240,8 @@ STAGEDIR= ${MAKEOBJDIRPREFIX}${.CURDIR}/stage packages: @mkdir -p ${MAKEOBJDIRPREFIX}${.CURDIR}/stage ${_+_}@cd ${.CURDIR}; \ - ${MAKE} buildworld >/dev/null ; \ - ${MAKE} DESTDIR=${DESTDIR:U${STAGEDIR}} -DNO_ROOT stageworld >/dev/null ; \ + ${MAKE} buildworld ; \ + ${MAKE} DESTDIR=${DESTDIR:U${STAGEDIR}} -DNO_ROOT stageworld ; \ ${MAKE} DESTDIR=${DESTDIR:U${STAGEDIR}} create-world-packages kernel-pkgs: @@ -1805,6 +1805,10 @@ gnu/lib/libstdc++__L: lib/msun__L gnu/lib/libsupc++__L: gnu/lib/libstdc++__L .endif +.if ${MK_LIBCPLUSPLUS} != "no" +_prebuild_libs+= lib/libc++ +.endif + lib/libgeom__L: lib/libexpat__L .if ${MK_LIBTHR} != "no" diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index 1678bd04d23c..ea41a49866fc 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -38,6 +38,61 @@ # xargs -n1 | sort | uniq -d; # done +# 20150315: new clang import which bumps version from 3.5.1 to 3.6.0. +OLD_FILES+=usr/include/clang/3.5.1/__wmmintrin_aes.h +OLD_FILES+=usr/include/clang/3.5.1/__wmmintrin_pclmul.h +OLD_FILES+=usr/include/clang/3.5.1/altivec.h +OLD_FILES+=usr/include/clang/3.5.1/ammintrin.h +OLD_FILES+=usr/include/clang/3.5.1/arm_acle.h +OLD_FILES+=usr/include/clang/3.5.1/arm_neon.h +OLD_FILES+=usr/include/clang/3.5.1/avx2intrin.h +OLD_FILES+=usr/include/clang/3.5.1/avxintrin.h +OLD_FILES+=usr/include/clang/3.5.1/bmi2intrin.h +OLD_FILES+=usr/include/clang/3.5.1/bmiintrin.h +OLD_FILES+=usr/include/clang/3.5.1/cpuid.h +OLD_FILES+=usr/include/clang/3.5.1/emmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/f16cintrin.h +OLD_FILES+=usr/include/clang/3.5.1/fma4intrin.h +OLD_FILES+=usr/include/clang/3.5.1/fmaintrin.h +OLD_FILES+=usr/include/clang/3.5.1/ia32intrin.h +OLD_FILES+=usr/include/clang/3.5.1/immintrin.h +OLD_FILES+=usr/include/clang/3.5.1/lzcntintrin.h +OLD_FILES+=usr/include/clang/3.5.1/mm3dnow.h +OLD_FILES+=usr/include/clang/3.5.1/mm_malloc.h +OLD_FILES+=usr/include/clang/3.5.1/mmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/module.modulemap +OLD_FILES+=usr/include/clang/3.5.1/nmmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/pmmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/popcntintrin.h +OLD_FILES+=usr/include/clang/3.5.1/prfchwintrin.h +OLD_FILES+=usr/include/clang/3.5.1/rdseedintrin.h +OLD_FILES+=usr/include/clang/3.5.1/rtmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/shaintrin.h +OLD_FILES+=usr/include/clang/3.5.1/smmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/tbmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/tmmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/wmmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/x86intrin.h +OLD_FILES+=usr/include/clang/3.5.1/xmmintrin.h +OLD_FILES+=usr/include/clang/3.5.1/xopintrin.h +OLD_DIRS+=usr/include/clang/3.5.1 +OLD_DIRS+=usr/include/clang +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan-i386.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan-x86_64.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan_cxx-i386.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-arm.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-i386.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-x86_64.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.san-i386.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.san-x86_64.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan-i386.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan-x86_64.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan_cxx-i386.a +OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a +OLD_DIRS+=usr/lib/clang/3.5.1/lib/freebsd +OLD_DIRS+=usr/lib/clang/3.5.1/lib +OLD_DIRS+=usr/lib/clang/3.5.1 # 20150302: binutils documentation distributed as a manpage OLD_FILES+=usr/share/doc/binutils/as.txt OLD_FILES+=usr/share/doc/binutils/ld.txt diff --git a/UPDATING b/UPDATING index 81f47deaf051..f26428cd8445 100644 --- a/UPDATING +++ b/UPDATING @@ -31,6 +31,11 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 11.x IS SLOW: disable the most expensive debugging functionality run "ln -s 'abort:false,junk:false' /etc/malloc.conf".) +20150315: + Clang, llvm and lldb have been upgraded to 3.6.0 release. Please see + the 20141231 entry below for information about prerequisites and + upgrading, if you are not already using 3.5.0 or higher. + 20150307: The 32-bit PowerPC kernel has been changed to a position-independent executable. This can only be booted with a version of loader(8) diff --git a/bin/ps/print.c b/bin/ps/print.c index bcd451e1fb59..40ecd07d0318 100644 --- a/bin/ps/print.c +++ b/bin/ps/print.c @@ -383,7 +383,6 @@ started(KINFO *k, VARENT *ve __unused) { time_t then; struct tm *tp; - static int use_ampm = -1; size_t buflen = 100; char *buf; @@ -394,16 +393,12 @@ started(KINFO *k, VARENT *ve __unused) if (buf == NULL) errx(1, "malloc failed"); - if (use_ampm < 0) - use_ampm = (*nl_langinfo(T_FMT_AMPM) != '\0'); then = k->ki_p->ki_start.tv_sec; tp = localtime(&then); if (now - k->ki_p->ki_start.tv_sec < 24 * 3600) { - (void)strftime(buf, buflen, - use_ampm ? "%l:%M%p" : "%k:%M ", tp); + (void)strftime(buf, buflen, "%H:%M ", tp); } else if (now - k->ki_p->ki_start.tv_sec < 7 * 86400) { - (void)strftime(buf, buflen, - use_ampm ? "%a%I%p" : "%a%H ", tp); + (void)strftime(buf, buflen, "%a%H ", tp); } else (void)strftime(buf, buflen, "%e%b%y", tp); return (buf); diff --git a/bin/ps/ps.1 b/bin/ps/ps.1 index 7e26dd905a74..6eead7606890 100644 --- a/bin/ps/ps.1 +++ b/bin/ps/ps.1 @@ -381,12 +381,12 @@ the real memory (resident set) size of the process (in 1024 byte units). The time the command started. If the command started less than 24 hours ago, the start time is displayed using the -.Dq Li %l:ps.1p +.Dq Li %H:%M format described in .Xr strftime 3 . If the command started less than 7 days ago, the start time is displayed using the -.Dq Li %a6.15p +.Dq Li %a%H format. Otherwise, the start time is displayed using the .Dq Li %e%b%y diff --git a/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c b/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c index c6f47af867d8..823e25e81edb 100644 --- a/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c +++ b/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c @@ -492,7 +492,7 @@ write_type(void *arg1, void *arg2) if (i > CTF_MAX_VLEN) { terminate("function %s has too many args: %d > %d\n", - i, CTF_MAX_VLEN); + tdesc_name(tp), i, CTF_MAX_VLEN); } ctt.ctt_info = CTF_TYPE_INFO(CTF_K_FUNCTION, isroot, i); diff --git a/contrib/compiler-rt/include/sanitizer/asan_interface.h b/contrib/compiler-rt/include/sanitizer/asan_interface.h index 435391486aaa..7763389ab257 100644 --- a/contrib/compiler-rt/include/sanitizer/asan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/asan_interface.h @@ -114,8 +114,7 @@ extern "C" { // Returns the old value. int __asan_set_error_exit_code(int exit_code); - // Sets the callback to be called right before death on error. - // Passing 0 will unset the callback. + // Deprecated. Call __sanitizer_set_death_callback instead. void __asan_set_death_callback(void (*callback)(void)); void __asan_set_error_report_callback(void (*callback)(const char*)); diff --git a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h index 9cb5ad88de6a..ef645e527119 100644 --- a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h +++ b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h @@ -62,18 +62,6 @@ extern "C" { void __sanitizer_unaligned_store32(void *p, uint32_t x); void __sanitizer_unaligned_store64(void *p, uint64_t x); - // Initialize coverage. - void __sanitizer_cov_init(); - // Record and dump coverage info. - void __sanitizer_cov_dump(); - // Open .sancov.packed in the coverage directory and return the file - // descriptor. Returns -1 on failure, or if coverage dumping is disabled. - // This is intended for use by sandboxing code. - intptr_t __sanitizer_maybe_open_cov_file(const char *name); - // Get the number of total unique covered entities (blocks, edges, calls). - // This can be useful for coverage-directed in-process fuzzers. - uintptr_t __sanitizer_get_total_unique_coverage(); - // Annotate the current state of a contiguous container, such as // std::vector, std::string or similar. // A contiguous container is a container that keeps all of its elements @@ -120,6 +108,9 @@ extern "C" { // Print the stack trace leading to this call. Useful for debugging user code. void __sanitizer_print_stack_trace(); + // Sets the callback to be called right before death on error. + // Passing 0 will unset the callback. + void __sanitizer_set_death_callback(void (*callback)(void)); #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/coverage_interface.h b/contrib/compiler-rt/include/sanitizer/coverage_interface.h new file mode 100644 index 000000000000..88a7e480081d --- /dev/null +++ b/contrib/compiler-rt/include/sanitizer/coverage_interface.h @@ -0,0 +1,46 @@ +//===-- sanitizer/coverage_interface.h --------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Public interface for sanitizer coverage. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_COVERAG_INTERFACE_H +#define SANITIZER_COVERAG_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + // Initialize coverage. + void __sanitizer_cov_init(); + // Record and dump coverage info. + void __sanitizer_cov_dump(); + // Open .sancov.packed in the coverage directory and return the file + // descriptor. Returns -1 on failure, or if coverage dumping is disabled. + // This is intended for use by sandboxing code. + intptr_t __sanitizer_maybe_open_cov_file(const char *name); + // Get the number of total unique covered entities (blocks, edges, calls). + // This can be useful for coverage-directed in-process fuzzers. + uintptr_t __sanitizer_get_total_unique_coverage(); + + // Reset the basic-block (edge) coverage to the initial state. + // Useful for in-process fuzzing to start collecting coverage from scratch. + // Experimental, will likely not work for multi-threaded process. + void __sanitizer_reset_coverage(); + // Set *data to the array of covered PCs and return the size of that array. + // Some of the entries in *data will be zero. + uintptr_t __sanitizer_get_coverage_guards(uintptr_t **data); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_COVERAG_INTERFACE_H diff --git a/contrib/compiler-rt/include/sanitizer/msan_interface.h b/contrib/compiler-rt/include/sanitizer/msan_interface.h index 5be586047ced..c3a19bf345ac 100644 --- a/contrib/compiler-rt/include/sanitizer/msan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/msan_interface.h @@ -38,7 +38,9 @@ extern "C" { contents). */ void __msan_unpoison_string(const volatile char *a); - /* Make memory region fully uninitialized (without changing its contents). */ + /* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ void __msan_poison(const volatile void *a, size_t size); /* Make memory region partially uninitialized (without changing its contents). diff --git a/contrib/compiler-rt/lib/asan/README.txt b/contrib/compiler-rt/lib/asan/README.txt index b9c43acd5fe4..8cc9bb17b59d 100644 --- a/contrib/compiler-rt/lib/asan/README.txt +++ b/contrib/compiler-rt/lib/asan/README.txt @@ -1,7 +1,6 @@ AddressSanitizer RT ================================ -This directory contains sources of the AddressSanitizer (asan) runtime library. -We are in the process of integrating AddressSanitizer with LLVM, stay tuned. +This directory contains sources of the AddressSanitizer (ASan) runtime library. Directory structure: README.txt : This file. @@ -13,14 +12,13 @@ tests/* : ASan unit tests. Also ASan runtime needs the following libraries: lib/interception/ : Machinery used to intercept function calls. -lib/sanitizer_common/ : Code shared between ASan and TSan. +lib/sanitizer_common/ : Code shared between various sanitizers. -Currently ASan runtime can be built by both make and cmake build systems. -(see compiler-rt/make and files Makefile.mk for make-based build and -files CMakeLists.txt for cmake-based build). +ASan runtime currently also embeds part of LeakSanitizer runtime for +leak detection (lib/lsan/lsan_common.{cc,h}). -ASan unit and output tests work only with cmake. You may run this -command from the root of your cmake build tree: +ASan runtime can only be built by CMake. You can run ASan tests +from the root of your CMake build tree: make check-asan diff --git a/contrib/compiler-rt/lib/asan/asan_activation.cc b/contrib/compiler-rt/lib/asan/asan_activation.cc index eb4a6db0b85e..3bc01984898d 100644 --- a/contrib/compiler-rt/lib/asan/asan_activation.cc +++ b/contrib/compiler-rt/lib/asan/asan_activation.cc @@ -16,40 +16,106 @@ #include "asan_allocator.h" #include "asan_flags.h" #include "asan_internal.h" +#include "asan_poisoning.h" +#include "asan_stack.h" #include "sanitizer_common/sanitizer_flags.h" namespace __asan { static struct AsanDeactivatedFlags { - int quarantine_size; - int max_redzone; + AllocatorOptions allocator_options; int malloc_context_size; bool poison_heap; - bool alloc_dealloc_mismatch; - bool allocator_may_return_null; + bool coverage; + const char *coverage_dir; + + void RegisterActivationFlags(FlagParser *parser, Flags *f, CommonFlags *cf) { +#define ASAN_ACTIVATION_FLAG(Type, Name) \ + RegisterFlag(parser, #Name, "", &f->Name); +#define COMMON_ACTIVATION_FLAG(Type, Name) \ + RegisterFlag(parser, #Name, "", &cf->Name); +#include "asan_activation_flags.inc" +#undef ASAN_ACTIVATION_FLAG +#undef COMMON_ACTIVATION_FLAG + + RegisterIncludeFlag(parser, cf); + } + + void OverrideFromActivationFlags() { + Flags f; + CommonFlags cf; + FlagParser parser; + RegisterActivationFlags(&parser, &f, &cf); + + // Copy the current activation flags. + allocator_options.CopyTo(&f, &cf); + cf.malloc_context_size = malloc_context_size; + f.poison_heap = poison_heap; + cf.coverage = coverage; + cf.coverage_dir = coverage_dir; + cf.verbosity = Verbosity(); + cf.help = false; // this is activation-specific help + + // Check if activation flags need to be overriden. + if (const char *env = GetEnv("ASAN_ACTIVATION_OPTIONS")) { + parser.ParseString(env); + } + + // Override from getprop asan.options. + char buf[100]; + GetExtraActivationFlags(buf, sizeof(buf)); + parser.ParseString(buf); + + SetVerbosity(cf.verbosity); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (cf.help) parser.PrintFlagDescriptions(); + + allocator_options.SetFrom(&f, &cf); + malloc_context_size = cf.malloc_context_size; + poison_heap = f.poison_heap; + coverage = cf.coverage; + coverage_dir = cf.coverage_dir; + } + + void Print() { + Report( + "quarantine_size_mb %d, max_redzone %d, poison_heap %d, " + "malloc_context_size %d, alloc_dealloc_mismatch %d, " + "allocator_may_return_null %d, coverage %d, coverage_dir %s\n", + allocator_options.quarantine_size_mb, allocator_options.max_redzone, + poison_heap, malloc_context_size, + allocator_options.alloc_dealloc_mismatch, + allocator_options.may_return_null, coverage, coverage_dir); + } } asan_deactivated_flags; static bool asan_is_deactivated; -void AsanStartDeactivated() { +void AsanDeactivate() { + CHECK(!asan_is_deactivated); VReport(1, "Deactivating ASan\n"); - // Save flag values. - asan_deactivated_flags.quarantine_size = flags()->quarantine_size; - asan_deactivated_flags.max_redzone = flags()->max_redzone; - asan_deactivated_flags.poison_heap = flags()->poison_heap; - asan_deactivated_flags.malloc_context_size = - common_flags()->malloc_context_size; - asan_deactivated_flags.alloc_dealloc_mismatch = - flags()->alloc_dealloc_mismatch; - asan_deactivated_flags.allocator_may_return_null = - common_flags()->allocator_may_return_null; - flags()->quarantine_size = 0; - flags()->max_redzone = 16; - flags()->poison_heap = false; - common_flags()->malloc_context_size = 0; - flags()->alloc_dealloc_mismatch = false; - common_flags()->allocator_may_return_null = true; + // Stash runtime state. + GetAllocatorOptions(&asan_deactivated_flags.allocator_options); + asan_deactivated_flags.malloc_context_size = GetMallocContextSize(); + asan_deactivated_flags.poison_heap = CanPoisonMemory(); + asan_deactivated_flags.coverage = common_flags()->coverage; + asan_deactivated_flags.coverage_dir = common_flags()->coverage_dir; + + // Deactivate the runtime. + SetCanPoisonMemory(false); + SetMallocContextSize(1); + ReInitializeCoverage(false, nullptr); + + AllocatorOptions disabled = asan_deactivated_flags.allocator_options; + disabled.quarantine_size_mb = 0; + disabled.min_redzone = 16; // Redzone must be at least 16 bytes long. + disabled.max_redzone = 16; + disabled.alloc_dealloc_mismatch = false; + disabled.may_return_null = true; + ReInitializeAllocator(disabled); asan_is_deactivated = true; } @@ -58,31 +124,19 @@ void AsanActivate() { if (!asan_is_deactivated) return; VReport(1, "Activating ASan\n"); - // Restore flag values. - // FIXME: this is not atomic, and there may be other threads alive. - flags()->quarantine_size = asan_deactivated_flags.quarantine_size; - flags()->max_redzone = asan_deactivated_flags.max_redzone; - flags()->poison_heap = asan_deactivated_flags.poison_heap; - common_flags()->malloc_context_size = - asan_deactivated_flags.malloc_context_size; - flags()->alloc_dealloc_mismatch = - asan_deactivated_flags.alloc_dealloc_mismatch; - common_flags()->allocator_may_return_null = - asan_deactivated_flags.allocator_may_return_null; + asan_deactivated_flags.OverrideFromActivationFlags(); - ParseExtraActivationFlags(); - - ReInitializeAllocator(); + SetCanPoisonMemory(asan_deactivated_flags.poison_heap); + SetMallocContextSize(asan_deactivated_flags.malloc_context_size); + ReInitializeCoverage(asan_deactivated_flags.coverage, + asan_deactivated_flags.coverage_dir); + ReInitializeAllocator(asan_deactivated_flags.allocator_options); asan_is_deactivated = false; - VReport( - 1, - "quarantine_size %d, max_redzone %d, poison_heap %d, " - "malloc_context_size %d, alloc_dealloc_mismatch %d, " - "allocator_may_return_null %d\n", - flags()->quarantine_size, flags()->max_redzone, flags()->poison_heap, - common_flags()->malloc_context_size, flags()->alloc_dealloc_mismatch, - common_flags()->allocator_may_return_null); + if (Verbosity()) { + Report("Activated with flags:\n"); + asan_deactivated_flags.Print(); + } } } // namespace __asan diff --git a/contrib/compiler-rt/lib/asan/asan_activation.h b/contrib/compiler-rt/lib/asan/asan_activation.h index dafb840a6042..d5e1ce433001 100644 --- a/contrib/compiler-rt/lib/asan/asan_activation.h +++ b/contrib/compiler-rt/lib/asan/asan_activation.h @@ -16,7 +16,7 @@ #define ASAN_ACTIVATION_H namespace __asan { -void AsanStartDeactivated(); +void AsanDeactivate(); void AsanActivate(); } // namespace __asan diff --git a/contrib/compiler-rt/lib/asan/asan_activation_flags.inc b/contrib/compiler-rt/lib/asan/asan_activation_flags.inc new file mode 100644 index 000000000000..d4c089ec6538 --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_activation_flags.inc @@ -0,0 +1,35 @@ +//===-- asan_activation_flags.inc -------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// A subset of ASan (and common) runtime flags supported at activation time. +// +//===----------------------------------------------------------------------===// +#ifndef ASAN_ACTIVATION_FLAG +# error "Define ASAN_ACTIVATION_FLAG prior to including this file!" +#endif + +#ifndef COMMON_ACTIVATION_FLAG +# error "Define COMMON_ACTIVATION_FLAG prior to including this file!" +#endif + +// ASAN_ACTIVATION_FLAG(Type, Name) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +ASAN_ACTIVATION_FLAG(int, redzone) +ASAN_ACTIVATION_FLAG(int, max_redzone) +ASAN_ACTIVATION_FLAG(int, quarantine_size_mb) +ASAN_ACTIVATION_FLAG(bool, alloc_dealloc_mismatch) +ASAN_ACTIVATION_FLAG(bool, poison_heap) + +COMMON_ACTIVATION_FLAG(bool, allocator_may_return_null) +COMMON_ACTIVATION_FLAG(int, malloc_context_size) +COMMON_ACTIVATION_FLAG(bool, coverage) +COMMON_ACTIVATION_FLAG(const char *, coverage_dir) +COMMON_ACTIVATION_FLAG(int, verbosity) +COMMON_ACTIVATION_FLAG(bool, help) diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.cc b/contrib/compiler-rt/lib/asan/asan_allocator.cc new file mode 100644 index 000000000000..fd63ac68c09e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_allocator.cc @@ -0,0 +1,909 @@ +//===-- asan_allocator.cc -------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Implementation of ASan's memory allocator, 2-nd version. +// This variant uses the allocator from sanitizer_common, i.e. the one shared +// with ThreadSanitizer and MemorySanitizer. +// +//===----------------------------------------------------------------------===// +#include "asan_allocator.h" + +#include "asan_mapping.h" +#include "asan_poisoning.h" +#include "asan_report.h" +#include "asan_stack.h" +#include "asan_thread.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_list.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_quarantine.h" +#include "lsan/lsan_common.h" + +namespace __asan { + +// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. +// We use adaptive redzones: for larger allocation larger redzones are used. +static u32 RZLog2Size(u32 rz_log) { + CHECK_LT(rz_log, 8); + return 16 << rz_log; +} + +static u32 RZSize2Log(u32 rz_size) { + CHECK_GE(rz_size, 16); + CHECK_LE(rz_size, 2048); + CHECK(IsPowerOfTwo(rz_size)); + u32 res = Log2(rz_size) - 4; + CHECK_EQ(rz_size, RZLog2Size(res)); + return res; +} + +static AsanAllocator &get_allocator(); + +// The memory chunk allocated from the underlying allocator looks like this: +// L L L L L L H H U U U U U U R R +// L -- left redzone words (0 or more bytes) +// H -- ChunkHeader (16 bytes), which is also a part of the left redzone. +// U -- user memory. +// R -- right redzone (0 or more bytes) +// ChunkBase consists of ChunkHeader and other bytes that overlap with user +// memory. + +// If the left redzone is greater than the ChunkHeader size we store a magic +// value in the first uptr word of the memory block and store the address of +// ChunkBase in the next uptr. +// M B L L L L L L L L L H H U U U U U U +// | ^ +// ---------------------| +// M -- magic value kAllocBegMagic +// B -- address of ChunkHeader pointing to the first 'H' +static const uptr kAllocBegMagic = 0xCC6E96B9; + +struct ChunkHeader { + // 1-st 8 bytes. + u32 chunk_state : 8; // Must be first. + u32 alloc_tid : 24; + + u32 free_tid : 24; + u32 from_memalign : 1; + u32 alloc_type : 2; + u32 rz_log : 3; + u32 lsan_tag : 2; + // 2-nd 8 bytes + // This field is used for small sizes. For large sizes it is equal to + // SizeClassMap::kMaxSize and the actual size is stored in the + // SecondaryAllocator's metadata. + u32 user_requested_size; + u32 alloc_context_id; +}; + +struct ChunkBase : ChunkHeader { + // Header2, intersects with user memory. + u32 free_context_id; +}; + +static const uptr kChunkHeaderSize = sizeof(ChunkHeader); +static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize; +COMPILER_CHECK(kChunkHeaderSize == 16); +COMPILER_CHECK(kChunkHeader2Size <= 16); + +// Every chunk of memory allocated by this allocator can be in one of 3 states: +// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated. +// CHUNK_ALLOCATED: the chunk is allocated and not yet freed. +// CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone. +enum { + CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it. + CHUNK_ALLOCATED = 2, + CHUNK_QUARANTINE = 3 +}; + +struct AsanChunk: ChunkBase { + uptr Beg() { return reinterpret_cast(this) + kChunkHeaderSize; } + uptr UsedSize(bool locked_version = false) { + if (user_requested_size != SizeClassMap::kMaxSize) + return user_requested_size; + return *reinterpret_cast( + get_allocator().GetMetaData(AllocBeg(locked_version))); + } + void *AllocBeg(bool locked_version = false) { + if (from_memalign) { + if (locked_version) + return get_allocator().GetBlockBeginFastLocked( + reinterpret_cast(this)); + return get_allocator().GetBlockBegin(reinterpret_cast(this)); + } + return reinterpret_cast(Beg() - RZLog2Size(rz_log)); + } + bool AddrIsInside(uptr addr, bool locked_version = false) { + return (addr >= Beg()) && (addr < Beg() + UsedSize(locked_version)); + } +}; + +struct QuarantineCallback { + explicit QuarantineCallback(AllocatorCache *cache) + : cache_(cache) { + } + + void Recycle(AsanChunk *m) { + CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); + atomic_store((atomic_uint8_t*)m, CHUNK_AVAILABLE, memory_order_relaxed); + CHECK_NE(m->alloc_tid, kInvalidTid); + CHECK_NE(m->free_tid, kInvalidTid); + PoisonShadow(m->Beg(), + RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), + kAsanHeapLeftRedzoneMagic); + void *p = reinterpret_cast(m->AllocBeg()); + if (p != m) { + uptr *alloc_magic = reinterpret_cast(p); + CHECK_EQ(alloc_magic[0], kAllocBegMagic); + // Clear the magic value, as allocator internals may overwrite the + // contents of deallocated chunk, confusing GetAsanChunk lookup. + alloc_magic[0] = 0; + CHECK_EQ(alloc_magic[1], reinterpret_cast(m)); + } + + // Statistics. + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.real_frees++; + thread_stats.really_freed += m->UsedSize(); + + get_allocator().Deallocate(cache_, p); + } + + void *Allocate(uptr size) { + return get_allocator().Allocate(cache_, size, 1, false); + } + + void Deallocate(void *p) { + get_allocator().Deallocate(cache_, p); + } + + AllocatorCache *cache_; +}; + +typedef Quarantine AsanQuarantine; +typedef AsanQuarantine::Cache QuarantineCache; + +void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const { + PoisonShadow(p, size, kAsanHeapLeftRedzoneMagic); + // Statistics. + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mmaps++; + thread_stats.mmaped += size; +} +void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const { + PoisonShadow(p, size, 0); + // We are about to unmap a chunk of user memory. + // Mark the corresponding shadow memory as not needed. + FlushUnneededASanShadowMemory(p, size); + // Statistics. + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.munmaps++; + thread_stats.munmaped += size; +} + +// We can not use THREADLOCAL because it is not supported on some of the +// platforms we care about (OSX 10.6, Android). +// static THREADLOCAL AllocatorCache cache; +AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) { + CHECK(ms); + return &ms->allocator_cache; +} + +QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) { + CHECK(ms); + CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache)); + return reinterpret_cast(ms->quarantine_cache); +} + +void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) { + quarantine_size_mb = f->quarantine_size_mb; + min_redzone = f->redzone; + max_redzone = f->max_redzone; + may_return_null = cf->allocator_may_return_null; + alloc_dealloc_mismatch = f->alloc_dealloc_mismatch; +} + +void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) { + f->quarantine_size_mb = quarantine_size_mb; + f->redzone = min_redzone; + f->max_redzone = max_redzone; + cf->allocator_may_return_null = may_return_null; + f->alloc_dealloc_mismatch = alloc_dealloc_mismatch; +} + +struct Allocator { + static const uptr kMaxAllowedMallocSize = + FIRST_32_SECOND_64(3UL << 30, 64UL << 30); + static const uptr kMaxThreadLocalQuarantine = + FIRST_32_SECOND_64(1 << 18, 1 << 20); + + AsanAllocator allocator; + AsanQuarantine quarantine; + StaticSpinMutex fallback_mutex; + AllocatorCache fallback_allocator_cache; + QuarantineCache fallback_quarantine_cache; + + // ------------------- Options -------------------------- + atomic_uint16_t min_redzone; + atomic_uint16_t max_redzone; + atomic_uint8_t alloc_dealloc_mismatch; + + // ------------------- Initialization ------------------------ + explicit Allocator(LinkerInitialized) + : quarantine(LINKER_INITIALIZED), + fallback_quarantine_cache(LINKER_INITIALIZED) {} + + void CheckOptions(const AllocatorOptions &options) const { + CHECK_GE(options.min_redzone, 16); + CHECK_GE(options.max_redzone, options.min_redzone); + CHECK_LE(options.max_redzone, 2048); + CHECK(IsPowerOfTwo(options.min_redzone)); + CHECK(IsPowerOfTwo(options.max_redzone)); + } + + void SharedInitCode(const AllocatorOptions &options) { + CheckOptions(options); + quarantine.Init((uptr)options.quarantine_size_mb << 20, + kMaxThreadLocalQuarantine); + atomic_store(&alloc_dealloc_mismatch, options.alloc_dealloc_mismatch, + memory_order_release); + atomic_store(&min_redzone, options.min_redzone, memory_order_release); + atomic_store(&max_redzone, options.max_redzone, memory_order_release); + } + + void Initialize(const AllocatorOptions &options) { + allocator.Init(options.may_return_null); + SharedInitCode(options); + } + + void ReInitialize(const AllocatorOptions &options) { + allocator.SetMayReturnNull(options.may_return_null); + SharedInitCode(options); + } + + void GetOptions(AllocatorOptions *options) const { + options->quarantine_size_mb = quarantine.GetSize() >> 20; + options->min_redzone = atomic_load(&min_redzone, memory_order_acquire); + options->max_redzone = atomic_load(&max_redzone, memory_order_acquire); + options->may_return_null = allocator.MayReturnNull(); + options->alloc_dealloc_mismatch = + atomic_load(&alloc_dealloc_mismatch, memory_order_acquire); + } + + // -------------------- Helper methods. ------------------------- + uptr ComputeRZLog(uptr user_requested_size) { + u32 rz_log = + user_requested_size <= 64 - 16 ? 0 : + user_requested_size <= 128 - 32 ? 1 : + user_requested_size <= 512 - 64 ? 2 : + user_requested_size <= 4096 - 128 ? 3 : + user_requested_size <= (1 << 14) - 256 ? 4 : + user_requested_size <= (1 << 15) - 512 ? 5 : + user_requested_size <= (1 << 16) - 1024 ? 6 : 7; + u32 min_rz = atomic_load(&min_redzone, memory_order_acquire); + u32 max_rz = atomic_load(&max_redzone, memory_order_acquire); + return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz)); + } + + // We have an address between two chunks, and we want to report just one. + AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk, + AsanChunk *right_chunk) { + // Prefer an allocated chunk over freed chunk and freed chunk + // over available chunk. + if (left_chunk->chunk_state != right_chunk->chunk_state) { + if (left_chunk->chunk_state == CHUNK_ALLOCATED) + return left_chunk; + if (right_chunk->chunk_state == CHUNK_ALLOCATED) + return right_chunk; + if (left_chunk->chunk_state == CHUNK_QUARANTINE) + return left_chunk; + if (right_chunk->chunk_state == CHUNK_QUARANTINE) + return right_chunk; + } + // Same chunk_state: choose based on offset. + sptr l_offset = 0, r_offset = 0; + CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset)); + CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset)); + if (l_offset < r_offset) + return left_chunk; + return right_chunk; + } + + // -------------------- Allocation/Deallocation routines --------------- + void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, + AllocType alloc_type, bool can_fill) { + if (UNLIKELY(!asan_inited)) + AsanInitFromRtl(); + Flags &fl = *flags(); + CHECK(stack); + const uptr min_alignment = SHADOW_GRANULARITY; + if (alignment < min_alignment) + alignment = min_alignment; + if (size == 0) { + // We'd be happy to avoid allocating memory for zero-size requests, but + // some programs/tests depend on this behavior and assume that malloc + // would not return NULL even for zero-size allocations. Moreover, it + // looks like operator new should never return NULL, and results of + // consecutive "new" calls must be different even if the allocated size + // is zero. + size = 1; + } + CHECK(IsPowerOfTwo(alignment)); + uptr rz_log = ComputeRZLog(size); + uptr rz_size = RZLog2Size(rz_log); + uptr rounded_size = RoundUpTo(Max(size, kChunkHeader2Size), alignment); + uptr needed_size = rounded_size + rz_size; + if (alignment > min_alignment) + needed_size += alignment; + bool using_primary_allocator = true; + // If we are allocating from the secondary allocator, there will be no + // automatic right redzone, so add the right redzone manually. + if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) { + needed_size += rz_size; + using_primary_allocator = false; + } + CHECK(IsAligned(needed_size, min_alignment)); + if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) { + Report("WARNING: AddressSanitizer failed to allocate %p bytes\n", + (void*)size); + return allocator.ReturnNullOrDie(); + } + + AsanThread *t = GetCurrentThread(); + void *allocated; + bool check_rss_limit = true; + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocated = + allocator.Allocate(cache, needed_size, 8, false, check_rss_limit); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocated = + allocator.Allocate(cache, needed_size, 8, false, check_rss_limit); + } + + if (!allocated) + return allocator.ReturnNullOrDie(); + + if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) { + // Heap poisoning is enabled, but the allocator provides an unpoisoned + // chunk. This is possible if CanPoisonMemory() was false for some + // time, for example, due to flags()->start_disabled. + // Anyway, poison the block before using it for anything else. + uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated); + PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic); + } + + uptr alloc_beg = reinterpret_cast(allocated); + uptr alloc_end = alloc_beg + needed_size; + uptr beg_plus_redzone = alloc_beg + rz_size; + uptr user_beg = beg_plus_redzone; + if (!IsAligned(user_beg, alignment)) + user_beg = RoundUpTo(user_beg, alignment); + uptr user_end = user_beg + size; + CHECK_LE(user_end, alloc_end); + uptr chunk_beg = user_beg - kChunkHeaderSize; + AsanChunk *m = reinterpret_cast(chunk_beg); + m->alloc_type = alloc_type; + m->rz_log = rz_log; + u32 alloc_tid = t ? t->tid() : 0; + m->alloc_tid = alloc_tid; + CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield? + m->free_tid = kInvalidTid; + m->from_memalign = user_beg != beg_plus_redzone; + if (alloc_beg != chunk_beg) { + CHECK_LE(alloc_beg+ 2 * sizeof(uptr), chunk_beg); + reinterpret_cast(alloc_beg)[0] = kAllocBegMagic; + reinterpret_cast(alloc_beg)[1] = chunk_beg; + } + if (using_primary_allocator) { + CHECK(size); + m->user_requested_size = size; + CHECK(allocator.FromPrimary(allocated)); + } else { + CHECK(!allocator.FromPrimary(allocated)); + m->user_requested_size = SizeClassMap::kMaxSize; + uptr *meta = reinterpret_cast(allocator.GetMetaData(allocated)); + meta[0] = size; + meta[1] = chunk_beg; + } + + m->alloc_context_id = StackDepotPut(*stack); + + uptr size_rounded_down_to_granularity = + RoundDownTo(size, SHADOW_GRANULARITY); + // Unpoison the bulk of the memory region. + if (size_rounded_down_to_granularity) + PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); + // Deal with the end of the region if size is not aligned to granularity. + if (size != size_rounded_down_to_granularity && CanPoisonMemory()) { + u8 *shadow = + (u8 *)MemToShadow(user_beg + size_rounded_down_to_granularity); + *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY - 1)) : 0; + } + + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mallocs++; + thread_stats.malloced += size; + thread_stats.malloced_redzones += needed_size - size; + uptr class_id = + Min(kNumberOfSizeClasses, SizeClassMap::ClassID(needed_size)); + thread_stats.malloced_by_size[class_id]++; + if (needed_size > SizeClassMap::kMaxSize) + thread_stats.malloc_large++; + + void *res = reinterpret_cast(user_beg); + if (can_fill && fl.max_malloc_fill_size) { + uptr fill_size = Min(size, (uptr)fl.max_malloc_fill_size); + REAL(memset)(res, fl.malloc_fill_byte, fill_size); + } +#if CAN_SANITIZE_LEAKS + m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored + : __lsan::kDirectlyLeaked; +#endif + // Must be the last mutation of metadata in this function. + atomic_store((atomic_uint8_t *)m, CHUNK_ALLOCATED, memory_order_release); + ASAN_MALLOC_HOOK(res, size); + return res; + } + + void AtomicallySetQuarantineFlag(AsanChunk *m, void *ptr, + BufferedStackTrace *stack) { + u8 old_chunk_state = CHUNK_ALLOCATED; + // Flip the chunk_state atomically to avoid race on double-free. + if (!atomic_compare_exchange_strong((atomic_uint8_t*)m, &old_chunk_state, + CHUNK_QUARANTINE, memory_order_acquire)) + ReportInvalidFree(ptr, old_chunk_state, stack); + CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state); + } + + // Expects the chunk to already be marked as quarantined by using + // AtomicallySetQuarantineFlag. + void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack, + AllocType alloc_type) { + CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); + + if (m->alloc_type != alloc_type) { + if (atomic_load(&alloc_dealloc_mismatch, memory_order_acquire)) { + ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type, + (AllocType)alloc_type); + } + } + + CHECK_GE(m->alloc_tid, 0); + if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area. + CHECK_EQ(m->free_tid, kInvalidTid); + AsanThread *t = GetCurrentThread(); + m->free_tid = t ? t->tid() : 0; + m->free_context_id = StackDepotPut(*stack); + // Poison the region. + PoisonShadow(m->Beg(), + RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), + kAsanHeapFreeMagic); + + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.frees++; + thread_stats.freed += m->UsedSize(); + + // Push into quarantine. + if (t) { + AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); + AllocatorCache *ac = GetAllocatorCache(ms); + quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m, + m->UsedSize()); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *ac = &fallback_allocator_cache; + quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m, + m->UsedSize()); + } + } + + void Deallocate(void *ptr, uptr delete_size, BufferedStackTrace *stack, + AllocType alloc_type) { + uptr p = reinterpret_cast(ptr); + if (p == 0) return; + + uptr chunk_beg = p - kChunkHeaderSize; + AsanChunk *m = reinterpret_cast(chunk_beg); + if (delete_size && flags()->new_delete_type_mismatch && + delete_size != m->UsedSize()) { + ReportNewDeleteSizeMismatch(p, delete_size, stack); + } + ASAN_FREE_HOOK(ptr); + // Must mark the chunk as quarantined before any changes to its metadata. + AtomicallySetQuarantineFlag(m, ptr, stack); + QuarantineChunk(m, ptr, stack, alloc_type); + } + + void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) { + CHECK(old_ptr && new_size); + uptr p = reinterpret_cast(old_ptr); + uptr chunk_beg = p - kChunkHeaderSize; + AsanChunk *m = reinterpret_cast(chunk_beg); + + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.reallocs++; + thread_stats.realloced += new_size; + + void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true); + if (new_ptr) { + u8 chunk_state = m->chunk_state; + if (chunk_state != CHUNK_ALLOCATED) + ReportInvalidFree(old_ptr, chunk_state, stack); + CHECK_NE(REAL(memcpy), (void*)0); + uptr memcpy_size = Min(new_size, m->UsedSize()); + // If realloc() races with free(), we may start copying freed memory. + // However, we will report racy double-free later anyway. + REAL(memcpy)(new_ptr, old_ptr, memcpy_size); + Deallocate(old_ptr, 0, stack, FROM_MALLOC); + } + return new_ptr; + } + + void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { + if (CallocShouldReturnNullDueToOverflow(size, nmemb)) + return allocator.ReturnNullOrDie(); + void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false); + // If the memory comes from the secondary allocator no need to clear it + // as it comes directly from mmap. + if (ptr && allocator.FromPrimary(ptr)) + REAL(memset)(ptr, 0, nmemb * size); + return ptr; + } + + void ReportInvalidFree(void *ptr, u8 chunk_state, BufferedStackTrace *stack) { + if (chunk_state == CHUNK_QUARANTINE) + ReportDoubleFree((uptr)ptr, stack); + else + ReportFreeNotMalloced((uptr)ptr, stack); + } + + void CommitBack(AsanThreadLocalMallocStorage *ms) { + AllocatorCache *ac = GetAllocatorCache(ms); + quarantine.Drain(GetQuarantineCache(ms), QuarantineCallback(ac)); + allocator.SwallowCache(ac); + } + + // -------------------------- Chunk lookup ---------------------- + + // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). + AsanChunk *GetAsanChunk(void *alloc_beg) { + if (!alloc_beg) return 0; + if (!allocator.FromPrimary(alloc_beg)) { + uptr *meta = reinterpret_cast(allocator.GetMetaData(alloc_beg)); + AsanChunk *m = reinterpret_cast(meta[1]); + return m; + } + uptr *alloc_magic = reinterpret_cast(alloc_beg); + if (alloc_magic[0] == kAllocBegMagic) + return reinterpret_cast(alloc_magic[1]); + return reinterpret_cast(alloc_beg); + } + + AsanChunk *GetAsanChunkByAddr(uptr p) { + void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast(p)); + return GetAsanChunk(alloc_beg); + } + + // Allocator must be locked when this function is called. + AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) { + void *alloc_beg = + allocator.GetBlockBeginFastLocked(reinterpret_cast(p)); + return GetAsanChunk(alloc_beg); + } + + uptr AllocationSize(uptr p) { + AsanChunk *m = GetAsanChunkByAddr(p); + if (!m) return 0; + if (m->chunk_state != CHUNK_ALLOCATED) return 0; + if (m->Beg() != p) return 0; + return m->UsedSize(); + } + + AsanChunkView FindHeapChunkByAddress(uptr addr) { + AsanChunk *m1 = GetAsanChunkByAddr(addr); + if (!m1) return AsanChunkView(m1); + sptr offset = 0; + if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) { + // The address is in the chunk's left redzone, so maybe it is actually + // a right buffer overflow from the other chunk to the left. + // Search a bit to the left to see if there is another chunk. + AsanChunk *m2 = 0; + for (uptr l = 1; l < GetPageSizeCached(); l++) { + m2 = GetAsanChunkByAddr(addr - l); + if (m2 == m1) continue; // Still the same chunk. + break; + } + if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, 1, &offset)) + m1 = ChooseChunk(addr, m2, m1); + } + return AsanChunkView(m1); + } + + void PrintStats() { + allocator.PrintStats(); + } + + void ForceLock() { + allocator.ForceLock(); + fallback_mutex.Lock(); + } + + void ForceUnlock() { + fallback_mutex.Unlock(); + allocator.ForceUnlock(); + } +}; + +static Allocator instance(LINKER_INITIALIZED); + +static AsanAllocator &get_allocator() { + return instance.allocator; +} + +bool AsanChunkView::IsValid() { + return chunk_ != 0 && chunk_->chunk_state != CHUNK_AVAILABLE; +} +uptr AsanChunkView::Beg() { return chunk_->Beg(); } +uptr AsanChunkView::End() { return Beg() + UsedSize(); } +uptr AsanChunkView::UsedSize() { return chunk_->UsedSize(); } +uptr AsanChunkView::AllocTid() { return chunk_->alloc_tid; } +uptr AsanChunkView::FreeTid() { return chunk_->free_tid; } + +static StackTrace GetStackTraceFromId(u32 id) { + CHECK(id); + StackTrace res = StackDepotGet(id); + CHECK(res.trace); + return res; +} + +StackTrace AsanChunkView::GetAllocStack() { + return GetStackTraceFromId(chunk_->alloc_context_id); +} + +StackTrace AsanChunkView::GetFreeStack() { + return GetStackTraceFromId(chunk_->free_context_id); +} + +void InitializeAllocator(const AllocatorOptions &options) { + instance.Initialize(options); +} + +void ReInitializeAllocator(const AllocatorOptions &options) { + instance.ReInitialize(options); +} + +void GetAllocatorOptions(AllocatorOptions *options) { + instance.GetOptions(options); +} + +AsanChunkView FindHeapChunkByAddress(uptr addr) { + return instance.FindHeapChunkByAddress(addr); +} + +void AsanThreadLocalMallocStorage::CommitBack() { + instance.CommitBack(this); +} + +void PrintInternalAllocatorStats() { + instance.PrintStats(); +} + +void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, + AllocType alloc_type) { + return instance.Allocate(size, alignment, stack, alloc_type, true); +} + +void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, 0, stack, alloc_type); +} + +void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, + AllocType alloc_type) { + instance.Deallocate(ptr, size, stack, alloc_type); +} + +void *asan_malloc(uptr size, BufferedStackTrace *stack) { + return instance.Allocate(size, 8, stack, FROM_MALLOC, true); +} + +void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { + return instance.Calloc(nmemb, size, stack); +} + +void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { + if (p == 0) + return instance.Allocate(size, 8, stack, FROM_MALLOC, true); + if (size == 0) { + instance.Deallocate(p, 0, stack, FROM_MALLOC); + return 0; + } + return instance.Reallocate(p, size, stack); +} + +void *asan_valloc(uptr size, BufferedStackTrace *stack) { + return instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true); +} + +void *asan_pvalloc(uptr size, BufferedStackTrace *stack) { + uptr PageSize = GetPageSizeCached(); + size = RoundUpTo(size, PageSize); + if (size == 0) { + // pvalloc(0) should allocate one page. + size = PageSize; + } + return instance.Allocate(size, PageSize, stack, FROM_MALLOC, true); +} + +int asan_posix_memalign(void **memptr, uptr alignment, uptr size, + BufferedStackTrace *stack) { + void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true); + CHECK(IsAligned((uptr)ptr, alignment)); + *memptr = ptr; + return 0; +} + +uptr asan_malloc_usable_size(void *ptr, uptr pc, uptr bp) { + if (ptr == 0) return 0; + uptr usable_size = instance.AllocationSize(reinterpret_cast(ptr)); + if (flags()->check_malloc_usable_size && (usable_size == 0)) { + GET_STACK_TRACE_FATAL(pc, bp); + ReportMallocUsableSizeNotOwned((uptr)ptr, &stack); + } + return usable_size; +} + +uptr asan_mz_size(const void *ptr) { + return instance.AllocationSize(reinterpret_cast(ptr)); +} + +void asan_mz_force_lock() { + instance.ForceLock(); +} + +void asan_mz_force_unlock() { + instance.ForceUnlock(); +} + +void AsanSoftRssLimitExceededCallback(bool exceeded) { + instance.allocator.SetRssLimitIsExceeded(exceeded); +} + +} // namespace __asan + +// --- Implementation of LSan-specific functions --- {{{1 +namespace __lsan { +void LockAllocator() { + __asan::get_allocator().ForceLock(); +} + +void UnlockAllocator() { + __asan::get_allocator().ForceUnlock(); +} + +void GetAllocatorGlobalRange(uptr *begin, uptr *end) { + *begin = (uptr)&__asan::get_allocator(); + *end = *begin + sizeof(__asan::get_allocator()); +} + +uptr PointsIntoChunk(void* p) { + uptr addr = reinterpret_cast(p); + __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(addr); + if (!m) return 0; + uptr chunk = m->Beg(); + if (m->chunk_state != __asan::CHUNK_ALLOCATED) + return 0; + if (m->AddrIsInside(addr, /*locked_version=*/true)) + return chunk; + if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(/*locked_version*/ true), + addr)) + return chunk; + return 0; +} + +uptr GetUserBegin(uptr chunk) { + __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk); + CHECK(m); + return m->Beg(); +} + +LsanMetadata::LsanMetadata(uptr chunk) { + metadata_ = reinterpret_cast(chunk - __asan::kChunkHeaderSize); +} + +bool LsanMetadata::allocated() const { + __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); + return m->chunk_state == __asan::CHUNK_ALLOCATED; +} + +ChunkTag LsanMetadata::tag() const { + __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); + return static_cast(m->lsan_tag); +} + +void LsanMetadata::set_tag(ChunkTag value) { + __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); + m->lsan_tag = value; +} + +uptr LsanMetadata::requested_size() const { + __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); + return m->UsedSize(/*locked_version=*/true); +} + +u32 LsanMetadata::stack_trace_id() const { + __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); + return m->alloc_context_id; +} + +void ForEachChunk(ForEachChunkCallback callback, void *arg) { + __asan::get_allocator().ForEachChunk(callback, arg); +} + +IgnoreObjectResult IgnoreObjectLocked(const void *p) { + uptr addr = reinterpret_cast(p); + __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr); + if (!m) return kIgnoreObjectInvalid; + if ((m->chunk_state == __asan::CHUNK_ALLOCATED) && m->AddrIsInside(addr)) { + if (m->lsan_tag == kIgnored) + return kIgnoreObjectAlreadyIgnored; + m->lsan_tag = __lsan::kIgnored; + return kIgnoreObjectSuccess; + } else { + return kIgnoreObjectInvalid; + } +} +} // namespace __lsan + +// ---------------------- Interface ---------------- {{{1 +using namespace __asan; // NOLINT + +// ASan allocator doesn't reserve extra bytes, so normally we would +// just return "size". We don't want to expose our redzone sizes, etc here. +uptr __sanitizer_get_estimated_allocated_size(uptr size) { + return size; +} + +int __sanitizer_get_ownership(const void *p) { + uptr ptr = reinterpret_cast(p); + return instance.AllocationSize(ptr) > 0; +} + +uptr __sanitizer_get_allocated_size(const void *p) { + if (p == 0) return 0; + uptr ptr = reinterpret_cast(p); + uptr allocated_size = instance.AllocationSize(ptr); + // Die if p is not malloced or if it is already freed. + if (allocated_size == 0) { + GET_STACK_TRACE_FATAL_HERE; + ReportSanitizerGetAllocatedSizeNotOwned(ptr, &stack); + } + return allocated_size; +} + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +// Provide default (no-op) implementation of malloc hooks. +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_malloc_hook(void *ptr, uptr size) { + (void)ptr; + (void)size; +} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_free_hook(void *ptr) { + (void)ptr; +} +} // extern "C" +#endif diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.h b/contrib/compiler-rt/lib/asan/asan_allocator.h index 6d3a99282a4a..3208d1f950cd 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.h +++ b/contrib/compiler-rt/lib/asan/asan_allocator.h @@ -9,12 +9,13 @@ // // This file is a part of AddressSanitizer, an address sanity checker. // -// ASan-private header for asan_allocator2.cc. +// ASan-private header for asan_allocator.cc. //===----------------------------------------------------------------------===// #ifndef ASAN_ALLOCATOR_H #define ASAN_ALLOCATOR_H +#include "asan_flags.h" #include "asan_internal.h" #include "asan_interceptors.h" #include "sanitizer_common/sanitizer_allocator.h" @@ -31,8 +32,20 @@ enum AllocType { static const uptr kNumberOfSizeClasses = 255; struct AsanChunk; -void InitializeAllocator(); -void ReInitializeAllocator(); +struct AllocatorOptions { + u32 quarantine_size_mb; + u16 min_redzone; + u16 max_redzone; + u8 may_return_null; + u8 alloc_dealloc_mismatch; + + void SetFrom(const Flags *f, const CommonFlags *cf); + void CopyTo(Flags *f, CommonFlags *cf); +}; + +void InitializeAllocator(const AllocatorOptions &options); +void ReInitializeAllocator(const AllocatorOptions &options); +void GetAllocatorOptions(AllocatorOptions *options); class AsanChunkView { public: @@ -127,12 +140,12 @@ typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, 16, typedef SizeClassAllocatorLocalCache AllocatorCache; typedef LargeMmapAllocator SecondaryAllocator; typedef CombinedAllocator Allocator; + SecondaryAllocator> AsanAllocator; struct AsanThreadLocalMallocStorage { uptr quarantine_cache[16]; - AllocatorCache allocator2_cache; + AllocatorCache allocator_cache; void CommitBack(); private: // These objects are allocated via mmap() and are zero-initialized. @@ -160,6 +173,7 @@ void asan_mz_force_lock(); void asan_mz_force_unlock(); void PrintInternalAllocatorStats(); +void AsanSoftRssLimitExceededCallback(bool exceeded); } // namespace __asan #endif // ASAN_ALLOCATOR_H diff --git a/contrib/compiler-rt/lib/asan/asan_allocator2.cc b/contrib/compiler-rt/lib/asan/asan_allocator2.cc deleted file mode 100644 index 52bdcf607f57..000000000000 --- a/contrib/compiler-rt/lib/asan/asan_allocator2.cc +++ /dev/null @@ -1,792 +0,0 @@ -//===-- asan_allocator2.cc ------------------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of AddressSanitizer, an address sanity checker. -// -// Implementation of ASan's memory allocator, 2-nd version. -// This variant uses the allocator from sanitizer_common, i.e. the one shared -// with ThreadSanitizer and MemorySanitizer. -// -//===----------------------------------------------------------------------===// -#include "asan_allocator.h" - -#include "asan_mapping.h" -#include "asan_poisoning.h" -#include "asan_report.h" -#include "asan_stack.h" -#include "asan_thread.h" -#include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_flags.h" -#include "sanitizer_common/sanitizer_internal_defs.h" -#include "sanitizer_common/sanitizer_list.h" -#include "sanitizer_common/sanitizer_stackdepot.h" -#include "sanitizer_common/sanitizer_quarantine.h" -#include "lsan/lsan_common.h" - -namespace __asan { - -void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const { - PoisonShadow(p, size, kAsanHeapLeftRedzoneMagic); - // Statistics. - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.mmaps++; - thread_stats.mmaped += size; -} -void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const { - PoisonShadow(p, size, 0); - // We are about to unmap a chunk of user memory. - // Mark the corresponding shadow memory as not needed. - FlushUnneededASanShadowMemory(p, size); - // Statistics. - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.munmaps++; - thread_stats.munmaped += size; -} - -// We can not use THREADLOCAL because it is not supported on some of the -// platforms we care about (OSX 10.6, Android). -// static THREADLOCAL AllocatorCache cache; -AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) { - CHECK(ms); - return &ms->allocator2_cache; -} - -static Allocator allocator; - -static const uptr kMaxAllowedMallocSize = - FIRST_32_SECOND_64(3UL << 30, 64UL << 30); - -static const uptr kMaxThreadLocalQuarantine = - FIRST_32_SECOND_64(1 << 18, 1 << 20); - -// Every chunk of memory allocated by this allocator can be in one of 3 states: -// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated. -// CHUNK_ALLOCATED: the chunk is allocated and not yet freed. -// CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone. -enum { - CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it. - CHUNK_ALLOCATED = 2, - CHUNK_QUARANTINE = 3 -}; - -// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. -// We use adaptive redzones: for larger allocation larger redzones are used. -static u32 RZLog2Size(u32 rz_log) { - CHECK_LT(rz_log, 8); - return 16 << rz_log; -} - -static u32 RZSize2Log(u32 rz_size) { - CHECK_GE(rz_size, 16); - CHECK_LE(rz_size, 2048); - CHECK(IsPowerOfTwo(rz_size)); - u32 res = Log2(rz_size) - 4; - CHECK_EQ(rz_size, RZLog2Size(res)); - return res; -} - -static uptr ComputeRZLog(uptr user_requested_size) { - u32 rz_log = - user_requested_size <= 64 - 16 ? 0 : - user_requested_size <= 128 - 32 ? 1 : - user_requested_size <= 512 - 64 ? 2 : - user_requested_size <= 4096 - 128 ? 3 : - user_requested_size <= (1 << 14) - 256 ? 4 : - user_requested_size <= (1 << 15) - 512 ? 5 : - user_requested_size <= (1 << 16) - 1024 ? 6 : 7; - return Min(Max(rz_log, RZSize2Log(flags()->redzone)), - RZSize2Log(flags()->max_redzone)); -} - -// The memory chunk allocated from the underlying allocator looks like this: -// L L L L L L H H U U U U U U R R -// L -- left redzone words (0 or more bytes) -// H -- ChunkHeader (16 bytes), which is also a part of the left redzone. -// U -- user memory. -// R -- right redzone (0 or more bytes) -// ChunkBase consists of ChunkHeader and other bytes that overlap with user -// memory. - -// If the left redzone is greater than the ChunkHeader size we store a magic -// value in the first uptr word of the memory block and store the address of -// ChunkBase in the next uptr. -// M B L L L L L L L L L H H U U U U U U -// | ^ -// ---------------------| -// M -- magic value kAllocBegMagic -// B -- address of ChunkHeader pointing to the first 'H' -static const uptr kAllocBegMagic = 0xCC6E96B9; - -struct ChunkHeader { - // 1-st 8 bytes. - u32 chunk_state : 8; // Must be first. - u32 alloc_tid : 24; - - u32 free_tid : 24; - u32 from_memalign : 1; - u32 alloc_type : 2; - u32 rz_log : 3; - u32 lsan_tag : 2; - // 2-nd 8 bytes - // This field is used for small sizes. For large sizes it is equal to - // SizeClassMap::kMaxSize and the actual size is stored in the - // SecondaryAllocator's metadata. - u32 user_requested_size; - u32 alloc_context_id; -}; - -struct ChunkBase : ChunkHeader { - // Header2, intersects with user memory. - u32 free_context_id; -}; - -static const uptr kChunkHeaderSize = sizeof(ChunkHeader); -static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize; -COMPILER_CHECK(kChunkHeaderSize == 16); -COMPILER_CHECK(kChunkHeader2Size <= 16); - -struct AsanChunk: ChunkBase { - uptr Beg() { return reinterpret_cast(this) + kChunkHeaderSize; } - uptr UsedSize(bool locked_version = false) { - if (user_requested_size != SizeClassMap::kMaxSize) - return user_requested_size; - return *reinterpret_cast( - allocator.GetMetaData(AllocBeg(locked_version))); - } - void *AllocBeg(bool locked_version = false) { - if (from_memalign) { - if (locked_version) - return allocator.GetBlockBeginFastLocked( - reinterpret_cast(this)); - return allocator.GetBlockBegin(reinterpret_cast(this)); - } - return reinterpret_cast(Beg() - RZLog2Size(rz_log)); - } - bool AddrIsInside(uptr addr, bool locked_version = false) { - return (addr >= Beg()) && (addr < Beg() + UsedSize(locked_version)); - } -}; - -bool AsanChunkView::IsValid() { - return chunk_ != 0 && chunk_->chunk_state != CHUNK_AVAILABLE; -} -uptr AsanChunkView::Beg() { return chunk_->Beg(); } -uptr AsanChunkView::End() { return Beg() + UsedSize(); } -uptr AsanChunkView::UsedSize() { return chunk_->UsedSize(); } -uptr AsanChunkView::AllocTid() { return chunk_->alloc_tid; } -uptr AsanChunkView::FreeTid() { return chunk_->free_tid; } - -static StackTrace GetStackTraceFromId(u32 id) { - CHECK(id); - StackTrace res = StackDepotGet(id); - CHECK(res.trace); - return res; -} - -StackTrace AsanChunkView::GetAllocStack() { - return GetStackTraceFromId(chunk_->alloc_context_id); -} - -StackTrace AsanChunkView::GetFreeStack() { - return GetStackTraceFromId(chunk_->free_context_id); -} - -struct QuarantineCallback; -typedef Quarantine AsanQuarantine; -typedef AsanQuarantine::Cache QuarantineCache; -static AsanQuarantine quarantine(LINKER_INITIALIZED); -static QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED); -static AllocatorCache fallback_allocator_cache; -static SpinMutex fallback_mutex; - -QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) { - CHECK(ms); - CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache)); - return reinterpret_cast(ms->quarantine_cache); -} - -struct QuarantineCallback { - explicit QuarantineCallback(AllocatorCache *cache) - : cache_(cache) { - } - - void Recycle(AsanChunk *m) { - CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); - atomic_store((atomic_uint8_t*)m, CHUNK_AVAILABLE, memory_order_relaxed); - CHECK_NE(m->alloc_tid, kInvalidTid); - CHECK_NE(m->free_tid, kInvalidTid); - PoisonShadow(m->Beg(), - RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), - kAsanHeapLeftRedzoneMagic); - void *p = reinterpret_cast(m->AllocBeg()); - if (p != m) { - uptr *alloc_magic = reinterpret_cast(p); - CHECK_EQ(alloc_magic[0], kAllocBegMagic); - // Clear the magic value, as allocator internals may overwrite the - // contents of deallocated chunk, confusing GetAsanChunk lookup. - alloc_magic[0] = 0; - CHECK_EQ(alloc_magic[1], reinterpret_cast(m)); - } - - // Statistics. - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.real_frees++; - thread_stats.really_freed += m->UsedSize(); - - allocator.Deallocate(cache_, p); - } - - void *Allocate(uptr size) { - return allocator.Allocate(cache_, size, 1, false); - } - - void Deallocate(void *p) { - allocator.Deallocate(cache_, p); - } - - AllocatorCache *cache_; -}; - -void InitializeAllocator() { - allocator.Init(); - quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine); -} - -void ReInitializeAllocator() { - quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine); -} - -static void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, - AllocType alloc_type, bool can_fill) { - if (UNLIKELY(!asan_inited)) - AsanInitFromRtl(); - Flags &fl = *flags(); - CHECK(stack); - const uptr min_alignment = SHADOW_GRANULARITY; - if (alignment < min_alignment) - alignment = min_alignment; - if (size == 0) { - // We'd be happy to avoid allocating memory for zero-size requests, but - // some programs/tests depend on this behavior and assume that malloc would - // not return NULL even for zero-size allocations. Moreover, it looks like - // operator new should never return NULL, and results of consecutive "new" - // calls must be different even if the allocated size is zero. - size = 1; - } - CHECK(IsPowerOfTwo(alignment)); - uptr rz_log = ComputeRZLog(size); - uptr rz_size = RZLog2Size(rz_log); - uptr rounded_size = RoundUpTo(Max(size, kChunkHeader2Size), alignment); - uptr needed_size = rounded_size + rz_size; - if (alignment > min_alignment) - needed_size += alignment; - bool using_primary_allocator = true; - // If we are allocating from the secondary allocator, there will be no - // automatic right redzone, so add the right redzone manually. - if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) { - needed_size += rz_size; - using_primary_allocator = false; - } - CHECK(IsAligned(needed_size, min_alignment)); - if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) { - Report("WARNING: AddressSanitizer failed to allocate %p bytes\n", - (void*)size); - return AllocatorReturnNull(); - } - - AsanThread *t = GetCurrentThread(); - void *allocated; - if (t) { - AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); - allocated = allocator.Allocate(cache, needed_size, 8, false); - } else { - SpinMutexLock l(&fallback_mutex); - AllocatorCache *cache = &fallback_allocator_cache; - allocated = allocator.Allocate(cache, needed_size, 8, false); - } - - if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && flags()->poison_heap) { - // Heap poisoning is enabled, but the allocator provides an unpoisoned - // chunk. This is possible if flags()->poison_heap was disabled for some - // time, for example, due to flags()->start_disabled. - // Anyway, poison the block before using it for anything else. - uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated); - PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic); - } - - uptr alloc_beg = reinterpret_cast(allocated); - uptr alloc_end = alloc_beg + needed_size; - uptr beg_plus_redzone = alloc_beg + rz_size; - uptr user_beg = beg_plus_redzone; - if (!IsAligned(user_beg, alignment)) - user_beg = RoundUpTo(user_beg, alignment); - uptr user_end = user_beg + size; - CHECK_LE(user_end, alloc_end); - uptr chunk_beg = user_beg - kChunkHeaderSize; - AsanChunk *m = reinterpret_cast(chunk_beg); - m->alloc_type = alloc_type; - m->rz_log = rz_log; - u32 alloc_tid = t ? t->tid() : 0; - m->alloc_tid = alloc_tid; - CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield? - m->free_tid = kInvalidTid; - m->from_memalign = user_beg != beg_plus_redzone; - if (alloc_beg != chunk_beg) { - CHECK_LE(alloc_beg+ 2 * sizeof(uptr), chunk_beg); - reinterpret_cast(alloc_beg)[0] = kAllocBegMagic; - reinterpret_cast(alloc_beg)[1] = chunk_beg; - } - if (using_primary_allocator) { - CHECK(size); - m->user_requested_size = size; - CHECK(allocator.FromPrimary(allocated)); - } else { - CHECK(!allocator.FromPrimary(allocated)); - m->user_requested_size = SizeClassMap::kMaxSize; - uptr *meta = reinterpret_cast(allocator.GetMetaData(allocated)); - meta[0] = size; - meta[1] = chunk_beg; - } - - m->alloc_context_id = StackDepotPut(*stack); - - uptr size_rounded_down_to_granularity = RoundDownTo(size, SHADOW_GRANULARITY); - // Unpoison the bulk of the memory region. - if (size_rounded_down_to_granularity) - PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); - // Deal with the end of the region if size is not aligned to granularity. - if (size != size_rounded_down_to_granularity && fl.poison_heap) { - u8 *shadow = (u8*)MemToShadow(user_beg + size_rounded_down_to_granularity); - *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY - 1)) : 0; - } - - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.mallocs++; - thread_stats.malloced += size; - thread_stats.malloced_redzones += needed_size - size; - uptr class_id = Min(kNumberOfSizeClasses, SizeClassMap::ClassID(needed_size)); - thread_stats.malloced_by_size[class_id]++; - if (needed_size > SizeClassMap::kMaxSize) - thread_stats.malloc_large++; - - void *res = reinterpret_cast(user_beg); - if (can_fill && fl.max_malloc_fill_size) { - uptr fill_size = Min(size, (uptr)fl.max_malloc_fill_size); - REAL(memset)(res, fl.malloc_fill_byte, fill_size); - } -#if CAN_SANITIZE_LEAKS - m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored - : __lsan::kDirectlyLeaked; -#endif - // Must be the last mutation of metadata in this function. - atomic_store((atomic_uint8_t *)m, CHUNK_ALLOCATED, memory_order_release); - ASAN_MALLOC_HOOK(res, size); - return res; -} - -static void ReportInvalidFree(void *ptr, u8 chunk_state, - BufferedStackTrace *stack) { - if (chunk_state == CHUNK_QUARANTINE) - ReportDoubleFree((uptr)ptr, stack); - else - ReportFreeNotMalloced((uptr)ptr, stack); -} - -static void AtomicallySetQuarantineFlag(AsanChunk *m, void *ptr, - BufferedStackTrace *stack) { - u8 old_chunk_state = CHUNK_ALLOCATED; - // Flip the chunk_state atomically to avoid race on double-free. - if (!atomic_compare_exchange_strong((atomic_uint8_t*)m, &old_chunk_state, - CHUNK_QUARANTINE, memory_order_acquire)) - ReportInvalidFree(ptr, old_chunk_state, stack); - CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state); -} - -// Expects the chunk to already be marked as quarantined by using -// AtomicallySetQuarantineFlag. -static void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack, - AllocType alloc_type) { - CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); - - if (m->alloc_type != alloc_type && flags()->alloc_dealloc_mismatch) - ReportAllocTypeMismatch((uptr)ptr, stack, - (AllocType)m->alloc_type, (AllocType)alloc_type); - - CHECK_GE(m->alloc_tid, 0); - if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area. - CHECK_EQ(m->free_tid, kInvalidTid); - AsanThread *t = GetCurrentThread(); - m->free_tid = t ? t->tid() : 0; - m->free_context_id = StackDepotPut(*stack); - // Poison the region. - PoisonShadow(m->Beg(), - RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), - kAsanHeapFreeMagic); - - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.frees++; - thread_stats.freed += m->UsedSize(); - - // Push into quarantine. - if (t) { - AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); - AllocatorCache *ac = GetAllocatorCache(ms); - quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), - m, m->UsedSize()); - } else { - SpinMutexLock l(&fallback_mutex); - AllocatorCache *ac = &fallback_allocator_cache; - quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), - m, m->UsedSize()); - } -} - -static void Deallocate(void *ptr, uptr delete_size, BufferedStackTrace *stack, - AllocType alloc_type) { - uptr p = reinterpret_cast(ptr); - if (p == 0) return; - - uptr chunk_beg = p - kChunkHeaderSize; - AsanChunk *m = reinterpret_cast(chunk_beg); - if (delete_size && flags()->new_delete_type_mismatch && - delete_size != m->UsedSize()) { - ReportNewDeleteSizeMismatch(p, delete_size, stack); - } - ASAN_FREE_HOOK(ptr); - // Must mark the chunk as quarantined before any changes to its metadata. - AtomicallySetQuarantineFlag(m, ptr, stack); - QuarantineChunk(m, ptr, stack, alloc_type); -} - -static void *Reallocate(void *old_ptr, uptr new_size, - BufferedStackTrace *stack) { - CHECK(old_ptr && new_size); - uptr p = reinterpret_cast(old_ptr); - uptr chunk_beg = p - kChunkHeaderSize; - AsanChunk *m = reinterpret_cast(chunk_beg); - - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.reallocs++; - thread_stats.realloced += new_size; - - void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true); - if (new_ptr) { - u8 chunk_state = m->chunk_state; - if (chunk_state != CHUNK_ALLOCATED) - ReportInvalidFree(old_ptr, chunk_state, stack); - CHECK_NE(REAL(memcpy), (void*)0); - uptr memcpy_size = Min(new_size, m->UsedSize()); - // If realloc() races with free(), we may start copying freed memory. - // However, we will report racy double-free later anyway. - REAL(memcpy)(new_ptr, old_ptr, memcpy_size); - Deallocate(old_ptr, 0, stack, FROM_MALLOC); - } - return new_ptr; -} - -// Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). -static AsanChunk *GetAsanChunk(void *alloc_beg) { - if (!alloc_beg) return 0; - if (!allocator.FromPrimary(alloc_beg)) { - uptr *meta = reinterpret_cast(allocator.GetMetaData(alloc_beg)); - AsanChunk *m = reinterpret_cast(meta[1]); - return m; - } - uptr *alloc_magic = reinterpret_cast(alloc_beg); - if (alloc_magic[0] == kAllocBegMagic) - return reinterpret_cast(alloc_magic[1]); - return reinterpret_cast(alloc_beg); -} - -static AsanChunk *GetAsanChunkByAddr(uptr p) { - void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast(p)); - return GetAsanChunk(alloc_beg); -} - -// Allocator must be locked when this function is called. -static AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) { - void *alloc_beg = - allocator.GetBlockBeginFastLocked(reinterpret_cast(p)); - return GetAsanChunk(alloc_beg); -} - -static uptr AllocationSize(uptr p) { - AsanChunk *m = GetAsanChunkByAddr(p); - if (!m) return 0; - if (m->chunk_state != CHUNK_ALLOCATED) return 0; - if (m->Beg() != p) return 0; - return m->UsedSize(); -} - -// We have an address between two chunks, and we want to report just one. -AsanChunk *ChooseChunk(uptr addr, - AsanChunk *left_chunk, AsanChunk *right_chunk) { - // Prefer an allocated chunk over freed chunk and freed chunk - // over available chunk. - if (left_chunk->chunk_state != right_chunk->chunk_state) { - if (left_chunk->chunk_state == CHUNK_ALLOCATED) - return left_chunk; - if (right_chunk->chunk_state == CHUNK_ALLOCATED) - return right_chunk; - if (left_chunk->chunk_state == CHUNK_QUARANTINE) - return left_chunk; - if (right_chunk->chunk_state == CHUNK_QUARANTINE) - return right_chunk; - } - // Same chunk_state: choose based on offset. - sptr l_offset = 0, r_offset = 0; - CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset)); - CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset)); - if (l_offset < r_offset) - return left_chunk; - return right_chunk; -} - -AsanChunkView FindHeapChunkByAddress(uptr addr) { - AsanChunk *m1 = GetAsanChunkByAddr(addr); - if (!m1) return AsanChunkView(m1); - sptr offset = 0; - if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) { - // The address is in the chunk's left redzone, so maybe it is actually - // a right buffer overflow from the other chunk to the left. - // Search a bit to the left to see if there is another chunk. - AsanChunk *m2 = 0; - for (uptr l = 1; l < GetPageSizeCached(); l++) { - m2 = GetAsanChunkByAddr(addr - l); - if (m2 == m1) continue; // Still the same chunk. - break; - } - if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, 1, &offset)) - m1 = ChooseChunk(addr, m2, m1); - } - return AsanChunkView(m1); -} - -void AsanThreadLocalMallocStorage::CommitBack() { - AllocatorCache *ac = GetAllocatorCache(this); - quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac)); - allocator.SwallowCache(GetAllocatorCache(this)); -} - -void PrintInternalAllocatorStats() { - allocator.PrintStats(); -} - -void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, - AllocType alloc_type) { - return Allocate(size, alignment, stack, alloc_type, true); -} - -void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { - Deallocate(ptr, 0, stack, alloc_type); -} - -void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, - AllocType alloc_type) { - Deallocate(ptr, size, stack, alloc_type); -} - -void *asan_malloc(uptr size, BufferedStackTrace *stack) { - return Allocate(size, 8, stack, FROM_MALLOC, true); -} - -void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { - if (CallocShouldReturnNullDueToOverflow(size, nmemb)) - return AllocatorReturnNull(); - void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false); - // If the memory comes from the secondary allocator no need to clear it - // as it comes directly from mmap. - if (ptr && allocator.FromPrimary(ptr)) - REAL(memset)(ptr, 0, nmemb * size); - return ptr; -} - -void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { - if (p == 0) - return Allocate(size, 8, stack, FROM_MALLOC, true); - if (size == 0) { - Deallocate(p, 0, stack, FROM_MALLOC); - return 0; - } - return Reallocate(p, size, stack); -} - -void *asan_valloc(uptr size, BufferedStackTrace *stack) { - return Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true); -} - -void *asan_pvalloc(uptr size, BufferedStackTrace *stack) { - uptr PageSize = GetPageSizeCached(); - size = RoundUpTo(size, PageSize); - if (size == 0) { - // pvalloc(0) should allocate one page. - size = PageSize; - } - return Allocate(size, PageSize, stack, FROM_MALLOC, true); -} - -int asan_posix_memalign(void **memptr, uptr alignment, uptr size, - BufferedStackTrace *stack) { - void *ptr = Allocate(size, alignment, stack, FROM_MALLOC, true); - CHECK(IsAligned((uptr)ptr, alignment)); - *memptr = ptr; - return 0; -} - -uptr asan_malloc_usable_size(void *ptr, uptr pc, uptr bp) { - if (ptr == 0) return 0; - uptr usable_size = AllocationSize(reinterpret_cast(ptr)); - if (flags()->check_malloc_usable_size && (usable_size == 0)) { - GET_STACK_TRACE_FATAL(pc, bp); - ReportMallocUsableSizeNotOwned((uptr)ptr, &stack); - } - return usable_size; -} - -uptr asan_mz_size(const void *ptr) { - return AllocationSize(reinterpret_cast(ptr)); -} - -void asan_mz_force_lock() { - allocator.ForceLock(); - fallback_mutex.Lock(); -} - -void asan_mz_force_unlock() { - fallback_mutex.Unlock(); - allocator.ForceUnlock(); -} - -} // namespace __asan - -// --- Implementation of LSan-specific functions --- {{{1 -namespace __lsan { -void LockAllocator() { - __asan::allocator.ForceLock(); -} - -void UnlockAllocator() { - __asan::allocator.ForceUnlock(); -} - -void GetAllocatorGlobalRange(uptr *begin, uptr *end) { - *begin = (uptr)&__asan::allocator; - *end = *begin + sizeof(__asan::allocator); -} - -uptr PointsIntoChunk(void* p) { - uptr addr = reinterpret_cast(p); - __asan::AsanChunk *m = __asan::GetAsanChunkByAddrFastLocked(addr); - if (!m) return 0; - uptr chunk = m->Beg(); - if (m->chunk_state != __asan::CHUNK_ALLOCATED) - return 0; - if (m->AddrIsInside(addr, /*locked_version=*/true)) - return chunk; - if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(/*locked_version*/ true), - addr)) - return chunk; - return 0; -} - -uptr GetUserBegin(uptr chunk) { - __asan::AsanChunk *m = - __asan::GetAsanChunkByAddrFastLocked(chunk); - CHECK(m); - return m->Beg(); -} - -LsanMetadata::LsanMetadata(uptr chunk) { - metadata_ = reinterpret_cast(chunk - __asan::kChunkHeaderSize); -} - -bool LsanMetadata::allocated() const { - __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); - return m->chunk_state == __asan::CHUNK_ALLOCATED; -} - -ChunkTag LsanMetadata::tag() const { - __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); - return static_cast(m->lsan_tag); -} - -void LsanMetadata::set_tag(ChunkTag value) { - __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); - m->lsan_tag = value; -} - -uptr LsanMetadata::requested_size() const { - __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); - return m->UsedSize(/*locked_version=*/true); -} - -u32 LsanMetadata::stack_trace_id() const { - __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); - return m->alloc_context_id; -} - -void ForEachChunk(ForEachChunkCallback callback, void *arg) { - __asan::allocator.ForEachChunk(callback, arg); -} - -IgnoreObjectResult IgnoreObjectLocked(const void *p) { - uptr addr = reinterpret_cast(p); - __asan::AsanChunk *m = __asan::GetAsanChunkByAddr(addr); - if (!m) return kIgnoreObjectInvalid; - if ((m->chunk_state == __asan::CHUNK_ALLOCATED) && m->AddrIsInside(addr)) { - if (m->lsan_tag == kIgnored) - return kIgnoreObjectAlreadyIgnored; - m->lsan_tag = __lsan::kIgnored; - return kIgnoreObjectSuccess; - } else { - return kIgnoreObjectInvalid; - } -} -} // namespace __lsan - -// ---------------------- Interface ---------------- {{{1 -using namespace __asan; // NOLINT - -// ASan allocator doesn't reserve extra bytes, so normally we would -// just return "size". We don't want to expose our redzone sizes, etc here. -uptr __sanitizer_get_estimated_allocated_size(uptr size) { - return size; -} - -int __sanitizer_get_ownership(const void *p) { - uptr ptr = reinterpret_cast(p); - return (AllocationSize(ptr) > 0); -} - -uptr __sanitizer_get_allocated_size(const void *p) { - if (p == 0) return 0; - uptr ptr = reinterpret_cast(p); - uptr allocated_size = AllocationSize(ptr); - // Die if p is not malloced or if it is already freed. - if (allocated_size == 0) { - GET_STACK_TRACE_FATAL_HERE; - ReportSanitizerGetAllocatedSizeNotOwned(ptr, &stack); - } - return allocated_size; -} - -#if !SANITIZER_SUPPORTS_WEAK_HOOKS -// Provide default (no-op) implementation of malloc hooks. -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void __sanitizer_malloc_hook(void *ptr, uptr size) { - (void)ptr; - (void)size; -} -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void __sanitizer_free_hook(void *ptr) { - (void)ptr; -} -} // extern "C" -#endif diff --git a/contrib/compiler-rt/lib/asan/asan_debugging.cc b/contrib/compiler-rt/lib/asan/asan_debugging.cc index 2b66dd5265fc..6fc5b690de99 100644 --- a/contrib/compiler-rt/lib/asan/asan_debugging.cc +++ b/contrib/compiler-rt/lib/asan/asan_debugging.cc @@ -81,8 +81,8 @@ void AsanLocateAddress(uptr addr, AddressDescription *descr) { GetInfoForHeapAddress(addr, descr); } -uptr AsanGetStack(uptr addr, uptr *trace, uptr size, u32 *thread_id, - bool alloc_stack) { +static uptr AsanGetStack(uptr addr, uptr *trace, u32 size, u32 *thread_id, + bool alloc_stack) { AsanChunkView chunk = FindHeapChunkByAddress(addr); if (!chunk.IsValid()) return 0; diff --git a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc index c95bc11f6cd3..bf4f1eb4c781 100644 --- a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc +++ b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc @@ -60,7 +60,7 @@ FakeStack *FakeStack::Create(uptr stack_size_log) { void FakeStack::Destroy(int tid) { PoisonAll(0); - if (common_flags()->verbosity >= 2) { + if (Verbosity() >= 2) { InternalScopedString str(kNumberOfSizeClasses * 50); for (uptr class_id = 0; class_id < kNumberOfSizeClasses; class_id++) str.append("%zd: %zd/%zd; ", class_id, hint_position_[class_id], @@ -192,20 +192,19 @@ static FakeStack *GetFakeStackFast() { return GetFakeStack(); } -ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size, uptr real_stack) { +ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) { FakeStack *fs = GetFakeStackFast(); - if (!fs) return real_stack; + if (!fs) return 0; + uptr local_stack; + uptr real_stack = reinterpret_cast(&local_stack); FakeFrame *ff = fs->Allocate(fs->stack_size_log(), class_id, real_stack); - if (!ff) - return real_stack; // Out of fake stack, return the real one. + if (!ff) return 0; // Out of fake stack. uptr ptr = reinterpret_cast(ff); SetShadow(ptr, size, class_id, 0); return ptr; } -ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size, uptr real_stack) { - if (ptr == real_stack) - return; +ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) { FakeStack::Deallocate(ptr, class_id); SetShadow(ptr, size, class_id, kMagic8); } @@ -216,12 +215,12 @@ ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size, uptr real_stack) { using namespace __asan; #define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \ - __asan_stack_malloc_##class_id(uptr size, uptr real_stack) { \ - return OnMalloc(class_id, size, real_stack); \ + __asan_stack_malloc_##class_id(uptr size) { \ + return OnMalloc(class_id, size); \ } \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \ - uptr ptr, uptr size, uptr real_stack) { \ - OnFree(ptr, class_id, size, real_stack); \ + uptr ptr, uptr size) { \ + OnFree(ptr, class_id, size); \ } DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(0) diff --git a/contrib/compiler-rt/lib/asan/asan_flags.cc b/contrib/compiler-rt/lib/asan/asan_flags.cc new file mode 100644 index 000000000000..efb7767d5d91 --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_flags.cc @@ -0,0 +1,160 @@ +//===-- asan_flags.cc -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// ASan flag parsing logic. +//===----------------------------------------------------------------------===// + +#include "asan_activation.h" +#include "asan_flags.h" +#include "asan_interface_internal.h" +#include "asan_stack.h" +#include "lsan/lsan_common.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" + +namespace __asan { + +Flags asan_flags_dont_use_directly; // use via flags(). + +static const char *MaybeCallAsanDefaultOptions() { + return (&__asan_default_options) ? __asan_default_options() : ""; +} + +static const char *MaybeUseAsanDefaultOptionsCompileDefinition() { +#ifdef ASAN_DEFAULT_OPTIONS +// Stringize the macro value. +# define ASAN_STRINGIZE(x) #x +# define ASAN_STRINGIZE_OPTIONS(options) ASAN_STRINGIZE(options) + return ASAN_STRINGIZE_OPTIONS(ASAN_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +void Flags::SetDefaults() { +#define ASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "asan_flags.inc" +#undef ASAN_FLAG +} + +static void RegisterAsanFlags(FlagParser *parser, Flags *f) { +#define ASAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "asan_flags.inc" +#undef ASAN_FLAG +} + +void InitializeFlags() { + // Set the default values and prepare for parsing ASan and common flags. + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.detect_leaks = CAN_SANITIZE_LEAKS; + cf.external_symbolizer_path = GetEnv("ASAN_SYMBOLIZER_PATH"); + cf.malloc_context_size = kDefaultMallocContextSize; + cf.intercept_tls_get_addr = true; + OverrideCommonFlags(cf); + } + Flags *f = flags(); + f->SetDefaults(); + + FlagParser asan_parser; + RegisterAsanFlags(&asan_parser, f); + RegisterCommonFlags(&asan_parser); + + // Set the default values and prepare for parsing LSan flags (which can also + // overwrite common flags). +#if CAN_SANITIZE_LEAKS + __lsan::Flags *lf = __lsan::flags(); + lf->SetDefaults(); + + FlagParser lsan_parser; + __lsan::RegisterLsanFlags(&lsan_parser, lf); + RegisterCommonFlags(&lsan_parser); +#endif + + // Override from ASan compile definition. + const char *asan_compile_def = MaybeUseAsanDefaultOptionsCompileDefinition(); + asan_parser.ParseString(asan_compile_def); + + // Override from user-specified string. + const char *asan_default_options = MaybeCallAsanDefaultOptions(); + asan_parser.ParseString(asan_default_options); + + // Override from command line. + asan_parser.ParseString(GetEnv("ASAN_OPTIONS")); +#if CAN_SANITIZE_LEAKS + lsan_parser.ParseString(GetEnv("LSAN_OPTIONS")); +#endif + + // Let activation flags override current settings. On Android they come + // from a system property. On other platforms this is no-op. + if (!flags()->start_deactivated) { + char buf[100]; + GetExtraActivationFlags(buf, sizeof(buf)); + asan_parser.ParseString(buf); + } + + SetVerbosity(common_flags()->verbosity); + + // TODO(eugenis): dump all flags at verbosity>=2? + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) { + // TODO(samsonov): print all of the flags (ASan, LSan, common). + asan_parser.PrintFlagDescriptions(); + } + + // Flag validation: + if (!CAN_SANITIZE_LEAKS && common_flags()->detect_leaks) { + Report("%s: detect_leaks is not supported on this platform.\n", + SanitizerToolName); + Die(); + } + // Make "strict_init_order" imply "check_initialization_order". + // TODO(samsonov): Use a single runtime flag for an init-order checker. + if (f->strict_init_order) { + f->check_initialization_order = true; + } + CHECK_LE((uptr)common_flags()->malloc_context_size, kStackTraceMax); + CHECK_LE(f->min_uar_stack_size_log, f->max_uar_stack_size_log); + CHECK_GE(f->redzone, 16); + CHECK_GE(f->max_redzone, f->redzone); + CHECK_LE(f->max_redzone, 2048); + CHECK(IsPowerOfTwo(f->redzone)); + CHECK(IsPowerOfTwo(f->max_redzone)); + + // quarantine_size is deprecated but we still honor it. + // quarantine_size can not be used together with quarantine_size_mb. + if (f->quarantine_size >= 0 && f->quarantine_size_mb >= 0) { + Report("%s: please use either 'quarantine_size' (deprecated) or " + "quarantine_size_mb, but not both\n", SanitizerToolName); + Die(); + } + if (f->quarantine_size >= 0) + f->quarantine_size_mb = f->quarantine_size >> 20; + if (f->quarantine_size_mb < 0) { + const int kDefaultQuarantineSizeMb = + (ASAN_LOW_MEMORY) ? 1UL << 6 : 1UL << 8; + f->quarantine_size_mb = kDefaultQuarantineSizeMb; + } +} + +} // namespace __asan + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char* __asan_default_options() { return ""; } +} // extern "C" +#endif diff --git a/contrib/compiler-rt/lib/asan/asan_flags.h b/contrib/compiler-rt/lib/asan/asan_flags.h index 3df4dd3050bc..4935161c30f1 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.h +++ b/contrib/compiler-rt/lib/asan/asan_flags.h @@ -16,6 +16,7 @@ #define ASAN_FLAGS_H #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_flag_parser.h" // ASan flag values can be defined in four ways: // 1) initialized with default values at startup. @@ -24,55 +25,24 @@ // 3) overriden from string returned by user-specified function // __asan_default_options(). // 4) overriden from env variable ASAN_OPTIONS. +// 5) overriden during ASan activation (for now used on Android only). namespace __asan { struct Flags { - // Flag descriptions are in asan_rtl.cc. - int quarantine_size; - int redzone; - int max_redzone; - bool debug; - int report_globals; - bool check_initialization_order; - bool replace_str; - bool replace_intrin; - bool mac_ignore_invalid_free; - bool detect_stack_use_after_return; - int min_uar_stack_size_log; - int max_uar_stack_size_log; - bool uar_noreserve; - int max_malloc_fill_size, malloc_fill_byte; - int exitcode; - bool allow_user_poisoning; - int sleep_before_dying; - bool check_malloc_usable_size; - bool unmap_shadow_on_exit; - bool abort_on_error; - bool print_stats; - bool print_legend; - bool atexit; - bool allow_reexec; - bool print_full_thread_history; - bool poison_heap; - bool poison_partial; - bool poison_array_cookie; - bool alloc_dealloc_mismatch; - bool new_delete_type_mismatch; - bool strict_memcmp; - bool strict_init_order; - bool start_deactivated; - int detect_invalid_pointer_pairs; - bool detect_container_overflow; - int detect_odr_violation; - bool dump_instruction_bytes; +#define ASAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "asan_flags.inc" +#undef ASAN_FLAG + + void SetDefaults(); }; extern Flags asan_flags_dont_use_directly; inline Flags *flags() { return &asan_flags_dont_use_directly; } -void InitializeFlags(Flags *f, const char *env); + +void InitializeFlags(); } // namespace __asan diff --git a/contrib/compiler-rt/lib/asan/asan_flags.inc b/contrib/compiler-rt/lib/asan/asan_flags.inc new file mode 100644 index 000000000000..53a8a4039e7e --- /dev/null +++ b/contrib/compiler-rt/lib/asan/asan_flags.inc @@ -0,0 +1,145 @@ +//===-- asan_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// ASan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef ASAN_FLAG +# error "Define ASAN_FLAG prior to including this file!" +#endif + +// ASAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +ASAN_FLAG(int, quarantine_size, -1, + "Deprecated, please use quarantine_size_mb.") +ASAN_FLAG(int, quarantine_size_mb, -1, + "Size (in Mb) of quarantine used to detect use-after-free " + "errors. Lower value may reduce memory usage but increase the " + "chance of false negatives.") +ASAN_FLAG(int, redzone, 16, + "Minimal size (in bytes) of redzones around heap objects. " + "Requirement: redzone >= 16, is a power of two.") +ASAN_FLAG(int, max_redzone, 2048, + "Maximal size (in bytes) of redzones around heap objects.") +ASAN_FLAG( + bool, debug, false, + "If set, prints some debugging information and does additional checks.") +ASAN_FLAG( + int, report_globals, 1, + "Controls the way to handle globals (0 - don't detect buffer overflow on " + "globals, 1 - detect buffer overflow, 2 - print data about registered " + "globals).") +ASAN_FLAG(bool, check_initialization_order, false, + "If set, attempts to catch initialization order issues.") +ASAN_FLAG( + bool, replace_str, true, + "If set, uses custom wrappers and replacements for libc string functions " + "to find more errors.") +ASAN_FLAG(bool, replace_intrin, true, + "If set, uses custom wrappers for memset/memcpy/memmove intinsics.") +ASAN_FLAG(bool, mac_ignore_invalid_free, false, + "Ignore invalid free() calls to work around some bugs. Used on OS X " + "only.") +ASAN_FLAG(bool, detect_stack_use_after_return, false, + "Enables stack-use-after-return checking at run-time.") +ASAN_FLAG(int, min_uar_stack_size_log, 16, // We can't do smaller anyway. + "Minimum fake stack size log.") +ASAN_FLAG(int, max_uar_stack_size_log, + 20, // 1Mb per size class, i.e. ~11Mb per thread + "Maximum fake stack size log.") +ASAN_FLAG(bool, uar_noreserve, false, + "Use mmap with 'noreserve' flag to allocate fake stack.") +ASAN_FLAG( + int, max_malloc_fill_size, 0x1000, // By default, fill only the first 4K. + "ASan allocator flag. max_malloc_fill_size is the maximal amount of " + "bytes that will be filled with malloc_fill_byte on malloc.") +ASAN_FLAG(int, malloc_fill_byte, 0xbe, + "Value used to fill the newly allocated memory.") +ASAN_FLAG(int, exitcode, ASAN_DEFAULT_FAILURE_EXITCODE, + "Override the program exit status if the tool found an error.") +ASAN_FLAG(bool, allow_user_poisoning, true, + "If set, user may manually mark memory regions as poisoned or " + "unpoisoned.") +ASAN_FLAG( + int, sleep_before_dying, 0, + "Number of seconds to sleep between printing an error report and " + "terminating the program. Useful for debugging purposes (e.g. when one " + "needs to attach gdb).") +ASAN_FLAG(bool, check_malloc_usable_size, true, + "Allows the users to work around the bug in Nvidia drivers prior to " + "295.*.") +ASAN_FLAG(bool, unmap_shadow_on_exit, false, + "If set, explicitly unmaps the (huge) shadow at exit.") +ASAN_FLAG( + bool, abort_on_error, false, + "If set, the tool calls abort() instead of _exit() after printing the " + "error report.") +ASAN_FLAG(bool, print_stats, false, + "Print various statistics after printing an error message or if " + "atexit=1.") +ASAN_FLAG(bool, print_legend, true, "Print the legend for the shadow bytes.") +ASAN_FLAG(bool, atexit, false, + "If set, prints ASan exit stats even after program terminates " + "successfully.") +ASAN_FLAG( + bool, print_full_thread_history, true, + "If set, prints thread creation stacks for the threads involved in the " + "report and their ancestors up to the main thread.") +ASAN_FLAG( + bool, poison_heap, true, + "Poison (or not) the heap memory on [de]allocation. Zero value is useful " + "for benchmarking the allocator or instrumentator.") +ASAN_FLAG(bool, poison_partial, true, + "If true, poison partially addressable 8-byte aligned words " + "(default=true). This flag affects heap and global buffers, but not " + "stack buffers.") +ASAN_FLAG(bool, poison_array_cookie, true, + "Poison (or not) the array cookie after operator new[].") + +// Turn off alloc/dealloc mismatch checker on Mac and Windows for now. +// https://code.google.com/p/address-sanitizer/issues/detail?id=131 +// https://code.google.com/p/address-sanitizer/issues/detail?id=309 +// TODO(glider,timurrrr): Fix known issues and enable this back. +ASAN_FLAG(bool, alloc_dealloc_mismatch, + (SANITIZER_MAC == 0) && (SANITIZER_WINDOWS == 0), + "Report errors on malloc/delete, new/free, new/delete[], etc.") + +ASAN_FLAG(bool, new_delete_type_mismatch, true, + "Report errors on mismatch betwen size of new and delete.") +ASAN_FLAG(bool, strict_memcmp, true, + "If true, assume that memcmp(p1, p2, n) always reads n bytes before " + "comparing p1 and p2.") +ASAN_FLAG( + bool, strict_init_order, false, + "If true, assume that dynamic initializers can never access globals from " + "other modules, even if the latter are already initialized.") +ASAN_FLAG( + bool, start_deactivated, false, + "If true, ASan tweaks a bunch of other flags (quarantine, redzone, heap " + "poisoning) to reduce memory consumption as much as possible, and " + "restores them to original values when the first instrumented module is " + "loaded into the process. This is mainly intended to be used on " + "Android. ") +ASAN_FLAG( + int, detect_invalid_pointer_pairs, 0, + "If non-zero, try to detect operations like <, <=, >, >= and - on " + "invalid pointer pairs (e.g. when pointers belong to different objects). " + "The bigger the value the harder we try.") +ASAN_FLAG( + bool, detect_container_overflow, true, + "If true, honor the container overflow annotations. " + "See https://code.google.com/p/address-sanitizer/wiki/ContainerOverflow") +ASAN_FLAG(int, detect_odr_violation, 2, + "If >=2, detect violation of One-Definition-Rule (ODR); " + "If ==1, detect ODR-violation only if the two variables " + "have different sizes") +ASAN_FLAG(bool, dump_instruction_bytes, false, + "If true, dump 16 bytes starting at the instruction that caused SEGV") +ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.") diff --git a/contrib/compiler-rt/lib/asan/asan_globals.cc b/contrib/compiler-rt/lib/asan/asan_globals.cc index be111d4fb4cf..06140bbb360a 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals.cc @@ -164,7 +164,7 @@ static void RegisterGlobal(const Global *g) { } } } - if (flags()->poison_heap) + if (CanPoisonMemory()) PoisonRedZones(*g); ListOfGlobals *l = new(allocator_for_globals) ListOfGlobals; l->g = g; @@ -182,11 +182,13 @@ static void RegisterGlobal(const Global *g) { static void UnregisterGlobal(const Global *g) { CHECK(asan_inited); + if (flags()->report_globals >= 2) + ReportGlobal(*g, "Removed"); CHECK(flags()->report_globals); CHECK(AddrIsInMem(g->beg)); CHECK(AddrIsAlignedByGranularity(g->beg)); CHECK(AddrIsAlignedByGranularity(g->size_with_redzone)); - if (flags()->poison_heap) + if (CanPoisonMemory()) PoisonShadowForGlobal(g, 0); // We unpoison the shadow memory for the global but we do not remove it from // the list because that would require O(n^2) time with the current list @@ -208,6 +210,20 @@ void StopInitOrderChecking() { } } +#if SANITIZER_WINDOWS // Should only be called on Windows. +SANITIZER_INTERFACE_ATTRIBUTE +void UnregisterGlobalsInRange(void *beg, void *end) { + if (!flags()->report_globals) + return; + BlockingMutexLock lock(&mu_for_globals); + for (ListOfGlobals *l = list_of_all_globals; l; l = l->next) { + void *address = (void *)l->g->beg; + if (beg <= address && address < end) + UnregisterGlobal(l->g); + } +} +#endif + } // namespace __asan // ---------------------- Interface ---------------- {{{1 @@ -249,7 +265,7 @@ void __asan_unregister_globals(__asan_global *globals, uptr n) { // initializer can only touch global variables in the same TU. void __asan_before_dynamic_init(const char *module_name) { if (!flags()->check_initialization_order || - !flags()->poison_heap) + !CanPoisonMemory()) return; bool strict_init_order = flags()->strict_init_order; CHECK(dynamic_init_globals); @@ -275,7 +291,7 @@ void __asan_before_dynamic_init(const char *module_name) { // TU are poisoned. It simply unpoisons all dynamically initialized globals. void __asan_after_dynamic_init() { if (!flags()->check_initialization_order || - !flags()->poison_heap) + !CanPoisonMemory()) return; CHECK(asan_inited); BlockingMutexLock lock(&mu_for_globals); diff --git a/contrib/compiler-rt/lib/asan/asan_init_version.h b/contrib/compiler-rt/lib/asan/asan_init_version.h index 77aea81bd298..6cf57c4aa2a8 100644 --- a/contrib/compiler-rt/lib/asan/asan_init_version.h +++ b/contrib/compiler-rt/lib/asan/asan_init_version.h @@ -25,8 +25,10 @@ extern "C" { // contains the function PC as the 3-rd field (see // DescribeAddressIfStack). // v3=>v4: added '__asan_global_source_location' to __asan_global. - #define __asan_init __asan_init_v4 - #define __asan_init_name "__asan_init_v4" + // v4=>v5: changed the semantics and format of __asan_stack_malloc_ and + // __asan_stack_free_ functions. + #define __asan_init __asan_init_v5 + #define __asan_init_name "__asan_init_v5" } #endif // ASAN_INIT_VERSION_H diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.cc b/contrib/compiler-rt/lib/asan/asan_interceptors.cc index 910cd3addcb0..3dc7ec67a3e5 100644 --- a/contrib/compiler-rt/lib/asan/asan_interceptors.cc +++ b/contrib/compiler-rt/lib/asan/asan_interceptors.cc @@ -142,14 +142,17 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ ASAN_READ_RANGE(ctx, ptr, size) #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + ASAN_INTERCEPTOR_ENTER(ctx, func); \ do { \ if (asan_init_is_running) \ return REAL(func)(__VA_ARGS__); \ - ASAN_INTERCEPTOR_ENTER(ctx, func); \ if (SANITIZER_MAC && UNLIKELY(!asan_inited)) \ return REAL(func)(__VA_ARGS__); \ ENSURE_ASAN_INITED(); \ } while (false) +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + } while (false) #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ do { \ } while (false) @@ -169,8 +172,9 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) } while (false) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res) CovUpdateMapping() -#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() CovUpdateMapping() +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ + CoverageUpdateMapping() +#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() CoverageUpdateMapping() #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited) #include "sanitizer_common/sanitizer_common_interceptors.inc" @@ -196,6 +200,12 @@ struct ThreadStartParam { }; static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { +#if SANITIZER_WINDOWS + // FIXME: this is a bandaid fix for PR22025. + AsanThread *t = (AsanThread*)arg; + SetCurrentThread(t); + return t->ThreadStart(GetTid(), /* signal_thread_is_registered */ nullptr); +#else ThreadStartParam *param = reinterpret_cast(arg); AsanThread *t = nullptr; while ((t = reinterpret_cast( @@ -203,6 +213,7 @@ static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { internal_sched_yield(); SetCurrentThread(t); return t->ThreadStart(GetTid(), ¶m->is_registered); +#endif } #if ASAN_INTERCEPT_PTHREAD_CREATE @@ -236,22 +247,26 @@ INTERCEPTOR(int, pthread_create, void *thread, } return result; } + +INTERCEPTOR(int, pthread_join, void *t, void **arg) { + return real_pthread_join(t, arg); +} + +DEFINE_REAL_PTHREAD_FUNCTIONS #endif // ASAN_INTERCEPT_PTHREAD_CREATE #if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION #if SANITIZER_ANDROID INTERCEPTOR(void*, bsd_signal, int signum, void *handler) { - if (!AsanInterceptsSignal(signum) || - common_flags()->allow_user_segv_handler) { + if (!IsDeadlySignal(signum) || common_flags()->allow_user_segv_handler) { return REAL(bsd_signal)(signum, handler); } return 0; } #else INTERCEPTOR(void*, signal, int signum, void *handler) { - if (!AsanInterceptsSignal(signum) || - common_flags()->allow_user_segv_handler) { + if (!IsDeadlySignal(signum) || common_flags()->allow_user_segv_handler) { return REAL(signal)(signum, handler); } return 0; @@ -260,8 +275,7 @@ INTERCEPTOR(void*, signal, int signum, void *handler) { INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, struct sigaction *oldact) { - if (!AsanInterceptsSignal(signum) || - common_flags()->allow_user_segv_handler) { + if (!IsDeadlySignal(signum) || common_flags()->allow_user_segv_handler) { return REAL(sigaction)(signum, act, oldact); } return 0; @@ -802,23 +816,14 @@ INTERCEPTOR_WINAPI(DWORD, CreateThread, if (flags()->strict_init_order) StopInitOrderChecking(); GET_STACK_TRACE_THREAD; + // FIXME: The CreateThread interceptor is not the same as a pthread_create + // one. This is a bandaid fix for PR22025. bool detached = false; // FIXME: how can we determine it on Windows? - ThreadStartParam param; - atomic_store(¶m.t, 0, memory_order_relaxed); - atomic_store(¶m.is_registered, 0, memory_order_relaxed); - DWORD result = REAL(CreateThread)(security, stack_size, asan_thread_start, - ¶m, thr_flags, tid); - if (result) { - u32 current_tid = GetCurrentTidOrInvalid(); - AsanThread *t = + u32 current_tid = GetCurrentTidOrInvalid(); + AsanThread *t = AsanThread::Create(start_routine, arg, current_tid, &stack, detached); - atomic_store(¶m.t, reinterpret_cast(t), memory_order_release); - // The pthread_create interceptor waits here, so we do the same for - // consistency. - while (atomic_load(¶m.is_registered, memory_order_acquire) == 0) - internal_sched_yield(); - } - return result; + return REAL(CreateThread)(security, stack_size, + asan_thread_start, t, thr_flags, tid); } namespace __asan { @@ -902,6 +907,7 @@ void InitializeAsanInterceptors() { // Intercept threading-related functions #if ASAN_INTERCEPT_PTHREAD_CREATE ASAN_INTERCEPT_FUNC(pthread_create); + ASAN_INTERCEPT_FUNC(pthread_join); #endif // Intercept atexit function. diff --git a/contrib/compiler-rt/lib/asan/asan_interface_internal.h b/contrib/compiler-rt/lib/asan/asan_interface_internal.h index edaf44d7893a..ea7540f6bb56 100644 --- a/contrib/compiler-rt/lib/asan/asan_interface_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_interface_internal.h @@ -9,8 +9,11 @@ // // This file is a part of AddressSanitizer, an address sanity checker. // -// This header can be included by the instrumented program to fetch -// data (mostly allocator statistics) from ASan runtime library. +// This header declares the AddressSanitizer runtime interface functions. +// The runtime library has to define these functions so the instrumented program +// could call them. +// +// See also include/sanitizer/asan_interface.h //===----------------------------------------------------------------------===// #ifndef ASAN_INTERFACE_INTERNAL_H #define ASAN_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h index 65d4a47d3d9e..ffd3ff82d71e 100644 --- a/contrib/compiler-rt/lib/asan/asan_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_internal.h @@ -93,8 +93,8 @@ void AsanCheckIncompatibleRT(); void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp); void AsanOnSIGSEGV(int, void *siginfo, void *context); +void DisableReexec(); void MaybeReexec(); -bool AsanInterceptsSignal(int signum); void ReadContextStack(void *context, uptr *stack, uptr *ssize); void AsanPlatformThreadInit(); void StopInitOrderChecking(); @@ -107,10 +107,10 @@ void PlatformTSDDtor(void *tsd); void AppendToErrorMessageBuffer(const char *buffer); -void ParseExtraActivationFlags(); - void *AsanDlSymNext(const char *sym); +void ReserveShadowMemoryRange(uptr beg, uptr end); + // Platform-specific options. #if SANITIZER_MAC bool PlatformHasDifferentMemcpyAndMemmove(); diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc index fdd009c960d8..8e8bafd47af6 100644 --- a/contrib/compiler-rt/lib/asan/asan_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_linux.cc @@ -68,6 +68,10 @@ asan_rt_version_t __asan_rt_version; namespace __asan { +void DisableReexec() { + // No need to re-exec on Linux. +} + void MaybeReexec() { // No need to re-exec on Linux. } @@ -220,10 +224,6 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #endif } -bool AsanInterceptsSignal(int signum) { - return signum == SIGSEGV && common_flags()->handle_segv; -} - void AsanPlatformThreadInit() { // Nothing here for now. } diff --git a/contrib/compiler-rt/lib/asan/asan_mac.cc b/contrib/compiler-rt/lib/asan/asan_mac.cc index ae0fa15b6523..b35368617dca 100644 --- a/contrib/compiler-rt/lib/asan/asan_mac.cc +++ b/contrib/compiler-rt/lib/asan/asan_mac.cc @@ -101,8 +101,15 @@ void LeakyResetEnv(const char *name, const char *name_value) { } } +static bool reexec_disabled = false; + +void DisableReexec() { + reexec_disabled = true; +} + void MaybeReexec() { - if (!flags()->allow_reexec) return; + if (reexec_disabled) return; + // Make sure the dynamic ASan runtime library is preloaded so that the // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec // ourselves. @@ -113,8 +120,10 @@ void MaybeReexec() { uptr old_env_len = dyld_insert_libraries ? internal_strlen(dyld_insert_libraries) : 0; uptr fname_len = internal_strlen(info.dli_fname); + const char *dylib_name = StripModuleName(info.dli_fname); + uptr dylib_name_len = internal_strlen(dylib_name); if (!dyld_insert_libraries || - !REAL(strstr)(dyld_insert_libraries, StripModuleName(info.dli_fname))) { + !REAL(strstr)(dyld_insert_libraries, dylib_name)) { // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime // library. char program_name[1024]; @@ -140,58 +149,74 @@ void MaybeReexec() { VReport(1, "exec()-ing the program with\n"); VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env); VReport(1, "to enable ASan wrappers.\n"); - VReport(1, "Set ASAN_OPTIONS=allow_reexec=0 to disable this.\n"); execv(program_name, *_NSGetArgv()); - } else { - // DYLD_INSERT_LIBRARIES is set and contains the runtime library. - if (old_env_len == fname_len) { - // It's just the runtime library name - fine to unset the variable. - LeakyResetEnv(kDyldInsertLibraries, NULL); - } else { - uptr env_name_len = internal_strlen(kDyldInsertLibraries); - // Allocate memory to hold the previous env var name, its value, the '=' - // sign and the '\0' char. - char *new_env = (char*)allocator_for_env.Allocate( - old_env_len + 2 + env_name_len); - CHECK(new_env); - internal_memset(new_env, '\0', old_env_len + 2 + env_name_len); - internal_strncpy(new_env, kDyldInsertLibraries, env_name_len); - new_env[env_name_len] = '='; - char *new_env_pos = new_env + env_name_len + 1; - // Iterate over colon-separated pieces of |dyld_insert_libraries|. - char *piece_start = dyld_insert_libraries; - char *piece_end = NULL; - char *old_env_end = dyld_insert_libraries + old_env_len; - do { - if (piece_start[0] == ':') piece_start++; - piece_end = REAL(strchr)(piece_start, ':'); - if (!piece_end) piece_end = dyld_insert_libraries + old_env_len; - if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break; - uptr piece_len = piece_end - piece_start; - - // If the current piece isn't the runtime library name, - // append it to new_env. - if ((piece_len != fname_len) || - (internal_strncmp(piece_start, info.dli_fname, fname_len) != 0)) { - if (new_env_pos != new_env + env_name_len + 1) { - new_env_pos[0] = ':'; - new_env_pos++; - } - internal_strncpy(new_env_pos, piece_start, piece_len); - } - // Move on to the next piece. - new_env_pos += piece_len; - piece_start = piece_end; - } while (piece_start < old_env_end); - - // Can't use setenv() here, because it requires the allocator to be - // initialized. - // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in - // a separate function called after InitializeAllocator(). - LeakyResetEnv(kDyldInsertLibraries, new_env); - } + // We get here only if execv() failed. + Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, " + "which is required for ASan to work. ASan tried to set the " + "environment variable and re-execute itself, but execv() failed, " + "possibly because of sandbox restrictions. Make sure to launch the " + "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env); + CHECK("execv failed" && 0); } + + // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove + // the dylib from the environment variable, because interceptors are installed + // and we don't want our children to inherit the variable. + + uptr env_name_len = internal_strlen(kDyldInsertLibraries); + // Allocate memory to hold the previous env var name, its value, the '=' + // sign and the '\0' char. + char *new_env = (char*)allocator_for_env.Allocate( + old_env_len + 2 + env_name_len); + CHECK(new_env); + internal_memset(new_env, '\0', old_env_len + 2 + env_name_len); + internal_strncpy(new_env, kDyldInsertLibraries, env_name_len); + new_env[env_name_len] = '='; + char *new_env_pos = new_env + env_name_len + 1; + + // Iterate over colon-separated pieces of |dyld_insert_libraries|. + char *piece_start = dyld_insert_libraries; + char *piece_end = NULL; + char *old_env_end = dyld_insert_libraries + old_env_len; + do { + if (piece_start[0] == ':') piece_start++; + piece_end = REAL(strchr)(piece_start, ':'); + if (!piece_end) piece_end = dyld_insert_libraries + old_env_len; + if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break; + uptr piece_len = piece_end - piece_start; + + char *filename_start = + (char *)internal_memrchr(piece_start, '/', piece_len); + uptr filename_len = piece_len; + if (filename_start) { + filename_start += 1; + filename_len = piece_len - (filename_start - piece_start); + } else { + filename_start = piece_start; + } + + // If the current piece isn't the runtime library name, + // append it to new_env. + if ((dylib_name_len != filename_len) || + (internal_memcmp(filename_start, dylib_name, dylib_name_len) != 0)) { + if (new_env_pos != new_env + env_name_len + 1) { + new_env_pos[0] = ':'; + new_env_pos++; + } + internal_strncpy(new_env_pos, piece_start, piece_len); + new_env_pos += piece_len; + } + // Move on to the next piece. + piece_start = piece_end; + } while (piece_start < old_env_end); + + // Can't use setenv() here, because it requires the allocator to be + // initialized. + // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in + // a separate function called after InitializeAllocator(). + if (new_env_pos == new_env + env_name_len + 1) new_env = NULL; + LeakyResetEnv(kDyldInsertLibraries, new_env); } // No-op. Mac does not support static linkage anyway. @@ -205,11 +230,6 @@ void AsanCheckDynamicRTPrereqs() {} // No-op. Mac does not support static linkage anyway. void AsanCheckIncompatibleRT() {} -bool AsanInterceptsSignal(int signum) { - return (signum == SIGSEGV || signum == SIGBUS) && - common_flags()->handle_segv; -} - void AsanPlatformThreadInit() { } @@ -312,7 +332,7 @@ asan_block_context_t *alloc_asan_context(void *ctxt, dispatch_function_t func, dispatch_function_t func) { \ GET_STACK_TRACE_THREAD; \ asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); \ - if (common_flags()->verbosity >= 2) { \ + if (Verbosity() >= 2) { \ Report(#dispatch_x_f "(): context: %p, pthread_self: %p\n", \ asan_ctxt, pthread_self()); \ PRINT_CURRENT_STACK(); \ @@ -330,7 +350,7 @@ INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, dispatch_function_t func) { GET_STACK_TRACE_THREAD; asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); - if (common_flags()->verbosity >= 2) { + if (Verbosity() >= 2) { Report("dispatch_after_f: %p\n", asan_ctxt); PRINT_CURRENT_STACK(); } @@ -343,7 +363,7 @@ INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, dispatch_function_t func) { GET_STACK_TRACE_THREAD; asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); - if (common_flags()->verbosity >= 2) { + if (Verbosity() >= 2) { Report("dispatch_group_async_f(): context: %p, pthread_self: %p\n", asan_ctxt, pthread_self()); PRINT_CURRENT_STACK(); @@ -373,13 +393,6 @@ void dispatch_source_set_event_handler(dispatch_source_t ds, void(^work)(void)); work(); \ } -// Forces the compiler to generate a frame pointer in the function. -#define ENABLE_FRAME_POINTER \ - do { \ - volatile uptr enable_fp; \ - enable_fp = GET_CURRENT_FRAME(); \ - } while (0) - INTERCEPTOR(void, dispatch_async, dispatch_queue_t dq, void(^work)(void)) { ENABLE_FRAME_POINTER; @@ -403,6 +416,10 @@ INTERCEPTOR(void, dispatch_after, INTERCEPTOR(void, dispatch_source_set_cancel_handler, dispatch_source_t ds, void(^work)(void)) { + if (!work) { + REAL(dispatch_source_set_cancel_handler)(ds, work); + return; + } ENABLE_FRAME_POINTER; GET_ASAN_BLOCK(work); REAL(dispatch_source_set_cancel_handler)(ds, asan_block); diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc b/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc index 79b6dfae10f6..d7a6307c9bdc 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc @@ -152,13 +152,17 @@ INTERCEPTOR(int, posix_memalign, void **memptr, size_t alignment, size_t size) { namespace { -// TODO(glider): the mz_* functions should be united with the Linux wrappers, -// as they are basically copied from there. -size_t mz_size(malloc_zone_t* zone, const void* ptr) { +// TODO(glider): the __asan_mz_* functions should be united with the Linux +// wrappers, as they are basically copied from there. +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +size_t __asan_mz_size(malloc_zone_t* zone, const void* ptr) { return asan_mz_size(ptr); } -void *mz_malloc(malloc_zone_t *zone, size_t size) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__asan_mz_malloc(malloc_zone_t *zone, uptr size) { if (UNLIKELY(!asan_inited)) { CHECK(system_malloc_zone); return malloc_zone_malloc(system_malloc_zone, size); @@ -167,7 +171,9 @@ void *mz_malloc(malloc_zone_t *zone, size_t size) { return asan_malloc(size, &stack); } -void *mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__asan_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { if (UNLIKELY(!asan_inited)) { // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. const size_t kCallocPoolSize = 1024; @@ -183,7 +189,9 @@ void *mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { return asan_calloc(nmemb, size, &stack); } -void *mz_valloc(malloc_zone_t *zone, size_t size) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__asan_mz_valloc(malloc_zone_t *zone, size_t size) { if (UNLIKELY(!asan_inited)) { CHECK(system_malloc_zone); return malloc_zone_valloc(system_malloc_zone, size); @@ -210,11 +218,15 @@ void ALWAYS_INLINE free_common(void *context, void *ptr) { } // TODO(glider): the allocation callbacks need to be refactored. -void mz_free(malloc_zone_t *zone, void *ptr) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void __asan_mz_free(malloc_zone_t *zone, void *ptr) { free_common(zone, ptr); } -void *mz_realloc(malloc_zone_t *zone, void *ptr, size_t size) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__asan_mz_realloc(malloc_zone_t *zone, void *ptr, size_t size) { if (!ptr) { GET_STACK_TRACE_MALLOC; return asan_malloc(size, &stack); @@ -233,15 +245,16 @@ void *mz_realloc(malloc_zone_t *zone, void *ptr, size_t size) { } } -void mz_destroy(malloc_zone_t* zone) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void __asan_mz_destroy(malloc_zone_t* zone) { // A no-op -- we will not be destroyed! - Report("mz_destroy() called -- ignoring\n"); + Report("__asan_mz_destroy() called -- ignoring\n"); } - // from AvailabilityMacros.h -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 -void *mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__asan_mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { if (UNLIKELY(!asan_inited)) { CHECK(system_malloc_zone); return malloc_zone_memalign(system_malloc_zone, align, size); @@ -252,12 +265,12 @@ void *mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { // This function is currently unused, and we build with -Werror. #if 0 -void mz_free_definite_size(malloc_zone_t* zone, void *ptr, size_t size) { +void __asan_mz_free_definite_size( + malloc_zone_t* zone, void *ptr, size_t size) { // TODO(glider): check that |size| is valid. UNIMPLEMENTED(); } #endif -#endif kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask, vm_address_t zone_address, @@ -299,13 +312,10 @@ void mi_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { internal_memcpy(stats, &malloc_stats, sizeof(malloc_statistics_t)); } -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 boolean_t mi_zone_locked(malloc_zone_t *zone) { // UNIMPLEMENTED(); return false; } -#endif } // unnamed namespace @@ -324,32 +334,25 @@ void ReplaceSystemMalloc() { asan_introspection.force_lock = &mi_force_lock; asan_introspection.force_unlock = &mi_force_unlock; asan_introspection.statistics = &mi_statistics; + asan_introspection.zone_locked = &mi_zone_locked; internal_memset(&asan_zone, 0, sizeof(malloc_zone_t)); - // Start with a version 4 zone which is used for OS X 10.4 and 10.5. - asan_zone.version = 4; + // Use version 6 for OSX >= 10.6. + asan_zone.version = 6; asan_zone.zone_name = "asan"; - asan_zone.size = &mz_size; - asan_zone.malloc = &mz_malloc; - asan_zone.calloc = &mz_calloc; - asan_zone.valloc = &mz_valloc; - asan_zone.free = &mz_free; - asan_zone.realloc = &mz_realloc; - asan_zone.destroy = &mz_destroy; + asan_zone.size = &__asan_mz_size; + asan_zone.malloc = &__asan_mz_malloc; + asan_zone.calloc = &__asan_mz_calloc; + asan_zone.valloc = &__asan_mz_valloc; + asan_zone.free = &__asan_mz_free; + asan_zone.realloc = &__asan_mz_realloc; + asan_zone.destroy = &__asan_mz_destroy; asan_zone.batch_malloc = 0; asan_zone.batch_free = 0; - asan_zone.introspect = &asan_introspection; - - // from AvailabilityMacros.h -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - // Switch to version 6 on OSX 10.6 to support memalign. - asan_zone.version = 6; asan_zone.free_definite_size = 0; - asan_zone.memalign = &mz_memalign; - asan_introspection.zone_locked = &mi_zone_locked; -#endif + asan_zone.memalign = &__asan_mz_memalign; + asan_zone.introspect = &asan_introspection; // Register the ASan zone. malloc_zone_register(&asan_zone); diff --git a/contrib/compiler-rt/lib/asan/asan_mapping.h b/contrib/compiler-rt/lib/asan/asan_mapping.h index 2746754152b6..5cb011d683d3 100644 --- a/contrib/compiler-rt/lib/asan/asan_mapping.h +++ b/contrib/compiler-rt/lib/asan/asan_mapping.h @@ -59,13 +59,20 @@ // || `[0x20000000, 0x23ffffff]` || LowShadow || // || `[0x00000000, 0x1fffffff]` || LowMem || // -// Default Linux/MIPS mapping: +// Default Linux/MIPS32 mapping: // || `[0x2aaa0000, 0xffffffff]` || HighMem || // || `[0x0fff4000, 0x2aa9ffff]` || HighShadow || // || `[0x0bff4000, 0x0fff3fff]` || ShadowGap || // || `[0x0aaa0000, 0x0bff3fff]` || LowShadow || // || `[0x00000000, 0x0aa9ffff]` || LowMem || // +// Default Linux/MIPS64 mapping: +// || `[0x4000000000, 0xffffffffff]` || HighMem || +// || `[0x2800000000, 0x3fffffffff]` || HighShadow || +// || `[0x2400000000, 0x27ffffffff]` || ShadowGap || +// || `[0x2000000000, 0x23ffffffff]` || LowShadow || +// || `[0x0000000000, 0x1fffffffff]` || LowMem || +// // Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: // || `[0x500000000000, 0x7fffffffffff]` || HighMem || // || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || @@ -79,6 +86,15 @@ // || `[0x48000000, 0x4bffffff]` || ShadowGap || // || `[0x40000000, 0x47ffffff]` || LowShadow || // || `[0x00000000, 0x3fffffff]` || LowMem || +// +// Default Windows/i386 mapping: +// (the exact location of HighShadow/HighMem may vary depending +// on WoW64, /LARGEADDRESSAWARE, etc). +// || `[0x50000000, 0xffffffff]` || HighMem || +// || `[0x3a000000, 0x4fffffff]` || HighShadow || +// || `[0x36000000, 0x39ffffff]` || ShadowGap || +// || `[0x30000000, 0x35ffffff]` || LowShadow || +// || `[0x00000000, 0x2fffffff]` || LowMem || static const u64 kDefaultShadowScale = 3; static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000 @@ -87,10 +103,11 @@ static const u64 kDefaultShadowOffset64 = 1ULL << 44; static const u64 kDefaultShort64bitShadowOffset = 0x7FFF8000; // < 2G. static const u64 kAArch64_ShadowOffset64 = 1ULL << 36; static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; -static const u64 kMIPS64_ShadowOffset64 = 1ULL << 36; +static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; static const u64 kPPC64_ShadowOffset64 = 1ULL << 41; static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 +static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 #define SHADOW_SCALE kDefaultShadowScale #if SANITIZER_ANDROID @@ -101,12 +118,12 @@ static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 # define SHADOW_OFFSET kMIPS32_ShadowOffset32 # elif SANITIZER_FREEBSD # define SHADOW_OFFSET kFreeBSD_ShadowOffset32 +# elif SANITIZER_IOS +# define SHADOW_OFFSET kIosShadowOffset32 +# elif SANITIZER_WINDOWS +# define SHADOW_OFFSET kWindowsShadowOffset32 # else -# if SANITIZER_IOS -# define SHADOW_OFFSET kIosShadowOffset32 -# else -# define SHADOW_OFFSET kDefaultShadowOffset32 -# endif +# define SHADOW_OFFSET kDefaultShadowOffset32 # endif # else # if defined(__aarch64__) diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.cc b/contrib/compiler-rt/lib/asan/asan_poisoning.cc index 1c6e92f69c65..e2b1f4dc4d5e 100644 --- a/contrib/compiler-rt/lib/asan/asan_poisoning.cc +++ b/contrib/compiler-rt/lib/asan/asan_poisoning.cc @@ -15,13 +15,24 @@ #include "asan_poisoning.h" #include "asan_report.h" #include "asan_stack.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_flags.h" namespace __asan { +static atomic_uint8_t can_poison_memory; + +void SetCanPoisonMemory(bool value) { + atomic_store(&can_poison_memory, value, memory_order_release); +} + +bool CanPoisonMemory() { + return atomic_load(&can_poison_memory, memory_order_acquire); +} + void PoisonShadow(uptr addr, uptr size, u8 value) { - if (!flags()->poison_heap) return; + if (!CanPoisonMemory()) return; CHECK(AddrIsAlignedByGranularity(addr)); CHECK(AddrIsInMem(addr)); CHECK(AddrIsAlignedByGranularity(addr + size)); @@ -34,7 +45,7 @@ void PoisonShadowPartialRightRedzone(uptr addr, uptr size, uptr redzone_size, u8 value) { - if (!flags()->poison_heap) return; + if (!CanPoisonMemory()) return; CHECK(AddrIsAlignedByGranularity(addr)); CHECK(AddrIsInMem(addr)); FastPoisonShadowPartialRightRedzone(addr, size, redzone_size, value); @@ -63,10 +74,10 @@ void FlushUnneededASanShadowMemory(uptr p, uptr size) { void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) { uptr end = ptr + size; - if (common_flags()->verbosity) { + if (Verbosity()) { Printf("__asan_%spoison_intra_object_redzone [%p,%p) %zd\n", poison ? "" : "un", ptr, end, size); - if (common_flags()->verbosity >= 2) + if (Verbosity() >= 2) PRINT_CURRENT_STACK(); } CHECK(size); diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.h b/contrib/compiler-rt/lib/asan/asan_poisoning.h index feda1a984544..3fc94649fb39 100644 --- a/contrib/compiler-rt/lib/asan/asan_poisoning.h +++ b/contrib/compiler-rt/lib/asan/asan_poisoning.h @@ -19,6 +19,10 @@ namespace __asan { +// Enable/disable memory poisoning. +void SetCanPoisonMemory(bool value); +bool CanPoisonMemory(); + // Poisons the shadow memory for "size" bytes starting from "addr". void PoisonShadow(uptr addr, uptr size, u8 value); @@ -34,7 +38,7 @@ void PoisonShadowPartialRightRedzone(uptr addr, // performance-critical code with care. ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, u8 value) { - DCHECK(flags()->poison_heap); + DCHECK(CanPoisonMemory()); uptr shadow_beg = MEM_TO_SHADOW(aligned_beg); uptr shadow_end = MEM_TO_SHADOW( aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1; @@ -60,15 +64,14 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, if (page_end != shadow_end) { REAL(memset)((void *)page_end, 0, shadow_end - page_end); } - void *res = MmapFixedNoReserve(page_beg, page_end - page_beg); - CHECK_EQ(page_beg, res); + ReserveShadowMemoryRange(page_beg, page_end - 1); } } } ALWAYS_INLINE void FastPoisonShadowPartialRightRedzone( uptr aligned_addr, uptr size, uptr redzone_size, u8 value) { - DCHECK(flags()->poison_heap); + DCHECK(CanPoisonMemory()); bool poison_partial = flags()->poison_partial; u8 *shadow = (u8*)MEM_TO_SHADOW(aligned_addr); for (uptr i = 0; i < redzone_size; i += SHADOW_GRANULARITY, shadow++) { diff --git a/contrib/compiler-rt/lib/asan/asan_report.cc b/contrib/compiler-rt/lib/asan/asan_report.cc index 0fb50276186b..8706d5decc0b 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.cc +++ b/contrib/compiler-rt/lib/asan/asan_report.cc @@ -53,7 +53,7 @@ void AppendToErrorMessageBuffer(const char *buffer) { buffer, remaining); error_message_buffer[error_message_buffer_size - 1] = '\0'; // FIXME: reallocate the buffer instead of truncating the message. - error_message_buffer_pos += remaining > length ? length : remaining; + error_message_buffer_pos += Min(remaining, length); } } @@ -937,6 +937,8 @@ using namespace __asan; // NOLINT void __asan_report_error(uptr pc, uptr bp, uptr sp, uptr addr, int is_write, uptr access_size) { + ENABLE_FRAME_POINTER; + // Determine the error type. const char *bug_descr = "unknown-crash"; if (AddrIsInMem(addr)) { diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc index 34fb111d5607..9126e71a6437 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtl.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc @@ -56,8 +56,6 @@ static void AsanDie() { } if (common_flags()->coverage) __sanitizer_cov_dump(); - if (death_callback) - death_callback(); if (flags()->abort_on_error) Abort(); internal__exit(flags()->exitcode); @@ -72,265 +70,9 @@ static void AsanCheckFailed(const char *file, int line, const char *cond, Die(); } -// -------------------------- Flags ------------------------- {{{1 -static const int kDefaultMallocContextSize = 30; - -Flags asan_flags_dont_use_directly; // use via flags(). - -static const char *MaybeCallAsanDefaultOptions() { - return (&__asan_default_options) ? __asan_default_options() : ""; -} - -static const char *MaybeUseAsanDefaultOptionsCompileDefinition() { -#ifdef ASAN_DEFAULT_OPTIONS -// Stringize the macro value. -# define ASAN_STRINGIZE(x) #x -# define ASAN_STRINGIZE_OPTIONS(options) ASAN_STRINGIZE(options) - return ASAN_STRINGIZE_OPTIONS(ASAN_DEFAULT_OPTIONS); -#else - return ""; -#endif -} - -static void ParseFlagsFromString(Flags *f, const char *str) { - CommonFlags *cf = common_flags(); - ParseCommonFlagsFromString(cf, str); - CHECK((uptr)cf->malloc_context_size <= kStackTraceMax); - // Please write meaningful flag descriptions when adding new flags. - ParseFlag(str, &f->quarantine_size, "quarantine_size", - "Size (in bytes) of quarantine used to detect use-after-free " - "errors. Lower value may reduce memory usage but increase the " - "chance of false negatives."); - ParseFlag(str, &f->redzone, "redzone", - "Minimal size (in bytes) of redzones around heap objects. " - "Requirement: redzone >= 16, is a power of two."); - ParseFlag(str, &f->max_redzone, "max_redzone", - "Maximal size (in bytes) of redzones around heap objects."); - CHECK_GE(f->redzone, 16); - CHECK_GE(f->max_redzone, f->redzone); - CHECK_LE(f->max_redzone, 2048); - CHECK(IsPowerOfTwo(f->redzone)); - CHECK(IsPowerOfTwo(f->max_redzone)); - - ParseFlag(str, &f->debug, "debug", - "If set, prints some debugging information and does additional checks."); - ParseFlag(str, &f->report_globals, "report_globals", - "Controls the way to handle globals (0 - don't detect buffer overflow on " - "globals, 1 - detect buffer overflow, 2 - print data about registered " - "globals)."); - - ParseFlag(str, &f->check_initialization_order, - "check_initialization_order", - "If set, attempts to catch initialization order issues."); - - ParseFlag(str, &f->replace_str, "replace_str", - "If set, uses custom wrappers and replacements for libc string functions " - "to find more errors."); - - ParseFlag(str, &f->replace_intrin, "replace_intrin", - "If set, uses custom wrappers for memset/memcpy/memmove intinsics."); - ParseFlag(str, &f->mac_ignore_invalid_free, "mac_ignore_invalid_free", - "Ignore invalid free() calls to work around some bugs. Used on OS X " - "only."); - ParseFlag(str, &f->detect_stack_use_after_return, - "detect_stack_use_after_return", - "Enables stack-use-after-return checking at run-time."); - ParseFlag(str, &f->min_uar_stack_size_log, "min_uar_stack_size_log", - "Minimum fake stack size log."); - ParseFlag(str, &f->max_uar_stack_size_log, "max_uar_stack_size_log", - "Maximum fake stack size log."); - ParseFlag(str, &f->uar_noreserve, "uar_noreserve", - "Use mmap with 'norserve' flag to allocate fake stack."); - ParseFlag(str, &f->max_malloc_fill_size, "max_malloc_fill_size", - "ASan allocator flag. max_malloc_fill_size is the maximal amount of " - "bytes that will be filled with malloc_fill_byte on malloc."); - ParseFlag(str, &f->malloc_fill_byte, "malloc_fill_byte", - "Value used to fill the newly allocated memory."); - ParseFlag(str, &f->exitcode, "exitcode", - "Override the program exit status if the tool found an error."); - ParseFlag(str, &f->allow_user_poisoning, "allow_user_poisoning", - "If set, user may manually mark memory regions as poisoned or " - "unpoisoned."); - ParseFlag(str, &f->sleep_before_dying, "sleep_before_dying", - "Number of seconds to sleep between printing an error report and " - "terminating the program. Useful for debugging purposes (e.g. when one " - "needs to attach gdb)."); - - ParseFlag(str, &f->check_malloc_usable_size, "check_malloc_usable_size", - "Allows the users to work around the bug in Nvidia drivers prior to " - "295.*."); - - ParseFlag(str, &f->unmap_shadow_on_exit, "unmap_shadow_on_exit", - "If set, explicitly unmaps the (huge) shadow at exit."); - ParseFlag(str, &f->abort_on_error, "abort_on_error", - "If set, the tool calls abort() instead of _exit() after printing the " - "error report."); - ParseFlag(str, &f->print_stats, "print_stats", - "Print various statistics after printing an error message or if " - "atexit=1."); - ParseFlag(str, &f->print_legend, "print_legend", - "Print the legend for the shadow bytes."); - ParseFlag(str, &f->atexit, "atexit", - "If set, prints ASan exit stats even after program terminates " - "successfully."); - - ParseFlag(str, &f->allow_reexec, "allow_reexec", - "Allow the tool to re-exec the program. This may interfere badly with " - "the debugger."); - - ParseFlag(str, &f->print_full_thread_history, - "print_full_thread_history", - "If set, prints thread creation stacks for the threads involved in the " - "report and their ancestors up to the main thread."); - - ParseFlag(str, &f->poison_heap, "poison_heap", - "Poison (or not) the heap memory on [de]allocation. Zero value is useful " - "for benchmarking the allocator or instrumentator."); - - ParseFlag(str, &f->poison_array_cookie, "poison_array_cookie", - "Poison (or not) the array cookie after operator new[]."); - - ParseFlag(str, &f->poison_partial, "poison_partial", - "If true, poison partially addressable 8-byte aligned words " - "(default=true). This flag affects heap and global buffers, but not " - "stack buffers."); - - ParseFlag(str, &f->alloc_dealloc_mismatch, "alloc_dealloc_mismatch", - "Report errors on malloc/delete, new/free, new/delete[], etc."); - - ParseFlag(str, &f->new_delete_type_mismatch, "new_delete_type_mismatch", - "Report errors on mismatch betwen size of new and delete."); - - ParseFlag(str, &f->strict_memcmp, "strict_memcmp", - "If true, assume that memcmp(p1, p2, n) always reads n bytes before " - "comparing p1 and p2."); - - ParseFlag(str, &f->strict_init_order, "strict_init_order", - "If true, assume that dynamic initializers can never access globals from " - "other modules, even if the latter are already initialized."); - - ParseFlag(str, &f->start_deactivated, "start_deactivated", - "If true, ASan tweaks a bunch of other flags (quarantine, redzone, heap " - "poisoning) to reduce memory consumption as much as possible, and " - "restores them to original values when the first instrumented module is " - "loaded into the process. This is mainly intended to be used on " - "Android. "); - - ParseFlag(str, &f->detect_invalid_pointer_pairs, - "detect_invalid_pointer_pairs", - "If non-zero, try to detect operations like <, <=, >, >= and - on " - "invalid pointer pairs (e.g. when pointers belong to different objects). " - "The bigger the value the harder we try."); - - ParseFlag(str, &f->detect_container_overflow, - "detect_container_overflow", - "If true, honor the container overflow annotations. " - "See https://code.google.com/p/address-sanitizer/wiki/ContainerOverflow"); - - ParseFlag(str, &f->detect_odr_violation, "detect_odr_violation", - "If >=2, detect violation of One-Definition-Rule (ODR); " - "If ==1, detect ODR-violation only if the two variables " - "have different sizes"); - - ParseFlag(str, &f->dump_instruction_bytes, "dump_instruction_bytes", - "If true, dump 16 bytes starting at the instruction that caused SEGV"); -} - -void InitializeFlags(Flags *f, const char *env) { - CommonFlags *cf = common_flags(); - SetCommonFlagsDefaults(cf); - cf->detect_leaks = CAN_SANITIZE_LEAKS; - cf->external_symbolizer_path = GetEnv("ASAN_SYMBOLIZER_PATH"); - cf->malloc_context_size = kDefaultMallocContextSize; - cf->intercept_tls_get_addr = true; - cf->coverage = false; - - internal_memset(f, 0, sizeof(*f)); - f->quarantine_size = (ASAN_LOW_MEMORY) ? 1UL << 26 : 1UL << 28; - f->redzone = 16; - f->max_redzone = 2048; - f->debug = false; - f->report_globals = 1; - f->check_initialization_order = false; - f->replace_str = true; - f->replace_intrin = true; - f->mac_ignore_invalid_free = false; - f->detect_stack_use_after_return = false; // Also needs the compiler flag. - f->min_uar_stack_size_log = 16; // We can't do smaller anyway. - f->max_uar_stack_size_log = 20; // 1Mb per size class, i.e. ~11Mb per thread. - f->uar_noreserve = false; - f->max_malloc_fill_size = 0x1000; // By default, fill only the first 4K. - f->malloc_fill_byte = 0xbe; - f->exitcode = ASAN_DEFAULT_FAILURE_EXITCODE; - f->allow_user_poisoning = true; - f->sleep_before_dying = 0; - f->check_malloc_usable_size = true; - f->unmap_shadow_on_exit = false; - f->abort_on_error = false; - f->print_stats = false; - f->print_legend = true; - f->atexit = false; - f->allow_reexec = true; - f->print_full_thread_history = true; - f->poison_heap = true; - f->poison_array_cookie = true; - f->poison_partial = true; - // Turn off alloc/dealloc mismatch checker on Mac and Windows for now. - // https://code.google.com/p/address-sanitizer/issues/detail?id=131 - // https://code.google.com/p/address-sanitizer/issues/detail?id=309 - // TODO(glider,timurrrr): Fix known issues and enable this back. - f->alloc_dealloc_mismatch = (SANITIZER_MAC == 0) && (SANITIZER_WINDOWS == 0); - f->new_delete_type_mismatch = true; - f->strict_memcmp = true; - f->strict_init_order = false; - f->start_deactivated = false; - f->detect_invalid_pointer_pairs = 0; - f->detect_container_overflow = true; - f->detect_odr_violation = 2; - f->dump_instruction_bytes = false; - - // Override from compile definition. - ParseFlagsFromString(f, MaybeUseAsanDefaultOptionsCompileDefinition()); - - // Override from user-specified string. - ParseFlagsFromString(f, MaybeCallAsanDefaultOptions()); - VReport(1, "Using the defaults from __asan_default_options: %s\n", - MaybeCallAsanDefaultOptions()); - - // Override from command line. - ParseFlagsFromString(f, env); - if (common_flags()->help) { - PrintFlagDescriptions(); - } - - if (!CAN_SANITIZE_LEAKS && cf->detect_leaks) { - Report("%s: detect_leaks is not supported on this platform.\n", - SanitizerToolName); - cf->detect_leaks = false; - } - - // Make "strict_init_order" imply "check_initialization_order". - // TODO(samsonov): Use a single runtime flag for an init-order checker. - if (f->strict_init_order) { - f->check_initialization_order = true; - } -} - -// Parse flags that may change between startup and activation. -// On Android they come from a system property. -// On other platforms this is no-op. -void ParseExtraActivationFlags() { - char buf[100]; - GetExtraActivationFlags(buf, sizeof(buf)); - ParseFlagsFromString(flags(), buf); - if (buf[0] != '\0') - VReport(1, "Extra activation flags: %s\n", buf); -} - // -------------------------- Globals --------------------- {{{1 int asan_inited; bool asan_init_is_running; -void (*death_callback)(void); #if !ASAN_FIXED_MAPPING uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; @@ -344,7 +86,8 @@ void ShowStatsAndAbort() { // ---------------------- mmap -------------------- {{{1 // Reserve memory range [beg, end]. -static void ReserveShadowMemoryRange(uptr beg, uptr end) { +// We need to use inclusive range because end+1 may not be representable. +void ReserveShadowMemoryRange(uptr beg, uptr end) { CHECK_EQ((beg % GetPageSizeCached()), 0); CHECK_EQ(((end + 1) % GetPageSizeCached()), 0); uptr size = end - beg + 1; @@ -355,6 +98,10 @@ static void ReserveShadowMemoryRange(uptr beg, uptr end) { "Perhaps you're using ulimit -v\n", size); Abort(); } + if (common_flags()->no_huge_pages_for_shadow) + NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(beg, size); } // --------------- LowLevelAllocateCallbac ---------- {{{1 @@ -500,7 +247,13 @@ static void InitializeHighMemEnd() { } static void ProtectGap(uptr a, uptr size) { - CHECK_EQ(a, (uptr)Mprotect(a, size)); + void *res = Mprotect(a, size); + if (a == (uptr)res) + return; + Report("ERROR: Failed to protect the shadow gap. " + "ASan cannot proceed correctly. ABORTING.\n"); + DumpProcessMap(); + Die(); } static void PrintAddressSpaceLayout() { @@ -539,7 +292,7 @@ static void PrintAddressSpaceLayout() { Printf("\n"); Printf("redzone=%zu\n", (uptr)flags()->redzone); Printf("max_redzone=%zu\n", (uptr)flags()->max_redzone); - Printf("quarantine_size=%zuM\n", (uptr)flags()->quarantine_size >> 20); + Printf("quarantine_size_mb=%zuM\n", (uptr)flags()->quarantine_size_mb); Printf("malloc_context_size=%zu\n", (uptr)common_flags()->malloc_context_size); @@ -561,8 +314,10 @@ static void AsanInitInternal() { // Initialize flags. This must be done early, because most of the // initialization steps look at flags(). - const char *options = GetEnv("ASAN_OPTIONS"); - InitializeFlags(flags(), options); + InitializeFlags(); + + SetCanPoisonMemory(flags()->poison_heap); + SetMallocContextSize(common_flags()->malloc_context_size); InitializeHighMemEnd(); @@ -574,20 +329,11 @@ static void AsanInitInternal() { SetCheckFailedCallback(AsanCheckFailed); SetPrintfAndReportCallback(AppendToErrorMessageBuffer); - if (!flags()->start_deactivated) - ParseExtraActivationFlags(); - __sanitizer_set_report_path(common_flags()->log_path); + + // Enable UAR detection, if required. __asan_option_detect_stack_use_after_return = flags()->detect_stack_use_after_return; - CHECK_LE(flags()->min_uar_stack_size_log, flags()->max_uar_stack_size_log); - - if (options) { - VReport(1, "Parsed ASAN_OPTIONS: %s\n", options); - } - - if (flags()->start_deactivated) - AsanStartDeactivated(); // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec(); @@ -618,8 +364,7 @@ static void AsanInitInternal() { } #endif - if (common_flags()->verbosity) - PrintAddressSpaceLayout(); + if (Verbosity()) PrintAddressSpaceLayout(); DisableCoreDumperIfNecessary(); @@ -649,6 +394,8 @@ static void AsanInitInternal() { } else { Report("Shadow memory range interleaves with an existing memory mapping. " "ASan cannot proceed correctly. ABORTING.\n"); + Report("ASan shadow was supposed to be located in the [%p-%p] range.\n", + shadow_start, kHighShadowEnd); DumpProcessMap(); Die(); } @@ -656,7 +403,12 @@ static void AsanInitInternal() { AsanTSDInit(PlatformTSDDtor); InstallDeadlySignalHandlers(AsanOnSIGSEGV); - InitializeAllocator(); + AllocatorOptions allocator_options; + allocator_options.SetFrom(flags(), common_flags()); + InitializeAllocator(allocator_options); + + MaybeStartBackgroudThread(); + SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited // should be set to 1 prior to initializing the threads. @@ -666,10 +418,12 @@ static void AsanInitInternal() { if (flags()->atexit) Atexit(asan_atexit); - if (common_flags()->coverage) { - __sanitizer_cov_init(); - Atexit(__sanitizer_cov_dump); - } + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); + + // Now that ASan runtime is (mostly) initialized, deactivate it if + // necessary, so that it can be re-activated when requested. + if (flags()->start_deactivated) + AsanDeactivate(); // interceptors InitTlsSize(); @@ -686,7 +440,7 @@ static void AsanInitInternal() { SanitizerInitializeUnwinder(); #if CAN_SANITIZE_LEAKS - __lsan::InitCommonLsan(false); + __lsan::InitCommonLsan(); if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) { Atexit(__lsan::DoLeakCheck); } @@ -724,13 +478,6 @@ static AsanInitializer asan_initializer; // ---------------------- Interface ---------------- {{{1 using namespace __asan; // NOLINT -#if !SANITIZER_SUPPORTS_WEAK_HOOKS -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -const char* __asan_default_options() { return ""; } -} // extern "C" -#endif - int NOINLINE __asan_set_error_exit_code(int exit_code) { int old = flags()->exitcode; flags()->exitcode = exit_code; @@ -764,7 +511,7 @@ void NOINLINE __asan_handle_no_return() { } void NOINLINE __asan_set_death_callback(void (*callback)(void)) { - death_callback = callback; + SetUserDieCallback(callback); } // Initialize as requested from instrumented application code. diff --git a/contrib/compiler-rt/lib/asan/asan_stack.cc b/contrib/compiler-rt/lib/asan/asan_stack.cc index 8188f3b5b6e9..cf7a587fa65a 100644 --- a/contrib/compiler-rt/lib/asan/asan_stack.cc +++ b/contrib/compiler-rt/lib/asan/asan_stack.cc @@ -13,6 +13,21 @@ //===----------------------------------------------------------------------===// #include "asan_internal.h" #include "asan_stack.h" +#include "sanitizer_common/sanitizer_atomic.h" + +namespace __asan { + +static atomic_uint32_t malloc_context_size; + +void SetMallocContextSize(u32 size) { + atomic_store(&malloc_context_size, size, memory_order_release); +} + +u32 GetMallocContextSize() { + return atomic_load(&malloc_context_size, memory_order_acquire); +} + +} // namespace __asan // ------------------ Interface -------------- {{{1 diff --git a/contrib/compiler-rt/lib/asan/asan_stack.h b/contrib/compiler-rt/lib/asan/asan_stack.h index a995256212e1..122967a152f8 100644 --- a/contrib/compiler-rt/lib/asan/asan_stack.h +++ b/contrib/compiler-rt/lib/asan/asan_stack.h @@ -21,6 +21,11 @@ namespace __asan { +static const u32 kDefaultMallocContextSize = 30; + +void SetMallocContextSize(u32 size); +u32 GetMallocContextSize(); + // Get the stack trace with the given pc and bp. // The pc will be in the position 0 of the resulting stack trace. // The bp may refer to the current frame or to the caller's frame. @@ -93,9 +98,8 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, #define GET_STACK_TRACE_THREAD \ GET_STACK_TRACE(kStackTraceMax, true) -#define GET_STACK_TRACE_MALLOC \ - GET_STACK_TRACE(common_flags()->malloc_context_size, \ - common_flags()->fast_unwind_on_malloc) +#define GET_STACK_TRACE_MALLOC \ + GET_STACK_TRACE(GetMallocContextSize(), common_flags()->fast_unwind_on_malloc) #define GET_STACK_TRACE_FREE GET_STACK_TRACE_MALLOC diff --git a/contrib/compiler-rt/lib/asan/asan_suppressions.cc b/contrib/compiler-rt/lib/asan/asan_suppressions.cc index ef554716faa0..62198aec64e7 100644 --- a/contrib/compiler-rt/lib/asan/asan_suppressions.cc +++ b/contrib/compiler-rt/lib/asan/asan_suppressions.cc @@ -15,57 +15,62 @@ #include "asan_suppressions.h" #include "asan_stack.h" +#include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_symbolizer.h" namespace __asan { -static bool suppressions_inited = false; +ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; +static SuppressionContext *suppression_ctx = nullptr; +static const char kInterceptorName[] = "interceptor_name"; +static const char kInterceptorViaFunction[] = "interceptor_via_fun"; +static const char kInterceptorViaLibrary[] = "interceptor_via_lib"; +static const char *kSuppressionTypes[] = { + kInterceptorName, kInterceptorViaFunction, kInterceptorViaLibrary}; void InitializeSuppressions() { - CHECK(!suppressions_inited); - SuppressionContext::InitIfNecessary(); - suppressions_inited = true; + CHECK_EQ(nullptr, suppression_ctx); + suppression_ctx = new (suppression_placeholder) // NOLINT + SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes)); + suppression_ctx->ParseFromFile(flags()->suppressions); } bool IsInterceptorSuppressed(const char *interceptor_name) { - CHECK(suppressions_inited); - SuppressionContext *ctx = SuppressionContext::Get(); + CHECK(suppression_ctx); Suppression *s; // Match "interceptor_name" suppressions. - return ctx->Match(interceptor_name, SuppressionInterceptorName, &s); + return suppression_ctx->Match(interceptor_name, kInterceptorName, &s); } bool HaveStackTraceBasedSuppressions() { - CHECK(suppressions_inited); - SuppressionContext *ctx = SuppressionContext::Get(); - return ctx->HasSuppressionType(SuppressionInterceptorViaFunction) || - ctx->HasSuppressionType(SuppressionInterceptorViaLibrary); + CHECK(suppression_ctx); + return suppression_ctx->HasSuppressionType(kInterceptorViaFunction) || + suppression_ctx->HasSuppressionType(kInterceptorViaLibrary); } bool IsStackTraceSuppressed(const StackTrace *stack) { - CHECK(suppressions_inited); if (!HaveStackTraceBasedSuppressions()) return false; - SuppressionContext *ctx = SuppressionContext::Get(); + CHECK(suppression_ctx); Symbolizer *symbolizer = Symbolizer::GetOrInit(); Suppression *s; for (uptr i = 0; i < stack->size && stack->trace[i]; i++) { uptr addr = stack->trace[i]; - if (ctx->HasSuppressionType(SuppressionInterceptorViaLibrary)) { + if (suppression_ctx->HasSuppressionType(kInterceptorViaLibrary)) { const char *module_name; uptr module_offset; // Match "interceptor_via_lib" suppressions. if (symbolizer->GetModuleNameAndOffsetForPC(addr, &module_name, &module_offset) && - ctx->Match(module_name, SuppressionInterceptorViaLibrary, &s)) { + suppression_ctx->Match(module_name, kInterceptorViaLibrary, &s)) { return true; } } - if (ctx->HasSuppressionType(SuppressionInterceptorViaFunction)) { + if (suppression_ctx->HasSuppressionType(kInterceptorViaFunction)) { SymbolizedStack *frames = symbolizer->SymbolizePC(addr); for (SymbolizedStack *cur = frames; cur; cur = cur->next) { const char *function_name = cur->info.function; @@ -73,7 +78,8 @@ bool IsStackTraceSuppressed(const StackTrace *stack) { continue; } // Match "interceptor_via_fun" suppressions. - if (ctx->Match(function_name, SuppressionInterceptorViaFunction, &s)) { + if (suppression_ctx->Match(function_name, kInterceptorViaFunction, + &s)) { frames->ClearAll(); return true; } diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc index 4f02b022fe79..693f0bcdee8e 100644 --- a/contrib/compiler-rt/lib/asan/asan_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_win.cc @@ -60,6 +60,10 @@ void PlatformTSDDtor(void *tsd) { AsanThread::TSDDtor(tsd); } // ---------------------- Various stuff ---------------- {{{1 +void DisableReexec() { + // No need to re-exec on Windows. +} + void MaybeReexec() { // No need to re-exec on Windows. } diff --git a/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc b/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc index b38a2d16087f..5d39e33096a8 100644 --- a/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc +++ b/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc @@ -294,7 +294,43 @@ INTERFACE_FUNCTION(__asan_stack_free_8) INTERFACE_FUNCTION(__asan_stack_free_9) INTERFACE_FUNCTION(__asan_stack_free_10) +// FIXME: we might want to have a sanitizer_win_dll_thunk? +INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container) +INTERFACE_FUNCTION(__sanitizer_cov) +INTERFACE_FUNCTION(__sanitizer_cov_dump) +INTERFACE_FUNCTION(__sanitizer_cov_indir_call16) +INTERFACE_FUNCTION(__sanitizer_cov_init) INTERFACE_FUNCTION(__sanitizer_cov_module_init) +INTERFACE_FUNCTION(__sanitizer_cov_trace_basic_block) +INTERFACE_FUNCTION(__sanitizer_cov_trace_func_enter) +INTERFACE_FUNCTION(__sanitizer_cov_with_check) +INTERFACE_FUNCTION(__sanitizer_free_hook) +INTERFACE_FUNCTION(__sanitizer_get_allocated_size) +INTERFACE_FUNCTION(__sanitizer_get_coverage_guards) +INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes) +INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size) +INTERFACE_FUNCTION(__sanitizer_get_free_bytes) +INTERFACE_FUNCTION(__sanitizer_get_heap_size) +INTERFACE_FUNCTION(__sanitizer_get_ownership) +INTERFACE_FUNCTION(__sanitizer_get_total_unique_coverage) +INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes) +INTERFACE_FUNCTION(__sanitizer_malloc_hook) +INTERFACE_FUNCTION(__sanitizer_maybe_open_cov_file) +INTERFACE_FUNCTION(__sanitizer_print_stack_trace) +INTERFACE_FUNCTION(__sanitizer_ptr_cmp) +INTERFACE_FUNCTION(__sanitizer_ptr_sub) +INTERFACE_FUNCTION(__sanitizer_report_error_summary) +INTERFACE_FUNCTION(__sanitizer_reset_coverage) +INTERFACE_FUNCTION(__sanitizer_sandbox_on_notify) +INTERFACE_FUNCTION(__sanitizer_set_death_callback) +INTERFACE_FUNCTION(__sanitizer_set_report_path) +INTERFACE_FUNCTION(__sanitizer_unaligned_load16) +INTERFACE_FUNCTION(__sanitizer_unaligned_load32) +INTERFACE_FUNCTION(__sanitizer_unaligned_load64) +INTERFACE_FUNCTION(__sanitizer_unaligned_store16) +INTERFACE_FUNCTION(__sanitizer_unaligned_store32) +INTERFACE_FUNCTION(__sanitizer_unaligned_store64) +INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container) // TODO(timurrrr): Add more interface functions on the as-needed basis. diff --git a/contrib/compiler-rt/lib/asan/asan_win_dynamic_runtime_thunk.cc b/contrib/compiler-rt/lib/asan/asan_win_dynamic_runtime_thunk.cc index 3a4de7dbf1fb..19456141c1ec 100644 --- a/contrib/compiler-rt/lib/asan/asan_win_dynamic_runtime_thunk.cc +++ b/contrib/compiler-rt/lib/asan/asan_win_dynamic_runtime_thunk.cc @@ -23,10 +23,11 @@ // Using #ifdef rather than relying on Makefiles etc. // simplifies the build procedure. #ifdef ASAN_DYNAMIC_RUNTIME_THUNK -extern "C" { -__declspec(dllimport) int __asan_set_seh_filter(); -__declspec(dllimport) int __asan_should_detect_stack_use_after_return(); +#include +#include +extern "C" { +//////////////////////////////////////////////////////////////////////////////// // Define a copy of __asan_option_detect_stack_use_after_return that should be // used when linking an MD runtime with a set of object files on Windows. // @@ -37,16 +38,82 @@ __declspec(dllimport) int __asan_should_detect_stack_use_after_return(); // with a MT or MD runtime and we don't want to use ugly __imp_ names on Windows // just to work around this issue, let's clone the a variable that is // constant after initialization anyways. +__declspec(dllimport) int __asan_should_detect_stack_use_after_return(); int __asan_option_detect_stack_use_after_return = __asan_should_detect_stack_use_after_return(); +} -// Set the ASan-specific SEH handler at the end of CRT initialization of each -// module (see asan_win.cc for the details). +//////////////////////////////////////////////////////////////////////////////// +// For some reason, the MD CRT doesn't call the C/C++ terminators as MT does. +// To work around this, for each DLL we schedule a call to +// UnregisterGlobalsInRange atexit() specifying the address range of the DLL +// image to unregister globals in that range. We don't do the same +// for the main module (.exe) as the asan_globals.cc allocator is destroyed +// by the time UnregisterGlobalsInRange is executed. +// See PR22545 for the details. +namespace __asan { +__declspec(dllimport) +void UnregisterGlobalsInRange(void *beg, void *end); +} + +namespace { +void *this_module_base, *this_module_end; + +void UnregisterGlobals() { + __asan::UnregisterGlobalsInRange(this_module_base, this_module_end); +} + +int ScheduleUnregisterGlobals() { + HMODULE this_module = 0; + // Increments the reference counter of the DLL module, so need to call + // FreeLibrary later. + if (!GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, + (LPCTSTR)&UnregisterGlobals, &this_module)) + return 1; + + // Skip the main module. + if (this_module == GetModuleHandle(0)) + return 0; + + MODULEINFO mi; + bool success = + GetModuleInformation(GetCurrentProcess(), this_module, &mi, sizeof(mi)); + if (!FreeLibrary(this_module)) + return 2; + if (!success) + return 3; + + this_module_base = mi.lpBaseOfDll; + this_module_end = (char*)mi.lpBaseOfDll + mi.SizeOfImage; + + return atexit(UnregisterGlobals); +} +} // namespace + +/////////////////////////////////////////////////////////////////////////////// +// ASan SEH handling. +extern "C" __declspec(dllimport) int __asan_set_seh_filter(); +static int SetSEHFilter() { return __asan_set_seh_filter(); } + +/////////////////////////////////////////////////////////////////////////////// +// We schedule some work at start-up by placing callbacks to our code to the +// list of CRT C initializers. +// +// First, declare sections we'll be using: +#pragma section(".CRT$XID", long, read) // NOLINT +#pragma section(".CRT$XIZ", long, read) // NOLINT + +// We need to call 'atexit(UnregisterGlobals);' after atexit() is initialized +// (.CRT$XIC) but before the C++ constructors (.CRT$XCA). +__declspec(allocate(".CRT$XID")) +static int (*__asan_schedule_unregister_globals)() = ScheduleUnregisterGlobals; + +// We need to set the ASan-specific SEH handler at the end of CRT initialization +// of each module (see also asan_win.cc). // // Unfortunately, putting a pointer to __asan_set_seh_filter into // __asan_intercept_seh gets optimized out, so we have to use an extra function. -static int SetSEHFilter() { return __asan_set_seh_filter(); } -#pragma section(".CRT$XIZ", long, read) // NOLINT -__declspec(allocate(".CRT$XIZ")) int (*__asan_seh_interceptor)() = SetSEHFilter; -} +extern "C" __declspec(allocate(".CRT$XIZ")) +int (*__asan_seh_interceptor)() = SetSEHFilter; + #endif // ASAN_DYNAMIC_RUNTIME_THUNK diff --git a/contrib/compiler-rt/lib/asan/scripts/asan_device_setup b/contrib/compiler-rt/lib/asan/scripts/asan_device_setup index a620f51b0836..104e07b722ca 100755 --- a/contrib/compiler-rt/lib/asan/scripts/asan_device_setup +++ b/contrib/compiler-rt/lib/asan/scripts/asan_device_setup @@ -18,6 +18,7 @@ revert=no extra_options= device= lib= +use_su=0 function usage { echo "usage: $0 [--revert] [--device device-id] [--lib path] [--extra-options options]" @@ -26,13 +27,70 @@ function usage { echo " --extra-options: Extra ASAN_OPTIONS." echo " --device: Install to the given device. Use 'adb devices' to find" echo " device-id." + echo " --use-su: Use 'su -c' prefix for every adb command instead of using" + echo " 'adb root' once." echo exit 1 } +function adb_push { + if [ $use_su -eq 0 ]; then + $ADB push "$1" "$2" + else + local FILENAME=$(basename $1) + $ADB push "$1" "/data/local/tmp/$FILENAME" + $ADB shell su -c "rm \\\"$2/$FILENAME\\\"" >&/dev/null + $ADB shell su -c "cat \\\"/data/local/tmp/$FILENAME\\\" > \\\"$2/$FILENAME\\\"" + $ADB shell su -c "rm \\\"/data/local/tmp/$FILENAME\\\"" + fi +} + +function adb_remount { + if [ $use_su -eq 0 ]; then + $ADB remount + else + local STORAGE=`$ADB shell mount | grep /system | cut -d ' ' -f1` + if [ "$STORAGE" != "" ]; then + echo Remounting $STORAGE at /system + $ADB shell su -c "mount -o remount,rw $STORAGE /system" + else + echo Failed to get storage device name for "/system" mount point + fi + fi +} + +function adb_shell { + if [ $use_su -eq 0 ]; then + $ADB shell $@ + else + $ADB shell su -c "$*" + fi +} + +function adb_root { + if [ $use_su -eq 0 ]; then + $ADB root + fi +} + +function adb_wait_for_device { + $ADB wait-for-device +} + +function adb_pull { + if [ $use_su -eq 0 ]; then + $ADB pull "$1" "$2" + else + local FILENAME=$(basename $1) + $ADB shell rm "/data/local/tmp/$FILENAME" >&/dev/null + $ADB shell su -c "[ -f \\\"$1\\\" ] && cat \\\"$1\\\" > \\\"/data/local/tmp/$FILENAME\\\" && chown root.shell \\\"/data/local/tmp/$FILENAME\\\" && chmod 755 \\\"/data/local/tmp/$FILENAME\\\"" && + $ADB pull "/data/local/tmp/$FILENAME" "$2" >&/dev/null && $ADB shell "rm \"/data/local/tmp/$FILENAME\"" + fi +} + function get_device_arch { # OUTVAR local _outvar=$1 - local _ABI=$($ADB shell getprop ro.product.cpu.abi) + local _ABI=$(adb_shell getprop ro.product.cpu.abi) local _ARCH= if [[ $_ABI == x86* ]]; then _ARCH=i686 @@ -74,6 +132,9 @@ while [[ $# > 0 ]]; do fi device="$1" ;; + --use-su) + use_su=1 + ;; *) usage ;; @@ -86,11 +147,25 @@ if [[ x$device != x ]]; then ADB="$ADB -s $device" fi +if [ $use_su -eq 1 ]; then + # Test if 'su' is present on the device + SU_TEST_OUT=`$ADB shell su -c "echo foo" 2>&1 | sed 's/\r$//'` + if [ $? != 0 -o "$SU_TEST_OUT" != "foo" ]; then + echo "ERROR: Cannot use 'su -c':" + echo "$ adb shell su -c \"echo foo\"" + echo $SU_TEST_OUT + echo "Check that 'su' binary is correctly installed on the device or omit" + echo " --use-su flag" + exit 1 + fi +fi + echo '>> Remounting /system rw' -$ADB root -$ADB wait-for-device -$ADB remount -$ADB wait-for-device +adb_wait_for_device +adb_root +adb_wait_for_device +adb_remount +adb_wait_for_device get_device_arch ARCH echo "Target architecture: $ARCH" @@ -99,22 +174,24 @@ ASAN_RT="libclang_rt.asan-$ARCH-android.so" if [[ x$revert == xyes ]]; then echo '>> Uninstalling ASan' - if ! $ADB shell readlink /system/bin/app_process | grep 'app_process' >&/dev/null; then + if ! adb_shell ls -l /system/bin/app_process | grep -o '\->.*app_process' >&/dev/null; then echo '>> Pre-L device detected.' - $ADB shell mv /system/bin/app_process.real /system/bin/app_process - $ADB shell rm /system/bin/asanwrapper - $ADB shell rm /system/lib/$ASAN_RT + adb_shell mv /system/bin/app_process.real /system/bin/app_process + adb_shell rm /system/bin/asanwrapper else - $ADB shell rm /system/bin/app_process.wrap - $ADB shell rm /system/bin/asanwrapper - $ADB shell rm /system/lib/$ASAN_RT - $ADB shell rm /system/bin/app_process - $ADB shell ln -s /system/bin/app_process32 /system/bin/app_process + adb_shell rm /system/bin/app_process.wrap + adb_shell rm /system/bin/asanwrapper + adb_shell rm /system/bin/app_process + adb_shell ln -s /system/bin/app_process32 /system/bin/app_process fi echo '>> Restarting shell' - $ADB shell stop - $ADB shell start + adb_shell stop + adb_shell start + + # Remove the library on the last step to give a chance to the 'su' binary to + # be executed without problem. + adb_shell rm /system/lib/$ASAN_RT echo '>> Done' exit 0 @@ -145,28 +222,28 @@ TMPDIROLD="$TMPDIRBASE/old" TMPDIR="$TMPDIRBASE/new" mkdir "$TMPDIROLD" -RELEASE=$($ADB shell getprop ro.build.version.release) +RELEASE=$(adb_shell getprop ro.build.version.release) PRE_L=0 if echo "$RELEASE" | grep '^4\.' >&/dev/null; then PRE_L=1 fi -if ! $ADB shell readlink /system/bin/app_process | grep 'app_process' >&/dev/null; then +if ! adb_shell ls -l /system/bin/app_process | grep -o '\->.*app_process' >&/dev/null; then - if $ADB pull /system/bin/app_process.real /dev/null >&/dev/null; then + if adb_pull /system/bin/app_process.real /dev/null >&/dev/null; then echo '>> Old-style ASan installation detected. Reverting.' - $ADB shell mv /system/bin/app_process.real /system/bin/app_process + adb_shell mv /system/bin/app_process.real /system/bin/app_process fi echo '>> Pre-L device detected. Setting up app_process symlink.' - $ADB shell mv /system/bin/app_process /system/bin/app_process32 - $ADB shell ln -s /system/bin/app_process32 /system/bin/app_process + adb_shell mv /system/bin/app_process /system/bin/app_process32 + adb_shell ln -s /system/bin/app_process32 /system/bin/app_process fi echo '>> Copying files from the device' -$ADB pull /system/bin/app_process.wrap "$TMPDIROLD" || true -$ADB pull /system/bin/asanwrapper "$TMPDIROLD" || true -$ADB pull /system/lib/"$ASAN_RT" "$TMPDIROLD" || true +adb_pull /system/bin/app_process.wrap "$TMPDIROLD" || true +adb_pull /system/bin/asanwrapper "$TMPDIROLD" || true +adb_pull /system/lib/"$ASAN_RT" "$TMPDIROLD" || true cp -r "$TMPDIROLD" "$TMPDIR" if [[ -f "$TMPDIR/app_process.wrap" ]]; then @@ -184,7 +261,7 @@ cp "$ASAN_RT_PATH/$ASAN_RT" "$TMPDIR/" ASAN_OPTIONS=start_deactivated=1,alloc_dealloc_mismatch=0 # On Android-L not allowing user segv handler breaks some applications. -if $ADB shell 'echo $LD_PRELOAD' | grep libsigchain.so >&/dev/null; then +if [[ PRE_L -eq 0 ]]; then ASAN_OPTIONS="$ASAN_OPTIONS,allow_user_segv_handler=1" fi @@ -212,52 +289,52 @@ EOF if ! ( cd "$TMPDIRBASE" && diff -qr old/ new/ ) ; then echo '>> Pushing files to the device' - $ADB push "$TMPDIR/$ASAN_RT" /system/lib/ - $ADB push "$TMPDIR/app_process.wrap" /system/bin/app_process.wrap - $ADB push "$TMPDIR/asanwrapper" /system/bin/asanwrapper + adb_push "$TMPDIR/$ASAN_RT" /system/lib/ + adb_push "$TMPDIR/app_process.wrap" /system/bin + adb_push "$TMPDIR/asanwrapper" /system/bin - $ADB shell rm /system/bin/app_process - $ADB shell ln -s /system/bin/app_process.wrap /system/bin/app_process + adb_shell rm /system/bin/app_process + adb_shell ln -s /system/bin/app_process.wrap /system/bin/app_process - $ADB shell chown root.shell \ + adb_shell chown root.shell \ /system/lib/"$ASAN_RT" \ /system/bin/app_process.wrap \ /system/bin/asanwrapper - $ADB shell chmod 644 \ + adb_shell chmod 644 \ /system/lib/"$ASAN_RT" - $ADB shell chmod 755 \ + adb_shell chmod 755 \ /system/bin/app_process.wrap \ /system/bin/asanwrapper # Make SELinux happy by keeping app_process wrapper and the shell # it runs on in zygote domain. ENFORCING=0 - if $ADB shell getenforce | grep Enforcing >/dev/null; then + if adb_shell getenforce | grep Enforcing >/dev/null; then # Sometimes shell is not allowed to change file contexts. # Temporarily switch to permissive. ENFORCING=1 - $ADB shell setenforce 0 + adb_shell setenforce 0 fi - $ADB shell cp /system/bin/sh /system/bin/sh-from-zygote + adb_shell cp /system/bin/sh /system/bin/sh-from-zygote if [[ PRE_L -eq 1 ]]; then CTX=u:object_r:system_file:s0 else CTX=u:object_r:zygote_exec:s0 fi - $ADB shell chcon $CTX \ + adb_shell chcon $CTX \ /system/bin/sh-from-zygote \ /system/bin/app_process.wrap \ /system/bin/app_process32 if [ $ENFORCING == 1 ]; then - $ADB shell setenforce 1 + adb_shell setenforce 1 fi echo '>> Restarting shell (asynchronous)' - $ADB shell stop - $ADB shell start + adb_shell stop + adb_shell start echo '>> Please wait until the device restarts' else diff --git a/contrib/compiler-rt/lib/asan/scripts/asan_symbolize.py b/contrib/compiler-rt/lib/asan/scripts/asan_symbolize.py index 5fca136b6950..59fceaaed814 100755 --- a/contrib/compiler-rt/lib/asan/scripts/asan_symbolize.py +++ b/contrib/compiler-rt/lib/asan/scripts/asan_symbolize.py @@ -11,11 +11,9 @@ import argparse import bisect import getopt import os -import pty import re import subprocess import sys -import termios symbolizers = {} DEBUG = False @@ -171,6 +169,9 @@ class UnbufferedLineConverter(object): output. Uses pty to trick the child into providing unbuffered output. """ def __init__(self, args, close_stderr=False): + # Local imports so that the script can start on Windows. + import pty + import termios pid, fd = pty.fork() if pid == 0: # We're the child. Transfer control to command. @@ -341,17 +342,23 @@ class BreakpadSymbolizer(Symbolizer): class SymbolizationLoop(object): def __init__(self, binary_name_filter=None, dsym_hint_producer=None): - # Used by clients who may want to supply a different binary name. - # E.g. in Chrome several binaries may share a single .dSYM. - self.binary_name_filter = binary_name_filter - self.dsym_hint_producer = dsym_hint_producer - self.system = os.uname()[0] - if self.system not in ['Linux', 'Darwin', 'FreeBSD']: - raise Exception('Unknown system') - self.llvm_symbolizers = {} - self.last_llvm_symbolizer = None - self.dsym_hints = set([]) - self.frame_no = 0 + if sys.platform == 'win32': + # ASan on Windows uses dbghelp.dll to symbolize in-process, which works + # even in sandboxed processes. Nothing needs to be done here. + self.process_line = self.process_line_echo + else: + # Used by clients who may want to supply a different binary name. + # E.g. in Chrome several binaries may share a single .dSYM. + self.binary_name_filter = binary_name_filter + self.dsym_hint_producer = dsym_hint_producer + self.system = os.uname()[0] + if self.system not in ['Linux', 'Darwin', 'FreeBSD']: + raise Exception('Unknown system') + self.llvm_symbolizers = {} + self.last_llvm_symbolizer = None + self.dsym_hints = set([]) + self.frame_no = 0 + self.process_line = self.process_line_posix def symbolize_address(self, addr, binary, offset): # On non-Darwin (i.e. on platforms without .dSYM debug info) always use @@ -366,12 +373,12 @@ class SymbolizationLoop(object): # 3. otherwise create a new symbolizer and pass all currently known # .dSYM hints to it. if not binary in self.llvm_symbolizers: - use_last_symbolizer = True + use_new_symbolizer = True if self.system == 'Darwin' and self.dsym_hint_producer: dsym_hints_for_binary = set(self.dsym_hint_producer(binary)) - use_last_symbolizer = bool(dsym_hints_for_binary - self.dsym_hints) + use_new_symbolizer = bool(dsym_hints_for_binary - self.dsym_hints) self.dsym_hints |= dsym_hints_for_binary - if self.last_llvm_symbolizer and use_last_symbolizer: + if self.last_llvm_symbolizer and not use_new_symbolizer: self.llvm_symbolizers[binary] = self.last_llvm_symbolizer else: self.last_llvm_symbolizer = LLVMSymbolizerFactory( @@ -405,14 +412,14 @@ class SymbolizationLoop(object): def process_logfile(self): self.frame_no = 0 - while True: - line = logfile.readline() - if not line: - break + for line in logfile: processed = self.process_line(line) print '\n'.join(processed) - def process_line(self, line): + def process_line_echo(self, line): + return [line.rstrip()] + + def process_line_posix(self, line): self.current_line = line.rstrip() #0 0x7f6e35cf2e45 (/blah/foo.so+0x11fe45) stack_trace_line_format = ( @@ -437,20 +444,23 @@ class SymbolizationLoop(object): if __name__ == '__main__': - parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, - description='ASan symbolization script', - epilog='''Example of use: - asan_symbolize.py -c "$HOME/opt/cross/bin/arm-linux-gnueabi-" -s "$HOME/SymbolFiles" < asan.log''') + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='ASan symbolization script', + epilog='Example of use:\n' + 'asan_symbolize.py -c "$HOME/opt/cross/bin/arm-linux-gnueabi-" ' + '-s "$HOME/SymbolFiles" < asan.log') parser.add_argument('path_to_cut', nargs='*', - help='pattern to be cut from the result file path ') + help='pattern to be cut from the result file path ') parser.add_argument('-d','--demangle', action='store_true', - help='demangle function names') + help='demangle function names') parser.add_argument('-s', metavar='SYSROOT', - help='set path to sysroot for sanitized binaries') + help='set path to sysroot for sanitized binaries') parser.add_argument('-c', metavar='CROSS_COMPILE', - help='set prefix for binutils') - parser.add_argument('-l','--logfile', default=sys.stdin, type=argparse.FileType('r'), - help='set log file name to parse, default is stdin') + help='set prefix for binutils') + parser.add_argument('-l','--logfile', default=sys.stdin, + type=argparse.FileType('r'), + help='set log file name to parse, default is stdin') args = parser.parse_args() if args.path_to_cut: fix_filename_patterns = args.path_to_cut diff --git a/contrib/compiler-rt/lib/asan/tests/asan_interface_test.cc b/contrib/compiler-rt/lib/asan/tests/asan_interface_test.cc index 50fdf1119f0b..a34c8528eae0 100644 --- a/contrib/compiler-rt/lib/asan/tests/asan_interface_test.cc +++ b/contrib/compiler-rt/lib/asan/tests/asan_interface_test.cc @@ -87,7 +87,7 @@ TEST(AddressSanitizerInterface, GetCurrentAllocatedBytesTest) { } TEST(AddressSanitizerInterface, GetHeapSizeTest) { - // asan_allocator2 does not keep huge chunks in free list, but unmaps them. + // ASan allocator does not keep huge chunks in free list, but unmaps them. // The chunk should be greater than the quarantine size, // otherwise it will be stuck in quarantine instead of being unmaped. static const size_t kLargeMallocSize = (1 << 28) + 1; // 256M diff --git a/contrib/compiler-rt/lib/asan/tests/asan_noinst_test.cc b/contrib/compiler-rt/lib/asan/tests/asan_noinst_test.cc index bb6af45bddf9..6a428fbbc2b9 100644 --- a/contrib/compiler-rt/lib/asan/tests/asan_noinst_test.cc +++ b/contrib/compiler-rt/lib/asan/tests/asan_noinst_test.cc @@ -31,18 +31,12 @@ // in this test. The static runtime library is linked explicitly (without // -fsanitize=address), thus the interceptors do not work correctly on OS X. -#if !defined(_WIN32) -extern "C" { -// Set specific ASan options for uninstrumented unittest. -const char* __asan_default_options() { - return "allow_reexec=0"; -} -} // extern "C" -#endif - // Make sure __asan_init is called before any test case is run. struct AsanInitCaller { - AsanInitCaller() { __asan_init(); } + AsanInitCaller() { + __asan::DisableReexec(); + __asan_init(); + } }; static AsanInitCaller asan_init_caller; diff --git a/contrib/compiler-rt/lib/asan/tests/asan_test.cc b/contrib/compiler-rt/lib/asan/tests/asan_test.cc index 67bcbaca1e40..952b05e21fe2 100644 --- a/contrib/compiler-rt/lib/asan/tests/asan_test.cc +++ b/contrib/compiler-rt/lib/asan/tests/asan_test.cc @@ -603,7 +603,8 @@ NOINLINE void SigLongJmpFunc1(sigjmp_buf buf) { } #if !defined(__ANDROID__) && !defined(__arm__) && \ - !defined(__powerpc64__) && !defined(__powerpc__) + !defined(__powerpc64__) && !defined(__powerpc__) && \ + !defined(__aarch64__) // Does not work on Power and ARM: // https://code.google.com/p/address-sanitizer/issues/detail?id=185 TEST(AddressSanitizer, BuiltinLongJmpTest) { @@ -1284,3 +1285,33 @@ TEST(AddressSanitizer, pthread_getschedparam) { ASSERT_EQ(0, res); } #endif + +#if SANITIZER_TEST_HAS_PRINTF_L +static int vsnprintf_l_wrapper(char *s, size_t n, + locale_t l, const char *format, ...) { + va_list va; + va_start(va, format); + int res = vsnprintf_l(s, n , l, format, va); + va_end(va); + return res; +} + +TEST(AddressSanitizer, snprintf_l) { + char buff[5]; + // Check that snprintf_l() works fine with Asan. + int res = snprintf_l(buff, 5, + _LIBCPP_GET_C_LOCALE, "%s", "snprintf_l()"); + EXPECT_EQ(12, res); + // Check that vsnprintf_l() works fine with Asan. + res = vsnprintf_l_wrapper(buff, 5, + _LIBCPP_GET_C_LOCALE, "%s", "vsnprintf_l()"); + EXPECT_EQ(13, res); + + EXPECT_DEATH(snprintf_l(buff, 10, + _LIBCPP_GET_C_LOCALE, "%s", "snprintf_l()"), + "AddressSanitizer: stack-buffer-overflow"); + EXPECT_DEATH(vsnprintf_l_wrapper(buff, 10, + _LIBCPP_GET_C_LOCALE, "%s", "vsnprintf_l()"), + "AddressSanitizer: stack-buffer-overflow"); +} +#endif diff --git a/contrib/compiler-rt/lib/builtins/atomic.c b/contrib/compiler-rt/lib/builtins/atomic.c index 02429a653d2b..35c8837dcecf 100644 --- a/contrib/compiler-rt/lib/builtins/atomic.c +++ b/contrib/compiler-rt/lib/builtins/atomic.c @@ -28,20 +28,14 @@ #include #include +#include "assembly.h" + // Clang objects if you redefine a builtin. This little hack allows us to // define a function with the same name as an intrinsic. -#if __APPLE__ -// mach-o has extra leading underscore -#pragma redefine_extname __atomic_load_c ___atomic_load -#pragma redefine_extname __atomic_store_c ___atomic_store -#pragma redefine_extname __atomic_exchange_c ___atomic_exchange -#pragma redefine_extname __atomic_compare_exchange_c ___atomic_compare_exchange -#else -#pragma redefine_extname __atomic_load_c __atomic_load -#pragma redefine_extname __atomic_store_c __atomic_store -#pragma redefine_extname __atomic_exchange_c __atomic_exchange -#pragma redefine_extname __atomic_compare_exchange_c __atomic_compare_exchange -#endif +#pragma redefine_extname __atomic_load_c SYMBOL_NAME(__atomic_load) +#pragma redefine_extname __atomic_store_c SYMBOL_NAME(__atomic_store) +#pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange) +#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME(__atomic_compare_exchange) /// Number of locks. This allocates one page on 32-bit platforms, two on /// 64-bit. This can be specified externally if a different trade between diff --git a/contrib/compiler-rt/lib/builtins/clear_cache.c b/contrib/compiler-rt/lib/builtins/clear_cache.c index 4b46e8b3e9be..b7e099dfb2c0 100644 --- a/contrib/compiler-rt/lib/builtins/clear_cache.c +++ b/contrib/compiler-rt/lib/builtins/clear_cache.c @@ -22,8 +22,53 @@ #include #endif -#if defined(__ANDROID__) && defined(__mips__) +#if defined(__mips__) && !defined(__FreeBSD__) #include + #include + #if defined(__ANDROID__) && defined(__LP64__) + /* + * clear_mips_cache - Invalidates instruction cache for Mips. + */ + static void clear_mips_cache(const void* Addr, size_t Size) { + asm volatile ( + ".set push\n" + ".set noreorder\n" + ".set noat\n" + "beq %[Size], $zero, 20f\n" /* If size == 0, branch around. */ + "nop\n" + "daddu %[Size], %[Addr], %[Size]\n" /* Calculate end address + 1 */ + "rdhwr $v0, $1\n" /* Get step size for SYNCI. + $1 is $HW_SYNCI_Step */ + "beq $v0, $zero, 20f\n" /* If no caches require + synchronization, branch + around. */ + "nop\n" + "10:\n" + "synci 0(%[Addr])\n" /* Synchronize all caches around + address. */ + "daddu %[Addr], %[Addr], $v0\n" /* Add step size. */ + "sltu $at, %[Addr], %[Size]\n" /* Compare current with end + address. */ + "bne $at, $zero, 10b\n" /* Branch if more to do. */ + "nop\n" + "sync\n" /* Clear memory hazards. */ + "20:\n" + "bal 30f\n" + "nop\n" + "30:\n" + "daddiu $ra, $ra, 12\n" /* $ra has a value of $pc here. + Add offset of 12 to point to the + instruction after the last nop. + */ + "jr.hb $ra\n" /* Return, clearing instruction + hazards. */ + "nop\n" + ".set pop\n" + : [Addr] "+r"(Addr), [Size] "+r"(Size) + :: "at", "ra", "v0", "memory" + ); + } + #endif #endif #if defined(__ANDROID__) && defined(__arm__) @@ -52,7 +97,7 @@ void __clear_cache(void *start, void *end) { sysarch(ARM_SYNC_ICACHE, &arg); #elif defined(__ANDROID__) - const register int start_reg __asm("r0") = (int) (intptr_t) start; + register int start_reg __asm("r0") = (int) (intptr_t) start; const register int end_reg __asm("r1") = (int) (intptr_t) end; const register int flags __asm("r2") = 0; const register int syscall_nr __asm("r7") = __ARM_NR_cacheflush; @@ -64,10 +109,20 @@ void __clear_cache(void *start, void *end) { #else compilerrt_abort(); #endif -#elif defined(__ANDROID__) && defined(__mips__) +#elif defined(__mips__) && !defined(__FreeBSD__) const uintptr_t start_int = (uintptr_t) start; const uintptr_t end_int = (uintptr_t) end; - _flush_cache(start, (end_int - start_int), BCACHE); + #if defined(__ANDROID__) && defined(__LP64__) + // Call synci implementation for short address range. + const uintptr_t address_range_limit = 256; + if ((end_int - start_int) <= address_range_limit) { + clear_mips_cache(start, (end_int - start_int)); + } else { + syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); + } + #else + syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE); + #endif #elif defined(__aarch64__) && !defined(__APPLE__) uint64_t xstart = (uint64_t)(uintptr_t) start; uint64_t xend = (uint64_t)(uintptr_t) end; diff --git a/contrib/compiler-rt/lib/builtins/fixunssfti.c b/contrib/compiler-rt/lib/builtins/fixunssfti.c index 9593153745c5..862d7bd6c7af 100644 --- a/contrib/compiler-rt/lib/builtins/fixunssfti.c +++ b/contrib/compiler-rt/lib/builtins/fixunssfti.c @@ -12,6 +12,9 @@ * ===----------------------------------------------------------------------=== */ +#define SINGLE_PRECISION +#include "fp_lib.h" + #if defined(CRT_HAS_128BIT) typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" diff --git a/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c b/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c index 869f4178e853..4b95cfd43b05 100644 --- a/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c +++ b/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c @@ -11,47 +11,7 @@ #include "int_lib.h" -/* - * _Unwind_* stuff based on C++ ABI public documentation - * http://refspecs.freestandards.org/abi-eh-1.21.html - */ - -typedef enum { - _URC_NO_REASON = 0, - _URC_FOREIGN_EXCEPTION_CAUGHT = 1, - _URC_FATAL_PHASE2_ERROR = 2, - _URC_FATAL_PHASE1_ERROR = 3, - _URC_NORMAL_STOP = 4, - _URC_END_OF_STACK = 5, - _URC_HANDLER_FOUND = 6, - _URC_INSTALL_CONTEXT = 7, - _URC_CONTINUE_UNWIND = 8 -} _Unwind_Reason_Code; - -typedef enum { - _UA_SEARCH_PHASE = 1, - _UA_CLEANUP_PHASE = 2, - _UA_HANDLER_FRAME = 4, - _UA_FORCE_UNWIND = 8, - _UA_END_OF_STACK = 16 -} _Unwind_Action; - -typedef struct _Unwind_Context* _Unwind_Context_t; - -struct _Unwind_Exception { - uint64_t exception_class; - void (*exception_cleanup)(_Unwind_Reason_Code reason, - struct _Unwind_Exception* exc); - uintptr_t private_1; - uintptr_t private_2; -}; - -COMPILER_RT_ABI const uint8_t* _Unwind_GetLanguageSpecificData(_Unwind_Context_t c); -COMPILER_RT_ABI void _Unwind_SetGR(_Unwind_Context_t c, int i, uintptr_t n); -COMPILER_RT_ABI void _Unwind_SetIP(_Unwind_Context_t, uintptr_t new_value); -COMPILER_RT_ABI uintptr_t _Unwind_GetIP(_Unwind_Context_t context); -COMPILER_RT_ABI uintptr_t _Unwind_GetRegionStart(_Unwind_Context_t context); - +#include /* * Pointer encodings documented at: @@ -185,12 +145,12 @@ static uintptr_t readEncodedPointer(const uint8_t** data, uint8_t encoding) COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_sj0(int version, _Unwind_Action actions, uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, - _Unwind_Context_t context) + struct _Unwind_Context *context) #else COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(int version, _Unwind_Action actions, uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, - _Unwind_Context_t context) + struct _Unwind_Context *context) #endif { /* Since C does not have catch clauses, there is nothing to do during */ @@ -199,7 +159,7 @@ __gcc_personality_v0(int version, _Unwind_Action actions, return _URC_CONTINUE_UNWIND; /* There is nothing to do if there is no LSDA for this frame. */ - const uint8_t* lsda = _Unwind_GetLanguageSpecificData(context); + const uint8_t* lsda = (uint8_t*)_Unwind_GetLanguageSpecificData(context); if ( lsda == (uint8_t*) 0 ) return _URC_CONTINUE_UNWIND; diff --git a/contrib/compiler-rt/lib/builtins/int_types.h b/contrib/compiler-rt/lib/builtins/int_types.h index 5107f71550ac..aedae14b2046 100644 --- a/contrib/compiler-rt/lib/builtins/int_types.h +++ b/contrib/compiler-rt/lib/builtins/int_types.h @@ -56,7 +56,8 @@ typedef union }s; } udwords; -#if __LP64__ +/* MIPS64 issue: PR 20098 */ +#if defined(__LP64__) && !(defined(__mips__) && defined(__clang__)) #define CRT_HAS_128BIT #endif diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.cc b/contrib/compiler-rt/lib/dfsan/dfsan.cc index 941edc5eba7a..de5b2ce107b4 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan.cc +++ b/contrib/compiler-rt/lib/dfsan/dfsan.cc @@ -22,6 +22,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_libc.h" #include "dfsan/dfsan.h" @@ -256,7 +257,7 @@ dfsan_read_label(const void *addr, uptr size) { return __dfsan_union_load(shadow_for(addr), size); } -SANITIZER_INTERFACE_ATTRIBUTE +extern "C" SANITIZER_INTERFACE_ATTRIBUTE const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) { return &__dfsan_label_info[label]; } @@ -310,16 +311,24 @@ dfsan_dump_labels(int fd) { } } -static void InitializeFlags(Flags &f, const char *env) { - f.warn_unimplemented = true; - f.warn_nonzero_labels = false; - f.strict_data_dependencies = true; - f.dump_labels_at_exit = ""; +void Flags::SetDefaults() { +#define DFSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "dfsan_flags.inc" +#undef DFSAN_FLAG +} - ParseFlag(env, &f.warn_unimplemented, "warn_unimplemented", ""); - ParseFlag(env, &f.warn_nonzero_labels, "warn_nonzero_labels", ""); - ParseFlag(env, &f.strict_data_dependencies, "strict_data_dependencies", ""); - ParseFlag(env, &f.dump_labels_at_exit, "dump_labels_at_exit", ""); +static void RegisterDfsanFlags(FlagParser *parser, Flags *f) { +#define DFSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "dfsan_flags.inc" +#undef DFSAN_FLAG +} + +static void InitializeFlags() { + FlagParser parser; + RegisterDfsanFlags(&parser, &flags()); + flags().SetDefaults(); + parser.ParseString(GetEnv("DFSAN_OPTIONS")); } static void dfsan_fini() { @@ -354,8 +363,7 @@ static void dfsan_init(int argc, char **argv, char **envp) { if (!(init_addr >= kUnusedAddr && init_addr < kAppAddr)) Mprotect(kUnusedAddr, kAppAddr - kUnusedAddr); - InitializeFlags(flags(), GetEnv("DFSAN_OPTIONS")); - + InitializeFlags(); InitializeInterceptors(); // Register the fini callback to run when the program terminates successfully diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.h b/contrib/compiler-rt/lib/dfsan/dfsan.h index bc38be08c9cd..ceba3533a233 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan.h +++ b/contrib/compiler-rt/lib/dfsan/dfsan.h @@ -56,17 +56,11 @@ inline const dfsan_label *shadow_for(const void *ptr) { } struct Flags { - // Whether to warn on unimplemented functions. - bool warn_unimplemented; - // Whether to warn on non-zero labels. - bool warn_nonzero_labels; - // Whether to propagate labels only when there is an obvious data dependency - // (e.g., when comparing strings, ignore the fact that the output of the - // comparison might be data-dependent on the content of the strings). This - // applies only to the custom functions defined in 'custom.c'. - bool strict_data_dependencies; - // The path of the file where to dump the labels when the program terminates. - const char* dump_labels_at_exit; +#define DFSAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "dfsan_flags.inc" +#undef DFSAN_FLAG + + void SetDefaults(); }; extern Flags flags_data; diff --git a/contrib/compiler-rt/lib/dfsan/dfsan_custom.cc b/contrib/compiler-rt/lib/dfsan/dfsan_custom.cc index 839a399faae0..318ecd6fb317 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan_custom.cc +++ b/contrib/compiler-rt/lib/dfsan/dfsan_custom.cc @@ -314,11 +314,12 @@ static void unpoison(const void *ptr, uptr size) { SANITIZER_INTERFACE_ATTRIBUTE void * __dfsw_dlopen(const char *filename, int flag, dfsan_label filename_label, dfsan_label flag_label, dfsan_label *ret_label) { - link_map *map = (link_map *)dlopen(filename, flag); + void *handle = dlopen(filename, flag); + link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE(handle); if (map) ForEachMappedRegion(map, unpoison); *ret_label = 0; - return (void *)map; + return handle; } struct pthread_create_info { diff --git a/contrib/compiler-rt/lib/dfsan/dfsan_flags.inc b/contrib/compiler-rt/lib/dfsan/dfsan_flags.inc new file mode 100644 index 000000000000..24fbfcb9e46f --- /dev/null +++ b/contrib/compiler-rt/lib/dfsan/dfsan_flags.inc @@ -0,0 +1,32 @@ +//===-- dfsan_flags.inc -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// DFSan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef DFSAN_FLAG +# error "Define DFSAN_FLAG prior to including this file!" +#endif + +// DFSAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +DFSAN_FLAG(bool, warn_unimplemented, true, + "Whether to warn on unimplemented functions.") +DFSAN_FLAG(bool, warn_nonzero_labels, false, + "Whether to warn on unimplemented functions.") +DFSAN_FLAG( + bool, strict_data_dependencies, true, + "Whether to propagate labels only when there is an obvious data dependency" + "(e.g., when comparing strings, ignore the fact that the output of the" + "comparison might be data-dependent on the content of the strings). This" + "applies only to the custom functions defined in 'custom.c'.") +DFSAN_FLAG(const char *, dump_labels_at_exit, "", "The path of the file where " + "to dump the labels when the " + "program terminates.") diff --git a/contrib/compiler-rt/lib/lsan/lsan.cc b/contrib/compiler-rt/lib/lsan/lsan.cc index 1598fcac4a76..6018f7bf6f49 100644 --- a/contrib/compiler-rt/lib/lsan/lsan.cc +++ b/contrib/compiler-rt/lib/lsan/lsan.cc @@ -15,6 +15,7 @@ #include "lsan.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "lsan_allocator.h" #include "lsan_common.h" @@ -34,13 +35,42 @@ bool WordIsPoisoned(uptr addr) { using namespace __lsan; // NOLINT +static void InitializeFlags() { + // Set all the default values. + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("LSAN_SYMBOLIZER_PATH"); + cf.malloc_context_size = 30; + cf.detect_leaks = true; + OverrideCommonFlags(cf); + } + + Flags *f = flags(); + f->SetDefaults(); + + FlagParser parser; + RegisterLsanFlags(&parser, f); + RegisterCommonFlags(&parser); + + parser.ParseString(GetEnv("LSAN_OPTIONS")); + + SetVerbosity(common_flags()->verbosity); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); +} + extern "C" void __lsan_init() { CHECK(!lsan_init_is_running); if (lsan_inited) return; lsan_init_is_running = true; SanitizerToolName = "LeakSanitizer"; - InitCommonLsan(true); + InitializeFlags(); + InitCommonLsan(); InitializeAllocator(); InitTlsSize(); InitializeInterceptors(); @@ -52,6 +82,9 @@ extern "C" void __lsan_init() { if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) Atexit(DoLeakCheck); + + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); + lsan_inited = true; lsan_init_is_running = false; } diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.cc b/contrib/compiler-rt/lib/lsan/lsan_allocator.cc index 8be2a2ad9224..67125dbb3e45 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.cc @@ -25,10 +25,6 @@ extern "C" void *memset(void *ptr, int value, uptr num); namespace __lsan { -static const uptr kMaxAllowedMallocSize = 8UL << 30; -static const uptr kAllocatorSpace = 0x600000000000ULL; -static const uptr kAllocatorSize = 0x40000000000ULL; // 4T. - struct ChunkMetadata { bool allocated : 8; // Must be first. ChunkTag tag : 2; @@ -36,8 +32,22 @@ struct ChunkMetadata { u32 stack_trace_id; }; +#if defined(__mips64) +static const uptr kMaxAllowedMallocSize = 4UL << 30; +static const uptr kRegionSizeLog = 20; +static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; +typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; +typedef CompactSizeClassMap SizeClassMap; +typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, + sizeof(ChunkMetadata), SizeClassMap, kRegionSizeLog, ByteMap> + PrimaryAllocator; +#else +static const uptr kMaxAllowedMallocSize = 8UL << 30; +static const uptr kAllocatorSpace = 0x600000000000ULL; +static const uptr kAllocatorSize = 0x40000000000ULL; // 4T. typedef SizeClassAllocator64 PrimaryAllocator; +#endif typedef SizeClassAllocatorLocalCache AllocatorCache; typedef LargeMmapAllocator<> SecondaryAllocator; typedef CombinedAllocatorallocator_may_return_null); } void AllocatorThreadFinish() { diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.cc b/contrib/compiler-rt/lib/lsan/lsan_common.cc index c2ba52e46911..b9e2a1104326 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common.cc @@ -16,6 +16,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_procmaps.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -36,52 +37,17 @@ bool DisabledInThisThread() { return disable_counter > 0; } Flags lsan_flags; -static void InitializeFlags(bool standalone) { - Flags *f = flags(); - // Default values. - f->report_objects = false; - f->resolution = 0; - f->max_leaks = 0; - f->exitcode = 23; - f->use_registers = true; - f->use_globals = true; - f->use_stacks = true; - f->use_tls = true; - f->use_root_regions = true; - f->use_unaligned = false; - f->use_poisoned = false; - f->log_pointers = false; - f->log_threads = false; +void Flags::SetDefaults() { +#define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "lsan_flags.inc" +#undef LSAN_FLAG +} - const char *options = GetEnv("LSAN_OPTIONS"); - if (options) { - ParseFlag(options, &f->use_registers, "use_registers", ""); - ParseFlag(options, &f->use_globals, "use_globals", ""); - ParseFlag(options, &f->use_stacks, "use_stacks", ""); - ParseFlag(options, &f->use_tls, "use_tls", ""); - ParseFlag(options, &f->use_root_regions, "use_root_regions", ""); - ParseFlag(options, &f->use_unaligned, "use_unaligned", ""); - ParseFlag(options, &f->use_poisoned, "use_poisoned", ""); - ParseFlag(options, &f->report_objects, "report_objects", ""); - ParseFlag(options, &f->resolution, "resolution", ""); - CHECK_GE(&f->resolution, 0); - ParseFlag(options, &f->max_leaks, "max_leaks", ""); - CHECK_GE(&f->max_leaks, 0); - ParseFlag(options, &f->log_pointers, "log_pointers", ""); - ParseFlag(options, &f->log_threads, "log_threads", ""); - ParseFlag(options, &f->exitcode, "exitcode", ""); - } - - // Set defaults for common flags (only in standalone mode) and parse - // them from LSAN_OPTIONS. - CommonFlags *cf = common_flags(); - if (standalone) { - SetCommonFlagsDefaults(cf); - cf->external_symbolizer_path = GetEnv("LSAN_SYMBOLIZER_PATH"); - cf->malloc_context_size = 30; - cf->detect_leaks = true; - } - ParseCommonFlagsFromString(cf, options); +void RegisterLsanFlags(FlagParser *parser, Flags *f) { +#define LSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "lsan_flags.inc" +#undef LSAN_FLAG } #define LOG_POINTERS(...) \ @@ -94,14 +60,23 @@ static void InitializeFlags(bool standalone) { if (flags()->log_threads) Report(__VA_ARGS__); \ } while (0); -static bool suppressions_inited = false; +ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; +static SuppressionContext *suppression_ctx = nullptr; +static const char kSuppressionLeak[] = "leak"; +static const char *kSuppressionTypes[] = { kSuppressionLeak }; void InitializeSuppressions() { - CHECK(!suppressions_inited); - SuppressionContext::InitIfNecessary(); + CHECK_EQ(nullptr, suppression_ctx); + suppression_ctx = new (suppression_placeholder) // NOLINT + SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes)); + suppression_ctx->ParseFromFile(flags()->suppressions); if (&__lsan_default_suppressions) - SuppressionContext::Get()->Parse(__lsan_default_suppressions()); - suppressions_inited = true; + suppression_ctx->Parse(__lsan_default_suppressions()); +} + +static SuppressionContext *GetSuppressionContext() { + CHECK(suppression_ctx); + return suppression_ctx; } struct RootRegion { @@ -117,8 +92,7 @@ void InitializeRootRegions() { root_regions = new(placeholder) InternalMmapVector(1); } -void InitCommonLsan(bool standalone) { - InitializeFlags(standalone); +void InitCommonLsan() { InitializeRootRegions(); if (common_flags()->detect_leaks) { // Initialization which can fail or print warnings should only be done if @@ -141,9 +115,11 @@ static inline bool CanBeAHeapPointer(uptr p) { // bound on heap addresses. const uptr kMinAddress = 4 * 4096; if (p < kMinAddress) return false; -#ifdef __x86_64__ +#if defined(__x86_64__) // Accept only canonical form user-space addresses. return ((p >> 47) == 0); +#elif defined(__mips64) + return ((p >> 40) == 0); #else return true; #endif @@ -367,7 +343,7 @@ static void CollectLeaksCb(uptr chunk, void *arg) { LsanMetadata m(chunk); if (!m.allocated()) return; if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked) { - uptr resolution = flags()->resolution; + u32 resolution = flags()->resolution; u32 stack_trace_id = 0; if (resolution > 0) { StackTrace stack = StackDepotGet(m.stack_trace_id()); @@ -383,7 +359,7 @@ static void CollectLeaksCb(uptr chunk, void *arg) { static void PrintMatchedSuppressions() { InternalMmapVector matched(1); - SuppressionContext::Get()->GetMatched(&matched); + GetSuppressionContext()->GetMatched(&matched); if (!matched.size()) return; const char *line = "-----------------------------------------------------"; @@ -462,17 +438,17 @@ static Suppression *GetSuppressionForAddr(uptr addr) { // Suppress by module name. const char *module_name; uptr module_offset; + SuppressionContext *suppressions = GetSuppressionContext(); if (Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(addr, &module_name, &module_offset) && - SuppressionContext::Get()->Match(module_name, SuppressionLeak, &s)) + suppressions->Match(module_name, kSuppressionLeak, &s)) return s; // Suppress by file or function name. SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr); for (SymbolizedStack *cur = frames; cur; cur = cur->next) { - if (SuppressionContext::Get()->Match(cur->info.function, SuppressionLeak, - &s) || - SuppressionContext::Get()->Match(cur->info.file, SuppressionLeak, &s)) { + if (suppressions->Match(cur->info.function, kSuppressionLeak, &s) || + suppressions->Match(cur->info.file, kSuppressionLeak, &s)) { break; } } diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.h b/contrib/compiler-rt/lib/lsan/lsan_common.h index 86ff12da6e0f..2c3a12ab6bd8 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.h +++ b/contrib/compiler-rt/lib/lsan/lsan_common.h @@ -21,12 +21,17 @@ #include "sanitizer_common/sanitizer_platform.h" #include "sanitizer_common/sanitizer_symbolizer.h" -#if SANITIZER_LINUX && defined(__x86_64__) && (SANITIZER_WORDSIZE == 64) +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips64)) \ + && (SANITIZER_WORDSIZE == 64) #define CAN_SANITIZE_LEAKS 1 #else #define CAN_SANITIZE_LEAKS 0 #endif +namespace __sanitizer { +class FlagParser; +} + namespace __lsan { // Chunk tags. @@ -38,44 +43,19 @@ enum ChunkTag { }; struct Flags { +#define LSAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "lsan_flags.inc" +#undef LSAN_FLAG + + void SetDefaults(); uptr pointer_alignment() const { return use_unaligned ? 1 : sizeof(uptr); } - - // Print addresses of leaked objects after main leak report. - bool report_objects; - // Aggregate two objects into one leak if this many stack frames match. If - // zero, the entire stack trace must match. - int resolution; - // The number of leaks reported. - int max_leaks; - // If nonzero kill the process with this exit code upon finding leaks. - int exitcode; - - // Flags controlling the root set of reachable memory. - // Global variables (.data and .bss). - bool use_globals; - // Thread stacks. - bool use_stacks; - // Thread registers. - bool use_registers; - // TLS and thread-specific storage. - bool use_tls; - // Regions added via __lsan_register_root_region(). - bool use_root_regions; - - // Consider unaligned pointers valid. - bool use_unaligned; - // Consider pointers found in poisoned memory to be valid. - bool use_poisoned; - - // Debug logging. - bool log_pointers; - bool log_threads; }; extern Flags lsan_flags; inline Flags *flags() { return &lsan_flags; } +void RegisterLsanFlags(FlagParser *parser, Flags *f); struct Leak { u32 id; @@ -131,7 +111,7 @@ enum IgnoreObjectResult { }; // Functions called from the parent tool. -void InitCommonLsan(bool standalone); +void InitCommonLsan(); void DoLeakCheck(); bool DisabledInThisThread(); diff --git a/contrib/compiler-rt/lib/lsan/lsan_flags.inc b/contrib/compiler-rt/lib/lsan/lsan_flags.inc new file mode 100644 index 000000000000..b19b3452b2fc --- /dev/null +++ b/contrib/compiler-rt/lib/lsan/lsan_flags.inc @@ -0,0 +1,45 @@ +//===-- lsan_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// LSan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef LSAN_FLAG +# error "Define LSAN_FLAG prior to including this file!" +#endif + +// LSAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +LSAN_FLAG(bool, report_objects, false, + "Print addresses of leaked objects after main leak report.") +LSAN_FLAG( + int, resolution, 0, + "Aggregate two objects into one leak if this many stack frames match. If " + "zero, the entire stack trace must match.") +LSAN_FLAG(int, max_leaks, 0, "The number of leaks reported.") +LSAN_FLAG(int, exitcode, 23, + "If nonzero kill the process with this exit code upon finding leaks.") + +// Flags controlling the root set of reachable memory. +LSAN_FLAG(bool, use_globals, true, + "Root set: include global variables (.data and .bss)") +LSAN_FLAG(bool, use_stacks, true, "Root set: include thread stacks") +LSAN_FLAG(bool, use_registers, true, "Root set: include thread registers") +LSAN_FLAG(bool, use_tls, true, + "Root set: include TLS and thread-specific storage") +LSAN_FLAG(bool, use_root_regions, true, + "Root set: include regions added via __lsan_register_root_region().") + +LSAN_FLAG(bool, use_unaligned, false, "Consider unaligned pointers valid.") +LSAN_FLAG(bool, use_poisoned, false, + "Consider pointers found in poisoned memory to be valid.") +LSAN_FLAG(bool, log_pointers, false, "Debug logging") +LSAN_FLAG(bool, log_threads, false, "Debug logging") +LSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") diff --git a/contrib/compiler-rt/lib/msan/msan.cc b/contrib/compiler-rt/lib/msan/msan.cc index 853e448fab8e..ed6efbdd682f 100644 --- a/contrib/compiler-rt/lib/msan/msan.cc +++ b/contrib/compiler-rt/lib/msan/msan.cc @@ -16,16 +16,17 @@ #include "msan_chained_origin_depot.h" #include "msan_origin.h" #include "msan_thread.h" +#include "msan_poisoning.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_procmaps.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_symbolizer.h" #include "sanitizer_common/sanitizer_stackdepot.h" - // ACHTUNG! No system header includes in this file. using namespace __sanitizer; @@ -96,19 +97,81 @@ static const char *StackOriginDescr[kNumStackOriginDescrs]; static uptr StackOriginPC[kNumStackOriginDescrs]; static atomic_uint32_t NumStackOriginDescrs; -static void ParseFlagsFromString(Flags *f, const char *str) { - CommonFlags *cf = common_flags(); - ParseCommonFlagsFromString(cf, str); - ParseFlag(str, &f->poison_heap_with_zeroes, "poison_heap_with_zeroes", ""); - ParseFlag(str, &f->poison_stack_with_zeroes, "poison_stack_with_zeroes", ""); - ParseFlag(str, &f->poison_in_malloc, "poison_in_malloc", ""); - ParseFlag(str, &f->poison_in_free, "poison_in_free", ""); - ParseFlag(str, &f->exit_code, "exit_code", ""); +void Flags::SetDefaults() { +#define MSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "msan_flags.inc" +#undef MSAN_FLAG +} + +// keep_going is an old name for halt_on_error, +// and it has inverse meaning. +class FlagHandlerKeepGoing : public FlagHandlerBase { + bool *halt_on_error_; + + public: + explicit FlagHandlerKeepGoing(bool *halt_on_error) + : halt_on_error_(halt_on_error) {} + bool Parse(const char *value) final { + bool tmp; + FlagHandler h(&tmp); + if (!h.Parse(value)) return false; + *halt_on_error_ = !tmp; + return true; + } +}; + +static void RegisterMsanFlags(FlagParser *parser, Flags *f) { +#define MSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "msan_flags.inc" +#undef MSAN_FLAG + + FlagHandlerKeepGoing *fh_keep_going = new (FlagParser::Alloc) // NOLINT + FlagHandlerKeepGoing(&f->halt_on_error); + parser->RegisterHandler("keep_going", fh_keep_going, + "deprecated, use halt_on_error"); +} + +static void InitializeFlags() { + Flags *f = flags(); + FlagParser parser; + RegisterMsanFlags(&parser, f); + RegisterCommonFlags(&parser); + + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("MSAN_SYMBOLIZER_PATH"); + cf.malloc_context_size = 20; + cf.handle_ioctl = true; + // FIXME: test and enable. + cf.check_printf = false; + cf.intercept_tls_get_addr = true; + OverrideCommonFlags(cf); + } + + f->SetDefaults(); + + // Override from user-specified string. + if (__msan_default_options) + parser.ParseString(__msan_default_options()); + + const char *msan_options = GetEnv("MSAN_OPTIONS"); + parser.ParseString(msan_options); + VPrintf(1, "MSAN_OPTIONS: %s\n", msan_options ? msan_options : ""); + + SetVerbosity(common_flags()->verbosity); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); + + // Check flag values: if (f->exit_code < 0 || f->exit_code > 127) { Printf("Exit code not in [0, 128) range: %d\n", f->exit_code); Die(); } - ParseFlag(str, &f->origin_history_size, "origin_history_size", ""); if (f->origin_history_size < 0 || f->origin_history_size > Origin::kMaxDepth) { Printf( @@ -117,8 +180,6 @@ static void ParseFlagsFromString(Flags *f, const char *str) { f->origin_history_size, Origin::kMaxDepth); Die(); } - ParseFlag(str, &f->origin_history_per_stack_limit, - "origin_history_per_stack_limit", ""); // Limiting to kStackDepotMaxUseCount / 2 to avoid overflow in // StackDepotHandle::inc_use_count_unsafe. if (f->origin_history_per_stack_limit < 0 || @@ -129,51 +190,7 @@ static void ParseFlagsFromString(Flags *f, const char *str) { f->origin_history_per_stack_limit, kStackDepotMaxUseCount / 2); Die(); } - - ParseFlag(str, &f->report_umrs, "report_umrs", ""); - ParseFlag(str, &f->wrap_signals, "wrap_signals", ""); - ParseFlag(str, &f->print_stats, "print_stats", ""); - ParseFlag(str, &f->atexit, "atexit", ""); - ParseFlag(str, &f->store_context_size, "store_context_size", ""); if (f->store_context_size < 1) f->store_context_size = 1; - - // keep_going is an old name for halt_on_error, - // and it has inverse meaning. - f->halt_on_error = !f->halt_on_error; - ParseFlag(str, &f->halt_on_error, "keep_going", ""); - f->halt_on_error = !f->halt_on_error; - ParseFlag(str, &f->halt_on_error, "halt_on_error", ""); -} - -static void InitializeFlags(Flags *f, const char *options) { - CommonFlags *cf = common_flags(); - SetCommonFlagsDefaults(cf); - cf->external_symbolizer_path = GetEnv("MSAN_SYMBOLIZER_PATH"); - cf->malloc_context_size = 20; - cf->handle_ioctl = true; - // FIXME: test and enable. - cf->check_printf = false; - cf->intercept_tls_get_addr = true; - - internal_memset(f, 0, sizeof(*f)); - f->poison_heap_with_zeroes = false; - f->poison_stack_with_zeroes = false; - f->poison_in_malloc = true; - f->poison_in_free = true; - f->exit_code = 77; - f->origin_history_size = Origin::kMaxDepth; - f->origin_history_per_stack_limit = 20000; - f->report_umrs = true; - f->wrap_signals = true; - f->print_stats = false; - f->atexit = false; - f->halt_on_error = !&__msan_keep_going; - f->store_context_size = 20; - - // Override from user-specified string. - if (__msan_default_options) - ParseFlagsFromString(f, __msan_default_options()); - ParseFlagsFromString(f, options); } void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, @@ -259,6 +276,7 @@ u32 ChainOrigin(u32 id, StackTrace *stack) { return id; Origin o = Origin::FromRawId(id); + stack->tag = StackTrace::TAG_UNKNOWN; Origin chained = Origin::CreateChainedOrigin(o, stack); return chained.raw_id(); } @@ -336,9 +354,7 @@ void __msan_init() { SetDieCallback(MsanDie); InitTlsSize(); - const char *msan_options = GetEnv("MSAN_OPTIONS"); - InitializeFlags(&msan_flags, msan_options); - if (common_flags()->help) PrintFlagDescriptions(); + InitializeFlags(); __sanitizer_set_report_path(common_flags()->log_path); InitializeInterceptors(); @@ -355,8 +371,6 @@ void __msan_init() { ReExec(); } - VPrintf(1, "MSAN_OPTIONS: %s\n", msan_options ? msan_options : ""); - __msan_clear_on_return(); if (__msan_get_track_origins()) VPrintf(1, "msan_track_origins\n"); @@ -372,10 +386,7 @@ void __msan_init() { Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); - if (common_flags()->coverage) { - __sanitizer_cov_init(); - Atexit(__sanitizer_cov_dump); - } + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); MsanTSDInit(MsanTSDDtor); @@ -485,24 +496,7 @@ void __msan_load_unpoisoned(void *src, uptr size, void *dst) { } void __msan_set_origin(const void *a, uptr size, u32 origin) { - // Origin mapping is 4 bytes per 4 bytes of application memory. - // Here we extend the range such that its left and right bounds are both - // 4 byte aligned. - if (!__msan_get_track_origins()) return; - uptr x = MEM_TO_ORIGIN((uptr)a); - uptr beg = x & ~3UL; // align down. - uptr end = (x + size + 3) & ~3UL; // align up. - u64 origin64 = ((u64)origin << 32) | origin; - // This is like memset, but the value is 32-bit. We unroll by 2 to write - // 64 bits at once. May want to unroll further to get 128-bit stores. - if (beg & 7ULL) { - *(u32*)beg = origin; - beg += 4; - } - for (uptr addr = beg; addr < (end & ~7UL); addr += 8) - *(u64*)addr = origin64; - if (end & 7ULL) - *(u32*)(end - 4) = origin; + if (__msan_get_track_origins()) SetOrigin(a, size, origin); } // 'descr' is created at compile time and contains '----' in the beginning. diff --git a/contrib/compiler-rt/lib/msan/msan.h b/contrib/compiler-rt/lib/msan/msan.h index 9bc1e4c6a447..ed18f21d0282 100644 --- a/contrib/compiler-rt/lib/msan/msan.h +++ b/contrib/compiler-rt/lib/msan/msan.h @@ -25,90 +25,90 @@ # define MSAN_REPLACE_OPERATORS_NEW_AND_DELETE 1 #endif -/* -C/C++ on FreeBSD -0000 0000 0000 - 00ff ffff ffff: Low memory: main binary, MAP_32BIT mappings and modules -0100 0000 0000 - 0fff ffff ffff: Bad1 -1000 0000 0000 - 30ff ffff ffff: Shadow -3100 0000 0000 - 37ff ffff ffff: Bad2 -3800 0000 0000 - 58ff ffff ffff: Origins -5900 0000 0000 - 5fff ffff ffff: Bad3 -6000 0000 0000 - 7fff ffff ffff: High memory: heap, modules and main thread stack +struct MappingDesc { + uptr start; + uptr end; + enum Type { + INVALID, APP, SHADOW, ORIGIN + } type; + const char *name; +}; -C/C++ on Linux/PIE -0000 0000 0000 - 1fff ffff ffff: Bad1 -2000 0000 0000 - 3fff ffff ffff: Shadow -4000 0000 0000 - 5fff ffff ffff: Origins -6000 0000 0000 - 7fff ffff ffff: Main memory - -C/C++ on Mips -0000 0000 0000 - 009f ffff ffff: Bad1 -00a0 0000 0000 - 00bf ffff ffff: Shadow -00c0 0000 0000 - 00df ffff ffff: Origins -00e0 0000 0000 - 00ff ffff ffff: Main memory -*/ #if SANITIZER_LINUX && defined(__mips64) -const uptr kLowMemBeg = 0; -const uptr kLowMemSize = 0; -const uptr kHighMemBeg = 0x00e000000000; -const uptr kHighMemSize = 0x002000000000; -const uptr kShadowBeg = 0x00a000000000; -const uptr kShadowSize = 0x002000000000; -const uptr kOriginsBeg = 0x00c000000000; -# define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x4000000000ULL) + +// Everything is above 0x00e000000000. +const MappingDesc kMemoryLayout[] = { + {0x000000000000ULL, 0x00a000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x00a000000000ULL, 0x00c000000000ULL, MappingDesc::SHADOW, "shadow"}, + {0x00c000000000ULL, 0x00e000000000ULL, MappingDesc::ORIGIN, "origin"}, + {0x00e000000000ULL, 0x010000000000ULL, MappingDesc::APP, "app"}}; + +#define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x4000000000ULL) +#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x002000000000) + #elif SANITIZER_FREEBSD && SANITIZER_WORDSIZE == 64 -const uptr kLowMemBeg = 0x000000000000; -const uptr kLowMemSize = 0x010000000000; -const uptr kHighMemBeg = 0x600000000000; -const uptr kHighMemSize = 0x200000000000; -const uptr kShadowBeg = 0x100000000000; -const uptr kShadowSize = 0x210000000000; -const uptr kOriginsBeg = 0x380000000000; + +// Low memory: main binary, MAP_32BIT mappings and modules +// High memory: heap, modules and main thread stack +const MappingDesc kMemoryLayout[] = { + {0x000000000000ULL, 0x010000000000ULL, MappingDesc::APP, "low memory"}, + {0x010000000000ULL, 0x100000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x100000000000ULL, 0x310000000000ULL, MappingDesc::SHADOW, "shadow"}, + {0x310000000000ULL, 0x380000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x380000000000ULL, 0x590000000000ULL, MappingDesc::ORIGIN, "origin"}, + {0x590000000000ULL, 0x600000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x600000000000ULL, 0x800000000000ULL, MappingDesc::APP, "high memory"}}; + // Maps low and high app ranges to contiguous space with zero base: // Low: 0000 0000 0000 - 00ff ffff ffff -> 2000 0000 0000 - 20ff ffff ffff // High: 6000 0000 0000 - 7fff ffff ffff -> 0000 0000 0000 - 1fff ffff ffff -# define LINEARIZE_MEM(mem) \ - (((uptr)(mem) & ~0xc00000000000ULL) ^ 0x200000000000ULL) -# define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x100000000000ULL) +#define LINEARIZE_MEM(mem) \ + (((uptr)(mem) & ~0xc00000000000ULL) ^ 0x200000000000ULL) +#define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x100000000000ULL) +#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x280000000000) + #elif SANITIZER_LINUX && SANITIZER_WORDSIZE == 64 -const uptr kLowMemBeg = 0; -const uptr kLowMemSize = 0; -const uptr kHighMemBeg = 0x600000000000; -const uptr kHighMemSize = 0x200000000000; -const uptr kShadowBeg = 0x200000000000; -const uptr kShadowSize = 0x200000000000; -const uptr kOriginsBeg = 0x400000000000; -# define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x400000000000ULL) + +// Requries PIE binary and ASLR enabled. +// Main thread stack and DSOs at 0x7f0000000000 (sometimes 0x7e0000000000). +// Heap at 0x600000000000. +const MappingDesc kMemoryLayout[] = { + {0x000000000000ULL, 0x200000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x200000000000ULL, 0x400000000000ULL, MappingDesc::SHADOW, "shadow"}, + {0x400000000000ULL, 0x600000000000ULL, MappingDesc::ORIGIN, "origin"}, + {0x600000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app"}}; + +#define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x400000000000ULL) +#define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x200000000000ULL) + #else #error "Unsupported platform" #endif -const uptr kBad1Beg = kLowMemBeg + kLowMemSize; -const uptr kBad1Size = kShadowBeg - kBad1Beg; - -const uptr kBad2Beg = kShadowBeg + kShadowSize; -const uptr kBad2Size = kOriginsBeg - kBad2Beg; - -const uptr kOriginsSize = kShadowSize; - -const uptr kBad3Beg = kOriginsBeg + kOriginsSize; -const uptr kBad3Size = kHighMemBeg - kBad3Beg; - -#define SHADOW_TO_ORIGIN(shadow) \ - (((uptr)(shadow)) + (kOriginsBeg - kShadowBeg)) +const uptr kMemoryLayoutSize = sizeof(kMemoryLayout) / sizeof(kMemoryLayout[0]); #define MEM_TO_ORIGIN(mem) (SHADOW_TO_ORIGIN(MEM_TO_SHADOW((mem)))) -#define MEM_IS_APP(mem) \ - ((kLowMemSize > 0 && (uptr)(mem) < kLowMemSize) || \ - (uptr)(mem) >= kHighMemBeg) +#ifndef __clang__ +__attribute__((optimize("unroll-loops"))) +#endif +inline bool addr_is_type(uptr addr, MappingDesc::Type mapping_type) { +// It is critical for performance that this loop is unrolled (because then it is +// simplified into just a few constant comparisons). +#ifdef __clang__ +#pragma unroll +#endif + for (unsigned i = 0; i < kMemoryLayoutSize; ++i) + if (kMemoryLayout[i].type == mapping_type && + addr >= kMemoryLayout[i].start && addr < kMemoryLayout[i].end) + return true; + return false; +} -#define MEM_IS_SHADOW(mem) \ - ((uptr)(mem) >= kShadowBeg && (uptr)(mem) < kShadowBeg + kShadowSize) - -#define MEM_IS_ORIGIN(mem) \ - ((uptr)(mem) >= kOriginsBeg && (uptr)(mem) < kOriginsBeg + kOriginsSize) +#define MEM_IS_APP(mem) addr_is_type((uptr)(mem), MappingDesc::APP) +#define MEM_IS_SHADOW(mem) addr_is_type((uptr)(mem), MappingDesc::SHADOW) +#define MEM_IS_ORIGIN(mem) addr_is_type((uptr)(mem), MappingDesc::ORIGIN) // These constants must be kept in sync with the ones in MemorySanitizer.cc. const int kMsanParamTlsSize = 800; @@ -125,6 +125,7 @@ char *GetProcSelfMaps(); void InitializeInterceptors(); void MsanAllocatorThreadFinish(); +void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size); void *MsanReallocate(StackTrace *stack, void *oldp, uptr size, uptr alignment, bool zeroise); void MsanDeallocate(StackTrace *stack, void *ptr); @@ -162,16 +163,12 @@ void ReportUMRInsideAddressRange(const char *what, const void *start, uptr size, void UnpoisonParam(uptr n); void UnpoisonThreadLocalState(); -u32 GetOriginIfPoisoned(uptr a, uptr size); -void SetOriginIfPoisoned(uptr addr, uptr src_shadow, uptr size, u32 src_origin); -void CopyOrigin(void *dst, const void *src, uptr size, StackTrace *stack); -void MovePoison(void *dst, const void *src, uptr size, StackTrace *stack); -void CopyPoison(void *dst, const void *src, uptr size, StackTrace *stack); - // Returns a "chained" origin id, pointing to the given stack trace followed by // the previous origin id. u32 ChainOrigin(u32 id, StackTrace *stack); +const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; + #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ if (__msan_get_track_origins() && msan_inited) \ diff --git a/contrib/compiler-rt/lib/msan/msan_allocator.cc b/contrib/compiler-rt/lib/msan/msan_allocator.cc index f21d71409ce2..698b6cddd30b 100644 --- a/contrib/compiler-rt/lib/msan/msan_allocator.cc +++ b/contrib/compiler-rt/lib/msan/msan_allocator.cc @@ -18,6 +18,7 @@ #include "msan_allocator.h" #include "msan_origin.h" #include "msan_thread.h" +#include "msan_poisoning.h" namespace __msan { @@ -73,7 +74,7 @@ static inline void Init() { if (inited) return; __msan_init(); inited = true; // this must happen before any threads are created. - allocator.Init(); + allocator.Init(common_flags()->allocator_may_return_null); } AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) { @@ -92,7 +93,7 @@ static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, if (size > kMaxAllowedMallocSize) { Report("WARNING: MemorySanitizer failed to allocate %p bytes\n", (void *)size); - return AllocatorReturnNull(); + return allocator.ReturnNullOrDie(); } MsanThread *t = GetCurrentThread(); void *allocated; @@ -112,6 +113,7 @@ static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, } else if (flags()->poison_in_malloc) { __msan_poison(allocated, size); if (__msan_get_track_origins()) { + stack->tag = StackTrace::TAG_ALLOC; Origin o = Origin::CreateHeapOrigin(stack); __msan_set_origin(allocated, size, o.raw_id()); } @@ -132,6 +134,7 @@ void MsanDeallocate(StackTrace *stack, void *p) { if (flags()->poison_in_free) { __msan_poison(p, size); if (__msan_get_track_origins()) { + stack->tag = StackTrace::TAG_DEALLOC; Origin o = Origin::CreateHeapOrigin(stack); __msan_set_origin(p, size, o.raw_id()); } @@ -147,6 +150,13 @@ void MsanDeallocate(StackTrace *stack, void *p) { } } +void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size) { + Init(); + if (CallocShouldReturnNullDueToOverflow(size, nmemb)) + return allocator.ReturnNullOrDie(); + return MsanReallocate(stack, 0, nmemb * size, sizeof(u64), true); +} + void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, uptr alignment, bool zeroise) { if (!old_p) @@ -161,15 +171,22 @@ void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, if (new_size <= actually_allocated_size) { // We are not reallocating here. meta->requested_size = new_size; - if (new_size > old_size) - __msan_poison((char*)old_p + old_size, new_size - old_size); + if (new_size > old_size) { + if (zeroise) { + __msan_clear_and_unpoison((char *)old_p + old_size, + new_size - old_size); + } else if (flags()->poison_in_malloc) { + stack->tag = StackTrace::TAG_ALLOC; + PoisonMemory((char *)old_p + old_size, new_size - old_size, stack); + } + } return old_p; } uptr memcpy_size = Min(new_size, old_size); void *new_p = MsanAllocate(stack, new_size, alignment, zeroise); // Printf("realloc: old_size %zd new_size %zd\n", old_size, new_size); if (new_p) { - __msan_memcpy(new_p, old_p, memcpy_size); + CopyMemory(new_p, old_p, memcpy_size, stack); MsanDeallocate(stack, old_p); } return new_p; diff --git a/contrib/compiler-rt/lib/msan/msan_flags.h b/contrib/compiler-rt/lib/msan/msan_flags.h index 9b93f118a985..4fc6d172a04a 100644 --- a/contrib/compiler-rt/lib/msan/msan_flags.h +++ b/contrib/compiler-rt/lib/msan/msan_flags.h @@ -9,28 +9,18 @@ // // This file is a part of MemorySanitizer. // -// MemorySanitizer allocator. //===----------------------------------------------------------------------===// #ifndef MSAN_FLAGS_H #define MSAN_FLAGS_H namespace __msan { -// Flags. struct Flags { - int exit_code; - int origin_history_size; - int origin_history_per_stack_limit; - bool poison_heap_with_zeroes; // default: false - bool poison_stack_with_zeroes; // default: false - bool poison_in_malloc; // default: true - bool poison_in_free; // default: true - bool report_umrs; - bool wrap_signals; - bool print_stats; - bool halt_on_error; - bool atexit; - int store_context_size; // like malloc_context_size, but for uninit stores +#define MSAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "msan_flags.inc" +#undef MSAN_FLAG + + void SetDefaults(); }; Flags *flags(); diff --git a/contrib/compiler-rt/lib/msan/msan_flags.inc b/contrib/compiler-rt/lib/msan/msan_flags.inc new file mode 100644 index 000000000000..cb58ffc4aba7 --- /dev/null +++ b/contrib/compiler-rt/lib/msan/msan_flags.inc @@ -0,0 +1,33 @@ +//===-- msan_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// MSan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef MSAN_FLAG +# error "Define MSAN_FLAG prior to including this file!" +#endif + +// MSAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +MSAN_FLAG(int, exit_code, 77, "") +MSAN_FLAG(int, origin_history_size, Origin::kMaxDepth, "") +MSAN_FLAG(int, origin_history_per_stack_limit, 20000, "") +MSAN_FLAG(bool, poison_heap_with_zeroes, false, "") +MSAN_FLAG(bool, poison_stack_with_zeroes, false, "") +MSAN_FLAG(bool, poison_in_malloc, true, "") +MSAN_FLAG(bool, poison_in_free, true, "") +MSAN_FLAG(bool, report_umrs, true, "") +MSAN_FLAG(bool, wrap_signals, true, "") +MSAN_FLAG(bool, print_stats, false, "") +MSAN_FLAG(bool, halt_on_error, !&__msan_keep_going, "") +MSAN_FLAG(bool, atexit, false, "") +MSAN_FLAG(int, store_context_size, 20, + "Like malloc_context_size, but for uninit stores.") diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc index bbdf18e162a3..4a243941b8a3 100644 --- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc +++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc @@ -20,6 +20,7 @@ #include "msan_chained_origin_depot.h" #include "msan_origin.h" #include "msan_thread.h" +#include "msan_poisoning.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_allocator.h" #include "sanitizer_common/sanitizer_allocator_interface.h" @@ -290,7 +291,7 @@ INTERCEPTOR(char *, strcpy, char *dest, const char *src) { // NOLINT GET_STORE_STACK_TRACE; SIZE_T n = REAL(strlen)(src); char *res = REAL(strcpy)(dest, src); // NOLINT - CopyPoison(dest, src, n + 1, &stack); + CopyShadowAndOrigin(dest, src, n + 1, &stack); return res; } @@ -301,7 +302,7 @@ INTERCEPTOR(char *, strncpy, char *dest, const char *src, SIZE_T n) { // NOLINT if (copy_size < n) copy_size++; // trailing \0 char *res = REAL(strncpy)(dest, src, n); // NOLINT - CopyPoison(dest, src, copy_size, &stack); + CopyShadowAndOrigin(dest, src, copy_size, &stack); __msan_unpoison(dest + copy_size, n - copy_size); return res; } @@ -311,16 +312,18 @@ INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT GET_STORE_STACK_TRACE; SIZE_T n = REAL(strlen)(src); char *res = REAL(stpcpy)(dest, src); // NOLINT - CopyPoison(dest, src, n + 1, &stack); + CopyShadowAndOrigin(dest, src, n + 1, &stack); return res; } INTERCEPTOR(char *, strdup, char *src) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; + // On FreeBSD strdup() leverages strlen(). + InterceptorScope interceptor_scope; SIZE_T n = REAL(strlen)(src); char *res = REAL(strdup)(src); - CopyPoison(res, src, n + 1, &stack); + CopyShadowAndOrigin(res, src, n + 1, &stack); return res; } @@ -330,7 +333,7 @@ INTERCEPTOR(char *, __strdup, char *src) { GET_STORE_STACK_TRACE; SIZE_T n = REAL(strlen)(src); char *res = REAL(__strdup)(src); - CopyPoison(res, src, n + 1, &stack); + CopyShadowAndOrigin(res, src, n + 1, &stack); return res; } #define MSAN_MAYBE_INTERCEPT___STRDUP INTERCEPT_FUNCTION(__strdup) @@ -341,9 +344,11 @@ INTERCEPTOR(char *, __strdup, char *src) { INTERCEPTOR(char *, strndup, char *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; + // On FreeBSD strndup() leverages strnlen(). + InterceptorScope interceptor_scope; SIZE_T copy_size = REAL(strnlen)(src, n); char *res = REAL(strndup)(src, n); - CopyPoison(res, src, copy_size, &stack); + CopyShadowAndOrigin(res, src, copy_size, &stack); __msan_unpoison(res + copy_size, 1); // \0 return res; } @@ -354,7 +359,7 @@ INTERCEPTOR(char *, __strndup, char *src, SIZE_T n) { GET_STORE_STACK_TRACE; SIZE_T copy_size = REAL(strnlen)(src, n); char *res = REAL(__strndup)(src, n); - CopyPoison(res, src, copy_size, &stack); + CopyShadowAndOrigin(res, src, copy_size, &stack); __msan_unpoison(res + copy_size, 1); // \0 return res; } @@ -377,7 +382,7 @@ INTERCEPTOR(char *, strcat, char *dest, const char *src) { // NOLINT SIZE_T src_size = REAL(strlen)(src); SIZE_T dest_size = REAL(strlen)(dest); char *res = REAL(strcat)(dest, src); // NOLINT - CopyPoison(dest + dest_size, src, src_size + 1, &stack); + CopyShadowAndOrigin(dest + dest_size, src, src_size + 1, &stack); return res; } @@ -387,7 +392,7 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT SIZE_T dest_size = REAL(strlen)(dest); SIZE_T copy_size = REAL(strnlen)(src, n); char *res = REAL(strncat)(dest, src, n); // NOLINT - CopyPoison(dest + dest_size, src, copy_size, &stack); + CopyShadowAndOrigin(dest + dest_size, src, copy_size, &stack); __msan_unpoison(dest + dest_size + copy_size, 1); // \0 return res; } @@ -576,7 +581,8 @@ INTERCEPTOR(wchar_t *, wcscpy, wchar_t *dest, const wchar_t *src) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; wchar_t *res = REAL(wcscpy)(dest, src); - CopyPoison(dest, src, sizeof(wchar_t) * (REAL(wcslen)(src) + 1), &stack); + CopyShadowAndOrigin(dest, src, sizeof(wchar_t) * (REAL(wcslen)(src) + 1), + &stack); return res; } @@ -585,7 +591,7 @@ INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; wchar_t *res = REAL(wmemcpy)(dest, src, n); - CopyPoison(dest, src, n * sizeof(wchar_t), &stack); + CopyShadowAndOrigin(dest, src, n * sizeof(wchar_t), &stack); return res; } @@ -593,7 +599,7 @@ INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; wchar_t *res = REAL(wmempcpy)(dest, src, n); - CopyPoison(dest, src, n * sizeof(wchar_t), &stack); + CopyShadowAndOrigin(dest, src, n * sizeof(wchar_t), &stack); return res; } @@ -609,7 +615,7 @@ INTERCEPTOR(wchar_t *, wmemmove, wchar_t *dest, const wchar_t *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; wchar_t *res = REAL(wmemmove)(dest, src, n); - MovePoison(dest, src, n * sizeof(wchar_t), &stack); + MoveShadowAndOrigin(dest, src, n * sizeof(wchar_t), &stack); return res; } @@ -699,7 +705,15 @@ INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) { #define MSAN_MAYBE_INTERCEPT___FXSTAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_FREEBSD +INTERCEPTOR(int, fstatat, int fd, char *pathname, void *buf, int flags) { + ENSURE_MSAN_INITED(); + int res = REAL(fstatat)(fd, pathname, buf, flags); + if (!res) __msan_unpoison(buf, __sanitizer::struct_stat_sz); + return res; +} +# define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(fstatat) +#else INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf, int flags) { ENSURE_MSAN_INITED(); @@ -707,9 +721,7 @@ INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf, if (!res) __msan_unpoison(buf, __sanitizer::struct_stat_sz); return res; } -#define MSAN_MAYBE_INTERCEPT___FXSTATAT INTERCEPT_FUNCTION(__fxstatat) -#else -#define MSAN_MAYBE_INTERCEPT___FXSTATAT +# define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(__fxstatat) #endif #if !SANITIZER_FREEBSD @@ -725,7 +737,16 @@ INTERCEPTOR(int, __fxstatat64, int magic, int fd, char *pathname, void *buf, #define MSAN_MAYBE_INTERCEPT___FXSTATAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_FREEBSD +INTERCEPTOR(int, stat, char *path, void *buf) { + ENSURE_MSAN_INITED(); + int res = REAL(stat)(path, buf); + if (!res) + __msan_unpoison(buf, __sanitizer::struct_stat_sz); + return res; +} +# define MSAN_INTERCEPT_STAT INTERCEPT_FUNCTION(stat) +#else INTERCEPTOR(int, __xstat, int magic, char *path, void *buf) { ENSURE_MSAN_INITED(); int res = REAL(__xstat)(magic, path, buf); @@ -733,9 +754,7 @@ INTERCEPTOR(int, __xstat, int magic, char *path, void *buf) { __msan_unpoison(buf, __sanitizer::struct_stat_sz); return res; } -#define MSAN_MAYBE_INTERCEPT___XSTAT INTERCEPT_FUNCTION(__xstat) -#else -#define MSAN_MAYBE_INTERCEPT___XSTAT +# define MSAN_INTERCEPT_STAT INTERCEPT_FUNCTION(__xstat) #endif #if !SANITIZER_FREEBSD @@ -849,14 +868,29 @@ INTERCEPTOR(int, getrlimit64, int resource, void *rlim) { #define MSAN_MAYBE_INTERCEPT_GETRLIMIT64 #endif -INTERCEPTOR(int, uname, void *utsname) { +#if SANITIZER_FREEBSD +// FreeBSD's define uname() as +// static __inline int uname(struct utsname *name) { +// return __xuname(SYS_NMLN, (void*)name); +// } +INTERCEPTOR(int, __xuname, int size, void *utsname) { ENSURE_MSAN_INITED(); - int res = REAL(uname)(utsname); - if (!res) { + int res = REAL(__xuname)(size, utsname); + if (!res) __msan_unpoison(utsname, __sanitizer::struct_utsname_sz); - } return res; } +#define MSAN_INTERCEPT_UNAME INTERCEPT_FUNCTION(__xuname) +#else +INTERCEPTOR(int, uname, struct utsname *utsname) { + ENSURE_MSAN_INITED(); + int res = REAL(uname)(utsname); + if (!res) + __msan_unpoison(utsname, __sanitizer::struct_utsname_sz); + return res; +} +#define MSAN_INTERCEPT_UNAME INTERCEPT_FUNCTION(uname) +#endif INTERCEPTOR(int, gethostname, char *name, SIZE_T len) { ENSURE_MSAN_INITED(); @@ -918,17 +952,15 @@ INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags, __msan_unpoison(buf, res); if (srcaddr) { SIZE_T sz = *addrlen; - __msan_unpoison(srcaddr, (sz < srcaddr_sz) ? sz : srcaddr_sz); + __msan_unpoison(srcaddr, Min(sz, srcaddr_sz)); } } return res; } INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { - if (CallocShouldReturnNullDueToOverflow(size, nmemb)) - return AllocatorReturnNull(); GET_MALLOC_STACK_TRACE; - if (!msan_inited) { + if (UNLIKELY(!msan_inited)) { // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. const SIZE_T kCallocPoolSize = 1024; static uptr calloc_memory_for_dlsym[kCallocPoolSize]; @@ -939,7 +971,7 @@ INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { CHECK(allocated < kCallocPoolSize); return mem; } - return MsanReallocate(&stack, 0, nmemb * size, sizeof(u64), true); + return MsanCalloc(&stack, nmemb, size); } INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { @@ -952,13 +984,11 @@ INTERCEPTOR(void *, malloc, SIZE_T size) { return MsanReallocate(&stack, 0, size, sizeof(u64), false); } -void __msan_allocated_memory(const void* data, uptr size) { +void __msan_allocated_memory(const void *data, uptr size) { GET_MALLOC_STACK_TRACE; - if (flags()->poison_in_malloc) - __msan_poison(data, size); - if (__msan_get_track_origins()) { - Origin o = Origin::CreateHeapOrigin(&stack); - __msan_set_origin(data, size, o.raw_id()); + if (flags()->poison_in_malloc) { + stack.tag = STACK_TRACE_TAG_POISON; + PoisonMemory(data, size, &stack); } } @@ -1328,6 +1358,9 @@ int OnExit() { InterceptorScope interceptor_scope; \ __msan_unpoison(__errno_location(), sizeof(int)); /* NOLINT */ \ ENSURE_MSAN_INITED(); +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + } while (false) #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ do { \ } while (false) @@ -1345,8 +1378,11 @@ int OnExit() { } while (false) // FIXME #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, map) \ - if (map) ForEachMappedRegion((link_map *)map, __msan_unpoison); +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ + do { \ + link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE((handle)); \ + if (map) ForEachMappedRegion(map, __msan_unpoison); \ + } while (false) #include "sanitizer_common/sanitizer_common_interceptors.inc" @@ -1360,53 +1396,26 @@ int OnExit() { #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) __msan_unpoison(p, s) #include "sanitizer_common/sanitizer_common_syscalls.inc" -static void PoisonShadow(uptr ptr, uptr size, u8 value) { - uptr PageSize = GetPageSizeCached(); - uptr shadow_beg = MEM_TO_SHADOW(ptr); - uptr shadow_end = MEM_TO_SHADOW(ptr + size); - if (value || - shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { - REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg); - } else { - uptr page_beg = RoundUpTo(shadow_beg, PageSize); - uptr page_end = RoundDownTo(shadow_end, PageSize); - - if (page_beg >= page_end) { - REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); - } else { - if (page_beg != shadow_beg) { - REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg); - } - if (page_end != shadow_end) { - REAL(memset)((void *)page_end, 0, shadow_end - page_end); - } - MmapFixedNoReserve(page_beg, page_end - page_beg); - } - } -} - // These interface functions reside here so that they can use // REAL(memset), etc. void __msan_unpoison(const void *a, uptr size) { if (!MEM_IS_APP(a)) return; - PoisonShadow((uptr)a, size, 0); + SetShadow(a, size, 0); } void __msan_poison(const void *a, uptr size) { if (!MEM_IS_APP(a)) return; - PoisonShadow((uptr)a, size, - __msan::flags()->poison_heap_with_zeroes ? 0 : -1); + SetShadow(a, size, __msan::flags()->poison_heap_with_zeroes ? 0 : -1); } void __msan_poison_stack(void *a, uptr size) { if (!MEM_IS_APP(a)) return; - PoisonShadow((uptr)a, size, - __msan::flags()->poison_stack_with_zeroes ? 0 : -1); + SetShadow(a, size, __msan::flags()->poison_stack_with_zeroes ? 0 : -1); } void __msan_clear_and_unpoison(void *a, uptr size) { REAL(memset)(a, 0, size); - PoisonShadow((uptr)a, size, 0); + SetShadow(a, size, 0); } void *__msan_memcpy(void *dest, const void *src, SIZE_T n) { @@ -1415,7 +1424,7 @@ void *__msan_memcpy(void *dest, const void *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; void *res = REAL(memcpy)(dest, src, n); - CopyPoison(dest, src, n, &stack); + CopyShadowAndOrigin(dest, src, n, &stack); return res; } @@ -1434,7 +1443,7 @@ void *__msan_memmove(void *dest, const void *src, SIZE_T n) { ENSURE_MSAN_INITED(); GET_STORE_STACK_TRACE; void *res = REAL(memmove)(dest, src, n); - MovePoison(dest, src, n, &stack); + MoveShadowAndOrigin(dest, src, n, &stack); return res; } @@ -1445,96 +1454,6 @@ void __msan_unpoison_string(const char* s) { namespace __msan { -u32 GetOriginIfPoisoned(uptr addr, uptr size) { - unsigned char *s = (unsigned char *)MEM_TO_SHADOW(addr); - for (uptr i = 0; i < size; ++i) - if (s[i]) - return *(u32 *)SHADOW_TO_ORIGIN(((uptr)s + i) & ~3UL); - return 0; -} - -void SetOriginIfPoisoned(uptr addr, uptr src_shadow, uptr size, - u32 src_origin) { - uptr dst_s = MEM_TO_SHADOW(addr); - uptr src_s = src_shadow; - uptr src_s_end = src_s + size; - - for (; src_s < src_s_end; ++dst_s, ++src_s) - if (*(u8 *)src_s) *(u32 *)SHADOW_TO_ORIGIN(dst_s &~3UL) = src_origin; -} - -void CopyOrigin(void *dst, const void *src, uptr size, StackTrace *stack) { - if (!__msan_get_track_origins()) return; - if (!MEM_IS_APP(dst) || !MEM_IS_APP(src)) return; - - uptr d = (uptr)dst; - uptr beg = d & ~3UL; - // Copy left unaligned origin if that memory is poisoned. - if (beg < d) { - u32 o = GetOriginIfPoisoned((uptr)src, d - beg); - if (o) { - if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack); - *(u32 *)MEM_TO_ORIGIN(beg) = o; - } - beg += 4; - } - - uptr end = (d + size) & ~3UL; - // If both ends fall into the same 4-byte slot, we are done. - if (end < beg) return; - - // Copy right unaligned origin if that memory is poisoned. - if (end < d + size) { - u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end); - if (o) { - if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack); - *(u32 *)MEM_TO_ORIGIN(end) = o; - } - } - - if (beg < end) { - // Align src up. - uptr s = ((uptr)src + 3) & ~3UL; - // FIXME: factor out to msan_copy_origin_aligned - if (__msan_get_track_origins() > 1) { - u32 *src = (u32 *)MEM_TO_ORIGIN(s); - u32 *src_s = (u32 *)MEM_TO_SHADOW(s); - u32 *src_end = (u32 *)MEM_TO_ORIGIN(s + (end - beg)); - u32 *dst = (u32 *)MEM_TO_ORIGIN(beg); - u32 src_o = 0; - u32 dst_o = 0; - for (; src < src_end; ++src, ++src_s, ++dst) { - if (!*src_s) continue; - if (*src != src_o) { - src_o = *src; - dst_o = ChainOrigin(src_o, stack); - } - *dst = dst_o; - } - } else { - REAL(memcpy)((void *)MEM_TO_ORIGIN(beg), (void *)MEM_TO_ORIGIN(s), - end - beg); - } - } -} - -void MovePoison(void *dst, const void *src, uptr size, StackTrace *stack) { - if (!MEM_IS_APP(dst)) return; - if (!MEM_IS_APP(src)) return; - if (src == dst) return; - REAL(memmove)((void *)MEM_TO_SHADOW((uptr)dst), - (void *)MEM_TO_SHADOW((uptr)src), size); - CopyOrigin(dst, src, size, stack); -} - -void CopyPoison(void *dst, const void *src, uptr size, StackTrace *stack) { - if (!MEM_IS_APP(dst)) return; - if (!MEM_IS_APP(src)) return; - REAL(memcpy)((void *)MEM_TO_SHADOW((uptr)dst), - (void *)MEM_TO_SHADOW((uptr)src), size); - CopyOrigin(dst, src, size, stack); -} - void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); @@ -1617,8 +1536,8 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(gettimeofday); INTERCEPT_FUNCTION(fcvt); MSAN_MAYBE_INTERCEPT___FXSTAT; - MSAN_MAYBE_INTERCEPT___FXSTATAT; - MSAN_MAYBE_INTERCEPT___XSTAT; + MSAN_INTERCEPT_FSTATAT; + MSAN_INTERCEPT_STAT; MSAN_MAYBE_INTERCEPT___LXSTAT; MSAN_MAYBE_INTERCEPT___FXSTAT64; MSAN_MAYBE_INTERCEPT___FXSTATAT64; @@ -1631,7 +1550,7 @@ void InitializeInterceptors() { MSAN_MAYBE_INTERCEPT_FGETS_UNLOCKED; INTERCEPT_FUNCTION(getrlimit); MSAN_MAYBE_INTERCEPT_GETRLIMIT64; - INTERCEPT_FUNCTION(uname); + MSAN_INTERCEPT_UNAME; INTERCEPT_FUNCTION(gethostname); MSAN_MAYBE_INTERCEPT_EPOLL_WAIT; MSAN_MAYBE_INTERCEPT_EPOLL_PWAIT; diff --git a/contrib/compiler-rt/lib/msan/msan_linux.cc b/contrib/compiler-rt/lib/msan/msan_linux.cc index 0b67b531d51c..6c185165fc50 100644 --- a/contrib/compiler-rt/lib/msan/msan_linux.cc +++ b/contrib/compiler-rt/lib/msan/msan_linux.cc @@ -64,41 +64,45 @@ static bool ProtectMemoryRange(uptr beg, uptr size) { return true; } +static void CheckMemoryLayoutSanity() { + uptr prev_end = 0; + for (unsigned i = 0; i < kMemoryLayoutSize; ++i) { + uptr start = kMemoryLayout[i].start; + uptr end = kMemoryLayout[i].end; + MappingDesc::Type type = kMemoryLayout[i].type; + CHECK_LT(start, end); + CHECK_EQ(prev_end, start); + CHECK(addr_is_type(start, type)); + CHECK(addr_is_type((start + end) / 2, type)); + CHECK(addr_is_type(end - 1, type)); + if (type == MappingDesc::APP) { + uptr addr = start; + CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr))); + CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr))); + CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr))); + + addr = (start + end) / 2; + CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr))); + CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr))); + CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr))); + + addr = end - 1; + CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr))); + CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr))); + CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr))); + } + prev_end = end; + } +} + bool InitShadow(bool map_shadow, bool init_origins) { // Let user know mapping parameters first. VPrintf(1, "__msan_init %p\n", &__msan_init); - ReportMapRange("Low Memory ", kLowMemBeg, kLowMemSize); - ReportMapRange("Bad1 ", kBad1Beg, kBad1Size); - ReportMapRange("Shadow ", kShadowBeg, kShadowSize); - ReportMapRange("Bad2 ", kBad2Beg, kBad2Size); - ReportMapRange("Origins ", kOriginsBeg, kOriginsSize); - ReportMapRange("Bad3 ", kBad3Beg, kBad3Size); - ReportMapRange("High Memory", kHighMemBeg, kHighMemSize); + for (unsigned i = 0; i < kMemoryLayoutSize; ++i) + VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start, + kMemoryLayout[i].end - 1); - // Check mapping sanity (the invariant). - CHECK_EQ(kLowMemBeg, 0); - CHECK_EQ(kBad1Beg, kLowMemBeg + kLowMemSize); - CHECK_EQ(kShadowBeg, kBad1Beg + kBad1Size); - CHECK_GT(kShadowSize, 0); - CHECK_GE(kShadowSize, kLowMemSize + kHighMemSize); - CHECK_EQ(kBad2Beg, kShadowBeg + kShadowSize); - CHECK_EQ(kOriginsBeg, kBad2Beg + kBad2Size); - CHECK_EQ(kOriginsSize, kShadowSize); - CHECK_EQ(kBad3Beg, kOriginsBeg + kOriginsSize); - CHECK_EQ(kHighMemBeg, kBad3Beg + kBad3Size); - CHECK_GT(kHighMemSize, 0); - CHECK_GE(kHighMemBeg + kHighMemSize, kHighMemBeg); // Tests for no overflow. - - if (kLowMemSize > 0) { - CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(kLowMemBeg))); - CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(kLowMemBeg + kLowMemSize - 1))); - CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(kLowMemBeg))); - CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(kLowMemBeg + kLowMemSize - 1))); - } - CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(kHighMemBeg))); - CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(kHighMemBeg + kHighMemSize - 1))); - CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(kHighMemBeg))); - CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(kHighMemBeg + kHighMemSize - 1))); + CheckMemoryLayoutSanity(); if (!MEM_IS_APP(&__msan_init)) { Printf("FATAL: Code %p is out of application range. Non-PIE build?\n", @@ -106,29 +110,23 @@ bool InitShadow(bool map_shadow, bool init_origins) { return false; } - if (!CheckMemoryRangeAvailability(kShadowBeg, kShadowSize) || - (init_origins && - !CheckMemoryRangeAvailability(kOriginsBeg, kOriginsSize)) || - !CheckMemoryRangeAvailability(kBad1Beg, kBad1Size) || - !CheckMemoryRangeAvailability(kBad2Beg, kBad2Size) || - !CheckMemoryRangeAvailability(kBad3Beg, kBad3Size)) { - return false; + for (unsigned i = 0; i < kMemoryLayoutSize; ++i) { + uptr start = kMemoryLayout[i].start; + uptr end = kMemoryLayout[i].end; + uptr size= end - start; + MappingDesc::Type type = kMemoryLayout[i].type; + if ((map_shadow && type == MappingDesc::SHADOW) || + (init_origins && type == MappingDesc::ORIGIN)) { + if (!CheckMemoryRangeAvailability(start, size)) return false; + if ((uptr)MmapFixedNoReserve(start, size) != start) return false; + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(start, size); + } else if (type == MappingDesc::INVALID) { + if (!CheckMemoryRangeAvailability(start, size)) return false; + if (!ProtectMemoryRange(start, size)) return false; + } } - if (!ProtectMemoryRange(kBad1Beg, kBad1Size) || - !ProtectMemoryRange(kBad2Beg, kBad2Size) || - !ProtectMemoryRange(kBad3Beg, kBad3Size)) { - return false; - } - - if (map_shadow) { - void *shadow = MmapFixedNoReserve(kShadowBeg, kShadowSize); - if (shadow != (void*)kShadowBeg) return false; - } - if (init_origins) { - void *origins = MmapFixedNoReserve(kOriginsBeg, kOriginsSize); - if (origins != (void*)kOriginsBeg) return false; - } return true; } @@ -137,7 +135,7 @@ void MsanDie() { __sanitizer_cov_dump(); if (death_callback) death_callback(); - _exit(flags()->exit_code); + internal__exit(flags()->exit_code); } static void MsanAtExit(void) { @@ -157,20 +155,26 @@ void InstallAtExitHandler() { static pthread_key_t tsd_key; static bool tsd_key_inited = false; + void MsanTSDInit(void (*destructor)(void *tsd)) { CHECK(!tsd_key_inited); tsd_key_inited = true; CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); } -void *MsanTSDGet() { - CHECK(tsd_key_inited); - return pthread_getspecific(tsd_key); +static THREADLOCAL MsanThread* msan_current_thread; + +MsanThread *GetCurrentThread() { + return msan_current_thread; } -void MsanTSDSet(void *tsd) { +void SetCurrentThread(MsanThread *t) { + // Make sure we do not reset the current MsanThread. + CHECK_EQ(0, msan_current_thread); + msan_current_thread = t; + // Make sure that MsanTSDDtor gets called at the end. CHECK(tsd_key_inited); - pthread_setspecific(tsd_key, tsd); + pthread_setspecific(tsd_key, (void *)t); } void MsanTSDDtor(void *tsd) { @@ -180,6 +184,9 @@ void MsanTSDDtor(void *tsd) { CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); return; } + msan_current_thread = nullptr; + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); MsanThread::TSDDtor(tsd); } diff --git a/contrib/compiler-rt/lib/msan/msan_poisoning.cc b/contrib/compiler-rt/lib/msan/msan_poisoning.cc new file mode 100644 index 000000000000..96411fdbc31b --- /dev/null +++ b/contrib/compiler-rt/lib/msan/msan_poisoning.cc @@ -0,0 +1,174 @@ +//===-- msan_poisoning.cc ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemorySanitizer. +// +//===----------------------------------------------------------------------===// + +#include "msan_poisoning.h" + +#include "interception/interception.h" +#include "msan_origin.h" +#include "sanitizer_common/sanitizer_common.h" + +DECLARE_REAL(void *, memset, void *dest, int c, uptr n) +DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n) +DECLARE_REAL(void *, memmove, void *dest, const void *src, uptr n) + +namespace __msan { + +u32 GetOriginIfPoisoned(uptr addr, uptr size) { + unsigned char *s = (unsigned char *)MEM_TO_SHADOW(addr); + for (uptr i = 0; i < size; ++i) + if (s[i]) return *(u32 *)SHADOW_TO_ORIGIN(((uptr)s + i) & ~3UL); + return 0; +} + +void SetOriginIfPoisoned(uptr addr, uptr src_shadow, uptr size, + u32 src_origin) { + uptr dst_s = MEM_TO_SHADOW(addr); + uptr src_s = src_shadow; + uptr src_s_end = src_s + size; + + for (; src_s < src_s_end; ++dst_s, ++src_s) + if (*(u8 *)src_s) *(u32 *)SHADOW_TO_ORIGIN(dst_s & ~3UL) = src_origin; +} + +void CopyOrigin(const void *dst, const void *src, uptr size, + StackTrace *stack) { + if (!MEM_IS_APP(dst) || !MEM_IS_APP(src)) return; + + uptr d = (uptr)dst; + uptr beg = d & ~3UL; + // Copy left unaligned origin if that memory is poisoned. + if (beg < d) { + u32 o = GetOriginIfPoisoned((uptr)src, d - beg); + if (o) { + if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack); + *(u32 *)MEM_TO_ORIGIN(beg) = o; + } + beg += 4; + } + + uptr end = (d + size) & ~3UL; + // If both ends fall into the same 4-byte slot, we are done. + if (end < beg) return; + + // Copy right unaligned origin if that memory is poisoned. + if (end < d + size) { + u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end); + if (o) { + if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack); + *(u32 *)MEM_TO_ORIGIN(end) = o; + } + } + + if (beg < end) { + // Align src up. + uptr s = ((uptr)src + 3) & ~3UL; + // FIXME: factor out to msan_copy_origin_aligned + if (__msan_get_track_origins() > 1) { + u32 *src = (u32 *)MEM_TO_ORIGIN(s); + u32 *src_s = (u32 *)MEM_TO_SHADOW(s); + u32 *src_end = (u32 *)MEM_TO_ORIGIN(s + (end - beg)); + u32 *dst = (u32 *)MEM_TO_ORIGIN(beg); + u32 src_o = 0; + u32 dst_o = 0; + for (; src < src_end; ++src, ++src_s, ++dst) { + if (!*src_s) continue; + if (*src != src_o) { + src_o = *src; + dst_o = ChainOrigin(src_o, stack); + } + *dst = dst_o; + } + } else { + REAL(memcpy)((void *)MEM_TO_ORIGIN(beg), (void *)MEM_TO_ORIGIN(s), + end - beg); + } + } +} + +void MoveShadowAndOrigin(const void *dst, const void *src, uptr size, + StackTrace *stack) { + if (!MEM_IS_APP(dst)) return; + if (!MEM_IS_APP(src)) return; + if (src == dst) return; + REAL(memmove)((void *)MEM_TO_SHADOW((uptr)dst), + (void *)MEM_TO_SHADOW((uptr)src), size); + if (__msan_get_track_origins()) CopyOrigin(dst, src, size, stack); +} + +void CopyShadowAndOrigin(const void *dst, const void *src, uptr size, + StackTrace *stack) { + if (!MEM_IS_APP(dst)) return; + if (!MEM_IS_APP(src)) return; + REAL(memcpy)((void *)MEM_TO_SHADOW((uptr)dst), + (void *)MEM_TO_SHADOW((uptr)src), size); + if (__msan_get_track_origins()) CopyOrigin(dst, src, size, stack); +} + +void CopyMemory(void *dst, const void *src, uptr size, StackTrace *stack) { + REAL(memcpy)(dst, src, size); + CopyShadowAndOrigin(dst, src, size, stack); +} + +void SetShadow(const void *ptr, uptr size, u8 value) { + uptr PageSize = GetPageSizeCached(); + uptr shadow_beg = MEM_TO_SHADOW(ptr); + uptr shadow_end = MEM_TO_SHADOW((uptr)ptr + size); + if (value || + shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { + REAL(memset)((void *)shadow_beg, value, shadow_end - shadow_beg); + } else { + uptr page_beg = RoundUpTo(shadow_beg, PageSize); + uptr page_end = RoundDownTo(shadow_end, PageSize); + + if (page_beg >= page_end) { + REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); + } else { + if (page_beg != shadow_beg) { + REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg); + } + if (page_end != shadow_end) { + REAL(memset)((void *)page_end, 0, shadow_end - page_end); + } + MmapFixedNoReserve(page_beg, page_end - page_beg); + } + } +} + +void SetOrigin(const void *dst, uptr size, u32 origin) { + // Origin mapping is 4 bytes per 4 bytes of application memory. + // Here we extend the range such that its left and right bounds are both + // 4 byte aligned. + uptr x = MEM_TO_ORIGIN((uptr)dst); + uptr beg = x & ~3UL; // align down. + uptr end = (x + size + 3) & ~3UL; // align up. + u64 origin64 = ((u64)origin << 32) | origin; + // This is like memset, but the value is 32-bit. We unroll by 2 to write + // 64 bits at once. May want to unroll further to get 128-bit stores. + if (beg & 7ULL) { + *(u32 *)beg = origin; + beg += 4; + } + for (uptr addr = beg; addr < (end & ~7UL); addr += 8) *(u64 *)addr = origin64; + if (end & 7ULL) *(u32 *)(end - 4) = origin; +} + +void PoisonMemory(const void *dst, uptr size, StackTrace *stack) { + SetShadow(dst, size, (u8)-1); + + if (__msan_get_track_origins()) { + Origin o = Origin::CreateHeapOrigin(stack); + SetOrigin(dst, size, o.raw_id()); + } +} + +} // namespace __msan diff --git a/contrib/compiler-rt/lib/msan/msan_poisoning.h b/contrib/compiler-rt/lib/msan/msan_poisoning.h new file mode 100644 index 000000000000..edacbeeab0a6 --- /dev/null +++ b/contrib/compiler-rt/lib/msan/msan_poisoning.h @@ -0,0 +1,59 @@ +//===-- msan_poisoning.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of MemorySanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef MSAN_POISONING_H +#define MSAN_POISONING_H + +#include "msan.h" + +namespace __msan { + +// Return origin for the first poisoned byte in the memory range, or 0. +u32 GetOriginIfPoisoned(uptr addr, uptr size); + +// Walk [addr, addr+size) app memory region, copying origin tags from the +// corresponding positions in [src_origin, src_origin+size) where the +// corresponding shadow in [src_shadow, src_shadow+size) is non-zero. +void SetOriginIfPoisoned(uptr addr, uptr src_shadow, uptr size, u32 src_origin); + +// Copy origin from src (app address) to dst (app address), creating chained +// origin ids as necessary, without overriding origin for fully initialized +// quads. +void CopyOrigin(const void *dst, const void *src, uptr size, StackTrace *stack); + +// memmove() shadow and origin. Dst and src are application addresses. +// See CopyOrigin() for the origin copying logic. +void MoveShadowAndOrigin(const void *dst, const void *src, uptr size, + StackTrace *stack); + +// memcpy() shadow and origin. Dst and src are application addresses. +// See CopyOrigin() for the origin copying logic. +void CopyShadowAndOrigin(const void *dst, const void *src, uptr size, + StackTrace *stack); + +// memcpy() app memory, and do "the right thing" to the corresponding shadow and +// origin regions. +void CopyMemory(void *dst, const void *src, uptr size, StackTrace *stack); + +// Fill shadow will value. Ptr is an application address. +void SetShadow(const void *ptr, uptr size, u8 value); + +// Set origin for the memory region. +void SetOrigin(const void *dst, uptr size, u32 origin); + +// Mark memory region uninitialized, with origins. +void PoisonMemory(const void *dst, uptr size, StackTrace *stack); + +} // namespace __msan + +#endif // MSAN_POISONING_H diff --git a/contrib/compiler-rt/lib/msan/msan_report.cc b/contrib/compiler-rt/lib/msan/msan_report.cc index 717c4a958c8f..33c28b2fba0e 100644 --- a/contrib/compiler-rt/lib/msan/msan_report.cc +++ b/contrib/compiler-rt/lib/msan/msan_report.cc @@ -75,8 +75,23 @@ static void DescribeOrigin(u32 id) { DescribeStackOrigin(so, pc); } else { StackTrace stack = o.getStackTraceForHeapOrigin(); - Printf(" %sUninitialized value was created by a heap allocation%s\n", - d.Origin(), d.End()); + switch (stack.tag) { + case StackTrace::TAG_ALLOC: + Printf(" %sUninitialized value was created by a heap allocation%s\n", + d.Origin(), d.End()); + break; + case StackTrace::TAG_DEALLOC: + Printf(" %sUninitialized value was created by a heap deallocation%s\n", + d.Origin(), d.End()); + break; + case STACK_TRACE_TAG_POISON: + Printf(" %sMemory was marked as uninitialized%s\n", d.Origin(), + d.End()); + break; + default: + Printf(" %sUninitialized value was created%s\n", d.Origin(), d.End()); + break; + } stack.Print(); } } @@ -255,7 +270,7 @@ void ReportUMRInsideAddressRange(const char *what, const void *start, uptr size, Printf("%sUninitialized bytes in %s%s%s at offset %zu inside [%p, %zu)%s\n", d.Warning(), d.Name(), what, d.Warning(), offset, start, size, d.End()); - if (__sanitizer::common_flags()->verbosity > 0) + if (__sanitizer::Verbosity()) DescribeMemoryRange(start, size); } diff --git a/contrib/compiler-rt/lib/msan/msan_thread.cc b/contrib/compiler-rt/lib/msan/msan_thread.cc index f29a4b053a36..e15a247c6bb8 100644 --- a/contrib/compiler-rt/lib/msan/msan_thread.cc +++ b/contrib/compiler-rt/lib/msan/msan_thread.cc @@ -79,15 +79,4 @@ thread_return_t MsanThread::ThreadStart() { return res; } -MsanThread *GetCurrentThread() { - return reinterpret_cast(MsanTSDGet()); -} - -void SetCurrentThread(MsanThread *t) { - // Make sure we do not reset the current MsanThread. - CHECK_EQ(0, MsanTSDGet()); - MsanTSDSet(t); - CHECK_EQ(t, MsanTSDGet()); -} - } // namespace __msan diff --git a/contrib/compiler-rt/lib/msan/tests/msan_test.cc b/contrib/compiler-rt/lib/msan/tests/msan_test.cc index 554265da6aa9..317f70cbc32e 100644 --- a/contrib/compiler-rt/lib/msan/tests/msan_test.cc +++ b/contrib/compiler-rt/lib/msan/tests/msan_test.cc @@ -21,13 +21,25 @@ #include "sanitizer/allocator_interface.h" #include "sanitizer/msan_interface.h" +#if defined(__FreeBSD__) +# define _KERNEL // To declare 'shminfo' structure. +# include +# undef _KERNEL +extern "C" { +// doesn't declare these functions in _KERNEL mode. +void *shmat(int, const void *, int); +int shmget(key_t, size_t, int); +int shmctl(int, int, struct shmid_ds *); +int shmdt(const void *); +} +#endif + #include #include #include #include #include #include -#include #include #include @@ -43,20 +55,40 @@ #include #include #include -#include #include #include -#include #include #include #include #include #include -#include -#include #include #include +#if !defined(__FreeBSD__) +# include +# include +# include +# include +# include +#else +# include +# include +# include +# include +# include +# include +# include +# define f_namelen f_namemax // FreeBSD names this statfs field so. +# define cpu_set_t cpuset_t +extern "C" { +// FreeBSD's defines mempcpy() to be a macro expanding into +// a __builtin___mempcpy_chk() call, but since Msan RTL defines it as an +// ordinary function, we can declare it here to complete the tests. +void *mempcpy(void *dest, const void *src, size_t n); +} +#endif + #if defined(__i386__) || defined(__x86_64__) # include # define MSAN_HAS_M128 1 @@ -68,7 +100,23 @@ # include #endif -static const size_t kPageSize = 4096; +// On FreeBSD procfs is not enabled by default. +#if defined(__FreeBSD__) +# define FILE_TO_READ "/bin/cat" +# define DIR_TO_READ "/bin" +# define SUBFILE_TO_READ "cat" +# define SYMLINK_TO_READ "/usr/bin/tar" +# define SUPERUSER_GROUP "wheel" +#else +# define FILE_TO_READ "/proc/self/stat" +# define DIR_TO_READ "/proc/self" +# define SUBFILE_TO_READ "stat" +# define SYMLINK_TO_READ "/proc/self/exe" +# define SUPERUSER_GROUP "root" +#endif + +const size_t kPageSize = 4096; +const size_t kMaxPathLength = 4096; typedef unsigned char U1; typedef unsigned short U2; // NOLINT @@ -122,11 +170,11 @@ void ExpectPoisonedWithOrigin(const T& t, unsigned origin) { EXPECT_EQ(origin, __msan_get_origin((void*)&t)); } -#define EXPECT_NOT_POISONED(x) ExpectNotPoisoned(x) +#define EXPECT_NOT_POISONED(x) EXPECT_EQ(true, TestForNotPoisoned((x))) template -void ExpectNotPoisoned(const T& t) { - EXPECT_EQ(-1, __msan_test_shadow((void*)&t, sizeof(t))); +bool TestForNotPoisoned(const T& t) { + return __msan_test_shadow((void*)&t, sizeof(t)) == -1; } static U8 poisoned_array[100]; @@ -493,10 +541,9 @@ static char *DynRetTestStr; TEST(MemorySanitizer, DynRet) { ReturnPoisoned(); - EXPECT_NOT_POISONED(clearenv()); + EXPECT_NOT_POISONED(atoi("0")); } - TEST(MemorySanitizer, DynRet1) { ReturnPoisoned(); } @@ -551,7 +598,7 @@ TEST(MemorySanitizer, strerror) { TEST(MemorySanitizer, strerror_r) { errno = 0; char buf[1000]; - char *res = strerror_r(EINVAL, buf, sizeof(buf)); + char *res = (char*) (size_t) strerror_r(EINVAL, buf, sizeof(buf)); ASSERT_EQ(0, errno); if (!res) res = buf; // POSIX version success. EXPECT_NOT_POISONED(strlen(res)); @@ -559,7 +606,7 @@ TEST(MemorySanitizer, strerror_r) { TEST(MemorySanitizer, fread) { char *x = new char[32]; - FILE *f = fopen("/proc/self/stat", "r"); + FILE *f = fopen(FILE_TO_READ, "r"); ASSERT_TRUE(f != NULL); fread(x, 1, 32, f); EXPECT_NOT_POISONED(x[0]); @@ -571,7 +618,7 @@ TEST(MemorySanitizer, fread) { TEST(MemorySanitizer, read) { char *x = new char[32]; - int fd = open("/proc/self/stat", O_RDONLY); + int fd = open(FILE_TO_READ, O_RDONLY); ASSERT_GT(fd, 0); int sz = read(fd, x, 32); ASSERT_EQ(sz, 32); @@ -584,7 +631,7 @@ TEST(MemorySanitizer, read) { TEST(MemorySanitizer, pread) { char *x = new char[32]; - int fd = open("/proc/self/stat", O_RDONLY); + int fd = open(FILE_TO_READ, O_RDONLY); ASSERT_GT(fd, 0); int sz = pread(fd, x, 32, 0); ASSERT_EQ(sz, 32); @@ -602,11 +649,11 @@ TEST(MemorySanitizer, readv) { iov[0].iov_len = 5; iov[1].iov_base = buf + 10; iov[1].iov_len = 2000; - int fd = open("/proc/self/stat", O_RDONLY); + int fd = open(FILE_TO_READ, O_RDONLY); ASSERT_GT(fd, 0); int sz = readv(fd, iov, 2); ASSERT_GE(sz, 0); - ASSERT_LT(sz, 5 + 2000); + ASSERT_LE(sz, 5 + 2000); ASSERT_GT((size_t)sz, iov[0].iov_len); EXPECT_POISONED(buf[0]); EXPECT_NOT_POISONED(buf[1]); @@ -626,11 +673,11 @@ TEST(MemorySanitizer, preadv) { iov[0].iov_len = 5; iov[1].iov_base = buf + 10; iov[1].iov_len = 2000; - int fd = open("/proc/self/stat", O_RDONLY); + int fd = open(FILE_TO_READ, O_RDONLY); ASSERT_GT(fd, 0); int sz = preadv(fd, iov, 2, 3); ASSERT_GE(sz, 0); - ASSERT_LT(sz, 5 + 2000); + ASSERT_LE(sz, 5 + 2000); ASSERT_GT((size_t)sz, iov[0].iov_len); EXPECT_POISONED(buf[0]); EXPECT_NOT_POISONED(buf[1]); @@ -652,15 +699,14 @@ TEST(MemorySanitizer, DISABLED_ioctl) { TEST(MemorySanitizer, readlink) { char *x = new char[1000]; - readlink("/proc/self/exe", x, 1000); + readlink(SYMLINK_TO_READ, x, 1000); EXPECT_NOT_POISONED(x[0]); delete [] x; } - TEST(MemorySanitizer, stat) { struct stat* st = new struct stat; - int res = stat("/proc/self/stat", st); + int res = stat(FILE_TO_READ, st); ASSERT_EQ(0, res); EXPECT_NOT_POISONED(st->st_dev); EXPECT_NOT_POISONED(st->st_mode); @@ -669,9 +715,9 @@ TEST(MemorySanitizer, stat) { TEST(MemorySanitizer, fstatat) { struct stat* st = new struct stat; - int dirfd = open("/proc/self", O_RDONLY); + int dirfd = open(DIR_TO_READ, O_RDONLY); ASSERT_GT(dirfd, 0); - int res = fstatat(dirfd, "stat", st, 0); + int res = fstatat(dirfd, SUBFILE_TO_READ, st, 0); ASSERT_EQ(0, res); EXPECT_NOT_POISONED(st->st_dev); EXPECT_NOT_POISONED(st->st_mode); @@ -763,6 +809,8 @@ TEST(MemorySanitizer, poll) { close(pipefd[1]); } +// There is no ppoll() on FreeBSD. +#if !defined (__FreeBSD__) TEST(MemorySanitizer, ppoll) { int* pipefd = new int[2]; int res = pipe(pipefd); @@ -787,6 +835,7 @@ TEST(MemorySanitizer, ppoll) { close(pipefd[0]); close(pipefd[1]); } +#endif TEST(MemorySanitizer, poll_positive) { int* pipefd = new int[2]; @@ -851,8 +900,11 @@ TEST(MemorySanitizer, accept) { res = fcntl(connect_socket, F_SETFL, O_NONBLOCK); ASSERT_EQ(0, res); res = connect(connect_socket, (struct sockaddr *)&sai, sizeof(sai)); - ASSERT_EQ(-1, res); - ASSERT_EQ(EINPROGRESS, errno); + // On FreeBSD this connection completes immediately. + if (res != 0) { + ASSERT_EQ(-1, res); + ASSERT_EQ(EINPROGRESS, errno); + } __msan_poison(&sai, sizeof(sai)); int new_sock = accept(listen_socket, (struct sockaddr *)&sai, &sz); @@ -973,7 +1025,6 @@ TEST(MemorySanitizer, recvmsg) { ASSERT_EQ(0, res); ASSERT_EQ(sizeof(client_sai), sz); - const char *s = "message text"; struct iovec iov; iov.iov_base = (void *)s; @@ -1125,12 +1176,15 @@ TEST(MemorySanitizer, getcwd_gnu) { free(res); } +// There's no get_current_dir_name() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, get_current_dir_name) { char* res = get_current_dir_name(); ASSERT_TRUE(res != NULL); EXPECT_NOT_POISONED(res[0]); free(res); } +#endif TEST(MemorySanitizer, shmctl) { int id = shmget(IPC_PRIVATE, 4096, 0644 | IPC_CREAT); @@ -1141,6 +1195,8 @@ TEST(MemorySanitizer, shmctl) { ASSERT_GT(res, -1); EXPECT_NOT_POISONED(ds); + // FreeBSD does not support shmctl(IPC_INFO) and shmctl(SHM_INFO). +#if !defined(__FreeBSD__) struct shminfo si; res = shmctl(id, IPC_INFO, (struct shmid_ds *)&si); ASSERT_GT(res, -1); @@ -1150,6 +1206,7 @@ TEST(MemorySanitizer, shmctl) { res = shmctl(id, SHM_INFO, (struct shmid_ds *)&s_i); ASSERT_GT(res, -1); EXPECT_NOT_POISONED(s_i); +#endif res = shmctl(id, IPC_RMID, 0); ASSERT_GT(res, -1); @@ -1157,7 +1214,7 @@ TEST(MemorySanitizer, shmctl) { TEST(MemorySanitizer, shmat) { void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(MAP_FAILED, p); ((char *)p)[10] = *GetPoisoned(); @@ -1183,6 +1240,8 @@ TEST(MemorySanitizer, shmat) { ASSERT_GT(res, -1); } +// There's no random_r() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, random_r) { int32_t x; char z[64]; @@ -1198,6 +1257,7 @@ TEST(MemorySanitizer, random_r) { ASSERT_EQ(0, res); EXPECT_NOT_POISONED(x); } +#endif TEST(MemorySanitizer, confstr) { char buf[3]; @@ -1215,6 +1275,16 @@ TEST(MemorySanitizer, confstr) { ASSERT_EQ(res, strlen(buf2) + 1); } +TEST(MemorySanitizer, opendir) { + DIR *dir = opendir("."); + closedir(dir); + + char name[10] = "."; + __msan_poison(name, sizeof(name)); + EXPECT_UMR(dir = opendir(name)); + closedir(dir); +} + TEST(MemorySanitizer, readdir) { DIR *dir = opendir("."); struct dirent *d = readdir(dir); @@ -1251,6 +1321,8 @@ TEST(MemorySanitizer, realpath_null) { free(res); } +// There's no canonicalize_file_name() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, canonicalize_file_name) { const char* relpath = "."; char* res = canonicalize_file_name(relpath); @@ -1258,6 +1330,7 @@ TEST(MemorySanitizer, canonicalize_file_name) { EXPECT_NOT_POISONED(res[0]); free(res); } +#endif extern char **environ; @@ -1655,26 +1728,35 @@ TEST(MemorySanitizer, modfl) { EXPECT_NOT_POISONED(y); } +// There's no sincos() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, sincos) { double s, c; sincos(0.2, &s, &c); EXPECT_NOT_POISONED(s); EXPECT_NOT_POISONED(c); } +#endif +// There's no sincosf() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, sincosf) { float s, c; sincosf(0.2, &s, &c); EXPECT_NOT_POISONED(s); EXPECT_NOT_POISONED(c); } +#endif +// There's no sincosl() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, sincosl) { long double s, c; sincosl(0.2, &s, &c); EXPECT_NOT_POISONED(s); EXPECT_NOT_POISONED(c); } +#endif TEST(MemorySanitizer, remquo) { int quo; @@ -1729,13 +1811,18 @@ TEST(MemorySanitizer, lgammaf_r) { EXPECT_NOT_POISONED(sgn); } +// There's no lgammal_r() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, lgammal_r) { int sgn; long double res = lgammal_r(1.1, &sgn); ASSERT_NE(0.0, res); EXPECT_NOT_POISONED(sgn); } +#endif +// There's no drand48_r() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, drand48_r) { struct drand48_data buf; srand48_r(0, &buf); @@ -1743,7 +1830,10 @@ TEST(MemorySanitizer, drand48_r) { drand48_r(&buf, &d); EXPECT_NOT_POISONED(d); } +#endif +// There's no lrand48_r() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, lrand48_r) { struct drand48_data buf; srand48_r(0, &buf); @@ -1751,6 +1841,7 @@ TEST(MemorySanitizer, lrand48_r) { lrand48_r(&buf, &d); EXPECT_NOT_POISONED(d); } +#endif TEST(MemorySanitizer, sprintf) { // NOLINT char buff[10]; @@ -2015,6 +2106,8 @@ TEST(MemorySanitizer, localtime_r) { EXPECT_NE(0U, strlen(time.tm_zone)); } +// There's no getmntent() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, getmntent) { FILE *fp = setmntent("/etc/fstab", "r"); struct mntent *mnt = getmntent(fp); @@ -2027,7 +2120,10 @@ TEST(MemorySanitizer, getmntent) { EXPECT_NOT_POISONED(mnt->mnt_passno); fclose(fp); } +#endif +// There's no getmntent_r() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, getmntent_r) { FILE *fp = setmntent("/etc/fstab", "r"); struct mntent mntbuf; @@ -2042,6 +2138,7 @@ TEST(MemorySanitizer, getmntent_r) { EXPECT_NOT_POISONED(mnt->mnt_passno); fclose(fp); } +#endif TEST(MemorySanitizer, ether) { const char *asc = "11:22:33:44:55:66"; @@ -2081,6 +2178,8 @@ TEST(MemorySanitizer, mmap) { } } +// There's no fcvt() on FreeBSD. +#if !defined(__FreeBSD__) // FIXME: enable and add ecvt. // FIXME: check why msandr does nt handle fcvt. TEST(MemorySanitizer, fcvt) { @@ -2096,7 +2195,10 @@ TEST(MemorySanitizer, fcvt) { EXPECT_NOT_POISONED(str[0]); ASSERT_NE(0U, strlen(str)); } +#endif +// There's no fcvt_long() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, fcvt_long) { int a, b; break_optimization(&a); @@ -2110,7 +2212,7 @@ TEST(MemorySanitizer, fcvt_long) { EXPECT_NOT_POISONED(str[0]); ASSERT_NE(0U, strlen(str)); } - +#endif TEST(MemorySanitizer, memchr) { char x[10]; @@ -2712,9 +2814,20 @@ TEST(MemorySanitizer, getrusage) { EXPECT_NOT_POISONED(usage.ru_nivcsw); } -#ifdef __GLIBC__ -extern char *program_invocation_name; -#else // __GLIBC__ +#if defined(__FreeBSD__) +static void GetProgramPath(char *buf, size_t sz) { + int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + int res = sysctl(mib, 4, buf, &sz, NULL, 0); + ASSERT_EQ(0, res); +} +#elif defined(__GLIBC__) +static void GetProgramPath(char *buf, size_t sz) { + extern char *program_invocation_name; + int res = snprintf(buf, sz, "%s", program_invocation_name); + ASSERT_GE(res, 0); + ASSERT_LT((size_t)res, sz); +} +#else # error "TODO: port this" #endif @@ -2749,21 +2862,29 @@ static int dl_phdr_callback(struct dl_phdr_info *info, size_t size, void *data) // Compute the path to our loadable DSO. We assume it's in the same // directory. Only use string routines that we intercept so far to do this. -static int PathToLoadable(char *buf, size_t sz) { - const char *basename = "libmsan_loadable.x86_64.so"; - char *argv0 = program_invocation_name; - char *last_slash = strrchr(argv0, '/'); - assert(last_slash); - int res = - snprintf(buf, sz, "%.*s/%s", int(last_slash - argv0), argv0, basename); - assert(res >= 0); - return (size_t)res < sz ? 0 : res; +static void GetPathToLoadable(char *buf, size_t sz) { + char program_path[kMaxPathLength]; + GetProgramPath(program_path, sizeof(program_path)); + + const char *last_slash = strrchr(program_path, '/'); + ASSERT_NE(nullptr, last_slash); + size_t dir_len = (size_t)(last_slash - program_path); +#if defined(__x86_64__) + static const char basename[] = "libmsan_loadable.x86_64.so"; +#elif defined(__MIPSEB__) || defined(MIPSEB) + static const char basename[] = "libmsan_loadable.mips64.so"; +#elif defined(__mips64) + static const char basename[] = "libmsan_loadable.mips64el.so"; +#endif + int res = snprintf(buf, sz, "%.*s/%s", + (int)dir_len, program_path, basename); + ASSERT_GE(res, 0); + ASSERT_LT((size_t)res, sz); } TEST(MemorySanitizer, dl_iterate_phdr) { - char path[4096]; - int res = PathToLoadable(path, sizeof(path)); - ASSERT_EQ(0, res); + char path[kMaxPathLength]; + GetPathToLoadable(path, sizeof(path)); // Having at least one dlopen'ed library in the process makes this more // entertaining. @@ -2773,15 +2894,13 @@ TEST(MemorySanitizer, dl_iterate_phdr) { int count = 0; int result = dl_iterate_phdr(dl_phdr_callback, &count); ASSERT_GT(count, 0); - + dlclose(lib); } - TEST(MemorySanitizer, dlopen) { - char path[4096]; - int res = PathToLoadable(path, sizeof(path)); - ASSERT_EQ(0, res); + char path[kMaxPathLength]; + GetPathToLoadable(path, sizeof(path)); // We need to clear shadow for globals when doing dlopen. In order to test // this, we have to poison the shadow for the DSO before we load it. In @@ -2806,19 +2925,22 @@ TEST(MemorySanitizer, dlopen) { // Regression test for a crash in dlopen() interceptor. TEST(MemorySanitizer, dlopenFailed) { - const char *path = "/libmsan_loadable_does_not_exist.x86_64.so"; + const char *path = "/libmsan_loadable_does_not_exist.so"; void *lib = dlopen(path, RTLD_LAZY); ASSERT_TRUE(lib == NULL); } #endif // MSAN_TEST_DISABLE_DLOPEN +// There's no sched_getaffinity() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, sched_getaffinity) { cpu_set_t mask; int res = sched_getaffinity(getpid(), sizeof(mask), &mask); ASSERT_EQ(0, res); EXPECT_NOT_POISONED(mask); } +#endif TEST(MemorySanitizer, scanf) { const char *input = "42 hello"; @@ -3048,11 +3170,14 @@ TEST(MemorySanitizer, posix_memalign) { free(p); } +// There's no memalign() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, memalign) { void *p = memalign(4096, 13); EXPECT_EQ(0U, (uintptr_t)p % kPageSize); free(p); } +#endif TEST(MemorySanitizer, valloc) { void *a = valloc(100); @@ -3060,6 +3185,8 @@ TEST(MemorySanitizer, valloc) { free(a); } +// There's no pvalloc() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, pvalloc) { void *p = pvalloc(kPageSize + 100); EXPECT_EQ(0U, (uintptr_t)p % kPageSize); @@ -3071,6 +3198,7 @@ TEST(MemorySanitizer, pvalloc) { EXPECT_EQ(kPageSize, __sanitizer_get_allocated_size(p)); free(p); } +#endif TEST(MemorySanitizer, inet_pton) { const char *s = "1:0:0:0:0:0:0:8"; @@ -3114,12 +3242,15 @@ TEST(MemorySanitizer, gethostname) { EXPECT_NOT_POISONED(strlen(buf)); } +// There's no sysinfo() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, sysinfo) { struct sysinfo info; int res = sysinfo(&info); ASSERT_EQ(0, res); EXPECT_NOT_POISONED(info); } +#endif TEST(MemorySanitizer, getpwuid) { struct passwd *p = getpwuid(0); // root @@ -3174,8 +3305,10 @@ TEST(MemorySanitizer, getgrnam_r) { struct group grp; struct group *grpres; char buf[10000]; - int res = getgrnam_r("root", &grp, buf, sizeof(buf), &grpres); + int res = getgrnam_r(SUPERUSER_GROUP, &grp, buf, sizeof(buf), &grpres); ASSERT_EQ(0, res); + // Note that getgrnam_r() returns 0 if the matching group is not found. + ASSERT_NE(nullptr, grpres); EXPECT_NOT_POISONED(grp.gr_name); ASSERT_TRUE(grp.gr_name != NULL); EXPECT_NOT_POISONED(grp.gr_name[0]); @@ -3207,6 +3340,8 @@ TEST(MemorySanitizer, getpwent_r) { EXPECT_NOT_POISONED(pwdres); } +// There's no fgetpwent() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, fgetpwent) { FILE *fp = fopen("/etc/passwd", "r"); struct passwd *p = fgetpwent(fp); @@ -3217,6 +3352,7 @@ TEST(MemorySanitizer, fgetpwent) { EXPECT_NOT_POISONED(p->pw_uid); fclose(fp); } +#endif TEST(MemorySanitizer, getgrent) { setgrent(); @@ -3228,6 +3364,8 @@ TEST(MemorySanitizer, getgrent) { EXPECT_NOT_POISONED(p->gr_gid); } +// There's no fgetgrent() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, fgetgrent) { FILE *fp = fopen("/etc/group", "r"); struct group *grp = fgetgrent(fp); @@ -3242,6 +3380,7 @@ TEST(MemorySanitizer, fgetgrent) { } fclose(fp); } +#endif TEST(MemorySanitizer, getgrent_r) { struct group grp; @@ -3257,6 +3396,8 @@ TEST(MemorySanitizer, getgrent_r) { EXPECT_NOT_POISONED(grpres); } +// There's no fgetgrent_r() on FreeBSD. +#if !defined(__FreeBSD__) TEST(MemorySanitizer, fgetgrent_r) { FILE *fp = fopen("/etc/group", "r"); struct group grp; @@ -3272,6 +3413,7 @@ TEST(MemorySanitizer, fgetgrent_r) { EXPECT_NOT_POISONED(grpres); fclose(fp); } +#endif TEST(MemorySanitizer, getgroups) { int n = getgroups(0, 0); @@ -3399,7 +3541,7 @@ TEST(MemorySanitizer, VolatileBitfield) { } TEST(MemorySanitizer, UnalignedLoad) { - char x[32]; + char x[32] __attribute__((aligned(8))); U4 origin = __LINE__; for (unsigned i = 0; i < sizeof(x) / 4; ++i) __msan_set_origin(x + 4 * i, 4, origin + i); @@ -3433,7 +3575,7 @@ TEST(MemorySanitizer, UnalignedLoad) { } TEST(MemorySanitizer, UnalignedStore16) { - char x[5]; + char x[5] __attribute__((aligned(4))); U2 y2 = 0; U4 origin = __LINE__; __msan_poison(&y2, 1); @@ -3444,11 +3586,10 @@ TEST(MemorySanitizer, UnalignedStore16) { EXPECT_POISONED_O(x[1], origin); EXPECT_NOT_POISONED(x[2]); EXPECT_POISONED_O(x[3], origin); - EXPECT_POISONED_O(x[4], origin); } TEST(MemorySanitizer, UnalignedStore32) { - char x[8]; + char x[8] __attribute__((aligned(4))); U4 y4 = 0; U4 origin = __LINE__; __msan_poison(&y4, 2); @@ -3466,7 +3607,7 @@ TEST(MemorySanitizer, UnalignedStore32) { } TEST(MemorySanitizer, UnalignedStore64) { - char x[16]; + char x[16] __attribute__((aligned(8))); U8 y8 = 0; U4 origin = __LINE__; __msan_poison(&y8, 3); @@ -3489,7 +3630,7 @@ TEST(MemorySanitizer, UnalignedStore64) { } TEST(MemorySanitizer, UnalignedStore16_precise) { - char x[8]; + char x[8] __attribute__((aligned(4))); U2 y = 0; U4 originx1 = __LINE__; U4 originx2 = __LINE__; @@ -3512,7 +3653,7 @@ TEST(MemorySanitizer, UnalignedStore16_precise) { } TEST(MemorySanitizer, UnalignedStore16_precise2) { - char x[8]; + char x[8] __attribute__((aligned(4))); U2 y = 0; U4 originx1 = __LINE__; U4 originx2 = __LINE__; @@ -3535,7 +3676,7 @@ TEST(MemorySanitizer, UnalignedStore16_precise2) { } TEST(MemorySanitizer, UnalignedStore64_precise) { - char x[12]; + char x[12] __attribute__((aligned(8))); U8 y = 0; U4 originx1 = __LINE__; U4 originx2 = __LINE__; @@ -3567,7 +3708,7 @@ TEST(MemorySanitizer, UnalignedStore64_precise) { } TEST(MemorySanitizer, UnalignedStore64_precise2) { - char x[12]; + char x[12] __attribute__((aligned(8))); U8 y = 0; U4 originx1 = __LINE__; U4 originx2 = __LINE__; @@ -3597,7 +3738,7 @@ TEST(MemorySanitizer, UnalignedStore64_precise2) { EXPECT_POISONED_O(x[11], originx3); } -#if defined(__clang__) +#if (defined(__x86_64__) && defined(__clang__)) namespace { typedef U1 V16x8 __attribute__((__vector_size__(16))); typedef U2 V8x16 __attribute__((__vector_size__(16))); @@ -4116,7 +4257,8 @@ TEST(MemorySanitizer, LargeAllocatorUnpoisonsOnFree) { // Allocate the page that was released to the OS in free() with the real mmap, // bypassing the interceptor. - char *q = (char *)real_mmap(p, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + char *q = (char *)real_mmap(p, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE((char *)0, q); ASSERT_TRUE(q <= p); diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c index 5aef3904b56d..346665fd5b3e 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c @@ -11,6 +11,7 @@ #include #include #include +#include #define UNCONST(ptr) ((void *)(uintptr_t)(ptr)) @@ -175,7 +176,11 @@ int __llvm_profile_write_file(void) { return -1; /* Write the file. */ - return writeFileWithName(__llvm_profile_CurrentFilename); + int rc = writeFileWithName(__llvm_profile_CurrentFilename); + if (rc && getenv("LLVM_PROFILE_VERBOSE_ERRORS")) + fprintf(stderr, "LLVM Profile: Failed to write file \"%s\": %s\n", + __llvm_profile_CurrentFilename, strerror(errno)); + return rc; } static void writeFileWithoutReturn(void) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc index 47509f83665b..03b3e83153de 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc @@ -14,7 +14,6 @@ #include "sanitizer_allocator.h" #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" -#include "sanitizer_flags.h" namespace __sanitizer { @@ -61,7 +60,7 @@ InternalAllocator *internal_allocator() { SpinMutexLock l(&internal_alloc_init_mu); if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) == 0) { - internal_allocator_instance->Init(); + internal_allocator_instance->Init(/* may_return_null*/ false); atomic_store(&internal_allocator_initialized, 1, memory_order_release); } } @@ -140,14 +139,12 @@ bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) { return (max / size) < n; } -void *AllocatorReturnNull() { - if (common_flags()->allocator_may_return_null) - return 0; +void NORETURN ReportAllocatorCannotReturnNull() { Report("%s's allocator is terminating the process instead of returning 0\n", SanitizerToolName); Report("If you don't like this behavior set allocator_may_return_null=1\n"); CHECK(0); - return 0; + Die(); } } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index 23218016b791..b5105f8c2555 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -23,8 +23,8 @@ namespace __sanitizer { -// Depending on allocator_may_return_null either return 0 or crash. -void *AllocatorReturnNull(); +// Prints error message and kills the program. +void NORETURN ReportAllocatorCannotReturnNull(); // SizeClassMap maps allocation sizes into size classes and back. // Class 0 corresponds to size 0. @@ -211,6 +211,7 @@ class AllocatorStats { void Init() { internal_memset(this, 0, sizeof(*this)); } + void InitLinkerInitialized() {} void Add(AllocatorStat i, uptr v) { v += atomic_load(&stats_[i], memory_order_relaxed); @@ -240,11 +241,14 @@ class AllocatorStats { // Global stats, used for aggregation and querying. class AllocatorGlobalStats : public AllocatorStats { public: - void Init() { - internal_memset(this, 0, sizeof(*this)); + void InitLinkerInitialized() { next_ = this; prev_ = this; } + void Init() { + internal_memset(this, 0, sizeof(*this)); + InitLinkerInitialized(); + } void Register(AllocatorStats *s) { SpinMutexLock l(&mu_); @@ -1002,9 +1006,14 @@ struct SizeClassAllocatorLocalCache { template class LargeMmapAllocator { public: - void Init() { - internal_memset(this, 0, sizeof(*this)); + void InitLinkerInitialized(bool may_return_null) { page_size_ = GetPageSizeCached(); + atomic_store(&may_return_null_, may_return_null, memory_order_relaxed); + } + + void Init(bool may_return_null) { + internal_memset(this, 0, sizeof(*this)); + InitLinkerInitialized(may_return_null); } void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) { @@ -1012,7 +1021,9 @@ class LargeMmapAllocator { uptr map_size = RoundUpMapSize(size); if (alignment > page_size_) map_size += alignment; - if (map_size < size) return AllocatorReturnNull(); // Overflow. + // Overflow. + if (map_size < size) + return ReturnNullOrDie(); uptr map_beg = reinterpret_cast( MmapOrDie(map_size, "LargeMmapAllocator")); CHECK(IsAligned(map_beg, page_size_)); @@ -1048,6 +1059,16 @@ class LargeMmapAllocator { return reinterpret_cast(res); } + void *ReturnNullOrDie() { + if (atomic_load(&may_return_null_, memory_order_acquire)) + return 0; + ReportAllocatorCannotReturnNull(); + } + + void SetMayReturnNull(bool may_return_null) { + atomic_store(&may_return_null_, may_return_null, memory_order_release); + } + void Deallocate(AllocatorStats *stat, void *p) { Header *h = GetHeader(p); { @@ -1226,6 +1247,7 @@ class LargeMmapAllocator { struct Stats { uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64]; } stats; + atomic_uint8_t may_return_null_; SpinMutex mutex_; }; @@ -1239,19 +1261,32 @@ template // NOLINT class CombinedAllocator { public: - void Init() { + void InitCommon(bool may_return_null) { primary_.Init(); - secondary_.Init(); + atomic_store(&may_return_null_, may_return_null, memory_order_relaxed); + } + + void InitLinkerInitialized(bool may_return_null) { + secondary_.InitLinkerInitialized(may_return_null); + stats_.InitLinkerInitialized(); + InitCommon(may_return_null); + } + + void Init(bool may_return_null) { + secondary_.Init(may_return_null); stats_.Init(); + InitCommon(may_return_null); } void *Allocate(AllocatorCache *cache, uptr size, uptr alignment, - bool cleared = false) { + bool cleared = false, bool check_rss_limit = false) { // Returning 0 on malloc(0) may break a lot of code. if (size == 0) size = 1; if (size + alignment < size) - return AllocatorReturnNull(); + return ReturnNullOrDie(); + if (check_rss_limit && RssLimitIsExceeded()) + return ReturnNullOrDie(); if (alignment > 8) size = RoundUpTo(size, alignment); void *res; @@ -1267,6 +1302,30 @@ class CombinedAllocator { return res; } + bool MayReturnNull() const { + return atomic_load(&may_return_null_, memory_order_acquire); + } + + void *ReturnNullOrDie() { + if (MayReturnNull()) + return 0; + ReportAllocatorCannotReturnNull(); + } + + void SetMayReturnNull(bool may_return_null) { + secondary_.SetMayReturnNull(may_return_null); + atomic_store(&may_return_null_, may_return_null, memory_order_release); + } + + bool RssLimitIsExceeded() { + return atomic_load(&rss_limit_is_exceeded_, memory_order_acquire); + } + + void SetRssLimitIsExceeded(bool rss_limit_is_exceeded) { + atomic_store(&rss_limit_is_exceeded_, rss_limit_is_exceeded, + memory_order_release); + } + void Deallocate(AllocatorCache *cache, void *p) { if (!p) return; if (primary_.PointerIsMine(p)) @@ -1379,6 +1438,8 @@ class CombinedAllocator { PrimaryAllocator primary_; SecondaryAllocator secondary_; AllocatorGlobalStats stats_; + atomic_uint8_t may_return_null_; + atomic_uint8_t rss_limit_is_exceeded_; }; // Returns true if calloc(size, n) should return 0 due to overflow in size*n. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h index 4409fd65bf31..9b9cfd0b5931 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -49,6 +49,15 @@ void *InternalAlloc(uptr size, InternalAllocatorCache *cache = 0); void InternalFree(void *p, InternalAllocatorCache *cache = 0); InternalAllocator *internal_allocator(); +enum InternalAllocEnum { + INTERNAL_ALLOC +}; + } // namespace __sanitizer +inline void *operator new(__sanitizer::operator_new_size_type size, + InternalAllocEnum) { + return InternalAlloc(size); +} + #endif // SANITIZER_ALLOCATOR_INTERNAL_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc index c3740f24a366..4be3c7abf756 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc @@ -12,13 +12,17 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common.h" +#include "sanitizer_allocator_internal.h" #include "sanitizer_flags.h" #include "sanitizer_libc.h" +#include "sanitizer_placement_new.h" namespace __sanitizer { const char *SanitizerToolName = "SanitizerTool"; +atomic_uint32_t current_verbosity; + uptr GetPageSizeCached() { static uptr PageSize; if (!PageSize) @@ -94,19 +98,23 @@ uptr stoptheworld_tracer_pid = 0; // writing to the same log file. uptr stoptheworld_tracer_ppid = 0; -static DieCallbackType DieCallback; +static DieCallbackType InternalDieCallback, UserDieCallback; void SetDieCallback(DieCallbackType callback) { - DieCallback = callback; + InternalDieCallback = callback; +} +void SetUserDieCallback(DieCallbackType callback) { + UserDieCallback = callback; } DieCallbackType GetDieCallback() { - return DieCallback; + return InternalDieCallback; } void NORETURN Die() { - if (DieCallback) { - DieCallback(); - } + if (UserDieCallback) + UserDieCallback(); + if (InternalDieCallback) + InternalDieCallback(); internal__exit(1); } @@ -125,8 +133,8 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, Die(); } -uptr ReadFileToBuffer(const char *file_name, char **buff, - uptr *buff_size, uptr max_len) { +uptr ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr max_len, int *errno_p) { uptr PageSize = GetPageSizeCached(); uptr kMinFileLen = PageSize; uptr read_len = 0; @@ -135,7 +143,7 @@ uptr ReadFileToBuffer(const char *file_name, char **buff, // The files we usually open are not seekable, so try different buffer sizes. for (uptr size = kMinFileLen; size <= max_len; size *= 2) { uptr openrv = OpenFile(file_name, /*write*/ false); - if (internal_iserror(openrv)) return 0; + if (internal_iserror(openrv, errno_p)) return 0; fd_t fd = openrv; UnmapOrDie(*buff, *buff_size); *buff = (char*)MmapOrDie(size, __func__); @@ -145,6 +153,10 @@ uptr ReadFileToBuffer(const char *file_name, char **buff, bool reached_eof = false; while (read_len + PageSize <= size) { uptr just_read = internal_read(fd, *buff + read_len, PageSize); + if (internal_iserror(just_read, errno_p)) { + UnmapOrDie(*buff, *buff_size); + return 0; + } if (just_read == 0) { reached_eof = true; break; @@ -233,20 +245,28 @@ void ReportErrorSummary(const char *error_type, const char *file, LoadedModule::LoadedModule(const char *module_name, uptr base_address) { full_name_ = internal_strdup(module_name); base_address_ = base_address; - n_ranges_ = 0; + ranges_.clear(); +} + +void LoadedModule::clear() { + InternalFree(full_name_); + while (!ranges_.empty()) { + AddressRange *r = ranges_.front(); + ranges_.pop_front(); + InternalFree(r); + } } void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable) { - CHECK_LT(n_ranges_, kMaxNumberOfAddressRanges); - ranges_[n_ranges_].beg = beg; - ranges_[n_ranges_].end = end; - exec_[n_ranges_] = executable; - n_ranges_++; + void *mem = InternalAlloc(sizeof(AddressRange)); + AddressRange *r = new(mem) AddressRange(beg, end, executable); + ranges_.push_back(r); } bool LoadedModule::containsAddress(uptr address) const { - for (uptr i = 0; i < n_ranges_; i++) { - if (ranges_[i].beg <= address && address < ranges_[i].end) + for (Iterator iter = ranges(); iter.hasNext();) { + const AddressRange *r = iter.next(); + if (r->beg <= address && address < r->end) return true; } return false; @@ -268,6 +288,48 @@ void DecreaseTotalMmap(uptr size) { atomic_fetch_sub(&g_total_mmaped, size, memory_order_relaxed); } +bool TemplateMatch(const char *templ, const char *str) { + if (str == 0 || str[0] == 0) + return false; + bool start = false; + if (templ && templ[0] == '^') { + start = true; + templ++; + } + bool asterisk = false; + while (templ && templ[0]) { + if (templ[0] == '*') { + templ++; + start = false; + asterisk = true; + continue; + } + if (templ[0] == '$') + return str[0] == 0 || asterisk; + if (str[0] == 0) + return false; + char *tpos = (char*)internal_strchr(templ, '*'); + char *tpos1 = (char*)internal_strchr(templ, '$'); + if (tpos == 0 || (tpos1 && tpos1 < tpos)) + tpos = tpos1; + if (tpos != 0) + tpos[0] = 0; + const char *str0 = str; + const char *spos = internal_strstr(str, templ); + str = spos + internal_strlen(templ); + templ = tpos; + if (tpos) + tpos[0] = tpos == tpos1 ? '$' : '*'; + if (spos == 0) + return false; + if (start && spos != str0) + return false; + start = false; + asterisk = false; + } + return true; +} + } // namespace __sanitizer using namespace __sanitizer; // NOLINT @@ -280,4 +342,9 @@ void __sanitizer_set_report_path(const char *path) { void __sanitizer_report_error_summary(const char *error_summary) { Printf("%s\n", error_summary); } + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_set_death_callback(void (*callback)(void)) { + SetUserDieCallback(callback); +} } // extern "C" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index c00dce66bb07..ff13ef164045 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// // -// This file is shared between AddressSanitizer and ThreadSanitizer -// run-time libraries. +// This file is shared between run-time libraries of sanitizers. +// // It declares common functions and classes that are used in both runtimes. // Implementation of some functions are provided in sanitizer_common, while // others must be defined by run-time library itself. @@ -16,10 +16,12 @@ #ifndef SANITIZER_COMMON_H #define SANITIZER_COMMON_H +#include "sanitizer_flags.h" +#include "sanitizer_interface_internal.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" +#include "sanitizer_list.h" #include "sanitizer_mutex.h" -#include "sanitizer_flags.h" namespace __sanitizer { struct StackTrace; @@ -40,6 +42,14 @@ const uptr kMaxThreadStackSize = 1 << 30; // 1Gb extern const char *SanitizerToolName; // Can be changed by the tool. +extern atomic_uint32_t current_verbosity; +INLINE void SetVerbosity(int verbosity) { + atomic_store(¤t_verbosity, verbosity, memory_order_relaxed); +} +INLINE int Verbosity() { + return atomic_load(¤t_verbosity, memory_order_relaxed); +} + uptr GetPageSize(); uptr GetPageSizeCached(); uptr GetMmapGranularity(); @@ -67,6 +77,8 @@ void FlushUnneededShadowMemory(uptr addr, uptr size); void IncreaseTotalMmap(uptr size); void DecreaseTotalMmap(uptr size); uptr GetRSS(); +void NoHugePagesInRegion(uptr addr, uptr length); +void DontDumpShadowMemory(uptr addr, uptr length); // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. @@ -135,11 +147,11 @@ void Report(const char *format, ...); void SetPrintfAndReportCallback(void (*callback)(const char *)); #define VReport(level, ...) \ do { \ - if ((uptr)common_flags()->verbosity >= (level)) Report(__VA_ARGS__); \ + if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \ } while (0) #define VPrintf(level, ...) \ do { \ - if ((uptr)common_flags()->verbosity >= (level)) Printf(__VA_ARGS__); \ + if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \ } while (0) // Can be used to prevent mixing error reports from different sanitizers. @@ -179,8 +191,8 @@ uptr OpenFile(const char *filename, bool write); // The resulting buffer is mmaped and stored in '*buff'. // The size of the mmaped region is stored in '*buff_size', // Returns the number of read bytes or 0 if file can not be opened. -uptr ReadFileToBuffer(const char *file_name, char **buff, - uptr *buff_size, uptr max_len); +uptr ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr max_len, int *errno_p = nullptr); // Maps given file to virtual memory, and returns pointer to it // (or NULL if the mapping failes). Stores the size of mmaped region // in '*buff_size'. @@ -214,10 +226,13 @@ void PrepareForSandboxing(__sanitizer_sandbox_arguments *args); void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args); void SetSandboxingCallback(void (*f)()); -void CovUpdateMapping(uptr caller_pc = 0); +void CoverageUpdateMapping(); void CovBeforeFork(); void CovAfterFork(int child_pid); +void InitializeCoverage(bool enabled, const char *coverage_dir); +void ReInitializeCoverage(bool enabled, const char *coverage_dir); + void InitTlsSize(); uptr GetTlsSize(); @@ -227,6 +242,7 @@ void SleepForMillis(int millis); u64 NanoTime(); int Atexit(void (*function)(void)); void SortArray(uptr *array, uptr size); +bool TemplateMatch(const char *templ, const char *str); // Exit void NORETURN Abort(); @@ -245,11 +261,18 @@ bool SanitizerGetThreadName(char *name, int max_len); // to do tool-specific job. typedef void (*DieCallbackType)(void); void SetDieCallback(DieCallbackType); +void SetUserDieCallback(DieCallbackType); DieCallbackType GetDieCallback(); typedef void (*CheckFailedCallbackType)(const char *, int, const char *, u64, u64); void SetCheckFailedCallback(CheckFailedCallbackType callback); +// Callback will be called if soft_rss_limit_mb is given and the limit is +// exceeded (exceeded==true) or if rss went down below the limit +// (exceeded==false). +// The callback should be registered once at the tool init time. +void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)); + // Functions related to signal handling. typedef void (*SignalHandlerType)(int, void *, void *); bool IsDeadlySignal(int signum); @@ -376,14 +399,14 @@ INLINE int ToLower(int c) { // small vectors. // WARNING: The current implementation supports only POD types. template -class InternalMmapVector { +class InternalMmapVectorNoCtor { public: - explicit InternalMmapVector(uptr initial_capacity) { + void Initialize(uptr initial_capacity) { capacity_ = Max(initial_capacity, (uptr)1); size_ = 0; - data_ = (T *)MmapOrDie(capacity_ * sizeof(T), "InternalMmapVector"); + data_ = (T *)MmapOrDie(capacity_ * sizeof(T), "InternalMmapVectorNoCtor"); } - ~InternalMmapVector() { + void Destroy() { UnmapOrDie(data_, capacity_ * sizeof(T)); } T &operator[](uptr i) { @@ -434,15 +457,24 @@ class InternalMmapVector { UnmapOrDie(old_data, capacity_ * sizeof(T)); capacity_ = new_capacity; } - // Disallow evil constructors. - InternalMmapVector(const InternalMmapVector&); - void operator=(const InternalMmapVector&); T *data_; uptr capacity_; uptr size_; }; +template +class InternalMmapVector : public InternalMmapVectorNoCtor { + public: + explicit InternalMmapVector(uptr initial_capacity) { + InternalMmapVectorNoCtor::Initialize(initial_capacity); + } + ~InternalMmapVector() { InternalMmapVectorNoCtor::Destroy(); } + // Disallow evil constructors. + InternalMmapVector(const InternalMmapVector&); + void operator=(const InternalMmapVector&); +}; + // HeapSort for arrays and InternalMmapVector. template void InternalSort(Container *v, uptr size, Compare comp) { @@ -501,28 +533,30 @@ uptr InternalBinarySearch(const Container &v, uptr first, uptr last, class LoadedModule { public: LoadedModule(const char *module_name, uptr base_address); + void clear(); void addAddressRange(uptr beg, uptr end, bool executable); bool containsAddress(uptr address) const; const char *full_name() const { return full_name_; } uptr base_address() const { return base_address_; } - uptr n_ranges() const { return n_ranges_; } - uptr address_range_start(int i) const { return ranges_[i].beg; } - uptr address_range_end(int i) const { return ranges_[i].end; } - bool address_range_executable(int i) const { return exec_[i]; } - - private: struct AddressRange { + AddressRange *next; uptr beg; uptr end; + bool executable; + + AddressRange(uptr beg, uptr end, bool executable) + : next(nullptr), beg(beg), end(end), executable(executable) {} }; - char *full_name_; + + typedef IntrusiveList::ConstIterator Iterator; + Iterator ranges() const { return Iterator(&ranges_); } + + private: + char *full_name_; // Owned. uptr base_address_; - static const uptr kMaxNumberOfAddressRanges = 6; - AddressRange ranges_[kMaxNumberOfAddressRanges]; - bool exec_[kMaxNumberOfAddressRanges]; - uptr n_ranges_; + IntrusiveList ranges_; }; // OS-dependent function that fills array with descriptions of at most @@ -556,6 +590,10 @@ INLINE void GetExtraActivationFlags(char *buf, uptr size) { *buf = '\0'; } INLINE void SanitizerInitializeUnwinder() {} #endif +void *internal_start_thread(void(*func)(void*), void *arg); +void internal_join_thread(void *th); +void MaybeStartBackgroudThread(); + // Make the compiler think that something is going on there. // Use this inside a loop that looks like memset/memcpy/etc to prevent the // compiler from recognising it and turning it into an actual call to diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 274e87c3d67d..87c33e186320 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -17,6 +17,7 @@ // COMMON_INTERCEPTOR_READ_RANGE // COMMON_INTERCEPTOR_WRITE_RANGE // COMMON_INTERCEPTOR_INITIALIZE_RANGE +// COMMON_INTERCEPTOR_DIR_ACQUIRE // COMMON_INTERCEPTOR_FD_ACQUIRE // COMMON_INTERCEPTOR_FD_RELEASE // COMMON_INTERCEPTOR_FD_ACCESS @@ -43,6 +44,8 @@ #if SANITIZER_FREEBSD #define pthread_setname_np pthread_set_name_np +#define inet_aton __inet_aton +#define inet_pton __inet_pton #endif #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE @@ -82,7 +85,7 @@ #endif #ifndef COMMON_INTERCEPTOR_LIBRARY_LOADED -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, map) {} +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) {} #endif #ifndef COMMON_INTERCEPTOR_LIBRARY_UNLOADED @@ -915,6 +918,16 @@ INTERCEPTOR(int, vsnprintf, char *str, SIZE_T size, const char *format, va_list ap) VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap) +#if SANITIZER_INTERCEPT_PRINTF_L +INTERCEPTOR(int, vsnprintf_l, char *str, SIZE_T size, void *loc, + const char *format, va_list ap) +VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf_l, str, size, loc, format, ap) + +INTERCEPTOR(int, snprintf_l, char *str, SIZE_T size, void *loc, + const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(snprintf_l, vsnprintf_l, str, size, loc, format) +#endif // SANITIZER_INTERCEPT_PRINTF_L + INTERCEPTOR(int, vsprintf, char *str, const char *format, va_list ap) VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap) @@ -991,6 +1004,14 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size, #define INIT_PRINTF #endif +#if SANITIZER_INTERCEPT_PRINTF_L +#define INIT_PRINTF_L \ + COMMON_INTERCEPT_FUNCTION(snprintf_l); \ + COMMON_INTERCEPT_FUNCTION(vsnprintf_l); +#else +#define INIT_PRINTF_L +#endif + #if SANITIZER_INTERCEPT_ISOC99_PRINTF #define INIT_ISOC99_PRINTF \ COMMON_INTERCEPT_FUNCTION(__isoc99_printf); \ @@ -1007,8 +1028,12 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size, #if SANITIZER_INTERCEPT_IOCTL #include "sanitizer_common_interceptors_ioctl.inc" -INTERCEPTOR(int, ioctl, int d, unsigned request, void *arg) { +INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) { void *ctx; + va_list ap; + va_start(ap, request); + void *arg = va_arg(ap, void *); + va_end(ap); COMMON_INTERCEPTOR_ENTER(ctx, ioctl, d, request, arg); CHECK(ioctl_initialized); @@ -1017,6 +1042,10 @@ INTERCEPTOR(int, ioctl, int d, unsigned request, void *arg) { // This effectively disables ioctl handling in TSan. if (!common_flags()->handle_ioctl) return REAL(ioctl)(d, request, arg); + // Although request is unsigned long, the rest of the interceptor uses it + // as just "unsigned" to save space, because we know that all values fit in + // "unsigned" - they are compile-time constants. + const ioctl_desc *desc = ioctl_lookup(request); ioctl_desc decoded_desc; if (!desc) { @@ -2139,6 +2168,16 @@ INTERCEPTOR(int, sysinfo, void *info) { #endif #if SANITIZER_INTERCEPT_READDIR +INTERCEPTOR(__sanitizer_dirent *, opendir, const char *path) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, opendir, path); + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + __sanitizer_dirent *res = REAL(opendir)(path); + if (res != 0) + COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path); + return res; +} + INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, readdir, dirp); @@ -2167,6 +2206,7 @@ INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry, } #define INIT_READDIR \ + COMMON_INTERCEPT_FUNCTION(opendir); \ COMMON_INTERCEPT_FUNCTION(readdir); \ COMMON_INTERCEPT_FUNCTION(readdir_r); #else @@ -2560,6 +2600,19 @@ INTERCEPTOR(int, sched_getaffinity, int pid, SIZE_T cpusetsize, void *mask) { #define INIT_SCHED_GETAFFINITY #endif +#if SANITIZER_INTERCEPT_SCHED_GETPARAM +INTERCEPTOR(int, sched_getparam, int pid, void *param) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sched_getparam, pid, param); + int res = REAL(sched_getparam)(pid, param); + if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, param, struct_sched_param_sz); + return res; +} +#define INIT_SCHED_GETPARAM COMMON_INTERCEPT_FUNCTION(sched_getparam); +#else +#define INIT_SCHED_GETPARAM +#endif + #if SANITIZER_INTERCEPT_STRERROR INTERCEPTOR(char *, strerror, int errnum) { void *ctx; @@ -3868,6 +3921,12 @@ INTERCEPTOR(__sanitizer_clock_t, times, void *tms) { #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr) +// If you see any crashes around this functions, there are 2 known issues with +// it: 1. __tls_get_addr can be called with mis-aligned stack due to: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 +// 2. It can be called recursively if sanitizer code uses __tls_get_addr +// to access thread local variables (it should not happen normally, +// because sanitizers use initial-exec tls model). INTERCEPTOR(void *, __tls_get_addr, void *arg) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr, arg); @@ -4762,6 +4821,7 @@ static void InitializeCommonInterceptors() { INIT_SCANF; INIT_ISOC99_SCANF; INIT_PRINTF; + INIT_PRINTF_L; INIT_ISOC99_PRINTF; INIT_FREXP; INIT_FREXPF_FREXPL; @@ -4812,6 +4872,7 @@ static void InitializeCommonInterceptors() { INIT_CANONICALIZE_FILE_NAME; INIT_CONFSTR; INIT_SCHED_GETAFFINITY; + INIT_SCHED_GETPARAM; INIT_STRERROR; INIT_STRERROR_R; INIT_XPG_STRERROR_R; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc index 20c1d5a78987..17ef6897ba26 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -13,6 +13,7 @@ #include "sanitizer_common.h" #include "sanitizer_flags.h" +#include "sanitizer_stackdepot.h" #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" @@ -59,6 +60,71 @@ void ReportErrorSummary(const char *error_type, StackTrace *stack) { #endif } +static void (*SoftRssLimitExceededCallback)(bool exceeded); +void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) { + CHECK_EQ(SoftRssLimitExceededCallback, nullptr); + SoftRssLimitExceededCallback = Callback; +} + +void BackgroundThread(void *arg) { + uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb; + uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb; + uptr prev_reported_rss = 0; + uptr prev_reported_stack_depot_size = 0; + bool reached_soft_rss_limit = false; + while (true) { + SleepForMillis(100); + uptr current_rss_mb = GetRSS() >> 20; + if (Verbosity()) { + // If RSS has grown 10% since last time, print some information. + if (prev_reported_rss * 11 / 10 < current_rss_mb) { + Printf("%s: RSS: %zdMb\n", SanitizerToolName, current_rss_mb); + prev_reported_rss = current_rss_mb; + } + // If stack depot has grown 10% since last time, print it too. + StackDepotStats *stack_depot_stats = StackDepotGetStats(); + if (prev_reported_stack_depot_size * 11 / 10 < + stack_depot_stats->allocated) { + Printf("%s: StackDepot: %zd ids; %zdM allocated\n", + SanitizerToolName, + stack_depot_stats->n_uniq_ids, + stack_depot_stats->allocated >> 20); + prev_reported_stack_depot_size = stack_depot_stats->allocated; + } + } + // Check RSS against the limit. + if (hard_rss_limit_mb && hard_rss_limit_mb < current_rss_mb) { + Report("%s: hard rss limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, hard_rss_limit_mb, current_rss_mb); + DumpProcessMap(); + Die(); + } + if (soft_rss_limit_mb) { + if (soft_rss_limit_mb < current_rss_mb && !reached_soft_rss_limit) { + reached_soft_rss_limit = true; + Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, soft_rss_limit_mb, current_rss_mb); + if (SoftRssLimitExceededCallback) + SoftRssLimitExceededCallback(true); + } else if (soft_rss_limit_mb >= current_rss_mb && + reached_soft_rss_limit) { + reached_soft_rss_limit = false; + if (SoftRssLimitExceededCallback) + SoftRssLimitExceededCallback(false); + } + } + } +} + +void MaybeStartBackgroudThread() { + if (!SANITIZER_LINUX) return; // Need to implement/test on other platforms. + // Start the background thread if one of the rss limits is given. + if (!common_flags()->hard_rss_limit_mb && + !common_flags()->soft_rss_limit_mb) return; + if (!&real_pthread_create) return; // Can't spawn the thread anyway. + internal_start_thread(BackgroundThread, nullptr); +} + } // namespace __sanitizer void NOINLINE diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc index a52338b62f5e..7e15d51ff35b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc @@ -2297,7 +2297,8 @@ PRE_SYSCALL(ni_syscall)() {} POST_SYSCALL(ni_syscall)(long res) {} PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { -#if !SANITIZER_ANDROID && (defined(__i386) || defined (__x86_64)) +#if !SANITIZER_ANDROID && \ + (defined(__i386) || defined(__x86_64) || defined(__mips64)) if (data) { if (request == ptrace_setregs) { PRE_READ((void *)data, struct_user_regs_struct_sz); @@ -2316,7 +2317,8 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { } POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) { -#if !SANITIZER_ANDROID && (defined(__i386) || defined (__x86_64)) +#if !SANITIZER_ANDROID && \ + (defined(__i386) || defined(__x86_64) || defined(__mips64)) if (res >= 0 && data) { // Note that this is different from the interceptor in // sanitizer_common_interceptors.inc. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc index 3fa8a18696f8..49887b1e91a9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -12,14 +12,16 @@ // // Compiler instrumentation: // For every interesting basic block the compiler injects the following code: -// if (Guard) { +// if (Guard < 0) { // __sanitizer_cov(&Guard); // } +// At the module start up time __sanitizer_cov_module_init sets the guards +// to consecutive negative numbers (-1, -2, -3, ...). // It's fine to call __sanitizer_cov more than once for a given block. // // Run-time: // - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC). -// and atomically set Guard to 1. +// and atomically set Guard to -Guard. // - __sanitizer_cov_dump: dump the coverage data to disk. // For every module of the current process that has coverage data // this will create a file module_name.PID.sancov. The file format is simple: @@ -56,23 +58,32 @@ static atomic_uintptr_t coverage_counter; static bool cov_sandboxed = false; static int cov_fd = kInvalidFd; static unsigned int cov_max_block_size = 0; +static bool coverage_enabled = false; +static const char *coverage_dir; namespace __sanitizer { class CoverageData { public: void Init(); + void Enable(); + void Disable(); + void ReInit(); void BeforeFork(); void AfterFork(int child_pid); void Extend(uptr npcs); - void Add(uptr pc, u8 *guard); + void Add(uptr pc, u32 *guard); void IndirCall(uptr caller, uptr callee, uptr callee_cache[], uptr cache_size); void DumpCallerCalleePairs(); void DumpTrace(); ALWAYS_INLINE - void TraceBasicaBlock(uptr *cache); + void TraceBasicBlock(s32 *id); + + void InitializeGuardArray(s32 *guards); + void InitializeGuards(s32 *guards, uptr n, const char *module_name); + void ReinitializeGuards(); uptr *data(); uptr size(); @@ -80,7 +91,7 @@ class CoverageData { private: // Maximal size pc array may ever grow. // We MmapNoReserve this space to ensure that the array is contiguous. - static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); + static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 24, 1 << 27); // The amount file mapping for the pc array is grown by. static const uptr kPcArrayMmapSize = 64 * 1024; @@ -96,45 +107,44 @@ class CoverageData { // Descriptor of the file mapped pc array. int pc_fd; + // Vector of coverage guard arrays, protected by mu. + InternalMmapVectorNoCtor guard_array_vec; + + // Vector of module (compilation unit) names. + InternalMmapVectorNoCtor comp_unit_name_vec; + // Caller-Callee (cc) array, size and current index. static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24); uptr **cc_array; atomic_uintptr_t cc_array_index; atomic_uintptr_t cc_array_size; - // Tracing (tr) pc and event arrays, their size and current index. + // Tracing event array, size and current pointer. // We record all events (basic block entries) in a global buffer of u32 - // values. Each such value is an index in the table of TracedPc objects. + // values. Each such value is the index in pc_array. // So far the tracing is highly experimental: // - not thread-safe; // - does not support long traces; // - not tuned for performance. - struct TracedPc { - uptr pc; - const char *module_name; - uptr module_offset; - }; static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30); u32 *tr_event_array; uptr tr_event_array_size; - uptr tr_event_array_index; + u32 *tr_event_pointer; static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); - TracedPc *tr_pc_array; - uptr tr_pc_array_size; - uptr tr_pc_array_index; StaticSpinMutex mu; void DirectOpen(); - void ReInit(); }; static CoverageData coverage_data; +void CovUpdateMapping(const char *path, uptr caller_pc = 0); + void CoverageData::DirectOpen() { InternalScopedString path(kMaxPathLength); internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw", - common_flags()->coverage_dir, internal_getpid()); + coverage_dir, internal_getpid()); pc_fd = OpenFile(path.data(), true); if (internal_iserror(pc_fd)) { Report(" Coverage: failed to open %s for writing\n", path.data()); @@ -142,19 +152,23 @@ void CoverageData::DirectOpen() { } pc_array_mapped_size = 0; - CovUpdateMapping(); + CovUpdateMapping(coverage_dir); } void CoverageData::Init() { + pc_fd = kInvalidFd; +} + +void CoverageData::Enable() { + if (pc_array) + return; pc_array = reinterpret_cast( MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit")); - pc_fd = kInvalidFd; + atomic_store(&pc_array_index, 0, memory_order_relaxed); if (common_flags()->coverage_direct) { atomic_store(&pc_array_size, 0, memory_order_relaxed); - atomic_store(&pc_array_index, 0, memory_order_relaxed); } else { atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed); - atomic_store(&pc_array_index, 0, memory_order_relaxed); } cc_array = reinterpret_cast(MmapNoReserveOrDie( @@ -162,30 +176,72 @@ void CoverageData::Init() { atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed); atomic_store(&cc_array_index, 0, memory_order_relaxed); - tr_event_array = reinterpret_cast( - MmapNoReserveOrDie(sizeof(tr_event_array[0]) * kTrEventArrayMaxSize, - "CovInit::tr_event_array")); + // Allocate tr_event_array with a guard page at the end. + tr_event_array = reinterpret_cast(MmapNoReserveOrDie( + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(), + "CovInit::tr_event_array")); + Mprotect(reinterpret_cast(&tr_event_array[kTrEventArrayMaxSize]), + GetMmapGranularity()); tr_event_array_size = kTrEventArrayMaxSize; - tr_event_array_index = 0; + tr_event_pointer = tr_event_array; +} - tr_pc_array = reinterpret_cast(MmapNoReserveOrDie( - sizeof(tr_pc_array[0]) * kTrEventArrayMaxSize, "CovInit::tr_pc_array")); - tr_pc_array_size = kTrEventArrayMaxSize; - tr_pc_array_index = 0; +void CoverageData::InitializeGuardArray(s32 *guards) { + Enable(); // Make sure coverage is enabled at this point. + s32 n = guards[0]; + for (s32 j = 1; j <= n; j++) { + uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + guards[j] = -static_cast(idx + 1); + } +} + +void CoverageData::Disable() { + if (pc_array) { + internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize); + pc_array = nullptr; + } + if (cc_array) { + internal_munmap(cc_array, sizeof(uptr *) * kCcArrayMaxSize); + cc_array = nullptr; + } + if (tr_event_array) { + internal_munmap(tr_event_array, + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + + GetMmapGranularity()); + tr_event_array = nullptr; + tr_event_pointer = nullptr; + } + if (pc_fd != kInvalidFd) { + internal_close(pc_fd); + pc_fd = kInvalidFd; + } +} + +void CoverageData::ReinitializeGuards() { + // Assuming single thread. + atomic_store(&pc_array_index, 0, memory_order_relaxed); + for (uptr i = 0; i < guard_array_vec.size(); i++) + InitializeGuardArray(guard_array_vec[i]); } void CoverageData::ReInit() { - internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize); - if (pc_fd != kInvalidFd) internal_close(pc_fd); - if (common_flags()->coverage_direct) { - // In memory-mapped mode we must extend the new file to the known array - // size. - uptr size = atomic_load(&pc_array_size, memory_order_relaxed); - Init(); - if (size) Extend(size); - } else { - Init(); + Disable(); + if (coverage_enabled) { + if (common_flags()->coverage_direct) { + // In memory-mapped mode we must extend the new file to the known array + // size. + uptr size = atomic_load(&pc_array_size, memory_order_relaxed); + Enable(); + if (size) Extend(size); + if (coverage_enabled) CovUpdateMapping(coverage_dir); + } else { + Enable(); + } } + // Re-initialize the guards. + // We are single-threaded now, no need to grab any lock. + CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0); + ReinitializeGuards(); } void CoverageData::BeforeFork() { @@ -203,15 +259,16 @@ void CoverageData::Extend(uptr npcs) { if (!common_flags()->coverage_direct) return; SpinMutexLock l(&mu); - if (pc_fd == kInvalidFd) DirectOpen(); - CHECK_NE(pc_fd, kInvalidFd); - uptr size = atomic_load(&pc_array_size, memory_order_relaxed); size += npcs * sizeof(uptr); - if (size > pc_array_mapped_size) { + if (coverage_enabled && size > pc_array_mapped_size) { + if (pc_fd == kInvalidFd) DirectOpen(); + CHECK_NE(pc_fd, kInvalidFd); + uptr new_mapped_size = pc_array_mapped_size; while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize; + CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize); // Extend the file and map the new space at the end of pc_array. uptr res = internal_ftruncate(pc_fd, new_mapped_size); @@ -220,29 +277,47 @@ void CoverageData::Extend(uptr npcs) { Printf("failed to extend raw coverage file: %d\n", err); Die(); } - void *p = MapWritableFileToMemory(pc_array + pc_array_mapped_size, + + uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size; + void *p = MapWritableFileToMemory((void *)next_map_base, new_mapped_size - pc_array_mapped_size, pc_fd, pc_array_mapped_size); - CHECK_EQ(p, pc_array + pc_array_mapped_size); + CHECK_EQ((uptr)p, next_map_base); pc_array_mapped_size = new_mapped_size; } atomic_store(&pc_array_size, size, memory_order_release); } -// Atomically add the pc to the vector. The atomically set the guard to 1. -// If the function is called more than once for a given PC it will -// be inserted multiple times, which is fine. -void CoverageData::Add(uptr pc, u8 *guard) { +void CoverageData::InitializeGuards(s32 *guards, uptr n, + const char *module_name) { + // The array 'guards' has n+1 elements, we use the element zero + // to store 'n'. + CHECK_LT(n, 1 << 30); + guards[0] = static_cast(n); + InitializeGuardArray(guards); + SpinMutexLock l(&mu); + comp_unit_name_vec.push_back(module_name); + guard_array_vec.push_back(guards); +} + +// If guard is negative, atomically set it to -guard and store the PC in +// pc_array. +void CoverageData::Add(uptr pc, u32 *guard) { + atomic_uint32_t *atomic_guard = reinterpret_cast(guard); + s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed); + if (guard_value >= 0) return; + + atomic_store(atomic_guard, -guard_value, memory_order_relaxed); if (!pc_array) return; - uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + + uptr idx = -guard_value - 1; + if (idx >= atomic_load(&pc_array_index, memory_order_acquire)) + return; // May happen after fork when pc_array_index becomes 0. CHECK_LT(idx * sizeof(uptr), atomic_load(&pc_array_size, memory_order_acquire)); pc_array[idx] = pc; atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); - // Set the guard. - atomic_uint8_t *atomic_guard = reinterpret_cast(guard); - atomic_store(atomic_guard, 1, memory_order_relaxed); } // Registers a pair caller=>callee. @@ -338,18 +413,19 @@ static void CovWritePacked(int pid, const char *module, const void *blob, // If packed = true and name == 0: ... // If packed = true and name != 0: .. (name is // user-supplied). -static int CovOpenFile(bool packed, const char* name) { +static int CovOpenFile(bool packed, const char *name, + const char *extension = "sancov") { InternalScopedString path(kMaxPathLength); if (!packed) { CHECK(name); - path.append("%s/%s.%zd.sancov", common_flags()->coverage_dir, name, - internal_getpid()); + path.append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(), + extension); } else { if (!name) - path.append("%s/%zd.sancov.packed", common_flags()->coverage_dir, - internal_getpid()); + path.append("%s/%zd.%s.packed", coverage_dir, internal_getpid(), + extension); else - path.append("%s/%s.sancov.packed", common_flags()->coverage_dir, name); + path.append("%s/%s.%s.packed", coverage_dir, name, extension); } uptr fd = OpenFile(path.data(), true); if (internal_iserror(fd)) { @@ -361,35 +437,49 @@ static int CovOpenFile(bool packed, const char* name) { // Dump trace PCs and trace events into two separate files. void CoverageData::DumpTrace() { - uptr max_idx = tr_event_array_index; + uptr max_idx = tr_event_pointer - tr_event_array; if (!max_idx) return; auto sym = Symbolizer::GetOrInit(); if (!sym) return; InternalScopedString out(32 << 20); - for (uptr i = 0; i < max_idx; i++) { - u32 pc_idx = tr_event_array[i]; - TracedPc *t = &tr_pc_array[pc_idx]; - if (!t->module_name) { - const char *module_name = ""; - uptr module_address = 0; - sym->GetModuleNameAndOffsetForPC(t->pc, &module_name, &module_address); - t->module_name = internal_strdup(module_name); - t->module_offset = module_address; - out.append("%s 0x%zx\n", t->module_name, t->module_offset); - } + for (uptr i = 0, n = size(); i < n; i++) { + const char *module_name = ""; + uptr module_address = 0; + sym->GetModuleNameAndOffsetForPC(pc_array[i], &module_name, + &module_address); + out.append("%s 0x%zx\n", module_name, module_address); } int fd = CovOpenFile(false, "trace-points"); if (fd < 0) return; internal_write(fd, out.data(), out.length()); internal_close(fd); + fd = CovOpenFile(false, "trace-compunits"); + if (fd < 0) return; + out.clear(); + for (uptr i = 0; i < comp_unit_name_vec.size(); i++) + out.append("%s\n", comp_unit_name_vec[i]); + internal_write(fd, out.data(), out.length()); + internal_close(fd); + fd = CovOpenFile(false, "trace-events"); if (fd < 0) return; - internal_write(fd, tr_event_array, max_idx * sizeof(tr_event_array[0])); + uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]); + u8 *event_bytes = reinterpret_cast(tr_event_array); + // The trace file could be huge, and may not be written with a single syscall. + while (bytes_to_write) { + uptr actually_written = internal_write(fd, event_bytes, bytes_to_write); + if (actually_written <= bytes_to_write) { + bytes_to_write -= actually_written; + event_bytes += actually_written; + } else { + break; + } + } internal_close(fd); - VReport(1, " CovDump: Trace: %zd PCs written\n", tr_pc_array_index); - VReport(1, " CovDump: Trace: %zd Events written\n", tr_event_array_index); + VReport(1, " CovDump: Trace: %zd PCs written\n", size()); + VReport(1, " CovDump: Trace: %zd Events written\n", max_idx); } // This function dumps the caller=>callee pairs into a file as a sequence of @@ -434,28 +524,45 @@ void CoverageData::DumpCallerCalleePairs() { // Record the current PC into the event buffer. // Every event is a u32 value (index in tr_pc_array_index) so we compute // it once and then cache in the provided 'cache' storage. -void CoverageData::TraceBasicaBlock(uptr *cache) { - CHECK(common_flags()->coverage); - uptr idx = *cache; - if (!idx) { - CHECK_LT(tr_pc_array_index, kTrPcArrayMaxSize); - idx = tr_pc_array_index++; - TracedPc *t = &tr_pc_array[idx]; - t->pc = GET_CALLER_PC(); - *cache = idx; - CHECK_LT(idx, 1U << 31); +// +// This function will eventually be inlined by the compiler. +void CoverageData::TraceBasicBlock(s32 *id) { + // Will trap here if + // 1. coverage is not enabled at run-time. + // 2. The array tr_event_array is full. + *tr_event_pointer = static_cast(*id - 1); + tr_event_pointer++; +} + +static void CovDumpAsBitSet() { + if (!common_flags()->coverage_bitset) return; + if (!coverage_data.size()) return; + int fd = CovOpenFile(/* packed */false, "combined", "bitset-sancov"); + if (fd < 0) return; + uptr n = coverage_data.size(); + uptr n_set_bits = 0; + InternalScopedBuffer out(n); + for (uptr i = 0; i < n; i++) { + uptr pc = coverage_data.data()[i]; + out[i] = pc ? '1' : '0'; + if (pc) + n_set_bits++; } - CHECK_LT(tr_event_array_index, tr_event_array_size); - tr_event_array[tr_event_array_index] = static_cast(idx); - tr_event_array_index++; + internal_write(fd, out.data(), n); + internal_close(fd); + VReport(1, " CovDump: bitset of %zd bits written, %zd bits are set\n", n, + n_set_bits); } // Dump the coverage on disk. static void CovDump() { - if (!common_flags()->coverage || common_flags()->coverage_direct) return; + if (!coverage_enabled || common_flags()->coverage_direct) return; #if !SANITIZER_WINDOWS if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) return; + CovDumpAsBitSet(); + coverage_data.DumpTrace(); + if (!common_flags()->coverage_pcs) return; uptr size = coverage_data.size(); InternalMmapVector offsets(size); uptr *vb = coverage_data.data(); @@ -491,8 +598,8 @@ static void CovDump() { } else { // One file per module per process. path.clear(); - path.append("%s/%s.%zd.sancov", common_flags()->coverage_dir, - module_name, internal_getpid()); + path.append("%s/%s.%zd.sancov", coverage_dir, module_name, + internal_getpid()); int fd = CovOpenFile(false /* packed */, module_name); if (fd > 0) { internal_write(fd, offsets.data(), offsets.size() * sizeof(u32)); @@ -506,13 +613,12 @@ static void CovDump() { if (cov_fd >= 0) internal_close(cov_fd); coverage_data.DumpCallerCalleePairs(); - coverage_data.DumpTrace(); #endif // !SANITIZER_WINDOWS } void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { if (!args) return; - if (!common_flags()->coverage) return; + if (!coverage_enabled) return; cov_sandboxed = args->coverage_sandboxed; if (!cov_sandboxed) return; cov_fd = args->coverage_fd; @@ -524,7 +630,7 @@ void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { int MaybeOpenCovFile(const char *name) { CHECK(name); - if (!common_flags()->coverage) return -1; + if (!coverage_enabled) return -1; return CovOpenFile(true /* packed */, name); } @@ -536,28 +642,60 @@ void CovAfterFork(int child_pid) { coverage_data.AfterFork(child_pid); } +void InitializeCoverage(bool enabled, const char *dir) { + if (coverage_enabled) + return; // May happen if two sanitizer enable coverage in the same process. + coverage_enabled = enabled; + coverage_dir = dir; + coverage_data.Init(); + if (enabled) coverage_data.Enable(); +#if !SANITIZER_WINDOWS + if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump); +#endif +} + +void ReInitializeCoverage(bool enabled, const char *dir) { + coverage_enabled = enabled; + coverage_dir = dir; + coverage_data.ReInit(); +} + +void CoverageUpdateMapping() { + if (coverage_enabled) + CovUpdateMapping(coverage_dir); +} + } // namespace __sanitizer extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u8 *guard) { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) { coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), guard); } +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) { + atomic_uint32_t *atomic_guard = reinterpret_cast(guard); + if (__sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) + __sanitizer_cov(guard); +} SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) { coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), callee, callee_cache16, 16); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { + coverage_enabled = true; + coverage_dir = common_flags()->coverage_dir; coverage_data.Init(); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(uptr npcs) { - if (!common_flags()->coverage || !common_flags()->coverage_direct) return; - if (SANITIZER_ANDROID) { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); } +SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_cov_module_init(s32 *guards, uptr npcs, const char *module_name) { + coverage_data.InitializeGuards(guards, npcs, module_name); + if (!common_flags()->coverage_direct) return; + if (SANITIZER_ANDROID && coverage_enabled) { // dlopen/dlclose interceptors do not work on Android, so we rely on // Extend() calls to update .sancov.map. - CovUpdateMapping(GET_CALLER_PC()); + CovUpdateMapping(coverage_dir, GET_CALLER_PC()); } coverage_data.Extend(npcs); } @@ -571,11 +709,23 @@ uptr __sanitizer_get_total_unique_coverage() { } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_func_enter(uptr *cache) { - coverage_data.TraceBasicaBlock(cache); +void __sanitizer_cov_trace_func_enter(s32 *id) { + coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_basic_block(uptr *cache) { - coverage_data.TraceBasicaBlock(cache); +void __sanitizer_cov_trace_basic_block(s32 *id) { + coverage_data.TraceBasicBlock(id); +} +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_reset_coverage() { + coverage_data.ReinitializeGuards(); + internal_bzero_aligned16( + coverage_data.data(), + RoundUpTo(coverage_data.size() * sizeof(coverage_data.data()[0]), 16)); +} +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_coverage_guards(uptr **data) { + *data = coverage_data.data(); + return coverage_data.size(); } } // extern "C" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc index fe72d06a2fb1..6b5e91fbc018 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc @@ -62,8 +62,8 @@ struct CachedMapping { static CachedMapping cached_mapping; static StaticSpinMutex mapping_mu; -void CovUpdateMapping(uptr caller_pc) { - if (!common_flags()->coverage || !common_flags()->coverage_direct) return; +void CovUpdateMapping(const char *coverage_dir, uptr caller_pc) { + if (!common_flags()->coverage_direct) return; SpinMutexLock l(&mapping_mu); @@ -71,36 +71,41 @@ void CovUpdateMapping(uptr caller_pc) { return; InternalScopedString text(kMaxTextSize); - InternalScopedBuffer modules(kMaxNumberOfModules); - CHECK(modules.data()); - int n_modules = GetListOfModules(modules.data(), kMaxNumberOfModules, - /* filter */ 0); - text.append("%d\n", sizeof(uptr) * 8); - for (int i = 0; i < n_modules; ++i) { - const char *module_name = StripModuleName(modules[i].full_name()); - for (unsigned j = 0; j < modules[i].n_ranges(); ++j) { - if (modules[i].address_range_executable(j)) { - uptr start = modules[i].address_range_start(j); - uptr end = modules[i].address_range_end(j); - uptr base = modules[i].base_address(); - text.append("%zx %zx %zx %s\n", start, end, base, module_name); - if (caller_pc && caller_pc >= start && caller_pc < end) - cached_mapping.SetModuleRange(start, end); + { + InternalScopedBuffer modules(kMaxNumberOfModules); + CHECK(modules.data()); + int n_modules = GetListOfModules(modules.data(), kMaxNumberOfModules, + /* filter */ 0); + + text.append("%d\n", sizeof(uptr) * 8); + for (int i = 0; i < n_modules; ++i) { + const char *module_name = StripModuleName(modules[i].full_name()); + uptr base = modules[i].base_address(); + for (auto iter = modules[i].ranges(); iter.hasNext();) { + const auto *range = iter.next(); + if (range->executable) { + uptr start = range->beg; + uptr end = range->end; + text.append("%zx %zx %zx %s\n", start, end, base, module_name); + if (caller_pc && caller_pc >= start && caller_pc < end) + cached_mapping.SetModuleRange(start, end); + } } + modules[i].clear(); } } int err; - InternalScopedString tmp_path(64 + - internal_strlen(common_flags()->coverage_dir)); + InternalScopedString tmp_path(64 + internal_strlen(coverage_dir)); uptr res = internal_snprintf((char *)tmp_path.data(), tmp_path.size(), - "%s/%zd.sancov.map.tmp", common_flags()->coverage_dir, - internal_getpid()); + "%s/%zd.sancov.map.tmp", coverage_dir, + internal_getpid()); CHECK_LE(res, tmp_path.size()); uptr map_fd = OpenFile(tmp_path.data(), true); - if (internal_iserror(map_fd)) { - Report(" Coverage: failed to open %s for writing\n", tmp_path.data()); + if (internal_iserror(map_fd, &err)) { + Report(" Coverage: failed to open %s for writing: %d\n", tmp_path.data(), + err); Die(); } @@ -111,9 +116,9 @@ void CovUpdateMapping(uptr caller_pc) { } internal_close(map_fd); - InternalScopedString path(64 + internal_strlen(common_flags()->coverage_dir)); + InternalScopedString path(64 + internal_strlen(coverage_dir)); res = internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.map", - common_flags()->coverage_dir, internal_getpid()); + coverage_dir, internal_getpid()); CHECK_LE(res, path.size()); res = internal_rename(tmp_path.data(), path.data()); if (internal_iserror(res, &err)) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector.h index 90e1cc4eb597..86d5743e9794 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_deadlock_detector.h @@ -50,6 +50,8 @@ class DeadlockDetectorTLS { if (epoch_ == current_epoch) return; bv_.clear(); epoch_ = current_epoch; + n_recursive_locks = 0; + n_all_locks_ = 0; } uptr getEpoch() const { return epoch_; } @@ -83,7 +85,8 @@ class DeadlockDetectorTLS { } } // Printf("remLock: %zx %zx\n", lock_id, epoch_); - CHECK(bv_.clearBit(lock_id)); + if (!bv_.clearBit(lock_id)) + return; // probably addLock happened before flush if (n_all_locks_) { for (sptr i = n_all_locks_ - 1; i >= 0; i--) { if (all_locks_with_contexts_[i].lock == static_cast(lock_id)) { @@ -175,6 +178,7 @@ class DeadlockDetector { recycled_nodes_.clear(); available_nodes_.setAll(); g_.clear(); + n_edges_ = 0; return getAvailableNode(data); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cc new file mode 100644 index 000000000000..d125002daf4c --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cc @@ -0,0 +1,153 @@ +//===-- sanitizer_flag_parser.cc ------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer/AddressSanitizer runtime. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_flag_parser.h" + +#include "sanitizer_common.h" +#include "sanitizer_libc.h" +#include "sanitizer_flags.h" +#include "sanitizer_flag_parser.h" + +namespace __sanitizer { + +LowLevelAllocator FlagParser::Alloc; + +class UnknownFlags { + static const int kMaxUnknownFlags = 20; + const char *unknown_flags_[kMaxUnknownFlags]; + int n_unknown_flags_; + + public: + void Add(const char *name) { + CHECK_LT(n_unknown_flags_, kMaxUnknownFlags); + unknown_flags_[n_unknown_flags_++] = name; + } + + void Report() { + if (!n_unknown_flags_) return; + Printf("WARNING: found %d unrecognized flag(s):\n", n_unknown_flags_); + for (int i = 0; i < n_unknown_flags_; ++i) + Printf(" %s\n", unknown_flags_[i]); + n_unknown_flags_ = 0; + } +}; + +UnknownFlags unknown_flags; + +void ReportUnrecognizedFlags() { + unknown_flags.Report(); +} + +char *FlagParser::ll_strndup(const char *s, uptr n) { + uptr len = internal_strnlen(s, n); + char *s2 = (char*)Alloc.Allocate(len + 1); + internal_memcpy(s2, s, len); + s2[len] = 0; + return s2; +} + +void FlagParser::PrintFlagDescriptions() { + Printf("Available flags for %s:\n", SanitizerToolName); + for (int i = 0; i < n_flags_; ++i) + Printf("\t%s\n\t\t- %s\n", flags_[i].name, flags_[i].desc); +} + +void FlagParser::fatal_error(const char *err) { + Printf("ERROR: %s\n", err); + Die(); +} + +bool FlagParser::is_space(char c) { + return c == ' ' || c == ',' || c == ':' || c == '\n' || c == '\t' || + c == '\r'; +} + +void FlagParser::skip_whitespace() { + while (is_space(buf_[pos_])) ++pos_; +} + +void FlagParser::parse_flag() { + uptr name_start = pos_; + while (buf_[pos_] != 0 && buf_[pos_] != '=' && !is_space(buf_[pos_])) ++pos_; + if (buf_[pos_] != '=') fatal_error("expected '='"); + char *name = ll_strndup(buf_ + name_start, pos_ - name_start); + + uptr value_start = ++pos_; + char *value; + if (buf_[pos_] == '\'' || buf_[pos_] == '"') { + char quote = buf_[pos_++]; + while (buf_[pos_] != 0 && buf_[pos_] != quote) ++pos_; + if (buf_[pos_] == 0) fatal_error("unterminated string"); + value = ll_strndup(buf_ + value_start + 1, pos_ - value_start - 1); + ++pos_; // consume the closing quote + } else { + while (buf_[pos_] != 0 && !is_space(buf_[pos_])) ++pos_; + if (buf_[pos_] != 0 && !is_space(buf_[pos_])) + fatal_error("expected separator or eol"); + value = ll_strndup(buf_ + value_start, pos_ - value_start); + } + + bool res = run_handler(name, value); + if (!res) fatal_error("Flag parsing failed."); +} + +void FlagParser::parse_flags() { + while (true) { + skip_whitespace(); + if (buf_[pos_] == 0) break; + parse_flag(); + } + + // Do a sanity check for certain flags. + if (common_flags_dont_use.malloc_context_size < 1) + common_flags_dont_use.malloc_context_size = 1; +} + +void FlagParser::ParseString(const char *s) { + if (!s) return; + // Backup current parser state to allow nested ParseString() calls. + const char *old_buf_ = buf_; + uptr old_pos_ = pos_; + buf_ = s; + pos_ = 0; + + parse_flags(); + + buf_ = old_buf_; + pos_ = old_pos_; +} + +bool FlagParser::run_handler(const char *name, const char *value) { + for (int i = 0; i < n_flags_; ++i) { + if (internal_strcmp(name, flags_[i].name) == 0) + return flags_[i].handler->Parse(value); + } + // Unrecognized flag. This is not a fatal error, we may print a warning later. + unknown_flags.Add(name); + return true; +} + +void FlagParser::RegisterHandler(const char *name, FlagHandlerBase *handler, + const char *desc) { + CHECK_LT(n_flags_, kMaxFlags); + flags_[n_flags_].name = name; + flags_[n_flags_].desc = desc; + flags_[n_flags_].handler = handler; + ++n_flags_; +} + +FlagParser::FlagParser() : n_flags_(0), buf_(nullptr), pos_(0) { + flags_ = (Flag *)Alloc.Allocate(sizeof(Flag) * kMaxFlags); +} + +} // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h new file mode 100644 index 000000000000..0ac7634cb876 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h @@ -0,0 +1,121 @@ +//===-- sanitizer_flag_parser.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer/AddressSanitizer runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_FLAG_REGISTRY_H +#define SANITIZER_FLAG_REGISTRY_H + +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_common.h" + +namespace __sanitizer { + +class FlagHandlerBase { + public: + virtual bool Parse(const char *value) { return false; } +}; + +template +class FlagHandler : public FlagHandlerBase { + T *t_; + + public: + explicit FlagHandler(T *t) : t_(t) {} + bool Parse(const char *value) final; +}; + +template <> +inline bool FlagHandler::Parse(const char *value) { + if (internal_strcmp(value, "0") == 0 || + internal_strcmp(value, "no") == 0 || + internal_strcmp(value, "false") == 0) { + *t_ = false; + return true; + } + if (internal_strcmp(value, "1") == 0 || + internal_strcmp(value, "yes") == 0 || + internal_strcmp(value, "true") == 0) { + *t_ = true; + return true; + } + Printf("ERROR: Invalid value for bool option: '%s'\n", value); + return false; +} + +template <> +inline bool FlagHandler::Parse(const char *value) { + *t_ = internal_strdup(value); + return true; +} + +template <> +inline bool FlagHandler::Parse(const char *value) { + char *value_end; + *t_ = internal_simple_strtoll(value, &value_end, 10); + bool ok = *value_end == 0; + if (!ok) Printf("ERROR: Invalid value for int option: '%s'\n", value); + return ok; +} + +template <> +inline bool FlagHandler::Parse(const char *value) { + char *value_end; + *t_ = internal_simple_strtoll(value, &value_end, 10); + bool ok = *value_end == 0; + if (!ok) Printf("ERROR: Invalid value for uptr option: '%s'\n", value); + return ok; +} + +class FlagParser { + static const int kMaxFlags = 200; + struct Flag { + const char *name; + const char *desc; + FlagHandlerBase *handler; + } *flags_; + int n_flags_; + + const char *buf_; + uptr pos_; + + public: + FlagParser(); + void RegisterHandler(const char *name, FlagHandlerBase *handler, + const char *desc); + void ParseString(const char *s); + void PrintFlagDescriptions(); + + static LowLevelAllocator Alloc; + + private: + void fatal_error(const char *err); + bool is_space(char c); + void skip_whitespace(); + void parse_flags(); + void parse_flag(); + bool run_handler(const char *name, const char *value); + char *ll_strndup(const char *s, uptr n); +}; + +template +static void RegisterFlag(FlagParser *parser, const char *name, const char *desc, + T *var) { + FlagHandler *fh = new (FlagParser::Alloc) FlagHandler(var); // NOLINT + parser->RegisterHandler(name, fh, desc); +} + +void ReportUnrecognizedFlags(); + +} // namespace __sanitizer + +#endif // SANITIZER_FLAG_REGISTRY_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc index 40b6ec067150..e835b46a24fc 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc @@ -16,6 +16,7 @@ #include "sanitizer_common.h" #include "sanitizer_libc.h" #include "sanitizer_list.h" +#include "sanitizer_flag_parser.h" namespace __sanitizer { @@ -34,274 +35,53 @@ IntrusiveList flag_descriptions; # define SANITIZER_NEEDS_SEGV 1 #endif -void SetCommonFlagsDefaults(CommonFlags *f) { - f->symbolize = true; - f->external_symbolizer_path = 0; - f->allow_addr2line = false; - f->strip_path_prefix = ""; - f->fast_unwind_on_check = false; - f->fast_unwind_on_fatal = false; - f->fast_unwind_on_malloc = true; - f->handle_ioctl = false; - f->malloc_context_size = 1; - f->log_path = "stderr"; - f->verbosity = 0; - f->detect_leaks = true; - f->leak_check_at_exit = true; - f->allocator_may_return_null = false; - f->print_summary = true; - f->check_printf = true; - // TODO(glider): tools may want to set different defaults for handle_segv. - f->handle_segv = SANITIZER_NEEDS_SEGV; - f->allow_user_segv_handler = false; - f->use_sigaltstack = true; - f->detect_deadlocks = false; - f->clear_shadow_mmap_threshold = 64 * 1024; - f->color = "auto"; - f->legacy_pthread_cond = false; - f->intercept_tls_get_addr = false; - f->coverage = false; - f->coverage_direct = SANITIZER_ANDROID; - f->coverage_dir = "."; - f->full_address_space = false; - f->suppressions = ""; - f->print_suppressions = true; - f->disable_coredump = (SANITIZER_WORDSIZE == 64); - f->symbolize_inline_frames = true; - f->stack_trace_format = "DEFAULT"; +void CommonFlags::SetDefaults() { +#define COMMON_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "sanitizer_flags.inc" +#undef COMMON_FLAG } -void ParseCommonFlagsFromString(CommonFlags *f, const char *str) { - ParseFlag(str, &f->symbolize, "symbolize", - "If set, use the online symbolizer from common sanitizer runtime to turn " - "virtual addresses to file/line locations."); - ParseFlag(str, &f->external_symbolizer_path, "external_symbolizer_path", - "Path to external symbolizer. If empty, the tool will search $PATH for " - "the symbolizer."); - ParseFlag(str, &f->allow_addr2line, "allow_addr2line", - "If set, allows online symbolizer to run addr2line binary to symbolize " - "stack traces (addr2line will only be used if llvm-symbolizer binary is " - "unavailable."); - ParseFlag(str, &f->strip_path_prefix, "strip_path_prefix", - "Strips this prefix from file paths in error reports."); - ParseFlag(str, &f->fast_unwind_on_check, "fast_unwind_on_check", - "If available, use the fast frame-pointer-based unwinder on " - "internal CHECK failures."); - ParseFlag(str, &f->fast_unwind_on_fatal, "fast_unwind_on_fatal", - "If available, use the fast frame-pointer-based unwinder on fatal " - "errors."); - ParseFlag(str, &f->fast_unwind_on_malloc, "fast_unwind_on_malloc", - "If available, use the fast frame-pointer-based unwinder on " - "malloc/free."); - ParseFlag(str, &f->handle_ioctl, "handle_ioctl", - "Intercept and handle ioctl requests."); - ParseFlag(str, &f->malloc_context_size, "malloc_context_size", - "Max number of stack frames kept for each allocation/deallocation."); - ParseFlag(str, &f->log_path, "log_path", - "Write logs to \"log_path.pid\". The special values are \"stdout\" and " - "\"stderr\". The default is \"stderr\"."); - ParseFlag(str, &f->verbosity, "verbosity", - "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output)."); - ParseFlag(str, &f->detect_leaks, "detect_leaks", - "Enable memory leak detection."); - ParseFlag(str, &f->leak_check_at_exit, "leak_check_at_exit", - "Invoke leak checking in an atexit handler. Has no effect if " - "detect_leaks=false, or if __lsan_do_leak_check() is called before the " - "handler has a chance to run."); - ParseFlag(str, &f->allocator_may_return_null, "allocator_may_return_null", - "If false, the allocator will crash instead of returning 0 on " - "out-of-memory."); - ParseFlag(str, &f->print_summary, "print_summary", - "If false, disable printing error summaries in addition to error " - "reports."); - ParseFlag(str, &f->check_printf, "check_printf", - "Check printf arguments."); - ParseFlag(str, &f->handle_segv, "handle_segv", - "If set, registers the tool's custom SEGV handler (both SIGBUS and " - "SIGSEGV on OSX)."); - ParseFlag(str, &f->allow_user_segv_handler, "allow_user_segv_handler", - "If set, allows user to register a SEGV handler even if the tool " - "registers one."); - ParseFlag(str, &f->use_sigaltstack, "use_sigaltstack", - "If set, uses alternate stack for signal handling."); - ParseFlag(str, &f->detect_deadlocks, "detect_deadlocks", - "If set, deadlock detection is enabled."); - ParseFlag(str, &f->clear_shadow_mmap_threshold, - "clear_shadow_mmap_threshold", - "Large shadow regions are zero-filled using mmap(NORESERVE) instead of " - "memset(). This is the threshold size in bytes."); - ParseFlag(str, &f->color, "color", - "Colorize reports: (always|never|auto)."); - ParseFlag(str, &f->legacy_pthread_cond, "legacy_pthread_cond", - "Enables support for dynamic libraries linked with libpthread 2.2.5."); - ParseFlag(str, &f->intercept_tls_get_addr, "intercept_tls_get_addr", - "Intercept __tls_get_addr."); - ParseFlag(str, &f->help, "help", "Print the flag descriptions."); - ParseFlag(str, &f->mmap_limit_mb, "mmap_limit_mb", - "Limit the amount of mmap-ed memory (excluding shadow) in Mb; " - "not a user-facing flag, used mosly for testing the tools"); - ParseFlag(str, &f->coverage, "coverage", - "If set, coverage information will be dumped at program shutdown (if the " - "coverage instrumentation was enabled at compile time)."); - ParseFlag(str, &f->coverage_direct, "coverage_direct", - "If set, coverage information will be dumped directly to a memory " - "mapped file. This way data is not lost even if the process is " - "suddenly killed."); - ParseFlag(str, &f->coverage_dir, "coverage_dir", - "Target directory for coverage dumps. Defaults to the current " - "directory."); - ParseFlag(str, &f->full_address_space, "full_address_space", - "Sanitize complete address space; " - "by default kernel area on 32-bit platforms will not be sanitized"); - ParseFlag(str, &f->suppressions, "suppressions", "Suppressions file name."); - ParseFlag(str, &f->print_suppressions, "print_suppressions", - "Print matched suppressions at exit."); - ParseFlag(str, &f->disable_coredump, "disable_coredump", - "Disable core dumping. By default, disable_core=1 on 64-bit to avoid " - "dumping a 16T+ core file. Ignored on OSes that don't dump core by" - "default and for sanitizers that don't reserve lots of virtual memory."); - ParseFlag(str, &f->symbolize_inline_frames, "symbolize_inline_frames", - "Print inlined frames in stacktraces. Defaults to true."); - ParseFlag(str, &f->stack_trace_format, "stack_trace_format", - "Format string used to render stack frames. " - "See sanitizer_stacktrace_printer.h for the format description. " - "Use DEFAULT to get default format."); - - // Do a sanity check for certain flags. - if (f->malloc_context_size < 1) - f->malloc_context_size = 1; +void CommonFlags::CopyFrom(const CommonFlags &other) { + internal_memcpy(this, &other, sizeof(*this)); } -static bool GetFlagValue(const char *env, const char *name, - const char **value, int *value_length) { - if (env == 0) - return false; - const char *pos = 0; - for (;;) { - pos = internal_strstr(env, name); - if (pos == 0) +class FlagHandlerInclude : public FlagHandlerBase { + static const uptr kMaxIncludeSize = 1 << 15; + FlagParser *parser_; + + public: + explicit FlagHandlerInclude(FlagParser *parser) : parser_(parser) {} + bool Parse(const char *value) final { + char *data; + uptr data_mapped_size; + int err; + uptr len = + ReadFileToBuffer(value, &data, &data_mapped_size, + Max(kMaxIncludeSize, GetPageSizeCached()), &err); + if (!len) { + Printf("Failed to read options from '%s': error %d\n", value, err); return false; - const char *name_end = pos + internal_strlen(name); - if ((pos != env && - ((pos[-1] >= 'a' && pos[-1] <= 'z') || pos[-1] == '_')) || - *name_end != '=') { - // Seems to be middle of another flag name or value. - env = pos + 1; - continue; } - pos = name_end; - break; + parser_->ParseString(data); + UnmapOrDie(data, data_mapped_size); + return true; } - const char *end; - if (pos[0] != '=') { - end = pos; - } else { - pos += 1; - if (pos[0] == '"') { - pos += 1; - end = internal_strchr(pos, '"'); - } else if (pos[0] == '\'') { - pos += 1; - end = internal_strchr(pos, '\''); - } else { - // Read until the next space or colon. - end = pos + internal_strcspn(pos, " :"); - } - if (end == 0) - end = pos + internal_strlen(pos); - } - *value = pos; - *value_length = end - pos; - return true; +}; + +void RegisterIncludeFlag(FlagParser *parser, CommonFlags *cf) { + FlagHandlerInclude *fh_include = + new (FlagParser::Alloc) FlagHandlerInclude(parser); // NOLINT + parser->RegisterHandler("include", fh_include, + "read more options from the given file"); } -static bool StartsWith(const char *flag, int flag_length, const char *value) { - if (!flag || !value) - return false; - int value_length = internal_strlen(value); - return (flag_length >= value_length) && - (0 == internal_strncmp(flag, value, value_length)); -} +void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf) { +#define COMMON_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &cf->Name); +#include "sanitizer_flags.inc" +#undef COMMON_FLAG -static LowLevelAllocator allocator_for_flags; - -// The linear scan is suboptimal, but the number of flags is relatively small. -bool FlagInDescriptionList(const char *name) { - IntrusiveList::Iterator it(&flag_descriptions); - while (it.hasNext()) { - if (!internal_strcmp(it.next()->name, name)) return true; - } - return false; -} - -void AddFlagDescription(const char *name, const char *description) { - if (FlagInDescriptionList(name)) return; - FlagDescription *new_description = new(allocator_for_flags) FlagDescription; - new_description->name = name; - new_description->description = description; - flag_descriptions.push_back(new_description); -} - -// TODO(glider): put the descriptions inside CommonFlags. -void PrintFlagDescriptions() { - IntrusiveList::Iterator it(&flag_descriptions); - Printf("Available flags for %s:\n", SanitizerToolName); - while (it.hasNext()) { - FlagDescription *descr = it.next(); - Printf("\t%s\n\t\t- %s\n", descr->name, descr->description); - } -} - -void ParseFlag(const char *env, bool *flag, - const char *name, const char *descr) { - const char *value; - int value_length; - AddFlagDescription(name, descr); - if (!GetFlagValue(env, name, &value, &value_length)) - return; - if (StartsWith(value, value_length, "0") || - StartsWith(value, value_length, "no") || - StartsWith(value, value_length, "false")) - *flag = false; - if (StartsWith(value, value_length, "1") || - StartsWith(value, value_length, "yes") || - StartsWith(value, value_length, "true")) - *flag = true; -} - -void ParseFlag(const char *env, int *flag, - const char *name, const char *descr) { - const char *value; - int value_length; - AddFlagDescription(name, descr); - if (!GetFlagValue(env, name, &value, &value_length)) - return; - *flag = static_cast(internal_atoll(value)); -} - -void ParseFlag(const char *env, uptr *flag, - const char *name, const char *descr) { - const char *value; - int value_length; - AddFlagDescription(name, descr); - if (!GetFlagValue(env, name, &value, &value_length)) - return; - *flag = static_cast(internal_atoll(value)); -} - -void ParseFlag(const char *env, const char **flag, - const char *name, const char *descr) { - const char *value; - int value_length; - AddFlagDescription(name, descr); - if (!GetFlagValue(env, name, &value, &value_length)) - return; - // Copy the flag value. Don't use locks here, as flags are parsed at - // tool startup. - char *value_copy = (char*)(allocator_for_flags.Allocate(value_length + 1)); - internal_memcpy(value_copy, value, value_length); - value_copy[value_length] = '\0'; - *flag = value_copy; + RegisterIncludeFlag(parser, cf); } } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h index 4791397a5761..fda6d710757e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.h @@ -18,62 +18,38 @@ namespace __sanitizer { -void ParseFlag(const char *env, bool *flag, - const char *name, const char *descr); -void ParseFlag(const char *env, int *flag, - const char *name, const char *descr); -void ParseFlag(const char *env, uptr *flag, - const char *name, const char *descr); -void ParseFlag(const char *env, const char **flag, - const char *name, const char *descr); - struct CommonFlags { - bool symbolize; - const char *external_symbolizer_path; - bool allow_addr2line; - const char *strip_path_prefix; - bool fast_unwind_on_check; - bool fast_unwind_on_fatal; - bool fast_unwind_on_malloc; - bool handle_ioctl; - int malloc_context_size; - const char *log_path; - int verbosity; - bool detect_leaks; - bool leak_check_at_exit; - bool allocator_may_return_null; - bool print_summary; - bool check_printf; - bool handle_segv; - bool allow_user_segv_handler; - bool use_sigaltstack; - bool detect_deadlocks; - uptr clear_shadow_mmap_threshold; - const char *color; - bool legacy_pthread_cond; - bool intercept_tls_get_addr; - bool help; - uptr mmap_limit_mb; - bool coverage; - bool coverage_direct; - const char *coverage_dir; - bool full_address_space; - const char *suppressions; - bool print_suppressions; - bool disable_coredump; - bool symbolize_inline_frames; - const char *stack_trace_format; +#define COMMON_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "sanitizer_flags.inc" +#undef COMMON_FLAG + + void SetDefaults(); + void CopyFrom(const CommonFlags &other); }; -inline CommonFlags *common_flags() { - extern CommonFlags common_flags_dont_use; +// Functions to get/set global CommonFlags shared by all sanitizer runtimes: +extern CommonFlags common_flags_dont_use; +inline const CommonFlags *common_flags() { return &common_flags_dont_use; } -void SetCommonFlagsDefaults(CommonFlags *f); -void ParseCommonFlagsFromString(CommonFlags *f, const char *str); -void PrintFlagDescriptions(); +inline void SetCommonFlagsDefaults() { + common_flags_dont_use.SetDefaults(); +} +// This function can only be used to setup tool-specific overrides for +// CommonFlags defaults. Generally, it should only be used right after +// SetCommonFlagsDefaults(), but before ParseCommonFlagsFromString(), and +// only during the flags initialization (i.e. before they are used for +// the first time). +inline void OverrideCommonFlags(const CommonFlags &cf) { + common_flags_dont_use.CopyFrom(cf); +} + +class FlagParser; +void RegisterCommonFlags(FlagParser *parser, + CommonFlags *cf = &common_flags_dont_use); +void RegisterIncludeFlag(FlagParser *parser, CommonFlags *cf); } // namespace __sanitizer #endif // SANITIZER_FLAGS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc new file mode 100644 index 000000000000..58f7f372228f --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc @@ -0,0 +1,148 @@ +//===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file describes common flags available in all sanitizers. +// +//===----------------------------------------------------------------------===// +#ifndef COMMON_FLAG +#error "Define COMMON_FLAG prior to including this file!" +#endif + +// COMMON_FLAG(Type, Name, DefaultValue, Description) +// Supported types: bool, const char *, int, uptr. +// Default value must be a compile-time constant. +// Description must be a string literal. + +COMMON_FLAG( + bool, symbolize, true, + "If set, use the online symbolizer from common sanitizer runtime to turn " + "virtual addresses to file/line locations.") +COMMON_FLAG( + const char *, external_symbolizer_path, 0, + "Path to external symbolizer. If empty, the tool will search $PATH for " + "the symbolizer.") +COMMON_FLAG( + bool, allow_addr2line, false, + "If set, allows online symbolizer to run addr2line binary to symbolize " + "stack traces (addr2line will only be used if llvm-symbolizer binary is " + "unavailable.") +COMMON_FLAG(const char *, strip_path_prefix, "", + "Strips this prefix from file paths in error reports.") +COMMON_FLAG(bool, fast_unwind_on_check, false, + "If available, use the fast frame-pointer-based unwinder on " + "internal CHECK failures.") +COMMON_FLAG(bool, fast_unwind_on_fatal, false, + "If available, use the fast frame-pointer-based unwinder on fatal " + "errors.") +COMMON_FLAG(bool, fast_unwind_on_malloc, true, + "If available, use the fast frame-pointer-based unwinder on " + "malloc/free.") +COMMON_FLAG(bool, handle_ioctl, false, "Intercept and handle ioctl requests.") +COMMON_FLAG(int, malloc_context_size, 1, + "Max number of stack frames kept for each allocation/deallocation.") +COMMON_FLAG( + const char *, log_path, "stderr", + "Write logs to \"log_path.pid\". The special values are \"stdout\" and " + "\"stderr\". The default is \"stderr\".") +COMMON_FLAG( + int, verbosity, 0, + "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).") +COMMON_FLAG(bool, detect_leaks, true, "Enable memory leak detection.") +COMMON_FLAG( + bool, leak_check_at_exit, true, + "Invoke leak checking in an atexit handler. Has no effect if " + "detect_leaks=false, or if __lsan_do_leak_check() is called before the " + "handler has a chance to run.") +COMMON_FLAG(bool, allocator_may_return_null, false, + "If false, the allocator will crash instead of returning 0 on " + "out-of-memory.") +COMMON_FLAG(bool, print_summary, true, + "If false, disable printing error summaries in addition to error " + "reports.") +COMMON_FLAG(bool, check_printf, true, "Check printf arguments.") +COMMON_FLAG(bool, handle_segv, SANITIZER_NEEDS_SEGV, + "If set, registers the tool's custom SEGV handler (both SIGBUS and " + "SIGSEGV on OSX).") +COMMON_FLAG(bool, allow_user_segv_handler, false, + "If set, allows user to register a SEGV handler even if the tool " + "registers one.") +COMMON_FLAG(bool, use_sigaltstack, true, + "If set, uses alternate stack for signal handling.") +COMMON_FLAG(bool, detect_deadlocks, false, + "If set, deadlock detection is enabled.") +COMMON_FLAG( + uptr, clear_shadow_mmap_threshold, 64 * 1024, + "Large shadow regions are zero-filled using mmap(NORESERVE) instead of " + "memset(). This is the threshold size in bytes.") +COMMON_FLAG(const char *, color, "auto", + "Colorize reports: (always|never|auto).") +COMMON_FLAG( + bool, legacy_pthread_cond, false, + "Enables support for dynamic libraries linked with libpthread 2.2.5.") +COMMON_FLAG(bool, intercept_tls_get_addr, false, "Intercept __tls_get_addr.") +COMMON_FLAG(bool, help, false, "Print the flag descriptions.") +COMMON_FLAG(uptr, mmap_limit_mb, 0, + "Limit the amount of mmap-ed memory (excluding shadow) in Mb; " + "not a user-facing flag, used mosly for testing the tools") +COMMON_FLAG(uptr, hard_rss_limit_mb, 0, + "Hard RSS limit in Mb." + " If non-zero, a background thread is spawned at startup" + " which periodically reads RSS and aborts the process if the" + " limit is reached") +COMMON_FLAG(uptr, soft_rss_limit_mb, 0, + "Soft RSS limit in Mb." + " If non-zero, a background thread is spawned at startup" + " which periodically reads RSS. If the limit is reached" + " all subsequent malloc/new calls will fail or return NULL" + " (depending on the value of allocator_may_return_null)" + " until the RSS goes below the soft limit." + " This limit does not affect memory allocations other than" + " malloc/new.") +COMMON_FLAG(bool, can_use_proc_maps_statm, true, + "If false, do not attempt to read /proc/maps/statm." + " Mostly useful for testing sanitizers.") +COMMON_FLAG( + bool, coverage, false, + "If set, coverage information will be dumped at program shutdown (if the " + "coverage instrumentation was enabled at compile time).") +// On by default, but works only if coverage == true. +COMMON_FLAG(bool, coverage_pcs, true, + "If set (and if 'coverage' is set too), the coverage information " + "will be dumped as a set of PC offsets for every module.") +COMMON_FLAG(bool, coverage_bitset, false, + "If set (and if 'coverage' is set too), the coverage information " + "will also be dumped as a bitset to a separate file.") +COMMON_FLAG(bool, coverage_direct, SANITIZER_ANDROID, + "If set, coverage information will be dumped directly to a memory " + "mapped file. This way data is not lost even if the process is " + "suddenly killed.") +COMMON_FLAG(const char *, coverage_dir, ".", + "Target directory for coverage dumps. Defaults to the current " + "directory.") +COMMON_FLAG(bool, full_address_space, false, + "Sanitize complete address space; " + "by default kernel area on 32-bit platforms will not be sanitized") +COMMON_FLAG(bool, print_suppressions, true, + "Print matched suppressions at exit.") +COMMON_FLAG( + bool, disable_coredump, (SANITIZER_WORDSIZE == 64), + "Disable core dumping. By default, disable_core=1 on 64-bit to avoid " + "dumping a 16T+ core file. Ignored on OSes that don't dump core by" + "default and for sanitizers that don't reserve lots of virtual memory.") +COMMON_FLAG(bool, use_madv_dontdump, true, + "If set, instructs kernel to not store the (huge) shadow " + "in core file.") +COMMON_FLAG(bool, symbolize_inline_frames, true, + "Print inlined frames in stacktraces. Defaults to true.") +COMMON_FLAG(const char *, stack_trace_format, "DEFAULT", + "Format string used to render stack frames. " + "See sanitizer_stacktrace_printer.h for the format description. " + "Use DEFAULT to get default format.") +COMMON_FLAG(bool, no_huge_pages_for_shadow, true, + "If true, the shadow is not allowed to use huge pages. ") diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h new file mode 100644 index 000000000000..94d9f4e9524a --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h @@ -0,0 +1,58 @@ +//===-- sanitizer_interface_internal.h --------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between run-time libraries of sanitizers. +// +// This header declares the sanitizer runtime interface functions. +// The runtime library has to define these functions so the instrumented program +// could call them. +// +// See also include/sanitizer/common_interface_defs.h +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_INTERFACE_INTERNAL_H +#define SANITIZER_INTERFACE_INTERNAL_H + +#include "sanitizer_internal_defs.h" + +extern "C" { + // Tell the tools to write their reports to "path." instead of stderr. + // The special values are "stdout" and "stderr". + SANITIZER_INTERFACE_ATTRIBUTE + void __sanitizer_set_report_path(const char *path); + + typedef struct { + int coverage_sandboxed; + __sanitizer::sptr coverage_fd; + unsigned int coverage_max_block_size; + } __sanitizer_sandbox_arguments; + + // Notify the tools that the sandbox is going to be turned on. + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void + __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); + + // This function is called by the tool when it has just finished reporting + // an error. 'error_summary' is a one-line string that summarizes + // the error message. This function can be overridden by the client. + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_report_error_summary(const char *error_summary); + + SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump(); + SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init(); + SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard); + SANITIZER_INTERFACE_ATTRIBUTE + void __sanitizer_annotate_contiguous_container(const void *beg, + const void *end, + const void *old_mid, + const void *new_mid); + SANITIZER_INTERFACE_ATTRIBUTE + int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, + const void *end); +} // extern "C" + +#endif // SANITIZER_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index 6f36f97f1c86..a969f305cd1a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -15,6 +15,10 @@ #include "sanitizer_platform.h" +#ifndef SANITIZER_DEBUG +# define SANITIZER_DEBUG 0 +#endif + // Only use SANITIZER_*ATTRIBUTE* before the function return type! #if SANITIZER_WINDOWS # define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport) @@ -48,11 +52,6 @@ # define __has_feature(x) 0 #endif -// Enable sanitizer compilation for pre-C++11 -#if __cplusplus < 201103L -#define nullptr 0 -#endif - // For portability reasons we do not include stddef.h, stdint.h or any other // system header, but we do need some basic types that are not defined // in a portable way by the language itself. @@ -86,8 +85,9 @@ typedef int fd_t; // WARNING: OFF_T may be different from OS type off_t, depending on the value of // _FILE_OFFSET_BITS. This definition of OFF_T matches the ABI of system calls // like pread and mmap, as opposed to pread64 and mmap64. -// Mac and Linux/x86-64 are special. -#if SANITIZER_MAC || (SANITIZER_LINUX && defined(__x86_64__)) +// FreeBSD, Mac and Linux/x86-64 are special. +#if SANITIZER_FREEBSD || SANITIZER_MAC || \ + (SANITIZER_LINUX && defined(__x86_64__)) typedef u64 OFF_T; #else typedef uptr OFF_T; @@ -101,41 +101,6 @@ typedef u32 operator_new_size_type; #endif } // namespace __sanitizer -extern "C" { - // Tell the tools to write their reports to "path." instead of stderr. - // The special values are "stdout" and "stderr". - SANITIZER_INTERFACE_ATTRIBUTE - void __sanitizer_set_report_path(const char *path); - - typedef struct { - int coverage_sandboxed; - __sanitizer::sptr coverage_fd; - unsigned int coverage_max_block_size; - } __sanitizer_sandbox_arguments; - - // Notify the tools that the sandbox is going to be turned on. - SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void - __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); - - // This function is called by the tool when it has just finished reporting - // an error. 'error_summary' is a one-line string that summarizes - // the error message. This function can be overridden by the client. - SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE - void __sanitizer_report_error_summary(const char *error_summary); - - SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump(); - SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init(); - SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u8 *guard); - SANITIZER_INTERFACE_ATTRIBUTE - void __sanitizer_annotate_contiguous_container(const void *beg, - const void *end, - const void *old_mid, - const void *new_mid); - SANITIZER_INTERFACE_ATTRIBUTE - int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, - const void *end); -} // extern "C" - using namespace __sanitizer; // NOLINT // ----------- ATTENTION ------------- @@ -245,7 +210,7 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, #define CHECK_GT(a, b) CHECK_IMPL((a), >, (b)) #define CHECK_GE(a, b) CHECK_IMPL((a), >=, (b)) -#if TSAN_DEBUG +#if SANITIZER_DEBUG #define DCHECK(a) CHECK(a) #define DCHECK_EQ(a, b) CHECK_EQ(a, b) #define DCHECK_NE(a, b) CHECK_NE(a, b) @@ -325,4 +290,11 @@ extern "C" void* _ReturnAddress(void); } while (internal_iserror(res, &rverrno) && rverrno == EINTR); \ } +// Forces the compiler to generate a frame pointer in the function. +#define ENABLE_FRAME_POINTER \ + do { \ + volatile uptr enable_fp; \ + enable_fp = GET_CURRENT_FRAME(); \ + } while (0) + #endif // SANITIZER_DEFS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc index d8bd1cf7a7e0..cb162a4c4984 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc @@ -28,6 +28,15 @@ void *internal_memchr(const void *s, int c, uptr n) { return 0; } +void *internal_memrchr(const void *s, int c, uptr n) { + const char *t = (const char *)s; + void *res = nullptr; + for (uptr i = 0; i < n; ++i, ++t) { + if (*t == c) res = reinterpret_cast(const_cast(t)); + } + return res; +} + int internal_memcmp(const void* s1, const void* s2, uptr n) { const char *t1 = (const char *)s1; const char *t2 = (const char *)s2; @@ -101,6 +110,14 @@ char* internal_strdup(const char *s) { return s2; } +char* internal_strndup(const char *s, uptr n) { + uptr len = internal_strnlen(s, n); + char *s2 = (char*)InternalAlloc(len + 1); + internal_memcpy(s2, s, len); + s2[len] = 0; + return s2; +} + int internal_strcmp(const char *s1, const char *s2) { while (true) { unsigned c1 = *s1; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h index 6995626821ab..c086b8a9139e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h @@ -26,6 +26,7 @@ namespace __sanitizer { // String functions s64 internal_atoll(const char *nptr); void *internal_memchr(const void *s, int c, uptr n); +void *internal_memrchr(const void *s, int c, uptr n); int internal_memcmp(const void* s1, const void* s2, uptr n); void *internal_memcpy(void *dest, const void *src, uptr n); void *internal_memmove(void *dest, const void *src, uptr n); @@ -38,6 +39,7 @@ char *internal_strchrnul(const char *s, int c); int internal_strcmp(const char *s1, const char *s2); uptr internal_strcspn(const char *s, const char *reject); char *internal_strdup(const char *s); +char *internal_strndup(const char *s, uptr n); uptr internal_strlen(const char *s); char *internal_strncat(char *dst, const char *src, uptr n); int internal_strncmp(const char *s1, const char *s2, uptr n); @@ -98,6 +100,25 @@ int internal_fork(); // Threading uptr internal_sched_yield(); +// These functions call appropriate pthread_ functions directly, bypassing +// the interceptor. They are weak and may not be present in some tools. +SANITIZER_WEAK_ATTRIBUTE +int real_pthread_create(void *th, void *attr, void *(*callback)(void *), + void *param); +SANITIZER_WEAK_ATTRIBUTE +int real_pthread_join(void *th, void **ret); + +#define DEFINE_REAL_PTHREAD_FUNCTIONS \ + namespace __sanitizer { \ + int real_pthread_create(void *th, void *attr, void *(*callback)(void *), \ + void *param) { \ + return REAL(pthread_create)(th, attr, callback, param); \ + } \ + int real_pthread_join(void *th, void **ret) { \ + return REAL(pthread_join(th, ret)); \ + } \ + } // namespace __sanitizer + // Error handling bool internal_iserror(uptr retval, int *rverrno = 0); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc index 8df0467b1e9b..cefb1dc97a17 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc @@ -19,24 +19,18 @@ namespace __sanitizer { LibIgnore::LibIgnore(LinkerInitialized) { } -void LibIgnore::Init(const SuppressionContext &supp) { +void LibIgnore::AddIgnoredLibrary(const char *name_templ) { BlockingMutexLock lock(&mutex_); - CHECK_EQ(count_, 0); - const uptr n = supp.SuppressionCount(); - for (uptr i = 0; i < n; i++) { - const Suppression *s = supp.SuppressionAt(i); - if (s->type != SuppressionLib) - continue; - if (count_ >= kMaxLibs) { - Report("%s: too many called_from_lib suppressions (max: %d)\n", - SanitizerToolName, kMaxLibs); - Die(); - } - Lib *lib = &libs_[count_++]; - lib->templ = internal_strdup(s->templ); - lib->name = 0; - lib->loaded = false; + if (count_ >= kMaxLibs) { + Report("%s: too many ignored libraries (max: %d)\n", SanitizerToolName, + kMaxLibs); + Die(); } + Lib *lib = &libs_[count_++]; + lib->templ = internal_strdup(name_templ); + lib->name = nullptr; + lib->real_name = nullptr; + lib->loaded = false; } void LibIgnore::OnLibraryLoaded(const char *name) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h index 8e1d584d8e3c..cd56c36c1c0e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h @@ -8,8 +8,8 @@ //===----------------------------------------------------------------------===// // // LibIgnore allows to ignore all interceptors called from a particular set -// of dynamic libraries. LibIgnore remembers all "called_from_lib" suppressions -// from the provided SuppressionContext; finds code ranges for the libraries; +// of dynamic libraries. LibIgnore can be initialized with several templates +// of names of libraries to be ignored. It finds code ranges for the libraries; // and checks whether the provided PC value belongs to the code ranges. // //===----------------------------------------------------------------------===// @@ -19,7 +19,6 @@ #include "sanitizer_internal_defs.h" #include "sanitizer_common.h" -#include "sanitizer_suppressions.h" #include "sanitizer_atomic.h" #include "sanitizer_mutex.h" @@ -29,8 +28,8 @@ class LibIgnore { public: explicit LibIgnore(LinkerInitialized); - // Fetches all "called_from_lib" suppressions from the SuppressionContext. - void Init(const SuppressionContext &supp); + // Must be called during initialization. + void AddIgnoredLibrary(const char *name_templ); // Must be called after a new dynamic library is loaded. void OnLibraryLoaded(const char *name); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index 36de1ec70e97..8029181a5173 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -31,6 +31,17 @@ #include #endif +// For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat' +// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To +// access stat from asm/stat.h, without conflicting with definition in +// sys/stat.h, we use this trick. +#if defined(__mips64) +#include +#define stat kernel_stat +#include +#undef stat +#endif + #include #include #include @@ -98,14 +109,16 @@ namespace __sanitizer { #endif // --------------- sanitizer_libc.h -uptr internal_mmap(void *addr, uptr length, int prot, int flags, - int fd, u64 offset) { +uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, + u64 offset) { #if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd, offset); #else + // mmap2 specifies file offset in 4096-byte units. + CHECK(IsAligned(offset, 4096)); return internal_syscall(SYSCALL(mmap2), addr, length, prot, flags, fd, - offset); + offset / 4096); #endif } @@ -179,6 +192,26 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { } #endif +#if defined(__mips64) +static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { + internal_memset(out, 0, sizeof(*out)); + out->st_dev = in->st_dev; + out->st_ino = in->st_ino; + out->st_mode = in->st_mode; + out->st_nlink = in->st_nlink; + out->st_uid = in->st_uid; + out->st_gid = in->st_gid; + out->st_rdev = in->st_rdev; + out->st_size = in->st_size; + out->st_blksize = in->st_blksize; + out->st_blocks = in->st_blocks; + out->st_atime = in->st_atime_nsec; + out->st_mtime = in->st_mtime_nsec; + out->st_ctime = in->st_ctime_nsec; + out->st_ino = in->st_ino; +} +#endif + uptr internal_stat(const char *path, void *buf) { #if SANITIZER_FREEBSD return internal_syscall(SYSCALL(stat), path, buf); @@ -186,7 +219,15 @@ uptr internal_stat(const char *path, void *buf) { return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); #elif SANITIZER_LINUX_USES_64BIT_SYSCALLS +# if defined(__mips64) + // For mips64, stat syscall fills buffer in the format of kernel_stat + struct kernel_stat kbuf; + int res = internal_syscall(SYSCALL(stat), path, &kbuf); + kernel_stat_to_stat(&kbuf, (struct stat *)buf); + return res; +# else return internal_syscall(SYSCALL(stat), (uptr)path, (uptr)buf); +# endif #else struct stat64 buf64; int res = internal_syscall(SYSCALL(stat64), path, &buf64); @@ -381,33 +422,6 @@ static void ReadNullSepFileToArray(const char *path, char ***arr, } #endif -uptr GetRSS() { - uptr fd = OpenFile("/proc/self/statm", false); - if ((sptr)fd < 0) - return 0; - char buf[64]; - uptr len = internal_read(fd, buf, sizeof(buf) - 1); - internal_close(fd); - if ((sptr)len <= 0) - return 0; - buf[len] = 0; - // The format of the file is: - // 1084 89 69 11 0 79 0 - // We need the second number which is RSS in 4K units. - char *pos = buf; - // Skip the first number. - while (*pos >= '0' && *pos <= '9') - pos++; - // Skip whitespaces. - while (!(*pos >= '0' && *pos <= '9') && *pos != 0) - pos++; - // Read the number. - uptr rss = 0; - while (*pos >= '0' && *pos <= '9') - rss = rss * 10 + *pos++ - '0'; - return rss * 4096; -} - static void GetArgsAndEnv(char*** argv, char*** envp) { #if !SANITIZER_GO if (&__libc_stack_end) { @@ -435,32 +449,18 @@ void ReExec() { Die(); } -// Stub implementation of GetThreadStackAndTls for Go. -#if SANITIZER_GO -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { - *stk_addr = 0; - *stk_size = 0; - *tls_addr = 0; - *tls_size = 0; -} -#endif // SANITIZER_GO - enum MutexState { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; -BlockingMutex::BlockingMutex(LinkerInitialized) { - CHECK_EQ(owner_, 0); -} - BlockingMutex::BlockingMutex() { internal_memset(this, 0, sizeof(*this)); } void BlockingMutex::Lock() { + CHECK_EQ(owner_, 0); atomic_uint32_t *m = reinterpret_cast(&opaque_storage_); if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked) return; @@ -760,6 +760,7 @@ bool LibraryNameIs(const char *full_name, const char *base_name) { #if !SANITIZER_ANDROID // Call cb for each region mapped by map. void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)) { + CHECK_NE(map, nullptr); #if !SANITIZER_FREEBSD typedef ElfW(Phdr) Elf_Phdr; typedef ElfW(Ehdr) Elf_Ehdr; @@ -859,6 +860,13 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, : "rsp", "memory", "r11", "rcx"); return res; } +#elif defined(__mips__) +// TODO(sagarthakur): clone function is to be rewritten in assembly. +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + return clone(fn, child_stack, flags, arg, parent_tidptr, + newtls, child_tidptr); +} #endif // defined(__x86_64__) && SANITIZER_LINUX #if SANITIZER_ANDROID @@ -896,9 +904,30 @@ void GetExtraActivationFlags(char *buf, uptr size) { #endif bool IsDeadlySignal(int signum) { - return (signum == SIGSEGV) && common_flags()->handle_segv; + return (signum == SIGSEGV || signum == SIGBUS) && common_flags()->handle_segv; } +#ifndef SANITIZER_GO +void *internal_start_thread(void(*func)(void *arg), void *arg) { + // Start the thread with signals blocked, otherwise it can steal user signals. + __sanitizer_sigset_t set, old; + internal_sigfillset(&set); + internal_sigprocmask(SIG_SETMASK, &set, &old); + void *th; + real_pthread_create(&th, 0, (void*(*)(void *arg))func, arg); + internal_sigprocmask(SIG_SETMASK, &old, 0); + return th; +} + +void internal_join_thread(void *th) { + real_pthread_join(th, 0); +} +#else +void *internal_start_thread(void (*func)(void *), void *arg) { return 0; } + +void internal_join_thread(void *th) {} +#endif + } // namespace __sanitizer #endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 3013c25f7c38..b2e603d3a23e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -43,7 +43,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5); // internal_sigaction instead. int internal_sigaction_norestorer(int signum, const void *act, void *oldact); void internal_sigdelset(__sanitizer_sigset_t *set, int signum); -#if defined(__x86_64__) +#if defined(__x86_64__) || defined(__mips__) uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr); #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 4e09081e7f54..c71b6257ebc3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -58,8 +58,10 @@ real_pthread_attr_getstack(void *attr, void **addr, size_t *size); } // extern "C" static int my_pthread_attr_getstack(void *attr, void **addr, size_t *size) { - if (real_pthread_attr_getstack) +#if !SANITIZER_GO + if (&real_pthread_attr_getstack) return real_pthread_attr_getstack((pthread_attr_t *)attr, addr, size); +#endif return pthread_attr_getstack((pthread_attr_t *)attr, addr, size); } @@ -67,8 +69,10 @@ SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act, void *oldact); int internal_sigaction(int signum, const void *act, void *oldact) { - if (real_sigaction) +#if !SANITIZER_GO + if (&real_sigaction) return real_sigaction(signum, act, oldact); +#endif return sigaction(signum, (const struct sigaction *)act, (struct sigaction *)oldact); } @@ -120,6 +124,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, *stack_bottom = (uptr)stackaddr; } +#if !SANITIZER_GO bool SetEnv(const char *name, const char *value) { void *f = dlsym(RTLD_NEXT, "setenv"); if (f == 0) @@ -130,6 +135,7 @@ bool SetEnv(const char *name, const char *value) { internal_memcpy(&setenv_f, &f, sizeof(f)); return setenv_f(name, value, 1) == 0; } +#endif bool SanitizerSetThreadName(const char *name) { #ifdef PR_SET_NAME @@ -162,8 +168,22 @@ static uptr g_tls_size; # define DL_INTERNAL_FUNCTION #endif +#if defined(__mips__) +// TlsPreTcbSize includes size of struct pthread_descr and size of tcb +// head structure. It lies before the static tls blocks. +static uptr TlsPreTcbSize() { + const uptr kTcbHead = 16; + const uptr kTlsAlign = 16; + const uptr kTlsPreTcbSize = + (ThreadDescriptorSize() + kTcbHead + kTlsAlign - 1) & ~(kTlsAlign - 1); + InitTlsSize(); + g_tls_size = (g_tls_size + kTlsPreTcbSize + kTlsAlign -1) & ~(kTlsAlign - 1); + return kTlsPreTcbSize; +} +#endif + void InitTlsSize() { -#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO typedef void (*get_tls_func)(size_t*, size_t*) DL_INTERNAL_FUNCTION; get_tls_func get_tls; void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info"); @@ -178,7 +198,8 @@ void InitTlsSize() { #endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID } -#if (defined(__x86_64__) || defined(__i386__)) && SANITIZER_LINUX +#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__)) \ + && SANITIZER_LINUX // sizeof(struct thread) from glibc. static atomic_uintptr_t kThreadDescriptorSize; @@ -186,6 +207,7 @@ uptr ThreadDescriptorSize() { uptr val = atomic_load(&kThreadDescriptorSize, memory_order_relaxed); if (val) return val; +#if defined(__x86_64__) || defined(__i386__) #ifdef _CS_GNU_LIBC_VERSION char buf[64]; uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf)); @@ -208,6 +230,8 @@ uptr ThreadDescriptorSize() { val = FIRST_32_SECOND_64(1168, 1776); else if (minor <= 12) val = FIRST_32_SECOND_64(1168, 2288); + else if (minor == 13) + val = FIRST_32_SECOND_64(1168, 2304); else val = FIRST_32_SECOND_64(1216, 2304); } @@ -215,6 +239,13 @@ uptr ThreadDescriptorSize() { atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); return val; } +#endif +#elif defined(__mips__) + // TODO(sagarthakur): add more values as per different glibc versions. + val = FIRST_32_SECOND_64(1152, 1776); + if (val) + atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); + return val; #endif return 0; } @@ -232,12 +263,24 @@ uptr ThreadSelf() { asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset)); # elif defined(__x86_64__) asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset)); +# elif defined(__mips__) + // MIPS uses TLS variant I. The thread pointer (in hardware register $29) + // points to the end of the TCB + 0x7000. The pthread_descr structure is + // immediately in front of the TCB. TlsPreTcbSize() includes the size of the + // TCB and the size of pthread_descr. + const uptr kTlsTcbOffset = 0x7000; + uptr thread_pointer; + asm volatile(".set push;\ + .set mips64r2;\ + rdhwr %0,$29;\ + .set pop" : "=r" (thread_pointer)); + descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize(); # else # error "unsupported CPU arch" # endif return descr_addr; } -#endif // (defined(__x86_64__) || defined(__i386__)) && SANITIZER_LINUX +#endif // (x86_64 || i386 || MIPS) && SANITIZER_LINUX #if SANITIZER_FREEBSD static void **ThreadSelfSegbase() { @@ -259,6 +302,7 @@ uptr ThreadSelf() { } #endif // SANITIZER_FREEBSD +#if !SANITIZER_GO static void GetTls(uptr *addr, uptr *size) { #if SANITIZER_LINUX # if defined(__x86_64__) || defined(__i386__) @@ -266,6 +310,9 @@ static void GetTls(uptr *addr, uptr *size) { *size = GetTlsSize(); *addr -= *size; *addr += ThreadDescriptorSize(); +# elif defined(__mips__) + *addr = ThreadSelf(); + *size = GetTlsSize(); # else *addr = 0; *size = 0; @@ -287,7 +334,9 @@ static void GetTls(uptr *addr, uptr *size) { # error "Unknown OS" #endif } +#endif +#if !SANITIZER_GO uptr GetTlsSize() { #if SANITIZER_FREEBSD uptr addr, size; @@ -297,9 +346,14 @@ uptr GetTlsSize() { return g_tls_size; #endif } +#endif void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size) { +#if SANITIZER_GO + // Stub implementation for Go. + *stk_addr = *stk_size = *tls_addr = *tls_size = 0; +#else GetTls(tls_addr, tls_size); uptr stack_top, stack_bottom; @@ -316,6 +370,7 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, *tls_addr = *stk_addr + *stk_size; } } +#endif } void AdjustStackSize(void *attr_) { @@ -420,6 +475,45 @@ void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) { #endif } +// getrusage does not give us the current RSS, only the max RSS. +// Still, this is better than nothing if /proc/self/statm is not available +// for some reason, e.g. due to a sandbox. +static uptr GetRSSFromGetrusage() { + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) // Failed, probably due to a sandbox. + return 0; + return usage.ru_maxrss << 10; // ru_maxrss is in Kb. +} + +uptr GetRSS() { + if (!common_flags()->can_use_proc_maps_statm) + return GetRSSFromGetrusage(); + uptr fd = OpenFile("/proc/self/statm", false); + if ((sptr)fd < 0) + return GetRSSFromGetrusage(); + char buf[64]; + uptr len = internal_read(fd, buf, sizeof(buf) - 1); + internal_close(fd); + if ((sptr)len <= 0) + return 0; + buf[len] = 0; + // The format of the file is: + // 1084 89 69 11 0 79 0 + // We need the second number which is RSS in pages. + char *pos = buf; + // Skip the first number. + while (*pos >= '0' && *pos <= '9') + pos++; + // Skip whitespaces. + while (!(*pos >= '0' && *pos <= '9') && *pos != 0) + pos++; + // Read the number. + uptr rss = 0; + while (*pos >= '0' && *pos <= '9') + rss = rss * 10 + *pos++ - '0'; + return rss * GetPageSizeCached(); +} + } // namespace __sanitizer #endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_list.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_list.h index a47bc7d45e3e..6dd9c8f7bca1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_list.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_list.h @@ -115,21 +115,25 @@ struct IntrusiveList { } } - class Iterator { + template + class IteratorBase { public: - explicit Iterator(IntrusiveList *list) + explicit IteratorBase(ListTy *list) : list_(list), current_(list->first_) { } - Item *next() { - Item *ret = current_; + ItemTy *next() { + ItemTy *ret = current_; if (current_) current_ = current_->next; return ret; } bool hasNext() const { return current_ != 0; } private: - IntrusiveList *list_; - Item *current_; + ListTy *list_; + ItemTy *current_; }; + typedef IteratorBase, Item> Iterator; + typedef IteratorBase, const Item> ConstIterator; + // private, don't use directly. uptr size_; Item *first_; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc index 98c5b94112cf..39a5c7e8d24f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -109,6 +109,10 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return readlink(path, buf, bufsize); } +uptr internal_unlink(const char *path) { + return unlink(path); +} + uptr internal_sched_yield() { return sched_yield(); } @@ -213,10 +217,6 @@ uptr GetPageSize() { return sysconf(_SC_PAGESIZE); } -BlockingMutex::BlockingMutex(LinkerInitialized) { - // We assume that OS_SPINLOCK_INIT is zero -} - BlockingMutex::BlockingMutex() { internal_memset(this, 0, sizeof(*this)); } @@ -298,7 +298,11 @@ MacosVersion GetMacosVersionInternal() { case '2': return MACOS_VERSION_MOUNTAIN_LION; case '3': return MACOS_VERSION_MAVERICKS; case '4': return MACOS_VERSION_YOSEMITE; - default: return MACOS_VERSION_UNKNOWN; + default: + if (IsDigit(version[1])) + return MACOS_VERSION_UNKNOWN_NEWER; + else + return MACOS_VERSION_UNKNOWN; } } default: return MACOS_VERSION_UNKNOWN; @@ -321,6 +325,9 @@ uptr GetRSS() { return 0; } +void *internal_start_thread(void (*func)(void *arg), void *arg) { return 0; } +void internal_join_thread(void *th) { } + } // namespace __sanitizer #endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index 3ed0ed3b00c2..9eed905187ec 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -27,6 +27,7 @@ enum MacosVersion { MACOS_VERSION_MOUNTAIN_LION, MACOS_VERSION_MAVERICKS, MACOS_VERSION_YOSEMITE, + MACOS_VERSION_UNKNOWN_NEWER }; MacosVersion GetMacosVersion(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h index c7589f76ed3b..d06fc45ff931 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h @@ -73,7 +73,13 @@ class SpinMutex : public StaticSpinMutex { class BlockingMutex { public: +#if SANITIZER_WINDOWS + // Windows does not currently support LinkerInitialized explicit BlockingMutex(LinkerInitialized); +#else + explicit constexpr BlockingMutex(LinkerInitialized) + : opaque_storage_ {0, }, owner_(0) {} +#endif BlockingMutex(); void Lock(); void Unlock(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index 7ca88fa972f3..438ecbaa2ec8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -57,7 +57,7 @@ #define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_MEMCHR 1 -#define SANITIZER_INTERCEPT_MEMRCHR SI_LINUX +#define SANITIZER_INTERCEPT_MEMRCHR SI_FREEBSD || SI_LINUX #define SANITIZER_INTERCEPT_READ SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_PREAD SI_NOT_WINDOWS @@ -70,7 +70,7 @@ #define SANITIZER_INTERCEPT_READV SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_WRITEV SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_PREADV SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PREADV SI_FREEBSD || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PWRITEV SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PREADV64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PWRITEV64 SI_LINUX_NOT_ANDROID @@ -85,6 +85,7 @@ #ifndef SANITIZER_INTERCEPT_PRINTF # define SANITIZER_INTERCEPT_PRINTF SI_NOT_WINDOWS +# define SANITIZER_INTERCEPT_PRINTF_L SI_FREEBSD # define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID #endif @@ -93,12 +94,13 @@ #define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \ - SI_MAC || SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_GETPWENT SI_MAC || SI_LINUX_NOT_ANDROID + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_GETPWENT \ + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_GETPWENT_R SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_GETPWENT_R SI_FREEBSD || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SETPWENT SI_MAC || SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_CLOCK_GETTIME SI_LINUX +#define SANITIZER_INTERCEPT_CLOCK_GETTIME SI_FREEBSD || SI_LINUX #define SANITIZER_INTERCEPT_GETITIMER SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_TIME SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID @@ -109,10 +111,10 @@ #define SANITIZER_INTERCEPT_GETNAMEINFO SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETSOCKNAME SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R SI_LINUX -#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_GETHOSTENT_R SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R SI_FREEBSD || SI_LINUX +#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R SI_FREEBSD || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R SI_FREEBSD || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_GETHOSTENT_R SI_FREEBSD || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_GETSOCKOPT SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_ACCEPT SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_ACCEPT4 SI_LINUX_NOT_ANDROID @@ -125,7 +127,7 @@ #define SANITIZER_INTERCEPT_READDIR SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PTRACE SI_LINUX_NOT_ANDROID && \ - (defined(__i386) || defined (__x86_64)) // NOLINT + (defined(__i386) || defined (__x86_64) || defined (__mips64)) // NOLINT #define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID @@ -133,12 +135,15 @@ #define SANITIZER_INTERCEPT_MBSTOWCS SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_MBSNRTOWCS SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_WCSTOMBS SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_WCSNRTOMBS SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WCSNRTOMBS \ + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_REALPATH SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_CONFSTR SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_CONFSTR \ + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STRERROR SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_STRERROR_R SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID @@ -147,7 +152,8 @@ #define SANITIZER_INTERCEPT_GETGROUPS SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_POLL SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_WORDEXP (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WORDEXP \ + SI_FREEBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SIGWAIT SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID @@ -158,21 +164,22 @@ #define SANITIZER_INTERCEPT_BACKTRACE SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STATFS SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_STATFS SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STATFS64 \ (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_STATVFS SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_STATVFS SI_FREEBSD || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_INITGROUPS SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_ETHER_HOST SI_MAC || SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_ETHER_R SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_ETHER_HOST \ + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_ETHER_R SI_FREEBSD || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_SHMCTL \ ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) #define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ - SI_MAC || SI_LINUX_NOT_ANDROID + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_NOT_WINDOWS @@ -193,7 +200,7 @@ #define SANITIZER_INTERCEPT_SINCOS SI_LINUX #define SANITIZER_INTERCEPT_REMQUO SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_LGAMMA SI_NOT_WINDOWS -#define SANITIZER_INTERCEPT_LGAMMA_R SI_LINUX +#define SANITIZER_INTERCEPT_LGAMMA_R SI_FREEBSD || SI_LINUX #define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_RAND_R SI_MAC || SI_LINUX_NOT_ANDROID diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index 808e92cb265a..8824c8088f2d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -116,6 +116,9 @@ #if SANITIZER_LINUX || SANITIZER_FREEBSD # include # include +# if defined(__mips64) +# include +# endif #endif #if !SANITIZER_ANDROID @@ -139,6 +142,9 @@ #include #include #include +#if defined(__mips64) +# include +#endif #include #include #include @@ -283,14 +289,19 @@ namespace __sanitizer { #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ - (defined(__i386) || defined(__x86_64)) + (defined(__i386) || defined(__x86_64) || defined(__mips64)) +#if defined(__mips64) + unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs); + unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t); +#else unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct); unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct); -#ifdef __x86_64 +#endif // __mips64 +#if (defined(__x86_64) || defined(__mips64)) unsigned struct_user_fpxregs_struct_sz = 0; #else unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct); -#endif +#endif // __x86_64 || __mips64 int ptrace_peektext = PTRACE_PEEKTEXT; int ptrace_peekdata = PTRACE_PEEKDATA; @@ -1060,7 +1071,13 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid); CHECK_SIZE_AND_OFFSET(ipc_perm, gid); CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); +#ifndef __GLIBC_PREREQ +#define __GLIBC_PREREQ(x, y) 0 +#endif +#if !defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21) +/* On aarch64 glibc 2.20 and earlier provided incorrect mode field. */ CHECK_SIZE_AND_OFFSET(ipc_perm, mode); +#endif CHECK_TYPE_SIZE(shmid_ds); CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 80a3ddb36670..bd20bea94e93 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -18,6 +18,15 @@ #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" +#if SANITIZER_FREEBSD +// FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that +// incroporates the map structure. +# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 544))) +#else +# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) ((link_map*)(handle)) +#endif // !SANITIZER_FREEBSD + namespace __sanitizer { extern unsigned struct_utsname_sz; extern unsigned struct_stat_sz; @@ -169,7 +178,7 @@ namespace __sanitizer { unsigned __seq; u64 __unused1; u64 __unused2; -#elif defined(__mips__) +#elif defined(__mips__) || defined(__aarch64__) unsigned int mode; unsigned short __seq; unsigned short __pad1; @@ -538,6 +547,10 @@ namespace __sanitizer { #if SANITIZER_FREEBSD typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; +#elif defined(__mips__) + struct __sanitizer_kernel_sigset_t { + u8 sig[16]; + }; #else struct __sanitizer_kernel_sigset_t { u8 sig[8]; @@ -686,7 +699,7 @@ namespace __sanitizer { #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ - (defined(__i386) || defined(__x86_64)) + (defined(__i386) || defined(__x86_64) || defined(__mips64)) extern unsigned struct_user_regs_struct_sz; extern unsigned struct_user_fpregs_struct_sz; extern unsigned struct_user_fpxregs_struct_sz; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc index 3b2c47bff22b..5bc41c2580fb 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc @@ -85,16 +85,15 @@ static uptr GetKernelAreaSize() { uptr GetMaxVirtualAddress() { #if SANITIZER_WORDSIZE == 64 -# if defined(__powerpc64__) +# if defined(__powerpc64__) || defined(__aarch64__) // On PowerPC64 we have two different address space layouts: 44- and 46-bit. // We somehow need to figure out which one we are using now and choose // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL. // Note that with 'ulimit -s unlimited' the stack is moved away from the top // of the address space, so simply checking the stack address is not enough. // This should (does) work for both PowerPC64 Endian modes. + // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit. return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1; -# elif defined(__aarch64__) - return (1ULL << 39) - 1; # elif defined(__mips64) return (1ULL << 40) - 1; // 0x000000ffffffffffUL; # else @@ -245,7 +244,8 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { while (proc_maps.Next(&start, &end, /*offset*/0, /*filename*/0, /*filename_size*/0, /*protection*/0)) { - if (!IntervalsAreSeparate(start, end, range_start, range_end)) + CHECK_NE(0, end); + if (!IntervalsAreSeparate(start, end - 1, range_start, range_end)) return false; } return true; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc index ed1e3729a00e..11828e6cdf51 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -44,6 +44,18 @@ void FlushUnneededShadowMemory(uptr addr, uptr size) { madvise((void*)addr, size, MADV_DONTNEED); } +void NoHugePagesInRegion(uptr addr, uptr size) { +#ifdef MADV_NOHUGEPAGE // May not be defined on old systems. + madvise((void *)addr, size, MADV_NOHUGEPAGE); +#endif // MADV_NOHUGEPAGE +} + +void DontDumpShadowMemory(uptr addr, uptr length) { +#ifdef MADV_DONTDUMP + madvise((void *)addr, length, MADV_DONTDUMP); +#endif +} + static rlim_t getlim(int res) { rlimit rlim; CHECK_EQ(0, getrlimit(res, &rlim)); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h index db4eb74505f9..404d3753f7e9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h @@ -49,11 +49,14 @@ class Quarantine { } void Init(uptr size, uptr cache_size) { - max_size_ = size; - min_size_ = size / 10 * 9; // 90% of max size. + atomic_store(&max_size_, size, memory_order_release); + atomic_store(&min_size_, size / 10 * 9, + memory_order_release); // 90% of max size. max_cache_size_ = cache_size; } + uptr GetSize() const { return atomic_load(&max_size_, memory_order_acquire); } + void Put(Cache *c, Callback cb, Node *ptr, uptr size) { c->Enqueue(cb, ptr, size); if (c->Size() > max_cache_size_) @@ -65,15 +68,15 @@ class Quarantine { SpinMutexLock l(&cache_mutex_); cache_.Transfer(c); } - if (cache_.Size() > max_size_ && recycle_mutex_.TryLock()) + if (cache_.Size() > GetSize() && recycle_mutex_.TryLock()) Recycle(cb); } private: // Read-only data. char pad0_[kCacheLineSize]; - uptr max_size_; - uptr min_size_; + atomic_uintptr_t max_size_; + atomic_uintptr_t min_size_; uptr max_cache_size_; char pad1_[kCacheLineSize]; SpinMutex cache_mutex_; @@ -83,9 +86,10 @@ class Quarantine { void NOINLINE Recycle(Callback cb) { Cache tmp; + uptr min_size = atomic_load(&min_size_, memory_order_acquire); { SpinMutexLock l(&cache_mutex_); - while (cache_.Size() > min_size_) { + while (cache_.Size() > min_size) { QuarantineBatch *b = cache_.DequeueBatch(); tmp.EnqueueBatch(b); } @@ -130,6 +134,7 @@ class QuarantineCache { size += sizeof(QuarantineBatch); // Count the batch in Quarantine size. } QuarantineBatch *b = list_.back(); + CHECK(b); b->batch[b->count++] = ptr; b->size += size; SizeAdd(size); @@ -168,6 +173,7 @@ class QuarantineCache { NOINLINE QuarantineBatch* AllocBatch(Callback cb) { QuarantineBatch *b = (QuarantineBatch *)cb.Allocate(sizeof(*b)); + CHECK(b); b->count = 0; b->size = 0; list_.push_back(b); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc index f10f1f973fd0..59b53f4dcd84 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc @@ -22,7 +22,8 @@ struct StackDepotNode { StackDepotNode *link; u32 id; atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20; - uptr size; + u32 size; + u32 tag; uptr stack[1]; // [size] static const u32 kTabSizeLog = 20; @@ -37,7 +38,8 @@ struct StackDepotNode { bool eq(u32 hash, const args_type &args) const { u32 hash_bits = atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask; - if ((hash & kHashMask) != hash_bits || args.size != size) return false; + if ((hash & kHashMask) != hash_bits || args.size != size || args.tag != tag) + return false; uptr i = 0; for (; i < size; i++) { if (stack[i] != args.trace[i]) return false; @@ -72,10 +74,11 @@ struct StackDepotNode { void store(const args_type &args, u32 hash) { atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed); size = args.size; + tag = args.tag; internal_memcpy(stack, args.trace, size * sizeof(uptr)); } args_type load() const { - return args_type(&stack[0], size); + return args_type(&stack[0], size, tag); } StackDepotHandle get_handle() { return StackDepotHandle(this); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cc index cf061fb8c73f..2deadb6e3560 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cc @@ -17,21 +17,6 @@ namespace __sanitizer { -uptr StackTrace::GetPreviousInstructionPc(uptr pc) { -#if defined(__arm__) - // Cancel Thumb bit. - pc = pc & (~1); -#endif -#if defined(__powerpc__) || defined(__powerpc64__) - // PCs are always 4 byte aligned. - return pc - 4; -#elif defined(__sparc__) || defined(__mips__) - return pc - 8; -#else - return pc - 1; -#endif -} - uptr StackTrace::GetNextInstructionPc(uptr pc) { #if defined(__mips__) return pc + 8; @@ -83,7 +68,7 @@ static inline uhwptr *GetCanonicFrame(uptr bp, } void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, - uptr stack_bottom, uptr max_depth) { + uptr stack_bottom, u32 max_depth) { CHECK_GE(max_depth, 2); trace_buffer[0] = pc; size = 1; @@ -120,7 +105,7 @@ void BufferedStackTrace::PopStackFrames(uptr count) { uptr BufferedStackTrace::LocatePcInTrace(uptr pc) { // Use threshold to find PC in stack trace, as PC we want to unwind from may // slightly differ from return address in the actual unwinded stack trace. - const int kPcThreshold = 288; + const int kPcThreshold = 304; for (uptr i = 0; i < size; ++i) { if (MatchPc(pc, trace[i], kPcThreshold)) return i; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index e755c052cb77..6c3a1511f337 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -17,7 +17,7 @@ namespace __sanitizer { -static const uptr kStackTraceMax = 256; +static const u32 kStackTraceMax = 256; #if SANITIZER_LINUX && (defined(__aarch64__) || defined(__powerpc__) || \ defined(__powerpc64__) || defined(__sparc__) || \ @@ -40,10 +40,18 @@ static const uptr kStackTraceMax = 256; struct StackTrace { const uptr *trace; - uptr size; + u32 size; + u32 tag; - StackTrace() : trace(nullptr), size(0) {} - StackTrace(const uptr *trace, uptr size) : trace(trace), size(size) {} + static const int TAG_UNKNOWN = 0; + static const int TAG_ALLOC = 1; + static const int TAG_DEALLOC = 2; + static const int TAG_CUSTOM = 100; // Tool specific tags start here. + + StackTrace() : trace(nullptr), size(0), tag(0) {} + StackTrace(const uptr *trace, u32 size) : trace(trace), size(size), tag(0) {} + StackTrace(const uptr *trace, u32 size, u32 tag) + : trace(trace), size(size), tag(tag) {} // Prints a symbolized stacktrace, followed by an empty line. void Print() const; @@ -57,12 +65,29 @@ struct StackTrace { } static uptr GetCurrentPc(); - static uptr GetPreviousInstructionPc(uptr pc); + static inline uptr GetPreviousInstructionPc(uptr pc); static uptr GetNextInstructionPc(uptr pc); typedef bool (*SymbolizeCallback)(const void *pc, char *out_buffer, int out_size); }; +// Performance-critical, must be in the header. +ALWAYS_INLINE +uptr StackTrace::GetPreviousInstructionPc(uptr pc) { +#if defined(__arm__) + // Cancel Thumb bit. + pc = pc & (~1); +#endif +#if defined(__powerpc__) || defined(__powerpc64__) + // PCs are always 4 byte aligned. + return pc - 4; +#elif defined(__sparc__) || defined(__mips__) + return pc - 8; +#else + return pc - 1; +#endif +} + // StackTrace that owns the buffer used to store the addresses. struct BufferedStackTrace : public StackTrace { uptr trace_buffer[kStackTraceMax]; @@ -71,15 +96,15 @@ struct BufferedStackTrace : public StackTrace { BufferedStackTrace() : StackTrace(trace_buffer, 0), top_frame_bp(0) {} void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0); - void Unwind(uptr max_depth, uptr pc, uptr bp, void *context, uptr stack_top, + void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind); private: void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, - uptr max_depth); - void SlowUnwindStack(uptr pc, uptr max_depth); + u32 max_depth); + void SlowUnwindStack(uptr pc, u32 max_depth); void SlowUnwindStackWithContext(uptr pc, void *context, - uptr max_depth); + u32 max_depth); void PopStackFrames(uptr count); uptr LocatePcInTrace(uptr pc); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc index 0d90980e6a68..0f98c7d5af4c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -44,7 +44,7 @@ void StackTrace::Print() const { Printf("\n"); } -void BufferedStackTrace::Unwind(uptr max_depth, uptr pc, uptr bp, void *context, +void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top, uptr stack_bottom, bool request_fast_unwind) { top_frame_bp = (max_depth > 0) ? bp : 0; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index d20b52483a91..ad20e39556d8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -14,7 +14,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_LINUX && defined(__x86_64__) +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)) #include "sanitizer_stoptheworld.h" @@ -89,36 +89,50 @@ class ThreadSuspender { bool SuspendThread(SuspendedThreadID thread_id); }; -bool ThreadSuspender::SuspendThread(SuspendedThreadID thread_id) { +bool ThreadSuspender::SuspendThread(SuspendedThreadID tid) { // Are we already attached to this thread? // Currently this check takes linear time, however the number of threads is // usually small. - if (suspended_threads_list_.Contains(thread_id)) + if (suspended_threads_list_.Contains(tid)) return false; int pterrno; - if (internal_iserror(internal_ptrace(PTRACE_ATTACH, thread_id, NULL, NULL), + if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, NULL, NULL), &pterrno)) { // Either the thread is dead, or something prevented us from attaching. // Log this event and move on. - VReport(1, "Could not attach to thread %d (errno %d).\n", thread_id, - pterrno); + VReport(1, "Could not attach to thread %d (errno %d).\n", tid, pterrno); return false; } else { - VReport(1, "Attached to thread %d.\n", thread_id); + VReport(1, "Attached to thread %d.\n", tid); // The thread is not guaranteed to stop before ptrace returns, so we must - // wait on it. - uptr waitpid_status; - HANDLE_EINTR(waitpid_status, internal_waitpid(thread_id, NULL, __WALL)); - int wperrno; - if (internal_iserror(waitpid_status, &wperrno)) { - // Got a ECHILD error. I don't think this situation is possible, but it - // doesn't hurt to report it. - VReport(1, "Waiting on thread %d failed, detaching (errno %d).\n", - thread_id, wperrno); - internal_ptrace(PTRACE_DETACH, thread_id, NULL, NULL); - return false; + // wait on it. Note: if the thread receives a signal concurrently, + // we can get notification about the signal before notification about stop. + // In such case we need to forward the signal to the thread, otherwise + // the signal will be missed (as we do PTRACE_DETACH with arg=0) and + // any logic relying on signals will break. After forwarding we need to + // continue to wait for stopping, because the thread is not stopped yet. + // We do ignore delivery of SIGSTOP, because we want to make stop-the-world + // as invisible as possible. + for (;;) { + int status; + uptr waitpid_status; + HANDLE_EINTR(waitpid_status, internal_waitpid(tid, &status, __WALL)); + int wperrno; + if (internal_iserror(waitpid_status, &wperrno)) { + // Got a ECHILD error. I don't think this situation is possible, but it + // doesn't hurt to report it. + VReport(1, "Waiting on thread %d failed, detaching (errno %d).\n", + tid, wperrno); + internal_ptrace(PTRACE_DETACH, tid, NULL, NULL); + return false; + } + if (WIFSTOPPED(status) && WSTOPSIG(status) != SIGSTOP) { + internal_ptrace(PTRACE_CONT, tid, 0, (void*)(uptr)WSTOPSIG(status)); + continue; + } + break; } - suspended_threads_list_.Append(thread_id); + suspended_threads_list_.Append(tid); return true; } } @@ -170,10 +184,9 @@ bool ThreadSuspender::SuspendAllThreads() { // Pointer to the ThreadSuspender instance for use in signal handler. static ThreadSuspender *thread_suspender_instance = NULL; -// Signals that should not be blocked (this is used in the parent thread as well -// as the tracer thread). -static const int kUnblockedSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, - SIGBUS, SIGXCPU, SIGXFSZ }; +// Synchronous signals that should not be blocked. +static const int kSyncSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGBUS, + SIGXCPU, SIGXFSZ }; // Structure for passing arguments into the tracer thread. struct TracerThreadArgument { @@ -188,7 +201,7 @@ struct TracerThreadArgument { static DieCallbackType old_die_callback; // Signal handler to wake up suspended threads when the tracer thread dies. -void TracerThreadSignalHandler(int signum, void *siginfo, void *) { +static void TracerThreadSignalHandler(int signum, void *siginfo, void *) { if (thread_suspender_instance != NULL) { if (signum == SIGABRT) thread_suspender_instance->KillAllThreads(); @@ -228,6 +241,7 @@ static int TracerThread(void* argument) { tracer_thread_argument->mutex.Lock(); tracer_thread_argument->mutex.Unlock(); + old_die_callback = GetDieCallback(); SetDieCallback(TracerThreadDieCallback); ThreadSuspender thread_suspender(internal_getppid()); @@ -242,17 +256,14 @@ static int TracerThread(void* argument) { handler_stack.ss_size = kHandlerStackSize; internal_sigaltstack(&handler_stack, NULL); - // Install our handler for fatal signals. Other signals should be blocked by - // the mask we inherited from the caller thread. - for (uptr signal_index = 0; signal_index < ARRAY_SIZE(kUnblockedSignals); - signal_index++) { - __sanitizer_sigaction new_sigaction; - internal_memset(&new_sigaction, 0, sizeof(new_sigaction)); - new_sigaction.sigaction = TracerThreadSignalHandler; - new_sigaction.sa_flags = SA_ONSTACK | SA_SIGINFO; - internal_sigfillset(&new_sigaction.sa_mask); - internal_sigaction_norestorer(kUnblockedSignals[signal_index], - &new_sigaction, NULL); + // Install our handler for synchronous signals. Other signals should be + // blocked by the mask we inherited from the parent thread. + for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) { + __sanitizer_sigaction act; + internal_memset(&act, 0, sizeof(act)); + act.sigaction = TracerThreadSignalHandler; + act.sa_flags = SA_ONSTACK | SA_SIGINFO; + internal_sigaction_norestorer(kSyncSignals[i], &act, 0); } int exit_code = 0; @@ -265,9 +276,11 @@ static int TracerThread(void* argument) { thread_suspender.ResumeAllThreads(); exit_code = 0; } + // Note, this is a bad race. If TracerThreadDieCallback is already started + // in another thread and observed that thread_suspender_instance != 0, + // it can call KillAllThreads on the destroyed variable. + SetDieCallback(old_die_callback); thread_suspender_instance = NULL; - handler_stack.ss_flags = SS_DISABLE; - internal_sigaltstack(&handler_stack, NULL); return exit_code; } @@ -299,53 +312,21 @@ class ScopedStackSpaceWithGuard { // into globals. static __sanitizer_sigset_t blocked_sigset; static __sanitizer_sigset_t old_sigset; -static __sanitizer_sigaction old_sigactions - [ARRAY_SIZE(kUnblockedSignals)]; class StopTheWorldScope { public: StopTheWorldScope() { - // Block all signals that can be blocked safely, and install - // default handlers for the remaining signals. - // We cannot allow user-defined handlers to run while the ThreadSuspender - // thread is active, because they could conceivably call some libc functions - // which modify errno (which is shared between the two threads). - internal_sigfillset(&blocked_sigset); - for (uptr signal_index = 0; signal_index < ARRAY_SIZE(kUnblockedSignals); - signal_index++) { - // Remove the signal from the set of blocked signals. - internal_sigdelset(&blocked_sigset, kUnblockedSignals[signal_index]); - // Install the default handler. - __sanitizer_sigaction new_sigaction; - internal_memset(&new_sigaction, 0, sizeof(new_sigaction)); - new_sigaction.handler = SIG_DFL; - internal_sigfillset(&new_sigaction.sa_mask); - internal_sigaction_norestorer(kUnblockedSignals[signal_index], - &new_sigaction, &old_sigactions[signal_index]); - } - int sigprocmask_status = - internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset); - CHECK_EQ(sigprocmask_status, 0); // sigprocmask should never fail // Make this process dumpable. Processes that are not dumpable cannot be // attached to. process_was_dumpable_ = internal_prctl(PR_GET_DUMPABLE, 0, 0, 0, 0); if (!process_was_dumpable_) internal_prctl(PR_SET_DUMPABLE, 1, 0, 0, 0); - old_die_callback = GetDieCallback(); } ~StopTheWorldScope() { - SetDieCallback(old_die_callback); // Restore the dumpable flag. if (!process_was_dumpable_) internal_prctl(PR_SET_DUMPABLE, 0, 0, 0, 0); - // Restore the signal handlers. - for (uptr signal_index = 0; signal_index < ARRAY_SIZE(kUnblockedSignals); - signal_index++) { - internal_sigaction_norestorer(kUnblockedSignals[signal_index], - &old_sigactions[signal_index], NULL); - } - internal_sigprocmask(SIG_SETMASK, &old_sigset, &old_sigset); } private: @@ -378,11 +359,36 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) { // Block the execution of TracerThread until after we have set ptrace // permissions. tracer_thread_argument.mutex.Lock(); + // Signal handling story. + // We don't want async signals to be delivered to the tracer thread, + // so we block all async signals before creating the thread. An async signal + // handler can temporary modify errno, which is shared with this thread. + // We ought to use pthread_sigmask here, because sigprocmask has undefined + // behavior in multithreaded programs. However, on linux sigprocmask is + // equivalent to pthread_sigmask with the exception that pthread_sigmask + // does not allow to block some signals used internally in pthread + // implementation. We are fine with blocking them here, we are really not + // going to pthread_cancel the thread. + // The tracer thread should not raise any synchronous signals. But in case it + // does, we setup a special handler for sync signals that properly kills the + // parent as well. Note: we don't pass CLONE_SIGHAND to clone, so handlers + // in the tracer thread won't interfere with user program. Double note: if a + // user does something along the lines of 'kill -11 pid', that can kill the + // process even if user setup own handler for SEGV. + // Thing to watch out for: this code should not change behavior of user code + // in any observable way. In particular it should not override user signal + // handlers. + internal_sigfillset(&blocked_sigset); + for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) + internal_sigdelset(&blocked_sigset, kSyncSignals[i]); + int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset); + CHECK_EQ(rv, 0); uptr tracer_pid = internal_clone( TracerThread, tracer_stack.Bottom(), CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED, &tracer_thread_argument, 0 /* parent_tidptr */, 0 /* newtls */, 0 /* child_tidptr */); + internal_sigprocmask(SIG_SETMASK, &old_sigset, 0); int local_errno = 0; if (internal_iserror(tracer_pid, &local_errno)) { VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno); @@ -459,4 +465,4 @@ uptr SuspendedThreadsList::RegisterCount() { } } // namespace __sanitizer -#endif // SANITIZER_LINUX && defined(__x86_64__) +#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc index 6b75036c7e50..2b697e955709 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc @@ -7,7 +7,7 @@ // //===----------------------------------------------------------------------===// // -// Suppression parsing/matching code shared between TSan and LSan. +// Suppression parsing/matching code. // //===----------------------------------------------------------------------===// @@ -21,97 +21,43 @@ namespace __sanitizer { -static const char *const kTypeStrings[SuppressionTypeCount] = { - "none", "race", "mutex", "thread", "signal", "leak", "called_from_lib", - "deadlock", "vptr_check", "interceptor_name", "interceptor_via_fun", - "interceptor_via_lib"}; - -bool TemplateMatch(char *templ, const char *str) { - if (str == 0 || str[0] == 0) - return false; - bool start = false; - if (templ && templ[0] == '^') { - start = true; - templ++; - } - bool asterisk = false; - while (templ && templ[0]) { - if (templ[0] == '*') { - templ++; - start = false; - asterisk = true; - continue; - } - if (templ[0] == '$') - return str[0] == 0 || asterisk; - if (str[0] == 0) - return false; - char *tpos = (char*)internal_strchr(templ, '*'); - char *tpos1 = (char*)internal_strchr(templ, '$'); - if (tpos == 0 || (tpos1 && tpos1 < tpos)) - tpos = tpos1; - if (tpos != 0) - tpos[0] = 0; - const char *str0 = str; - const char *spos = internal_strstr(str, templ); - str = spos + internal_strlen(templ); - templ = tpos; - if (tpos) - tpos[0] = tpos == tpos1 ? '$' : '*'; - if (spos == 0) - return false; - if (start && spos != str0) - return false; - start = false; - asterisk = false; - } - return true; +SuppressionContext::SuppressionContext(const char *suppression_types[], + int suppression_types_num) + : suppression_types_(suppression_types), + suppression_types_num_(suppression_types_num), suppressions_(1), + can_parse_(true) { + CHECK_LE(suppression_types_num_, kMaxSuppressionTypes); + internal_memset(has_suppression_type_, 0, suppression_types_num_); } -ALIGNED(64) static char placeholder[sizeof(SuppressionContext)]; -static SuppressionContext *suppression_ctx = 0; - -SuppressionContext::SuppressionContext() : suppressions_(1), can_parse_(true) { - internal_memset(has_suppresson_type_, 0, sizeof(has_suppresson_type_)); -} - -SuppressionContext *SuppressionContext::Get() { - CHECK(suppression_ctx); - return suppression_ctx; -} - -void SuppressionContext::InitIfNecessary() { - if (suppression_ctx) +void SuppressionContext::ParseFromFile(const char *filename) { + if (filename[0] == '\0') return; - suppression_ctx = new(placeholder) SuppressionContext; - if (common_flags()->suppressions[0] == '\0') - return; - char *suppressions_from_file; + char *file_contents; uptr buffer_size; - uptr contents_size = - ReadFileToBuffer(common_flags()->suppressions, &suppressions_from_file, - &buffer_size, 1 << 26 /* max_len */); + uptr contents_size = ReadFileToBuffer(filename, &file_contents, &buffer_size, + 1 << 26 /* max_len */); if (contents_size == 0) { Printf("%s: failed to read suppressions file '%s'\n", SanitizerToolName, - common_flags()->suppressions); + filename); Die(); } - suppression_ctx->Parse(suppressions_from_file); + Parse(file_contents); } -bool SuppressionContext::Match(const char *str, SuppressionType type, +bool SuppressionContext::Match(const char *str, const char *type, Suppression **s) { - if (!has_suppresson_type_[type]) - return false; can_parse_ = false; - uptr i; - for (i = 0; i < suppressions_.size(); i++) - if (type == suppressions_[i].type && - TemplateMatch(suppressions_[i].templ, str)) - break; - if (i == suppressions_.size()) return false; - *s = &suppressions_[i]; - return true; + if (!HasSuppressionType(type)) + return false; + for (uptr i = 0; i < suppressions_.size(); i++) { + Suppression &cur = suppressions_[i]; + if (0 == internal_strcmp(cur.type, type) && TemplateMatch(cur.templ, str)) { + *s = &cur; + return true; + } + } + return false; } static const char *StripPrefix(const char *str, const char *prefix) { @@ -139,26 +85,26 @@ void SuppressionContext::Parse(const char *str) { while (line != end2 && (end2[-1] == ' ' || end2[-1] == '\t')) end2--; int type; - for (type = 0; type < SuppressionTypeCount; type++) { - const char *next_char = StripPrefix(line, kTypeStrings[type]); + for (type = 0; type < suppression_types_num_; type++) { + const char *next_char = StripPrefix(line, suppression_types_[type]); if (next_char && *next_char == ':') { line = ++next_char; break; } } - if (type == SuppressionTypeCount) { + if (type == suppression_types_num_) { Printf("%s: failed to parse suppressions\n", SanitizerToolName); Die(); } Suppression s; - s.type = static_cast(type); + s.type = suppression_types_[type]; s.templ = (char*)InternalAlloc(end2 - line + 1); internal_memcpy(s.templ, line, end2 - line); s.templ[end2 - line] = 0; s.hit_count = 0; s.weight = 0; suppressions_.push_back(s); - has_suppresson_type_[s.type] = true; + has_suppression_type_[type] = true; } if (end[0] == 0) break; @@ -170,8 +116,12 @@ uptr SuppressionContext::SuppressionCount() const { return suppressions_.size(); } -bool SuppressionContext::HasSuppressionType(SuppressionType type) const { - return has_suppresson_type_[type]; +bool SuppressionContext::HasSuppressionType(const char *type) const { + for (int i = 0; i < suppression_types_num_; i++) { + if (0 == internal_strcmp(type, suppression_types_[i])) + return has_suppression_type_[i]; + } + return false; } const Suppression *SuppressionContext::SuppressionAt(uptr i) const { @@ -186,9 +136,4 @@ void SuppressionContext::GetMatched( matched->push_back(&suppressions_[i]); } -const char *SuppressionTypeString(SuppressionType t) { - CHECK(t < SuppressionTypeCount); - return kTypeStrings[t]; -} - } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.h index 453731456169..02dbf6f9690b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.h @@ -7,7 +7,7 @@ // //===----------------------------------------------------------------------===// // -// Suppression parsing/matching code shared between TSan and LSan. +// Suppression parsing/matching code. // //===----------------------------------------------------------------------===// #ifndef SANITIZER_SUPPRESSIONS_H @@ -18,24 +18,8 @@ namespace __sanitizer { -enum SuppressionType { - SuppressionNone, - SuppressionRace, - SuppressionMutex, - SuppressionThread, - SuppressionSignal, - SuppressionLeak, - SuppressionLib, - SuppressionDeadlock, - SuppressionVptrCheck, - SuppressionInterceptorName, - SuppressionInterceptorViaFunction, - SuppressionInterceptorViaLibrary, - SuppressionTypeCount -}; - struct Suppression { - SuppressionType type; + const char *type; char *templ; unsigned hit_count; uptr weight; @@ -43,33 +27,29 @@ struct Suppression { class SuppressionContext { public: + // Create new SuppressionContext capable of parsing given suppression types. + SuppressionContext(const char *supprression_types[], + int suppression_types_num); + + void ParseFromFile(const char *filename); void Parse(const char *str); - bool Match(const char* str, SuppressionType type, Suppression **s); + + bool Match(const char *str, const char *type, Suppression **s); uptr SuppressionCount() const; - bool HasSuppressionType(SuppressionType type) const; + bool HasSuppressionType(const char *type) const; const Suppression *SuppressionAt(uptr i) const; void GetMatched(InternalMmapVector *matched); - // Create a SuppressionContext singleton if it hasn't been created earlier. - // Not thread safe. Must be called early during initialization (but after - // runtime flags are parsed). - static void InitIfNecessary(); - // Returns a SuppressionContext singleton. - static SuppressionContext *Get(); - private: - SuppressionContext(); + static const int kMaxSuppressionTypes = 16; + const char **const suppression_types_; + const int suppression_types_num_; + InternalMmapVector suppressions_; - bool has_suppresson_type_[SuppressionTypeCount]; + bool has_suppression_type_[kMaxSuppressionTypes]; bool can_parse_; - - friend class SuppressionContextTest; }; -const char *SuppressionTypeString(SuppressionType t); - -bool TemplateMatch(char *templ, const char *str); - } // namespace __sanitizer #endif // SANITIZER_SUPPRESSIONS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc index bb8ba6b81674..9317a78eef13 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc @@ -173,9 +173,9 @@ SymbolizedStack *LibbacktraceSymbolizer::SymbolizeCode(uptr addr, return data.first; } -bool LibbacktraceSymbolizer::SymbolizeData(DataInfo *info) { - backtrace_syminfo((backtrace_state *)state_, info->address, - SymbolizeDataCallback, ErrorCallback, info); +bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { + backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeDataCallback, + ErrorCallback, info); return true; } @@ -192,7 +192,7 @@ SymbolizedStack *LibbacktraceSymbolizer::SymbolizeCode(uptr addr, return nullptr; } -bool LibbacktraceSymbolizer::SymbolizeData(DataInfo *info) { +bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { return false; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h index a335cb23788c..1ff005042a11 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h @@ -35,7 +35,7 @@ class LibbacktraceSymbolizer { SymbolizedStack *SymbolizeCode(uptr addr, const char *module_name, uptr module_offset); - bool SymbolizeData(DataInfo *info); + bool SymbolizeData(uptr addr, DataInfo *info); // May return NULL if demangling failed. static char *Demangle(const char *name, bool always_alloc = false); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index d46c249e6cab..69ac18e8426f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -596,7 +596,7 @@ class POSIXSymbolizer : public Symbolizer { // First, try to use libbacktrace symbolizer (if it's available). if (libbacktrace_symbolizer_ != 0) { mu_.CheckLocked(); - if (libbacktrace_symbolizer_->SymbolizeData(info)) + if (libbacktrace_symbolizer_->SymbolizeData(addr, info)) return true; } const char *str = SendCommand(true, module_name, module_offset); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cc index 6bb7d3805604..ed96a3a895a8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cc @@ -30,25 +30,7 @@ class WinSymbolizer : public Symbolizer { SymbolizedStack *frame = SymbolizedStack::New(addr); BlockingMutexLock l(&dbghelp_mu_); - if (!initialized_) { - if (!TrySymInitialize()) { - // OK, maybe the client app has called SymInitialize already. - // That's a bit unfortunate for us as all the DbgHelp functions are - // single-threaded and we can't coordinate with the app. - // FIXME: Can we stop the other threads at this point? - // Anyways, we have to reconfigure stuff to make sure that SymInitialize - // has all the appropriate options set. - // Cross our fingers and reinitialize DbgHelp. - Report("*** WARNING: Failed to initialize DbgHelp! ***\n"); - Report("*** Most likely this means that the app is already ***\n"); - Report("*** using DbgHelp, possibly with incompatible flags. ***\n"); - Report("*** Due to technical reasons, symbolization might crash ***\n"); - Report("*** or produce wrong results. ***\n"); - SymCleanup(GetCurrentProcess()); - TrySymInitialize(); - } - initialized_ = true; - } + InitializeIfNeeded(); // See http://msdn.microsoft.com/en-us/library/ms680578(VS.85).aspx char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)]; @@ -100,6 +82,58 @@ class WinSymbolizer : public Symbolizer { // FIXME: Implement GetModuleNameAndOffsetForPC(). private: + void InitializeIfNeeded() { + if (initialized_) + return; + if (!TrySymInitialize()) { + // OK, maybe the client app has called SymInitialize already. + // That's a bit unfortunate for us as all the DbgHelp functions are + // single-threaded and we can't coordinate with the app. + // FIXME: Can we stop the other threads at this point? + // Anyways, we have to reconfigure stuff to make sure that SymInitialize + // has all the appropriate options set. + // Cross our fingers and reinitialize DbgHelp. + Report("*** WARNING: Failed to initialize DbgHelp! ***\n"); + Report("*** Most likely this means that the app is already ***\n"); + Report("*** using DbgHelp, possibly with incompatible flags. ***\n"); + Report("*** Due to technical reasons, symbolization might crash ***\n"); + Report("*** or produce wrong results. ***\n"); + SymCleanup(GetCurrentProcess()); + TrySymInitialize(); + } + initialized_ = true; + + // When an executable is run from a location different from the one where it + // was originally built, we may not see the nearby PDB files. + // To work around this, let's append the directory of the main module + // to the symbol search path. All the failures below are not fatal. + const size_t kSymPathSize = 2048; + static wchar_t path_buffer[kSymPathSize + 1 + MAX_PATH]; + if (!SymGetSearchPathW(GetCurrentProcess(), path_buffer, kSymPathSize)) { + Report("*** WARNING: Failed to SymGetSearchPathW ***\n"); + return; + } + size_t sz = wcslen(path_buffer); + if (sz) { + CHECK_EQ(0, wcscat_s(path_buffer, L";")); + sz++; + } + DWORD res = GetModuleFileNameW(NULL, path_buffer + sz, MAX_PATH); + if (res == 0 || res == MAX_PATH) { + Report("*** WARNING: Failed to getting the EXE directory ***\n"); + return; + } + // Write the zero character in place of the last backslash to get the + // directory of the main module at the end of path_buffer. + wchar_t *last_bslash = wcsrchr(path_buffer + sz, L'\\'); + CHECK_NE(last_bslash, 0); + *last_bslash = L'\0'; + if (!SymSetSearchPathW(GetCurrentProcess(), path_buffer)) { + Report("*** WARNING: Failed to SymSetSearchPathW\n"); + return; + } + } + bool TrySymInitialize() { SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_UNDNAME | SYMOPT_LOAD_LINES); return SymInitialize(GetCurrentProcess(), 0, TRUE); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc index 88d237f4e3ce..15cf05f06087 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -11,13 +11,13 @@ // //===----------------------------------------------------------------------===// -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC # define SYSCALL(name) SYS_ ## name #else # define SYSCALL(name) __NR_ ## name #endif -#if SANITIZER_FREEBSD && defined(__x86_64__) +#if (SANITIZER_FREEBSD || SANITIZER_MAC) && defined(__x86_64__) # define internal_syscall __syscall # else # define internal_syscall syscall diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_posix_libcdep.cc index a98e61771c02..7ab2efbd75bd 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_posix_libcdep.cc @@ -96,7 +96,7 @@ uptr Unwind_GetIP(struct _Unwind_Context *ctx) { struct UnwindTraceArg { BufferedStackTrace *stack; - uptr max_depth; + u32 max_depth; }; _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { @@ -108,7 +108,7 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { return UNWIND_CONTINUE; } -void BufferedStackTrace::SlowUnwindStack(uptr pc, uptr max_depth) { +void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); size = 0; UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)}; @@ -128,7 +128,7 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, uptr max_depth) { } void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - uptr max_depth) { + u32 max_depth) { CHECK_GE(max_depth, 2); if (!unwind_backtrace_signal_arch) { SlowUnwindStack(pc, max_depth); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc index 3e9014199651..335cecabe118 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "sanitizer_common.h" @@ -122,18 +123,34 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) { } void *Mprotect(uptr fixed_addr, uptr size) { - return VirtualAlloc((LPVOID)fixed_addr, size, - MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS); + void *res = VirtualAlloc((LPVOID)fixed_addr, size, + MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS); + if (res == 0) + Report("WARNING: %s failed to " + "mprotect %p (%zd) bytes at %p (error code: %d)\n", + SanitizerToolName, size, size, fixed_addr, GetLastError()); + return res; } void FlushUnneededShadowMemory(uptr addr, uptr size) { // This is almost useless on 32-bits. - // FIXME: add madvice-analog when we move to 64-bits. + // FIXME: add madvise-analog when we move to 64-bits. +} + +void NoHugePagesInRegion(uptr addr, uptr size) { + // FIXME: probably similar to FlushUnneededShadowMemory. +} + +void DontDumpShadowMemory(uptr addr, uptr length) { + // This is almost useless on 32-bits. + // FIXME: add madvise-analog when we move to 64-bits. } bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { - // FIXME: shall we do anything here on Windows? - return true; + MEMORY_BASIC_INFORMATION mbi; + CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi))); + return mbi.Protect == PAGE_NOACCESS && + (uptr)mbi.BaseAddress + mbi.RegionSize >= range_end; } void *MapFileToMemory(const char *file_name, uptr *buff_size) { @@ -187,9 +204,80 @@ u32 GetUid() { UNIMPLEMENTED(); } -void DumpProcessMap() { - UNIMPLEMENTED(); +namespace { +struct ModuleInfo { + HMODULE handle; + uptr base_address; + uptr end_address; +}; + +int CompareModulesBase(const void *pl, const void *pr) { + const ModuleInfo &l = *(ModuleInfo *)pl, &r = *(ModuleInfo *)pr; + if (l.base_address < r.base_address) + return -1; + return l.base_address > r.base_address; } +} // namespace + +#ifndef SANITIZER_GO +void DumpProcessMap() { + Report("Dumping process modules:\n"); + HANDLE cur_process = GetCurrentProcess(); + + // Query the list of modules. Start by assuming there are no more than 256 + // modules and retry if that's not sufficient. + ModuleInfo *modules; + size_t num_modules; + { + HMODULE *hmodules = 0; + uptr modules_buffer_size = sizeof(HMODULE) * 256; + DWORD bytes_required; + while (!hmodules) { + hmodules = (HMODULE *)MmapOrDie(modules_buffer_size, __FUNCTION__); + CHECK(EnumProcessModules(cur_process, hmodules, modules_buffer_size, + &bytes_required)); + if (bytes_required > modules_buffer_size) { + // Either there turned out to be more than 256 hmodules, or new hmodules + // could have loaded since the last try. Retry. + UnmapOrDie(hmodules, modules_buffer_size); + hmodules = 0; + modules_buffer_size = bytes_required; + } + } + + num_modules = bytes_required / sizeof(HMODULE); + modules = + (ModuleInfo *)MmapOrDie(num_modules * sizeof(ModuleInfo), __FUNCTION__); + for (size_t i = 0; i < num_modules; ++i) { + modules[i].handle = hmodules[i]; + MODULEINFO mi; + if (!GetModuleInformation(cur_process, hmodules[i], &mi, sizeof(mi))) + continue; + modules[i].base_address = (uptr)mi.lpBaseOfDll; + modules[i].end_address = (uptr)mi.lpBaseOfDll + mi.SizeOfImage; + } + UnmapOrDie(hmodules, modules_buffer_size); + } + + qsort(modules, num_modules, sizeof(ModuleInfo), CompareModulesBase); + + for (size_t i = 0; i < num_modules; ++i) { + const ModuleInfo &mi = modules[i]; + char module_name[MAX_PATH]; + bool got_module_name = GetModuleFileNameA( + mi.handle, module_name, sizeof(module_name)); + if (mi.end_address != 0) { + Printf("\t%p-%p %s\n", mi.base_address, mi.end_address, + got_module_name ? module_name : "[no name]"); + } else if (got_module_name) { + Printf("\t??\?-??? %s\n", module_name); + } else { + Printf("\t???\n"); + } + } + UnmapOrDie(modules, num_modules * sizeof(ModuleInfo)); +} +#endif void DisableCoreDumperIfNecessary() { // Do nothing. @@ -238,8 +326,9 @@ u64 NanoTime() { } void Abort() { - abort(); - internal__exit(-1); // abort is not NORETURN on Windows. + if (::IsDebuggerPresent()) + __debugbreak(); + internal__exit(3); } uptr GetListOfModules(LoadedModule *modules, uptr max_modules, @@ -379,6 +468,9 @@ uptr GetRSS() { return 0; } +void *internal_start_thread(void (*func)(void *arg), void *arg) { return 0; } +void internal_join_thread(void *th) { } + // ---------------------- BlockingMutex ---------------- {{{1 const uptr LOCK_UNINITIALIZED = 0; const uptr LOCK_READY = (uptr)-1; @@ -448,7 +540,7 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, } #if !SANITIZER_GO -void BufferedStackTrace::SlowUnwindStack(uptr pc, uptr max_depth) { +void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { CHECK_GE(max_depth, 2); // FIXME: CaptureStackBackTrace might be too slow for us. // FIXME: Compare with StackWalk64. @@ -464,7 +556,7 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, uptr max_depth) { } void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, - uptr max_depth) { + u32 max_depth) { CONTEXT ctx = *(CONTEXT *)context; STACKFRAME64 stack_frame; memset(&stack_frame, 0, sizeof(stack_frame)); diff --git a/contrib/compiler-rt/lib/sanitizer_common/scripts/check_lint.sh b/contrib/compiler-rt/lib/sanitizer_common/scripts/check_lint.sh index 267273d97794..7ed05d73563a 100755 --- a/contrib/compiler-rt/lib/sanitizer_common/scripts/check_lint.sh +++ b/contrib/compiler-rt/lib/sanitizer_common/scripts/check_lint.sh @@ -32,7 +32,14 @@ LSAN_LIT_TEST_LINT_FILTER=${LSAN_RTL_LINT_FILTER},-whitespace/line_length DFSAN_RTL_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int,-runtime/printf,-runtime/references,-readability/function COMMON_RTL_INC_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int,-runtime/sizeof,-runtime/printf,-readability/fn_size SANITIZER_INCLUDES_LINT_FILTER=${COMMON_LINT_FILTER},-runtime/int -MKTEMP="mktemp -q /tmp/tmp.XXXXXXXXXX" + +MKTEMP_DIR=$(mktemp -qd /tmp/check_lint.XXXXXXXXXX) +MKTEMP="mktemp -q ${MKTEMP_DIR}/tmp.XXXXXXXXXX" +cleanup() { + rm -rf $MKTEMP_DIR +} +trap cleanup EXIT + cd ${LLVM_CHECKOUT} EXITSTATUS=0 diff --git a/contrib/compiler-rt/lib/sanitizer_common/scripts/sancov.py b/contrib/compiler-rt/lib/sanitizer_common/scripts/sancov.py index 476953015280..566116eb2334 100755 --- a/contrib/compiler-rt/lib/sanitizer_common/scripts/sancov.py +++ b/contrib/compiler-rt/lib/sanitizer_common/scripts/sancov.py @@ -118,7 +118,7 @@ def UnpackOneRawFile(path, map_path): if len(pc_list) == 0: continue assert path.endswith('.sancov.raw') dst_path = module_path + '.' + os.path.basename(path)[:-4] - print "writing %d PCs to %s" % (len(pc_list), dst_path) + print >> sys.stderr, "%s: writing %d PCs to %s" % (prog_name, len(pc_list), dst_path) arr = array.array('I') arr.fromlist(sorted(pc_list)) with open(dst_path, 'ab') as f2: diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc index f61d58dea7d9..be8fc91aa861 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc @@ -14,7 +14,6 @@ #include "sanitizer_common/sanitizer_allocator.h" #include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_test_utils.h" #include "sanitizer_pthread_wrappers.h" @@ -27,9 +26,9 @@ #include // Too slow for debug build -#if TSAN_DEBUG == 0 +#if !SANITIZER_DEBUG -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 static const uptr kAllocatorSpace = 0x700000000000ULL; static const uptr kAllocatorSize = 0x010000000000ULL; // 1T. static const u64 kAddressSpaceSize = 1ULL << 47; @@ -39,6 +38,8 @@ typedef SizeClassAllocator64< typedef SizeClassAllocator64< kAllocatorSpace, kAllocatorSize, 16, CompactSizeClassMap> Allocator64Compact; +#elif defined(__mips64) +static const u64 kAddressSpaceSize = 1ULL << 40; #else static const u64 kAddressSpaceSize = 1ULL << 32; #endif @@ -140,7 +141,7 @@ void TestSizeClassAllocator() { delete a; } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64) { TestSizeClassAllocator(); } @@ -184,7 +185,7 @@ void SizeClassAllocatorMetadataStress() { delete a; } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64MetadataStress) { SizeClassAllocatorMetadataStress(); } @@ -192,7 +193,7 @@ TEST(SanitizerCommon, SizeClassAllocator64MetadataStress) { TEST(SanitizerCommon, SizeClassAllocator64CompactMetadataStress) { SizeClassAllocatorMetadataStress(); } -#endif // SANITIZER_WORDSIZE == 64 +#endif // SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator32CompactMetadataStress) { SizeClassAllocatorMetadataStress(); } @@ -221,7 +222,7 @@ void SizeClassAllocatorGetBlockBeginStress() { delete a; } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64GetBlockBegin) { SizeClassAllocatorGetBlockBeginStress(); } @@ -231,7 +232,7 @@ TEST(SanitizerCommon, SizeClassAllocator64CompactGetBlockBegin) { TEST(SanitizerCommon, SizeClassAllocator32CompactGetBlockBegin) { SizeClassAllocatorGetBlockBeginStress(); } -#endif // SANITIZER_WORDSIZE == 64 +#endif // SANITIZER_CAN_USE_ALLOCATOR64 struct TestMapUnmapCallback { static int map_count, unmap_count; @@ -241,7 +242,7 @@ struct TestMapUnmapCallback { int TestMapUnmapCallback::map_count; int TestMapUnmapCallback::unmap_count; -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64MapUnmapCallback) { TestMapUnmapCallback::map_count = 0; TestMapUnmapCallback::unmap_count = 0; @@ -297,7 +298,7 @@ TEST(SanitizerCommon, LargeMmapAllocatorMapUnmapCallback) { TestMapUnmapCallback::map_count = 0; TestMapUnmapCallback::unmap_count = 0; LargeMmapAllocator a; - a.Init(); + a.Init(/* may_return_null */ false); AllocatorStats stats; stats.Init(); void *x = a.Allocate(&stats, 1 << 20, 1); @@ -322,7 +323,7 @@ void FailInAssertionOnOOM() { a.TestOnlyUnmap(); } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64Overflow) { EXPECT_DEATH(FailInAssertionOnOOM(), "Out of memory"); } @@ -331,7 +332,7 @@ TEST(SanitizerCommon, SizeClassAllocator64Overflow) { #if !defined(_WIN32) // FIXME: This currently fails on Windows. TEST(SanitizerCommon, LargeMmapAllocator) { LargeMmapAllocator<> a; - a.Init(); + a.Init(/* may_return_null */ false); AllocatorStats stats; stats.Init(); @@ -413,25 +414,22 @@ void TestCombinedAllocator() { CombinedAllocator Allocator; Allocator *a = new Allocator; - a->Init(); + a->Init(/* may_return_null */ true); AllocatorCache cache; memset(&cache, 0, sizeof(cache)); a->InitCache(&cache); - bool allocator_may_return_null = common_flags()->allocator_may_return_null; - common_flags()->allocator_may_return_null = true; EXPECT_EQ(a->Allocate(&cache, -1, 1), (void*)0); EXPECT_EQ(a->Allocate(&cache, -1, 1024), (void*)0); EXPECT_EQ(a->Allocate(&cache, (uptr)-1 - 1024, 1), (void*)0); EXPECT_EQ(a->Allocate(&cache, (uptr)-1 - 1024, 1024), (void*)0); EXPECT_EQ(a->Allocate(&cache, (uptr)-1 - 1023, 1024), (void*)0); - common_flags()->allocator_may_return_null = false; + // Set to false + a->SetMayReturnNull(false); EXPECT_DEATH(a->Allocate(&cache, -1, 1), "allocator is terminating the process"); - // Restore the original value. - common_flags()->allocator_may_return_null = allocator_may_return_null; const uptr kNumAllocs = 100000; const uptr kNumIter = 10; @@ -465,7 +463,7 @@ void TestCombinedAllocator() { a->TestOnlyUnmap(); } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, CombinedAllocator64) { TestCombinedAllocator, @@ -521,7 +519,7 @@ void TestSizeClassAllocatorLocalCache() { delete a; } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64LocalCache) { TestSizeClassAllocatorLocalCache< SizeClassAllocatorLocalCache >(); @@ -538,7 +536,7 @@ TEST(SanitizerCommon, SizeClassAllocator32CompactLocalCache) { SizeClassAllocatorLocalCache >(); } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 typedef SizeClassAllocatorLocalCache AllocatorCache; static AllocatorCache static_allocator_cache; @@ -694,7 +692,7 @@ void TestSizeClassAllocatorIteration() { delete a; } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 TEST(SanitizerCommon, SizeClassAllocator64Iteration) { TestSizeClassAllocatorIteration(); } @@ -706,7 +704,7 @@ TEST(SanitizerCommon, SizeClassAllocator32Iteration) { TEST(SanitizerCommon, LargeMmapAllocatorIteration) { LargeMmapAllocator<> a; - a.Init(); + a.Init(/* may_return_null */ false); AllocatorStats stats; stats.Init(); @@ -733,7 +731,7 @@ TEST(SanitizerCommon, LargeMmapAllocatorIteration) { TEST(SanitizerCommon, LargeMmapAllocatorBlockBegin) { LargeMmapAllocator<> a; - a.Init(); + a.Init(/* may_return_null */ false); AllocatorStats stats; stats.Init(); @@ -769,7 +767,7 @@ TEST(SanitizerCommon, LargeMmapAllocatorBlockBegin) { } -#if SANITIZER_WORDSIZE == 64 +#if SANITIZER_CAN_USE_ALLOCATOR64 // Regression test for out-of-memory condition in PopulateFreeList(). TEST(SanitizerCommon, SizeClassAllocator64PopulateFreeListOOM) { // In a world where regions are small and chunks are huge... @@ -859,4 +857,4 @@ TEST(SanitizerCommon, ThreadedTwoLevelByteMap) { EXPECT_EQ((uptr)TestMapUnmapCallback::unmap_count, m.size1()); } -#endif // #if TSAN_DEBUG==0 +#endif // #if !SANITIZER_DEBUG diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc index 8c8363353507..7835eef76d06 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_deadlock_detector_test.cc @@ -268,7 +268,7 @@ void RunMultipleEpochsTest() { } EXPECT_EQ(d.testOnlyGetEpoch(), 4 * d.size()); -#if TSAN_DEBUG == 0 +#if !SANITIZER_DEBUG // EXPECT_DEATH clones a thread with 4K stack, // which is overflown by tsan memory accesses functions in debug mode. diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_flags_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_flags_test.cc index 1055f5d24d6b..3e5d8381ed3a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_flags_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_flags_test.cc @@ -12,7 +12,9 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "gtest/gtest.h" #include @@ -20,58 +22,79 @@ namespace __sanitizer { static const char kFlagName[] = "flag_name"; +static const char kFlagDesc[] = "flag description"; template static void TestFlag(T start_value, const char *env, T final_value) { T flag = start_value; - ParseFlag(env, &flag, kFlagName, "flag description"); + + FlagParser parser; + RegisterFlag(&parser, kFlagName, kFlagDesc, &flag); + + parser.ParseString(env); + EXPECT_EQ(final_value, flag); } -static void TestStrFlag(const char *start_value, const char *env, - const char *final_value) { +template <> +void TestFlag(const char *start_value, const char *env, + const char *final_value) { const char *flag = start_value; - ParseFlag(env, &flag, kFlagName, "flag description"); + + FlagParser parser; + RegisterFlag(&parser, kFlagName, kFlagDesc, &flag); + + parser.ParseString(env); + EXPECT_EQ(0, internal_strcmp(final_value, flag)); } TEST(SanitizerCommon, BooleanFlags) { - TestFlag(true, "--flag_name", true); - TestFlag(false, "flag_name", false); - TestFlag(false, "--flag_name=1", true); - TestFlag(true, "asdas flag_name=0 asdas", false); - TestFlag(true, " --flag_name=0 ", false); + TestFlag(false, "flag_name=1", true); TestFlag(false, "flag_name=yes", true); TestFlag(false, "flag_name=true", true); + TestFlag(true, "flag_name=0", false); TestFlag(true, "flag_name=no", false); TestFlag(true, "flag_name=false", false); } TEST(SanitizerCommon, IntFlags) { TestFlag(-11, 0, -11); - TestFlag(-11, "flag_name", -11); - TestFlag(-11, "--flag_name=", 0); - TestFlag(-11, "--flag_name=42", 42); - TestFlag(-11, "--flag_name=-42", -42); + TestFlag(-11, "flag_name=0", 0); + TestFlag(-11, "flag_name=42", 42); + TestFlag(-11, "flag_name=-42", -42); + + // Unrecognized flags are ignored. + TestFlag(-11, "--flag_name=42", -11); + TestFlag(-11, "zzzzzzz=42", -11); + + EXPECT_DEATH(TestFlag(-11, "flag_name", 0), "expected '='"); + EXPECT_DEATH(TestFlag(-11, "flag_name=42U", 0), + "Invalid value for int option"); } TEST(SanitizerCommon, StrFlags) { - TestStrFlag("zzz", 0, "zzz"); - TestStrFlag("zzz", "flag_name", "zzz"); - TestStrFlag("zzz", "--flag_name=", ""); - TestStrFlag("", "--flag_name=abc", "abc"); - TestStrFlag("", "--flag_name='abc zxc'", "abc zxc"); - TestStrFlag("", "--flag_name='abc zxcc'", "abc zxcc"); - TestStrFlag("", "--flag_name=\"abc qwe\" asd", "abc qwe"); - TestStrFlag("", "other_flag_name=zzz", ""); + TestFlag("zzz", 0, "zzz"); + TestFlag("zzz", "flag_name=", ""); + TestFlag("zzz", "flag_name=abc", "abc"); + TestFlag("", "flag_name=abc", "abc"); + TestFlag("", "flag_name='abc zxc'", "abc zxc"); + // TestStrFlag("", "flag_name=\"abc qwe\" asd", "abc qwe"); } static void TestTwoFlags(const char *env, bool expected_flag1, - const char *expected_flag2) { + const char *expected_flag2, + const char *name1 = "flag1", + const char *name2 = "flag2") { bool flag1 = !expected_flag1; const char *flag2 = ""; - ParseFlag(env, &flag1, "flag1", "flag1 description"); - ParseFlag(env, &flag2, "flag2", "flag2 description"); + + FlagParser parser; + RegisterFlag(&parser, name1, kFlagDesc, &flag1); + RegisterFlag(&parser, name2, kFlagDesc, &flag2); + + parser.ParseString(env); + EXPECT_EQ(expected_flag1, flag1); EXPECT_EQ(0, internal_strcmp(flag2, expected_flag2)); } @@ -81,6 +104,39 @@ TEST(SanitizerCommon, MultipleFlags) { TestTwoFlags("flag2='qxx' flag1=0", false, "qxx"); TestTwoFlags("flag1=false:flag2='zzz'", false, "zzz"); TestTwoFlags("flag2=qxx:flag1=yes", true, "qxx"); + TestTwoFlags("flag2=qxx\nflag1=yes", true, "qxx"); + TestTwoFlags("flag2=qxx\r\nflag1=yes", true, "qxx"); + TestTwoFlags("flag2=qxx\tflag1=yes", true, "qxx"); +} + +TEST(SanitizerCommon, CommonSuffixFlags) { + TestTwoFlags("flag=1 other_flag='zzz'", true, "zzz", "flag", "other_flag"); + TestTwoFlags("other_flag='zzz' flag=1", true, "zzz", "flag", "other_flag"); + TestTwoFlags("other_flag=' flag=0 ' flag=1", true, " flag=0 ", "flag", + "other_flag"); + TestTwoFlags("flag=1 other_flag=' flag=0 '", true, " flag=0 ", "flag", + "other_flag"); +} + +TEST(SanitizerCommon, CommonFlags) { + CommonFlags cf; + FlagParser parser; + RegisterCommonFlags(&parser, &cf); + + cf.SetDefaults(); + EXPECT_TRUE(cf.symbolize); + EXPECT_STREQ(".", cf.coverage_dir); + + cf.symbolize = false; + cf.coverage = true; + cf.coverage_direct = true; + cf.log_path = "path/one"; + + parser.ParseString("symbolize=1:coverage_direct=false log_path='path/two'"); + EXPECT_TRUE(cf.symbolize); + EXPECT_TRUE(cf.coverage); + EXPECT_FALSE(cf.coverage_direct); + EXPECT_STREQ("path/two", cf.log_path); } } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cc index 660710d5bb7e..8712d2c1b2a5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_libc_test.cc @@ -55,6 +55,19 @@ struct stat_and_more { unsigned char z; }; +static void temp_file_name(char *buf, size_t bufsize, const char *prefix) { + const char *tmpdir = "/tmp"; +#if SANITIZER_ANDROID + // I don't know a way to query temp directory location on Android without + // going through Java interfaces. The code below is not ideal, but should + // work. May require "adb root", but it is needed for almost any use of ASan + // on Android already. + tmpdir = GetEnv("EXTERNAL_STORAGE"); +#endif + u32 uid = GetUid(); + internal_snprintf(buf, bufsize, "%s/%s%d", tmpdir, prefix, uid); +} + // FIXME: File manipulations are not yet supported on Windows #if !defined(_WIN32) TEST(SanitizerCommon, FileOps) { @@ -63,28 +76,16 @@ TEST(SanitizerCommon, FileOps) { const char *str2 = "zxcv"; uptr len2 = internal_strlen(str2); - u32 uid = GetUid(); - char temp_filename[128]; -#if SANITIZER_ANDROID - // I don't know a way to query temp directory location on Android without - // going through Java interfaces. The code below is not ideal, but should - // work. May require "adb root", but it is needed for almost any use of ASan - // on Android already. - internal_snprintf(temp_filename, sizeof(temp_filename), - "%s/sanitizer_common.tmp.%d", - GetEnv("EXTERNAL_STORAGE"), uid); -#else - internal_snprintf(temp_filename, sizeof(temp_filename), - "/tmp/sanitizer_common.tmp.%d", uid); -#endif - uptr openrv = OpenFile(temp_filename, true); + char tmpfile[128]; + temp_file_name(tmpfile, sizeof(tmpfile), "sanitizer_common.fileops.tmp."); + uptr openrv = OpenFile(tmpfile, true); EXPECT_FALSE(internal_iserror(openrv)); fd_t fd = openrv; EXPECT_EQ(len1, internal_write(fd, str1, len1)); EXPECT_EQ(len2, internal_write(fd, str2, len2)); internal_close(fd); - openrv = OpenFile(temp_filename, false); + openrv = OpenFile(tmpfile, false); EXPECT_FALSE(internal_iserror(openrv)); fd = openrv; uptr fsize = internal_filesize(fd); @@ -92,8 +93,8 @@ TEST(SanitizerCommon, FileOps) { #if SANITIZER_TEST_HAS_STAT_H struct stat st1, st2, st3; - EXPECT_EQ(0u, internal_stat(temp_filename, &st1)); - EXPECT_EQ(0u, internal_lstat(temp_filename, &st2)); + EXPECT_EQ(0u, internal_stat(tmpfile, &st1)); + EXPECT_EQ(0u, internal_lstat(tmpfile, &st2)); EXPECT_EQ(0u, internal_fstat(fd, &st3)); EXPECT_EQ(fsize, (uptr)st3.st_size); @@ -115,6 +116,7 @@ TEST(SanitizerCommon, FileOps) { EXPECT_EQ(len2, internal_read(fd, buf, len2)); EXPECT_EQ(0, internal_memcmp(buf, str2, len2)); internal_close(fd); + internal_unlink(tmpfile); } #endif @@ -125,3 +127,35 @@ TEST(SanitizerCommon, InternalStrFunctions) { EXPECT_EQ(0, internal_strchr(haystack, 'z')); EXPECT_EQ(haystack + 8, internal_strchrnul(haystack, 'z')); } + +// FIXME: File manipulations are not yet supported on Windows +#if !defined(_WIN32) && !SANITIZER_MAC +TEST(SanitizerCommon, InternalMmapWithOffset) { + char tmpfile[128]; + temp_file_name(tmpfile, sizeof(tmpfile), + "sanitizer_common.internalmmapwithoffset.tmp."); + uptr res = OpenFile(tmpfile, true); + ASSERT_FALSE(internal_iserror(res)); + fd_t fd = res; + + uptr page_size = GetPageSizeCached(); + res = internal_ftruncate(fd, page_size * 2); + ASSERT_FALSE(internal_iserror(res)); + + res = internal_lseek(fd, page_size, SEEK_SET); + ASSERT_FALSE(internal_iserror(res)); + + res = internal_write(fd, "AB", 2); + ASSERT_FALSE(internal_iserror(res)); + + char *p = (char *)MapWritableFileToMemory(nullptr, page_size, fd, page_size); + ASSERT_NE(nullptr, p); + + ASSERT_EQ('A', p[0]); + ASSERT_EQ('B', p[1]); + + internal_close(fd); + internal_munmap(p, page_size); + internal_unlink(tmpfile); +} +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cc index 592d9c3eeaf5..11342b775cc7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cc @@ -255,6 +255,14 @@ TEST(SanitizerCommon, LibraryNameIs) { } } +#if defined(__mips64) +// Effectively, this is a test for ThreadDescriptorSize() which is used to +// compute ThreadSelf(). +TEST(SanitizerLinux, ThreadSelfTest) { + ASSERT_EQ(pthread_self(), ThreadSelf()); +} +#endif + } // namespace __sanitizer #endif // SANITIZER_LINUX diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_printf_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_printf_test.cc index d0b46ac94ff2..5e39e0a591d6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_printf_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_printf_test.cc @@ -28,14 +28,11 @@ TEST(Printf, Basic) { (unsigned)10, (unsigned long)11, // NOLINT (void*)0x123, "_string_"); EXPECT_EQ(len, strlen(buf)); - void *ptr; - if (sizeof(ptr) == 4) { - EXPECT_STREQ("a-1b-2c4294967292e5fahbq" - "0x00000123e_string_r", buf); - } else { - EXPECT_STREQ("a-1b-2c4294967292e5fahbq" - "0x000000000123e_string_r", buf); - } + + std::string expectedString = "a-1b-2c4294967292e5fahbq0x"; + expectedString += std::string(SANITIZER_POINTER_FORMAT_LENGTH - 3, '0'); + expectedString += "123e_string_r"; + EXPECT_STREQ(expectedString.c_str(), buf); } TEST(Printf, OverflowStr) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc index 495b726dcc45..abe4ef43093f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_test.cc @@ -48,6 +48,7 @@ TEST(MemoryMappingLayout, DumpListOfModules) { if (strstr(modules[i].full_name(), binary_name) != 0) found = true; } + modules[i].clear(); } EXPECT_TRUE(found); free(modules); diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc index 0699243283df..e8c30d07e78c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc @@ -58,117 +58,77 @@ TEST(Suppressions, Match) { EXPECT_FALSE(MyMatch("foo$^bar", "foobar")); } -TEST(Suppressions, TypeStrings) { - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionNone), "none")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionRace), "race")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionMutex), "mutex")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionThread), "thread")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionSignal), "signal")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionLeak), "leak")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionLib), - "called_from_lib")); - CHECK( - !internal_strcmp(SuppressionTypeString(SuppressionDeadlock), "deadlock")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionVptrCheck), - "vptr_check")); - CHECK(!internal_strcmp(SuppressionTypeString(SuppressionInterceptorName), - "interceptor_name")); - CHECK( - !internal_strcmp(SuppressionTypeString(SuppressionInterceptorViaFunction), - "interceptor_via_fun")); - CHECK( - !internal_strcmp(SuppressionTypeString(SuppressionInterceptorViaLibrary), - "interceptor_via_lib")); - // Ensure this test is up-to-date when suppression types are added. - CHECK_EQ(12, SuppressionTypeCount); -} +static const char *kTestSuppressionTypes[] = {"race", "thread", "mutex", + "signal"}; class SuppressionContextTest : public ::testing::Test { public: - virtual void SetUp() { ctx_ = new(placeholder_) SuppressionContext; } - virtual void TearDown() { ctx_->~SuppressionContext(); } + SuppressionContextTest() + : ctx_(kTestSuppressionTypes, ARRAY_SIZE(kTestSuppressionTypes)) {} protected: - InternalMmapVector *Suppressions() { - return &ctx_->suppressions_; + SuppressionContext ctx_; + + void CheckSuppressions(unsigned count, std::vector types, + std::vector templs) const { + EXPECT_EQ(count, ctx_.SuppressionCount()); + for (unsigned i = 0; i < count; i++) { + const Suppression *s = ctx_.SuppressionAt(i); + EXPECT_STREQ(types[i], s->type); + EXPECT_STREQ(templs[i], s->templ); + } } - SuppressionContext *ctx_; - ALIGNED(64) char placeholder_[sizeof(SuppressionContext)]; }; TEST_F(SuppressionContextTest, Parse) { - ctx_->Parse( - "race:foo\n" - " race:bar\n" // NOLINT - "race:baz \n" // NOLINT - "# a comment\n" - "race:quz\n" - ); // NOLINT - EXPECT_EQ((unsigned)4, ctx_->SuppressionCount()); - EXPECT_EQ((*Suppressions())[3].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[3].templ, "quz")); - EXPECT_EQ((*Suppressions())[2].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[2].templ, "baz")); - EXPECT_EQ((*Suppressions())[1].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[1].templ, "bar")); - EXPECT_EQ((*Suppressions())[0].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[0].templ, "foo")); + ctx_.Parse("race:foo\n" + " race:bar\n" // NOLINT + "race:baz \n" // NOLINT + "# a comment\n" + "race:quz\n"); // NOLINT + CheckSuppressions(4, {"race", "race", "race", "race"}, + {"foo", "bar", "baz", "quz"}); } TEST_F(SuppressionContextTest, Parse2) { - ctx_->Parse( + ctx_.Parse( " # first line comment\n" // NOLINT " race:bar \n" // NOLINT "race:baz* *baz\n" "# a comment\n" "# last line comment\n" ); // NOLINT - EXPECT_EQ((unsigned)2, ctx_->SuppressionCount()); - EXPECT_EQ((*Suppressions())[1].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[1].templ, "baz* *baz")); - EXPECT_EQ((*Suppressions())[0].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[0].templ, "bar")); + CheckSuppressions(2, {"race", "race"}, {"bar", "baz* *baz"}); } TEST_F(SuppressionContextTest, Parse3) { - ctx_->Parse( + ctx_.Parse( "# last suppression w/o line-feed\n" "race:foo\n" "race:bar" ); // NOLINT - EXPECT_EQ((unsigned)2, ctx_->SuppressionCount()); - EXPECT_EQ((*Suppressions())[1].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[1].templ, "bar")); - EXPECT_EQ((*Suppressions())[0].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[0].templ, "foo")); + CheckSuppressions(2, {"race", "race"}, {"foo", "bar"}); } TEST_F(SuppressionContextTest, ParseType) { - ctx_->Parse( + ctx_.Parse( "race:foo\n" "thread:bar\n" "mutex:baz\n" "signal:quz\n" ); // NOLINT - EXPECT_EQ((unsigned)4, ctx_->SuppressionCount()); - EXPECT_EQ((*Suppressions())[3].type, SuppressionSignal); - EXPECT_EQ(0, strcmp((*Suppressions())[3].templ, "quz")); - EXPECT_EQ((*Suppressions())[2].type, SuppressionMutex); - EXPECT_EQ(0, strcmp((*Suppressions())[2].templ, "baz")); - EXPECT_EQ((*Suppressions())[1].type, SuppressionThread); - EXPECT_EQ(0, strcmp((*Suppressions())[1].templ, "bar")); - EXPECT_EQ((*Suppressions())[0].type, SuppressionRace); - EXPECT_EQ(0, strcmp((*Suppressions())[0].templ, "foo")); + CheckSuppressions(4, {"race", "thread", "mutex", "signal"}, + {"foo", "bar", "baz", "quz"}); } TEST_F(SuppressionContextTest, HasSuppressionType) { - ctx_->Parse( + ctx_.Parse( "race:foo\n" "thread:bar\n"); - EXPECT_TRUE(ctx_->HasSuppressionType(SuppressionRace)); - EXPECT_TRUE(ctx_->HasSuppressionType(SuppressionThread)); - EXPECT_FALSE(ctx_->HasSuppressionType(SuppressionMutex)); - EXPECT_FALSE(ctx_->HasSuppressionType(SuppressionSignal)); + EXPECT_TRUE(ctx_.HasSuppressionType("race")); + EXPECT_TRUE(ctx_.HasSuppressionType("thread")); + EXPECT_FALSE(ctx_.HasSuppressionType("mutex")); + EXPECT_FALSE(ctx_.HasSuppressionType("signal")); } } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_test_utils.h b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_test_utils.h index 64db37f341d3..9c162a66f547 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_test_utils.h +++ b/contrib/compiler-rt/lib/sanitizer_common/tests/sanitizer_test_utils.h @@ -118,4 +118,10 @@ static inline uint32_t my_rand() { # define SANITIZER_TEST_HAS_STRNLEN 0 #endif +#if defined(__FreeBSD__) +# define SANITIZER_TEST_HAS_PRINTF_L 1 +#else +# define SANITIZER_TEST_HAS_PRINTF_L 0 +#endif + #endif // SANITIZER_TEST_UTILS_H diff --git a/contrib/compiler-rt/lib/tsan/Makefile.old b/contrib/compiler-rt/lib/tsan/Makefile.old deleted file mode 100644 index b982e663a0cf..000000000000 --- a/contrib/compiler-rt/lib/tsan/Makefile.old +++ /dev/null @@ -1,106 +0,0 @@ -DEBUG=0 -LDFLAGS=-ldl -lrt -lpthread -pie -CXXFLAGS = -std=c++11 -fPIE -fno-rtti -g -Wall -Werror \ - -DGTEST_HAS_RTTI=0 -DTSAN_DEBUG=$(DEBUG) -DSANITIZER_DEBUG=$(DEBUG) -CLANG=clang -FILECHECK=FileCheck -# Silence warnings that Clang produces for gtest code. -# Use -Wno-attributes so that gcc doesn't complain about unknown warning types. -CXXFLAGS += -Wno-attributes -ifeq ($(DEBUG), 0) - CXXFLAGS += -O3 -endif -ifeq ($(CXX), $(CLANG)++) - CXXFLAGS+= -Wno-unused-private-field -Wno-static-in-inline -Wgnu -endif - -LIBTSAN=rtl/libtsan.a -GTEST_ROOT=third_party/googletest -GTEST_INCLUDE=-I$(GTEST_ROOT)/include -GTEST_BUILD_DIR=$(GTEST_ROOT)/build -GTEST_LIB_NAME=gtest-all.o -GTEST_LIB=$(GTEST_BUILD_DIR)/$(GTEST_LIB_NAME) - -SANITIZER_TESTS_PATH=../sanitizer_common/tests -SANITIZER_COMMON_TESTS_SRC=$(wildcard $(SANITIZER_TESTS_PATH)/*_test.cc) -SANITIZER_COMMON_EXCLUDED_TESTS=$(SANITIZER_TESTS_PATH)/sanitizer_nolibc_test.cc -SANITIZER_COMMON_GOOD_TESTS=$(filter-out $(SANITIZER_COMMON_EXCLUDED_TESTS), $(SANITIZER_COMMON_TESTS_SRC)) -SANITIZER_COMMON_TESTS_OBJ=$(patsubst %.cc,%.o,$(SANITIZER_COMMON_GOOD_TESTS)) -RTL_TEST_SRC=$(wildcard tests/rtl/*.cc) -RTL_TEST_OBJ=$(patsubst %.cc,%.o,$(RTL_TEST_SRC)) -UNIT_TEST_SRC=$(wildcard tests/unit/*_test.cc) -UNIT_TEST_OBJ=$(patsubst %.cc,%.o,$(UNIT_TEST_SRC)) -UNIT_TEST_HDR=$(wildcard rtl/*.h) $(wildcard ../sanitizer_common/*.h) -LIT_TESTS_PATH=../../test/tsan - -INCLUDES=-Irtl -I.. -I../../include $(GTEST_INCLUDE) - -all: libtsan test - -help: - @ echo "A little help is always welcome!" - @ echo "The most useful targets are:" - @ echo " make install_deps # Install third-party dependencies required for building" - @ echo " make presubmit # Run it every time before committing" - @ echo - @ echo "For more info, see http://code.google.com/p/thread-sanitizer/wiki/Development" - -$(LIBTSAN): libtsan - -libtsan: - $(MAKE) -C rtl -f Makefile.old DEBUG=$(DEBUG) - -%.o: %.cc $(UNIT_TEST_HDR) $(LIBTSAN) - $(CXX) $(CXXFLAGS) $(CFLAGS) $(INCLUDES) -o $@ -c $< - -tsan_test: $(UNIT_TEST_OBJ) $(RTL_TEST_OBJ) \ - $(SANITIZER_COMMON_TESTS_OBJ) $(LIBTSAN) $(GTEST_LIB) - $(CXX) -Wl,--whole-archive $^ -Wl,--no-whole-archive -o $@ $(LDFLAGS) - -test: libtsan tsan_test - -run: all - (ulimit -s 8192; ./tsan_test) - CC=$(CLANG) CXX=$(CLANG)++ FILECHECK=$(FILECHECK) $(LIT_TESTS_PATH)/test_output.sh - -presubmit: - ../sanitizer_common/scripts/check_lint.sh - # Debug build with clang. - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=1 -j 16 CC=$(CLANG) CXX=$(CLANG)++ - # Release build with clang. - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=0 -j 16 CC=$(CLANG) CXX=$(CLANG)++ - ./check_memcpy.sh - # Debug build with gcc - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=1 -j 16 CC=gcc CXX=g++ - # Release build with gcc - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=0 -j 16 CC=gcc CXX=g++ - ./check_memcpy.sh - ./check_analyze.sh - # Sanity check for Go runtime - (cd go && ./buildgo.sh) - # Check cmake build - ./check_cmake.sh - @ echo PRESUBMIT PASSED - -install_deps: - rm -rf third_party - mkdir third_party - (cd third_party && \ - svn co -r613 http://googletest.googlecode.com/svn/trunk googletest \ - ) - -$(GTEST_LIB): - mkdir -p $(GTEST_BUILD_DIR) && \ - cd $(GTEST_BUILD_DIR) && \ - $(MAKE) -f ../make/Makefile CXXFLAGS="$(CXXFLAGS)" CFLAGS="$(CFLAGS)" CC=$(CC) CXX=$(CXX) $(GTEST_LIB_NAME) - -clean: - rm -f asm_*.s libtsan.nm libtsan.objdump */*.o tsan_test - rm -rf $(GTEST_BUILD_DIR) - $(MAKE) clean -C rtl -f Makefile.old - rm -f go/*.s - rm -rf build diff --git a/contrib/compiler-rt/lib/tsan/check_analyze.sh b/contrib/compiler-rt/lib/tsan/check_analyze.sh index 08bfc7a76313..4b33393ef648 100755 --- a/contrib/compiler-rt/lib/tsan/check_analyze.sh +++ b/contrib/compiler-rt/lib/tsan/check_analyze.sh @@ -8,17 +8,6 @@ PrintRes() { PrintRes -wmops="write1 \ - write2 \ - write4 \ - write8" -rmops="read1 \ - read2 \ - read4 \ - read8" -func="func_entry \ - func_exit" - check() { res=$(PrintRes | egrep "$1 .* $2 $3; ") if [ "$res" == "" ]; then @@ -27,19 +16,25 @@ check() { fi } -for f in $wmops; do - check $f rsp 3 - check $f push 1 +for f in write1; do + check $f rsp 1 + check $f push 2 + check $f pop 2 +done + +for f in write2 write4 write8; do + check $f rsp 1 + check $f push 3 + check $f pop 3 +done + +for f in read1 read2 read4 read8; do + check $f rsp 1 + check $f push 5 check $f pop 5 done -for f in $rmops; do - check $f rsp 3 - check $f push 1 - check $f pop 4 -done - -for f in $func; do +for f in func_entry func_exit; do check $f rsp 0 check $f push 0 check $f pop 0 diff --git a/contrib/compiler-rt/lib/tsan/dd/dd_rtl.cc b/contrib/compiler-rt/lib/tsan/dd/dd_rtl.cc index 41b75bf755b8..99b8ee597cd1 100644 --- a/contrib/compiler-rt/lib/tsan/dd/dd_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/dd/dd_rtl.cc @@ -11,6 +11,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -64,20 +65,27 @@ u32 Callback::Unwind() { return CurrentStackTrace(thr, 3); } -void InitializeFlags(Flags *f, const char *env) { - internal_memset(f, 0, sizeof(*f)); +static void InitializeFlags() { + Flags *f = flags(); // Default values. f->second_deadlock_stack = false; - CommonFlags *cf = common_flags(); - SetCommonFlagsDefaults(cf); - // Override some common flags defaults. - cf->allow_addr2line = true; + SetCommonFlagsDefaults(); + { + // Override some common flags defaults. + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.allow_addr2line = true; + OverrideCommonFlags(cf); + } // Override from command line. - ParseFlag(env, &f->second_deadlock_stack, "second_deadlock_stack", ""); - ParseCommonFlagsFromString(cf, env); + FlagParser parser; + RegisterFlag(&parser, "second_deadlock_stack", "", &f->second_deadlock_stack); + RegisterCommonFlags(&parser); + parser.ParseString(GetEnv("DSAN_OPTIONS")); + SetVerbosity(common_flags()->verbosity); } void Initialize() { @@ -85,7 +93,7 @@ void Initialize() { ctx = new(ctx_mem) Context(); InitializeInterceptors(); - InitializeFlags(flags(), GetEnv("DSAN_OPTIONS")); + InitializeFlags(); ctx->dd = DDetector::Create(flags()); } diff --git a/contrib/compiler-rt/lib/tsan/go/build.bat b/contrib/compiler-rt/lib/tsan/go/build.bat index 8f8087fc6352..7d393dc0e025 100644 --- a/contrib/compiler-rt/lib/tsan/go/build.bat +++ b/contrib/compiler-rt/lib/tsan/go/build.bat @@ -1,4 +1,4 @@ -type tsan_go.cc ..\rtl\tsan_interface_atomic.cc ..\rtl\tsan_clock.cc ..\rtl\tsan_flags.cc ..\rtl\tsan_md5.cc ..\rtl\tsan_mutex.cc ..\rtl\tsan_report.cc ..\rtl\tsan_rtl.cc ..\rtl\tsan_rtl_mutex.cc ..\rtl\tsan_rtl_report.cc ..\rtl\tsan_rtl_thread.cc ..\rtl\tsan_stat.cc ..\rtl\tsan_suppressions.cc ..\rtl\tsan_sync.cc ..\rtl\tsan_stack_trace.cc ..\..\sanitizer_common\sanitizer_allocator.cc ..\..\sanitizer_common\sanitizer_common.cc ..\..\sanitizer_common\sanitizer_flags.cc ..\..\sanitizer_common\sanitizer_stacktrace.cc ..\..\sanitizer_common\sanitizer_libc.cc ..\..\sanitizer_common\sanitizer_printf.cc ..\..\sanitizer_common\sanitizer_suppressions.cc ..\..\sanitizer_common\sanitizer_thread_registry.cc ..\rtl\tsan_platform_windows.cc ..\..\sanitizer_common\sanitizer_win.cc ..\..\sanitizer_common\sanitizer_deadlock_detector1.cc ..\..\sanitizer_common\sanitizer_stackdepot.cc ..\..\sanitizer_common\sanitizer_persistent_allocator.cc > gotsan.cc +type tsan_go.cc ..\rtl\tsan_interface_atomic.cc ..\rtl\tsan_clock.cc ..\rtl\tsan_flags.cc ..\rtl\tsan_md5.cc ..\rtl\tsan_mutex.cc ..\rtl\tsan_report.cc ..\rtl\tsan_rtl.cc ..\rtl\tsan_rtl_mutex.cc ..\rtl\tsan_rtl_report.cc ..\rtl\tsan_rtl_thread.cc ..\rtl\tsan_stat.cc ..\rtl\tsan_suppressions.cc ..\rtl\tsan_sync.cc ..\rtl\tsan_stack_trace.cc ..\..\sanitizer_common\sanitizer_allocator.cc ..\..\sanitizer_common\sanitizer_common.cc ..\..\sanitizer_common\sanitizer_flags.cc ..\..\sanitizer_common\sanitizer_stacktrace.cc ..\..\sanitizer_common\sanitizer_libc.cc ..\..\sanitizer_common\sanitizer_printf.cc ..\..\sanitizer_common\sanitizer_suppressions.cc ..\..\sanitizer_common\sanitizer_thread_registry.cc ..\rtl\tsan_platform_windows.cc ..\..\sanitizer_common\sanitizer_win.cc ..\..\sanitizer_common\sanitizer_deadlock_detector1.cc ..\..\sanitizer_common\sanitizer_stackdepot.cc ..\..\sanitizer_common\sanitizer_persistent_allocator.cc ..\..\sanitizer_common\sanitizer_flag_parser.cc ..\..\sanitizer_common\sanitizer_symbolizer.cc > gotsan.cc -gcc -c -o race_windows_amd64.syso gotsan.cc -I..\rtl -I..\.. -I..\..\sanitizer_common -I..\..\..\include -m64 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO -DTSAN_SHADOW_COUNT=4 -Wno-error=attributes -Wno-attributes -Wno-format -DTSAN_DEBUG=0 -O3 -fomit-frame-pointer +gcc -c -o race_windows_amd64.syso gotsan.cc -I..\rtl -I..\.. -I..\..\sanitizer_common -I..\..\..\include -m64 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO -Wno-error=attributes -Wno-attributes -Wno-format -Wno-maybe-uninitialized -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer -std=c++11 diff --git a/contrib/compiler-rt/lib/tsan/go/buildgo.sh b/contrib/compiler-rt/lib/tsan/go/buildgo.sh index 4df2e96c5a8c..5ac60349e081 100755 --- a/contrib/compiler-rt/lib/tsan/go/buildgo.sh +++ b/contrib/compiler-rt/lib/tsan/go/buildgo.sh @@ -1,3 +1,5 @@ +#!/bin/sh + set -e SRCS=" @@ -19,6 +21,7 @@ SRCS=" ../../sanitizer_common/sanitizer_allocator.cc ../../sanitizer_common/sanitizer_common.cc ../../sanitizer_common/sanitizer_deadlock_detector2.cc + ../../sanitizer_common/sanitizer_flag_parser.cc ../../sanitizer_common/sanitizer_flags.cc ../../sanitizer_common/sanitizer_libc.cc ../../sanitizer_common/sanitizer_persistent_allocator.cc @@ -34,7 +37,8 @@ if [ "`uname -a | grep Linux`" != "" ]; then SUFFIX="linux_amd64" OSCFLAGS="-fPIC -ffreestanding -Wno-maybe-uninitialized -Wno-unused-const-variable -Werror -Wno-unknown-warning-option" OSLDFLAGS="-lpthread -fPIC -fpie" - SRCS+=" + SRCS=" + $SRCS ../rtl/tsan_platform_linux.cc ../../sanitizer_common/sanitizer_posix.cc ../../sanitizer_common/sanitizer_posix_libcdep.cc @@ -47,7 +51,8 @@ elif [ "`uname -a | grep FreeBSD`" != "" ]; then SUFFIX="freebsd_amd64" OSCFLAGS="-fno-strict-aliasing -fPIC -Werror" OSLDFLAGS="-lpthread -fPIC -fpie" - SRCS+=" + SRCS=" + $SRCS ../rtl/tsan_platform_linux.cc ../../sanitizer_common/sanitizer_posix.cc ../../sanitizer_common/sanitizer_posix_libcdep.cc @@ -60,7 +65,8 @@ elif [ "`uname -a | grep Darwin`" != "" ]; then SUFFIX="darwin_amd64" OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option" OSLDFLAGS="-lpthread -fPIC -fpie" - SRCS+=" + SRCS=" + $SRCS ../rtl/tsan_platform_mac.cc ../../sanitizer_common/sanitizer_mac.cc ../../sanitizer_common/sanitizer_posix.cc @@ -71,7 +77,8 @@ elif [ "`uname -a | grep MINGW`" != "" ]; then SUFFIX="windows_amd64" OSCFLAGS="-Wno-error=attributes -Wno-attributes -Wno-unused-const-variable -Wno-unknown-warning-option" OSLDFLAGS="" - SRCS+=" + SRCS=" + $SRCS ../rtl/tsan_platform_windows.cc ../../sanitizer_common/sanitizer_win.cc " @@ -80,24 +87,44 @@ else exit 1 fi -SRCS+=$ADD_SRCS +CC=${CC:-gcc} +IN_TMPDIR=${IN_TMPDIR:-0} +SILENT=${SILENT:-0} -rm -f gotsan.cc -for F in $SRCS; do - cat $F >> gotsan.cc -done - -FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++11 -m64 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO -DTSAN_SHADOW_COUNT=4 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS" -if [ "$DEBUG" == "" ]; then - FLAGS+=" -DTSAN_DEBUG=0 -O3 -msse3 -fomit-frame-pointer" +if [ $IN_TMPDIR != "0" ]; then + DIR=$(mktemp -qd /tmp/gotsan.XXXXXXXXXX) + cleanup() { + rm -rf $DIR + } + trap cleanup EXIT else - FLAGS+=" -DTSAN_DEBUG=1 -g" + DIR=. fi -CC=${CC:-gcc} +SRCS="$SRCS $ADD_SRCS" -echo $CC gotsan.cc -c -o race_$SUFFIX.syso $FLAGS $CFLAGS -$CC gotsan.cc -c -o race_$SUFFIX.syso $FLAGS $CFLAGS +rm -f $DIR/gotsan.cc +for F in $SRCS; do + cat $F >> $DIR/gotsan.cc +done -$CC test.c race_$SUFFIX.syso -m64 -o test $OSLDFLAGS -GORACE="exitcode=0 atexit_sleep_ms=0" ./test +FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++11 -m64 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS" +if [ "$DEBUG" = "" ]; then + FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -msse3 -fomit-frame-pointer" +else + FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g" +fi + +if [ "$SILENT" != "1" ]; then + echo $CC gotsan.cc -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS +fi +$CC $DIR/gotsan.cc -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS + +$CC test.c $DIR/race_$SUFFIX.syso -m64 -o $DIR/test $OSLDFLAGS + +export GORACE="exitcode=0 atexit_sleep_ms=0" +if [ "$SILENT" != "1" ]; then + $DIR/test +else + $DIR/test 2>/dev/null +fi diff --git a/contrib/compiler-rt/lib/tsan/go/tsan_go.cc b/contrib/compiler-rt/lib/tsan/go/tsan_go.cc index cccf72cedd27..ea0beb74215b 100644 --- a/contrib/compiler-rt/lib/tsan/go/tsan_go.cc +++ b/contrib/compiler-rt/lib/tsan/go/tsan_go.cc @@ -28,13 +28,6 @@ bool IsExpectedReport(uptr addr, uptr size) { return false; } -void *internal_start_thread(void(*func)(void*), void *arg) { - return 0; -} - -void internal_join_thread(void *th) { -} - ReportLocation *SymbolizeData(uptr addr) { return 0; } diff --git a/contrib/compiler-rt/lib/tsan/rtl/Makefile.old b/contrib/compiler-rt/lib/tsan/rtl/Makefile.old deleted file mode 100644 index 79c761ce3f4e..000000000000 --- a/contrib/compiler-rt/lib/tsan/rtl/Makefile.old +++ /dev/null @@ -1,62 +0,0 @@ -CXXFLAGS = -std=c++11 -fPIE -g -Wall -Werror -fno-builtin -msse3 -DTSAN_DEBUG=$(DEBUG) -DSANITIZER_DEBUG=$(DEBUG) -CLANG=clang -ifeq ($(DEBUG), 0) - CXXFLAGS += -O3 -endif - -# For interception. FIXME: move interception one level higher. -INTERCEPTION=../../interception -COMMON=../../sanitizer_common -INCLUDES= -I../.. -I../../../include -EXTRA_CXXFLAGS=-fno-exceptions -fno-rtti -NO_SYSROOT=--sysroot=. -CXXFLAGS+=$(EXTRA_CXXFLAGS) -CXXFLAGS+=$(CFLAGS) -ifeq ($(DEBUG), 0) - CXXFLAGS+=-fomit-frame-pointer -ifeq ($(CXX), g++) - CXXFLAGS+=-Wframe-larger-than=512 -endif # CXX=g++ -endif # DEBUG=0 - -ifeq ($(CXX), $(CLANG)++) - # Global constructors are banned. - CXXFLAGS+=-Wglobal-constructors -endif - - - -all: libtsan.a - -LIBTSAN_HEADERS=$(wildcard *.h) \ - $(wildcard $(INTERCEPTION)/*.h) \ - $(wildcard $(COMMON)/*.h) -LIBTSAN_SRC=$(wildcard *.cc) -LIBTSAN_ASM_SRC=$(wildcard *.S) -INTERCEPTION_SRC=$(wildcard $(INTERCEPTION)/*.cc) -COMMON_SRC=$(wildcard $(COMMON)/*.cc) - -LIBTSAN_OBJ=$(patsubst %.cc,%.o,$(LIBTSAN_SRC)) \ - $(patsubst %.S,%.o,$(LIBTSAN_ASM_SRC)) \ - $(patsubst $(INTERCEPTION)/%.cc,%.o,$(INTERCEPTION_SRC)) \ - $(patsubst $(COMMON)/%.cc,%.o,$(COMMON_SRC)) - -%_linux.o: %_linux.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -%.o: %.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) $(NO_SYSROOT) -c $< -%.o: $(INTERCEPTION)/%.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ -%.o: $(COMMON)/%.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ -%.o: %.S - $(CXX) $(INCLUDES) -o $@ -c $< - -libtsan.a: $(LIBTSAN_OBJ) - ar ru $@ $(LIBTSAN_OBJ) - -libtsan_dummy.a: tsan_dummy_rtl.o - ar ru $@ $< - -clean: - rm -f *.o *.a diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc index f2b39a182b39..59e3de435f1b 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc @@ -104,8 +104,8 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused) } void ThreadClock::acquire(ClockCache *c, const SyncClock *src) { - DCHECK(nclk_ <= kMaxTid); - DCHECK(src->size_ <= kMaxTid); + DCHECK_LE(nclk_, kMaxTid); + DCHECK_LE(src->size_, kMaxTid); CPP_STAT_INC(StatClockAcquire); // Check if it's empty -> no need to do anything. @@ -215,8 +215,8 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) const { } void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) const { - DCHECK(nclk_ <= kMaxTid); - DCHECK(dst->size_ <= kMaxTid); + DCHECK_LE(nclk_, kMaxTid); + DCHECK_LE(dst->size_, kMaxTid); CPP_STAT_INC(StatClockStore); // Check if we need to resize dst. diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_defs.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_defs.h index 7ed3796b5012..910a483127d5 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_defs.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_defs.h @@ -18,9 +18,14 @@ #include "sanitizer_common/sanitizer_libc.h" #include "tsan_stat.h" -#ifndef TSAN_DEBUG -#define TSAN_DEBUG 0 -#endif // TSAN_DEBUG +// Setup defaults for compile definitions. +#ifndef TSAN_NO_HISTORY +# define TSAN_NO_HISTORY 0 +#endif + +#ifndef TSAN_COLLECT_STATS +# define TSAN_COLLECT_STATS 0 +#endif namespace __tsan { @@ -39,23 +44,17 @@ const char *const kTsanOptionsEnv = "TSAN_OPTIONS"; const int kTidBits = 13; const unsigned kMaxTid = 1 << kTidBits; +#ifndef SANITIZER_GO const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit. +#else +const unsigned kMaxTidInClock = kMaxTid; // Go does not track freed memory. +#endif const int kClkBits = 42; const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1; const uptr kShadowStackSize = 64 * 1024; -#ifdef TSAN_SHADOW_COUNT -# if TSAN_SHADOW_COUNT == 2 \ - || TSAN_SHADOW_COUNT == 4 || TSAN_SHADOW_COUNT == 8 -const uptr kShadowCnt = TSAN_SHADOW_COUNT; -# else -# error "TSAN_SHADOW_COUNT must be one of 2,4,8" -# endif -#else // Count of shadow values in a shadow cell. -#define TSAN_SHADOW_COUNT 4 const uptr kShadowCnt = 4; -#endif // That many user bytes are mapped onto a single shadow cell. const uptr kShadowCell = 8; @@ -73,22 +72,16 @@ const uptr kMetaShadowCell = 8; // Size of a single meta shadow value (u32). const uptr kMetaShadowSize = 4; -#if defined(TSAN_NO_HISTORY) && TSAN_NO_HISTORY +#if TSAN_NO_HISTORY const bool kCollectHistory = false; #else const bool kCollectHistory = true; #endif -#if defined(TSAN_COLLECT_STATS) && TSAN_COLLECT_STATS -const bool kCollectStats = true; -#else -const bool kCollectStats = false; -#endif - // The following "build consistency" machinery ensures that all source files // are built in the same configuration. Inconsistent builds lead to // hard to debug crashes. -#if TSAN_DEBUG +#if SANITIZER_DEBUG void build_consistency_debug(); #else void build_consistency_release(); @@ -100,18 +93,8 @@ void build_consistency_stats(); void build_consistency_nostats(); #endif -#if TSAN_SHADOW_COUNT == 1 -void build_consistency_shadow1(); -#elif TSAN_SHADOW_COUNT == 2 -void build_consistency_shadow2(); -#elif TSAN_SHADOW_COUNT == 4 -void build_consistency_shadow4(); -#else -void build_consistency_shadow8(); -#endif - static inline void USED build_consistency() { -#if TSAN_DEBUG +#if SANITIZER_DEBUG build_consistency_debug(); #else build_consistency_release(); @@ -121,15 +104,6 @@ static inline void USED build_consistency() { #else build_consistency_nostats(); #endif -#if TSAN_SHADOW_COUNT == 1 - build_consistency_shadow1(); -#elif TSAN_SHADOW_COUNT == 2 - build_consistency_shadow2(); -#elif TSAN_SHADOW_COUNT == 4 - build_consistency_shadow4(); -#else - build_consistency_shadow8(); -#endif } template diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc index 5dc331f59469..1e81ef3d0003 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" #include "sanitizer_common/sanitizer_libc.h" #include "tsan_flags.h" #include "tsan_rtl.h" @@ -33,80 +34,46 @@ const char *WEAK __tsan_default_options() { } #endif -static void ParseFlags(Flags *f, const char *env) { - ParseFlag(env, &f->enable_annotations, "enable_annotations", ""); - ParseFlag(env, &f->suppress_equal_stacks, "suppress_equal_stacks", ""); - ParseFlag(env, &f->suppress_equal_addresses, "suppress_equal_addresses", ""); - ParseFlag(env, &f->report_bugs, "report_bugs", ""); - ParseFlag(env, &f->report_thread_leaks, "report_thread_leaks", ""); - ParseFlag(env, &f->report_destroy_locked, "report_destroy_locked", ""); - ParseFlag(env, &f->report_mutex_bugs, "report_mutex_bugs", ""); - ParseFlag(env, &f->report_signal_unsafe, "report_signal_unsafe", ""); - ParseFlag(env, &f->report_atomic_races, "report_atomic_races", ""); - ParseFlag(env, &f->force_seq_cst_atomics, "force_seq_cst_atomics", ""); - ParseFlag(env, &f->print_benign, "print_benign", ""); - ParseFlag(env, &f->exitcode, "exitcode", ""); - ParseFlag(env, &f->halt_on_error, "halt_on_error", ""); - ParseFlag(env, &f->atexit_sleep_ms, "atexit_sleep_ms", ""); - ParseFlag(env, &f->profile_memory, "profile_memory", ""); - ParseFlag(env, &f->flush_memory_ms, "flush_memory_ms", ""); - ParseFlag(env, &f->flush_symbolizer_ms, "flush_symbolizer_ms", ""); - ParseFlag(env, &f->memory_limit_mb, "memory_limit_mb", ""); - ParseFlag(env, &f->stop_on_start, "stop_on_start", ""); - ParseFlag(env, &f->running_on_valgrind, "running_on_valgrind", ""); - ParseFlag(env, &f->history_size, "history_size", ""); - ParseFlag(env, &f->io_sync, "io_sync", ""); - ParseFlag(env, &f->die_after_fork, "die_after_fork", ""); - +void Flags::SetDefaults() { +#define TSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "tsan_flags.inc" +#undef TSAN_FLAG // DDFlags - ParseFlag(env, &f->second_deadlock_stack, "second_deadlock_stack", ""); + second_deadlock_stack = false; +} + +void RegisterTsanFlags(FlagParser *parser, Flags *f) { +#define TSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "tsan_flags.inc" +#undef TSAN_FLAG } void InitializeFlags(Flags *f, const char *env) { - internal_memset(f, 0, sizeof(*f)); + FlagParser parser; + RegisterTsanFlags(&parser, f); + RegisterCommonFlags(&parser); - // Default values. - f->enable_annotations = true; - f->suppress_equal_stacks = true; - f->suppress_equal_addresses = true; - f->report_bugs = true; - f->report_thread_leaks = true; - f->report_destroy_locked = true; - f->report_mutex_bugs = true; - f->report_signal_unsafe = true; - f->report_atomic_races = true; - f->force_seq_cst_atomics = false; - f->print_benign = false; - f->exitcode = 66; - f->halt_on_error = false; - f->atexit_sleep_ms = 1000; - f->profile_memory = ""; - f->flush_memory_ms = 0; - f->flush_symbolizer_ms = 5000; - f->memory_limit_mb = 0; - f->stop_on_start = false; - f->running_on_valgrind = false; - f->history_size = kGoMode ? 1 : 2; // There are a lot of goroutines in Go. - f->io_sync = 1; - f->die_after_fork = true; + f->SetDefaults(); - // DDFlags - f->second_deadlock_stack = false; - - CommonFlags *cf = common_flags(); - SetCommonFlagsDefaults(cf); - // Override some common flags defaults. - cf->allow_addr2line = true; - cf->detect_deadlocks = true; - cf->print_suppressions = false; - cf->stack_trace_format = " #%n %f %S %M"; + SetCommonFlagsDefaults(); + { + // Override some common flags defaults. + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.allow_addr2line = true; +#ifndef SANITIZER_GO + cf.detect_deadlocks = true; +#endif + cf.print_suppressions = false; + cf.stack_trace_format = " #%n %f %S %M"; + OverrideCommonFlags(cf); + } // Let a frontend override. - ParseFlags(f, __tsan_default_options()); - ParseCommonFlagsFromString(cf, __tsan_default_options()); + parser.ParseString(__tsan_default_options()); // Override from command line. - ParseFlags(f, env); - ParseCommonFlagsFromString(cf, env); + parser.ParseString(env); // Sanity check. if (!f->report_bugs) { @@ -115,7 +82,11 @@ void InitializeFlags(Flags *f, const char *env) { f->report_signal_unsafe = false; } - if (cf->help) PrintFlagDescriptions(); + SetVerbosity(common_flags()->verbosity); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); if (f->history_size < 0 || f->history_size > 7) { Printf("ThreadSanitizer: incorrect value for history_size" diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.h index 621ca139236f..e2f6b3c9f021 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.h @@ -20,65 +20,12 @@ namespace __tsan { struct Flags : DDFlags { - // Enable dynamic annotations, otherwise they are no-ops. - bool enable_annotations; - // Suppress a race report if we've already output another race report - // with the same stack. - bool suppress_equal_stacks; - // Suppress a race report if we've already output another race report - // on the same address. - bool suppress_equal_addresses; - // Turns off bug reporting entirely (useful for benchmarking). - bool report_bugs; - // Report thread leaks at exit? - bool report_thread_leaks; - // Report destruction of a locked mutex? - bool report_destroy_locked; - // Report incorrect usages of mutexes and mutex annotations? - bool report_mutex_bugs; - // Report violations of async signal-safety - // (e.g. malloc() call from a signal handler). - bool report_signal_unsafe; - // Report races between atomic and plain memory accesses. - bool report_atomic_races; - // If set, all atomics are effectively sequentially consistent (seq_cst), - // regardless of what user actually specified. - bool force_seq_cst_atomics; - // Print matched "benign" races at exit. - bool print_benign; - // Override exit status if something was reported. - int exitcode; - // Exit after first reported error. - bool halt_on_error; - // Sleep in main thread before exiting for that many ms - // (useful to catch "at exit" races). - int atexit_sleep_ms; - // If set, periodically write memory profile to that file. - const char *profile_memory; - // Flush shadow memory every X ms. - int flush_memory_ms; - // Flush symbolizer caches every X ms. - int flush_symbolizer_ms; - // Resident memory limit in MB to aim at. - // If the process consumes more memory, then TSan will flush shadow memory. - int memory_limit_mb; - // Stops on start until __tsan_resume() is called (for debugging). - bool stop_on_start; - // Controls whether RunningOnValgrind() returns true or false. - bool running_on_valgrind; - // Per-thread history size, controls how many previous memory accesses - // are remembered per thread. Possible values are [0..7]. - // history_size=0 amounts to 32K memory accesses. Each next value doubles - // the amount of memory accesses, up to history_size=7 that amounts to - // 4M memory accesses. The default value is 2 (128K memory accesses). - int history_size; - // Controls level of synchronization implied by IO operations. - // 0 - no synchronization - // 1 - reasonable level of synchronization (write->read) - // 2 - global synchronization of all IO operations - int io_sync; - // Die after multi-threaded fork if the child creates new threads. - bool die_after_fork; +#define TSAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "tsan_flags.inc" +#undef TSAN_FLAG + + void SetDefaults(); + void ParseFromString(const char *str); }; Flags *flags(); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc new file mode 100644 index 000000000000..e4994685fa0d --- /dev/null +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc @@ -0,0 +1,79 @@ +//===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// TSan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_FLAG +# error "Define TSAN_FLAG prior to including this file!" +#endif + +// TSAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +TSAN_FLAG(bool, enable_annotations, true, + "Enable dynamic annotations, otherwise they are no-ops.") +// Suppress a race report if we've already output another race report +// with the same stack. +TSAN_FLAG(bool, suppress_equal_stacks, true, + "Suppress a race report if we've already output another race report " + "with the same stack.") +TSAN_FLAG(bool, suppress_equal_addresses, true, + "Suppress a race report if we've already output another race report " + "on the same address.") + +TSAN_FLAG(bool, report_bugs, true, + "Turns off bug reporting entirely (useful for benchmarking).") +TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?") +TSAN_FLAG(bool, report_destroy_locked, true, + "Report destruction of a locked mutex?") +TSAN_FLAG(bool, report_mutex_bugs, true, + "Report incorrect usages of mutexes and mutex annotations?") +TSAN_FLAG(bool, report_signal_unsafe, true, + "Report violations of async signal-safety " + "(e.g. malloc() call from a signal handler).") +TSAN_FLAG(bool, report_atomic_races, true, + "Report races between atomic and plain memory accesses.") +TSAN_FLAG( + bool, force_seq_cst_atomics, false, + "If set, all atomics are effectively sequentially consistent (seq_cst), " + "regardless of what user actually specified.") +TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.") +TSAN_FLAG(int, exitcode, 66, "Override exit status if something was reported.") +TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.") +TSAN_FLAG(int, atexit_sleep_ms, 1000, + "Sleep in main thread before exiting for that many ms " + "(useful to catch \"at exit\" races).") +TSAN_FLAG(const char *, profile_memory, "", + "If set, periodically write memory profile to that file.") +TSAN_FLAG(int, flush_memory_ms, 0, "Flush shadow memory every X ms.") +TSAN_FLAG(int, flush_symbolizer_ms, 5000, "Flush symbolizer caches every X ms.") +TSAN_FLAG( + int, memory_limit_mb, 0, + "Resident memory limit in MB to aim at." + "If the process consumes more memory, then TSan will flush shadow memory.") +TSAN_FLAG(bool, stop_on_start, false, + "Stops on start until __tsan_resume() is called (for debugging).") +TSAN_FLAG(bool, running_on_valgrind, false, + "Controls whether RunningOnValgrind() returns true or false.") +TSAN_FLAG( + int, history_size, kGoMode ? 1 : 3, // There are a lot of goroutines in Go. + "Per-thread history size, controls how many previous memory accesses " + "are remembered per thread. Possible values are [0..7]. " + "history_size=0 amounts to 32K memory accesses. Each next value doubles " + "the amount of memory accesses, up to history_size=7 that amounts to " + "4M memory accesses. The default value is 2 (128K memory accesses).") +TSAN_FLAG(int, io_sync, 1, + "Controls level of synchronization implied by IO operations. " + "0 - no synchronization " + "1 - reasonable level of synchronization (write->read)" + "2 - global synchronization of all IO operations.") +TSAN_FLAG(bool, die_after_fork, true, + "Die after multi-threaded fork if the child creates new threads.") +TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 5bede0ec7d0c..31ff7d56aac7 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -39,17 +39,27 @@ using namespace __tsan; // NOLINT #define stderr __stderrp #endif +#ifdef __mips__ +const int kSigCount = 129; +#else const int kSigCount = 65; +#endif struct my_siginfo_t { // The size is determined by looking at sizeof of real siginfo_t on linux. u64 opaque[128 / sizeof(u64)]; }; +#ifdef __mips__ +struct ucontext_t { + u64 opaque[768 / sizeof(u64) + 1]; +}; +#else struct ucontext_t { // The size is determined by looking at sizeof of real ucontext_t on linux. u64 opaque[936 / sizeof(u64) + 1]; }; +#endif extern "C" int pthread_attr_init(void *attr); extern "C" int pthread_attr_destroy(void *attr); @@ -72,6 +82,7 @@ extern "C" void *__libc_malloc(uptr size); extern "C" void *__libc_calloc(uptr size, uptr n); extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" void __libc_free(void *ptr); +extern "C" int dirfd(void *dirp); #if !SANITIZER_FREEBSD extern "C" int mallopt(int param, int value); #endif @@ -88,8 +99,13 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; +#ifdef __mips__ +const int SIGBUS = 10; +const int SIGSYS = 12; +#else const int SIGBUS = 7; const int SIGSYS = 31; +#endif void *const MAP_FAILED = (void*)-1; const int PTHREAD_BARRIER_SERIAL_THREAD = -1; const int MAP_FIXED = 0x10; @@ -101,21 +117,27 @@ typedef long long_t; // NOLINT # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ -typedef void (*sighandler_t)(int sig); - #define errno (*__errno_location()) +typedef void (*sighandler_t)(int sig); +typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); + struct sigaction_t { +#ifdef __mips__ + u32 sa_flags; +#endif union { sighandler_t sa_handler; - void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx); + sigactionhandler_t sa_sigaction; }; #if SANITIZER_FREEBSD int sa_flags; __sanitizer_sigset_t sa_mask; #else __sanitizer_sigset_t sa_mask; +#ifndef __mips__ int sa_flags; +#endif void (*sa_restorer)(); #endif }; @@ -123,8 +145,13 @@ struct sigaction_t { const sighandler_t SIG_DFL = (sighandler_t)0; const sighandler_t SIG_IGN = (sighandler_t)1; const sighandler_t SIG_ERR = (sighandler_t)-1; +#ifdef __mips__ +const int SA_SIGINFO = 8; +const int SIG_SETMASK = 3; +#else const int SA_SIGINFO = 4; const int SIG_SETMASK = 2; +#endif namespace std { struct nothrow_t {}; @@ -155,7 +182,13 @@ static LibIgnore *libignore() { } void InitializeLibIgnore() { - libignore()->Init(*SuppressionContext::Get()); + const SuppressionContext &supp = *Suppressions(); + const uptr n = supp.SuppressionCount(); + for (uptr i = 0; i < n; i++) { + const Suppression *s = supp.SuppressionAt(i); + if (0 == internal_strcmp(s->type, kSuppressionLib)) + libignore()->AddIgnoredLibrary(s->templ); + } libignore()->OnLibraryLoaded(0); } @@ -505,14 +538,10 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { if (cur_thread()->in_symbolizer) return __libc_calloc(size, n); - if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) - return AllocatorReturnNull(); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); - p = user_alloc(thr, pc, n * size); - if (p) - internal_memset(p, 0, n * size); + p = user_calloc(thr, pc, size, n); } invoke_malloc_hook(p, n * size); return p; @@ -952,6 +981,8 @@ TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { return res; } +DEFINE_REAL_PTHREAD_FUNCTIONS + TSAN_INTERCEPTOR(int, pthread_detach, void *th) { SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); int tid = ThreadTid(thr, pc, (uptr)th); @@ -1826,12 +1857,11 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) { return res; } -TSAN_INTERCEPTOR(void*, opendir, char *path) { - SCOPED_TSAN_INTERCEPTOR(opendir, path); - void *res = REAL(opendir)(path); - if (res != 0) - Acquire(thr, pc, Dir2addr(path)); - return res; +TSAN_INTERCEPTOR(int, closedir, void *dirp) { + SCOPED_TSAN_INTERCEPTOR(closedir, dirp); + int fd = dirfd(dirp); + FdClose(thr, pc, fd); + return REAL(closedir)(dirp); } #if !SANITIZER_FREEBSD @@ -1875,15 +1905,18 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, // Ensure that the handler does not spoil errno. const int saved_errno = errno; errno = 99; - // Need to remember pc before the call, because the handler can reset it. - uptr pc = sigact ? + // This code races with sigaction. Be careful to not read sa_sigaction twice. + // Also need to remember pc for reporting before the call, + // because the handler can reset it. + volatile uptr pc = sigact ? (uptr)sigactions[sig].sa_sigaction : (uptr)sigactions[sig].sa_handler; - pc += 1; // return address is expected, OutputReport() will undo this - if (sigact) - sigactions[sig].sa_sigaction(sig, info, uctx); - else - sigactions[sig].sa_handler(sig); + if (pc != (uptr)SIG_DFL && pc != (uptr)SIG_IGN) { + if (sigact) + ((sigactionhandler_t)pc)(sig, info, uctx); + else + ((sighandler_t)pc)(sig); + } // We do not detect errno spoiling for SIGTERM, // because some SIGTERM handlers do spoil errno but reraise SIGTERM, // tsan reports false positive in such case. @@ -1893,7 +1926,9 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, // signal; and it looks too fragile to intercept all ways to reraise a signal. if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { VarSizeStackTrace stack; - ObtainCurrentStack(thr, pc, &stack); + // StackTrace::GetNestInstructionPc(pc) is used because return address is + // expected, OutputReport() will undo this. + ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); if (!IsFiredSuppression(ctx, rep, stack)) { @@ -1919,11 +1954,8 @@ void ProcessPendingSignals(ThreadState *thr) { SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed) { signal->armed = false; - if (sigactions[sig].sa_handler != SIG_DFL - && sigactions[sig].sa_handler != SIG_IGN) { - CallUserSignalHandler(thr, false, true, signal->sigaction, - sig, &signal->siginfo, &signal->ctx); - } + CallUserSignalHandler(thr, false, true, signal->sigaction, sig, + &signal->siginfo, &signal->ctx); } } pthread_sigmask(SIG_SETMASK, &oldset, 0); @@ -2005,7 +2037,19 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { internal_memcpy(old, &sigactions[sig], sizeof(*old)); if (act == 0) return 0; - internal_memcpy(&sigactions[sig], act, sizeof(*act)); + // Copy act into sigactions[sig]. + // Can't use struct copy, because compiler can emit call to memcpy. + // Can't use internal_memcpy, because it copies byte-by-byte, + // and signal handler reads the sa_handler concurrently. It it can read + // some bytes from old value and some bytes from new value. + // Use volatile to prevent insertion of memcpy. + sigactions[sig].sa_handler = *(volatile sighandler_t*)&act->sa_handler; + sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags; + internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, + sizeof(sigactions[sig].sa_mask)); +#if !SANITIZER_FREEBSD + sigactions[sig].sa_restorer = act->sa_restorer; +#endif sigaction_t newact; internal_memcpy(&newact, act, sizeof(newact)); REAL(sigfillset)(&newact.sa_mask); @@ -2171,6 +2215,16 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #undef SANITIZER_INTERCEPT_FGETPWENT #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS +// __tls_get_addr can be called with mis-aligned stack due to: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 +// There are two potential issues: +// 1. Sanitizer code contains a MOVDQA spill (it does not seem to be the case +// right now). or 2. ProcessPendingSignal calls user handler which contains +// MOVDQA spill (this happens right now). +// Since the interceptor only initializes memory for msan, the simplest solution +// is to disable the interceptor in tsan (other sanitizers do not call +// signal handlers from COMMON_INTERCEPTOR_ENTER). +#undef SANITIZER_INTERCEPT_TLS_GET_ADDR #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) @@ -2209,12 +2263,15 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, if (fd >= 0) FdClose(thr, pc, fd); \ } -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res) \ +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ libignore()->OnLibraryLoaded(filename) #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ libignore()->OnLibraryUnloaded() +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) + #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) @@ -2530,7 +2587,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(abort); TSAN_INTERCEPT(puts); TSAN_INTERCEPT(rmdir); - TSAN_INTERCEPT(opendir); + TSAN_INTERCEPT(closedir); TSAN_MAYBE_INTERCEPT_EPOLL_CTL; TSAN_MAYBE_INTERCEPT_EPOLL_WAIT; @@ -2569,19 +2626,4 @@ void InitializeInterceptors() { FdInit(); } -void *internal_start_thread(void(*func)(void *arg), void *arg) { - // Start the thread with signals blocked, otherwise it can steal user signals. - __sanitizer_sigset_t set, old; - internal_sigfillset(&set); - internal_sigprocmask(SIG_SETMASK, &set, &old); - void *th; - REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg); - internal_sigprocmask(SIG_SETMASK, &old, 0); - return th; -} - -void internal_join_thread(void *th) { - REAL(pthread_join)(th, 0); -} - } // namespace __tsan diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.cc index 9de3808e79ff..9bc9a696363d 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.cc @@ -38,57 +38,80 @@ void __tsan_write16(void *addr) { MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8); } -u16 __tsan_unaligned_read2(const uu16 *addr) { +// __tsan_unaligned_read/write calls are emitted by compiler. + +void __tsan_unaligned_read2(const void *addr) { UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false); - return *addr; } -u32 __tsan_unaligned_read4(const uu32 *addr) { +void __tsan_unaligned_read4(const void *addr) { UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false); - return *addr; } -u64 __tsan_unaligned_read8(const uu64 *addr) { +void __tsan_unaligned_read8(const void *addr) { UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false); - return *addr; } -void __tsan_unaligned_write2(uu16 *addr, u16 v) { +void __tsan_unaligned_read16(const void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, false, false); +} + +void __tsan_unaligned_write2(void *addr) { UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false); - *addr = v; } -void __tsan_unaligned_write4(uu32 *addr, u32 v) { +void __tsan_unaligned_write4(void *addr) { UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false); - *addr = v; } -void __tsan_unaligned_write8(uu64 *addr, u64 v) { +void __tsan_unaligned_write8(void *addr) { UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false); - *addr = v; } +void __tsan_unaligned_write16(void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, true, false); +} + +// __sanitizer_unaligned_load/store are for user instrumentation. + extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -uint16_t __sanitizer_unaligned_load16(void *addr) - ALIAS("__tsan_unaligned_read2"); -SANITIZER_INTERFACE_ATTRIBUTE -uint32_t __sanitizer_unaligned_load32(void *addr) - ALIAS("__tsan_unaligned_read4"); -SANITIZER_INTERFACE_ATTRIBUTE -uint64_t __sanitizer_unaligned_load64(void *addr) - ALIAS("__tsan_unaligned_read8"); -SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_unaligned_store16(void *addr, uint16_t v) - ALIAS("__tsan_unaligned_write2"); -SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_unaligned_store32(void *addr, uint32_t v) - ALIAS("__tsan_unaligned_write4"); -SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_unaligned_store64(void *addr, uint64_t v) - ALIAS("__tsan_unaligned_write8"); +u16 __sanitizer_unaligned_load16(const uu16 *addr) { + __tsan_unaligned_read2(addr); + return *addr; } +SANITIZER_INTERFACE_ATTRIBUTE +u32 __sanitizer_unaligned_load32(const uu32 *addr) { + __tsan_unaligned_read4(addr); + return *addr; +} + +SANITIZER_INTERFACE_ATTRIBUTE +u64 __sanitizer_unaligned_load64(const uu64 *addr) { + __tsan_unaligned_read8(addr); + return *addr; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store16(uu16 *addr, u16 v) { + __tsan_unaligned_write2(addr); + *addr = v; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store32(uu32 *addr, u32 v) { + __tsan_unaligned_write4(addr); + *addr = v; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store64(uu64 *addr, u64 v) { + __tsan_unaligned_write8(addr); + *addr = v; +} +} // extern "C" + void __tsan_acquire(void *addr) { Acquire(cur_thread(), CALLERPC, (uptr)addr); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.h index 70450697d480..a05e6f0f6d09 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface.h @@ -41,12 +41,15 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4(void *addr); SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8(void *addr); SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16(void *addr); -SANITIZER_INTERFACE_ATTRIBUTE u16 __tsan_unaligned_read2(const uu16 *addr); -SANITIZER_INTERFACE_ATTRIBUTE u32 __tsan_unaligned_read4(const uu32 *addr); -SANITIZER_INTERFACE_ATTRIBUTE u64 __tsan_unaligned_read8(const uu64 *addr); -SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(uu16 *addr, u16 v); -SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(uu32 *addr, u32 v); -SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(uu64 *addr, u64 v); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr); SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc index 9b699511674a..27031991438c 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc @@ -33,14 +33,14 @@ typedef unsigned short a16; // NOLINT typedef unsigned int a32; typedef unsigned long long a64; // NOLINT #if !defined(SANITIZER_GO) && (defined(__SIZEOF_INT128__) \ - || (__clang_major__ * 100 + __clang_minor__ >= 302)) + || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64) __extension__ typedef __int128 a128; # define __TSAN_HAS_INT128 1 #else # define __TSAN_HAS_INT128 0 #endif -#ifndef SANITIZER_GO +#if !defined(SANITIZER_GO) && __TSAN_HAS_INT128 // Protects emulation of 128-bit atomic operations. static StaticSpinMutex mutex128; #endif @@ -125,7 +125,8 @@ template T func_cas(volatile T *v, T cmp, T xch) { // Atomic ops are executed under tsan internal mutex, // here we assume that the atomic variables are not accessed // from non-instrumented code. -#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(SANITIZER_GO) +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(SANITIZER_GO) \ + && __TSAN_HAS_INT128 a128 func_xchg(volatile a128 *v, a128 op) { SpinMutexLock lock(&mutex128); a128 cmp = *v; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.cc index 8615349f657f..0aea63d11671 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.cc @@ -219,3 +219,33 @@ int __tsan_java_mutex_unlock_rec(jptr addr) { return MutexUnlock(thr, pc, addr, true); } + +void __tsan_java_acquire(jptr addr) { + SCOPED_JAVA_FUNC(__tsan_java_acquire); + DPrintf("#%d: java_acquire(%p)\n", thr->tid, addr); + CHECK_NE(jctx, 0); + CHECK_GE(addr, jctx->heap_begin); + CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + Acquire(thr, caller_pc, addr); +} + +void __tsan_java_release(jptr addr) { + SCOPED_JAVA_FUNC(__tsan_java_release); + DPrintf("#%d: java_release(%p)\n", thr->tid, addr); + CHECK_NE(jctx, 0); + CHECK_GE(addr, jctx->heap_begin); + CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + Release(thr, caller_pc, addr); +} + +void __tsan_java_release_store(jptr addr) { + SCOPED_JAVA_FUNC(__tsan_java_release); + DPrintf("#%d: java_release_store(%p)\n", thr->tid, addr); + CHECK_NE(jctx, 0); + CHECK_GE(addr, jctx->heap_begin); + CHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + ReleaseStore(thr, caller_pc, addr); +} diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.h index 1f793df712de..30153a1d8505 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_java.h @@ -79,6 +79,14 @@ void __tsan_java_mutex_lock_rec(jptr addr, int rec) INTERFACE_ATTRIBUTE; // the same recursion level. int __tsan_java_mutex_unlock_rec(jptr addr) INTERFACE_ATTRIBUTE; +// Raw acquire/release primitives. +// Can be used to establish happens-before edges on volatile/final fields, +// in atomic operations, etc. release_store is the same as release, but it +// breaks release sequence on addr (see C++ standard 1.10/7 for details). +void __tsan_java_acquire(jptr addr) INTERFACE_ATTRIBUTE; +void __tsan_java_release(jptr addr) INTERFACE_ATTRIBUTE; +void __tsan_java_release_store(jptr addr) INTERFACE_ATTRIBUTE; + #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc index 285bdb34d91d..ebb3f77fb992 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc @@ -45,7 +45,7 @@ Allocator *allocator() { } void InitializeAllocator() { - allocator()->Init(); + allocator()->Init(common_flags()->allocator_may_return_null); } void AllocatorThreadStart(ThreadState *thr) { @@ -78,7 +78,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) - return AllocatorReturnNull(); + return allocator()->ReturnNullOrDie(); void *p = allocator()->Allocate(&thr->alloc_cache, sz, align); if (p == 0) return 0; @@ -89,6 +89,15 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { return p; } +void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { + if (CallocShouldReturnNullDueToOverflow(size, n)) + return allocator()->ReturnNullOrDie(); + void *p = user_alloc(thr, pc, n * size); + if (p) + internal_memset(p, 0, n * size); + return p; +} + void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { if (ctx && ctx->initialized) OnUserFree(thr, pc, (uptr)p, true); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h index 7d41fa864a26..5ff956d827f6 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h @@ -27,6 +27,7 @@ void AllocatorPrintStats(); // For user allocations. void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align = kDefaultAlignment, bool signal = true); +void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); // Does not accept NULL. void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.cc index 9ea9bae21b50..dc5a462a8081 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.cc @@ -25,7 +25,7 @@ namespace __tsan { // then Report mutex can be locked while under Threads mutex. // The leaf mutexes can be locked under any other mutexes. // Recursive locking is not supported. -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO const MutexType MutexTypeLeaf = (MutexType)-1; static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { /*0 MutexTypeInvalid*/ {}, @@ -47,7 +47,7 @@ static bool CanLockAdj[MutexTypeCount][MutexTypeCount]; #endif void InitializeMutex() { -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO // Build the "can lock" adjacency matrix. // If [i][j]==true, then one can lock mutex j while under mutex i. const int N = MutexTypeCount; @@ -128,7 +128,7 @@ InternalDeadlockDetector::InternalDeadlockDetector() { // Rely on zero initialization because some mutexes can be locked before ctor. } -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO void InternalDeadlockDetector::Lock(MutexType t) { // Printf("LOCK %d @%zu\n", t, seq_ + 1); CHECK_GT(t, MutexTypeInvalid); @@ -170,7 +170,7 @@ void InternalDeadlockDetector::CheckNoLocks() { #endif void CheckNoLocks(ThreadState *thr) { -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO thr->internal_deadlock_detector.CheckNoLocks(); #endif } @@ -208,7 +208,7 @@ class Backoff { Mutex::Mutex(MutexType type, StatType stat_type) { CHECK_GT(type, MutexTypeInvalid); CHECK_LT(type, MutexTypeCount); -#if TSAN_DEBUG +#if SANITIZER_DEBUG type_ = type; #endif #if TSAN_COLLECT_STATS @@ -222,7 +222,7 @@ Mutex::~Mutex() { } void Mutex::Lock() { -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO cur_thread()->internal_deadlock_detector.Lock(type_); #endif uptr cmp = kUnlocked; @@ -247,13 +247,13 @@ void Mutex::Unlock() { uptr prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release); (void)prev; DCHECK_NE(prev & kWriteLock, 0); -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO cur_thread()->internal_deadlock_detector.Unlock(type_); #endif } void Mutex::ReadLock() { -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO cur_thread()->internal_deadlock_detector.Lock(type_); #endif uptr prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); @@ -275,7 +275,7 @@ void Mutex::ReadUnlock() { (void)prev; DCHECK_EQ(prev & kWriteLock, 0); DCHECK_GT(prev & ~kWriteLock, 0); -#if TSAN_DEBUG && !SANITIZER_GO +#if SANITIZER_DEBUG && !SANITIZER_GO cur_thread()->internal_deadlock_detector.Unlock(type_); #endif } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.h index 7bb1c48fcac8..88fad57c78a0 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mutex.h @@ -52,7 +52,7 @@ class Mutex { private: atomic_uintptr_t state_; -#if TSAN_DEBUG +#if SANITIZER_DEBUG MutexType type_; #endif #if TSAN_COLLECT_STATS diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h index 270a7519dd0a..135e16027132 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -26,8 +26,9 @@ namespace __tsan { #if !defined(SANITIZER_GO) +#if defined(__x86_64__) /* -C/C++ on linux and freebsd +C/C++ on linux/x86_64 and freebsd/x86_64 0000 0000 1000 - 0100 0000 0000: main binary and/or MAP_32BIT mappings 0100 0000 0000 - 0200 0000 0000: - 0200 0000 0000 - 1000 0000 0000: shadow @@ -40,7 +41,6 @@ C/C++ on linux and freebsd 7e00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack */ - const uptr kMetaShadowBeg = 0x300000000000ull; const uptr kMetaShadowEnd = 0x400000000000ull; const uptr kTraceMemBeg = 0x600000000000ull; @@ -55,6 +55,38 @@ const uptr kHiAppMemBeg = 0x7e8000000000ull; const uptr kHiAppMemEnd = 0x800000000000ull; const uptr kAppMemMsk = 0x7c0000000000ull; const uptr kAppMemXor = 0x020000000000ull; +const uptr kVdsoBeg = 0xf000000000000000ull; +#elif defined(__mips64) +/* +C/C++ on linux/mips64 +0100 0000 00 - 0200 0000 00: main binary +0200 0000 00 - 1400 0000 00: - +1400 0000 00 - 2400 0000 00: shadow +2400 0000 00 - 3000 0000 00: - +3000 0000 00 - 4000 0000 00: metainfo (memory blocks and sync objects) +4000 0000 00 - 6000 0000 00: - +6000 0000 00 - 6200 0000 00: traces +6200 0000 00 - fe00 0000 00: - +fe00 0000 00 - ff00 0000 00: heap +ff00 0000 00 - ff80 0000 00: - +ff80 0000 00 - ffff ffff ff: modules and main thread stack +*/ +const uptr kMetaShadowBeg = 0x3000000000ull; +const uptr kMetaShadowEnd = 0x4000000000ull; +const uptr kTraceMemBeg = 0x6000000000ull; +const uptr kTraceMemEnd = 0x6200000000ull; +const uptr kShadowBeg = 0x1400000000ull; +const uptr kShadowEnd = 0x2400000000ull; +const uptr kHeapMemBeg = 0xfe00000000ull; +const uptr kHeapMemEnd = 0xff00000000ull; +const uptr kLoAppMemBeg = 0x0100000000ull; +const uptr kLoAppMemEnd = 0x0200000000ull; +const uptr kHiAppMemBeg = 0xff80000000ull; +const uptr kHiAppMemEnd = 0xffffffffffull; +const uptr kAppMemMsk = 0xfc00000000ull; +const uptr kAppMemXor = 0x0400000000ull; +const uptr kVdsoBeg = 0xfffff00000ull; +#endif ALWAYS_INLINE bool IsAppMem(uptr mem) { @@ -171,8 +203,8 @@ static USED uptr UserRegions[] = { 0000 1000 0000 - 00f8 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 0100 0000 0000: - -0100 0000 0000 - 0380 0000 0000: shadow -0380 0000 0000 - 0560 0000 0000: - +0100 0000 0000 - 0500 0000 0000: shadow +0500 0000 0000 - 0560 0000 0000: - 0560 0000 0000 - 0760 0000 0000: traces 0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) 07d0 0000 0000 - 8000 0000 0000: - @@ -183,7 +215,7 @@ const uptr kMetaShadowEnd = 0x07d000000000ull; const uptr kTraceMemBeg = 0x056000000000ull; const uptr kTraceMemEnd = 0x076000000000ull; const uptr kShadowBeg = 0x010000000000ull; -const uptr kShadowEnd = 0x038000000000ull; +const uptr kShadowEnd = 0x050000000000ull; const uptr kAppMemBeg = 0x000000001000ull; const uptr kAppMemEnd = 0x00e000000000ull; @@ -205,21 +237,21 @@ bool IsMetaMem(uptr mem) { ALWAYS_INLINE uptr MemToShadow(uptr x) { DCHECK(IsAppMem(x)); - return ((x & ~(kShadowCell - 1)) * kShadowCnt) | kShadowBeg; + return ((x & ~(kShadowCell - 1)) * kShadowCnt) + kShadowBeg; } ALWAYS_INLINE u32 *MemToMeta(uptr x) { DCHECK(IsAppMem(x)); return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ - kMetaShadowCell * kMetaShadowSize) | kMetaShadowEnd); + kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); } ALWAYS_INLINE uptr ShadowToMem(uptr s) { CHECK(IsShadowMem(s)); // FIXME(dvyukov): this is most likely wrong as the mapping is not bijection. - return (x & ~kShadowBeg) / kShadowCnt; + return (s - kShadowBeg) / kShadowCnt; } static USED uptr UserRegions[] = { @@ -252,9 +284,6 @@ void InitializePlatform(); void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); -void *internal_start_thread(void(*func)(void*), void *arg); -void internal_join_thread(void *th); - // Says whether the addr relates to a global var. // Guesses with high probability, may yield both false positives and negatives. bool IsGlobalVar(uptr addr); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc index 4dcfa558529c..659e8d8a8345 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc @@ -66,8 +66,6 @@ namespace __tsan { static uptr g_data_start; static uptr g_data_end; -const uptr kPageSize = 4096; - enum { MemTotal = 0, MemShadow = 1, @@ -173,7 +171,7 @@ static void MapRodata() { *p = kShadowRodata; internal_write(fd, marker.data(), marker.size()); // Map the file into memory. - uptr page = internal_mmap(0, kPageSize, PROT_READ | PROT_WRITE, + uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); if (internal_iserror(page)) { internal_close(fd); @@ -215,10 +213,18 @@ void InitializeShadowMemory() { // Frequently a thread uses only a small part of stack and similarly // a program uses a small part of large mmap. On some programs // we see 20% memory usage reduction without huge pages for this range. -#ifdef MADV_NOHUGEPAGE - madvise((void*)MemToShadow(0x7f0000000000ULL), - 0x10000000000ULL * kShadowMultiplier, MADV_NOHUGEPAGE); + // FIXME: don't use constants here. +#if defined(__x86_64__) + const uptr kMadviseRangeBeg = 0x7f0000000000ull; + const uptr kMadviseRangeSize = 0x010000000000ull; +#elif defined(__mips64) + const uptr kMadviseRangeBeg = 0xff00000000ull; + const uptr kMadviseRangeSize = 0x0100000000ull; #endif + NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg), + kMadviseRangeSize * kShadowMultiplier); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); DPrintf("memory shadow: %zx-%zx (%zuGB)\n", kShadowBeg, kShadowEnd, (kShadowEnd - kShadowBeg) >> 30); @@ -232,6 +238,8 @@ void InitializeShadowMemory() { "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg); Die(); } + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(meta, meta_size); DPrintf("meta shadow: %zx-%zx (%zuGB)\n", meta, meta + meta_size, meta_size >> 30); @@ -286,9 +294,9 @@ static void CheckAndProtect() { if (IsAppMem(p)) continue; if (p >= kHeapMemEnd && - p < kHeapMemEnd + PrimaryAllocator::AdditionalSize()) + p < HeapEnd()) continue; - if (p >= 0xf000000000000000ull) // vdso + if (p >= kVdsoBeg) // vdso break; Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end); Die(); @@ -301,7 +309,7 @@ static void CheckAndProtect() { // Protect the whole range for now, so that user does not map something here. ProtectRange(kTraceMemBeg, kTraceMemEnd); ProtectRange(kTraceMemEnd, kHeapMemBeg); - ProtectRange(kHeapMemEnd + PrimaryAllocator::AdditionalSize(), kHiAppMemBeg); + ProtectRange(HeapEnd(), kHiAppMemBeg); } #endif // #ifndef SANITIZER_GO diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc index 15b9f9d2cb19..63f1748e13ce 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc @@ -60,6 +60,8 @@ void InitializeShadowMemory() { "to link with -pie.\n"); Die(); } + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); DPrintf("kShadow %zx-%zx (%zuGB)\n", kShadowBeg, kShadowEnd, (kShadowEnd - kShadowBeg) >> 30); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc index c22f12a1bfa5..7e69cb4ecfbc 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc @@ -356,8 +356,9 @@ void PrintStack(const ReportStack *ent) { SymbolizedStack *frame = ent->frames; for (int i = 0; frame; frame = frame->next, i++) { const AddressInfo &info = frame->info; - Printf(" %s()\n %s:%d +0x%zx\n", info.function, info.file, info.line, - (void *)info.module_offset); + Printf(" %s()\n %s:%d +0x%zx\n", info.function, + StripPathPrefix(info.file, common_flags()->strip_path_prefix), + info.line, (void *)info.module_offset); } } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index 7cb7008e2980..b76f3e05dde6 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -67,8 +67,17 @@ static char thread_registry_placeholder[sizeof(ThreadRegistry)]; static ThreadContextBase *CreateThreadContext(u32 tid) { // Map thread trace when context is created. MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event)); - MapThreadTrace(GetThreadTraceHeader(tid), sizeof(Trace)); - new(ThreadTrace(tid)) Trace(); + const uptr hdr = GetThreadTraceHeader(tid); + MapThreadTrace(hdr, sizeof(Trace)); + new((void*)hdr) Trace(); + // We are going to use only a small part of the trace with the default + // value of history_size. However, the constructor writes to the whole trace. + // Unmap the unused part. + uptr hdr_end = hdr + sizeof(Trace); + hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts()); + hdr_end = RoundUp(hdr_end, GetPageSizeCached()); + if (hdr_end < hdr + sizeof(Trace)) + UnmapOrDie((void*)hdr_end, hdr + sizeof(Trace) - hdr_end); void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext)); return new(mem) ThreadContext(tid); } @@ -117,6 +126,7 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, { } +#ifndef SANITIZER_GO static void MemoryProfiler(Context *ctx, fd_t fd, int i) { uptr n_threads; uptr n_running_threads; @@ -127,13 +137,11 @@ static void MemoryProfiler(Context *ctx, fd_t fd, int i) { } static void BackgroundThread(void *arg) { -#ifndef SANITIZER_GO // This is a non-initialized non-user thread, nothing to see here. // We don't use ScopedIgnoreInterceptors, because we want ignores to be // enabled even when the thread function exits (e.g. during pthread thread // shutdown code). cur_thread()->ignore_interceptors++; -#endif const u64 kMs2Ns = 1000 * 1000; fd_t mprof_fd = kInvalidFd; @@ -191,7 +199,6 @@ static void BackgroundThread(void *arg) { if (mprof_fd != kInvalidFd) MemoryProfiler(ctx, mprof_fd, i); -#ifndef SANITIZER_GO // Flush symbolizer cache if requested. if (flags()->flush_symbolizer_ms > 0) { u64 last = atomic_load(&ctx->last_symbolize_time_ns, @@ -203,7 +210,6 @@ static void BackgroundThread(void *arg) { atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); } } -#endif } } @@ -211,13 +217,14 @@ static void StartBackgroundThread() { ctx->background_thread = internal_start_thread(&BackgroundThread, 0); } -#ifndef SANITIZER_GO +#ifndef __mips__ static void StopBackgroundThread() { atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed); internal_join_thread(ctx->background_thread); ctx->background_thread = 0; } #endif +#endif void DontNeedShadowFor(uptr addr, uptr size) { uptr shadow_beg = MemToShadow(addr); @@ -282,11 +289,11 @@ static void CheckShadowMapping() { if (p < beg || p >= end) continue; const uptr s = MemToShadow(p); - VPrintf(3, " checking pointer %p -> %p\n", p, s); + const uptr m = (uptr)MemToMeta(p); + VPrintf(3, " checking pointer %p: shadow=%p meta=%p\n", p, s, m); CHECK(IsAppMem(p)); CHECK(IsShadowMem(s)); CHECK_EQ(p & ~(kShadowCell - 1), ShadowToMem(s)); - const uptr m = (uptr)MemToMeta(p); CHECK(IsMetaMem(m)); } } @@ -325,10 +332,13 @@ void Initialize(ThreadState *thr) { #ifndef SANITIZER_GO InitializeLibIgnore(); Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); -#endif + // On MIPS, TSan initialization is run before + // __pthread_initialize_minimal_internal() is finished, so we can not spawn + // new threads. +#ifndef __mips__ StartBackgroundThread(); -#ifndef SANITIZER_GO SetSandboxingCallback(StopBackgroundThread); +#endif #endif if (common_flags()->detect_deadlocks) ctx->dd = DDetector::Create(flags()); @@ -365,8 +375,7 @@ int Finalize(ThreadState *thr) { ctx->report_mtx.Unlock(); #ifndef SANITIZER_GO - if (common_flags()->verbosity) - AllocatorPrintStats(); + if (Verbosity()) AllocatorPrintStats(); #endif ThreadFinalize(thr); @@ -395,8 +404,11 @@ int Finalize(ThreadState *thr) { failed = OnFinalize(failed); +#if TSAN_COLLECT_STATS StatAggregate(ctx->stat, thr->stat); StatOutput(ctx->stat); +#endif + return failed ? flags()->exitcode : 0; } @@ -420,7 +432,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc) { VPrintf(1, "ThreadSanitizer: forked new process with pid %d," " parent had %d threads\n", (int)internal_getpid(), (int)nthread); if (nthread == 1) { - internal_start_thread(&BackgroundThread, 0); + StartBackgroundThread(); } else { // We've just forked a multi-threaded process. We cannot reasonably function // after that (some mutexes may be locked before fork). So just enable @@ -565,43 +577,26 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, // it's just not worth it (performance- and complexity-wise). Shadow old(0); - if (kShadowCnt == 1) { - int idx = 0; + + // It release mode we manually unroll the loop, + // because empirically gcc generates better code this way. + // However, we can't afford unrolling in debug mode, because the function + // consumes almost 4K of stack. Gtest gives only 4K of stack to death test + // threads, which is not enough for the unrolled loop. +#if SANITIZER_DEBUG + for (int idx = 0; idx < 4; idx++) { #include "tsan_update_shadow_word_inl.h" - } else if (kShadowCnt == 2) { - int idx = 0; -#include "tsan_update_shadow_word_inl.h" - idx = 1; -#include "tsan_update_shadow_word_inl.h" - } else if (kShadowCnt == 4) { - int idx = 0; -#include "tsan_update_shadow_word_inl.h" - idx = 1; -#include "tsan_update_shadow_word_inl.h" - idx = 2; -#include "tsan_update_shadow_word_inl.h" - idx = 3; -#include "tsan_update_shadow_word_inl.h" - } else if (kShadowCnt == 8) { - int idx = 0; -#include "tsan_update_shadow_word_inl.h" - idx = 1; -#include "tsan_update_shadow_word_inl.h" - idx = 2; -#include "tsan_update_shadow_word_inl.h" - idx = 3; -#include "tsan_update_shadow_word_inl.h" - idx = 4; -#include "tsan_update_shadow_word_inl.h" - idx = 5; -#include "tsan_update_shadow_word_inl.h" - idx = 6; -#include "tsan_update_shadow_word_inl.h" - idx = 7; -#include "tsan_update_shadow_word_inl.h" - } else { - CHECK(false); } +#else + int idx = 0; +#include "tsan_update_shadow_word_inl.h" + idx = 1; +#include "tsan_update_shadow_word_inl.h" + idx = 2; +#include "tsan_update_shadow_word_inl.h" + idx = 3; +#include "tsan_update_shadow_word_inl.h" +#endif // we did not find any races and had already stored // the current access info, so we are done @@ -652,7 +647,7 @@ bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) { return false; } -#if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4 +#if defined(__SSE3__) #define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \ _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \ (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64)) @@ -712,11 +707,12 @@ bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) { ALWAYS_INLINE bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) { -#if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4 +#if defined(__SSE3__) bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write); // NOTE: this check can fail if the shadow is concurrently mutated - // by other threads. - DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write)); + // by other threads. But it still can be useful if you modify + // ContainsSameAccessFast and want to ensure that it's not completely broken. + // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write)); return res; #else return ContainsSameAccessSlow(s, a, sync_epoch, is_write); @@ -733,7 +729,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2], (uptr)shadow_mem[3]); -#if TSAN_DEBUG +#if SANITIZER_DEBUG if (!IsAppMem(addr)) { Printf("Access to non app mem %zx\n", addr); DCHECK(IsAppMem(addr)); @@ -843,7 +839,7 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, } } else { // The region is big, reset only beginning and end. - const uptr kPageSize = 4096; + const uptr kPageSize = GetPageSizeCached(); u64 *begin = (u64*)MemToShadow(addr); u64 *end = begin + size / kShadowCell * kShadowCnt; u64 *p = begin; @@ -990,7 +986,7 @@ bool MD5Hash::operator==(const MD5Hash &other) const { return hash[0] == other.hash[0] && hash[1] == other.hash[1]; } -#if TSAN_DEBUG +#if SANITIZER_DEBUG void build_consistency_debug() {} #else void build_consistency_release() {} @@ -1002,16 +998,6 @@ void build_consistency_stats() {} void build_consistency_nostats() {} #endif -#if TSAN_SHADOW_COUNT == 1 -void build_consistency_shadow1() {} -#elif TSAN_SHADOW_COUNT == 2 -void build_consistency_shadow2() {} -#elif TSAN_SHADOW_COUNT == 4 -void build_consistency_shadow4() {} -#else -void build_consistency_shadow8() {} -#endif - } // namespace __tsan #ifndef SANITIZER_GO diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h index 8d886875159b..7a60e5c55af4 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -54,8 +54,21 @@ namespace __tsan { #ifndef SANITIZER_GO struct MapUnmapCallback; +#ifdef __mips64 +static const uptr kAllocatorSpace = 0; +static const uptr kAllocatorSize = SANITIZER_MMAP_RANGE_SIZE; +static const uptr kAllocatorRegionSizeLog = 20; +static const uptr kAllocatorNumRegions = + kAllocatorSize >> kAllocatorRegionSizeLog; +typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, + MapUnmapCallback> ByteMap; +typedef SizeClassAllocator32 PrimaryAllocator; +#else typedef SizeClassAllocator64 PrimaryAllocator; +#endif typedef SizeClassAllocatorLocalCache AllocatorCache; typedef LargeMmapAllocator SecondaryAllocator; typedef CombinedAllocator jmp_bufs; int ignore_interceptors; #endif +#if TSAN_COLLECT_STATS u64 stat[StatCnt]; +#endif const int tid; const int unique_id; bool in_symbolizer; @@ -365,7 +380,9 @@ struct ThreadState { const uptr tls_size; ThreadContext *tctx; +#if SANITIZER_DEBUG && !SANITIZER_GO InternalDeadlockDetector internal_deadlock_detector; +#endif DDPhysicalThread *dd_pt; DDLogicalThread *dd_lt; @@ -539,15 +556,20 @@ void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) { } +#if TSAN_COLLECT_STATS void StatAggregate(u64 *dst, u64 *src); void StatOutput(u64 *stat); +#endif + void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { - if (kCollectStats) - thr->stat[typ] += n; +#if TSAN_COLLECT_STATS + thr->stat[typ] += n; +#endif } void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) { - if (kCollectStats) - thr->stat[typ] = n; +#if TSAN_COLLECT_STATS + thr->stat[typ] = n; +#endif } void MapShadow(uptr addr, uptr size); @@ -685,7 +707,7 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); // The trick is that the call preserves all registers and the compiler // does not treat it as a call. // If it does not work for you, use normal call. -#if TSAN_DEBUG == 0 +#if !SANITIZER_DEBUG && defined(__x86_64__) // The caller may not create the stack frame for itself at all, // so we create a reserve stack frame for it (1024b must be enough). #define HACKY_CALL(f) \ @@ -729,6 +751,16 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, *evp = ev; } +#ifndef SANITIZER_GO +uptr ALWAYS_INLINE HeapEnd() { +#if SANITIZER_CAN_USE_ALLOCATOR64 + return kHeapMemEnd + PrimaryAllocator::AdditionalSize(); +#else + return kHeapMemEnd; +#endif +} +#endif + } // namespace __tsan #endif // TSAN_RTL_H diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc index 0481b23b7be0..dc9438e6371b 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc @@ -87,7 +87,7 @@ static void StackStripMain(SymbolizedStack *frames) { // can actually happen if we do not instrument some code, // so it's only a debug print. However we must try hard to not miss it // due to our fault. - DPrintf("Bottom stack frame of stack %zx is missed\n", stack->info.address); + DPrintf("Bottom stack frame is missed\n"); } #else // The last frame always point into runtime (gosched0, goexit0, runtime.main). @@ -112,16 +112,10 @@ static ReportStack *SymbolizeStack(StackTrace trace) { for (uptr si = 0; si < trace.size; si++) { const uptr pc = trace.trace[si]; uptr pc1 = pc; -#ifndef SANITIZER_GO // We obtain the return address, but we're interested in the previous // instruction. if ((pc & kExternalPCBit) == 0) pc1 = StackTrace::GetPreviousInstructionPc(pc); -#else - // FIXME(dvyukov): Go sometimes uses address of a function as top pc. - if (si != trace.size - 1) - pc1 -= 1; -#endif SymbolizedStack *ent = SymbolizeCode(pc1); CHECK_NE(ent, 0); SymbolizedStack *last = ent; @@ -251,7 +245,8 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { void ScopedReport::AddThread(int unique_tid, bool suppressable) { #ifndef SANITIZER_GO - AddThread(FindThreadByUidLocked(unique_tid), suppressable); + if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid)) + AddThread(tctx, suppressable); #endif } @@ -397,7 +392,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, InternalScopedBuffer stack(kShadowStackSize); for (uptr i = 0; i < hdr->stack0.size; i++) { stack[i] = hdr->stack0.trace[i]; - DPrintf2(" #%02lu: pc=%zx\n", i, stack[i]); + DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]); } if (mset) *mset = hdr->mset0; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc index 7b7b27c024f6..8ed1fbf2edae 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc @@ -111,12 +111,13 @@ void ThreadContext::OnStarted(void *arg) { thr->dd_pt = ctx->dd->CreatePhysicalThread(); thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id); } + thr->fast_state.SetHistorySize(flags()->history_size); + // Commit switch to the new part of the trace. + // TraceAddEvent will reset stack0/mset0 in the new part for us. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + thr->fast_synch_epoch = epoch0; AcquireImpl(thr, 0, &sync); - thr->fast_state.SetHistorySize(flags()->history_size); - const uptr trace = (epoch0 / kTracePartSize) % TraceParts(); - Trace *thr_trace = ThreadTrace(thr->tid); - thr_trace->headers[trace].epoch0 = epoch0; StatInc(thr, StatSyncAcquire); sync.Reset(&thr->clock_cache); DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " @@ -144,7 +145,9 @@ void ThreadContext::OnFinished() { AllocatorThreadFinish(thr); #endif thr->~ThreadState(); +#if TSAN_COLLECT_STATS StatAggregate(ctx->stat, thr->stat); +#endif thr = 0; } @@ -238,6 +241,7 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; +#ifndef SANITIZER_GO GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { @@ -258,6 +262,7 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { thr_end, tls_addr + tls_size - thr_end); } } +#endif ThreadRegistry *tr = ctx->thread_registry; OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; @@ -329,7 +334,7 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, thr->tid, (void*)pc, (void*)addr, (int)size, is_write); -#if TSAN_DEBUG +#if SANITIZER_DEBUG if (!IsAppMem(addr)) { Printf("Access to non app mem %zx\n", addr); DCHECK(IsAppMem(addr)); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc index 350a2ba48253..15fa43d6f8a1 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc @@ -15,17 +15,14 @@ namespace __tsan { +#if TSAN_COLLECT_STATS + void StatAggregate(u64 *dst, u64 *src) { - if (!kCollectStats) - return; for (int i = 0; i < StatCnt; i++) dst[i] += src[i]; } void StatOutput(u64 *stat) { - if (!kCollectStats) - return; - stat[StatShadowNonZero] = stat[StatShadowProcessed] - stat[StatShadowZero]; static const char *name[StatCnt] = {}; @@ -176,4 +173,6 @@ void StatOutput(u64 *stat) { Printf("%s: %16zu\n", name[i], (uptr)stat[i]); } +#endif + } // namespace __tsan diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc index 299fc80fd262..5413f04af1d2 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc @@ -41,63 +41,74 @@ extern "C" const char *WEAK __tsan_default_suppressions() { namespace __tsan { -static bool suppressions_inited = false; +ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; +static SuppressionContext *suppression_ctx = nullptr; +static const char *kSuppressionTypes[] = { + kSuppressionRace, kSuppressionMutex, kSuppressionThread, + kSuppressionSignal, kSuppressionLib, kSuppressionDeadlock}; void InitializeSuppressions() { - CHECK(!suppressions_inited); - SuppressionContext::InitIfNecessary(); + CHECK_EQ(nullptr, suppression_ctx); + suppression_ctx = new (suppression_placeholder) // NOLINT + SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes)); + suppression_ctx->ParseFromFile(flags()->suppressions); #ifndef SANITIZER_GO - SuppressionContext::Get()->Parse(__tsan_default_suppressions()); - SuppressionContext::Get()->Parse(std_suppressions); + suppression_ctx->Parse(__tsan_default_suppressions()); + suppression_ctx->Parse(std_suppressions); #endif - suppressions_inited = true; } -SuppressionType conv(ReportType typ) { +SuppressionContext *Suppressions() { + CHECK(suppression_ctx); + return suppression_ctx; +} + +static const char *conv(ReportType typ) { if (typ == ReportTypeRace) - return SuppressionRace; + return kSuppressionRace; else if (typ == ReportTypeVptrRace) - return SuppressionRace; + return kSuppressionRace; else if (typ == ReportTypeUseAfterFree) - return SuppressionRace; + return kSuppressionRace; else if (typ == ReportTypeVptrUseAfterFree) - return SuppressionRace; + return kSuppressionRace; else if (typ == ReportTypeThreadLeak) - return SuppressionThread; + return kSuppressionThread; else if (typ == ReportTypeMutexDestroyLocked) - return SuppressionMutex; + return kSuppressionMutex; else if (typ == ReportTypeMutexDoubleLock) - return SuppressionMutex; + return kSuppressionMutex; else if (typ == ReportTypeMutexBadUnlock) - return SuppressionMutex; + return kSuppressionMutex; else if (typ == ReportTypeMutexBadReadLock) - return SuppressionMutex; + return kSuppressionMutex; else if (typ == ReportTypeMutexBadReadUnlock) - return SuppressionMutex; + return kSuppressionMutex; else if (typ == ReportTypeSignalUnsafe) - return SuppressionSignal; + return kSuppressionSignal; else if (typ == ReportTypeErrnoInSignal) - return SuppressionNone; + return kSuppressionNone; else if (typ == ReportTypeDeadlock) - return SuppressionDeadlock; + return kSuppressionDeadlock; Printf("ThreadSanitizer: unknown report type %d\n", typ), Die(); } uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp) { - if (!SuppressionContext::Get()->SuppressionCount() || stack == 0 || + CHECK(suppression_ctx); + if (!suppression_ctx->SuppressionCount() || stack == 0 || !stack->suppressable) return 0; - SuppressionType stype = conv(typ); - if (stype == SuppressionNone) + const char *stype = conv(typ); + if (0 == internal_strcmp(stype, kSuppressionNone)) return 0; Suppression *s; for (const SymbolizedStack *frame = stack->frames; frame; frame = frame->next) { const AddressInfo &info = frame->info; - if (SuppressionContext::Get()->Match(info.function, stype, &s) || - SuppressionContext::Get()->Match(info.file, stype, &s) || - SuppressionContext::Get()->Match(info.module, stype, &s)) { + if (suppression_ctx->Match(info.function, stype, &s) || + suppression_ctx->Match(info.file, stype, &s) || + suppression_ctx->Match(info.module, stype, &s)) { DPrintf("ThreadSanitizer: matched suppression '%s'\n", s->templ); s->hit_count++; *sp = s; @@ -108,16 +119,17 @@ uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp) { } uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) { - if (!SuppressionContext::Get()->SuppressionCount() || loc == 0 || + CHECK(suppression_ctx); + if (!suppression_ctx->SuppressionCount() || loc == 0 || loc->type != ReportLocationGlobal || !loc->suppressable) return 0; - SuppressionType stype = conv(typ); - if (stype == SuppressionNone) + const char *stype = conv(typ); + if (0 == internal_strcmp(stype, kSuppressionNone)) return 0; Suppression *s; const DataInfo &global = loc->global; - if (SuppressionContext::Get()->Match(global.name, stype, &s) || - SuppressionContext::Get()->Match(global.module, stype, &s)) { + if (suppression_ctx->Match(global.name, stype, &s) || + suppression_ctx->Match(global.module, stype, &s)) { DPrintf("ThreadSanitizer: matched suppression '%s'\n", s->templ); s->hit_count++; *sp = s; @@ -128,7 +140,8 @@ uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) { void PrintMatchedSuppressions() { InternalMmapVector matched(1); - SuppressionContext::Get()->GetMatched(&matched); + CHECK(suppression_ctx); + suppression_ctx->GetMatched(&matched); if (!matched.size()) return; int hit_count = 0; @@ -137,8 +150,8 @@ void PrintMatchedSuppressions() { Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count, (int)internal_getpid()); for (uptr i = 0; i < matched.size(); i++) { - Printf("%d %s:%s\n", matched[i]->hit_count, - SuppressionTypeString(matched[i]->type), matched[i]->templ); + Printf("%d %s:%s\n", matched[i]->hit_count, matched[i]->type, + matched[i]->templ); } } } // namespace __tsan diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.h index c618b3db4c2d..e6d279c33a4e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.h @@ -18,7 +18,16 @@ namespace __tsan { +const char kSuppressionNone[] = "none"; +const char kSuppressionRace[] = "race"; +const char kSuppressionMutex[] = "mutex"; +const char kSuppressionThread[] = "thread"; +const char kSuppressionSignal[] = "signal"; +const char kSuppressionLib[] = "called_from_lib"; +const char kSuppressionDeadlock[] = "deadlock"; + void InitializeSuppressions(); +SuppressionContext *Suppressions(); void PrintMatchedSuppressions(); uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp); uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h index 1da8752f6fa7..2569c7e42a47 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h @@ -20,9 +20,9 @@ namespace __tsan { -const int kTracePartSizeBits = 14; +const int kTracePartSizeBits = 13; const int kTracePartSize = 1 << kTracePartSizeBits; -const int kTraceParts = 4 * 1024 * 1024 / kTracePartSize; +const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize; const int kTraceSize = kTracePartSize * kTraceParts; // Must fit into 3 bits. @@ -54,13 +54,15 @@ struct TraceHeader { }; struct Trace { - TraceHeader headers[kTraceParts]; Mutex mtx; #ifndef SANITIZER_GO // Must be last to catch overflow as paging fault. // Go shadow stack is dynamically allocated. uptr shadow_stack[kShadowStackSize]; #endif + // Must be the last field, because we unmap the unused part in + // CreateThreadContext. + TraceHeader headers[kTraceParts]; Trace() : mtx(MutexTypeTrace, StatMtxTrace) { diff --git a/contrib/compiler-rt/lib/tsan/tests/rtl/tsan_string.cc b/contrib/compiler-rt/lib/tsan/tests/rtl/tsan_string.cc index c402f7cbd679..75adc6c85ee9 100644 --- a/contrib/compiler-rt/lib/tsan/tests/rtl/tsan_string.cc +++ b/contrib/compiler-rt/lib/tsan/tests/rtl/tsan_string.cc @@ -46,9 +46,6 @@ TEST(ThreadSanitizer, MemcpyRace1) { t2.Memcpy(data, data2, 10, true); } -// The test fails with TSAN_SHADOW_COUNT=2, -// because the old racy access is evicted. -#if defined(TSAN_SHADOW_COUNT) && TSAN_SHADOW_COUNT >= 4 TEST(ThreadSanitizer, MemcpyRace2) { char *data = new char[10]; char *data1 = new char[10]; @@ -57,7 +54,6 @@ TEST(ThreadSanitizer, MemcpyRace2) { t1.Memcpy(data+5, data1, 1); t2.Memcpy(data+3, data2, 4, true); } -#endif TEST(ThreadSanitizer, MemcpyRace3) { char *data = new char[10]; diff --git a/contrib/compiler-rt/lib/tsan/tests/unit/tsan_clock_test.cc b/contrib/compiler-rt/lib/tsan/tests/unit/tsan_clock_test.cc index a1fd2b7f6e99..92071827d3d8 100644 --- a/contrib/compiler-rt/lib/tsan/tests/unit/tsan_clock_test.cc +++ b/contrib/compiler-rt/lib/tsan/tests/unit/tsan_clock_test.cc @@ -211,8 +211,8 @@ TEST(Clock, Growth) { } } -const int kThreads = 4; -const int kClocks = 4; +const uptr kThreads = 4; +const uptr kClocks = 4; // SimpleSyncClock and SimpleThreadClock implement the same thing as // SyncClock and ThreadClock, but in a very simple way. diff --git a/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc b/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc index d969989df768..bfaefe648705 100644 --- a/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc +++ b/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc @@ -136,7 +136,7 @@ TEST(Mman, Stats) { } TEST(Mman, CallocOverflow) { -#if TSAN_DEBUG +#if SANITIZER_DEBUG // EXPECT_DEATH clones a thread with 4K stack, // which is overflown by tsan memory accesses functions in debug mode. return; diff --git a/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mutex_test.cc b/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mutex_test.cc index c39841ddcbb1..cce7f073b92f 100644 --- a/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mutex_test.cc +++ b/contrib/compiler-rt/lib/tsan/tests/unit/tsan_mutex_test.cc @@ -64,7 +64,7 @@ class TestData { const int kThreads = 8; const int kWriteRate = 1024; -#if TSAN_DEBUG +#if SANITIZER_DEBUG const int kIters = 16*1024; #else const int kIters = 64*1024; diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc index 76ce2bd39996..4f2a2a9f3562 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc @@ -14,9 +14,11 @@ #include "ubsan_diag.h" #include "ubsan_init.h" #include "ubsan_flags.h" +#include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_stacktrace_printer.h" +#include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_symbolizer.h" #include @@ -66,39 +68,9 @@ class Decorator : public SanitizerCommonDecorator { }; } -Location __ubsan::getCallerLocation(uptr CallerLoc) { - if (!CallerLoc) - return Location(); - - uptr Loc = StackTrace::GetPreviousInstructionPc(CallerLoc); - return getFunctionLocation(Loc, 0); -} - -Location __ubsan::getFunctionLocation(uptr Loc, const char **FName) { - if (!Loc) - return Location(); +SymbolizedStack *__ubsan::getSymbolizedLocation(uptr PC) { InitIfNecessary(); - - SymbolizedStack *Frames = Symbolizer::GetOrInit()->SymbolizePC(Loc); - const AddressInfo &Info = Frames->info; - - if (!Info.module) { - Frames->ClearAll(); - return Location(Loc); - } - - if (FName && Info.function) - *FName = internal_strdup(Info.function); - - if (!Info.file) { - ModuleLocation MLoc(internal_strdup(Info.module), Info.module_offset); - Frames->ClearAll(); - return MLoc; - } - - SourceLocation SLoc(internal_strdup(Info.file), Info.line, Info.column); - Frames->ClearAll(); - return SLoc; + return Symbolizer::GetOrInit()->SymbolizePC(PC); } Diag &Diag::operator<<(const TypeDescriptor &V) { @@ -142,15 +114,22 @@ static void renderLocation(Location Loc) { SLoc.getColumn(), common_flags()->strip_path_prefix); break; } - case Location::LK_Module: { - ModuleLocation MLoc = Loc.getModuleLocation(); - RenderModuleLocation(&LocBuffer, MLoc.getModuleName(), MLoc.getOffset(), - common_flags()->strip_path_prefix); - break; - } case Location::LK_Memory: LocBuffer.append("%p", Loc.getMemoryLocation()); break; + case Location::LK_Symbolized: { + const AddressInfo &Info = Loc.getSymbolizedStack()->info; + if (Info.file) { + RenderSourceLocation(&LocBuffer, Info.file, Info.line, Info.column, + common_flags()->strip_path_prefix); + } else if (Info.module) { + RenderModuleLocation(&LocBuffer, Info.module, Info.module_offset, + common_flags()->strip_path_prefix); + } else { + LocBuffer.append("%p", Info.address); + } + break; + } case Location::LK_Null: LocBuffer.append(""); break; @@ -356,11 +335,24 @@ ScopedReport::~ScopedReport() { Die(); } -bool __ubsan::MatchSuppression(const char *Str, SuppressionType Type) { - Suppression *s; +ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; +static SuppressionContext *suppression_ctx = nullptr; +static const char kVptrCheck[] = "vptr_check"; +static const char *kSuppressionTypes[] = { kVptrCheck }; + +void __ubsan::InitializeSuppressions() { + CHECK_EQ(nullptr, suppression_ctx); + suppression_ctx = new (suppression_placeholder) // NOLINT + SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes)); + suppression_ctx->ParseFromFile(flags()->suppressions); +} + +bool __ubsan::IsVptrCheckSuppressed(const char *TypeName) { // If .preinit_array is not used, it is possible that the UBSan runtime is not // initialized. if (!SANITIZER_CAN_USE_PREINIT_ARRAY) InitIfNecessary(); - return SuppressionContext::Get()->Match(Str, Type, &s); + CHECK(suppression_ctx); + Suppression *s; + return suppression_ctx->Match(TypeName, kVptrCheck, &s); } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h index 296ec0d3f61a..44dca90b7a53 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h @@ -15,79 +15,84 @@ #include "ubsan_value.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#include "sanitizer_common/sanitizer_suppressions.h" +#include "sanitizer_common/sanitizer_symbolizer.h" namespace __ubsan { -/// \brief A location within a loaded module in the program. These are used when -/// the location can't be resolved to a SourceLocation. -class ModuleLocation { - const char *ModuleName; - uptr Offset; +class SymbolizedStackHolder { + SymbolizedStack *Stack; + + void clear() { + if (Stack) + Stack->ClearAll(); + } public: - ModuleLocation() : ModuleName(0), Offset(0) {} - ModuleLocation(const char *ModuleName, uptr Offset) - : ModuleName(ModuleName), Offset(Offset) {} - const char *getModuleName() const { return ModuleName; } - uptr getOffset() const { return Offset; } + explicit SymbolizedStackHolder(SymbolizedStack *Stack = nullptr) + : Stack(Stack) {} + ~SymbolizedStackHolder() { clear(); } + void reset(SymbolizedStack *S) { + if (Stack != S) + clear(); + Stack = S; + } + const SymbolizedStack *get() const { return Stack; } }; +SymbolizedStack *getSymbolizedLocation(uptr PC); + +inline SymbolizedStack *getCallerLocation(uptr CallerPC) { + CHECK(CallerPC); + uptr PC = StackTrace::GetPreviousInstructionPc(CallerPC); + return getSymbolizedLocation(PC); +} + /// A location of some data within the program's address space. typedef uptr MemoryLocation; /// \brief Location at which a diagnostic can be emitted. Either a -/// SourceLocation, a ModuleLocation, or a MemoryLocation. +/// SourceLocation, a MemoryLocation, or a SymbolizedStack. class Location { public: - enum LocationKind { LK_Null, LK_Source, LK_Module, LK_Memory }; + enum LocationKind { LK_Null, LK_Source, LK_Memory, LK_Symbolized }; private: LocationKind Kind; // FIXME: In C++11, wrap these in an anonymous union. SourceLocation SourceLoc; - ModuleLocation ModuleLoc; MemoryLocation MemoryLoc; + const SymbolizedStack *SymbolizedLoc; // Not owned. public: Location() : Kind(LK_Null) {} Location(SourceLocation Loc) : Kind(LK_Source), SourceLoc(Loc) {} - Location(ModuleLocation Loc) : - Kind(LK_Module), ModuleLoc(Loc) {} Location(MemoryLocation Loc) : Kind(LK_Memory), MemoryLoc(Loc) {} + // SymbolizedStackHolder must outlive Location object. + Location(const SymbolizedStackHolder &Stack) : + Kind(LK_Symbolized), SymbolizedLoc(Stack.get()) {} LocationKind getKind() const { return Kind; } bool isSourceLocation() const { return Kind == LK_Source; } - bool isModuleLocation() const { return Kind == LK_Module; } bool isMemoryLocation() const { return Kind == LK_Memory; } + bool isSymbolizedStack() const { return Kind == LK_Symbolized; } SourceLocation getSourceLocation() const { CHECK(isSourceLocation()); return SourceLoc; } - ModuleLocation getModuleLocation() const { - CHECK(isModuleLocation()); - return ModuleLoc; - } MemoryLocation getMemoryLocation() const { CHECK(isMemoryLocation()); return MemoryLoc; } + const SymbolizedStack *getSymbolizedStack() const { + CHECK(isSymbolizedStack()); + return SymbolizedLoc; + } }; -/// Try to obtain a location for the caller. This might fail, and produce either -/// an invalid location or a module location for the caller. -Location getCallerLocation(uptr CallerLoc = GET_CALLER_PC()); - -/// Try to obtain a location for the given function pointer. This might fail, -/// and produce either an invalid location or a module location for the caller. -/// If FName is non-null and the name of the function is known, set *FName to -/// the function name, otherwise *FName is unchanged. -Location getFunctionLocation(uptr Loc, const char **FName); - /// A diagnostic severity level. enum DiagLevel { DL_Error, ///< An error. @@ -230,7 +235,8 @@ public: ~ScopedReport(); }; -bool MatchSuppression(const char *Str, SuppressionType Type); +void InitializeSuppressions(); +bool IsVptrCheckSuppressed(const char *TypeName); } // namespace __ubsan diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc index eda11f1b265c..0dbffc9b102a 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc @@ -14,6 +14,7 @@ #include "ubsan_flags.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" namespace __ubsan { @@ -21,36 +22,51 @@ static const char *MaybeCallUbsanDefaultOptions() { return (&__ubsan_default_options) ? __ubsan_default_options() : ""; } -void InitializeCommonFlags() { - CommonFlags *cf = common_flags(); - SetCommonFlagsDefaults(cf); - cf->print_summary = false; - // Override from user-specified string. - ParseCommonFlagsFromString(cf, MaybeCallUbsanDefaultOptions()); - // Override from environment variable. - ParseCommonFlagsFromString(cf, GetEnv("UBSAN_OPTIONS")); -} - Flags ubsan_flags; -static void ParseFlagsFromString(Flags *f, const char *str) { - if (!str) - return; - ParseFlag(str, &f->halt_on_error, "halt_on_error", - "Crash the program after printing the first error report"); - ParseFlag(str, &f->print_stacktrace, "print_stacktrace", - "Include full stacktrace into an error report"); +void Flags::SetDefaults() { +#define UBSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "ubsan_flags.inc" +#undef UBSAN_FLAG } -void InitializeFlags() { +void RegisterUbsanFlags(FlagParser *parser, Flags *f) { +#define UBSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "ubsan_flags.inc" +#undef UBSAN_FLAG +} + +void InitializeFlags(bool standalone) { Flags *f = flags(); - // Default values. - f->halt_on_error = false; - f->print_stacktrace = false; + FlagParser parser; + RegisterUbsanFlags(&parser, f); + + if (standalone) { + RegisterCommonFlags(&parser); + + SetCommonFlagsDefaults(); + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.print_summary = false; + OverrideCommonFlags(cf); + } else { + // Ignore common flags if not standalone. + // This is inconsistent with LSan, which allows common flags in LSAN_FLAGS. + // This is caused by undefined initialization order between ASan and UBsan, + // which makes it impossible to make sure that common flags from ASAN_OPTIONS + // have not been used (in __asan_init) before they are overwritten with flags + // from UBSAN_OPTIONS. + CommonFlags cf_ignored; + RegisterCommonFlags(&parser, &cf_ignored); + } + + f->SetDefaults(); // Override from user-specified string. - ParseFlagsFromString(f, MaybeCallUbsanDefaultOptions()); + parser.ParseString(MaybeCallUbsanDefaultOptions()); // Override from environment variable. - ParseFlagsFromString(f, GetEnv("UBSAN_OPTIONS")); + parser.ParseString(GetEnv("UBSAN_OPTIONS")); + SetVerbosity(common_flags()->verbosity); } } // namespace __ubsan diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.h b/contrib/compiler-rt/lib/ubsan/ubsan_flags.h index c496469f5f41..b47f14e1e2fd 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.h @@ -18,15 +18,17 @@ namespace __ubsan { struct Flags { - bool halt_on_error; - bool print_stacktrace; +#define UBSAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "ubsan_flags.inc" +#undef UBSAN_FLAG + + void SetDefaults(); }; extern Flags ubsan_flags; inline Flags *flags() { return &ubsan_flags; } -void InitializeCommonFlags(); -void InitializeFlags(); +void InitializeFlags(bool standalone); } // namespace __ubsan diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc new file mode 100644 index 000000000000..9ca31d13a9b6 --- /dev/null +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc @@ -0,0 +1,25 @@ +//===-- ubsan_flags.inc -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// UBSan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef UBSAN_FLAG +# error "Define UBSAN_FLAG prior to including this file!" +#endif + +// UBSAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +UBSAN_FLAG(bool, halt_on_error, false, + "Crash the program after printing the first error report") +UBSAN_FLAG(bool, print_stacktrace, false, + "Include full stacktrace into an error report") +UBSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") + diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc index a0ecff943592..78e7508f7f93 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc @@ -37,14 +37,17 @@ const char *TypeCheckKinds[] = { } static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer, - Location FallbackLoc, ReportOptions Opts) { + ReportOptions Opts) { Location Loc = Data->Loc.acquire(); // Use the SourceLocation from Data to track deduplication, even if 'invalid' if (ignoreReport(Loc.getSourceLocation(), Opts)) return; - if (Data->Loc.isInvalid()) + SymbolizedStackHolder FallbackLoc; + if (Data->Loc.isInvalid()) { + FallbackLoc.reset(getCallerLocation(Opts.pc)); Loc = FallbackLoc; + } ScopedReport R(Opts, Loc); @@ -67,12 +70,12 @@ static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer, void __ubsan::__ubsan_handle_type_mismatch(TypeMismatchData *Data, ValueHandle Pointer) { GET_REPORT_OPTIONS(false); - handleTypeMismatchImpl(Data, Pointer, getCallerLocation(), Opts); + handleTypeMismatchImpl(Data, Pointer, Opts); } void __ubsan::__ubsan_handle_type_mismatch_abort(TypeMismatchData *Data, ValueHandle Pointer) { GET_REPORT_OPTIONS(true); - handleTypeMismatchImpl(Data, Pointer, getCallerLocation(), Opts); + handleTypeMismatchImpl(Data, Pointer, Opts); Die(); } @@ -288,7 +291,8 @@ void __ubsan::__ubsan_handle_vla_bound_not_positive_abort(VLABoundData *Data, static void handleFloatCastOverflow(FloatCastOverflowData *Data, ValueHandle From, ReportOptions Opts) { // TODO: Add deduplication once a SourceLocation is generated for this check. - Location Loc = getCallerLocation(); + SymbolizedStackHolder CallerLoc(getCallerLocation(Opts.pc)); + Location Loc = CallerLoc; ScopedReport R(Opts, Loc); Diag(Loc, DL_Error, @@ -337,16 +341,21 @@ void __ubsan::__ubsan_handle_load_invalid_value_abort(InvalidValueData *Data, static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, ValueHandle Function, ReportOptions Opts) { - const char *FName = "(unknown)"; + SourceLocation CallLoc = Data->Loc.acquire(); + if (ignoreReport(CallLoc, Opts)) + return; - Location Loc = getFunctionLocation(Function, &FName); + ScopedReport R(Opts, CallLoc); - ScopedReport R(Opts, Loc); + SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); + const char *FName = FLoc.get()->info.function; + if (!FName) + FName = "(unknown)"; - Diag(Data->Loc, DL_Error, + Diag(CallLoc, DL_Error, "call to function %0 through pointer to incorrect function type %1") - << FName << Data->Type; - Diag(Loc, DL_Note, "%0 defined here") << FName; + << FName << Data->Type; + Diag(FLoc, DL_Note, "%0 defined here") << FName; } void diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc index 5704c1e6342d..4718e6eacee2 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc @@ -36,8 +36,7 @@ static void HandleDynamicTypeCacheMiss( // Check if error report should be suppressed. DynamicTypeInfo DTI = getDynamicTypeInfo((void*)Pointer); - if (DTI.isValid() && - MatchSuppression(DTI.getMostDerivedTypeName(), SuppressionVptrCheck)) + if (DTI.isValid() && IsVptrCheckSuppressed(DTI.getMostDerivedTypeName())) return; SourceLocation Loc = Data->Loc.acquire(); diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc index 6080e304c122..219273d51921 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// +#include "ubsan_diag.h" #include "ubsan_init.h" #include "ubsan_flags.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_mutex.h" -#include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_symbolizer.h" using namespace __ubsan; @@ -31,6 +31,7 @@ void __ubsan::InitIfNecessary() { #endif if (LIKELY(ubsan_inited)) return; + bool standalone = false; if (0 == internal_strcmp(SanitizerToolName, "SanitizerTool")) { // WARNING: If this condition holds, then either UBSan runs in a standalone // mode, or initializer for another sanitizer hasn't run yet. In a latter @@ -38,11 +39,12 @@ void __ubsan::InitIfNecessary() { // common flags. It means, that we are not allowed to *use* common flags // in this function. SanitizerToolName = "UndefinedBehaviorSanitizer"; - InitializeCommonFlags(); + standalone = true; } // Initialize UBSan-specific flags. - InitializeFlags(); - SuppressionContext::InitIfNecessary(); + InitializeFlags(standalone); + InitializeSuppressions(); + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); ubsan_inited = true; } diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_type_hash.cc b/contrib/compiler-rt/lib/ubsan/ubsan_type_hash.cc index 808a4332d01f..a388bcc6d72e 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_type_hash.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_type_hash.cc @@ -115,8 +115,7 @@ __ubsan::__ubsan_vptr_type_cache[__ubsan::VptrTypeCacheSize]; /// \brief Determine whether \p Derived has a \p Base base class subobject at /// offset \p Offset. -static bool isDerivedFromAtOffset(sptr Object, - const abi::__class_type_info *Derived, +static bool isDerivedFromAtOffset(const abi::__class_type_info *Derived, const abi::__class_type_info *Base, sptr Offset) { if (Derived->__type_name == Base->__type_name) @@ -124,7 +123,7 @@ static bool isDerivedFromAtOffset(sptr Object, if (const abi::__si_class_type_info *SI = dynamic_cast(Derived)) - return isDerivedFromAtOffset(Object, SI->__base_type, Base, Offset); + return isDerivedFromAtOffset(SI->__base_type, Base, Offset); const abi::__vmi_class_type_info *VTI = dynamic_cast(Derived); @@ -139,13 +138,13 @@ static bool isDerivedFromAtOffset(sptr Object, sptr OffsetHere = VTI->base_info[base].__offset_flags >> abi::__base_class_type_info::__offset_shift; if (VTI->base_info[base].__offset_flags & - abi::__base_class_type_info::__virtual_mask) { - sptr VTable = *reinterpret_cast(Object); - OffsetHere = *reinterpret_cast(VTable + OffsetHere); - } - if (isDerivedFromAtOffset(Object + OffsetHere, - VTI->base_info[base].__base_type, Base, - Offset - OffsetHere)) + abi::__base_class_type_info::__virtual_mask) + // For now, just punt on virtual bases and say 'yes'. + // FIXME: OffsetHere is the offset in the vtable of the virtual base + // offset. Read the vbase offset out of the vtable and use it. + return true; + if (isDerivedFromAtOffset(VTI->base_info[base].__base_type, + Base, Offset - OffsetHere)) return true; } @@ -154,15 +153,14 @@ static bool isDerivedFromAtOffset(sptr Object, /// \brief Find the derived-most dynamic base class of \p Derived at offset /// \p Offset. -static const abi::__class_type_info * -findBaseAtOffset(sptr Object, const abi::__class_type_info *Derived, - sptr Offset) { +static const abi::__class_type_info *findBaseAtOffset( + const abi::__class_type_info *Derived, sptr Offset) { if (!Offset) return Derived; if (const abi::__si_class_type_info *SI = dynamic_cast(Derived)) - return findBaseAtOffset(Object, SI->__base_type, Offset); + return findBaseAtOffset(SI->__base_type, Offset); const abi::__vmi_class_type_info *VTI = dynamic_cast(Derived); @@ -174,13 +172,12 @@ findBaseAtOffset(sptr Object, const abi::__class_type_info *Derived, sptr OffsetHere = VTI->base_info[base].__offset_flags >> abi::__base_class_type_info::__offset_shift; if (VTI->base_info[base].__offset_flags & - abi::__base_class_type_info::__virtual_mask) { - sptr VTable = *reinterpret_cast(Object); - OffsetHere = *reinterpret_cast(VTable + OffsetHere); - } - if (const abi::__class_type_info *Base = findBaseAtOffset( - Object + OffsetHere, VTI->base_info[base].__base_type, - Offset - OffsetHere)) + abi::__base_class_type_info::__virtual_mask) + // FIXME: Can't handle virtual bases yet. + continue; + if (const abi::__class_type_info *Base = + findBaseAtOffset(VTI->base_info[base].__base_type, + Offset - OffsetHere)) return Base; } @@ -232,8 +229,7 @@ bool __ubsan::checkDynamicType(void *Object, void *Type, HashValue Hash) { return false; abi::__class_type_info *Base = (abi::__class_type_info*)Type; - if (!isDerivedFromAtOffset(reinterpret_cast(Object), Derived, Base, - -Vtable->Offset)) + if (!isDerivedFromAtOffset(Derived, Base, -Vtable->Offset)) return false; // Success. Cache this result. @@ -247,9 +243,8 @@ __ubsan::DynamicTypeInfo __ubsan::getDynamicTypeInfo(void *Object) { if (!Vtable) return DynamicTypeInfo(0, 0, 0); const abi::__class_type_info *ObjectType = findBaseAtOffset( - reinterpret_cast(Object), - static_cast(Vtable->TypeInfo), - -Vtable->Offset); + static_cast(Vtable->TypeInfo), + -Vtable->Offset); return DynamicTypeInfo(Vtable->TypeInfo->__type_name, -Vtable->Offset, ObjectType ? ObjectType->__type_name : ""); } diff --git a/contrib/llvm/include/llvm-c/BitReader.h b/contrib/llvm/include/llvm-c/BitReader.h index 7af209bdfdce..f3b388bc4fb4 100644 --- a/contrib/llvm/include/llvm-c/BitReader.h +++ b/contrib/llvm/include/llvm-c/BitReader.h @@ -16,8 +16,8 @@ |* *| \*===----------------------------------------------------------------------===*/ -#ifndef LLVM_C_BITCODEREADER_H -#define LLVM_C_BITCODEREADER_H +#ifndef LLVM_C_BITREADER_H +#define LLVM_C_BITREADER_H #include "llvm-c/Core.h" diff --git a/contrib/llvm/include/llvm-c/BitWriter.h b/contrib/llvm/include/llvm-c/BitWriter.h index f605e2453f1c..f25ad3a445f5 100644 --- a/contrib/llvm/include/llvm-c/BitWriter.h +++ b/contrib/llvm/include/llvm-c/BitWriter.h @@ -16,8 +16,8 @@ |* *| \*===----------------------------------------------------------------------===*/ -#ifndef LLVM_C_BITCODEWRITER_H -#define LLVM_C_BITCODEWRITER_H +#ifndef LLVM_C_BITWRITER_H +#define LLVM_C_BITWRITER_H #include "llvm-c/Core.h" @@ -45,6 +45,9 @@ int LLVMWriteBitcodeToFD(LLVMModuleRef M, int FD, int ShouldClose, descriptor. Returns 0 on success. Closes the Handle. */ int LLVMWriteBitcodeToFileHandle(LLVMModuleRef M, int Handle); +/** Writes a module to a new memory buffer and returns it. */ +LLVMMemoryBufferRef LLVMWriteBitcodeToMemoryBuffer(LLVMModuleRef M); + /** * @} */ diff --git a/contrib/llvm/include/llvm-c/Core.h b/contrib/llvm/include/llvm-c/Core.h index fdff77bc5e51..8873fdb67730 100644 --- a/contrib/llvm/include/llvm-c/Core.h +++ b/contrib/llvm/include/llvm-c/Core.h @@ -560,6 +560,10 @@ LLVMModuleRef LLVMModuleCreateWithName(const char *ModuleID); */ LLVMModuleRef LLVMModuleCreateWithNameInContext(const char *ModuleID, LLVMContextRef C); +/** + * Return an exact copy of the specified module. + */ +LLVMModuleRef LLVMCloneModule(LLVMModuleRef M); /** * Destroy a module instance. @@ -1153,8 +1157,6 @@ LLVMTypeRef LLVMX86MMXType(void); macro(Argument) \ macro(BasicBlock) \ macro(InlineAsm) \ - macro(MDNode) \ - macro(MDString) \ macro(User) \ macro(Constant) \ macro(BlockAddress) \ @@ -1303,6 +1305,9 @@ LLVMBool LLVMIsUndef(LLVMValueRef Val); LLVMValueRef LLVMIsA##name(LLVMValueRef Val); LLVM_FOR_EACH_VALUE_SUBCLASS(LLVM_DECLARE_VALUE_CAST) +LLVMValueRef LLVMIsAMDNode(LLVMValueRef Val); +LLVMValueRef LLVMIsAMDString(LLVMValueRef Val); + /** * @} */ @@ -1377,6 +1382,13 @@ LLVMValueRef LLVMGetUsedValue(LLVMUseRef U); */ LLVMValueRef LLVMGetOperand(LLVMValueRef Val, unsigned Index); +/** + * Obtain the use of an operand at a specific index in a llvm::User value. + * + * @see llvm::User::getOperandUse() + */ +LLVMUseRef LLVMGetOperandUse(LLVMValueRef Val, unsigned Index); + /** * Set an operand at a specific index in a llvm::User value. * @@ -1537,6 +1549,14 @@ unsigned long long LLVMConstIntGetZExtValue(LLVMValueRef ConstantVal); */ long long LLVMConstIntGetSExtValue(LLVMValueRef ConstantVal); +/** + * Obtain the double value for an floating point constant value. + * losesInfo indicates if some precision was lost in the conversion. + * + * @see llvm::ConstantFP::getDoubleValue + */ +double LLVMConstRealGetDouble(LLVMValueRef ConstantVal, LLVMBool *losesInfo); + /** * @} */ @@ -1569,6 +1589,20 @@ LLVMValueRef LLVMConstStringInContext(LLVMContextRef C, const char *Str, LLVMValueRef LLVMConstString(const char *Str, unsigned Length, LLVMBool DontNullTerminate); +/** + * Returns true if the specified constant is an array of i8. + * + * @see ConstantDataSequential::getAsString() + */ +LLVMBool LLVMIsConstantString(LLVMValueRef c); + +/** + * Get the given constant data sequential as a string. + * + * @see ConstantDataSequential::getAsString() + */ +const char *LLVMGetAsString(LLVMValueRef c, size_t* out); + /** * Create an anonymous ConstantStruct with the specified values. * @@ -1606,6 +1640,13 @@ LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy, LLVMValueRef *ConstantVals, unsigned Count); +/** + * Get an element at specified index as a constant. + * + * @see ConstantDataSequential::getElementAsConstant() + */ +LLVMValueRef LLVMGetElementAsConstant(LLVMValueRef c, unsigned idx); + /** * Create a ConstantVector from values. * @@ -2376,6 +2417,26 @@ LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst); */ LLVMIntPredicate LLVMGetICmpPredicate(LLVMValueRef Inst); +/** + * Obtain the float predicate of an instruction. + * + * This is only valid for instructions that correspond to llvm::FCmpInst + * or llvm::ConstantExpr whose opcode is llvm::Instruction::FCmp. + * + * @see llvm::FCmpInst::getPredicate() + */ +LLVMRealPredicate LLVMGetFCmpPredicate(LLVMValueRef Inst); + +/** + * Create a copy of 'this' instruction that is identical in all ways + * except the following: + * * The instruction has no parent + * * The instruction has no name + * + * @see llvm::Instruction::clone() + */ +LLVMValueRef LLVMInstructionClone(LLVMValueRef Inst); + /** * @defgroup LLVMCCoreValueInstructionCall Call Sites and Invocations * @@ -2436,6 +2497,63 @@ void LLVMSetTailCall(LLVMValueRef CallInst, LLVMBool IsTailCall); * @} */ +/** + * @defgroup LLVMCCoreValueInstructionTerminator Terminators + * + * Functions in this group only apply to instructions that map to + * llvm::TerminatorInst instances. + * + * @{ + */ + +/** + * Return the number of successors that this terminator has. + * + * @see llvm::TerminatorInst::getNumSuccessors + */ +unsigned LLVMGetNumSuccessors(LLVMValueRef Term); + +/** + * Return the specified successor. + * + * @see llvm::TerminatorInst::getSuccessor + */ +LLVMBasicBlockRef LLVMGetSuccessor(LLVMValueRef Term, unsigned i); + +/** + * Update the specified successor to point at the provided block. + * + * @see llvm::TerminatorInst::setSuccessor + */ +void LLVMSetSuccessor(LLVMValueRef Term, unsigned i, LLVMBasicBlockRef block); + +/** + * Return if a branch is conditional. + * + * This only works on llvm::BranchInst instructions. + * + * @see llvm::BranchInst::isConditional + */ +LLVMBool LLVMIsConditional(LLVMValueRef Branch); + +/** + * Return the condition of a branch instruction. + * + * This only works on llvm::BranchInst instructions. + * + * @see llvm::BranchInst::getCondition + */ +LLVMValueRef LLVMGetCondition(LLVMValueRef Branch); + +/** + * Set the condition of a branch instruction. + * + * This only works on llvm::BranchInst instructions. + * + * @see llvm::BranchInst::setCondition + */ +void LLVMSetCondition(LLVMValueRef Branch, LLVMValueRef Cond); + /** * Obtain the default destination basic block of a switch instruction. * @@ -2445,6 +2563,10 @@ void LLVMSetTailCall(LLVMValueRef CallInst, LLVMBool IsTailCall); */ LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef SwitchInstr); +/** + * @} + */ + /** * @defgroup LLVMCCoreValueInstructionPHINode PHI Nodes * diff --git a/contrib/llvm/include/llvm-c/Disassembler.h b/contrib/llvm/include/llvm-c/Disassembler.h index 8f31150ad91d..d6cbe31cbb4e 100644 --- a/contrib/llvm/include/llvm-c/Disassembler.h +++ b/contrib/llvm/include/llvm-c/Disassembler.h @@ -174,8 +174,8 @@ extern "C" { * by passing a block of information in the DisInfo parameter and specifying the * TagType and callback functions as described above. These can all be passed * as NULL. If successful, this returns a disassembler context. If not, it - * returns NULL. This function is equivalent to calling LLVMCreateDisasmCPU() - * with an empty CPU name. + * returns NULL. This function is equivalent to calling + * LLVMCreateDisasmCPUFeatures() with an empty CPU name and feature set. */ LLVMDisasmContextRef LLVMCreateDisasm(const char *TripleName, void *DisInfo, int TagType, LLVMOpInfoCallback GetOpInfo, @@ -186,13 +186,27 @@ LLVMDisasmContextRef LLVMCreateDisasm(const char *TripleName, void *DisInfo, * disassembly is supported by passing a block of information in the DisInfo * parameter and specifying the TagType and callback functions as described * above. These can all be passed * as NULL. If successful, this returns a - * disassembler context. If not, it returns NULL. + * disassembler context. If not, it returns NULL. This function is equivalent + * to calling LLVMCreateDisasmCPUFeatures() with an empty feature set. */ LLVMDisasmContextRef LLVMCreateDisasmCPU(const char *Triple, const char *CPU, void *DisInfo, int TagType, LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp); +/** + * Create a disassembler for the TripleName, a specific CPU and specific feature + * string. Symbolic disassembly is supported by passing a block of information + * in the DisInfo parameter and specifying the TagType and callback functions as + * described above. These can all be passed * as NULL. If successful, this + * returns a disassembler context. If not, it returns NULL. + */ +LLVMDisasmContextRef +LLVMCreateDisasmCPUFeatures(const char *Triple, const char *CPU, + const char *Features, void *DisInfo, int TagType, + LLVMOpInfoCallback GetOpInfo, + LLVMSymbolLookupCallback SymbolLookUp); + /** * Set the disassembler's options. Returns 1 if it can set the Options and 0 * otherwise. diff --git a/contrib/llvm/include/llvm-c/ExecutionEngine.h b/contrib/llvm/include/llvm-c/ExecutionEngine.h index 7cdf0d78d5b6..eb3ecabfa8a8 100644 --- a/contrib/llvm/include/llvm-c/ExecutionEngine.h +++ b/contrib/llvm/include/llvm-c/ExecutionEngine.h @@ -34,7 +34,6 @@ extern "C" { * @{ */ -void LLVMLinkInJIT(void); void LLVMLinkInMCJIT(void); void LLVMLinkInInterpreter(void); @@ -171,6 +170,10 @@ void LLVMAddGlobalMapping(LLVMExecutionEngineRef EE, LLVMValueRef Global, void *LLVMGetPointerToGlobal(LLVMExecutionEngineRef EE, LLVMValueRef Global); +uint64_t LLVMGetGlobalValueAddress(LLVMExecutionEngineRef EE, const char *Name); + +uint64_t LLVMGetFunctionAddress(LLVMExecutionEngineRef EE, const char *Name); + /*===-- Operations on memory managers -------------------------------------===*/ typedef uint8_t *(*LLVMMemoryManagerAllocateCodeSectionCallback)( diff --git a/contrib/llvm/include/llvm-c/Initialization.h b/contrib/llvm/include/llvm-c/Initialization.h index ada473818da6..44194f8ea311 100644 --- a/contrib/llvm/include/llvm-c/Initialization.h +++ b/contrib/llvm/include/llvm-c/Initialization.h @@ -13,8 +13,8 @@ |* *| \*===----------------------------------------------------------------------===*/ -#ifndef LLVM_C_INITIALIZEPASSES_H -#define LLVM_C_INITIALIZEPASSES_H +#ifndef LLVM_C_INITIALIZATION_H +#define LLVM_C_INITIALIZATION_H #include "llvm-c/Core.h" diff --git a/contrib/llvm/include/llvm-c/Linker.h b/contrib/llvm/include/llvm-c/Linker.h index 9f337cfba477..a932c6d0f078 100644 --- a/contrib/llvm/include/llvm-c/Linker.h +++ b/contrib/llvm/include/llvm-c/Linker.h @@ -21,6 +21,7 @@ extern "C" { #endif +/* Note: LLVMLinkerPreserveSource has no effect. */ typedef enum { LLVMLinkerDestroySource = 0, /* Allow source module to be destroyed. */ LLVMLinkerPreserveSource = 1 /* Preserve the source module. */ diff --git a/contrib/llvm/include/llvm-c/Support.h b/contrib/llvm/include/llvm-c/Support.h index 4e6ff220b100..a9216d0364ad 100644 --- a/contrib/llvm/include/llvm-c/Support.h +++ b/contrib/llvm/include/llvm-c/Support.h @@ -47,6 +47,17 @@ typedef struct LLVMOpaqueMemoryBuffer *LLVMMemoryBufferRef; */ LLVMBool LLVMLoadLibraryPermanently(const char* Filename); +/** + * This function parses the given arguments using the LLVM command line parser. + * Note that the only stable thing about this function is its signature; you + * cannot rely on any particular set of command line arguments being interpreted + * the same way across LLVM versions. + * + * @see llvm::cl::ParseCommandLineOptions() + */ +void LLVMParseCommandLineOptions(int argc, const char *const *argv, + const char *Overview); + #ifdef __cplusplus } #endif diff --git a/contrib/llvm/include/llvm-c/Transforms/Scalar.h b/contrib/llvm/include/llvm-c/Transforms/Scalar.h index 0ca72cec0cd2..7ad1ad1d0564 100644 --- a/contrib/llvm/include/llvm-c/Transforms/Scalar.h +++ b/contrib/llvm/include/llvm-c/Transforms/Scalar.h @@ -35,6 +35,9 @@ extern "C" { /** See llvm::createAggressiveDCEPass function. */ void LLVMAddAggressiveDCEPass(LLVMPassManagerRef PM); +/** See llvm::createAlignmentFromAssumptionsPass function. */ +void LLVMAddAlignmentFromAssumptionsPass(LLVMPassManagerRef PM); + /** See llvm::createCFGSimplificationPass function. */ void LLVMAddCFGSimplificationPass(LLVMPassManagerRef PM); @@ -86,6 +89,9 @@ void LLVMAddMemCpyOptPass(LLVMPassManagerRef PM); /** See llvm::createPartiallyInlineLibCallsPass function. */ void LLVMAddPartiallyInlineLibCallsPass(LLVMPassManagerRef PM); +/** See llvm::createLowerSwitchPass function. */ +void LLVMAddLowerSwitchPass(LLVMPassManagerRef PM); + /** See llvm::createPromoteMemoryToRegisterPass function. */ void LLVMAddPromoteMemoryToRegisterPass(LLVMPassManagerRef PM); @@ -132,6 +138,9 @@ void LLVMAddLowerExpectIntrinsicPass(LLVMPassManagerRef PM); /** See llvm::createTypeBasedAliasAnalysisPass function */ void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM); +/** See llvm::createScopedNoAliasAAPass function */ +void LLVMAddScopedNoAliasAAPass(LLVMPassManagerRef PM); + /** See llvm::createBasicAliasAnalysisPass function */ void LLVMAddBasicAliasAnalysisPass(LLVMPassManagerRef PM); diff --git a/contrib/llvm/include/llvm-c/lto.h b/contrib/llvm/include/llvm-c/lto.h index c525710ae1a1..3f30d6d535d2 100644 --- a/contrib/llvm/include/llvm-c/lto.h +++ b/contrib/llvm/include/llvm-c/lto.h @@ -40,7 +40,7 @@ typedef bool lto_bool_t; * @{ */ -#define LTO_API_VERSION 10 +#define LTO_API_VERSION 11 /** * \since prior to LTO_API_VERSION=3 @@ -177,6 +177,35 @@ extern lto_module_t lto_module_create_from_memory_with_path(const void* mem, size_t length, const char *path); +/** + * \brief Loads an object file in its own context. + * + * Loads an object file in its own LLVMContext. This function call is + * thread-safe. However, modules created this way should not be merged into an + * lto_code_gen_t using \a lto_codegen_add_module(). + * + * Returns NULL on error (check lto_get_error_message() for details). + * + * \since LTO_API_VERSION=11 + */ +extern lto_module_t +lto_module_create_in_local_context(const void *mem, size_t length, + const char *path); + +/** + * \brief Loads an object file in the codegen context. + * + * Loads an object file into the same context as \c cg. The module is safe to + * add using \a lto_codegen_add_module(). + * + * Returns NULL on error (check lto_get_error_message() for details). + * + * \since LTO_API_VERSION=11 + */ +extern lto_module_t +lto_module_create_in_codegen_context(const void *mem, size_t length, + const char *path, lto_code_gen_t cg); + /** * Loads an object file from disk. The seek point of fd is not preserved. * Returns NULL on error (check lto_get_error_message() for details). @@ -324,11 +353,26 @@ extern void lto_codegen_set_diagnostic_handler(lto_code_gen_t, * Instantiates a code generator. * Returns NULL on error (check lto_get_error_message() for details). * + * All modules added using \a lto_codegen_add_module() must have been created + * in the same context as the codegen. + * * \since prior to LTO_API_VERSION=3 */ extern lto_code_gen_t lto_codegen_create(void); +/** + * \brief Instantiate a code generator in its own context. + * + * Instantiates a code generator in its own context. Modules added via \a + * lto_codegen_add_module() must have all been created in the same context, + * using \a lto_module_create_in_codegen_context(). + * + * \since LTO_API_VERSION=11 + */ +extern lto_code_gen_t +lto_codegen_create_in_local_context(void); + /** * Frees all code generator and all memory it internally allocated. * Upon return the lto_code_gen_t is no longer valid. @@ -342,6 +386,10 @@ lto_codegen_dispose(lto_code_gen_t); * Add an object module to the set of modules for which code will be generated. * Returns true on error (check lto_get_error_message() for details). * + * \c cg and \c mod must both be in the same context. See \a + * lto_codegen_create_in_local_context() and \a + * lto_module_create_in_codegen_context(). + * * \since prior to LTO_API_VERSION=3 */ extern lto_bool_t diff --git a/contrib/llvm/include/llvm/ADT/APFloat.h b/contrib/llvm/include/llvm/ADT/APFloat.h index 50f1463d7eaa..26aae773624c 100644 --- a/contrib/llvm/include/llvm/ADT/APFloat.h +++ b/contrib/llvm/include/llvm/ADT/APFloat.h @@ -304,6 +304,38 @@ public: /// IEEE-754R 5.3.1: nextUp/nextDown. opStatus next(bool nextDown); + /// \brief Operator+ overload which provides the default + /// \c nmNearestTiesToEven rounding mode and *no* error checking. + APFloat operator+(const APFloat &RHS) const { + APFloat Result = *this; + Result.add(RHS, rmNearestTiesToEven); + return Result; + } + + /// \brief Operator- overload which provides the default + /// \c nmNearestTiesToEven rounding mode and *no* error checking. + APFloat operator-(const APFloat &RHS) const { + APFloat Result = *this; + Result.subtract(RHS, rmNearestTiesToEven); + return Result; + } + + /// \brief Operator* overload which provides the default + /// \c nmNearestTiesToEven rounding mode and *no* error checking. + APFloat operator*(const APFloat &RHS) const { + APFloat Result = *this; + Result.multiply(RHS, rmNearestTiesToEven); + return Result; + } + + /// \brief Operator/ overload which provides the default + /// \c nmNearestTiesToEven rounding mode and *no* error checking. + APFloat operator/(const APFloat &RHS) const { + APFloat Result = *this; + Result.divide(RHS, rmNearestTiesToEven); + return Result; + } + /// @} /// \name Sign operations. @@ -313,6 +345,13 @@ public: void clearSign(); void copySign(const APFloat &); + /// \brief A static helper to produce a copy of an APFloat value with its sign + /// copied from some other APFloat. + static APFloat copySign(APFloat Value, const APFloat &Sign) { + Value.copySign(Sign); + return std::move(Value); + } + /// @} /// \name Conversions @@ -452,6 +491,36 @@ public: /// return true. bool getExactInverse(APFloat *inv) const; + /// \brief Enumeration of \c ilogb error results. + enum IlogbErrorKinds { + IEK_Zero = INT_MIN+1, + IEK_NaN = INT_MIN, + IEK_Inf = INT_MAX + }; + + /// \brief Returns the exponent of the internal representation of the APFloat. + /// + /// Because the radix of APFloat is 2, this is equivalent to floor(log2(x)). + /// For special APFloat values, this returns special error codes: + /// + /// NaN -> \c IEK_NaN + /// 0 -> \c IEK_Zero + /// Inf -> \c IEK_Inf + /// + friend int ilogb(const APFloat &Arg) { + if (Arg.isNaN()) + return IEK_NaN; + if (Arg.isZero()) + return IEK_Zero; + if (Arg.isInfinity()) + return IEK_Inf; + + return Arg.exponent; + } + + /// \brief Returns: X * 2^Exp for integral exponents. + friend APFloat scalbn(APFloat X, int Exp); + private: /// \name Simple Queries @@ -573,11 +642,41 @@ private: unsigned int sign : 1; }; -/// See friend declaration above. +/// See friend declarations above. /// -/// This additional declaration is required in order to compile LLVM with IBM +/// These additional declarations are required in order to compile LLVM with IBM /// xlC compiler. hash_code hash_value(const APFloat &Arg); +APFloat scalbn(APFloat X, int Exp); + +/// \brief Returns the absolute value of the argument. +inline APFloat abs(APFloat X) { + X.clearSign(); + return X; +} + +/// Implements IEEE minNum semantics. Returns the smaller of the 2 arguments if +/// both are not NaN. If either argument is a NaN, returns the other argument. +LLVM_READONLY +inline APFloat minnum(const APFloat &A, const APFloat &B) { + if (A.isNaN()) + return B; + if (B.isNaN()) + return A; + return (B.compare(A) == APFloat::cmpLessThan) ? B : A; +} + +/// Implements IEEE maxNum semantics. Returns the larger of the 2 arguments if +/// both are not NaN. If either argument is a NaN, returns the other argument. +LLVM_READONLY +inline APFloat maxnum(const APFloat &A, const APFloat &B) { + if (A.isNaN()) + return B; + if (B.isNaN()) + return A; + return (A.compare(B) == APFloat::cmpLessThan) ? B : A; +} + } // namespace llvm #endif // LLVM_ADT_APFLOAT_H diff --git a/contrib/llvm/include/llvm/ADT/APInt.h b/contrib/llvm/include/llvm/ADT/APInt.h index aa3c3f67ec10..025397d9ce45 100644 --- a/contrib/llvm/include/llvm/ADT/APInt.h +++ b/contrib/llvm/include/llvm/ADT/APInt.h @@ -91,6 +91,8 @@ class APInt { APINT_WORD_SIZE = static_cast(sizeof(uint64_t)) }; + friend struct DenseMapAPIntKeyInfo; + /// \brief Fast internal constructor /// /// This constructor is used only internally for speed of construction of @@ -277,7 +279,6 @@ public: /// Simply makes *this a copy of that. /// @brief Copy Constructor. APInt(const APInt &that) : BitWidth(that.BitWidth), VAL(0) { - assert(BitWidth && "bitwidth too small"); if (isSingleWord()) VAL = that.VAL; else @@ -656,13 +657,24 @@ public: /// @brief Move assignment operator. APInt &operator=(APInt &&that) { - if (!isSingleWord()) + if (!isSingleWord()) { + // The MSVC STL shipped in 2013 requires that self move assignment be a + // no-op. Otherwise algorithms like stable_sort will produce answers + // where half of the output is left in a moved-from state. + if (this == &that) + return *this; delete[] pVal; + } - BitWidth = that.BitWidth; - VAL = that.VAL; + // Use memcpy so that type based alias analysis sees both VAL and pVal + // as modified. + memcpy(&VAL, &that.VAL, sizeof(uint64_t)); + // If 'this == &that', avoid zeroing our own bitwidth by storing to 'that' + // first. + unsigned ThatBitWidth = that.BitWidth; that.BitWidth = 0; + BitWidth = ThatBitWidth; return *this; } @@ -936,7 +948,8 @@ public: APInt sdiv_ov(const APInt &RHS, bool &Overflow) const; APInt smul_ov(const APInt &RHS, bool &Overflow) const; APInt umul_ov(const APInt &RHS, bool &Overflow) const; - APInt sshl_ov(unsigned Amt, bool &Overflow) const; + APInt sshl_ov(const APInt &Amt, bool &Overflow) const; + APInt ushl_ov(const APInt &Amt, bool &Overflow) const; /// \brief Array-indexing support. /// diff --git a/contrib/llvm/include/llvm/ADT/APSInt.h b/contrib/llvm/include/llvm/ADT/APSInt.h index ee34e9b53088..a6693f7992cd 100644 --- a/contrib/llvm/include/llvm/ADT/APSInt.h +++ b/contrib/llvm/include/llvm/ADT/APSInt.h @@ -269,19 +269,15 @@ public: else if (I2.getBitWidth() > I1.getBitWidth()) return isSameValue(I1.extend(I2.getBitWidth()), I2); - // We have a signedness mismatch. Turn the signed value into an unsigned - // value. - if (I1.isSigned()) { - if (I1.isNegative()) - return false; + assert(I1.isSigned() != I2.isSigned()); - return APSInt(I1, true) == I2; - } - - if (I2.isNegative()) + // We have a signedness mismatch. Check for negative values and do an + // unsigned compare if signs match. + if ((I1.isSigned() && I1.isNegative()) || + (!I1.isSigned() && I2.isNegative())) return false; - return I1 == APSInt(I2, true); + return I1.eq(I2); } /// Profile - Used to insert APSInt objects, or objects that contain APSInt diff --git a/contrib/llvm/include/llvm/ADT/ArrayRef.h b/contrib/llvm/include/llvm/ADT/ArrayRef.h index 0fff505d8d01..8c14a423c8f5 100644 --- a/contrib/llvm/include/llvm/ADT/ArrayRef.h +++ b/contrib/llvm/include/llvm/ADT/ArrayRef.h @@ -11,6 +11,7 @@ #define LLVM_ADT_ARRAYREF_H #include "llvm/ADT/None.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include @@ -43,6 +44,19 @@ namespace llvm { /// The number of elements. size_type Length; + /// \brief A dummy "optional" type that is only created by implicit + /// conversion from a reference to T. + /// + /// This type must *only* be used in a function argument or as a copy of + /// a function argument, as otherwise it will hold a pointer to a temporary + /// past that temporaries' lifetime. + struct TRefOrNothing { + const T *TPtr; + + TRefOrNothing() : TPtr(nullptr) {} + TRefOrNothing(const T &TRef) : TPtr(&TRef) {} + }; + public: /// @name Constructors /// @{ @@ -90,6 +104,14 @@ namespace llvm { Length(Vec.size()) {} #endif + /// Construct an ArrayRef from ArrayRef. This uses SFINAE to + /// ensure that only ArrayRefs of pointers can be converted. + template + ArrayRef(const ArrayRef &A, + typename std::enable_if< + std::is_convertible::value>::type* = 0) + : Data(A.data()), Length(A.size()) {} + /// @} /// @name Simple Operations /// @{ @@ -131,7 +153,13 @@ namespace llvm { bool equals(ArrayRef RHS) const { if (Length != RHS.Length) return false; - return std::equal(begin(), end(), RHS.begin()); + // Don't use std::equal(), since it asserts in MSVC on nullptr iterators. + for (auto L = begin(), LE = end(), R = RHS.begin(); L != LE; ++L, ++R) + // Match std::equal() in using == (instead of !=) to minimize API + // requirements of ArrayRef'ed types. + if (!(*L == *R)) + return false; + return true; } /// slice(n) - Chop off the first N elements of the array. @@ -175,6 +203,47 @@ namespace llvm { return std::vector(Data, Data+Length); } + /// @} + /// @{ + /// @name Convenience methods + + /// @brief Predicate for testing that the array equals the exact sequence of + /// arguments. + /// + /// Will return false if the size is not equal to the exact number of + /// arguments given or if the array elements don't equal the argument + /// elements in order. Currently supports up to 16 arguments, but can + /// easily be extended. + bool equals(TRefOrNothing Arg0 = TRefOrNothing(), + TRefOrNothing Arg1 = TRefOrNothing(), + TRefOrNothing Arg2 = TRefOrNothing(), + TRefOrNothing Arg3 = TRefOrNothing(), + TRefOrNothing Arg4 = TRefOrNothing(), + TRefOrNothing Arg5 = TRefOrNothing(), + TRefOrNothing Arg6 = TRefOrNothing(), + TRefOrNothing Arg7 = TRefOrNothing(), + TRefOrNothing Arg8 = TRefOrNothing(), + TRefOrNothing Arg9 = TRefOrNothing(), + TRefOrNothing Arg10 = TRefOrNothing(), + TRefOrNothing Arg11 = TRefOrNothing(), + TRefOrNothing Arg12 = TRefOrNothing(), + TRefOrNothing Arg13 = TRefOrNothing(), + TRefOrNothing Arg14 = TRefOrNothing(), + TRefOrNothing Arg15 = TRefOrNothing()) { + TRefOrNothing Args[] = {Arg0, Arg1, Arg2, Arg3, Arg4, Arg5, + Arg6, Arg7, Arg8, Arg9, Arg10, Arg11, + Arg12, Arg13, Arg14, Arg15}; + if (size() > array_lengthof(Args)) + return false; + + for (unsigned i = 0, e = size(); i != e; ++i) + if (Args[i].TPtr == nullptr || (*this)[i] != *Args[i].TPtr) + return false; + + // Either the size is exactly as many args, or the next arg must be null. + return size() == array_lengthof(Args) || Args[size()].TPtr == nullptr; + } + /// @} }; diff --git a/contrib/llvm/include/llvm/ADT/BitVector.h b/contrib/llvm/include/llvm/ADT/BitVector.h index 34e2284311b3..a40f694485bf 100644 --- a/contrib/llvm/include/llvm/ADT/BitVector.h +++ b/contrib/llvm/include/llvm/ADT/BitVector.h @@ -239,6 +239,7 @@ public: } BitVector &set(unsigned Idx) { + assert(Bits && "Bits never allocated"); Bits[Idx / BITWORD_SIZE] |= BitWord(1) << (Idx % BITWORD_SIZE); return *this; } @@ -450,6 +451,7 @@ public: // Grow the bitvector to have enough elements. Capacity = RHSWords; + assert(Capacity > 0 && "negative capacity?"); BitWord *NewBits = (BitWord *)std::malloc(Capacity * sizeof(BitWord)); std::memcpy(NewBits, RHS.Bits, Capacity * sizeof(BitWord)); @@ -545,6 +547,7 @@ private: void grow(unsigned NewSize) { Capacity = std::max(NumBitWords(NewSize), Capacity * 2); + assert(Capacity > 0 && "realloc-ing zero space"); Bits = (BitWord *)std::realloc(Bits, Capacity * sizeof(BitWord)); clear_unused_bits(); diff --git a/contrib/llvm/include/llvm/ADT/DenseMap.h b/contrib/llvm/include/llvm/ADT/DenseMap.h index 85f37b9051b1..050f8ac150dd 100644 --- a/contrib/llvm/include/llvm/ADT/DenseMap.h +++ b/contrib/llvm/include/llvm/ADT/DenseMap.h @@ -31,26 +31,35 @@ namespace llvm { -template, - bool IsConst = false> +namespace detail { +// We extend a pair to allow users to override the bucket type with their own +// implementation without requiring two members. +template +struct DenseMapPair : public std::pair { + KeyT &getFirst() { return std::pair::first; } + const KeyT &getFirst() const { return std::pair::first; } + ValueT &getSecond() { return std::pair::second; } + const ValueT &getSecond() const { return std::pair::second; } +}; +} + +template < + typename KeyT, typename ValueT, typename KeyInfoT = DenseMapInfo, + typename Bucket = detail::DenseMapPair, bool IsConst = false> class DenseMapIterator; -template +template class DenseMapBase { -protected: - typedef std::pair BucketT; - public: typedef unsigned size_type; typedef KeyT key_type; typedef ValueT mapped_type; typedef BucketT value_type; - typedef DenseMapIterator iterator; - typedef DenseMapIterator const_iterator; + typedef DenseMapIterator iterator; + typedef DenseMapIterator + const_iterator; inline iterator begin() { // When the map is empty, avoid the overhead of AdvancePastEmptyBuckets(). return empty() ? end() : iterator(getBuckets(), getBucketsEnd()); @@ -88,12 +97,12 @@ public: const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { - if (!KeyInfoT::isEqual(P->first, EmptyKey)) { - if (!KeyInfoT::isEqual(P->first, TombstoneKey)) { - P->second.~ValueT(); + if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { + if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { + P->getSecond().~ValueT(); decrementNumEntries(); } - P->first = EmptyKey; + P->getFirst() = EmptyKey; } } assert(getNumEntries() == 0 && "Node count imbalance!"); @@ -144,7 +153,7 @@ public: ValueT lookup(const KeyT &Val) const { const BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return TheBucket->second; + return TheBucket->getSecond(); return ValueT(); } @@ -191,16 +200,16 @@ public: if (!LookupBucketFor(Val, TheBucket)) return false; // not in map. - TheBucket->second.~ValueT(); - TheBucket->first = getTombstoneKey(); + TheBucket->getSecond().~ValueT(); + TheBucket->getFirst() = getTombstoneKey(); decrementNumEntries(); incrementNumTombstones(); return true; } void erase(iterator I) { BucketT *TheBucket = &*I; - TheBucket->second.~ValueT(); - TheBucket->first = getTombstoneKey(); + TheBucket->getSecond().~ValueT(); + TheBucket->getFirst() = getTombstoneKey(); decrementNumEntries(); incrementNumTombstones(); } @@ -250,10 +259,10 @@ protected: const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { - if (!KeyInfoT::isEqual(P->first, EmptyKey) && - !KeyInfoT::isEqual(P->first, TombstoneKey)) - P->second.~ValueT(); - P->first.~KeyT(); + if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && + !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) + P->getSecond().~ValueT(); + P->getFirst().~KeyT(); } #ifndef NDEBUG @@ -269,7 +278,7 @@ protected: "# initial buckets must be a power of two!"); const KeyT EmptyKey = getEmptyKey(); for (BucketT *B = getBuckets(), *E = getBucketsEnd(); B != E; ++B) - new (&B->first) KeyT(EmptyKey); + new (&B->getFirst()) KeyT(EmptyKey); } void moveFromOldBuckets(BucketT *OldBucketsBegin, BucketT *OldBucketsEnd) { @@ -279,21 +288,21 @@ protected: const KeyT EmptyKey = getEmptyKey(); const KeyT TombstoneKey = getTombstoneKey(); for (BucketT *B = OldBucketsBegin, *E = OldBucketsEnd; B != E; ++B) { - if (!KeyInfoT::isEqual(B->first, EmptyKey) && - !KeyInfoT::isEqual(B->first, TombstoneKey)) { + if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) && + !KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) { // Insert the key/value into the new table. BucketT *DestBucket; - bool FoundVal = LookupBucketFor(B->first, DestBucket); + bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket); (void)FoundVal; // silence warning. assert(!FoundVal && "Key already in new map?"); - DestBucket->first = std::move(B->first); - new (&DestBucket->second) ValueT(std::move(B->second)); + DestBucket->getFirst() = std::move(B->getFirst()); + new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond())); incrementNumEntries(); // Free the value. - B->second.~ValueT(); + B->getSecond().~ValueT(); } - B->first.~KeyT(); + B->getFirst().~KeyT(); } #ifndef NDEBUG @@ -304,7 +313,9 @@ protected: } template - void copyFrom(const DenseMapBase& other) { + void copyFrom( + const DenseMapBase &other) { + assert(&other != this); assert(getNumBuckets() == other.getNumBuckets()); setNumEntries(other.getNumEntries()); @@ -315,10 +326,12 @@ protected: getNumBuckets() * sizeof(BucketT)); else for (size_t i = 0; i < getNumBuckets(); ++i) { - new (&getBuckets()[i].first) KeyT(other.getBuckets()[i].first); - if (!KeyInfoT::isEqual(getBuckets()[i].first, getEmptyKey()) && - !KeyInfoT::isEqual(getBuckets()[i].first, getTombstoneKey())) - new (&getBuckets()[i].second) ValueT(other.getBuckets()[i].second); + new (&getBuckets()[i].getFirst()) + KeyT(other.getBuckets()[i].getFirst()); + if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) && + !KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey())) + new (&getBuckets()[i].getSecond()) + ValueT(other.getBuckets()[i].getSecond()); } } @@ -395,8 +408,8 @@ private: BucketT *TheBucket) { TheBucket = InsertIntoBucketImpl(Key, TheBucket); - TheBucket->first = Key; - new (&TheBucket->second) ValueT(Value); + TheBucket->getFirst() = Key; + new (&TheBucket->getSecond()) ValueT(Value); return TheBucket; } @@ -404,16 +417,16 @@ private: BucketT *TheBucket) { TheBucket = InsertIntoBucketImpl(Key, TheBucket); - TheBucket->first = Key; - new (&TheBucket->second) ValueT(std::move(Value)); + TheBucket->getFirst() = Key; + new (&TheBucket->getSecond()) ValueT(std::move(Value)); return TheBucket; } BucketT *InsertIntoBucket(KeyT &&Key, ValueT &&Value, BucketT *TheBucket) { TheBucket = InsertIntoBucketImpl(Key, TheBucket); - TheBucket->first = std::move(Key); - new (&TheBucket->second) ValueT(std::move(Value)); + TheBucket->getFirst() = std::move(Key); + new (&TheBucket->getSecond()) ValueT(std::move(Value)); return TheBucket; } @@ -445,7 +458,7 @@ private: // If we are writing over a tombstone, remember this. const KeyT EmptyKey = getEmptyKey(); - if (!KeyInfoT::isEqual(TheBucket->first, EmptyKey)) + if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey)) decrementNumTombstones(); return TheBucket; @@ -479,14 +492,14 @@ private: while (1) { const BucketT *ThisBucket = BucketsPtr + BucketNo; // Found Val's bucket? If so, return it. - if (KeyInfoT::isEqual(Val, ThisBucket->first)) { + if (KeyInfoT::isEqual(Val, ThisBucket->getFirst())) { FoundBucket = ThisBucket; return true; } // If we found an empty bucket, the key doesn't exist in the set. // Insert it and return the default value. - if (KeyInfoT::isEqual(ThisBucket->first, EmptyKey)) { + if (KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey)) { // If we've already seen a tombstone while probing, fill it in instead // of the empty bucket we eventually probed to. FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket; @@ -495,7 +508,8 @@ private: // If this is a tombstone, remember it. If Val ends up not in the map, we // prefer to return it than something that would require more probing. - if (KeyInfoT::isEqual(ThisBucket->first, TombstoneKey) && !FoundTombstone) + if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) && + !FoundTombstone) FoundTombstone = ThisBucket; // Remember the first tombstone found. // Otherwise, it's a hash collision or a tombstone, continue quadratic @@ -524,16 +538,15 @@ public: } }; -template > -class DenseMap - : public DenseMapBase, - KeyT, ValueT, KeyInfoT> { +template , + typename BucketT = detail::DenseMapPair> +class DenseMap : public DenseMapBase, + KeyT, ValueT, KeyInfoT, BucketT> { // Lift some types from the dependent base class into this class for // simplicity of referring to them. - typedef DenseMapBase BaseT; - typedef typename BaseT::BucketT BucketT; - friend class DenseMapBase; + typedef DenseMapBase BaseT; + friend class DenseMapBase; BucketT *Buckets; unsigned NumEntries; @@ -574,7 +587,8 @@ public: } DenseMap& operator=(const DenseMap& other) { - copyFrom(other); + if (&other != this) + copyFrom(other); return *this; } @@ -675,17 +689,17 @@ private: } }; -template > +template , + typename BucketT = detail::DenseMapPair> class SmallDenseMap - : public DenseMapBase, - KeyT, ValueT, KeyInfoT> { + : public DenseMapBase< + SmallDenseMap, KeyT, + ValueT, KeyInfoT, BucketT> { // Lift some types from the dependent base class into this class for // simplicity of referring to them. - typedef DenseMapBase BaseT; - typedef typename BaseT::BucketT BucketT; - friend class DenseMapBase; + typedef DenseMapBase BaseT; + friend class DenseMapBase; unsigned Small : 1; unsigned NumEntries : 31; @@ -742,23 +756,23 @@ public: for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { BucketT *LHSB = &getInlineBuckets()[i], *RHSB = &RHS.getInlineBuckets()[i]; - bool hasLHSValue = (!KeyInfoT::isEqual(LHSB->first, EmptyKey) && - !KeyInfoT::isEqual(LHSB->first, TombstoneKey)); - bool hasRHSValue = (!KeyInfoT::isEqual(RHSB->first, EmptyKey) && - !KeyInfoT::isEqual(RHSB->first, TombstoneKey)); + bool hasLHSValue = (!KeyInfoT::isEqual(LHSB->getFirst(), EmptyKey) && + !KeyInfoT::isEqual(LHSB->getFirst(), TombstoneKey)); + bool hasRHSValue = (!KeyInfoT::isEqual(RHSB->getFirst(), EmptyKey) && + !KeyInfoT::isEqual(RHSB->getFirst(), TombstoneKey)); if (hasLHSValue && hasRHSValue) { // Swap together if we can... std::swap(*LHSB, *RHSB); continue; } // Swap separately and handle any assymetry. - std::swap(LHSB->first, RHSB->first); + std::swap(LHSB->getFirst(), RHSB->getFirst()); if (hasLHSValue) { - new (&RHSB->second) ValueT(std::move(LHSB->second)); - LHSB->second.~ValueT(); + new (&RHSB->getSecond()) ValueT(std::move(LHSB->getSecond())); + LHSB->getSecond().~ValueT(); } else if (hasRHSValue) { - new (&LHSB->second) ValueT(std::move(RHSB->second)); - RHSB->second.~ValueT(); + new (&LHSB->getSecond()) ValueT(std::move(RHSB->getSecond())); + RHSB->getSecond().~ValueT(); } } return; @@ -783,12 +797,12 @@ public: for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { BucketT *NewB = &LargeSide.getInlineBuckets()[i], *OldB = &SmallSide.getInlineBuckets()[i]; - new (&NewB->first) KeyT(std::move(OldB->first)); - OldB->first.~KeyT(); - if (!KeyInfoT::isEqual(NewB->first, EmptyKey) && - !KeyInfoT::isEqual(NewB->first, TombstoneKey)) { - new (&NewB->second) ValueT(std::move(OldB->second)); - OldB->second.~ValueT(); + new (&NewB->getFirst()) KeyT(std::move(OldB->getFirst())); + OldB->getFirst().~KeyT(); + if (!KeyInfoT::isEqual(NewB->getFirst(), EmptyKey) && + !KeyInfoT::isEqual(NewB->getFirst(), TombstoneKey)) { + new (&NewB->getSecond()) ValueT(std::move(OldB->getSecond())); + OldB->getSecond().~ValueT(); } } @@ -799,7 +813,8 @@ public: } SmallDenseMap& operator=(const SmallDenseMap& other) { - copyFrom(other); + if (&other != this) + copyFrom(other); return *this; } @@ -849,16 +864,16 @@ public: const KeyT EmptyKey = this->getEmptyKey(); const KeyT TombstoneKey = this->getTombstoneKey(); for (BucketT *P = getBuckets(), *E = P + InlineBuckets; P != E; ++P) { - if (!KeyInfoT::isEqual(P->first, EmptyKey) && - !KeyInfoT::isEqual(P->first, TombstoneKey)) { + if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && + !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { assert(size_t(TmpEnd - TmpBegin) < InlineBuckets && "Too many inline buckets!"); - new (&TmpEnd->first) KeyT(std::move(P->first)); - new (&TmpEnd->second) ValueT(std::move(P->second)); + new (&TmpEnd->getFirst()) KeyT(std::move(P->getFirst())); + new (&TmpEnd->getSecond()) ValueT(std::move(P->getSecond())); ++TmpEnd; - P->second.~ValueT(); + P->getSecond().~ValueT(); } - P->first.~KeyT(); + P->getFirst().~KeyT(); } // Now make this map use the large rep, and move all the entries back @@ -969,13 +984,12 @@ private: } }; -template +template class DenseMapIterator { - typedef std::pair Bucket; - typedef DenseMapIterator ConstIterator; - friend class DenseMapIterator; + typedef DenseMapIterator ConstIterator; + friend class DenseMapIterator; + public: typedef ptrdiff_t difference_type; typedef typename std::conditional::type @@ -996,9 +1010,9 @@ public: // If IsConst is true this is a converting constructor from iterator to // const_iterator and the default copy constructor is used. // Otherwise this is a copy constructor for iterator. - DenseMapIterator(const DenseMapIterator& I) - : Ptr(I.Ptr), End(I.End) {} + DenseMapIterator( + const DenseMapIterator &I) + : Ptr(I.Ptr), End(I.End) {} reference operator*() const { return *Ptr; @@ -1028,9 +1042,8 @@ private: const KeyT Empty = KeyInfoT::getEmptyKey(); const KeyT Tombstone = KeyInfoT::getTombstoneKey(); - while (Ptr != End && - (KeyInfoT::isEqual(Ptr->first, Empty) || - KeyInfoT::isEqual(Ptr->first, Tombstone))) + while (Ptr != End && (KeyInfoT::isEqual(Ptr->getFirst(), Empty) || + KeyInfoT::isEqual(Ptr->getFirst(), Tombstone))) ++Ptr; } }; diff --git a/contrib/llvm/include/llvm/ADT/DenseSet.h b/contrib/llvm/include/llvm/ADT/DenseSet.h index 37a81b0c7ee2..d34024005dfe 100644 --- a/contrib/llvm/include/llvm/ADT/DenseSet.h +++ b/contrib/llvm/include/llvm/ADT/DenseSet.h @@ -18,18 +18,34 @@ namespace llvm { +namespace detail { +struct DenseSetEmpty {}; + +// Use the empty base class trick so we can create a DenseMap where the buckets +// contain only a single item. +template class DenseSetPair : public DenseSetEmpty { + KeyT key; + +public: + KeyT &getFirst() { return key; } + const KeyT &getFirst() const { return key; } + DenseSetEmpty &getSecond() { return *this; } + const DenseSetEmpty &getSecond() const { return *this; } +}; +} + /// DenseSet - This implements a dense probed hash-table based set. -/// -/// FIXME: This is currently implemented directly in terms of DenseMap, this -/// should be optimized later if there is a need. template > class DenseSet { - typedef DenseMap MapTy; + typedef DenseMap> MapTy; + static_assert(sizeof(typename MapTy::value_type) == sizeof(ValueT), + "DenseMap buckets unexpectedly large!"); MapTy TheMap; public: typedef ValueT key_type; typedef ValueT value_type; - typedef unsigned size_type; + typedef unsigned size_type; explicit DenseSet(unsigned NumInitBuckets = 0) : TheMap(NumInitBuckets) {} @@ -45,7 +61,7 @@ public: TheMap.clear(); } - /// Return 1 if the specified key is in the set, 0 otherwise. + /// Return 1 if the specified key is in the set, 0 otherwise. size_type count(const ValueT &V) const { return TheMap.count(V); } @@ -72,8 +88,8 @@ public: Iterator(const typename MapTy::iterator &i) : I(i) {} - ValueT& operator*() { return I->first; } - ValueT* operator->() { return &I->first; } + ValueT &operator*() { return I->getFirst(); } + ValueT *operator->() { return &I->getFirst(); } Iterator& operator++() { ++I; return *this; } bool operator==(const Iterator& X) const { return I == X.I; } @@ -92,8 +108,8 @@ public: ConstIterator(const typename MapTy::const_iterator &i) : I(i) {} - const ValueT& operator*() { return I->first; } - const ValueT* operator->() { return &I->first; } + const ValueT &operator*() { return I->getFirst(); } + const ValueT *operator->() { return &I->getFirst(); } ConstIterator& operator++() { ++I; return *this; } bool operator==(const ConstIterator& X) const { return I == X.I; } @@ -110,11 +126,27 @@ public: const_iterator end() const { return ConstIterator(TheMap.end()); } iterator find(const ValueT &V) { return Iterator(TheMap.find(V)); } + + /// Alternative version of find() which allows a different, and possibly less + /// expensive, key type. + /// The DenseMapInfo is responsible for supplying methods + /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key type + /// used. + template + iterator find_as(const LookupKeyT &Val) { + return Iterator(TheMap.find_as(Val)); + } + template + const_iterator find_as(const LookupKeyT &Val) const { + return ConstIterator(TheMap.find_as(Val)); + } + void erase(Iterator I) { return TheMap.erase(I.I); } void erase(ConstIterator CI) { return TheMap.erase(CI.I); } std::pair insert(const ValueT &V) { - return TheMap.insert(std::make_pair(V, 0)); + detail::DenseSetEmpty Empty; + return TheMap.insert(std::make_pair(V, Empty)); } // Range insertion of values. diff --git a/contrib/llvm/include/llvm/ADT/DepthFirstIterator.h b/contrib/llvm/include/llvm/ADT/DepthFirstIterator.h index dfba43f3ac85..6cd9e68aea56 100644 --- a/contrib/llvm/include/llvm/ADT/DepthFirstIterator.h +++ b/contrib/llvm/include/llvm/ADT/DepthFirstIterator.h @@ -33,10 +33,10 @@ #ifndef LLVM_ADT_DEPTHFIRSTITERATOR_H #define LLVM_ADT_DEPTHFIRSTITERATOR_H -#include "llvm/ADT/iterator_range.h" #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/iterator_range.h" #include #include @@ -231,6 +231,13 @@ df_ext_iterator df_ext_end(const T& G, SetTy &S) { return df_ext_iterator::end(G, S); } +template +iterator_range> depth_first_ext(const T& G, + SetTy &S) { + return iterator_range>(df_ext_begin(G, S), + df_ext_end(G, S)); +} + // Provide global definitions of inverse depth first iterators... template idf_ext_end(const T& G, SetTy &S) { return idf_ext_iterator::end(Inverse(G), S); } +template +iterator_range> inverse_depth_first_ext(const T& G, + SetTy &S) { + return iterator_range>(idf_ext_begin(G, S), + idf_ext_end(G, S)); +} + } // End llvm namespace #endif diff --git a/contrib/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h b/contrib/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h index f9df3781257e..c859c98d06b2 100644 --- a/contrib/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h +++ b/contrib/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h @@ -197,6 +197,9 @@ public: private: void retain() { if (Obj) IntrusiveRefCntPtrInfo::retain(Obj); } void release() { if (Obj) IntrusiveRefCntPtrInfo::release(Obj); } + + template + friend class IntrusiveRefCntPtr; }; template diff --git a/contrib/llvm/include/llvm/ADT/MapVector.h b/contrib/llvm/include/llvm/ADT/MapVector.h index 4e1fc1527270..1331b15b2d29 100644 --- a/contrib/llvm/include/llvm/ADT/MapVector.h +++ b/contrib/llvm/include/llvm/ADT/MapVector.h @@ -18,6 +18,7 @@ #define LLVM_ADT_MAPVECTOR_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #include namespace llvm { @@ -37,26 +38,20 @@ class MapVector { public: typedef typename VectorType::iterator iterator; typedef typename VectorType::const_iterator const_iterator; + typedef typename VectorType::reverse_iterator reverse_iterator; + typedef typename VectorType::const_reverse_iterator const_reverse_iterator; - size_type size() const { - return Vector.size(); - } + size_type size() const { return Vector.size(); } - iterator begin() { - return Vector.begin(); - } + iterator begin() { return Vector.begin(); } + const_iterator begin() const { return Vector.begin(); } + iterator end() { return Vector.end(); } + const_iterator end() const { return Vector.end(); } - const_iterator begin() const { - return Vector.begin(); - } - - iterator end() { - return Vector.end(); - } - - const_iterator end() const { - return Vector.end(); - } + reverse_iterator rbegin() { return Vector.rbegin(); } + const_reverse_iterator rbegin() const { return Vector.rbegin(); } + reverse_iterator rend() { return Vector.rend(); } + const_reverse_iterator rend() const { return Vector.rend(); } bool empty() const { return Vector.empty(); @@ -147,6 +142,17 @@ public: return Next; } + /// \brief Remove all elements with the key value Key. + /// + /// Returns the number of elements removed. + size_type erase(const KeyT &Key) { + auto Iterator = find(Key); + if (Iterator == end()) + return 0; + erase(Iterator); + return 1; + } + /// \brief Remove the elements that match the predicate. /// /// Erase all elements that match \c Pred in a single pass. Takes linear @@ -176,6 +182,14 @@ void MapVector::remove_if(Function Pred) { Vector.erase(O, Vector.end()); } +/// \brief A MapVector that performs no allocations if smaller than a certain +/// size. +template +struct SmallMapVector + : MapVector, + SmallVector, N>> { +}; + } // end namespace llvm #endif diff --git a/contrib/llvm/include/llvm/ADT/Optional.h b/contrib/llvm/include/llvm/ADT/Optional.h index ae8344da76a6..591872e6591a 100644 --- a/contrib/llvm/include/llvm/ADT/Optional.h +++ b/contrib/llvm/include/llvm/ADT/Optional.h @@ -20,6 +20,7 @@ #include "llvm/Support/AlignOf.h" #include "llvm/Support/Compiler.h" #include +#include #include namespace llvm { @@ -29,6 +30,8 @@ class Optional { AlignedCharArrayUnion storage; bool hasVal; public: + typedef T value_type; + Optional(NoneType) : hasVal(false) {} explicit Optional() : hasVal(false) {} Optional(const T &y) : hasVal(true) { @@ -67,6 +70,61 @@ public: return *this; } +#if LLVM_HAS_VARIADIC_TEMPLATES + + /// Create a new object by constructing it in place with the given arguments. + template + void emplace(ArgTypes &&...Args) { + reset(); + hasVal = true; + new (storage.buffer) T(std::forward(Args)...); + } + +#else + + /// Create a new object by default-constructing it in place. + void emplace() { + reset(); + hasVal = true; + new (storage.buffer) T(); + } + + /// Create a new object by constructing it in place with the given arguments. + template + void emplace(T1 &&A1) { + reset(); + hasVal = true; + new (storage.buffer) T(std::forward(A1)); + } + + /// Create a new object by constructing it in place with the given arguments. + template + void emplace(T1 &&A1, T2 &&A2) { + reset(); + hasVal = true; + new (storage.buffer) T(std::forward(A1), std::forward(A2)); + } + + /// Create a new object by constructing it in place with the given arguments. + template + void emplace(T1 &&A1, T2 &&A2, T3 &&A3) { + reset(); + hasVal = true; + new (storage.buffer) T(std::forward(A1), std::forward(A2), + std::forward(A3)); + } + + /// Create a new object by constructing it in place with the given arguments. + template + void emplace(T1 &&A1, T2 &&A2, T3 &&A3, T4 &&A4) { + reset(); + hasVal = true; + new (storage.buffer) T(std::forward(A1), std::forward(A2), + std::forward(A3), std::forward(A4)); + } + +#endif // LLVM_HAS_VARIADIC_TEMPLATES + static inline Optional create(const T* y) { return y ? Optional(*y) : Optional(); } @@ -117,9 +175,19 @@ public: const T& operator*() const LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } T& operator*() LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } + template + LLVM_CONSTEXPR T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION { + return hasValue() ? getValue() : std::forward(value); + } + #if LLVM_HAS_RVALUE_REFERENCE_THIS T&& getValue() && { assert(hasVal); return std::move(*getPointer()); } T&& operator*() && { assert(hasVal); return std::move(*getPointer()); } + + template + T getValueOr(U &&value) && { + return hasValue() ? std::move(getValue()) : std::forward(value); + } #endif }; diff --git a/contrib/llvm/include/llvm/ADT/PostOrderIterator.h b/contrib/llvm/include/llvm/ADT/PostOrderIterator.h index dd8cc74b714e..dfadc3b85db6 100644 --- a/contrib/llvm/include/llvm/ADT/PostOrderIterator.h +++ b/contrib/llvm/include/llvm/ADT/PostOrderIterator.h @@ -57,7 +57,7 @@ public: // Return true if edge destination should be visited. template bool insertEdge(NodeType *From, NodeType *To) { - return Visited.insert(To); + return Visited.insert(To).second; } // Called after all children of BB have been visited. @@ -76,8 +76,9 @@ public: // Return true if edge destination should be visited, called with From = 0 for // the root node. // Graph edges can be pruned by specializing this function. - template - bool insertEdge(NodeType *From, NodeType *To) { return Visited.insert(To); } + template bool insertEdge(NodeType *From, NodeType *To) { + return Visited.insert(To).second; + } // Called after all children of BB have been visited. template diff --git a/contrib/llvm/include/llvm/ADT/STLExtras.h b/contrib/llvm/include/llvm/ADT/STLExtras.h index 1cef3933b5d6..4e56e4d74470 100644 --- a/contrib/llvm/include/llvm/ADT/STLExtras.h +++ b/contrib/llvm/include/llvm/ADT/STLExtras.h @@ -77,8 +77,11 @@ class function_ref { } public: - template - function_ref(Callable &&callable) + template + function_ref(Callable &&callable, + typename std::enable_if< + !std::is_same::type, + function_ref>::value>::type * = nullptr) : callback(callback_fn::type>), callable(reinterpret_cast(&callable)) {} Ret operator()(Params ...params) const { @@ -100,7 +103,10 @@ class function_ref { public: template - function_ref(Callable &&callable) + function_ref(Callable &&callable, + typename std::enable_if< + !std::is_same::type, + function_ref>::value>::type * = nullptr) : callback(callback_fn::type>), callable(reinterpret_cast(&callable)) {} Ret operator()() const { return callback(callable); } @@ -119,7 +125,10 @@ class function_ref { public: template - function_ref(Callable &&callable) + function_ref(Callable &&callable, + typename std::enable_if< + !std::is_same::type, + function_ref>::value>::type * = nullptr) : callback(callback_fn::type>), callable(reinterpret_cast(&callable)) {} Ret operator()(Param1 param1) { @@ -141,7 +150,10 @@ class function_ref { public: template - function_ref(Callable &&callable) + function_ref(Callable &&callable, + typename std::enable_if< + !std::is_same::type, + function_ref>::value>::type * = nullptr) : callback(callback_fn::type>), callable(reinterpret_cast(&callable)) {} Ret operator()(Param1 param1, Param2 param2) { @@ -167,7 +179,10 @@ class function_ref { public: template - function_ref(Callable &&callable) + function_ref(Callable &&callable, + typename std::enable_if< + !std::is_same::type, + function_ref>::value>::type * = nullptr) : callback(callback_fn::type>), callable(reinterpret_cast(&callable)) {} Ret operator()(Param1 param1, Param2 param2, Param3 param3) { @@ -530,6 +545,12 @@ make_unique(size_t n) { #endif +struct FreeDeleter { + void operator()(void* v) { + ::free(v); + } +}; + template struct pair_hash { size_t operator()(const std::pair &P) const { diff --git a/contrib/llvm/include/llvm/ADT/ScopedHashTable.h b/contrib/llvm/include/llvm/ADT/ScopedHashTable.h index 02a6ea345834..2f60ecc92043 100644 --- a/contrib/llvm/include/llvm/ADT/ScopedHashTable.h +++ b/contrib/llvm/include/llvm/ADT/ScopedHashTable.h @@ -148,7 +148,7 @@ public: /// ScopeTy - This is a helpful typedef that allows clients to get easy access /// to the name of the scope for this hash table. typedef ScopedHashTableScope ScopeTy; - typedef unsigned size_type; + typedef unsigned size_type; private: typedef ScopedHashTableVal ValTy; DenseMap TopLevelMap; @@ -171,7 +171,7 @@ public: AllocatorTy &getAllocator() { return Allocator; } const AllocatorTy &getAllocator() const { return Allocator; } - /// Return 1 if the specified key is in the table, 0 otherwise. + /// Return 1 if the specified key is in the table, 0 otherwise. size_type count(const K &Key) const { return TopLevelMap.count(Key); } diff --git a/contrib/llvm/include/llvm/ADT/SetVector.h b/contrib/llvm/include/llvm/ADT/SetVector.h index 1e7d237045aa..a7fd408c854a 100644 --- a/contrib/llvm/include/llvm/ADT/SetVector.h +++ b/contrib/llvm/include/llvm/ADT/SetVector.h @@ -100,7 +100,7 @@ public: /// \brief Insert a new element into the SetVector. /// \returns true iff the element was inserted into the SetVector. bool insert(const value_type &X) { - bool result = set_.insert(X); + bool result = set_.insert(X).second; if (result) vector_.push_back(X); return result; @@ -110,7 +110,7 @@ public: template void insert(It Start, It End) { for (; Start != End; ++Start) - if (set_.insert(*Start)) + if (set_.insert(*Start).second) vector_.push_back(*Start); } diff --git a/contrib/llvm/include/llvm/ADT/SmallBitVector.h b/contrib/llvm/include/llvm/ADT/SmallBitVector.h index 0922017ea61a..1e2f365b1040 100644 --- a/contrib/llvm/include/llvm/ADT/SmallBitVector.h +++ b/contrib/llvm/include/llvm/ADT/SmallBitVector.h @@ -54,7 +54,7 @@ class SmallBitVector { }; public: - typedef unsigned size_type; + typedef unsigned size_type; // Encapsulation of a single bit. class reference { SmallBitVector &TheVector; @@ -292,8 +292,12 @@ public: } SmallBitVector &set(unsigned Idx) { - if (isSmall()) + if (isSmall()) { + assert(Idx <= static_cast( + std::numeric_limits::digits) && + "undefined behavior"); setSmallBits(getSmallBits() | (uintptr_t(1) << Idx)); + } else getPointer()->set(Idx); return *this; diff --git a/contrib/llvm/include/llvm/ADT/SmallPtrSet.h b/contrib/llvm/include/llvm/ADT/SmallPtrSet.h index 74f3fd43cec4..cb1c5e1fa96a 100644 --- a/contrib/llvm/include/llvm/ADT/SmallPtrSet.h +++ b/contrib/llvm/include/llvm/ADT/SmallPtrSet.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace llvm { @@ -100,7 +101,7 @@ protected: /// insert_imp - This returns true if the pointer was new to the set, false if /// it was already in the set. This is hidden from the client so that the /// derived class can check that the right type of pointer is passed in. - bool insert_imp(const void * Ptr); + std::pair insert_imp(const void *Ptr); /// erase_imp - If the set contains the specified pointer, remove it and /// return true, otherwise return false. This is hidden from the client so @@ -240,6 +241,8 @@ struct RoundUpToPowerOfTwo { template class SmallPtrSetImpl : public SmallPtrSetImplBase { typedef PointerLikeTypeTraits PtrTraits; + + SmallPtrSetImpl(const SmallPtrSetImpl&) LLVM_DELETED_FUNCTION; protected: // Constructors that forward to the base. SmallPtrSetImpl(const void **SmallStorage, const SmallPtrSetImpl &that) @@ -251,10 +254,16 @@ protected: : SmallPtrSetImplBase(SmallStorage, SmallSize) {} public: - /// insert - This returns true if the pointer was new to the set, false if it - /// was already in the set. - bool insert(PtrType Ptr) { - return insert_imp(PtrTraits::getAsVoidPointer(Ptr)); + typedef SmallPtrSetIterator iterator; + typedef SmallPtrSetIterator const_iterator; + + /// Inserts Ptr if and only if there is no element in the container equal to + /// Ptr. The bool component of the returned pair is true if and only if the + /// insertion takes place, and the iterator component of the pair points to + /// the element equal to Ptr. + std::pair insert(PtrType Ptr) { + auto p = insert_imp(PtrTraits::getAsVoidPointer(Ptr)); + return std::make_pair(iterator(p.first, CurArray + CurArraySize), p.second); } /// erase - If the set contains the specified pointer, remove it and return @@ -274,8 +283,6 @@ public: insert(*I); } - typedef SmallPtrSetIterator iterator; - typedef SmallPtrSetIterator const_iterator; inline iterator begin() const { return iterator(CurArray, CurArray+CurArraySize); } diff --git a/contrib/llvm/include/llvm/ADT/SmallSet.h b/contrib/llvm/include/llvm/ADT/SmallSet.h index bb1971eb7c5d..bc6493554c8b 100644 --- a/contrib/llvm/include/llvm/ADT/SmallSet.h +++ b/contrib/llvm/include/llvm/ADT/SmallSet.h @@ -14,6 +14,7 @@ #ifndef LLVM_ADT_SMALLSET_H #define LLVM_ADT_SMALLSET_H +#include "llvm/ADT/None.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include @@ -60,16 +61,21 @@ public: /// insert - Insert an element into the set if it isn't already there. /// Returns true if the element is inserted (it was not in the set before). - bool insert(const T &V) { + /// The first value of the returned pair is unused and provided for + /// partial compatibility with the standard library self-associative container + /// concept. + // FIXME: Add iterators that abstract over the small and large form, and then + // return those here. + std::pair insert(const T &V) { if (!isSmall()) - return Set.insert(V).second; + return std::make_pair(None, Set.insert(V).second); VIterator I = vfind(V); if (I != Vector.end()) // Don't reinsert if it already exists. - return false; + return std::make_pair(None, false); if (Vector.size() < N) { Vector.push_back(V); - return true; + return std::make_pair(None, true); } // Otherwise, grow from vector to set. @@ -78,7 +84,7 @@ public: Vector.pop_back(); } Set.insert(V); - return true; + return std::make_pair(None, true); } template diff --git a/contrib/llvm/include/llvm/ADT/SmallVector.h b/contrib/llvm/include/llvm/ADT/SmallVector.h index 82538e9bd108..44a352119b09 100644 --- a/contrib/llvm/include/llvm/ADT/SmallVector.h +++ b/contrib/llvm/include/llvm/ADT/SmallVector.h @@ -29,8 +29,7 @@ namespace llvm { -/// SmallVectorBase - This is all the non-templated stuff common to all -/// SmallVectors. +/// This is all the non-templated stuff common to all SmallVectors. class SmallVectorBase { protected: void *BeginX, *EndX, *CapacityX; @@ -39,12 +38,12 @@ protected: SmallVectorBase(void *FirstEl, size_t Size) : BeginX(FirstEl), EndX(FirstEl), CapacityX((char*)FirstEl+Size) {} - /// grow_pod - This is an implementation of the grow() method which only works + /// This is an implementation of the grow() method which only works /// on POD-like data types and is out of line to reduce code duplication. void grow_pod(void *FirstEl, size_t MinSizeInBytes, size_t TSize); public: - /// size_in_bytes - This returns size()*sizeof(T). + /// This returns size()*sizeof(T). size_t size_in_bytes() const { return size_t((char*)EndX - (char*)BeginX); } @@ -59,10 +58,9 @@ public: template struct SmallVectorStorage; -/// SmallVectorTemplateCommon - This is the part of SmallVectorTemplateBase -/// which does not depend on whether the type T is a POD. The extra dummy -/// template argument is used by ArrayRef to avoid unnecessarily requiring T -/// to be complete. +/// This is the part of SmallVectorTemplateBase which does not depend on whether +/// the type T is a POD. The extra dummy template argument is used by ArrayRef +/// to avoid unnecessarily requiring T to be complete. template class SmallVectorTemplateCommon : public SmallVectorBase { private: @@ -82,13 +80,13 @@ protected: SmallVectorBase::grow_pod(&FirstEl, MinSizeInBytes, TSize); } - /// isSmall - Return true if this is a smallvector which has not had dynamic + /// Return true if this is a smallvector which has not had dynamic /// memory allocated for it. bool isSmall() const { return BeginX == static_cast(&FirstEl); } - /// resetToSmall - Put this vector in a state of being small. + /// Put this vector in a state of being small. void resetToSmall() { BeginX = EndX = CapacityX = &FirstEl; } @@ -128,21 +126,20 @@ public: size_type size() const { return end()-begin(); } size_type max_size() const { return size_type(-1) / sizeof(T); } - /// capacity - Return the total number of elements in the currently allocated - /// buffer. + /// Return the total number of elements in the currently allocated buffer. size_t capacity() const { return capacity_ptr() - begin(); } - /// data - Return a pointer to the vector's buffer, even if empty(). + /// Return a pointer to the vector's buffer, even if empty(). pointer data() { return pointer(begin()); } - /// data - Return a pointer to the vector's buffer, even if empty(). + /// Return a pointer to the vector's buffer, even if empty(). const_pointer data() const { return const_pointer(begin()); } - reference operator[](unsigned idx) { - assert(begin() + idx < end()); + reference operator[](size_type idx) { + assert(idx < size()); return begin()[idx]; } - const_reference operator[](unsigned idx) const { - assert(begin() + idx < end()); + const_reference operator[](size_type idx) const { + assert(idx < size()); return begin()[idx]; } @@ -179,7 +176,7 @@ protected: } } - /// move - Use move-assignment to move the range [I, E) onto the + /// Use move-assignment to move the range [I, E) onto the /// objects starting with "Dest". This is just 's /// std::move, but not all stdlibs actually provide that. template @@ -189,7 +186,7 @@ protected: return Dest; } - /// move_backward - Use move-assignment to move the range + /// Use move-assignment to move the range /// [I, E) onto the objects ending at "Dest", moving objects /// in reverse order. This is just 's /// std::move_backward, but not all stdlibs actually provide that. @@ -200,25 +197,24 @@ protected: return Dest; } - /// uninitialized_move - Move the range [I, E) into the uninitialized - /// memory starting with "Dest", constructing elements as needed. + /// Move the range [I, E) into the uninitialized memory starting with "Dest", + /// constructing elements as needed. template static void uninitialized_move(It1 I, It1 E, It2 Dest) { for (; I != E; ++I, ++Dest) ::new ((void*) &*Dest) T(::std::move(*I)); } - /// uninitialized_copy - Copy the range [I, E) onto the uninitialized - /// memory starting with "Dest", constructing elements as needed. + /// Copy the range [I, E) onto the uninitialized memory starting with "Dest", + /// constructing elements as needed. template static void uninitialized_copy(It1 I, It1 E, It2 Dest) { std::uninitialized_copy(I, E, Dest); } - /// grow - Grow the allocated memory (without initializing new - /// elements), doubling the size of the allocated memory. - /// Guarantees space for at least one more element, or MinSize more - /// elements if specified. + /// Grow the allocated memory (without initializing new elements), doubling + /// the size of the allocated memory. Guarantees space for at least one more + /// element, or MinSize more elements if specified. void grow(size_t MinSize = 0); public: @@ -240,6 +236,51 @@ public: this->setEnd(this->end()-1); this->end()->~T(); } + +#if LLVM_HAS_VARIADIC_TEMPLATES + template void emplace_back(ArgTypes &&... Args) { + if (LLVM_UNLIKELY(this->EndX >= this->CapacityX)) + this->grow(); + ::new ((void *)this->end()) T(std::forward(Args)...); + this->setEnd(this->end() + 1); + } +#else +private: + template void emplace_back_impl(Constructor construct) { + if (LLVM_UNLIKELY(this->EndX >= this->CapacityX)) + this->grow(); + construct((void *)this->end()); + this->setEnd(this->end() + 1); + } + +public: + void emplace_back() { + emplace_back_impl([](void *Mem) { ::new (Mem) T(); }); + } + template void emplace_back(T1 &&A1) { + emplace_back_impl([&](void *Mem) { ::new (Mem) T(std::forward(A1)); }); + } + template void emplace_back(T1 &&A1, T2 &&A2) { + emplace_back_impl([&](void *Mem) { + ::new (Mem) T(std::forward(A1), std::forward(A2)); + }); + } + template + void emplace_back(T1 &&A1, T2 &&A2, T3 &&A3) { + T(std::forward(A1), std::forward(A2), std::forward(A3)); + emplace_back_impl([&](void *Mem) { + ::new (Mem) + T(std::forward(A1), std::forward(A2), std::forward(A3)); + }); + } + template + void emplace_back(T1 &&A1, T2 &&A2, T3 &&A3, T4 &&A4) { + emplace_back_impl([&](void *Mem) { + ::new (Mem) T(std::forward(A1), std::forward(A2), + std::forward(A3), std::forward(A4)); + }); + } +#endif // LLVM_HAS_VARIADIC_TEMPLATES }; // Define this out-of-line to dissuade the C++ compiler from inlining it. @@ -279,22 +320,21 @@ protected: // No need to do a destroy loop for POD's. static void destroy_range(T *, T *) {} - /// move - Use move-assignment to move the range [I, E) onto the + /// Use move-assignment to move the range [I, E) onto the /// objects starting with "Dest". For PODs, this is just memcpy. template static It2 move(It1 I, It1 E, It2 Dest) { return ::std::copy(I, E, Dest); } - /// move_backward - Use move-assignment to move the range - /// [I, E) onto the objects ending at "Dest", moving objects - /// in reverse order. + /// Use move-assignment to move the range [I, E) onto the objects ending at + /// "Dest", moving objects in reverse order. template static It2 move_backward(It1 I, It1 E, It2 Dest) { return ::std::copy_backward(I, E, Dest); } - /// uninitialized_move - Move the range [I, E) onto the uninitialized memory + /// Move the range [I, E) onto the uninitialized memory /// starting with "Dest", constructing elements into it as needed. template static void uninitialized_move(It1 I, It1 E, It2 Dest) { @@ -302,7 +342,7 @@ protected: uninitialized_copy(I, E, Dest); } - /// uninitialized_copy - Copy the range [I, E) onto the uninitialized memory + /// Copy the range [I, E) onto the uninitialized memory /// starting with "Dest", constructing elements into it as needed. template static void uninitialized_copy(It1 I, It1 E, It2 Dest) { @@ -310,7 +350,7 @@ protected: std::uninitialized_copy(I, E, Dest); } - /// uninitialized_copy - Copy the range [I, E) onto the uninitialized memory + /// Copy the range [I, E) onto the uninitialized memory /// starting with "Dest", constructing elements into it as needed. template static void uninitialized_copy(T1 *I, T1 *E, T2 *Dest) { @@ -320,7 +360,7 @@ protected: memcpy(Dest, I, (E-I)*sizeof(T)); } - /// grow - double the size of the allocated memory, guaranteeing space for at + /// Double the size of the allocated memory, guaranteeing space for at /// least one more element or MinSize if specified. void grow(size_t MinSize = 0) { this->grow_pod(MinSize*sizeof(T), sizeof(T)); @@ -339,9 +379,8 @@ public: }; -/// SmallVectorImpl - This class consists of common code factored out of the -/// SmallVector class to reduce code duplication based on the SmallVector 'N' -/// template parameter. +/// This class consists of common code factored out of the SmallVector class to +/// reduce code duplication based on the SmallVector 'N' template parameter. template class SmallVectorImpl : public SmallVectorTemplateBase::value> { typedef SmallVectorTemplateBase::value > SuperClass; @@ -373,7 +412,7 @@ public: this->EndX = this->BeginX; } - void resize(unsigned N) { + void resize(size_type N) { if (N < this->size()) { this->destroy_range(this->begin()+N, this->end()); this->setEnd(this->begin()+N); @@ -386,7 +425,7 @@ public: } } - void resize(unsigned N, const T &NV) { + void resize(size_type N, const T &NV) { if (N < this->size()) { this->destroy_range(this->begin()+N, this->end()); this->setEnd(this->begin()+N); @@ -398,7 +437,7 @@ public: } } - void reserve(unsigned N) { + void reserve(size_type N) { if (this->capacity() < N) this->grow(N); } @@ -411,8 +450,7 @@ public: void swap(SmallVectorImpl &RHS); - /// append - Add the specified range to the end of the SmallVector. - /// + /// Add the specified range to the end of the SmallVector. template void append(in_iter in_start, in_iter in_end) { size_type NumInputs = std::distance(in_start, in_end); @@ -427,8 +465,7 @@ public: this->setEnd(this->end() + NumInputs); } - /// append - Add the specified range to the end of the SmallVector. - /// + /// Add the specified range to the end of the SmallVector. void append(size_type NumInputs, const T &Elt) { // Grow allocated space if needed. if (NumInputs > size_type(this->capacity_ptr()-this->end())) @@ -439,7 +476,7 @@ public: this->setEnd(this->end() + NumInputs); } - void assign(unsigned NumElts, const T &Elt) { + void assign(size_type NumElts, const T &Elt) { clear(); if (this->capacity() < NumElts) this->grow(NumElts); @@ -545,7 +582,7 @@ public: assert(I <= this->end() && "Inserting past the end of the vector."); // Ensure there is enough space. - reserve(static_cast(this->size() + NumToInsert)); + reserve(this->size() + NumToInsert); // Uninvalidate the iterator. I = this->begin()+InsertElt; @@ -599,7 +636,7 @@ public: size_t NumToInsert = std::distance(From, To); // Ensure there is enough space. - reserve(static_cast(this->size() + NumToInsert)); + reserve(this->size() + NumToInsert); // Uninvalidate the iterator. I = this->begin()+InsertElt; @@ -666,7 +703,7 @@ public: /// of the buffer when they know that more elements are available, and only /// update the size later. This avoids the cost of value initializing elements /// which will only be overwritten. - void set_size(unsigned N) { + void set_size(size_type N) { assert(N <= this->capacity()); this->setEnd(this->begin() + N); } @@ -692,7 +729,7 @@ void SmallVectorImpl::swap(SmallVectorImpl &RHS) { // Swap the shared elements. size_t NumShared = this->size(); if (NumShared > RHS.size()) NumShared = RHS.size(); - for (unsigned i = 0; i != static_cast(NumShared); ++i) + for (size_type i = 0; i != NumShared; ++i) std::swap((*this)[i], RHS[i]); // Copy over the extra elts. @@ -833,7 +870,7 @@ struct SmallVectorStorage { template struct SmallVectorStorage {}; template struct SmallVectorStorage {}; -/// SmallVector - This is a 'vector' (really, a variable-sized array), optimized +/// This is a 'vector' (really, a variable-sized array), optimized /// for the case when the array is small. It contains some number of elements /// in-place, which allows it to avoid heap allocation when the actual number of /// elements is below that threshold. This allows normal "small" cases to be @@ -843,13 +880,13 @@ template struct SmallVectorStorage {}; /// template class SmallVector : public SmallVectorImpl { - /// Storage - Inline space for elements which aren't stored in the base class. + /// Inline space for elements which aren't stored in the base class. SmallVectorStorage Storage; public: SmallVector() : SmallVectorImpl(N) { } - explicit SmallVector(unsigned Size, const T &Value = T()) + explicit SmallVector(size_t Size, const T &Value = T()) : SmallVectorImpl(N) { this->assign(Size, Value); } diff --git a/contrib/llvm/include/llvm/ADT/SparseBitVector.h b/contrib/llvm/include/llvm/ADT/SparseBitVector.h index 36754d682355..d5bde2963fbd 100644 --- a/contrib/llvm/include/llvm/ADT/SparseBitVector.h +++ b/contrib/llvm/include/llvm/ADT/SparseBitVector.h @@ -45,7 +45,7 @@ struct SparseBitVectorElement : public ilist_node > { public: typedef unsigned long BitWord; - typedef unsigned size_type; + typedef unsigned size_type; enum { BITWORD_SIZE = sizeof(BitWord) * CHAR_BIT, BITWORDS_PER_ELEMENT = (ElementSize + BITWORD_SIZE - 1) / BITWORD_SIZE, diff --git a/contrib/llvm/include/llvm/ADT/SparseMultiSet.h b/contrib/llvm/include/llvm/ADT/SparseMultiSet.h index dc1273eb7ff6..f858536b6ed8 100644 --- a/contrib/llvm/include/llvm/ADT/SparseMultiSet.h +++ b/contrib/llvm/include/llvm/ADT/SparseMultiSet.h @@ -185,7 +185,7 @@ public: typedef const ValueT &const_reference; typedef ValueT *pointer; typedef const ValueT *const_pointer; - typedef unsigned size_type; + typedef unsigned size_type; SparseMultiSet() : Sparse(nullptr), Universe(0), FreelistIdx(SMSNode::INVALID), NumFree(0) {} diff --git a/contrib/llvm/include/llvm/ADT/SparseSet.h b/contrib/llvm/include/llvm/ADT/SparseSet.h index 632d52ad9d82..9a13440000ac 100644 --- a/contrib/llvm/include/llvm/ADT/SparseSet.h +++ b/contrib/llvm/include/llvm/ADT/SparseSet.h @@ -124,7 +124,7 @@ class SparseSet { typedef typename KeyFunctorT::argument_type KeyT; typedef SmallVector DenseT; - typedef unsigned size_type; + typedef unsigned size_type; DenseT Dense; SparseT *Sparse; unsigned Universe; diff --git a/contrib/llvm/include/llvm/ADT/StringMap.h b/contrib/llvm/include/llvm/ADT/StringMap.h index c40e5e2b3d87..3437607a0bd0 100644 --- a/contrib/llvm/include/llvm/ADT/StringMap.h +++ b/contrib/llvm/include/llvm/ADT/StringMap.h @@ -117,8 +117,9 @@ public: explicit StringMapEntry(unsigned strLen) : StringMapEntryBase(strLen), second() {} - StringMapEntry(unsigned strLen, ValueTy V) - : StringMapEntryBase(strLen), second(std::move(V)) {} + template + StringMapEntry(unsigned strLen, InitTy &&V) + : StringMapEntryBase(strLen), second(std::forward(V)) {} StringRef getKey() const { return StringRef(getKeyData(), getKeyLength()); @@ -138,10 +139,9 @@ public: /// Create - Create a StringMapEntry for the specified key and default /// construct the value. - template - static StringMapEntry *Create(StringRef Key, - AllocatorTy &Allocator, - InitType InitVal) { + template + static StringMapEntry *Create(StringRef Key, AllocatorTy &Allocator, + InitType &&InitVal) { unsigned KeyLength = Key.size(); // Allocate a new item with space for the string at the end and a null @@ -154,7 +154,7 @@ public: static_cast(Allocator.Allocate(AllocSize,Alignment)); // Default construct the value. - new (NewItem) StringMapEntry(KeyLength, std::move(InitVal)); + new (NewItem) StringMapEntry(KeyLength, std::forward(InitVal)); // Copy the string information. char *StrBuffer = const_cast(NewItem->getKeyData()); @@ -170,28 +170,15 @@ public: /// Create - Create a StringMapEntry with normal malloc/free. template - static StringMapEntry *Create(StringRef Key, InitType InitVal) { + static StringMapEntry *Create(StringRef Key, InitType &&InitVal) { MallocAllocator A; - return Create(Key, A, std::move(InitVal)); + return Create(Key, A, std::forward(InitVal)); } static StringMapEntry *Create(StringRef Key) { return Create(Key, ValueTy()); } - /// GetStringMapEntryFromValue - Given a value that is known to be embedded - /// into a StringMapEntry, return the StringMapEntry itself. - static StringMapEntry &GetStringMapEntryFromValue(ValueTy &V) { - StringMapEntry *EPtr = 0; - char *Ptr = reinterpret_cast(&V) - - (reinterpret_cast(&EPtr->second) - - reinterpret_cast(EPtr)); - return *reinterpret_cast(Ptr); - } - static const StringMapEntry &GetStringMapEntryFromValue(const ValueTy &V) { - return GetStringMapEntryFromValue(const_cast(V)); - } - /// GetStringMapEntryFromKeyData - Given key data that is known to be embedded /// into a StringMapEntry, return the StringMapEntry itself. static StringMapEntry &GetStringMapEntryFromKeyData(const char *KeyData) { @@ -296,7 +283,7 @@ public: } ValueTy &operator[](StringRef Key) { - return GetOrCreateValue(Key).getValue(); + return insert(std::make_pair(Key, ValueTy())).first->second; } /// count - Return 1 if the element is in the map, 0 otherwise. @@ -363,18 +350,6 @@ public: NumTombstones = 0; } - /// GetOrCreateValue - Look up the specified key in the table. If a value - /// exists, return it. Otherwise, default construct a value, insert it, and - /// return. - template - MapEntryTy &GetOrCreateValue(StringRef Key, InitTy Val) { - return *insert(std::make_pair(Key, std::move(Val))).first; - } - - MapEntryTy &GetOrCreateValue(StringRef Key) { - return GetOrCreateValue(Key, ValueTy()); - } - /// remove - Remove the specified key/value pair from the map, but do not /// erase it. This aborts if the key is not in the map. void remove(MapEntryTy *KeyValue) { diff --git a/contrib/llvm/include/llvm/ADT/StringRef.h b/contrib/llvm/include/llvm/ADT/StringRef.h index 1f413e80553f..6111c42da9dc 100644 --- a/contrib/llvm/include/llvm/ADT/StringRef.h +++ b/contrib/llvm/include/llvm/ADT/StringRef.h @@ -51,12 +51,6 @@ namespace llvm { /// The length of the string. size_t Length; - // Workaround PR5482: nearly all gcc 4.x miscompile StringRef and std::min() - // Changing the arg of min to be an integer, instead of a reference to an - // integer works around this bug. - static size_t min(size_t a, size_t b) { return a < b ? a : b; } - static size_t max(size_t a, size_t b) { return a > b ? a : b; } - // Workaround memcmp issue with null pointers (undefined behavior) // by providing a specialized version static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) { @@ -97,6 +91,13 @@ namespace llvm { iterator end() const { return Data + Length; } + const unsigned char *bytes_begin() const { + return reinterpret_cast(begin()); + } + const unsigned char *bytes_end() const { + return reinterpret_cast(end()); + } + /// @} /// @name String Operations /// @{ @@ -124,7 +125,7 @@ namespace llvm { } // copy - Allocate copy in Allocator and return StringRef to it. - template StringRef copy(Allocator &A) { + template StringRef copy(Allocator &A) const { char *S = A.template Allocate(Length); std::copy(begin(), end(), S); return StringRef(S, Length); @@ -146,7 +147,7 @@ namespace llvm { /// is lexicographically less than, equal to, or greater than the \p RHS. int compare(StringRef RHS) const { // Check the prefix for a mismatch. - if (int Res = compareMemory(Data, RHS.Data, min(Length, RHS.Length))) + if (int Res = compareMemory(Data, RHS.Data, std::min(Length, RHS.Length))) return Res < 0 ? -1 : 1; // Otherwise the prefixes match, so we only need to check the lengths. @@ -237,7 +238,7 @@ namespace llvm { /// \returns The index of the first occurrence of \p C, or npos if not /// found. size_t find(char C, size_t From = 0) const { - for (size_t i = min(From, Length), e = Length; i != e; ++i) + for (size_t i = std::min(From, Length), e = Length; i != e; ++i) if (Data[i] == C) return i; return npos; @@ -254,7 +255,7 @@ namespace llvm { /// \returns The index of the last occurrence of \p C, or npos if not /// found. size_t rfind(char C, size_t From = npos) const { - From = min(From, Length); + From = std::min(From, Length); size_t i = From; while (i != 0) { --i; @@ -353,8 +354,11 @@ namespace llvm { typename std::enable_if::is_signed, bool>::type getAsInteger(unsigned Radix, T &Result) const { unsigned long long ULLVal; + // The additional cast to unsigned long long is required to avoid the + // Visual C++ warning C4805: '!=' : unsafe mix of type 'bool' and type + // 'unsigned __int64' when instantiating getAsInteger with T = bool. if (getAsUnsignedInteger(*this, Radix, ULLVal) || - static_cast(ULLVal) != ULLVal) + static_cast(static_cast(ULLVal)) != ULLVal) return true; Result = ULLVal; return false; @@ -396,8 +400,8 @@ namespace llvm { /// exceeds the number of characters remaining in the string, the string /// suffix (starting with \p Start) will be returned. StringRef substr(size_t Start, size_t N = npos) const { - Start = min(Start, Length); - return StringRef(Data + Start, min(N, Length - Start)); + Start = std::min(Start, Length); + return StringRef(Data + Start, std::min(N, Length - Start)); } /// Return a StringRef equal to 'this' but with the first \p N elements @@ -425,8 +429,8 @@ namespace llvm { /// number of characters remaining in the string, the string suffix /// (starting with \p Start) will be returned. StringRef slice(size_t Start, size_t End) const { - Start = min(Start, Length); - End = min(max(Start, End), Length); + Start = std::min(Start, Length); + End = std::min(std::max(Start, End), Length); return StringRef(Data + Start, End - Start); } diff --git a/contrib/llvm/include/llvm/ADT/StringSet.h b/contrib/llvm/include/llvm/ADT/StringSet.h index 7bea577f34d3..3e0cc200b6dd 100644 --- a/contrib/llvm/include/llvm/ADT/StringSet.h +++ b/contrib/llvm/include/llvm/ADT/StringSet.h @@ -24,20 +24,9 @@ namespace llvm { typedef llvm::StringMap base; public: - /// insert - Insert the specified key into the set. If the key already - /// exists in the set, return false and ignore the request, otherwise insert - /// it and return true. - bool insert(StringRef Key) { - // Get or create the map entry for the key; if it doesn't exist the value - // type will be default constructed which we use to detect insert. - // - // We use '+' as the sentinel value in the map. + std::pair insert(StringRef Key) { assert(!Key.empty()); - StringMapEntry &Entry = this->GetOrCreateValue(Key); - if (Entry.getValue() == '+') - return false; - Entry.setValue('+'); - return true; + return base::insert(std::make_pair(Key, '\0')); } }; } diff --git a/contrib/llvm/include/llvm/ADT/TinyPtrVector.h b/contrib/llvm/include/llvm/ADT/TinyPtrVector.h index 5669b2a81a40..15137f5ebf8c 100644 --- a/contrib/llvm/include/llvm/ADT/TinyPtrVector.h +++ b/contrib/llvm/include/llvm/ADT/TinyPtrVector.h @@ -96,10 +96,17 @@ public: return *this; } + /// Constructor from a single element. + explicit TinyPtrVector(EltTy Elt) : Val(Elt) {} + + /// Constructor from an ArrayRef. + explicit TinyPtrVector(ArrayRef Elts) + : Val(new VecTy(Elts.begin(), Elts.end())) {} + // implicit conversion operator to ArrayRef. operator ArrayRef() const { if (Val.isNull()) - return ArrayRef(); + return None; if (Val.template is()) return *Val.getAddrOfPtr1(); return *Val.template get(); diff --git a/contrib/llvm/include/llvm/ADT/Triple.h b/contrib/llvm/include/llvm/ADT/Triple.h index b96f11435520..8a685995256b 100644 --- a/contrib/llvm/include/llvm/ADT/Triple.h +++ b/contrib/llvm/include/llvm/ADT/Triple.h @@ -48,8 +48,6 @@ public: arm, // ARM (little endian): arm, armv.*, xscale armeb, // ARM (big endian): armeb - arm64, // ARM64 (little endian): arm64 - arm64_be, // ARM64 (big endian): arm64_be aarch64, // AArch64 (little endian): aarch64 aarch64_be, // AArch64 (big endian): aarch64_be hexagon, // Hexagon: hexagon @@ -62,6 +60,7 @@ public: ppc64, // PPC64: powerpc64, ppu ppc64le, // PPC64LE: powerpc64le r600, // R600: AMD GPUs HD2XXX - HD6XXX + amdgcn, // AMDGCN: AMD GCN GPUs sparc, // Sparc: sparc sparcv9, // Sparcv9: Sparcv9 systemz, // SystemZ: s390x @@ -74,7 +73,11 @@ public: nvptx, // NVPTX: 32-bit nvptx64, // NVPTX: 64-bit le32, // le32: generic little-endian 32-bit CPU (PNaCl / Emscripten) - amdil, // amdil: amd IL + le64, // le64: generic little-endian 64-bit CPU (PNaCl / Emscripten) + amdil, // AMDIL + amdil64, // AMDIL with 64-bit pointers + hsail, // AMD HSAIL + hsail64, // AMD HSAIL with 64-bit pointers spir, // SPIR: standard portable IR for OpenCL 32-bit version spir64, // SPIR: standard portable IR for OpenCL 64-bit version kalimba // Kalimba: generic kalimba @@ -92,7 +95,11 @@ public: ARMSubArch_v6t2, ARMSubArch_v5, ARMSubArch_v5te, - ARMSubArch_v4t + ARMSubArch_v4t, + + KalimbaSubArch_v3, + KalimbaSubArch_v4, + KalimbaSubArch_v5 }; enum VendorType { UnknownVendor, @@ -112,8 +119,6 @@ public: enum OSType { UnknownOS, - AuroraUX, - Cygwin, Darwin, DragonFly, FreeBSD, @@ -122,7 +127,6 @@ public: Linux, Lv2, // PS3 MacOSX, - MinGW32, // i*86-pc-mingw32, *-w64-mingw32 NetBSD, OpenBSD, Solaris, @@ -135,7 +139,8 @@ public: Bitrig, AIX, CUDA, // NVIDIA CUDA - NVCL // NVIDIA OpenCL + NVCL, // NVIDIA OpenCL + AMDHSA // AMD HSA Runtime }; enum EnvironmentType { UnknownEnvironment, @@ -361,10 +366,28 @@ public: return isMacOSX() || isiOS(); } + bool isOSNetBSD() const { + return getOS() == Triple::NetBSD; + } + + bool isOSOpenBSD() const { + return getOS() == Triple::OpenBSD; + } + bool isOSFreeBSD() const { return getOS() == Triple::FreeBSD; } + bool isOSDragonFly() const { return getOS() == Triple::DragonFly; } + + bool isOSSolaris() const { + return getOS() == Triple::Solaris; + } + + bool isOSBitrig() const { + return getOS() == Triple::Bitrig; + } + bool isWindowsMSVCEnvironment() const { return getOS() == Triple::Win32 && (getEnvironment() == Triple::UnknownEnvironment || @@ -380,13 +403,11 @@ public: } bool isWindowsCygwinEnvironment() const { - return getOS() == Triple::Cygwin || - (getOS() == Triple::Win32 && getEnvironment() == Triple::Cygnus); + return getOS() == Triple::Win32 && getEnvironment() == Triple::Cygnus; } bool isWindowsGNUEnvironment() const { - return getOS() == Triple::MinGW32 || - (getOS() == Triple::Win32 && getEnvironment() == Triple::GNU); + return getOS() == Triple::Win32 && getEnvironment() == Triple::GNU; } /// \brief Tests for either Cygwin or MinGW OS @@ -396,7 +417,8 @@ public: /// \brief Is this a "Windows" OS targeting a "MSVCRT.dll" environment. bool isOSMSVCRT() const { - return isWindowsMSVCEnvironment() || isWindowsGNUEnvironment(); + return isWindowsMSVCEnvironment() || isWindowsGNUEnvironment() || + isWindowsItaniumEnvironment(); } /// \brief Tests whether the OS is Windows. @@ -475,10 +497,6 @@ public: /// environment components with a single string. void setOSAndEnvironmentName(StringRef Str); - /// getArchNameForAssembler - Get an architecture name that is understood by - /// the target assembler. - const char *getArchNameForAssembler(); - /// @} /// @name Helpers to build variants of a particular triple. /// @{ diff --git a/contrib/llvm/include/llvm/ADT/Twine.h b/contrib/llvm/include/llvm/ADT/Twine.h index 4be3ee6f82db..05d2fea117cf 100644 --- a/contrib/llvm/include/llvm/ADT/Twine.h +++ b/contrib/llvm/include/llvm/ADT/Twine.h @@ -80,7 +80,7 @@ namespace llvm { /// StringRef) codegen as desired. class Twine { /// NodeKind - Represent the type of an argument. - enum NodeKind { + enum NodeKind : unsigned char { /// An empty string; the result of concatenating anything with it is also /// empty. NullKind, @@ -153,12 +153,10 @@ namespace llvm { /// RHS - The suffix in the concatenation, which may be uninitialized for /// Null or Empty kinds. Child RHS; - // enums stored as unsigned chars to save on space while some compilers - // don't support specifying the backing type for an enum /// LHSKind - The NodeKind of the left hand side, \see getLHSKind(). - unsigned char LHSKind; - /// RHSKind - The NodeKind of the left hand side, \see getLHSKind(). - unsigned char RHSKind; + NodeKind LHSKind; + /// RHSKind - The NodeKind of the right hand side, \see getRHSKind(). + NodeKind RHSKind; private: /// Construct a nullary twine; the kind must be NullKind or EmptyKind. @@ -238,10 +236,10 @@ namespace llvm { } /// getLHSKind - Get the NodeKind of the left-hand side. - NodeKind getLHSKind() const { return (NodeKind) LHSKind; } + NodeKind getLHSKind() const { return LHSKind; } /// getRHSKind - Get the NodeKind of the right-hand side. - NodeKind getRHSKind() const { return (NodeKind) RHSKind; } + NodeKind getRHSKind() const { return RHSKind; } /// printOneChild - Print one child from a twine. void printOneChild(raw_ostream &OS, Child Ptr, NodeKind Kind) const; diff --git a/contrib/llvm/include/llvm/ADT/VariadicFunction.h b/contrib/llvm/include/llvm/ADT/VariadicFunction.h index 0497aa70887c..403130c623eb 100644 --- a/contrib/llvm/include/llvm/ADT/VariadicFunction.h +++ b/contrib/llvm/include/llvm/ADT/VariadicFunction.h @@ -105,7 +105,7 @@ template )> struct VariadicFunction { ResultT operator()() const { - return Func(ArrayRef()); + return Func(None); } #define LLVM_DEFINE_OVERLOAD(N) \ @@ -152,7 +152,7 @@ template )> struct VariadicFunction1 { ResultT operator()(Param0T P0) const { - return Func(P0, ArrayRef()); + return Func(P0, None); } #define LLVM_DEFINE_OVERLOAD(N) \ @@ -199,7 +199,7 @@ template )> struct VariadicFunction2 { ResultT operator()(Param0T P0, Param1T P1) const { - return Func(P0, P1, ArrayRef()); + return Func(P0, P1, None); } #define LLVM_DEFINE_OVERLOAD(N) \ @@ -248,7 +248,7 @@ template )> struct VariadicFunction3 { ResultT operator()(Param0T P0, Param1T P1, Param2T P2) const { - return Func(P0, P1, P2, ArrayRef()); + return Func(P0, P1, P2, None); } #define LLVM_DEFINE_OVERLOAD(N) \ diff --git a/contrib/llvm/include/llvm/ADT/ilist.h b/contrib/llvm/include/llvm/ADT/ilist.h index bc148452f217..8c19a6f4547a 100644 --- a/contrib/llvm/include/llvm/ADT/ilist.h +++ b/contrib/llvm/include/llvm/ADT/ilist.h @@ -579,60 +579,6 @@ public: void splice(iterator where, iplist &L2, iterator first, iterator last) { if (first != last) transfer(where, L2, first, last); } - - - - //===----------------------------------------------------------------------=== - // High-Level Functionality that shouldn't really be here, but is part of list - // - - // These two functions are actually called remove/remove_if in list<>, but - // they actually do the job of erase, rename them accordingly. - // - void erase(const NodeTy &val) { - for (iterator I = begin(), E = end(); I != E; ) { - iterator next = I; ++next; - if (*I == val) erase(I); - I = next; - } - } - template void erase_if(Pr1 pred) { - for (iterator I = begin(), E = end(); I != E; ) { - iterator next = I; ++next; - if (pred(*I)) erase(I); - I = next; - } - } - - template void unique(Pr2 pred) { - if (empty()) return; - for (iterator I = begin(), E = end(), Next = begin(); ++Next != E;) { - if (pred(*I)) - erase(Next); - else - I = Next; - Next = I; - } - } - void unique() { unique(op_equal); } - - template void merge(iplist &right, Pr3 pred) { - iterator first1 = begin(), last1 = end(); - iterator first2 = right.begin(), last2 = right.end(); - while (first1 != last1 && first2 != last2) - if (pred(*first2, *first1)) { - iterator next = first2; - transfer(first1, right, first2, ++next); - first2 = next; - } else { - ++first1; - } - if (first2 != last2) transfer(last1, right, first2, last2); - } - void merge(iplist &right) { return merge(right, op_less); } - - template void sort(Pr3 pred); - void sort() { sort(op_less); } }; diff --git a/contrib/llvm/include/llvm/ADT/ilist_node.h b/contrib/llvm/include/llvm/ADT/ilist_node.h index 85aa7a4b1f7f..26d0b55e4093 100644 --- a/contrib/llvm/include/llvm/ADT/ilist_node.h +++ b/contrib/llvm/include/llvm/ADT/ilist_node.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_ADT_ILISTNODE_H -#define LLVM_ADT_ILISTNODE_H +#ifndef LLVM_ADT_ILIST_NODE_H +#define LLVM_ADT_ILIST_NODE_H namespace llvm { diff --git a/contrib/llvm/include/llvm/ADT/iterator.h b/contrib/llvm/include/llvm/ADT/iterator.h index 56041dbb106c..e2c9e5ea6bda 100644 --- a/contrib/llvm/include/llvm/ADT/iterator.h +++ b/contrib/llvm/include/llvm/ADT/iterator.h @@ -10,8 +10,8 @@ #ifndef LLVM_ADT_ITERATOR_H #define LLVM_ADT_ITERATOR_H -#include #include +#include namespace llvm { diff --git a/contrib/llvm/include/llvm/ADT/iterator_range.h b/contrib/llvm/include/llvm/ADT/iterator_range.h index dd17d6c8f7b4..523a86f02e08 100644 --- a/contrib/llvm/include/llvm/ADT/iterator_range.h +++ b/contrib/llvm/include/llvm/ADT/iterator_range.h @@ -32,7 +32,6 @@ class iterator_range { IteratorT begin_iterator, end_iterator; public: - iterator_range() {} iterator_range(IteratorT begin_iterator, IteratorT end_iterator) : begin_iterator(std::move(begin_iterator)), end_iterator(std::move(end_iterator)) {} @@ -48,6 +47,10 @@ public: template iterator_range make_range(T x, T y) { return iterator_range(std::move(x), std::move(y)); } + +template iterator_range make_range(std::pair p) { + return iterator_range(std::move(p.first), std::move(p.second)); +} } #endif diff --git a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h index 689766446445..763f37298811 100644 --- a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -39,6 +39,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Metadata.h" namespace llvm { @@ -112,13 +113,14 @@ public: /// there are restrictions on stepping out of one object and into another. /// See http://llvm.org/docs/LangRef.html#pointeraliasing uint64_t Size; - /// TBAATag - The metadata node which describes the TBAA type of - /// the location, or null if there is no known unique tag. - const MDNode *TBAATag; + /// AATags - The metadata nodes which describes the aliasing of the + /// location (each member is null if that kind of information is + /// unavailable).. + AAMDNodes AATags; explicit Location(const Value *P = nullptr, uint64_t S = UnknownSize, - const MDNode *N = nullptr) - : Ptr(P), Size(S), TBAATag(N) {} + const AAMDNodes &N = AAMDNodes()) + : Ptr(P), Size(S), AATags(N) {} Location getWithNewPtr(const Value *NewPtr) const { Location Copy(*this); @@ -132,9 +134,9 @@ public: return Copy; } - Location getWithoutTBAATag() const { + Location getWithoutAATags() const { Location Copy(*this); - Copy.TBAATag = nullptr; + Copy.AATags = AAMDNodes(); return Copy; } }; @@ -500,7 +502,7 @@ public: /// /// canBasicBlockModify - Return true if it is possible for execution of the - /// specified basic block to modify the value pointed to by Ptr. + /// specified basic block to modify the location Loc. bool canBasicBlockModify(const BasicBlock &BB, const Location &Loc); /// canBasicBlockModify - A convenience wrapper. @@ -508,17 +510,20 @@ public: return canBasicBlockModify(BB, Location(P, Size)); } - /// canInstructionRangeModify - Return true if it is possible for the - /// execution of the specified instructions to modify the value pointed to by - /// Ptr. The instructions to consider are all of the instructions in the - /// range of [I1,I2] INCLUSIVE. I1 and I2 must be in the same basic block. - bool canInstructionRangeModify(const Instruction &I1, const Instruction &I2, - const Location &Loc); + /// canInstructionRangeModRef - Return true if it is possible for the + /// execution of the specified instructions to mod\ref (according to the + /// mode) the location Loc. The instructions to consider are all + /// of the instructions in the range of [I1,I2] INCLUSIVE. + /// I1 and I2 must be in the same basic block. + bool canInstructionRangeModRef(const Instruction &I1, + const Instruction &I2, const Location &Loc, + const ModRefResult Mode); - /// canInstructionRangeModify - A convenience wrapper. - bool canInstructionRangeModify(const Instruction &I1, const Instruction &I2, - const Value *Ptr, uint64_t Size) { - return canInstructionRangeModify(I1, I2, Location(Ptr, Size)); + /// canInstructionRangeModRef - A convenience wrapper. + bool canInstructionRangeModRef(const Instruction &I1, + const Instruction &I2, const Value *Ptr, + uint64_t Size, const ModRefResult Mode) { + return canInstructionRangeModRef(I1, I2, Location(Ptr, Size), Mode); } //===--------------------------------------------------------------------===// @@ -566,25 +571,23 @@ public: template<> struct DenseMapInfo { static inline AliasAnalysis::Location getEmptyKey() { - return - AliasAnalysis::Location(DenseMapInfo::getEmptyKey(), - 0, nullptr); + return AliasAnalysis::Location(DenseMapInfo::getEmptyKey(), + 0); } static inline AliasAnalysis::Location getTombstoneKey() { - return - AliasAnalysis::Location(DenseMapInfo::getTombstoneKey(), - 0, nullptr); + return AliasAnalysis::Location( + DenseMapInfo::getTombstoneKey(), 0); } static unsigned getHashValue(const AliasAnalysis::Location &Val) { return DenseMapInfo::getHashValue(Val.Ptr) ^ DenseMapInfo::getHashValue(Val.Size) ^ - DenseMapInfo::getHashValue(Val.TBAATag); + DenseMapInfo::getHashValue(Val.AATags); } static bool isEqual(const AliasAnalysis::Location &LHS, const AliasAnalysis::Location &RHS) { return LHS.Ptr == RHS.Ptr && LHS.Size == RHS.Size && - LHS.TBAATag == RHS.TBAATag; + LHS.AATags == RHS.AATags; } }; diff --git a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h index e32b6d628b7f..036d58dfa810 100644 --- a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h +++ b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h @@ -20,6 +20,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" #include @@ -40,11 +41,11 @@ class AliasSet : public ilist_node { PointerRec **PrevInList, *NextInList; AliasSet *AS; uint64_t Size; - const MDNode *TBAAInfo; + AAMDNodes AAInfo; public: PointerRec(Value *V) : Val(V), PrevInList(nullptr), NextInList(nullptr), AS(nullptr), Size(0), - TBAAInfo(DenseMapInfo::getEmptyKey()) {} + AAInfo(DenseMapInfo::getEmptyKey()) {} Value *getValue() const { return Val; } @@ -56,27 +57,27 @@ class AliasSet : public ilist_node { return &NextInList; } - void updateSizeAndTBAAInfo(uint64_t NewSize, const MDNode *NewTBAAInfo) { + void updateSizeAndAAInfo(uint64_t NewSize, const AAMDNodes &NewAAInfo) { if (NewSize > Size) Size = NewSize; - if (TBAAInfo == DenseMapInfo::getEmptyKey()) - // We don't have a TBAAInfo yet. Set it to NewTBAAInfo. - TBAAInfo = NewTBAAInfo; - else if (TBAAInfo != NewTBAAInfo) - // NewTBAAInfo conflicts with TBAAInfo. - TBAAInfo = DenseMapInfo::getTombstoneKey(); + if (AAInfo == DenseMapInfo::getEmptyKey()) + // We don't have a AAInfo yet. Set it to NewAAInfo. + AAInfo = NewAAInfo; + else if (AAInfo != NewAAInfo) + // NewAAInfo conflicts with AAInfo. + AAInfo = DenseMapInfo::getTombstoneKey(); } uint64_t getSize() const { return Size; } - /// getTBAAInfo - Return the TBAAInfo, or null if there is no + /// getAAInfo - Return the AAInfo, or null if there is no /// information or conflicting information. - const MDNode *getTBAAInfo() const { - // If we have missing or conflicting TBAAInfo, return null. - if (TBAAInfo == DenseMapInfo::getEmptyKey() || - TBAAInfo == DenseMapInfo::getTombstoneKey()) - return nullptr; - return TBAAInfo; + AAMDNodes getAAInfo() const { + // If we have missing or conflicting AAInfo, return null. + if (AAInfo == DenseMapInfo::getEmptyKey() || + AAInfo == DenseMapInfo::getTombstoneKey()) + return AAMDNodes(); + return AAInfo; } AliasSet *getAliasSet(AliasSetTracker &AST) { @@ -204,7 +205,7 @@ public: Value *getPointer() const { return CurNode->getValue(); } uint64_t getSize() const { return CurNode->getSize(); } - const MDNode *getTBAAInfo() const { return CurNode->getTBAAInfo(); } + AAMDNodes getAAInfo() const { return CurNode->getAAInfo(); } iterator& operator++() { // Preincrement assert(CurNode && "Advancing past AliasSet.end()!"); @@ -250,7 +251,7 @@ private: void removeFromTracker(AliasSetTracker &AST); void addPointer(AliasSetTracker &AST, PointerRec &Entry, uint64_t Size, - const MDNode *TBAAInfo, + const AAMDNodes &AAInfo, bool KnownMustAlias = false); void addUnknownInst(Instruction *I, AliasAnalysis &AA); void removeUnknownInst(AliasSetTracker &AST, Instruction *I) { @@ -270,7 +271,7 @@ public: /// aliasesPointer - Return true if the specified pointer "may" (or must) /// alias one of the members in the set. /// - bool aliasesPointer(const Value *Ptr, uint64_t Size, const MDNode *TBAAInfo, + bool aliasesPointer(const Value *Ptr, uint64_t Size, const AAMDNodes &AAInfo, AliasAnalysis &AA) const; bool aliasesUnknownInst(Instruction *Inst, AliasAnalysis &AA) const; }; @@ -325,7 +326,7 @@ public: /// These methods return true if inserting the instruction resulted in the /// addition of a new alias set (i.e., the pointer did not alias anything). /// - bool add(Value *Ptr, uint64_t Size, const MDNode *TBAAInfo); // Add a location + bool add(Value *Ptr, uint64_t Size, const AAMDNodes &AAInfo); // Add a loc. bool add(LoadInst *LI); bool add(StoreInst *SI); bool add(VAArgInst *VAAI); @@ -338,7 +339,7 @@ public: /// be aliased by the specified instruction. These methods return true if any /// alias sets were eliminated. // Remove a location - bool remove(Value *Ptr, uint64_t Size, const MDNode *TBAAInfo); + bool remove(Value *Ptr, uint64_t Size, const AAMDNodes &AAInfo); bool remove(LoadInst *LI); bool remove(StoreInst *SI); bool remove(VAArgInst *VAAI); @@ -357,20 +358,24 @@ public: /// true if a new alias set is created to contain the pointer (because the /// pointer didn't alias anything). AliasSet &getAliasSetForPointer(Value *P, uint64_t Size, - const MDNode *TBAAInfo, + const AAMDNodes &AAInfo, bool *New = nullptr); /// getAliasSetForPointerIfExists - Return the alias set containing the /// location specified if one exists, otherwise return null. AliasSet *getAliasSetForPointerIfExists(Value *P, uint64_t Size, - const MDNode *TBAAInfo) { - return findAliasSetForPointer(P, Size, TBAAInfo); + const AAMDNodes &AAInfo) { + return findAliasSetForPointer(P, Size, AAInfo); } /// containsPointer - Return true if the specified location is represented by /// this alias set, false otherwise. This does not modify the AST object or /// alias sets. - bool containsPointer(Value *P, uint64_t Size, const MDNode *TBAAInfo) const; + bool containsPointer(Value *P, uint64_t Size, const AAMDNodes &AAInfo) const; + + /// Return true if the specified instruction "may" (or must) alias one of the + /// members in any of the sets. + bool containsUnknown(Instruction *I) const; /// getAliasAnalysis - Return the underlying alias analysis object used by /// this tracker. @@ -417,16 +422,16 @@ private: return *Entry; } - AliasSet &addPointer(Value *P, uint64_t Size, const MDNode *TBAAInfo, + AliasSet &addPointer(Value *P, uint64_t Size, const AAMDNodes &AAInfo, AliasSet::AccessType E, bool &NewSet) { NewSet = false; - AliasSet &AS = getAliasSetForPointer(P, Size, TBAAInfo, &NewSet); + AliasSet &AS = getAliasSetForPointer(P, Size, AAInfo, &NewSet); AS.AccessTy |= E; return AS; } AliasSet *findAliasSetForPointer(const Value *Ptr, uint64_t Size, - const MDNode *TBAAInfo); + const AAMDNodes &AAInfo); AliasSet *findAliasSetForUnknownInst(Instruction *Inst); }; diff --git a/contrib/llvm/include/llvm/Analysis/AssumptionCache.h b/contrib/llvm/include/llvm/Analysis/AssumptionCache.h new file mode 100644 index 000000000000..b129e6796328 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/AssumptionCache.h @@ -0,0 +1,142 @@ +//===- llvm/Analysis/AssumptionCache.h - Track @llvm.assume ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains a pass that keeps track of @llvm.assume intrinsics in +// the functions of a module (allowing assumptions within any function to be +// found cheaply by other parts of the optimizer). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_ASSUMPTIONCACHE_H +#define LLVM_ANALYSIS_ASSUMPTIONCACHE_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Pass.h" +#include + +namespace llvm { + +/// \brief A cache of @llvm.assume calls within a function. +/// +/// This cache provides fast lookup of assumptions within a function by caching +/// them and amortizing the cost of scanning for them across all queries. The +/// cache is also conservatively self-updating so that it will never return +/// incorrect results about a function even as the function is being mutated. +/// However, flushing the cache and rebuilding it (or explicitly updating it) +/// may allow it to discover new assumptions. +class AssumptionCache { + /// \brief The function for which this cache is handling assumptions. + /// + /// We track this to lazily populate our assumptions. + Function &F; + + /// \brief Vector of weak value handles to calls of the @llvm.assume + /// intrinsic. + SmallVector AssumeHandles; + + /// \brief Flag tracking whether we have scanned the function yet. + /// + /// We want to be as lazy about this as possible, and so we scan the function + /// at the last moment. + bool Scanned; + + /// \brief Scan the function for assumptions and add them to the cache. + void scanFunction(); + +public: + /// \brief Construct an AssumptionCache from a function by scanning all of + /// its instructions. + AssumptionCache(Function &F) : F(F), Scanned(false) {} + + /// \brief Add an @llvm.assume intrinsic to this function's cache. + /// + /// The call passed in must be an instruction within this fuction and must + /// not already be in the cache. + void registerAssumption(CallInst *CI); + + /// \brief Clear the cache of @llvm.assume intrinsics for a function. + /// + /// It will be re-scanned the next time it is requested. + void clear() { + AssumeHandles.clear(); + Scanned = false; + } + + /// \brief Access the list of assumption handles currently tracked for this + /// fuction. + /// + /// Note that these produce weak handles that may be null. The caller must + /// handle that case. + /// FIXME: We should replace this with pointee_iterator> + /// when we can write that to filter out the null values. Then caller code + /// will become simpler. + MutableArrayRef assumptions() { + if (!Scanned) + scanFunction(); + return AssumeHandles; + } +}; + +/// \brief An immutable pass that tracks lazily created \c AssumptionCache +/// objects. +/// +/// This is essentially a workaround for the legacy pass manager's weaknesses +/// which associates each assumption cache with Function and clears it if the +/// function is deleted. The nature of the AssumptionCache is that it is not +/// invalidated by any changes to the function body and so this is sufficient +/// to be conservatively correct. +class AssumptionCacheTracker : public ImmutablePass { + /// A callback value handle applied to function objects, which we use to + /// delete our cache of intrinsics for a function when it is deleted. + class FunctionCallbackVH : public CallbackVH { + AssumptionCacheTracker *ACT; + void deleted() override; + + public: + typedef DenseMapInfo DMI; + + FunctionCallbackVH(Value *V, AssumptionCacheTracker *ACT = nullptr) + : CallbackVH(V), ACT(ACT) {} + }; + + friend FunctionCallbackVH; + + typedef DenseMap, + FunctionCallbackVH::DMI> FunctionCallsMap; + FunctionCallsMap AssumptionCaches; + +public: + /// \brief Get the cached assumptions for a function. + /// + /// If no assumptions are cached, this will scan the function. Otherwise, the + /// existing cache will be returned. + AssumptionCache &getAssumptionCache(Function &F); + + AssumptionCacheTracker(); + ~AssumptionCacheTracker(); + + void releaseMemory() override { AssumptionCaches.shrink_and_clear(); } + + void verifyAnalysis() const override; + bool doFinalization(Module &) override { + verifyAnalysis(); + return false; + } + + static char ID; // Pass identification, replacement for typeid +}; + +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index bb256c7bbcc8..57b515420925 100644 --- a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -260,7 +260,7 @@ public: /// loop. /// /// This function should only be called when distributing mass. As long as - /// there are no irreducilbe edges to Node, then it will have complexity + /// there are no irreducible edges to Node, then it will have complexity /// O(1) in this context. /// /// In general, the complexity is O(L), where L is the number of loop diff --git a/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index 4414c84f6b7a..89eef68d8431 100644 --- a/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -111,6 +111,10 @@ public: void setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors, uint32_t Weight); + static uint32_t getBranchWeightStackProtector(bool IsLikely) { + return IsLikely ? (1u << 20) - 1 : 1; + } + private: // Since we allow duplicate edges from one basic block to another, we use // a pair (PredBlock and an index in the successors) to specify an edge. diff --git a/contrib/llvm/include/llvm/Analysis/CFGPrinter.h b/contrib/llvm/include/llvm/Analysis/CFGPrinter.h index e6d2ed1a6864..035764837e6f 100644 --- a/contrib/llvm/include/llvm/Analysis/CFGPrinter.h +++ b/contrib/llvm/include/llvm/Analysis/CFGPrinter.h @@ -72,13 +72,13 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { OutStr.erase(OutStr.begin()+i, OutStr.begin()+Idx); --i; } else if (ColNum == MaxColumns) { // Wrap lines. - if (LastSpace) { - OutStr.insert(LastSpace, "\\l..."); - ColNum = i - LastSpace; - LastSpace = 0; - i += 3; // The loop will advance 'i' again. - } - // Else keep trying to find a space. + // Wrap very long names even though we can't find a space. + if (!LastSpace) + LastSpace = i; + OutStr.insert(LastSpace, "\\l..."); + ColNum = i - LastSpace; + LastSpace = 0; + i += 3; // The loop will advance 'i' again. } else ++ColNum; diff --git a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h index 09101ae6d0d1..0d4fe932481b 100644 --- a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -18,138 +18,28 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_ANALYSIS_CGSCC_PASS_MANAGER_H -#define LLVM_ANALYSIS_CGSCC_PASS_MANAGER_H +#ifndef LLVM_ANALYSIS_CGSCCPASSMANAGER_H +#define LLVM_ANALYSIS_CGSCCPASSMANAGER_H -#include "llvm/IR/PassManager.h" #include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/IR/PassManager.h" namespace llvm { -class CGSCCAnalysisManager; +/// \brief The CGSCC pass manager. +/// +/// See the documentation for the PassManager template for details. It runs +/// a sequency of SCC passes over each SCC that the manager is run over. This +/// typedef serves as a convenient way to refer to this construct. +typedef PassManager CGSCCPassManager; -class CGSCCPassManager { -public: - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - CGSCCPassManager() {} - CGSCCPassManager(CGSCCPassManager &&Arg) : Passes(std::move(Arg.Passes)) {} - CGSCCPassManager &operator=(CGSCCPassManager &&RHS) { - Passes = std::move(RHS.Passes); - return *this; - } - - /// \brief Run all of the CGSCC passes in this pass manager over a SCC. - PreservedAnalyses run(LazyCallGraph::SCC *C, - CGSCCAnalysisManager *AM = nullptr); - - template void addPass(CGSCCPassT Pass) { - Passes.emplace_back(new CGSCCPassModel(std::move(Pass))); - } - - static StringRef name() { return "CGSCCPassManager"; } - -private: - // Pull in the concept type and model template specialized for SCCs. - typedef detail::PassConcept - CGSCCPassConcept; - template - struct CGSCCPassModel - : detail::PassModel { - CGSCCPassModel(PassT Pass) - : detail::PassModel( - std::move(Pass)) {} - }; - - CGSCCPassManager(const CGSCCPassManager &) LLVM_DELETED_FUNCTION; - CGSCCPassManager &operator=(const CGSCCPassManager &) LLVM_DELETED_FUNCTION; - - std::vector> Passes; -}; - -/// \brief A function analysis manager to coordinate and cache analyses run over -/// a module. -class CGSCCAnalysisManager : public detail::AnalysisManagerBase< - CGSCCAnalysisManager, LazyCallGraph::SCC *> { - friend class detail::AnalysisManagerBase; - typedef detail::AnalysisManagerBase BaseT; - typedef BaseT::ResultConceptT ResultConceptT; - typedef BaseT::PassConceptT PassConceptT; - -public: - // Most public APIs are inherited from the CRTP base class. - - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - CGSCCAnalysisManager() {} - CGSCCAnalysisManager(CGSCCAnalysisManager &&Arg) - : BaseT(std::move(static_cast(Arg))), - CGSCCAnalysisResults(std::move(Arg.CGSCCAnalysisResults)) {} - CGSCCAnalysisManager &operator=(CGSCCAnalysisManager &&RHS) { - BaseT::operator=(std::move(static_cast(RHS))); - CGSCCAnalysisResults = std::move(RHS.CGSCCAnalysisResults); - return *this; - } - - /// \brief Returns true if the analysis manager has an empty results cache. - bool empty() const; - - /// \brief Clear the function analysis result cache. - /// - /// This routine allows cleaning up when the set of functions itself has - /// potentially changed, and thus we can't even look up a a result and - /// invalidate it directly. Notably, this does *not* call invalidate - /// functions as there is nothing to be done for them. - void clear(); - -private: - CGSCCAnalysisManager(const CGSCCAnalysisManager &) LLVM_DELETED_FUNCTION; - CGSCCAnalysisManager & - operator=(const CGSCCAnalysisManager &) LLVM_DELETED_FUNCTION; - - /// \brief Get a function pass result, running the pass if necessary. - ResultConceptT &getResultImpl(void *PassID, LazyCallGraph::SCC *C); - - /// \brief Get a cached function pass result or return null. - ResultConceptT *getCachedResultImpl(void *PassID, - LazyCallGraph::SCC *C) const; - - /// \brief Invalidate a function pass result. - void invalidateImpl(void *PassID, LazyCallGraph::SCC *C); - - /// \brief Invalidate the results for a function.. - void invalidateImpl(LazyCallGraph::SCC *C, const PreservedAnalyses &PA); - - /// \brief List of function analysis pass IDs and associated concept pointers. - /// - /// Requires iterators to be valid across appending new entries and arbitrary - /// erases. Provides both the pass ID and concept pointer such that it is - /// half of a bijection and provides storage for the actual result concept. - typedef std::list< - std::pair>>> CGSCCAnalysisResultListT; - - /// \brief Map type from function pointer to our custom list type. - typedef DenseMap - CGSCCAnalysisResultListMapT; - - /// \brief Map from function to a list of function analysis results. - /// - /// Provides linear time removal of all analysis results for a function and - /// the ultimate storage for a particular cached analysis result. - CGSCCAnalysisResultListMapT CGSCCAnalysisResultLists; - - /// \brief Map type from a pair of analysis ID and function pointer to an - /// iterator into a particular result list. - typedef DenseMap, - CGSCCAnalysisResultListT::iterator> CGSCCAnalysisResultMapT; - - /// \brief Map from an analysis ID and function to a particular cached - /// analysis result. - CGSCCAnalysisResultMapT CGSCCAnalysisResults; -}; +/// \brief The CGSCC analysis manager. +/// +/// See the documentation for the AnalysisManager template for detail +/// documentation. This typedef serves as a convenient way to refer to this +/// construct in the adaptors and proxies used to integrate this into the larger +/// pass manager infrastructure. +typedef AnalysisManager CGSCCAnalysisManager; /// \brief A module analysis which acts as a proxy for a CGSCC analysis /// manager. @@ -187,7 +77,7 @@ public: /// Regardless of whether this analysis is marked as preserved, all of the /// analyses in the \c CGSCCAnalysisManager are potentially invalidated /// based on the set of preserved analyses. - bool invalidate(Module *M, const PreservedAnalyses &PA); + bool invalidate(Module &M, const PreservedAnalyses &PA); private: CGSCCAnalysisManager *CGAM; @@ -195,12 +85,13 @@ public: static void *ID() { return (void *)&PassID; } + static StringRef name() { return "CGSCCAnalysisManagerModuleProxy"; } + explicit CGSCCAnalysisManagerModuleProxy(CGSCCAnalysisManager &CGAM) : CGAM(&CGAM) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. - CGSCCAnalysisManagerModuleProxy( - const CGSCCAnalysisManagerModuleProxy &Arg) + CGSCCAnalysisManagerModuleProxy(const CGSCCAnalysisManagerModuleProxy &Arg) : CGAM(Arg.CGAM) {} CGSCCAnalysisManagerModuleProxy(CGSCCAnalysisManagerModuleProxy &&Arg) : CGAM(std::move(Arg.CGAM)) {} @@ -219,7 +110,7 @@ public: /// In debug builds, it will also assert that the analysis manager is empty /// as no queries should arrive at the CGSCC analysis manager prior to /// this analysis being requested. - Result run(Module *M); + Result run(Module &M); private: static char PassID; @@ -257,7 +148,7 @@ public: const ModuleAnalysisManager &getManager() const { return *MAM; } /// \brief Handle invalidation by ignoring it, this pass is immutable. - bool invalidate(LazyCallGraph::SCC *) { return false; } + bool invalidate(LazyCallGraph::SCC &) { return false; } private: const ModuleAnalysisManager *MAM; @@ -265,12 +156,13 @@ public: static void *ID() { return (void *)&PassID; } + static StringRef name() { return "ModuleAnalysisManagerCGSCCProxy"; } + ModuleAnalysisManagerCGSCCProxy(const ModuleAnalysisManager &MAM) : MAM(&MAM) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. - ModuleAnalysisManagerCGSCCProxy( - const ModuleAnalysisManagerCGSCCProxy &Arg) + ModuleAnalysisManagerCGSCCProxy(const ModuleAnalysisManagerCGSCCProxy &Arg) : MAM(Arg.MAM) {} ModuleAnalysisManagerCGSCCProxy(ModuleAnalysisManagerCGSCCProxy &&Arg) : MAM(std::move(Arg.MAM)) {} @@ -283,7 +175,7 @@ public: /// \brief Run the analysis pass and create our proxy result object. /// Nothing to see here, it just forwards the \c MAM reference into the /// result. - Result run(LazyCallGraph::SCC *) { return Result(*MAM); } + Result run(LazyCallGraph::SCC &) { return Result(*MAM); } private: static char PassID; @@ -323,7 +215,7 @@ public: } /// \brief Runs the CGSCC pass across every SCC in the module. - PreservedAnalyses run(Module *M, ModuleAnalysisManager *AM) { + PreservedAnalyses run(Module &M, ModuleAnalysisManager *AM) { assert(AM && "We need analyses to compute the call graph!"); // Setup the CGSCC analysis manager from its proxy. @@ -335,15 +227,17 @@ public: PreservedAnalyses PA = PreservedAnalyses::all(); for (LazyCallGraph::SCC &C : CG.postorder_sccs()) { - PreservedAnalyses PassPA = Pass.run(&C, &CGAM); + PreservedAnalyses PassPA = Pass.run(C, &CGAM); // We know that the CGSCC pass couldn't have invalidated any other // SCC's analyses (that's the contract of a CGSCC pass), so - // directly handle the CGSCC analysis manager's invalidation here. + // directly handle the CGSCC analysis manager's invalidation here. We + // also update the preserved set of analyses to reflect that invalidated + // analyses are now safe to preserve. // FIXME: This isn't quite correct. We need to handle the case where the // pass updated the CG, particularly some child of the current SCC, and // invalidate its analyses. - CGAM.invalidate(&C, PassPA); + PassPA = CGAM.invalidate(C, std::move(PassPA)); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -409,7 +303,7 @@ public: /// Regardless of whether this analysis is marked as preserved, all of the /// analyses in the \c FunctionAnalysisManager are potentially invalidated /// based on the set of preserved analyses. - bool invalidate(LazyCallGraph::SCC *C, const PreservedAnalyses &PA); + bool invalidate(LazyCallGraph::SCC &C, const PreservedAnalyses &PA); private: FunctionAnalysisManager *FAM; @@ -417,6 +311,8 @@ public: static void *ID() { return (void *)&PassID; } + static StringRef name() { return "FunctionAnalysisManagerCGSCCProxy"; } + explicit FunctionAnalysisManagerCGSCCProxy(FunctionAnalysisManager &FAM) : FAM(&FAM) {} // We have to explicitly define all the special member functions because MSVC @@ -441,7 +337,7 @@ public: /// In debug builds, it will also assert that the analysis manager is empty /// as no queries should arrive at the function analysis manager prior to /// this analysis being requested. - Result run(LazyCallGraph::SCC *C); + Result run(LazyCallGraph::SCC &C); private: static char PassID; @@ -479,7 +375,7 @@ public: const CGSCCAnalysisManager &getManager() const { return *CGAM; } /// \brief Handle invalidation by ignoring it, this pass is immutable. - bool invalidate(Function *) { return false; } + bool invalidate(Function &) { return false; } private: const CGSCCAnalysisManager *CGAM; @@ -487,6 +383,8 @@ public: static void *ID() { return (void *)&PassID; } + static StringRef name() { return "CGSCCAnalysisManagerFunctionProxy"; } + CGSCCAnalysisManagerFunctionProxy(const CGSCCAnalysisManager &CGAM) : CGAM(&CGAM) {} // We have to explicitly define all the special member functions because MSVC @@ -505,7 +403,7 @@ public: /// \brief Run the analysis pass and create our proxy result object. /// Nothing to see here, it just forwards the \c CGAM reference into the /// result. - Result run(Function *) { return Result(*CGAM); } + Result run(Function &) { return Result(*CGAM); } private: static char PassID; @@ -531,7 +429,8 @@ public: : Pass(Arg.Pass) {} CGSCCToFunctionPassAdaptor(CGSCCToFunctionPassAdaptor &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(CGSCCToFunctionPassAdaptor &LHS, CGSCCToFunctionPassAdaptor &RHS) { + friend void swap(CGSCCToFunctionPassAdaptor &LHS, + CGSCCToFunctionPassAdaptor &RHS) { using std::swap; swap(LHS.Pass, RHS.Pass); } @@ -541,21 +440,23 @@ public: } /// \brief Runs the function pass across every function in the module. - PreservedAnalyses run(LazyCallGraph::SCC *C, CGSCCAnalysisManager *AM) { + PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager *AM) { FunctionAnalysisManager *FAM = nullptr; if (AM) // Setup the function analysis manager from its proxy. FAM = &AM->getResult(C).getManager(); PreservedAnalyses PA = PreservedAnalyses::all(); - for (LazyCallGraph::Node *N : *C) { - PreservedAnalyses PassPA = Pass.run(&N->getFunction(), FAM); + for (LazyCallGraph::Node *N : C) { + PreservedAnalyses PassPA = Pass.run(N->getFunction(), FAM); // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. + // Also, update the preserved analyses to reflect that once invalidated + // these can again be preserved. if (FAM) - FAM->invalidate(&N->getFunction(), PassPA); + PassPA = FAM->invalidate(N->getFunction(), std::move(PassPA)); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -585,7 +486,6 @@ CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT Pass) { return std::move(CGSCCToFunctionPassAdaptor(std::move(Pass))); } - } #endif diff --git a/contrib/llvm/include/llvm/Analysis/CallGraph.h b/contrib/llvm/include/llvm/Analysis/CallGraph.h index 9a6a4a76eb73..76d9073799f8 100644 --- a/contrib/llvm/include/llvm/Analysis/CallGraph.h +++ b/contrib/llvm/include/llvm/Analysis/CallGraph.h @@ -58,7 +58,6 @@ #include "llvm/IR/Function.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" -#include "llvm/Support/IncludeFile.h" #include namespace llvm { @@ -418,13 +417,24 @@ template <> struct GraphTraits { template <> struct GraphTraits { typedef const CallGraphNode NodeType; - typedef NodeType::const_iterator ChildIteratorType; + + typedef CallGraphNode::CallRecord CGNPairTy; + typedef std::pointer_to_unary_function + CGNDerefFun; static NodeType *getEntryNode(const CallGraphNode *CGN) { return CGN; } + + typedef mapped_iterator + ChildIteratorType; + static inline ChildIteratorType child_begin(NodeType *N) { - return N->begin(); + return map_iterator(N->begin(), CGNDerefFun(CGNDeref)); } - static inline ChildIteratorType child_end(NodeType *N) { return N->end(); } + static inline ChildIteratorType child_end(NodeType *N) { + return map_iterator(N->end(), CGNDerefFun(CGNDeref)); + } + + static const CallGraphNode *CGNDeref(CGNPairTy P) { return P.second; } }; template <> @@ -451,17 +461,24 @@ template <> struct GraphTraits : public GraphTraits< const CallGraphNode *> { static NodeType *getEntryNode(const CallGraph *CGN) { - return CGN->getExternalCallingNode(); + return CGN->getExternalCallingNode(); // Start at the external node! } + typedef std::pair PairTy; + typedef std::pointer_to_unary_function + DerefFun; + // nodes_iterator/begin/end - Allow iteration over all nodes in the graph - typedef CallGraph::const_iterator nodes_iterator; - static nodes_iterator nodes_begin(const CallGraph *CG) { return CG->begin(); } - static nodes_iterator nodes_end(const CallGraph *CG) { return CG->end(); } + typedef mapped_iterator nodes_iterator; + static nodes_iterator nodes_begin(const CallGraph *CG) { + return map_iterator(CG->begin(), DerefFun(CGdereference)); + } + static nodes_iterator nodes_end(const CallGraph *CG) { + return map_iterator(CG->end(), DerefFun(CGdereference)); + } + + static const CallGraphNode &CGdereference(PairTy P) { return *P.second; } }; } // End llvm namespace -// Make sure that any clients of this file link in CallGraph.cpp -FORCE_DEFINING_FILE_TO_BE_LINKED(CallGraph) - #endif diff --git a/contrib/llvm/include/llvm/Analysis/CodeMetrics.h b/contrib/llvm/include/llvm/Analysis/CodeMetrics.h index 04b39c176946..2f5969129e02 100644 --- a/contrib/llvm/include/llvm/Analysis/CodeMetrics.h +++ b/contrib/llvm/include/llvm/Analysis/CodeMetrics.h @@ -16,10 +16,13 @@ #define LLVM_ANALYSIS_CODEMETRICS_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/IR/CallSite.h" namespace llvm { +class AssumptionCache; class BasicBlock; +class Loop; class Function; class Instruction; class DataLayout; @@ -85,7 +88,18 @@ struct CodeMetrics { NumInlineCandidates(0), NumVectorInsts(0), NumRets(0) {} /// \brief Add information about a block to the current state. - void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI); + void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI, + SmallPtrSetImpl &EphValues); + + /// \brief Collect a loop's ephemeral values (those used only by an assume + /// or similar intrinsics in the loop). + static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, + SmallPtrSetImpl &EphValues); + + /// \brief Collect a functions's ephemeral values (those used only by an + /// assume or similar intrinsics in the function). + static void collectEphemeralValues(const Function *L, AssumptionCache *AC, + SmallPtrSetImpl &EphValues); }; } diff --git a/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h b/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h index 53c832cbebee..cb74e9f32d3d 100644 --- a/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -66,15 +66,15 @@ public: bool runOnFunction(Function &F) override { GraphT Graph = AnalysisGraphTraitsT::getGraph(&getAnalysis()); std::string Filename = Name + "." + F.getName().str() + ".dot"; - std::string ErrorInfo; + std::error_code EC; errs() << "Writing '" << Filename << "'..."; - raw_fd_ostream File(Filename.c_str(), ErrorInfo, sys::fs::F_Text); + raw_fd_ostream File(Filename, EC, sys::fs::F_Text); std::string GraphName = DOTGraphTraits::getGraphName(Graph); std::string Title = GraphName + " for '" + F.getName().str() + "' function"; - if (ErrorInfo.empty()) + if (!EC) WriteGraph(File, Graph, IsSimple, Title); else errs() << " error opening file for writing!"; @@ -129,14 +129,14 @@ public: bool runOnModule(Module &M) override { GraphT Graph = AnalysisGraphTraitsT::getGraph(&getAnalysis()); std::string Filename = Name + ".dot"; - std::string ErrorInfo; + std::error_code EC; errs() << "Writing '" << Filename << "'..."; - raw_fd_ostream File(Filename.c_str(), ErrorInfo, sys::fs::F_Text); + raw_fd_ostream File(Filename, EC, sys::fs::F_Text); std::string Title = DOTGraphTraits::getGraphName(Graph); - if (ErrorInfo.empty()) + if (!EC) WriteGraph(File, Graph, IsSimple, Title); else errs() << " error opening file for writing!"; diff --git a/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h index 279755e47622..1041e3f0a4a9 100644 --- a/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -287,9 +287,9 @@ namespace llvm { /// The flag PossiblyLoopIndependent should be set by the caller /// if it appears that control flow can reach from Src to Dst /// without traversing a loop back edge. - Dependence *depends(Instruction *Src, - Instruction *Dst, - bool PossiblyLoopIndependent); + std::unique_ptr depends(Instruction *Src, + Instruction *Dst, + bool PossiblyLoopIndependent); /// getSplitIteration - Give a dependence that's splittable at some /// particular level, return the iteration that should be used to split @@ -331,7 +331,7 @@ namespace llvm { /// /// breaks the dependence and allows us to vectorize/parallelize /// both loops. - const SCEV *getSplitIteration(const Dependence *Dep, unsigned Level); + const SCEV *getSplitIteration(const Dependence &Dep, unsigned Level); private: AliasAnalysis *AA; @@ -523,6 +523,12 @@ namespace llvm { /// in LoopNest. bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const; + /// Makes sure both subscripts (i.e. Pair->Src and Pair->Dst) share the same + /// integer type by sign-extending one of them when necessary. + /// Sign-extending a subscript is safe because getelementptr assumes the + /// array subscripts are signed. + void unifySubscriptType(Subscript *Pair); + /// removeMatchingExtensions - Examines a subscript pair. /// If the source and destination are identically sign (or zero) /// extended, it strips off the extension in an effort to @@ -911,7 +917,7 @@ namespace llvm { bool tryDelinearize(const SCEV *SrcSCEV, const SCEV *DstSCEV, SmallVectorImpl &Pair, - const SCEV *ElementSize) const; + const SCEV *ElementSize); public: static char ID; // Class identification, replacement for typeinfo diff --git a/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h b/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h index f42b9cbbfedd..996700efdb60 100644 --- a/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h +++ b/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h @@ -102,7 +102,9 @@ public: void print(raw_ostream &OS) const; /// dump - Dump the dominance frontier to dbgs(). +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void dump() const; +#endif }; //===------------------------------------- diff --git a/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h b/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h index 04df2cc35d46..735bfb83671d 100644 --- a/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h +++ b/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h @@ -15,8 +15,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_ANALYSIS_DOMINANCEFRONTIER_IMPL_H -#define LLVM_ANALYSIS_DOMINANCEFRONTIER_IMPL_H +#ifndef LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H +#define LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/Debug.h" @@ -172,9 +172,7 @@ ForwardDominanceFrontierBase::calculate(const DomTreeT &DT, DomSetType &S = this->Frontiers[currentBB]; // Visit each block only once. - if (visited.count(currentBB) == 0) { - visited.insert(currentBB); - + if (visited.insert(currentBB).second) { // Loop over CFG successors to calculate DFlocal[currentNode] for (auto SI = BlockTraits::child_begin(currentBB), SE = BlockTraits::child_end(currentBB); diff --git a/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h b/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h deleted file mode 100644 index 574c947f4ebb..000000000000 --- a/contrib/llvm/include/llvm/Analysis/FindUsedTypes.h +++ /dev/null @@ -1,66 +0,0 @@ -//===- llvm/Analysis/FindUsedTypes.h - Find all Types in use ----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This pass is used to seek out all of the types in use by the program. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_ANALYSIS_FINDUSEDTYPES_H -#define LLVM_ANALYSIS_FINDUSEDTYPES_H - -#include "llvm/ADT/SetVector.h" -#include "llvm/Pass.h" - -namespace llvm { - -class Type; -class Value; - -class FindUsedTypes : public ModulePass { - SetVector UsedTypes; -public: - static char ID; // Pass identification, replacement for typeid - FindUsedTypes() : ModulePass(ID) { - initializeFindUsedTypesPass(*PassRegistry::getPassRegistry()); - } - - /// getTypes - After the pass has been run, return the set containing all of - /// the types used in the module. - /// - const SetVector &getTypes() const { return UsedTypes; } - - /// Print the types found in the module. If the optional Module parameter is - /// passed in, then the types are printed symbolically if possible, using the - /// symbol table from the module. - /// - void print(raw_ostream &o, const Module *M) const override; - -private: - /// IncorporateType - Incorporate one type and all of its subtypes into the - /// collection of used types. - /// - void IncorporateType(Type *Ty); - - /// IncorporateValue - Incorporate all of the types used by this value. - /// - void IncorporateValue(const Value *V); - -public: - /// run - This incorporates all types used by the specified module - bool runOnModule(Module &M) override; - - /// getAnalysisUsage - We do not modify anything. - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesAll(); - } -}; - -} // End llvm namespace - -#endif diff --git a/contrib/llvm/include/llvm/Analysis/FunctionTargetTransformInfo.h b/contrib/llvm/include/llvm/Analysis/FunctionTargetTransformInfo.h new file mode 100644 index 000000000000..fce5a1a92bd9 --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/FunctionTargetTransformInfo.h @@ -0,0 +1,49 @@ +//===- llvm/Analysis/FunctionTargetTransformInfo.h --------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass wraps a TargetTransformInfo in a FunctionPass so that it can +// forward along the current Function so that we can make target specific +// decisions based on the particular subtarget specified for each Function. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_FUNCTIONTARGETTRANSFORMINFO_H +#define LLVM_ANALYSIS_FUNCTIONTARGETTRANSFORMINFO_H + +#include "TargetTransformInfo.h" +#include "llvm/Pass.h" + +namespace llvm { +class FunctionTargetTransformInfo final : public FunctionPass { +private: + const Function *Fn; + const TargetTransformInfo *TTI; + + FunctionTargetTransformInfo(const FunctionTargetTransformInfo &) + LLVM_DELETED_FUNCTION; + void operator=(const FunctionTargetTransformInfo &) LLVM_DELETED_FUNCTION; + +public: + static char ID; + FunctionTargetTransformInfo(); + + // Implementation boilerplate. + void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override; + bool runOnFunction(Function &F) override; + + // Shimmed functions from TargetTransformInfo. + void + getUnrollingPreferences(Loop *L, + TargetTransformInfo::UnrollingPreferences &UP) const { + TTI->getUnrollingPreferences(Fn, L, UP); + } +}; +} +#endif diff --git a/contrib/llvm/include/llvm/Analysis/IVUsers.h b/contrib/llvm/include/llvm/Analysis/IVUsers.h index 6038872207c3..d1f037021773 100644 --- a/contrib/llvm/include/llvm/Analysis/IVUsers.h +++ b/contrib/llvm/include/llvm/Analysis/IVUsers.h @@ -174,7 +174,7 @@ public: /// dump - This method is used for debugging. void dump() const; protected: - bool AddUsersImpl(Instruction *I, SmallPtrSet &SimpleLoopNests); + bool AddUsersImpl(Instruction *I, SmallPtrSetImpl &SimpleLoopNests); }; Pass *createIVUsersPass(); diff --git a/contrib/llvm/include/llvm/Analysis/InlineCost.h b/contrib/llvm/include/llvm/Analysis/InlineCost.h index aaed716b6a11..a064cfc897b0 100644 --- a/contrib/llvm/include/llvm/Analysis/InlineCost.h +++ b/contrib/llvm/include/llvm/Analysis/InlineCost.h @@ -19,6 +19,7 @@ #include namespace llvm { +class AssumptionCacheTracker; class CallSite; class DataLayout; class Function; @@ -100,6 +101,7 @@ public: /// \brief Cost analyzer used by inliner. class InlineCostAnalysis : public CallGraphSCCPass { const TargetTransformInfo *TTI; + AssumptionCacheTracker *ACT; public: static char ID; diff --git a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h index 2367c0bf77aa..b88e0899f19a 100644 --- a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -37,6 +37,7 @@ namespace llvm { template class ArrayRef; + class AssumptionCache; class DominatorTree; class Instruction; class DataLayout; @@ -50,150 +51,193 @@ namespace llvm { Value *SimplifyAddInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifySubInst - Given operands for a Sub, see if we can /// fold the result. If not, this returns null. Value *SimplifySubInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// Given operands for an FAdd, see if we can fold the result. If not, this /// returns null. Value *SimplifyFAddInst(Value *LHS, Value *RHS, FastMathFlags FMF, - const DataLayout *TD = nullptr, - const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DataLayout *TD = nullptr, + const TargetLibraryInfo *TLI = nullptr, + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// Given operands for an FSub, see if we can fold the result. If not, this /// returns null. Value *SimplifyFSubInst(Value *LHS, Value *RHS, FastMathFlags FMF, - const DataLayout *TD = nullptr, - const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DataLayout *TD = nullptr, + const TargetLibraryInfo *TLI = nullptr, + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// Given operands for an FMul, see if we can fold the result. If not, this /// returns null. - Value *SimplifyFMulInst(Value *LHS, Value *RHS, - FastMathFlags FMF, + Value *SimplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyMulInst - Given operands for a Mul, see if we can /// fold the result. If not, this returns null. Value *SimplifyMulInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifySDivInst - Given operands for an SDiv, see if we can /// fold the result. If not, this returns null. Value *SimplifySDivInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyUDivInst - Given operands for a UDiv, see if we can /// fold the result. If not, this returns null. Value *SimplifyUDivInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyFDivInst - Given operands for an FDiv, see if we can /// fold the result. If not, this returns null. Value *SimplifyFDivInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifySRemInst - Given operands for an SRem, see if we can /// fold the result. If not, this returns null. Value *SimplifySRemInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyURemInst - Given operands for a URem, see if we can /// fold the result. If not, this returns null. Value *SimplifyURemInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyFRemInst - Given operands for an FRem, see if we can /// fold the result. If not, this returns null. Value *SimplifyFRemInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyShlInst - Given operands for a Shl, see if we can /// fold the result. If not, this returns null. Value *SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyLShrInst - Given operands for a LShr, see if we can /// fold the result. If not, this returns null. Value *SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyAShrInst - Given operands for a AShr, see if we can /// fold the result. If not, this returns null. Value *SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyAndInst - Given operands for an And, see if we can /// fold the result. If not, this returns null. Value *SimplifyAndInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyOrInst - Given operands for an Or, see if we can /// fold the result. If not, this returns null. Value *SimplifyOrInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyXorInst - Given operands for a Xor, see if we can /// fold the result. If not, this returns null. Value *SimplifyXorInst(Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyICmpInst - Given operands for an ICmpInst, see if we can /// fold the result. If not, this returns null. Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + Instruction *CxtI = nullptr); /// SimplifyFCmpInst - Given operands for an FCmpInst, see if we can /// fold the result. If not, this returns null. Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifySelectInst - Given operands for a SelectInst, see if we can fold /// the result. If not, this returns null. Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyGEPInst - Given operands for an GetElementPtrInst, see if we can /// fold the result. If not, this returns null. Value *SimplifyGEPInst(ArrayRef Ops, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyInsertValueInst - Given operands for an InsertValueInst, see if we /// can fold the result. If not, this returns null. @@ -201,13 +245,17 @@ namespace llvm { ArrayRef Idxs, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyTruncInst - Given operands for an TruncInst, see if we can fold /// the result. If not, this returns null. Value *SimplifyTruncInst(Value *Op, Type *Ty, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); //=== Helper functions for higher up the class hierarchy. @@ -217,14 +265,18 @@ namespace llvm { Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyBinOp - Given operands for a BinaryOperator, see if we can /// fold the result. If not, this returns null. Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// \brief Given a function and iterators over arguments, see if we can fold /// the result. @@ -233,7 +285,9 @@ namespace llvm { Value *SimplifyCall(Value *V, User::op_iterator ArgBegin, User::op_iterator ArgEnd, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// \brief Given a function and set of arguments, see if we can fold the /// result. @@ -242,14 +296,16 @@ namespace llvm { Value *SimplifyCall(Value *V, ArrayRef Args, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr); /// SimplifyInstruction - See if we can compute a simplified version of this /// instruction. If not, this returns null. Value *SimplifyInstruction(Instruction *I, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); - + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr); /// \brief Replace all uses of 'I' with 'SimpleV' and simplify the uses /// recursively. @@ -262,7 +318,8 @@ namespace llvm { bool replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr); /// \brief Recursively attempt to simplify an instruction. /// @@ -273,7 +330,8 @@ namespace llvm { bool recursivelySimplifyInstruction(Instruction *I, const DataLayout *TD = nullptr, const TargetLibraryInfo *TLI = nullptr, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + AssumptionCache *AC = nullptr); } // end namespace llvm #endif diff --git a/contrib/llvm/include/llvm/Analysis/IntervalIterator.h b/contrib/llvm/include/llvm/Analysis/IntervalIterator.h index 73aff76efe83..3b51d44099fa 100644 --- a/contrib/llvm/include/llvm/Analysis/IntervalIterator.h +++ b/contrib/llvm/include/llvm/Analysis/IntervalIterator.h @@ -165,10 +165,10 @@ private: // bool ProcessInterval(NodeTy *Node) { BasicBlock *Header = getNodeHeader(Node); - if (Visited.count(Header)) return false; + if (!Visited.insert(Header).second) + return false; Interval *Int = new Interval(Header); - Visited.insert(Header); // The header has now been visited! // Check all of our successors to see if they are in the interval... for (typename GT::ChildIteratorType I = GT::child_begin(Node), diff --git a/contrib/llvm/include/llvm/Analysis/JumpInstrTableInfo.h b/contrib/llvm/include/llvm/Analysis/JumpInstrTableInfo.h index 54760aa02466..591e794a3901 100644 --- a/contrib/llvm/include/llvm/Analysis/JumpInstrTableInfo.h +++ b/contrib/llvm/include/llvm/Analysis/JumpInstrTableInfo.h @@ -16,7 +16,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/Pass.h" - #include namespace llvm { @@ -37,7 +36,9 @@ class JumpInstrTableInfo : public ImmutablePass { public: static char ID; - JumpInstrTableInfo(); + /// The default byte alignment for jump tables is 16, which is large but + /// usually safe. + JumpInstrTableInfo(uint64_t ByteAlign = 16); virtual ~JumpInstrTableInfo(); const char *getPassName() const override { return "Jump-Instruction Table Info"; @@ -52,9 +53,19 @@ public: /// Gets the tables. const JumpTables &getTables() const { return Tables; } + /// Gets the alignment in bytes of a jumptable entry. + uint64_t entryByteAlignment() const { return ByteAlignment; } private: JumpTables Tables; + + /// A power-of-two alignment of a jumptable entry. + uint64_t ByteAlignment; }; + +/// Creates a JumpInstrTableInfo pass with the given bound on entry size. This +/// bound specifies the maximum number of bytes needed to represent an +/// unconditional jump or a trap instruction in the back end currently in use. +ModulePass *createJumpInstrTableInfoPass(unsigned Bound); } #endif /* LLVM_ANALYSIS_JUMPINSTRTABLEINFO_H */ diff --git a/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h b/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h index 70a4df51c1ed..b0b9068de34b 100644 --- a/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h +++ b/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h @@ -32,8 +32,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_ANALYSIS_LAZY_CALL_GRAPH -#define LLVM_ANALYSIS_LAZY_CALL_GRAPH +#ifndef LLVM_ANALYSIS_LAZYCALLGRAPH_H +#define LLVM_ANALYSIS_LAZYCALLGRAPH_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerUnion.h" @@ -46,11 +46,11 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/Support/Allocator.h" #include namespace llvm { -class ModuleAnalysisManager; class PreservedAnalyses; class raw_ostream; @@ -252,6 +252,12 @@ public: /// \brief Test if this SCC is a descendant of \a C. bool isDescendantOf(const SCC &C) const; + /// \brief Short name useful for debugging or logging. + /// + /// We use the name of the first function in the SCC to name the SCC for + /// the purposes of debugging and logging. + StringRef getName() const { return (*begin())->getFunction().getName(); } + ///@{ /// \name Mutation API /// @@ -537,11 +543,13 @@ public: static void *ID() { return (void *)&PassID; } - /// \brief Compute the \c LazyCallGraph for a the module \c M. + static StringRef name() { return "Lazy CallGraph Analysis"; } + + /// \brief Compute the \c LazyCallGraph for the module \c M. /// /// This just builds the set of entry points to the call graph. The rest is /// built lazily as it is walked. - LazyCallGraph run(Module *M) { return LazyCallGraph(*M); } + LazyCallGraph run(Module &M) { return LazyCallGraph(M); } private: static char PassID; @@ -556,7 +564,7 @@ class LazyCallGraphPrinterPass { public: explicit LazyCallGraphPrinterPass(raw_ostream &OS); - PreservedAnalyses run(Module *M, ModuleAnalysisManager *AM); + PreservedAnalyses run(Module &M, ModuleAnalysisManager *AM); static StringRef name() { return "LazyCallGraphPrinterPass"; } }; diff --git a/contrib/llvm/include/llvm/Analysis/LazyValueInfo.h b/contrib/llvm/include/llvm/Analysis/LazyValueInfo.h index 2fe7386e7302..9a67d52c3696 100644 --- a/contrib/llvm/include/llvm/Analysis/LazyValueInfo.h +++ b/contrib/llvm/include/llvm/Analysis/LazyValueInfo.h @@ -18,16 +18,20 @@ #include "llvm/Pass.h" namespace llvm { + class AssumptionCache; class Constant; class DataLayout; + class DominatorTree; + class Instruction; class TargetLibraryInfo; class Value; -/// LazyValueInfo - This pass computes, caches, and vends lazy value constraint -/// information. +/// This pass computes, caches, and vends lazy value constraint information. class LazyValueInfo : public FunctionPass { + AssumptionCache *AC; const DataLayout *DL; class TargetLibraryInfo *TLI; + DominatorTree *DT; void *PImpl; LazyValueInfo(const LazyValueInfo&) LLVM_DELETED_FUNCTION; void operator=(const LazyValueInfo&) LLVM_DELETED_FUNCTION; @@ -38,7 +42,7 @@ public: } ~LazyValueInfo() { assert(!PImpl && "releaseMemory not called"); } - /// Tristate - This is used to return true/false/dunno results. + /// This is used to return true/false/dunno results. enum Tristate { Unknown = -1, False = 0, True = 1 }; @@ -46,26 +50,33 @@ public: // Public query interface. - /// getPredicateOnEdge - Determine whether the specified value comparison - /// with a constant is known to be true or false on the specified CFG edge. + /// Determine whether the specified value comparison with a constant is known + /// to be true or false on the specified CFG edge. /// Pred is a CmpInst predicate. Tristate getPredicateOnEdge(unsigned Pred, Value *V, Constant *C, - BasicBlock *FromBB, BasicBlock *ToBB); + BasicBlock *FromBB, BasicBlock *ToBB, + Instruction *CxtI = nullptr); - - /// getConstant - Determine whether the specified value is known to be a + /// Determine whether the specified value comparison with a constant is known + /// to be true or false at the specified instruction + /// (from an assume intrinsic). Pred is a CmpInst predicate. + Tristate getPredicateAt(unsigned Pred, Value *V, Constant *C, + Instruction *CxtI); + + /// Determine whether the specified value is known to be a /// constant at the end of the specified block. Return null if not. - Constant *getConstant(Value *V, BasicBlock *BB); + Constant *getConstant(Value *V, BasicBlock *BB, Instruction *CxtI = nullptr); - /// getConstantOnEdge - Determine whether the specified value is known to be a + /// Determine whether the specified value is known to be a /// constant on the specified edge. Return null if not. - Constant *getConstantOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB); + Constant *getConstantOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB, + Instruction *CxtI = nullptr); - /// threadEdge - Inform the analysis cache that we have threaded an edge from + /// Inform the analysis cache that we have threaded an edge from /// PredBB to OldSucc to be from PredBB to NewSucc instead. void threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc, BasicBlock *NewSucc); - /// eraseBlock - Inform the analysis cache that we have erased a block. + /// Inform the analysis cache that we have erased a block. void eraseBlock(BasicBlock *BB); // Implementation boilerplate. diff --git a/contrib/llvm/include/llvm/Analysis/Loads.h b/contrib/llvm/include/llvm/Analysis/Loads.h index 25c59288f34b..0fe34539f8f7 100644 --- a/contrib/llvm/include/llvm/Analysis/Loads.h +++ b/contrib/llvm/include/llvm/Analysis/Loads.h @@ -44,14 +44,14 @@ bool isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, /// If it is set to 0, it will scan the whole block. You can also optionally /// specify an alias analysis implementation, which makes this more precise. /// -/// If TBAATag is non-null and a load or store is found, the TBAA tag from the -/// load or store is recorded there. If there is no TBAA tag or if no access +/// If AATags is non-null and a load or store is found, the AA tags from the +/// load or store are recorded there. If there are no AA tags or if no access /// is found, it is left unmodified. Value *FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan = 6, AliasAnalysis *AA = nullptr, - MDNode **TBAATag = nullptr); + AAMDNodes *AATags = nullptr); } diff --git a/contrib/llvm/include/llvm/Analysis/LoopPass.h b/contrib/llvm/include/llvm/Analysis/LoopPass.h index 726e28636ac6..8650000fcfb6 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopPass.h +++ b/contrib/llvm/include/llvm/Analysis/LoopPass.h @@ -82,6 +82,11 @@ public: /// deleteAnalysisValue - Delete analysis info associated with value V. virtual void deleteAnalysisValue(Value *V, Loop *L) {} + /// Delete analysis info associated with Loop L. + /// Called to notify a Pass that a loop has been deleted and any + /// associated analysis values can be deleted. + virtual void deleteAnalysisLoop(Loop *L) {} + protected: /// skipOptnoneFunction - Containing function has Attribute::OptimizeNone /// and most transformation passes should skip it. @@ -152,6 +157,10 @@ public: /// that implement simple analysis interface. void deleteSimpleAnalysisValue(Value *V, Loop *L); + /// Invoke deleteAnalysisLoop hook for all passes that implement simple + /// analysis interface. + void deleteSimpleAnalysisLoop(Loop *L); + private: std::deque LQ; bool skipThisLoop; diff --git a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h index 1c4441bea670..67fd70a4561f 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -28,6 +28,7 @@ namespace llvm { class Instruction; class CallSite; class AliasAnalysis; + class AssumptionCache; class DataLayout; class MemoryDependenceAnalysis; class PredIteratorCache; @@ -281,12 +282,12 @@ namespace llvm { /// Size - The maximum size of the dereferences of the /// pointer. May be UnknownSize if the sizes are unknown. uint64_t Size; - /// TBAATag - The TBAA tag associated with dereferences of the - /// pointer. May be null if there are no tags or conflicting tags. - const MDNode *TBAATag; + /// AATags - The AA tags associated with dereferences of the + /// pointer. The members may be null if there are no tags or + /// conflicting tags. + AAMDNodes AATags; - NonLocalPointerInfo() - : Size(AliasAnalysis::UnknownSize), TBAATag(nullptr) {} + NonLocalPointerInfo() : Size(AliasAnalysis::UnknownSize) {} }; /// CachedNonLocalPointerInfo - This map stores the cached results of doing @@ -325,6 +326,7 @@ namespace llvm { AliasAnalysis *AA; const DataLayout *DL; DominatorTree *DT; + AssumptionCache *AC; std::unique_ptr PredCache; public: @@ -364,12 +366,16 @@ namespace llvm { /// getNonLocalPointerDependency - Perform a full dependency query for an - /// access to the specified (non-volatile) memory location, returning the - /// set of instructions that either define or clobber the value. + /// access to the QueryInst's specified memory location, returning the set + /// of instructions that either define or clobber the value. /// - /// This method assumes the pointer has a "NonLocal" dependency within BB. - void getNonLocalPointerDependency(const AliasAnalysis::Location &Loc, - bool isLoad, BasicBlock *BB, + /// Warning: For a volatile query instruction, the dependencies will be + /// accurate, and thus usable for reordering, but it is never legal to + /// remove the query instruction. + /// + /// This method assumes the pointer has a "NonLocal" dependency within + /// QueryInst's parent basic block. + void getNonLocalPointerDependency(Instruction *QueryInst, SmallVectorImpl &Result); /// removeInstruction - Remove an instruction from the dependence analysis, diff --git a/contrib/llvm/include/llvm/Analysis/PHITransAddr.h b/contrib/llvm/include/llvm/Analysis/PHITransAddr.h index 69f59071f94f..38730d8ea4f3 100644 --- a/contrib/llvm/include/llvm/Analysis/PHITransAddr.h +++ b/contrib/llvm/include/llvm/Analysis/PHITransAddr.h @@ -18,6 +18,7 @@ #include "llvm/IR/Instruction.h" namespace llvm { + class AssumptionCache; class DominatorTree; class DataLayout; class TargetLibraryInfo; @@ -41,12 +42,15 @@ class PHITransAddr { /// TLI - The target library info if known, otherwise null. const TargetLibraryInfo *TLI; - + + /// A cache of @llvm.assume calls used by SimplifyInstruction. + AssumptionCache *AC; + /// InstInputs - The inputs for our symbolic address. SmallVector InstInputs; public: - PHITransAddr(Value *addr, const DataLayout *DL) - : Addr(addr), DL(DL), TLI(nullptr) { + PHITransAddr(Value *addr, const DataLayout *DL, AssumptionCache *AC) + : Addr(addr), DL(DL), TLI(nullptr), AC(AC) { // If the address is an instruction, the whole thing is considered an input. if (Instruction *I = dyn_cast(Addr)) InstInputs.push_back(I); diff --git a/contrib/llvm/include/llvm/Analysis/Passes.h b/contrib/llvm/include/llvm/Analysis/Passes.h index fd65ae5ca5b2..10a56059ae10 100644 --- a/contrib/llvm/include/llvm/Analysis/Passes.h +++ b/contrib/llvm/include/llvm/Analysis/Passes.h @@ -64,6 +64,13 @@ namespace llvm { // ImmutablePass *createBasicAliasAnalysisPass(); + //===--------------------------------------------------------------------===// + // + // createCFLAliasAnalysisPass - This pass implements a set-based approach to + // alias analysis. + // + ImmutablePass *createCFLAliasAnalysisPass(); + //===--------------------------------------------------------------------===// // /// createLibCallAliasAnalysisPass - Create an alias analysis pass that knows @@ -86,6 +93,13 @@ namespace llvm { // ImmutablePass *createTypeBasedAliasAnalysisPass(); + //===--------------------------------------------------------------------===// + // + // createScopedNoAliasAAPass - This pass implements metadata-based + // scoped noalias analysis. + // + ImmutablePass *createScopedNoAliasAAPass(); + //===--------------------------------------------------------------------===// // // createObjCARCAliasAnalysisPass - This pass implements ObjC-ARC-based @@ -93,6 +107,8 @@ namespace llvm { // ImmutablePass *createObjCARCAliasAnalysisPass(); + FunctionPass *createPAEvalPass(); + //===--------------------------------------------------------------------===// // /// createLazyValueInfoPass - This creates an instance of the LazyValueInfo diff --git a/contrib/llvm/include/llvm/Analysis/PostDominators.h b/contrib/llvm/include/llvm/Analysis/PostDominators.h index d330755a0871..72cd35754c8a 100644 --- a/contrib/llvm/include/llvm/Analysis/PostDominators.h +++ b/contrib/llvm/include/llvm/Analysis/PostDominators.h @@ -19,7 +19,7 @@ namespace llvm { /// PostDominatorTree Class - Concrete subclass of DominatorTree that is used to -/// compute the a post-dominator tree. +/// compute the post-dominator tree. /// struct PostDominatorTree : public FunctionPass { static char ID; // Pass identification, replacement for typeid diff --git a/contrib/llvm/include/llvm/Analysis/RegionInfo.h b/contrib/llvm/include/llvm/Analysis/RegionInfo.h index 49c88fd5caeb..6ff7f97d01f5 100644 --- a/contrib/llvm/include/llvm/Analysis/RegionInfo.h +++ b/contrib/llvm/include/llvm/Analysis/RegionInfo.h @@ -424,8 +424,10 @@ public: void print(raw_ostream &OS, bool printTree = true, unsigned level = 0, PrintStyle Style = PrintNone) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// @brief Print the region to stderr. void dump() const; +#endif /// @brief Check if the region contains a BasicBlock. /// @@ -732,7 +734,9 @@ public: static typename RegionT::PrintStyle printStyle; void print(raw_ostream &OS) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void dump() const; +#endif void releaseMemory(); diff --git a/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h b/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h index 4266b84c32c4..b0dc26312aaa 100644 --- a/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h @@ -12,11 +12,11 @@ #ifndef LLVM_ANALYSIS_REGIONINFOIMPL_H #define LLVM_ANALYSIS_REGIONINFOIMPL_H -#include "llvm/Analysis/RegionInfo.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/PostDominators.h" +#include "llvm/Analysis/RegionInfo.h" #include "llvm/Analysis/RegionIterator.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -25,7 +25,7 @@ #include #include -using namespace llvm; +namespace llvm { #define DEBUG_TYPE "region" @@ -916,4 +916,8 @@ void RegionInfoBase::calculate(FuncT &F) { buildRegionsTree(DT->getNode(BB), TopLevelRegion); } +#undef DEBUG_TYPE + +} // end namespace llvm + #endif diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h index 617e54541ee1..f394e335257f 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -35,6 +35,7 @@ namespace llvm { class APInt; + class AssumptionCache; class Constant; class ConstantInt; class DominatorTree; @@ -128,9 +129,11 @@ namespace llvm { /// purposes. void print(raw_ostream &OS) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// dump - This method is used for debugging. /// void dump() const; +#endif }; // Specialize FoldingSetTrait for SCEV to avoid needing to compute @@ -221,6 +224,9 @@ namespace llvm { /// Function *F; + /// The tracker for @llvm.assume intrinsics in this function. + AssumptionCache *AC; + /// LI - The loop information for the function we are currently analyzing. /// LoopInfo *LI; @@ -257,24 +263,13 @@ namespace llvm { /// loop exit's branch condition evaluates to the not-taken path. This is a /// temporary pair of exact and max expressions that are eventually /// summarized in ExitNotTakenInfo and BackedgeTakenInfo. - /// - /// If MustExit is true, then the exit must be taken when the BECount - /// reaches Exact (and before surpassing Max). If MustExit is false, then - /// BECount may exceed Exact or Max if the loop exits via another branch. In - /// either case, the loop may exit early via another branch. - /// - /// MustExit is true for most cases. However, an exit guarded by an - /// (in)equality on a nonunit stride may be skipped. struct ExitLimit { const SCEV *Exact; const SCEV *Max; - bool MustExit; - /*implicit*/ ExitLimit(const SCEV *E) - : Exact(E), Max(E), MustExit(true) {} + /*implicit*/ ExitLimit(const SCEV *E) : Exact(E), Max(E) {} - ExitLimit(const SCEV *E, const SCEV *M, bool MustExit) - : Exact(E), Max(M), MustExit(MustExit) {} + ExitLimit(const SCEV *E, const SCEV *M) : Exact(E), Max(M) {} /// hasAnyInfo - Test whether this ExitLimit contains any computed /// information, or whether it's all SCEVCouldNotCompute values. @@ -749,6 +744,13 @@ namespace llvm { bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS); + /// \brief Returns the maximum trip count of the loop if it is a single-exit + /// loop and we can compute a small maximum for that loop. + /// + /// Implemented in terms of the \c getSmallConstantTripCount overload with + /// the single exiting block passed to it. See that routine for details. + unsigned getSmallConstantTripCount(Loop *L); + /// getSmallConstantTripCount - Returns the maximum trip count of this loop /// as a normal unsigned value. Returns 0 if the trip count is unknown or /// not constant. This "trip count" assumes that control exits via @@ -758,6 +760,14 @@ namespace llvm { /// the loop exits prematurely via another branch. unsigned getSmallConstantTripCount(Loop *L, BasicBlock *ExitingBlock); + /// \brief Returns the largest constant divisor of the trip count of the + /// loop if it is a single-exit loop and we can compute a small maximum for + /// that loop. + /// + /// Implemented in terms of the \c getSmallConstantTripMultiple overload with + /// the single exiting block passed to it. See that routine for details. + unsigned getSmallConstantTripMultiple(Loop *L); + /// getSmallConstantTripMultiple - Returns the largest constant divisor of /// the trip count of this loop as a normal unsigned value, if /// possible. This means that the actual trip count is always a multiple of diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index 2f1b1c3841f3..ff82db19b9e7 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -14,8 +14,8 @@ #ifndef LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H #define LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H -#include "llvm/ADT/iterator_range.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Support/ErrorHandling.h" @@ -577,7 +577,7 @@ namespace llvm { SmallPtrSet Visited; void push(const SCEV *S) { - if (Visited.insert(S) && Visitor.follow(S)) + if (Visited.insert(S).second && Visitor.follow(S)) Worklist.push_back(S); } public: @@ -624,7 +624,7 @@ namespace llvm { } }; - /// Use SCEVTraversal to visit all nodes in the givien expression tree. + /// Use SCEVTraversal to visit all nodes in the given expression tree. template void visitAll(const SCEV *Root, SV& Visitor) { SCEVTraversal T(Visitor); diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h index f57f3eb009a1..4bd5dd8a221e 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -28,6 +28,7 @@ namespace llvm { +class Function; class GlobalValue; class Loop; class Type; @@ -183,7 +184,7 @@ public: /// should probably move to simpler cost metrics using the above. /// Alternatively, we could split the cost interface into distinct code-size /// and execution-speed costs. This would allow modelling the core of this - /// query more accurately as the a call is a single small instruction, but + /// query more accurately as a call is a single small instruction, but /// incurs significant execution cost. virtual bool isLoweredToCall(const Function *F) const; @@ -227,7 +228,8 @@ public: /// \brief Get target-customized preferences for the generic loop unrolling /// transformation. The caller will initialize UP with the current /// target-independent defaults. - virtual void getUnrollingPreferences(Loop *L, UnrollingPreferences &UP) const; + virtual void getUnrollingPreferences(const Function *F, Loop *L, + UnrollingPreferences &UP) const; /// @} @@ -268,6 +270,13 @@ public: int64_t BaseOffset, bool HasBaseReg, int64_t Scale) const; + /// \brief Return true if the target works with masked instruction + /// AVX2 allows masks for consecutive load and store for i32 and i64 elements. + /// AVX-512 architecture will also allow masks for non-consecutive memory + /// accesses. + virtual bool isLegalMaskedStore(Type *DataType, int Consecutive) const; + virtual bool isLegalMaskedLoad (Type *DataType, int Consecutive) const; + /// \brief Return the cost of the scaling factor used in the addressing /// mode represented by AM for this target, for a load/store /// of the specified type. @@ -335,6 +344,9 @@ public: OK_NonUniformConstantValue // Operand is a non uniform constant value. }; + /// \brief Additional properties of an operand's values. + enum OperandValueProperties { OP_None = 0, OP_PowerOf2 = 1 }; + /// \return The number of scalar or vector registers that the target has. /// If 'Vectors' is true, it returns the number of vector registers. If it is /// set to false, it returns the number of scalar registers. @@ -343,15 +355,18 @@ public: /// \return The width of the largest scalar or vector register type. virtual unsigned getRegisterBitWidth(bool Vector) const; - /// \return The maximum unroll factor that the vectorizer should try to + /// \return The maximum interleave factor that any transform should try to /// perform for this target. This number depends on the level of parallelism /// and the number of execution units in the CPU. - virtual unsigned getMaximumUnrollFactor() const; + virtual unsigned getMaxInterleaveFactor() const; /// \return The expected cost of arithmetic ops, such as mul, xor, fsub, etc. - virtual unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, - OperandValueKind Opd1Info = OK_AnyValue, - OperandValueKind Opd2Info = OK_AnyValue) const; + virtual unsigned + getArithmeticInstrCost(unsigned Opcode, Type *Ty, + OperandValueKind Opd1Info = OK_AnyValue, + OperandValueKind Opd2Info = OK_AnyValue, + OperandValueProperties Opd1PropInfo = OP_None, + OperandValueProperties Opd2PropInfo = OP_None) const; /// \return The cost of a shuffle instruction of kind Kind and of type Tp. /// The index and subtype parameters are used by the subvector insertion and @@ -416,6 +431,13 @@ public: virtual unsigned getAddressComputationCost(Type *Ty, bool IsComplex = false) const; + /// \returns The cost, if any, of keeping values of the given types alive + /// over a callsite. + /// + /// Some types may require the use of register classes that do not have + /// any callee-saved registers, so would require a spill and fill. + virtual unsigned getCostOfKeepingLiveOverCall(ArrayRef Tys) const; + /// @} /// Analysis group identification. diff --git a/contrib/llvm/include/llvm/Analysis/ValueTracking.h b/contrib/llvm/include/llvm/Analysis/ValueTracking.h index 83b5408fb1c2..cc588381727d 100644 --- a/contrib/llvm/include/llvm/Analysis/ValueTracking.h +++ b/contrib/llvm/include/llvm/Analysis/ValueTracking.h @@ -25,6 +25,8 @@ namespace llvm { class DataLayout; class StringRef; class MDNode; + class AssumptionCache; + class DominatorTree; class TargetLibraryInfo; /// Determine which bits of V are known to be either zero or one and return @@ -35,8 +37,11 @@ namespace llvm { /// where V is a vector, the known zero and known one values are the /// same width as the vector element, and the bit is set only if it is true /// for all of the elements in the vector. - void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, - const DataLayout *TD = nullptr, unsigned Depth = 0); + void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, + const DataLayout *TD = nullptr, unsigned Depth = 0, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); /// Compute known bits from the range metadata. /// \p KnownZero the set of bits that are known to be zero void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, @@ -45,21 +50,29 @@ namespace llvm { /// ComputeSignBit - Determine whether the sign bit is known to be zero or /// one. Convenience wrapper around computeKnownBits. void ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne, - const DataLayout *TD = nullptr, unsigned Depth = 0); + const DataLayout *TD = nullptr, unsigned Depth = 0, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); /// isKnownToBeAPowerOfTwo - Return true if the given value is known to have /// exactly one bit set when defined. For vectors return true if every /// element is known to be a power of two when defined. Supports values with /// integer or pointer type and vectors of integers. If 'OrZero' is set then /// returns true if the given value is either a power of two or zero. - bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero = false, unsigned Depth = 0); + bool isKnownToBeAPowerOfTwo(Value *V, bool OrZero = false, unsigned Depth = 0, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); /// isKnownNonZero - Return true if the given value is known to be non-zero /// when defined. For vectors return true if every element is known to be /// non-zero when defined. Supports values with integer or pointer type and /// vectors of integers. bool isKnownNonZero(Value *V, const DataLayout *TD = nullptr, - unsigned Depth = 0); + unsigned Depth = 0, AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use /// this predicate to simplify operations downstream. Mask is known to be @@ -70,10 +83,12 @@ namespace llvm { /// where V is a vector, the mask, known zero, and known one values are the /// same width as the vector element, and the bit is set only if it is true /// for all of the elements in the vector. - bool MaskedValueIsZero(Value *V, const APInt &Mask, - const DataLayout *TD = nullptr, unsigned Depth = 0); + bool MaskedValueIsZero(Value *V, const APInt &Mask, + const DataLayout *TD = nullptr, unsigned Depth = 0, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); - /// ComputeNumSignBits - Return the number of times the sign bit of the /// register is replicated into the other bits. We know that at least 1 bit /// is always equal to the sign bit (itself), but other cases can give us @@ -83,7 +98,9 @@ namespace llvm { /// 'Op' must have a scalar integer type. /// unsigned ComputeNumSignBits(Value *Op, const DataLayout *TD = nullptr, - unsigned Depth = 0); + unsigned Depth = 0, AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); /// ComputeMultiple - This function computes the integer multiple of Base that /// equals V. If successful, it returns true and returns the multiple in @@ -191,6 +208,24 @@ namespace llvm { /// and byval arguments. bool isKnownNonNull(const Value *V, const TargetLibraryInfo *TLI = nullptr); + /// Return true if it is valid to use the assumptions provided by an + /// assume intrinsic, I, at the point in the control-flow identified by the + /// context instruction, CxtI. + bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, + const DataLayout *DL = nullptr, + const DominatorTree *DT = nullptr); + + enum class OverflowResult { AlwaysOverflows, MayOverflow, NeverOverflows }; + OverflowResult computeOverflowForUnsignedMul(Value *LHS, Value *RHS, + const DataLayout *DL, + AssumptionCache *AC, + const Instruction *CxtI, + const DominatorTree *DT); + OverflowResult computeOverflowForUnsignedAdd(Value *LHS, Value *RHS, + const DataLayout *DL, + AssumptionCache *AC, + const Instruction *CxtI, + const DominatorTree *DT); } // end namespace llvm #endif diff --git a/contrib/llvm/include/llvm/AsmParser/Parser.h b/contrib/llvm/include/llvm/AsmParser/Parser.h index 165c46d6f271..7ef78d73da17 100644 --- a/contrib/llvm/include/llvm/AsmParser/Parser.h +++ b/contrib/llvm/include/llvm/AsmParser/Parser.h @@ -14,12 +14,11 @@ #ifndef LLVM_ASMPARSER_PARSER_H #define LLVM_ASMPARSER_PARSER_H -#include +#include "llvm/Support/MemoryBuffer.h" namespace llvm { class Module; -class MemoryBuffer; class SMDiagnostic; class LLVMContext; @@ -29,11 +28,12 @@ class LLVMContext; /// that this does not verify that the generated Module is valid, so you should /// run the verifier after parsing the file to check that it is okay. /// @brief Parse LLVM Assembly from a file -Module *ParseAssemblyFile( - const std::string &Filename, ///< The name of the file to parse - SMDiagnostic &Error, ///< Error result info. - LLVMContext &Context ///< Context in which to allocate globals info. -); +/// @param Filename The name of the file to parse +/// @param Error Error result info. +/// @param Context Context in which to allocate globals info. +std::unique_ptr parseAssemblyFile(StringRef Filename, + SMDiagnostic &Error, + LLVMContext &Context); /// The function is a secondary interface to the LLVM Assembly Parser. It parses /// an ASCII string that (presumably) contains LLVM Assembly code. It returns a @@ -41,23 +41,31 @@ Module *ParseAssemblyFile( /// that this does not verify that the generated Module is valid, so you should /// run the verifier after parsing the file to check that it is okay. /// @brief Parse LLVM Assembly from a string -Module *ParseAssemblyString( - const char *AsmString, ///< The string containing assembly - Module *M, ///< A module to add the assembly too. - SMDiagnostic &Error, ///< Error result info. - LLVMContext &Context -); +/// @param AsmString The string containing assembly +/// @param Error Error result info. +/// @param Context Context in which to allocate globals info. +std::unique_ptr parseAssemblyString(StringRef AsmString, + SMDiagnostic &Error, + LLVMContext &Context); + +/// parseAssemblyFile and parseAssemblyString are wrappers around this function. +/// @brief Parse LLVM Assembly from a MemoryBuffer. +/// @param F The MemoryBuffer containing assembly +/// @param Err Error result info. +/// @param Context Context in which to allocate globals info. +std::unique_ptr parseAssembly(MemoryBufferRef F, SMDiagnostic &Err, + LLVMContext &Context); /// This function is the low-level interface to the LLVM Assembly Parser. -/// ParseAssemblyFile and ParseAssemblyString are wrappers around this function. -/// @brief Parse LLVM Assembly from a MemoryBuffer. This function *always* -/// takes ownership of the MemoryBuffer. -Module *ParseAssembly( - MemoryBuffer *F, ///< The MemoryBuffer containing assembly - Module *M, ///< A module to add the assembly too. - SMDiagnostic &Err, ///< Error result info. - LLVMContext &Context -); +/// This is kept as an independent function instead of being inlined into +/// parseAssembly for the convenience of interactive users that want to add +/// recently parsed bits to an existing module. +/// +/// @param F The MemoryBuffer containing assembly +/// @param M The module to add data to. +/// @param Err Error result info. +/// @return true on error. +bool parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err); } // End llvm namespace diff --git a/contrib/llvm/include/llvm/Bitcode/BitCodes.h b/contrib/llvm/include/llvm/Bitcode/BitCodes.h index b510daf33147..ed2dcf84f924 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitCodes.h +++ b/contrib/llvm/include/llvm/Bitcode/BitCodes.h @@ -18,6 +18,7 @@ #ifndef LLVM_BITCODE_BITCODES_H #define LLVM_BITCODE_BITCODES_H +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/ErrorHandling.h" @@ -161,16 +162,13 @@ template <> struct isPodLike { static const bool value=true; }; /// BitCodeAbbrev - This class represents an abbreviation record. An /// abbreviation allows a complex record that has redundancy to be stored in a /// specialized format instead of the fully-general, fully-vbr, format. -class BitCodeAbbrev { +class BitCodeAbbrev : public RefCountedBase { SmallVector OperandList; - unsigned char RefCount; // Number of things using this. ~BitCodeAbbrev() {} + // Only RefCountedBase is allowed to delete. + friend class RefCountedBase; + public: - BitCodeAbbrev() : RefCount(1) {} - - void addRef() { ++RefCount; } - void dropRef() { if (--RefCount == 0) delete this; } - unsigned getNumOperandInfos() const { return static_cast(OperandList.size()); } diff --git a/contrib/llvm/include/llvm/Bitcode/BitcodeWriterPass.h b/contrib/llvm/include/llvm/Bitcode/BitcodeWriterPass.h index 898cd523bd08..8fe9b7e8434d 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitcodeWriterPass.h +++ b/contrib/llvm/include/llvm/Bitcode/BitcodeWriterPass.h @@ -12,8 +12,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_BITCODE_BITCODE_WRITER_PASS_H -#define LLVM_BITCODE_BITCODE_WRITER_PASS_H +#ifndef LLVM_BITCODE_BITCODEWRITERPASS_H +#define LLVM_BITCODE_BITCODEWRITERPASS_H #include "llvm/ADT/StringRef.h" @@ -41,7 +41,7 @@ public: /// \brief Run the bitcode writer pass, and output the module to the selected /// output stream. - PreservedAnalyses run(Module *M); + PreservedAnalyses run(Module &M); static StringRef name() { return "BitcodeWriterPass"; } }; diff --git a/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h b/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h index 6f478b72127d..865a3e668428 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h +++ b/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h @@ -17,39 +17,37 @@ #include "llvm/Bitcode/BitCodes.h" #include "llvm/Support/Endian.h" -#include "llvm/Support/StreamableMemoryObject.h" +#include "llvm/Support/StreamingMemoryObject.h" #include #include #include namespace llvm { - class Deserializer; +class Deserializer; -/// BitstreamReader - This class is used to read from an LLVM bitcode stream, -/// maintaining information that is global to decoding the entire file. While -/// a file is being read, multiple cursors can be independently advanced or -/// skipped around within the file. These are represented by the -/// BitstreamCursor class. +/// This class is used to read from an LLVM bitcode stream, maintaining +/// information that is global to decoding the entire file. While a file is +/// being read, multiple cursors can be independently advanced or skipped around +/// within the file. These are represented by the BitstreamCursor class. class BitstreamReader { public: - /// BlockInfo - This contains information emitted to BLOCKINFO_BLOCK blocks. - /// These describe abbreviations that all blocks of the specified ID inherit. + /// This contains information emitted to BLOCKINFO_BLOCK blocks. These + /// describe abbreviations that all blocks of the specified ID inherit. struct BlockInfo { unsigned BlockID; - std::vector Abbrevs; + std::vector> Abbrevs; std::string Name; std::vector > RecordNames; }; private: - std::unique_ptr BitcodeBytes; + std::unique_ptr BitcodeBytes; std::vector BlockInfoRecords; - /// IgnoreBlockInfoNames - This is set to true if we don't care about the - /// block/record name information in the BlockInfo block. Only llvm-bcanalyzer - /// uses this. + /// This is set to true if we don't care about the block/record name + /// information in the BlockInfo block. Only llvm-bcanalyzer uses this. bool IgnoreBlockInfoNames; BitstreamReader(const BitstreamReader&) LLVM_DELETED_FUNCTION; @@ -58,13 +56,24 @@ public: BitstreamReader() : IgnoreBlockInfoNames(true) { } - BitstreamReader(const unsigned char *Start, const unsigned char *End) { - IgnoreBlockInfoNames = true; + BitstreamReader(const unsigned char *Start, const unsigned char *End) + : IgnoreBlockInfoNames(true) { init(Start, End); } - BitstreamReader(StreamableMemoryObject *bytes) { - BitcodeBytes.reset(bytes); + BitstreamReader(std::unique_ptr BitcodeBytes) + : BitcodeBytes(std::move(BitcodeBytes)), IgnoreBlockInfoNames(true) {} + + BitstreamReader(BitstreamReader &&Other) { + *this = std::move(Other); + } + + BitstreamReader &operator=(BitstreamReader &&Other) { + BitcodeBytes = std::move(Other.BitcodeBytes); + // Explicitly swap block info, so that nothing gets destroyed twice. + std::swap(BlockInfoRecords, Other.BlockInfoRecords); + IgnoreBlockInfoNames = Other.IgnoreBlockInfoNames; + return *this; } void init(const unsigned char *Start, const unsigned char *End) { @@ -72,22 +81,9 @@ public: BitcodeBytes.reset(getNonStreamedMemoryObject(Start, End)); } - StreamableMemoryObject &getBitcodeBytes() { return *BitcodeBytes; } + MemoryObject &getBitcodeBytes() { return *BitcodeBytes; } - ~BitstreamReader() { - // Free the BlockInfoRecords. - while (!BlockInfoRecords.empty()) { - BlockInfo &Info = BlockInfoRecords.back(); - // Free blockinfo abbrev info. - for (unsigned i = 0, e = static_cast(Info.Abbrevs.size()); - i != e; ++i) - Info.Abbrevs[i]->dropRef(); - BlockInfoRecords.pop_back(); - } - } - - /// CollectBlockInfoNames - This is called by clients that want block/record - /// name information. + /// This is called by clients that want block/record name information. void CollectBlockInfoNames() { IgnoreBlockInfoNames = false; } bool isIgnoringBlockInfoNames() { return IgnoreBlockInfoNames; } @@ -95,13 +91,13 @@ public: // Block Manipulation //===--------------------------------------------------------------------===// - /// hasBlockInfoRecords - Return true if we've already read and processed the - /// block info block for this Bitstream. We only process it for the first - /// cursor that walks over it. + /// Return true if we've already read and processed the block info block for + /// this Bitstream. We only process it for the first cursor that walks over + /// it. bool hasBlockInfoRecords() const { return !BlockInfoRecords.empty(); } - /// getBlockInfo - If there is block info for the specified ID, return it, - /// otherwise return null. + /// If there is block info for the specified ID, return it, otherwise return + /// null. const BlockInfo *getBlockInfo(unsigned BlockID) const { // Common case, the most recent entry matches BlockID. if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID) @@ -123,23 +119,26 @@ public: BlockInfoRecords.back().BlockID = BlockID; return BlockInfoRecords.back(); } + + /// Takes block info from the other bitstream reader. + /// + /// This is a "take" operation because BlockInfo records are non-trivial, and + /// indeed rather expensive. + void takeBlockInfo(BitstreamReader &&Other) { + assert(!hasBlockInfoRecords()); + BlockInfoRecords = std::move(Other.BlockInfoRecords); + } }; - -/// BitstreamEntry - When advancing through a bitstream cursor, each advance can -/// discover a few different kinds of entries: -/// Error - Malformed bitcode was found. -/// EndBlock - We've reached the end of the current block, (or the end of the -/// file, which is treated like a series of EndBlock records. -/// SubBlock - This is the start of a new subblock of a specific ID. -/// Record - This is a record with a specific AbbrevID. -/// +/// When advancing through a bitstream cursor, each advance can discover a few +/// different kinds of entries: struct BitstreamEntry { enum { - Error, - EndBlock, - SubBlock, - Record + Error, // Malformed bitcode was found. + EndBlock, // We've reached the end of the current block, (or the end of the + // file, which is treated like a series of EndBlock records. + SubBlock, // This is the start of a new subblock of a specific ID. + Record // This is a record with a specific AbbrevID. } Kind; unsigned ID; @@ -158,9 +157,9 @@ struct BitstreamEntry { } }; -/// BitstreamCursor - This represents a position within a bitcode file. There -/// may be multiple independent cursors reading within one bitstream, each -/// maintaining their own local state. +/// This represents a position within a bitcode file. There may be multiple +/// independent cursors reading within one bitstream, each maintaining their own +/// local state. /// /// Unlike iterators, BitstreamCursors are heavy-weight objects that should not /// be passed by value. @@ -169,92 +168,74 @@ class BitstreamCursor { BitstreamReader *BitStream; size_t NextChar; + // The size of the bicode. 0 if we don't know it yet. + size_t Size; - /// CurWord/word_t - This is the current data we have pulled from the stream - /// but have not returned to the client. This is specifically and - /// intentionally defined to follow the word size of the host machine for - /// efficiency. We use word_t in places that are aware of this to make it - /// perfectly explicit what is going on. - typedef uint32_t word_t; + /// This is the current data we have pulled from the stream but have not + /// returned to the client. This is specifically and intentionally defined to + /// follow the word size of the host machine for efficiency. We use word_t in + /// places that are aware of this to make it perfectly explicit what is going + /// on. + typedef size_t word_t; word_t CurWord; - /// BitsInCurWord - This is the number of bits in CurWord that are valid. This - /// is always from [0...31/63] inclusive (depending on word size). + /// This is the number of bits in CurWord that are valid. This is always from + /// [0...bits_of(size_t)-1] inclusive. unsigned BitsInCurWord; - // CurCodeSize - This is the declared size of code values used for the current - // block, in bits. + // This is the declared size of code values used for the current block, in + // bits. unsigned CurCodeSize; - /// CurAbbrevs - Abbrevs installed at in this block. - std::vector CurAbbrevs; + /// Abbrevs installed at in this block. + std::vector> CurAbbrevs; struct Block { unsigned PrevCodeSize; - std::vector PrevAbbrevs; + std::vector> PrevAbbrevs; explicit Block(unsigned PCS) : PrevCodeSize(PCS) {} }; - /// BlockScope - This tracks the codesize of parent blocks. + /// This tracks the codesize of parent blocks. SmallVector BlockScope; public: - BitstreamCursor() : BitStream(nullptr), NextChar(0) {} - BitstreamCursor(const BitstreamCursor &RHS) - : BitStream(nullptr), NextChar(0) { - operator=(RHS); - } + BitstreamCursor() { init(nullptr); } - explicit BitstreamCursor(BitstreamReader &R) : BitStream(&R) { + explicit BitstreamCursor(BitstreamReader &R) { init(&R); } + + void init(BitstreamReader *R) { + freeState(); + + BitStream = R; NextChar = 0; - CurWord = 0; + Size = 0; BitsInCurWord = 0; CurCodeSize = 2; } - void init(BitstreamReader &R) { - freeState(); - - BitStream = &R; - NextChar = 0; - CurWord = 0; - BitsInCurWord = 0; - CurCodeSize = 2; - } - - ~BitstreamCursor() { - freeState(); - } - - void operator=(const BitstreamCursor &RHS); - void freeState(); - bool isEndPos(size_t pos) { - return BitStream->getBitcodeBytes().isObjectEnd(static_cast(pos)); - } - bool canSkipToPos(size_t pos) const { // pos can be skipped to if it is a valid address or one byte past the end. return pos == 0 || BitStream->getBitcodeBytes().isValidAddress( static_cast(pos - 1)); } - uint32_t getWord(size_t pos) { - uint8_t buf[4] = { 0xFF, 0xFF, 0xFF, 0xFF }; - BitStream->getBitcodeBytes().readBytes(pos, sizeof(buf), buf); - return *reinterpret_cast(buf); - } - bool AtEndOfStream() { - return BitsInCurWord == 0 && isEndPos(NextChar); + if (BitsInCurWord != 0) + return false; + if (Size != 0) + return Size == NextChar; + fillCurWord(); + return BitsInCurWord == 0; } - /// getAbbrevIDWidth - Return the number of bits used to encode an abbrev #. + /// Return the number of bits used to encode an abbrev #. unsigned getAbbrevIDWidth() const { return CurCodeSize; } - /// GetCurrentBitNo - Return the bit # of the bit we are reading. + /// Return the bit # of the bit we are reading. uint64_t GetCurrentBitNo() const { return NextChar*CHAR_BIT - BitsInCurWord; } @@ -268,19 +249,17 @@ public: /// Flags that modify the behavior of advance(). enum { - /// AF_DontPopBlockAtEnd - If this flag is used, the advance() method does - /// not automatically pop the block scope when the end of a block is - /// reached. + /// If this flag is used, the advance() method does not automatically pop + /// the block scope when the end of a block is reached. AF_DontPopBlockAtEnd = 1, - /// AF_DontAutoprocessAbbrevs - If this flag is used, abbrev entries are - /// returned just like normal records. + /// If this flag is used, abbrev entries are returned just like normal + /// records. AF_DontAutoprocessAbbrevs = 2 }; - /// advance - Advance the current bitstream, returning the next entry in the - /// stream. - BitstreamEntry advance(unsigned Flags = 0) { + /// Advance the current bitstream, returning the next entry in the stream. + BitstreamEntry advance(unsigned Flags = 0) { while (1) { unsigned Code = ReadCode(); if (Code == bitc::END_BLOCK) { @@ -305,8 +284,8 @@ public: } } - /// advanceSkippingSubblocks - This is a convenience function for clients that - /// don't expect any subblocks. This just skips over them automatically. + /// This is a convenience function for clients that don't expect any + /// subblocks. This just skips over them automatically. BitstreamEntry advanceSkippingSubblocks(unsigned Flags = 0) { while (1) { // If we found a normal entry, return it. @@ -320,7 +299,7 @@ public: } } - /// JumpToBit - Reset the stream to the specified bit number. + /// Reset the stream to the specified bit number. void JumpToBit(uint64_t BitNo) { uintptr_t ByteNo = uintptr_t(BitNo/8) & ~(sizeof(word_t)-1); unsigned WordBitNo = unsigned(BitNo & (sizeof(word_t)*8-1)); @@ -329,75 +308,72 @@ public: // Move the cursor to the right word. NextChar = ByteNo; BitsInCurWord = 0; - CurWord = 0; // Skip over any bits that are already consumed. - if (WordBitNo) { - if (sizeof(word_t) > 4) - Read64(WordBitNo); - else - Read(WordBitNo); - } + if (WordBitNo) + Read(WordBitNo); } - - uint32_t Read(unsigned NumBits) { - assert(NumBits && NumBits <= 32 && - "Cannot return zero or more than 32 bits!"); - - // If the field is fully contained by CurWord, return it quickly. - if (BitsInCurWord >= NumBits) { - uint32_t R = uint32_t(CurWord) & (~0U >> (32-NumBits)); - CurWord >>= NumBits; - BitsInCurWord -= NumBits; - return R; - } - - // If we run out of data, stop at the end of the stream. - if (isEndPos(NextChar)) { - CurWord = 0; - BitsInCurWord = 0; - return 0; - } - - uint32_t R = uint32_t(CurWord); + void fillCurWord() { + assert(Size == 0 || NextChar < (unsigned)Size); // Read the next word from the stream. uint8_t Array[sizeof(word_t)] = {0}; - BitStream->getBitcodeBytes().readBytes(NextChar, sizeof(Array), Array); + uint64_t BytesRead = + BitStream->getBitcodeBytes().readBytes(Array, sizeof(Array), NextChar); - // Handle big-endian byte-swapping if necessary. - support::detail::packed_endian_specific_integral - EndianValue; - memcpy(&EndianValue, Array, sizeof(Array)); + // If we run out of data, stop at the end of the stream. + if (BytesRead == 0) { + Size = NextChar; + return; + } - CurWord = EndianValue; - - NextChar += sizeof(word_t); - - // Extract NumBits-BitsInCurWord from what we just read. - unsigned BitsLeft = NumBits-BitsInCurWord; - - // Be careful here, BitsLeft is in the range [1..32]/[1..64] inclusive. - R |= uint32_t((CurWord & (word_t(~0ULL) >> (sizeof(word_t)*8-BitsLeft))) - << BitsInCurWord); - - // BitsLeft bits have just been used up from CurWord. BitsLeft is in the - // range [1..32]/[1..64] so be careful how we shift. - if (BitsLeft != sizeof(word_t)*8) - CurWord >>= BitsLeft; - else - CurWord = 0; - BitsInCurWord = sizeof(word_t)*8-BitsLeft; - return R; + CurWord = + support::endian::read( + Array); + NextChar += BytesRead; + BitsInCurWord = BytesRead * 8; } - uint64_t Read64(unsigned NumBits) { - if (NumBits <= 32) return Read(NumBits); + word_t Read(unsigned NumBits) { + static const unsigned BitsInWord = sizeof(word_t) * 8; - uint64_t V = Read(32); - return V | (uint64_t)Read(NumBits-32) << 32; + assert(NumBits && NumBits <= BitsInWord && + "Cannot return zero or more than BitsInWord bits!"); + + static const unsigned Mask = sizeof(word_t) > 4 ? 0x3f : 0x1f; + + // If the field is fully contained by CurWord, return it quickly. + if (BitsInCurWord >= NumBits) { + word_t R = CurWord & (~word_t(0) >> (BitsInWord - NumBits)); + + // Use a mask to avoid undefined behavior. + CurWord >>= (NumBits & Mask); + + BitsInCurWord -= NumBits; + return R; + } + + word_t R = BitsInCurWord ? CurWord : 0; + unsigned BitsLeft = NumBits - BitsInCurWord; + + fillCurWord(); + + // If we run out of data, stop at the end of the stream. + if (BitsLeft > BitsInCurWord) + return 0; + + word_t R2 = CurWord & (~word_t(0) >> (BitsInWord - BitsLeft)); + + // Use a mask to avoid undefined behavior. + CurWord >>= (BitsLeft & Mask); + + BitsInCurWord -= BitsLeft; + + R |= R2 << (NumBits - BitsLeft); + + return R; } uint32_t ReadVBR(unsigned NumBits) { @@ -418,8 +394,8 @@ public: } } - // ReadVBR64 - Read a VBR that may have a value up to 64-bits in size. The - // chunk size of the VBR must still be <= 32 bits though. + // Read a VBR that may have a value up to 64-bits in size. The chunk size of + // the VBR must still be <= 32 bits though. uint64_t ReadVBR64(unsigned NumBits) { uint32_t Piece = Read(NumBits); if ((Piece & (1U << (NumBits-1))) == 0) @@ -450,7 +426,6 @@ private: } BitsInCurWord = 0; - CurWord = 0; } public: @@ -462,15 +437,13 @@ public: // Block header: // [ENTER_SUBBLOCK, blockid, newcodelen, , blocklen] - /// ReadSubBlockID - Having read the ENTER_SUBBLOCK code, read the BlockID for - /// the block. + /// Having read the ENTER_SUBBLOCK code, read the BlockID for the block. unsigned ReadSubBlockID() { return ReadVBR(bitc::BlockIDWidth); } - /// SkipBlock - Having read the ENTER_SUBBLOCK abbrevid and a BlockID, skip - /// over the body of this block. If the block record is malformed, return - /// true. + /// Having read the ENTER_SUBBLOCK abbrevid and a BlockID, skip over the body + /// of this block. If the block record is malformed, return true. bool SkipBlock() { // Read and ignore the codelen value. Since we are skipping this block, we // don't care what code widths are used inside of it. @@ -488,8 +461,8 @@ public: return false; } - /// EnterSubBlock - Having read the ENTER_SUBBLOCK abbrevid, enter - /// the block, and return true if the block has an error. + /// Having read the ENTER_SUBBLOCK abbrevid, enter the block, and return true + /// if the block has an error. bool EnterSubBlock(unsigned BlockID, unsigned *NumWordsP = nullptr); bool ReadBlockEnd() { @@ -508,12 +481,7 @@ private: void popBlockScope() { CurCodeSize = BlockScope.back().PrevCodeSize; - // Delete abbrevs from popped scope. - for (unsigned i = 0, e = static_cast(CurAbbrevs.size()); - i != e; ++i) - CurAbbrevs[i]->dropRef(); - - BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); + CurAbbrevs = std::move(BlockScope.back().PrevAbbrevs); BlockScope.pop_back(); } @@ -521,23 +489,16 @@ private: // Record Processing //===--------------------------------------------------------------------===// -private: - void readAbbreviatedLiteral(const BitCodeAbbrevOp &Op, - SmallVectorImpl &Vals); - void readAbbreviatedField(const BitCodeAbbrevOp &Op, - SmallVectorImpl &Vals); - void skipAbbreviatedField(const BitCodeAbbrevOp &Op); - public: - /// getAbbrev - Return the abbreviation for the specified AbbrevId. + /// Return the abbreviation for the specified AbbrevId. const BitCodeAbbrev *getAbbrev(unsigned AbbrevID) { unsigned AbbrevNo = AbbrevID-bitc::FIRST_APPLICATION_ABBREV; assert(AbbrevNo < CurAbbrevs.size() && "Invalid abbrev #!"); - return CurAbbrevs[AbbrevNo]; + return CurAbbrevs[AbbrevNo].get(); } - /// skipRecord - Read the current record and discard it. + /// Read the current record and discard it. void skipRecord(unsigned AbbrevID); unsigned readRecord(unsigned AbbrevID, SmallVectorImpl &Vals, diff --git a/contrib/llvm/include/llvm/Bitcode/BitstreamWriter.h b/contrib/llvm/include/llvm/Bitcode/BitstreamWriter.h index dcfebd9cc105..9e2c2fa4a156 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitstreamWriter.h +++ b/contrib/llvm/include/llvm/Bitcode/BitstreamWriter.h @@ -40,12 +40,12 @@ class BitstreamWriter { unsigned BlockInfoCurBID; /// CurAbbrevs - Abbrevs installed at in this block. - std::vector CurAbbrevs; + std::vector> CurAbbrevs; struct Block { unsigned PrevCodeSize; unsigned StartSizeWord; - std::vector PrevAbbrevs; + std::vector> PrevAbbrevs; Block(unsigned PCS, unsigned SSW) : PrevCodeSize(PCS), StartSizeWord(SSW) {} }; @@ -56,7 +56,7 @@ class BitstreamWriter { /// These describe abbreviations that all blocks of the specified ID inherit. struct BlockInfo { unsigned BlockID; - std::vector Abbrevs; + std::vector> Abbrevs; }; std::vector BlockInfoRecords; @@ -99,16 +99,6 @@ public: ~BitstreamWriter() { assert(CurBit == 0 && "Unflushed data remaining"); assert(BlockScope.empty() && CurAbbrevs.empty() && "Block imbalance"); - - // Free the BlockInfoRecords. - while (!BlockInfoRecords.empty()) { - BlockInfo &Info = BlockInfoRecords.back(); - // Free blockinfo abbrev info. - for (unsigned i = 0, e = static_cast(Info.Abbrevs.size()); - i != e; ++i) - Info.Abbrevs[i]->dropRef(); - BlockInfoRecords.pop_back(); - } } /// \brief Retrieve the current position in the stream, in bits. @@ -231,22 +221,13 @@ public: // If there is a blockinfo for this BlockID, add all the predefined abbrevs // to the abbrev list. if (BlockInfo *Info = getBlockInfo(BlockID)) { - for (unsigned i = 0, e = static_cast(Info->Abbrevs.size()); - i != e; ++i) { - CurAbbrevs.push_back(Info->Abbrevs[i]); - Info->Abbrevs[i]->addRef(); - } + CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), + Info->Abbrevs.end()); } } void ExitBlock() { assert(!BlockScope.empty() && "Block scope imbalance!"); - - // Delete all abbrevs. - for (unsigned i = 0, e = static_cast(CurAbbrevs.size()); - i != e; ++i) - CurAbbrevs[i]->dropRef(); - const Block &B = BlockScope.back(); // Block tail: @@ -263,7 +244,7 @@ public: // Restore the inner block's code size and abbrev table. CurCodeSize = B.PrevCodeSize; - BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); + CurAbbrevs = std::move(B.PrevAbbrevs); BlockScope.pop_back(); } @@ -317,7 +298,7 @@ private: unsigned BlobLen = (unsigned) Blob.size(); unsigned AbbrevNo = Abbrev-bitc::FIRST_APPLICATION_ABBREV; assert(AbbrevNo < CurAbbrevs.size() && "Invalid abbrev #!"); - BitCodeAbbrev *Abbv = CurAbbrevs[AbbrevNo]; + const BitCodeAbbrev *Abbv = CurAbbrevs[AbbrevNo].get(); EmitCode(Abbrev); diff --git a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h index ee2efa2257b1..043ecd3d65d7 100644 --- a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -137,14 +137,14 @@ namespace bitc { enum MetadataCodes { METADATA_STRING = 1, // MDSTRING: [values] - // 2 is unused. - // 3 is unused. + METADATA_VALUE = 2, // VALUE: [type num, value num] + METADATA_NODE = 3, // NODE: [n x md num] METADATA_NAME = 4, // STRING: [values] - // 5 is unused. + METADATA_DISTINCT_NODE = 5, // DISTINCT_NODE: [n x md num] METADATA_KIND = 6, // [n x [id, name]] - // 7 is unused. - METADATA_NODE = 8, // NODE: [n x (type num, value num)] - METADATA_FN_NODE = 9, // FN_NODE: [n x (type num, value num)] + METADATA_LOCATION = 7, // [distinct, line, col, scope, inlined-at?] + METADATA_OLD_NODE = 8, // OLD_NODE: [n x (type num, value num)] + METADATA_OLD_FN_NODE = 9, // OLD_FN_NODE: [n x (type num, value num)] METADATA_NAMED_NODE = 10, // NAMED_NODE: [n x mdnodes] METADATA_ATTACHMENT = 11 // [m x [value, [n x [id, mdnode]]] }; @@ -330,7 +330,8 @@ namespace bitc { }; enum UseListCodes { - USELIST_CODE_ENTRY = 1 // USELIST_CODE_ENTRY: TBD. + USELIST_CODE_DEFAULT = 1, // DEFAULT: [index..., value-id] + USELIST_CODE_BB = 2 // BB: [index..., bb-id] }; enum AttributeKindCodes { diff --git a/contrib/llvm/include/llvm/Bitcode/ReaderWriter.h b/contrib/llvm/include/llvm/Bitcode/ReaderWriter.h index 8cf573544f8d..48bdabc88391 100644 --- a/contrib/llvm/include/llvm/Bitcode/ReaderWriter.h +++ b/contrib/llvm/include/llvm/Bitcode/ReaderWriter.h @@ -14,12 +14,14 @@ #ifndef LLVM_BITCODE_READERWRITER_H #define LLVM_BITCODE_READERWRITER_H +#include "llvm/IR/DiagnosticInfo.h" #include "llvm/Support/ErrorOr.h" +#include "llvm/Support/MemoryBuffer.h" +#include #include namespace llvm { class BitstreamWriter; - class MemoryBuffer; class DataStreamer; class LLVMContext; class Module; @@ -27,30 +29,30 @@ namespace llvm { class raw_ostream; /// Read the header of the specified bitcode buffer and prepare for lazy - /// deserialization of function bodies. If successful, this takes ownership - /// of 'buffer. On error, this *does not* take ownership of Buffer. - ErrorOr getLazyBitcodeModule(MemoryBuffer *Buffer, - LLVMContext &Context); + /// deserialization of function bodies. If successful, this moves Buffer. On + /// error, this *does not* move Buffer. + ErrorOr + getLazyBitcodeModule(std::unique_ptr &&Buffer, + LLVMContext &Context, + DiagnosticHandlerFunction DiagnosticHandler = nullptr); - /// getStreamedBitcodeModule - Read the header of the specified stream - /// and prepare for lazy deserialization and streaming of function bodies. - /// On error, this returns null, and fills in *ErrMsg with an error - /// description if ErrMsg is non-null. - Module *getStreamedBitcodeModule(const std::string &name, - DataStreamer *streamer, - LLVMContext &Context, - std::string *ErrMsg = nullptr); + /// Read the header of the specified stream and prepare for lazy + /// deserialization and streaming of function bodies. + ErrorOr> getStreamedBitcodeModule( + StringRef Name, DataStreamer *Streamer, LLVMContext &Context, + DiagnosticHandlerFunction DiagnosticHandler = nullptr); /// Read the header of the specified bitcode buffer and extract just the - /// triple information. If successful, this returns a string and *does not* - /// take ownership of 'buffer'. On error, this returns "". - std::string getBitcodeTargetTriple(MemoryBuffer *Buffer, - LLVMContext &Context); + /// triple information. If successful, this returns a string. On error, this + /// returns "". + std::string + getBitcodeTargetTriple(MemoryBufferRef Buffer, LLVMContext &Context, + DiagnosticHandlerFunction DiagnosticHandler = nullptr); /// Read the specified bitcode file, returning the module. - /// This method *never* takes ownership of Buffer. - ErrorOr parseBitcodeFile(MemoryBuffer *Buffer, - LLVMContext &Context); + ErrorOr + parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context, + DiagnosticHandlerFunction DiagnosticHandler = nullptr); /// WriteBitcodeToFile - Write the specified module to the specified /// raw output stream. For streams where it matters, the given stream @@ -139,6 +141,32 @@ namespace llvm { BufEnd = BufPtr+Size; return false; } + + const std::error_category &BitcodeErrorCategory(); + enum class BitcodeError { InvalidBitcodeSignature, CorruptedBitcode }; + inline std::error_code make_error_code(BitcodeError E) { + return std::error_code(static_cast(E), BitcodeErrorCategory()); + } + + class BitcodeDiagnosticInfo : public DiagnosticInfo { + const Twine &Msg; + std::error_code EC; + + public: + BitcodeDiagnosticInfo(std::error_code EC, DiagnosticSeverity Severity, + const Twine &Msg); + void print(DiagnosticPrinter &DP) const override; + std::error_code getError() const { return EC; }; + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_Bitcode; + } + }; + } // End llvm namespace +namespace std { +template <> struct is_error_code_enum : std::true_type {}; +} + #endif diff --git a/contrib/llvm/include/llvm/CodeGen/Analysis.h b/contrib/llvm/include/llvm/CodeGen/Analysis.h index b791ba09adaf..c4b94ede4f55 100644 --- a/contrib/llvm/include/llvm/CodeGen/Analysis.h +++ b/contrib/llvm/include/llvm/CodeGen/Analysis.h @@ -22,7 +22,7 @@ #include "llvm/IR/Instructions.h" namespace llvm { -class GlobalVariable; +class GlobalValue; class TargetLoweringBase; class TargetLowering; class TargetMachine; @@ -31,10 +31,21 @@ class SDValue; class SelectionDAG; struct EVT; -/// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence -/// of insertvalue or extractvalue indices that identify a member, return -/// the linearized index of the start of the member. +/// \brief Compute the linearized index of a member in a nested +/// aggregate/struct/array. /// +/// Given an LLVM IR aggregate type and a sequence of insertvalue or +/// extractvalue indices that identify a member, return the linearized index of +/// the start of the member, i.e the number of element in memory before the +/// seeked one. This is disconnected from the number of bytes. +/// +/// \param Ty is the type indexed by \p Indices. +/// \param Indices is an optional pointer in the indices list to the current +/// index. +/// \param IndicesEnd is the end of the indices list. +/// \param CurIndex is the current index in the recursion. +/// +/// \returns \p CurIndex plus the linear index in \p Ty the indices list. unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, @@ -59,7 +70,7 @@ void ComputeValueVTs(const TargetLowering &TLI, Type *Ty, uint64_t StartingOffset = 0); /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. -GlobalVariable *ExtractTypeInfo(Value *V); +GlobalValue *ExtractTypeInfo(Value *V); /// hasInlineAsmMemConstraint - Return true if the inline asm instruction being /// processed uses a memory 'm' constraint. @@ -97,6 +108,13 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const ReturnInst *Ret, const TargetLoweringBase &TLI); +// True if GV can be left out of the object symbol table. This is the case +// for linkonce_odr values whose address is not significant. While legal, it is +// not normally profitable to omit them from the .o symbol table. Using this +// analysis makes sense when the information can be passed down to the linker +// or we are in LTO. +bool canBeOmittedFromSymbolTable(const GlobalValue *GV); + } // End llvm namespace #endif diff --git a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h index e1c9a14c9009..e3ce57ad1850 100644 --- a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -44,6 +44,7 @@ class MachineModuleInfo; class MCAsmInfo; class MCCFIInstruction; class MCContext; +class MCExpr; class MCInst; class MCInstrInfo; class MCSection; @@ -132,6 +133,7 @@ public: virtual ~AsmPrinter(); DwarfDebug *getDwarfDebug() { return DD; } + DwarfDebug *getDwarfDebug() const { return DD; } /// Return true if assembly output should contain comments. /// @@ -203,6 +205,8 @@ public: void emitCFIInstruction(const MachineInstr &MI); + void emitFrameAlloc(const MachineInstr &MI); + enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; CFIMoveType needsCFIMoves(); @@ -238,6 +242,9 @@ public: /// alignment (if present) and a comment describing it if appropriate. void EmitBasicBlockStart(const MachineBasicBlock &MBB) const; + /// Lower the specified LLVM Constant to an MCExpr. + const MCExpr *lowerConstant(const Constant *CV); + /// \brief Print a general LLVM constant to the .s file. void EmitGlobalConstant(const Constant *CV); @@ -264,6 +271,9 @@ public: /// function. virtual void EmitFunctionBodyEnd() {} + /// Targets can override this to emit stuff at the end of a basic block. + virtual void EmitBasicBlockEnd(const MachineBasicBlock &MBB) {} + /// Targets should implement this to emit instructions. virtual void EmitInstruction(const MachineInstr *) { llvm_unreachable("EmitInstruction not implemented"); @@ -346,12 +356,6 @@ public: void EmitLabelDifference(const MCSymbol *Hi, const MCSymbol *Lo, unsigned Size) const; - /// Emit something like ".long Hi+Offset-Lo" where the size in bytes of the - /// directive is specified by Size and Hi/Lo specify the labels. This - /// implicitly uses .set if it is available. - void EmitLabelOffsetDifference(const MCSymbol *Hi, uint64_t Offset, - const MCSymbol *Lo, unsigned Size) const; - /// Emit something like ".long Label+Offset" where the size in bytes of the /// directive is specified by Size and Label specifies the label. This /// implicitly uses .set if it is available. @@ -402,6 +406,13 @@ public: /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified. virtual unsigned getISAEncoding() { return 0; } + /// Emit a dwarf register operation for describing + /// - a small value occupying only part of a register or + /// - a register representing only part of a value. + void EmitDwarfOpPiece(ByteStreamer &Streamer, unsigned SizeInBits, + unsigned OffsetInBits = 0) const; + + /// \brief Emit a partial DWARF register operation. /// \param MLoc the register /// \param PieceSize size and @@ -418,7 +429,7 @@ public: unsigned PieceSize = 0, unsigned PieceOffset = 0) const; - /// Emit dwarf register operation. + /// EmitDwarfRegOp - Emit a dwarf register operation. /// \param Indirect whether this is a register-indirect address virtual void EmitDwarfRegOp(ByteStreamer &BS, const MachineLocation &MLoc, bool Indirect) const; @@ -461,6 +472,10 @@ public: unsigned AsmVariant, const char *ExtraCode, raw_ostream &OS); + /// Let the target do anything it needs to do before emitting inlineasm. + /// \p StartInfo - the subtarget info before parsing inline asm + virtual void emitInlineAsmStart(const MCSubtargetInfo &StartInfo) const; + /// Let the target do anything it needs to do after emitting inlineasm. /// This callback can be used restore the original mode in case the /// inlineasm contains directives to switch modes. diff --git a/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h b/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h index 0d79b1d41bdb..91fb0a9d7e77 100644 --- a/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h +++ b/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h @@ -30,8 +30,10 @@ namespace llvm { /// @param UseDefFreq Expected number of executed use and def instructions /// per function call. Derived from block frequencies. /// @param Size Size of live interval as returnexd by getSize() + /// @param NumInstr Number of instructions using this live interval /// - static inline float normalizeSpillWeight(float UseDefFreq, unsigned Size) { + static inline float normalizeSpillWeight(float UseDefFreq, unsigned Size, + unsigned NumInstr) { // The constant 25 instructions is added to avoid depending too much on // accidental SlotIndex gaps for small intervals. The effect is that small // intervals have a spill weight that is mostly proportional to the number @@ -44,7 +46,7 @@ namespace llvm { /// spill weight and allocation hint. class VirtRegAuxInfo { public: - typedef float (*NormalizingFn)(float, unsigned); + typedef float (*NormalizingFn)(float, unsigned, unsigned); private: MachineFunction &MF; diff --git a/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h b/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h index abe00a167fd6..dd7703b1dbf7 100644 --- a/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h +++ b/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h @@ -35,18 +35,18 @@ public: SExt, // The value is sign extended in the location. ZExt, // The value is zero extended in the location. AExt, // The value is extended with undefined upper bits. + SExtUpper, // The value is in the upper bits of the location and should be + // sign extended when retrieved. + ZExtUpper, // The value is in the upper bits of the location and should be + // zero extended when retrieved. + AExtUpper, // The value is in the upper bits of the location and should be + // extended with undefined upper bits when retrieved. BCvt, // The value is bit-converted in the location. VExt, // The value is vector-widened in the location. // FIXME: Not implemented yet. Code that uses AExt to mean // vector-widen should be fixed to use VExt instead. FPExt, // The floating-point value is fp-extended in the location. - Indirect, // The location contains pointer to the value. - SExtUpper, // The value is in the upper bits of the location and should be - // sign extended when retrieved. - ZExtUpper, // The value is in the upper bits of the location and should be - // zero extended when retrieved. - AExtUpper // The value is in the upper bits of the location and should be - // extended with undefined upper bits when retrieved. + Indirect // The location contains pointer to the value. // TODO: a subset of the value is in the location. }; @@ -158,6 +158,16 @@ public: } }; +/// Describes a register that needs to be forwarded from the prologue to a +/// musttail call. +struct ForwardedRegister { + ForwardedRegister(unsigned VReg, MCPhysReg PReg, MVT VT) + : VReg(VReg), PReg(PReg), VT(VT) {} + unsigned VReg; + MCPhysReg PReg; + MVT VT; +}; + /// CCAssignFn - This function assigns a location for Val, updating State to /// reflect the change. It returns 'true' if it failed to handle Val. typedef bool CCAssignFn(unsigned ValNo, MVT ValVT, @@ -184,7 +194,6 @@ private: CallingConv::ID CallingConv; bool IsVarArg; MachineFunction &MF; - const TargetMachine &TM; const TargetRegisterInfo &TRI; SmallVectorImpl &Locs; LLVMContext &Context; @@ -248,15 +257,13 @@ protected: public: CCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF, - const TargetMachine &TM, SmallVectorImpl &locs, - LLVMContext &C); + SmallVectorImpl &locs, LLVMContext &C); void addLoc(const CCValAssign &V) { Locs.push_back(V); } LLVMContext &getContext() const { return Context; } - const TargetMachine &getTarget() const { return TM; } MachineFunction &getMachineFunction() const { return MF; } CallingConv::ID getCallingConv() const { return CallingConv; } bool isVarArg() const { return IsVarArg; } @@ -348,8 +355,12 @@ public: /// AllocateRegBlock - Attempt to allocate a block of RegsRequired consecutive /// registers. If this is not possible, return zero. Otherwise, return the first /// register of the block that were allocated, marking the entire block as allocated. - unsigned AllocateRegBlock(const uint16_t *Regs, unsigned NumRegs, unsigned RegsRequired) { - for (unsigned StartIdx = 0; StartIdx <= NumRegs - RegsRequired; ++StartIdx) { + unsigned AllocateRegBlock(ArrayRef Regs, unsigned RegsRequired) { + if (RegsRequired > Regs.size()) + return 0; + + for (unsigned StartIdx = 0; StartIdx <= Regs.size() - RegsRequired; + ++StartIdx) { bool BlockAvailable = true; // Check for already-allocated regs in this block for (unsigned BlockIdx = 0; BlockIdx < RegsRequired; ++BlockIdx) { @@ -387,8 +398,8 @@ public: /// AllocateStack - Allocate a chunk of stack space with the specified size /// and alignment. unsigned AllocateStack(unsigned Size, unsigned Align) { - assert(Align && ((Align-1) & Align) == 0); // Align is power of 2. - StackOffset = ((StackOffset + Align-1) & ~(Align-1)); + assert(Align && ((Align - 1) & Align) == 0); // Align is power of 2. + StackOffset = ((StackOffset + Align - 1) & ~(Align - 1)); unsigned Result = StackOffset; StackOffset += Size; MF.getFrameInfo()->ensureMaxAlignment(Align); @@ -469,6 +480,19 @@ public: return PendingLocs; } + /// Compute the remaining unused register parameters that would be used for + /// the given value type. This is useful when varargs are passed in the + /// registers that normal prototyped parameters would be passed in, or for + /// implementing perfect forwarding. + void getRemainingRegParmsForType(SmallVectorImpl &Regs, MVT VT, + CCAssignFn Fn); + + /// Compute the set of registers that need to be preserved and forwarded to + /// any musttail calls. + void analyzeMustTailForwardedRegisters( + SmallVectorImpl &Forwards, ArrayRef RegParmTypes, + CCAssignFn Fn); + private: /// MarkAllocated - Mark a register and all of its aliases as allocated. void MarkAllocated(unsigned Reg); diff --git a/contrib/llvm/include/llvm/CodeGen/CommandFlags.h b/contrib/llvm/include/llvm/CodeGen/CommandFlags.h index 449d93418a4c..973c5954f9ad 100644 --- a/contrib/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/contrib/llvm/include/llvm/CodeGen/CommandFlags.h @@ -54,6 +54,16 @@ RelocModel("relocation-model", "Relocatable external references, non-relocatable code"), clEnumValEnd)); +cl::opt +TMModel("thread-model", + cl::desc("Choose threading model"), + cl::init(ThreadModel::POSIX), + cl::values(clEnumValN(ThreadModel::POSIX, "posix", + "POSIX thread model"), + clEnumValN(ThreadModel::Single, "single", + "Single thread model"), + clEnumValEnd)); + cl::opt CMModel("code-model", cl::desc("Choose code model"), @@ -82,11 +92,6 @@ FileType("filetype", cl::init(TargetMachine::CGFT_AssemblyFile), "Emit nothing, for performance testing"), clEnumValEnd)); -cl::opt -DisableRedZone("disable-red-zone", - cl::desc("Do not emit code that uses the red zone."), - cl::init(false)); - cl::opt EnableFPMAD("enable-fp-mad", cl::desc("Enable less precise MAD instructions to be generated"), @@ -180,8 +185,8 @@ EnablePIE("enable-pie", cl::init(false)); cl::opt -UseInitArray("use-init-array", - cl::desc("Use .init_array instead of .ctors."), +UseCtors("use-ctors", + cl::desc("Use .ctors instead of .init_array."), cl::init(false)); cl::opt StopAfter("stop-after", @@ -217,6 +222,44 @@ JTableType("jump-table-type", "Create one table per unique function type."), clEnumValEnd)); +cl::opt +FCFI("fcfi", + cl::desc("Apply forward-edge control-flow integrity"), + cl::init(false)); + +cl::opt +CFIType("cfi-type", + cl::desc("Choose the type of Control-Flow Integrity check to add"), + cl::init(CFIntegrity::Sub), + cl::values( + clEnumValN(CFIntegrity::Sub, "sub", + "Subtract the pointer from the table base, then mask."), + clEnumValN(CFIntegrity::Ror, "ror", + "Use rotate to check the offset from a table base."), + clEnumValN(CFIntegrity::Add, "add", + "Mask out the high bits and add to an aligned base."), + clEnumValEnd)); + +cl::opt +CFIEnforcing("cfi-enforcing", + cl::desc("Enforce CFI or pass the violation to a function."), + cl::init(false)); + +// Note that this option is linked to the cfi-enforcing option above: if +// cfi-enforcing is set, then the cfi-func-name option is entirely ignored. If +// cfi-enforcing is false and no cfi-func-name is set, then a default function +// will be generated that ignores all CFI violations. The expected signature for +// functions called with CFI violations is +// +// void (i8*, i8*) +// +// The first pointer is a C string containing the name of the function in which +// the violation occurs, and the second pointer is the pointer that violated +// CFI. +cl::opt +CFIFuncName("cfi-func-name", cl::desc("The name of the CFI function to call"), + cl::init("")); + // Common utility function tightly tied to the options listed here. Initializes // a TargetOptions object with CodeGen flags and returns it. static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { @@ -238,12 +281,18 @@ static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { Options.StackAlignmentOverride = OverrideStackAlignment; Options.TrapFuncName = TrapFuncName; Options.PositionIndependentExecutable = EnablePIE; - Options.UseInitArray = UseInitArray; + Options.UseInitArray = !UseCtors; Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; Options.MCOptions = InitMCTargetOptionsFromFlags(); Options.JTType = JTableType; + Options.FCFI = FCFI; + Options.CFIType = CFIType; + Options.CFIEnforcing = CFIEnforcing; + Options.CFIFuncName = CFIFuncName; + + Options.ThreadModel = TMModel; return Options; } diff --git a/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h b/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h index 9d25fd377b7e..f9cdc2a469ff 100644 --- a/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h +++ b/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h @@ -91,7 +91,6 @@ public: // API call is made to prune the dependence. class VLIWPacketizerList { protected: - const TargetMachine &TM; const MachineFunction &MF; const TargetInstrInfo *TII; @@ -107,9 +106,7 @@ protected: std::map MIToSUnit; public: - VLIWPacketizerList( - MachineFunction &MF, MachineLoopInfo &MLI, MachineDominatorTree &MDT, - bool IsPostRA); + VLIWPacketizerList(MachineFunction &MF, MachineLoopInfo &MLI, bool IsPostRA); virtual ~VLIWPacketizerList(); diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h b/contrib/llvm/include/llvm/CodeGen/DIE.h similarity index 99% rename from contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h rename to contrib/llvm/include/llvm/CodeGen/DIE.h index ef05f1707810..e310aef3dcbb 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h +++ b/contrib/llvm/include/llvm/CodeGen/DIE.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef CODEGEN_ASMPRINTER_DIE_H__ -#define CODEGEN_ASMPRINTER_DIE_H__ +#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_DIE_H +#define LLVM_LIB_CODEGEN_ASMPRINTER_DIE_H #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SmallVector.h" @@ -381,10 +381,10 @@ public: /// class DIEString : public DIEValue { const DIEValue *Access; - const StringRef Str; + StringRef Str; public: - DIEString(const DIEValue *Acc, const StringRef S) + DIEString(const DIEValue *Acc, StringRef S) : DIEValue(isString), Access(Acc), Str(S) {} /// getString - Grab the string out of the object. diff --git a/contrib/llvm/include/llvm/CodeGen/FastISel.h b/contrib/llvm/include/llvm/CodeGen/FastISel.h index 0d1b1dc09560..1dca2ce1ab22 100644 --- a/contrib/llvm/include/llvm/CodeGen/FastISel.h +++ b/contrib/llvm/include/llvm/CodeGen/FastISel.h @@ -18,72 +18,52 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/Target/TargetLowering.h" #include "llvm/IR/CallingConv.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/Target/TargetLowering.h" namespace llvm { -class AllocaInst; -class Constant; -class ConstantFP; -class CallInst; -class DataLayout; -class FunctionLoweringInfo; -class Instruction; -class IntrinsicInst; -class LoadInst; -class MVT; -class MachineConstantPool; -class MachineFrameInfo; -class MachineFunction; -class MachineInstr; -class MachineRegisterInfo; -class TargetInstrInfo; -class TargetLibraryInfo; -class TargetLowering; -class TargetMachine; -class TargetRegisterClass; -class TargetRegisterInfo; -class User; -class Value; - -/// This is a fast-path instruction selection class that generates poor code and -/// doesn't support illegal types or non-trivial lowering, but runs quickly. +/// \brief This is a fast-path instruction selection class that generates poor +/// code and doesn't support illegal types or non-trivial lowering, but runs +/// quickly. class FastISel { - public: +public: struct ArgListEntry { Value *Val; Type *Ty; - bool isSExt : 1; - bool isZExt : 1; - bool isInReg : 1; - bool isSRet : 1; - bool isNest : 1; - bool isByVal : 1; - bool isInAlloca : 1; - bool isReturned : 1; + bool IsSExt : 1; + bool IsZExt : 1; + bool IsInReg : 1; + bool IsSRet : 1; + bool IsNest : 1; + bool IsByVal : 1; + bool IsInAlloca : 1; + bool IsReturned : 1; uint16_t Alignment; ArgListEntry() - : Val(nullptr), Ty(nullptr), isSExt(false), isZExt(false), isInReg(false), - isSRet(false), isNest(false), isByVal(false), isInAlloca(false), - isReturned(false), Alignment(0) { } + : Val(nullptr), Ty(nullptr), IsSExt(false), IsZExt(false), + IsInReg(false), IsSRet(false), IsNest(false), IsByVal(false), + IsInAlloca(false), IsReturned(false), Alignment(0) {} + /// \brief Set CallLoweringInfo attribute flags based on a call instruction + /// and called function attributes. void setAttributes(ImmutableCallSite *CS, unsigned AttrIdx); }; typedef std::vector ArgListTy; struct CallLoweringInfo { Type *RetTy; - bool RetSExt : 1; - bool RetZExt : 1; - bool IsVarArg : 1; - bool IsInReg : 1; - bool DoesNotReturn : 1; + bool RetSExt : 1; + bool RetZExt : 1; + bool IsVarArg : 1; + bool IsInReg : 1; + bool DoesNotReturn : 1; bool IsReturnValueUsed : 1; - // IsTailCall should be modified by implementations of - // FastLowerCall that perform tail call conversions. + // \brief IsTailCall Should be modified by implementations of FastLowerCall + // that perform tail call conversions. bool IsTailCall; unsigned NumFixedArgs; @@ -96,6 +76,8 @@ class FastISel { unsigned ResultReg; unsigned NumResultRegs; + bool IsPatchPoint; + SmallVector OutVals; SmallVector OutFlags; SmallVector OutRegs; @@ -103,12 +85,11 @@ class FastISel { SmallVector InRegs; CallLoweringInfo() - : RetTy(nullptr), RetSExt(false), RetZExt(false), IsVarArg(false), - IsInReg(false), DoesNotReturn(false), IsReturnValueUsed(true), - IsTailCall(false), NumFixedArgs(-1), CallConv(CallingConv::C), - Callee(nullptr), SymName(nullptr), CS(nullptr), Call(nullptr), - ResultReg(0), NumResultRegs(0) - {} + : RetTy(nullptr), RetSExt(false), RetZExt(false), IsVarArg(false), + IsInReg(false), DoesNotReturn(false), IsReturnValueUsed(true), + IsTailCall(false), NumFixedArgs(-1), CallConv(CallingConv::C), + Callee(nullptr), SymName(nullptr), CS(nullptr), Call(nullptr), + ResultReg(0), NumResultRegs(0), IsPatchPoint(false) {} CallLoweringInfo &setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, @@ -124,8 +105,8 @@ class FastISel { RetZExt = Call.paramHasAttr(0, Attribute::ZExt); CallConv = Call.getCallingConv(); - NumFixedArgs = FuncTy->getNumParams(); Args = std::move(ArgsList); + NumFixedArgs = FuncTy->getNumParams(); CS = &Call; @@ -148,8 +129,8 @@ class FastISel { RetZExt = Call.paramHasAttr(0, Attribute::ZExt); CallConv = Call.getCallingConv(); - NumFixedArgs = (FixedArgs == ~0U) ? FuncTy->getNumParams() : FixedArgs; Args = std::move(ArgsList); + NumFixedArgs = (FixedArgs == ~0U) ? FuncTy->getNumParams() : FixedArgs; CS = &Call; @@ -162,8 +143,19 @@ class FastISel { RetTy = ResultTy; Callee = Target; CallConv = CC; - NumFixedArgs = (FixedArgs == ~0U) ? Args.size() : FixedArgs; Args = std::move(ArgsList); + NumFixedArgs = (FixedArgs == ~0U) ? Args.size() : FixedArgs; + return *this; + } + + CallLoweringInfo &setCallee(CallingConv::ID CC, Type *ResultTy, + const char *Target, ArgListTy &&ArgsList, + unsigned FixedArgs = ~0U) { + RetTy = ResultTy; + SymName = Target; + CallConv = CC; + Args = std::move(ArgsList); + NumFixedArgs = (FixedArgs == ~0U) ? Args.size() : FixedArgs; return *this; } @@ -172,10 +164,13 @@ class FastISel { return *this; } - ArgListTy &getArgs() { - return Args; + CallLoweringInfo &setIsPatchPoint(bool Value = true) { + IsPatchPoint = Value; + return *this; } + ArgListTy &getArgs() { return Args; } + void clearOuts() { OutVals.clear(); OutFlags.clear(); @@ -202,61 +197,64 @@ protected: const TargetLowering &TLI; const TargetRegisterInfo &TRI; const TargetLibraryInfo *LibInfo; + bool SkipTargetIndependentISel; - /// The position of the last instruction for materializing constants for use - /// in the current block. It resets to EmitStartPt when it makes sense (for - /// example, it's usually profitable to avoid function calls between the + /// \brief The position of the last instruction for materializing constants + /// for use in the current block. It resets to EmitStartPt when it makes sense + /// (for example, it's usually profitable to avoid function calls between the /// definition and the use) MachineInstr *LastLocalValue; - /// The top most instruction in the current block that is allowed for emitting - /// local variables. LastLocalValue resets to EmitStartPt when it makes sense - /// (for example, on function calls) + /// \brief The top most instruction in the current block that is allowed for + /// emitting local variables. LastLocalValue resets to EmitStartPt when it + /// makes sense (for example, on function calls) MachineInstr *EmitStartPt; public: - /// Return the position of the last instruction emitted for materializing - /// constants for use in the current block. + /// \brief Return the position of the last instruction emitted for + /// materializing constants for use in the current block. MachineInstr *getLastLocalValue() { return LastLocalValue; } - /// Update the position of the last instruction emitted for materializing - /// constants for use in the current block. + /// \brief Update the position of the last instruction emitted for + /// materializing constants for use in the current block. void setLastLocalValue(MachineInstr *I) { EmitStartPt = I; LastLocalValue = I; } - /// Set the current block to which generated machine instructions will be - /// appended, and clear the local CSE map. + /// \brief Set the current block to which generated machine instructions will + /// be appended, and clear the local CSE map. void startNewBlock(); - /// Return current debug location information. + /// \brief Return current debug location information. DebugLoc getCurDebugLoc() const { return DbgLoc; } - - /// Do "fast" instruction selection for function arguments and append machine - /// instructions to the current block. Return true if it is successful. - bool LowerArguments(); - /// Do "fast" instruction selection for the given LLVM IR instruction, and - /// append generated machine instructions to the current block. Return true if - /// selection was successful. - bool SelectInstruction(const Instruction *I); + /// \brief Do "fast" instruction selection for function arguments and append + /// the machine instructions to the current block. Returns true when + /// successful. + bool lowerArguments(); - /// Do "fast" instruction selection for the given LLVM IR operator + /// \brief Do "fast" instruction selection for the given LLVM IR instruction + /// and append the generated machine instructions to the current block. + /// Returns true if selection was successful. + bool selectInstruction(const Instruction *I); + + /// \brief Do "fast" instruction selection for the given LLVM IR operator /// (Instruction or ConstantExpr), and append generated machine instructions /// to the current block. Return true if selection was successful. - bool SelectOperator(const User *I, unsigned Opcode); + bool selectOperator(const User *I, unsigned Opcode); - /// Create a virtual register and arrange for it to be assigned the value for - /// the given LLVM value. + /// \brief Create a virtual register and arrange for it to be assigned the + /// value for the given LLVM value. unsigned getRegForValue(const Value *V); - /// Look up the value to see if its value is already cached in a register. It - /// may be defined by instructions across blocks or defined locally. + /// \brief Look up the value to see if its value is already cached in a + /// register. It may be defined by instructions across blocks or defined + /// locally. unsigned lookUpRegForValue(const Value *V); - /// This is a wrapper around getRegForValue that also takes care of truncating - /// or sign-extending the given getelementptr index value. + /// \brief This is a wrapper around getRegForValue that also takes care of + /// truncating or sign-extending the given getelementptr index value. std::pair getRegForGEPIndex(const Value *V); /// \brief We're checking to see if we can fold \p LI into \p FoldInst. Note @@ -284,11 +282,11 @@ public: return false; } - /// Reset InsertPt to prepare for inserting instructions into the current - /// block. + /// \brief Reset InsertPt to prepare for inserting instructions into the + /// current block. void recomputeInsertPt(); - /// Remove all dead instructions between the I and E. + /// \brief Remove all dead instructions between the I and E. void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E); @@ -297,221 +295,195 @@ public: DebugLoc DL; }; - /// Prepare InsertPt to begin inserting instructions into the local value area - /// and return the old insert position. + /// \brief Prepare InsertPt to begin inserting instructions into the local + /// value area and return the old insert position. SavePoint enterLocalValueArea(); - /// Reset InsertPt to the given old insert position. + /// \brief Reset InsertPt to the given old insert position. void leaveLocalValueArea(SavePoint Old); virtual ~FastISel(); protected: - explicit FastISel(FunctionLoweringInfo &funcInfo, - const TargetLibraryInfo *libInfo); + explicit FastISel(FunctionLoweringInfo &FuncInfo, + const TargetLibraryInfo *LibInfo, + bool SkipTargetIndependentISel = false); - /// This method is called by target-independent code when the normal FastISel - /// process fails to select an instruction. This gives targets a chance to - /// emit code for anything that doesn't fit into FastISel's framework. It - /// returns true if it was successful. - virtual bool TargetSelectInstruction(const Instruction *I) = 0; - - /// This method is called by target-independent code to do target specific - /// argument lowering. It returns true if it was successful. - virtual bool FastLowerArguments(); + /// \brief This method is called by target-independent code when the normal + /// FastISel process fails to select an instruction. This gives targets a + /// chance to emit code for anything that doesn't fit into FastISel's + /// framework. It returns true if it was successful. + virtual bool fastSelectInstruction(const Instruction *I) = 0; - /// \brief This method is called by target-independent code to do target + /// \brief This method is called by target-independent code to do target- + /// specific argument lowering. It returns true if it was successful. + virtual bool fastLowerArguments(); + + /// \brief This method is called by target-independent code to do target- /// specific call lowering. It returns true if it was successful. - virtual bool FastLowerCall(CallLoweringInfo &CLI); + virtual bool fastLowerCall(CallLoweringInfo &CLI); - /// \brief This method is called by target-independent code to do target + /// \brief This method is called by target-independent code to do target- /// specific intrinsic lowering. It returns true if it was successful. - virtual bool FastLowerIntrinsicCall(const IntrinsicInst *II); + virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type and opcode be emitted. - virtual unsigned FastEmit_(MVT VT, - MVT RetVT, - unsigned Opcode); + virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and register operand be emitted. - virtual unsigned FastEmit_r(MVT VT, - MVT RetVT, - unsigned Opcode, - unsigned Op0, bool Op0IsKill); + virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, + bool Op0IsKill); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and register operands be emitted. - virtual unsigned FastEmit_rr(MVT VT, - MVT RetVT, - unsigned Opcode, - unsigned Op0, bool Op0IsKill, - unsigned Op1, bool Op1IsKill); + virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, + bool Op0IsKill, unsigned Op1, bool Op1IsKill); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and register and immediate - /// operands be emitted. - virtual unsigned FastEmit_ri(MVT VT, - MVT RetVT, - unsigned Opcode, - unsigned Op0, bool Op0IsKill, - uint64_t Imm); + // operands be emitted. + virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, + bool Op0IsKill, uint64_t Imm); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and register and floating-point /// immediate operands be emitted. - virtual unsigned FastEmit_rf(MVT VT, - MVT RetVT, - unsigned Opcode, - unsigned Op0, bool Op0IsKill, - const ConstantFP *FPImm); + virtual unsigned fastEmit_rf(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, + bool Op0IsKill, const ConstantFP *FPImm); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and register and immediate /// operands be emitted. - virtual unsigned FastEmit_rri(MVT VT, - MVT RetVT, - unsigned Opcode, - unsigned Op0, bool Op0IsKill, - unsigned Op1, bool Op1IsKill, - uint64_t Imm); + virtual unsigned fastEmit_rri(MVT VT, MVT RetVT, unsigned Opcode, + unsigned Op0, bool Op0IsKill, unsigned Op1, + bool Op1IsKill, uint64_t Imm); - /// \brief This method is a wrapper of FastEmit_ri. - /// + /// \brief This method is a wrapper of fastEmit_ri. + /// /// It first tries to emit an instruction with an immediate operand using - /// FastEmit_ri. If that fails, it materializes the immediate into a register - /// and try FastEmit_rr instead. - unsigned FastEmit_ri_(MVT VT, - unsigned Opcode, - unsigned Op0, bool Op0IsKill, + /// fastEmit_ri. If that fails, it materializes the immediate into a register + /// and try fastEmit_rr instead. + unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and immediate operand be emitted. - virtual unsigned FastEmit_i(MVT VT, - MVT RetVT, - unsigned Opcode, - uint64_t Imm); + virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm); - /// This method is called by target-independent code to request that an + /// \brief This method is called by target-independent code to request that an /// instruction with the given type, opcode, and floating-point immediate /// operand be emitted. - virtual unsigned FastEmit_f(MVT VT, - MVT RetVT, - unsigned Opcode, + virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm); - /// Emit a MachineInstr with no operands and a result register in the given - /// register class. - unsigned FastEmitInst_(unsigned MachineInstOpcode, + /// \brief Emit a MachineInstr with no operands and a result register in the + /// given register class. + unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC); - /// Emit a MachineInstr with one register operand and a result register in the - /// given register class. - unsigned FastEmitInst_r(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill); + /// \brief Emit a MachineInstr with one register operand and a result register + /// in the given register class. + unsigned fastEmitInst_r(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill); - /// Emit a MachineInstr with two register operands and a result register in - /// the given register class. - unsigned FastEmitInst_rr(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - unsigned Op1, bool Op1IsKill); - - /// Emit a MachineInstr with three register operands and a result register in - /// the given register class. - unsigned FastEmitInst_rrr(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - unsigned Op1, bool Op1IsKill, - unsigned Op2, bool Op2IsKill); - - /// Emit a MachineInstr with a register operand, an immediate, and a result + /// \brief Emit a MachineInstr with two register operands and a result /// register in the given register class. - unsigned FastEmitInst_ri(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - uint64_t Imm); + unsigned fastEmitInst_rr(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, unsigned Op1, bool Op1IsKill); - /// Emit a MachineInstr with one register operand and two immediate operands. - unsigned FastEmitInst_rii(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - uint64_t Imm1, uint64_t Imm2); - - /// Emit a MachineInstr with two register operands and a result register in - /// the given register class. - unsigned FastEmitInst_rf(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - const ConstantFP *FPImm); - - /// Emit a MachineInstr with two register operands, an immediate, and a result + /// \brief Emit a MachineInstr with three register operands and a result /// register in the given register class. - unsigned FastEmitInst_rri(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - unsigned Op1, bool Op1IsKill, + unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, unsigned Op1, bool Op1IsKill, + unsigned Op2, bool Op2IsKill); + + /// \brief Emit a MachineInstr with a register operand, an immediate, and a + /// result register in the given register class. + unsigned fastEmitInst_ri(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, uint64_t Imm); + + /// \brief Emit a MachineInstr with one register operand and two immediate + /// operands. + unsigned fastEmitInst_rii(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, uint64_t Imm1, uint64_t Imm2); + + /// \brief Emit a MachineInstr with two register operands and a result + /// register in the given register class. + unsigned fastEmitInst_rf(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, const ConstantFP *FPImm); + + /// \brief Emit a MachineInstr with two register operands, an immediate, and a + /// result register in the given register class. + unsigned fastEmitInst_rri(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm); - /// Emit a MachineInstr with two register operands, two immediates operands, - /// and a result register in the given register class. - unsigned FastEmitInst_rrii(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, - unsigned Op0, bool Op0IsKill, - unsigned Op1, bool Op1IsKill, + /// \brief Emit a MachineInstr with two register operands, two immediates + /// operands, and a result register in the given register class. + unsigned fastEmitInst_rrii(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, unsigned Op0, + bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm1, uint64_t Imm2); - /// Emit a MachineInstr with a single immediate operand, and a result register - /// in the given register class. - unsigned FastEmitInst_i(unsigned MachineInstrOpcode, - const TargetRegisterClass *RC, - uint64_t Imm); + /// \brief Emit a MachineInstr with a single immediate operand, and a result + /// register in the given register class. + unsigned fastEmitInst_i(unsigned MachineInstrOpcode, + const TargetRegisterClass *RC, uint64_t Imm); - /// Emit a MachineInstr with a two immediate operands. - unsigned FastEmitInst_ii(unsigned MachineInstrOpcode, - const TargetRegisterClass *RC, - uint64_t Imm1, uint64_t Imm2); + /// \brief Emit a MachineInstr with a two immediate operands. + unsigned fastEmitInst_ii(unsigned MachineInstrOpcode, + const TargetRegisterClass *RC, uint64_t Imm1, + uint64_t Imm2); - /// Emit a MachineInstr for an extract_subreg from a specified index of a - /// superregister to a specified type. - unsigned FastEmitInst_extractsubreg(MVT RetVT, - unsigned Op0, bool Op0IsKill, + /// \brief Emit a MachineInstr for an extract_subreg from a specified index of + /// a superregister to a specified type. + unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx); - /// Emit MachineInstrs to compute the value of Op with all but the least - /// significant bit set to zero. - unsigned FastEmitZExtFromI1(MVT VT, - unsigned Op0, bool Op0IsKill); + /// \brief Emit MachineInstrs to compute the value of Op with all but the + /// least significant bit set to zero. + unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill); - /// Emit an unconditional branch to the given block, unless it is the + /// \brief Emit an unconditional branch to the given block, unless it is the /// immediate (fall-through) successor, and update the CFG. - void FastEmitBranch(MachineBasicBlock *MBB, DebugLoc DL); + void fastEmitBranch(MachineBasicBlock *MBB, DebugLoc DL); - void UpdateValueMap(const Value* I, unsigned Reg, unsigned NumRegs = 1); + /// \brief Update the value map to include the new mapping for this + /// instruction, or insert an extra copy to get the result in a previous + /// determined register. + /// + /// NOTE: This is only necessary because we might select a block that uses a + /// value before we select the block that defines the value. It might be + /// possible to fix this by selecting blocks in reverse postorder. + void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs = 1); unsigned createResultReg(const TargetRegisterClass *RC); - /// Try to constrain Op so that it is usable by argument OpNum of the provided - /// MCInstrDesc. If this fails, create a new virtual register in the correct - /// class and COPY the value there. + /// \brief Try to constrain Op so that it is usable by argument OpNum of the + /// provided MCInstrDesc. If this fails, create a new virtual register in the + /// correct class and COPY the value there. unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum); - /// Emit a constant in a register using target-specific logic, such as + /// \brief Emit a constant in a register using target-specific logic, such as /// constant pool loads. - virtual unsigned TargetMaterializeConstant(const Constant* C) { - return 0; - } + virtual unsigned fastMaterializeConstant(const Constant *C) { return 0; } - /// Emit an alloca address in a register using target-specific logic. - virtual unsigned TargetMaterializeAlloca(const AllocaInst* C) { - return 0; - } + /// \brief Emit an alloca address in a register using target-specific logic. + virtual unsigned fastMaterializeAlloca(const AllocaInst *C) { return 0; } - virtual unsigned TargetMaterializeFloatZero(const ConstantFP* CF) { + /// \brief Emit the floating-point constant +0.0 in a register using target- + /// specific logic. + virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF) { return 0; } @@ -524,36 +496,46 @@ protected: /// - \c Add has a constant operand. bool canFoldAddIntoGEP(const User *GEP, const Value *Add); - /// Test whether the given value has exactly one use. - bool hasTrivialKill(const Value *V) const; + /// \brief Test whether the given value has exactly one use. + bool hasTrivialKill(const Value *V); /// \brief Create a machine mem operand from the given instruction. MachineMemOperand *createMachineMemOperandFor(const Instruction *I) const; - bool LowerCallTo(const CallInst *CI, const char *SymName, unsigned NumArgs); - bool LowerCallTo(CallLoweringInfo &CLI); + CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const; + + bool lowerCallTo(const CallInst *CI, const char *SymName, unsigned NumArgs); + bool lowerCallTo(CallLoweringInfo &CLI); + + bool isCommutativeIntrinsic(IntrinsicInst const *II) { + switch (II->getIntrinsicID()) { + case Intrinsic::sadd_with_overflow: + case Intrinsic::uadd_with_overflow: + case Intrinsic::smul_with_overflow: + case Intrinsic::umul_with_overflow: + return true; + default: + return false; + } + } + + + bool lowerCall(const CallInst *I); + /// \brief Select and emit code for a binary operator instruction, which has + /// an opcode which directly corresponds to the given ISD opcode. + bool selectBinaryOp(const User *I, unsigned ISDOpcode); + bool selectFNeg(const User *I); + bool selectGetElementPtr(const User *I); + bool selectStackmap(const CallInst *I); + bool selectPatchpoint(const CallInst *I); + bool selectCall(const User *Call); + bool selectIntrinsicCall(const IntrinsicInst *II); + bool selectBitCast(const User *I); + bool selectCast(const User *I, unsigned Opcode); + bool selectExtractValue(const User *I); + bool selectInsertValue(const User *I); private: - bool SelectBinaryOp(const User *I, unsigned ISDOpcode); - - bool SelectFNeg(const User *I); - - bool SelectGetElementPtr(const User *I); - - bool SelectStackmap(const CallInst *I); - bool SelectPatchpoint(const CallInst *I); - bool LowerCall(const CallInst *I); - bool SelectCall(const User *Call); - bool SelectIntrinsicCall(const IntrinsicInst *II); - - bool SelectBitCast(const User *I); - - bool SelectCast(const User *I, unsigned Opcode); - - bool SelectExtractValue(const User *I); - - bool SelectInsertValue(const User *I); - /// \brief Handle PHI nodes in successor blocks. /// /// Emit code to ensure constants are copied into registers when needed. @@ -561,18 +543,27 @@ private: /// nodes as input. We cannot just directly add them, because expansion might /// result in multiple MBB's for one BB. As such, the start of the BB might /// correspond to a different MBB than the end. - bool HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB); + bool handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB); - /// Helper for getRegForVale. This function is called when the value isn't - /// already available in a register and must be materialized with new + /// \brief Helper for materializeRegForValue to materialize a constant in a + /// target-independent way. + unsigned materializeConstant(const Value *V, MVT VT); + + /// \brief Helper for getRegForVale. This function is called when the value + /// isn't already available in a register and must be materialized with new /// instructions. unsigned materializeRegForValue(const Value *V, MVT VT); - /// Clears LocalValueMap and moves the area for the new local variables to the - /// beginning of the block. It helps to avoid spilling cached variables across - /// heavy instructions like calls. + /// \brief Clears LocalValueMap and moves the area for the new local variables + /// to the beginning of the block. It helps to avoid spilling cached variables + /// across heavy instructions like calls. void flushLocalValueMap(); + /// \brief Insertion point before trying to select the current instruction. + MachineBasicBlock::iterator SavedInsertPt; + + /// \brief Add a stackmap or patchpoint intrinsic call's live variable + /// operands to a stackmap or patchpoint machine instruction. bool addStackMapLiveVars(SmallVectorImpl &Ops, const CallInst *CI, unsigned StartIdx); bool lowerCallOperands(const CallInst *CI, unsigned ArgIdx, unsigned NumArgs, @@ -580,6 +571,6 @@ private: CallLoweringInfo &CLI); }; -} +} // end namespace llvm #endif diff --git a/contrib/llvm/include/llvm/CodeGen/ForwardControlFlowIntegrity.h b/contrib/llvm/include/llvm/CodeGen/ForwardControlFlowIntegrity.h new file mode 100644 index 000000000000..ec8e2ef243b7 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/ForwardControlFlowIntegrity.h @@ -0,0 +1,122 @@ +//===-- ForwardControlFlowIntegrity.h: Forward-Edge CFI ---------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass instruments indirect calls with checks to ensure that these calls +// pass through the appropriate jump-instruction table generated by +// JumpInstrTables. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H +#define LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Pass.h" +#include "llvm/Target/TargetOptions.h" +#include + +namespace llvm { + +class AnalysisUsage; +class BasicBlock; +class Constant; +class Function; +class Instruction; +class Module; +class Value; + +/// ForwardControlFlowIntegrity uses the information from JumpInstrTableInfo to +/// prepend checks to indirect calls to make sure that these calls target valid +/// locations. +class ForwardControlFlowIntegrity : public ModulePass { +public: + static char ID; + + ForwardControlFlowIntegrity(); + ForwardControlFlowIntegrity(JumpTable::JumpTableType JTT, + CFIntegrity CFIType, + bool CFIEnforcing, std::string CFIFuncName); + ~ForwardControlFlowIntegrity() override; + + /// Runs the CFI pass on a given module. This works best if the module in + /// question is the result of link-time optimization (see lib/LTO). + bool runOnModule(Module &M) override; + const char *getPassName() const override { + return "Forward Control-Flow Integrity"; + } + void getAnalysisUsage(AnalysisUsage &AU) const override; + +private: + typedef SmallVector CallSet; + + /// A structure that is used to keep track of constant table information. + struct CFIConstants { + Constant *StartValue; + Constant *MaskValue; + Constant *Size; + }; + + /// A map from function type to the base of the table for this type and a mask + /// for the table + typedef DenseMap CFITables; + + CallSet IndirectCalls; + + /// The type of jumptable implementation. + JumpTable::JumpTableType JTType; + + /// The type of CFI check to add before each indirect call. + CFIntegrity CFIType; + + /// A value that controls whether or not CFI violations cause a halt. + bool CFIEnforcing; + + /// The name of the function to call in case of a CFI violation when + /// CFIEnforcing is false. There is a default function that ignores + /// violations. + std::string CFIFuncName; + + /// The alignment of each entry in the table, from JumpInstrTableInfo. The + /// JumpInstrTableInfo class always makes this a power of two. + uint64_t ByteAlignment; + + /// The base-2 logarithm of ByteAlignment, needed for some of the transforms + /// (like CFIntegrity::Ror) + unsigned LogByteAlignment; + + /// Adds checks to each indirect call site to make sure that it is calling a + /// function in our jump table. + void updateIndirectCalls(Module &M, CFITables &CFIT); + + /// Walks the instructions to find all the indirect calls. + void getIndirectCalls(Module &M); + + /// Adds a function that handles violations in non-enforcing mode + /// (!CFIEnforcing). The default warning function simply returns, since the + /// exact details of how to handle CFI violations depend on the application. + void addWarningFunction(Module &M); + + /// Rewrites a function pointer in a call/invoke instruction to force it into + /// a table. + void rewriteFunctionPointer(Module &M, Instruction *I, Value *FunPtr, + Constant *JumpTableStart, Constant *JumpTableMask, + Constant *JumpTableSize); + + /// Inserts a check and a call to a warning function at a given instruction + /// that must be an indirect call. + void insertWarning(Module &M, BasicBlock *Block, Instruction *I, + Value *FunPtr); +}; + +ModulePass * +createForwardControlFlowIntegrityPass(JumpTable::JumpTableType JTT, + CFIntegrity CFIType, + bool CFIEnforcing, StringRef CFIFuncName); +} + +#endif // LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H diff --git a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index 9636b51e303d..7c574df4ba41 100644 --- a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -20,6 +20,7 @@ #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" @@ -50,10 +51,10 @@ class Value; /// function that is used when lowering a region of the function. /// class FunctionLoweringInfo { - const TargetMachine &TM; public: const Function *Fn; MachineFunction *MF; + const TargetLowering *TLI; MachineRegisterInfo *RegInfo; BranchProbabilityInfo *BPI; /// CanLowerReturn - true iff the function's return value can be lowered to @@ -87,6 +88,12 @@ public: /// RegFixups - Registers which need to be replaced after isel is done. DenseMap RegFixups; + /// StatepointStackSlots - A list of temporary stack slots (frame indices) + /// used to spill values at a statepoint. We store them here to enable + /// reuse of the same stack slots across different statepoints in different + /// basic blocks. + SmallVector StatepointStackSlots; + /// MBB - The current block. MachineBasicBlock *MBB; @@ -106,6 +113,10 @@ public: KnownZero(1, 0) {} }; + /// Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) + /// for a value. + DenseMap PreferredExtendType; + /// VisitedBBs - The set of basic blocks visited thus far by instruction /// selection. SmallPtrSet VisitedBBs; @@ -115,14 +126,13 @@ public: /// TODO: This isn't per-function state, it's per-basic-block state. But /// there's no other convenient place for it to live right now. std::vector > PHINodesToUpdate; + unsigned OrigNumPHINodesToUpdate; /// If the current MBB is a landing pad, the exception pointer and exception /// selector registers are copied into these virtual registers by /// SelectionDAGISel::PrepareEHLandingPad(). unsigned ExceptionPointerVirtReg, ExceptionSelectorVirtReg; - explicit FunctionLoweringInfo(const TargetMachine &TM) : TM(TM) {} - /// set - Initialize this FunctionLoweringInfo with the given Function /// and its associated MachineFunction. /// @@ -196,6 +206,9 @@ public: return; unsigned Reg = It->second; + if (Reg == 0) + return; + LiveOutRegInfo.grow(Reg); LiveOutRegInfo[Reg].IsValid = false; } diff --git a/contrib/llvm/include/llvm/CodeGen/GCMetadata.h b/contrib/llvm/include/llvm/CodeGen/GCMetadata.h index ddcc823ecd9e..c7f1ab87fcb1 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCMetadata.h +++ b/contrib/llvm/include/llvm/CodeGen/GCMetadata.h @@ -37,7 +37,6 @@ #include "llvm/ADT/StringMap.h" #include "llvm/IR/DebugLoc.h" #include "llvm/Pass.h" - #include namespace llvm { @@ -80,8 +79,8 @@ namespace llvm { }; - /// GCFunctionInfo - Garbage collection metadata for a single function. - /// + /// Garbage collection metadata for a single function. Currently, this + /// information only applies to GCStrategies which use GCRoot. class GCFunctionInfo { public: typedef std::vector::iterator iterator; @@ -160,21 +159,37 @@ namespace llvm { size_t live_size(const iterator &p) const { return roots_size(); } }; - - /// GCModuleInfo - Garbage collection metadata for a whole module. - /// + /// An analysis pass which caches information about the entire Module. + /// Records both the function level information used by GCRoots and a + /// cache of the 'active' gc strategy objects for the current Module. class GCModuleInfo : public ImmutablePass { typedef StringMap strategy_map_type; typedef std::vector> list_type; - typedef DenseMap finfo_map_type; strategy_map_type StrategyMap; list_type StrategyList; - finfo_map_type FInfoMap; GCStrategy *getOrCreateStrategy(const Module *M, const std::string &Name); public: + /// List of per function info objects. In theory, Each of these + /// may be associated with a different GC. + typedef std::vector> FuncInfoVec; + + FuncInfoVec::iterator funcinfo_begin() { return Functions.begin(); } + FuncInfoVec::iterator funcinfo_end() { return Functions.end(); } + + + private: + /// Owning list of all GCFunctionInfos associated with this Module + FuncInfoVec Functions; + + /// Non-owning map to bypass linear search when finding the GCFunctionInfo + /// associated with a particular Function. + typedef DenseMap finfo_map_type; + finfo_map_type FInfoMap; + public: + typedef list_type::const_iterator iterator; static char ID; @@ -191,8 +206,9 @@ namespace llvm { iterator begin() const { return StrategyList.begin(); } iterator end() const { return StrategyList.end(); } - /// get - Look up function metadata. - /// + /// get - Look up function metadata. This is currently assumed + /// have the side effect of initializing the associated GCStrategy. That + /// will soon change. GCFunctionInfo &getFunctionInfo(const Function &F); }; diff --git a/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h b/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h index 4a6b5ac19c36..25fafba93f8b 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h +++ b/contrib/llvm/include/llvm/CodeGen/GCMetadataPrinter.h @@ -32,16 +32,11 @@ namespace llvm { /// defaults from Registry. typedef Registry GCMetadataPrinterRegistry; - /// GCMetadataPrinter - Emits GC metadata as assembly code. - /// + /// GCMetadataPrinter - Emits GC metadata as assembly code. Instances are + /// created, managed, and owned by the AsmPrinter. class GCMetadataPrinter { - public: - typedef GCStrategy::list_type list_type; - typedef GCStrategy::iterator iterator; - private: GCStrategy *S; - friend class AsmPrinter; protected: @@ -55,16 +50,15 @@ namespace llvm { public: GCStrategy &getStrategy() { return *S; } - const Module &getModule() const { return S->getModule(); } - /// begin/end - Iterate over the collected function metadata. - iterator begin() { return S->begin(); } - iterator end() { return S->end(); } - - /// beginAssembly/finishAssembly - Emit module metadata as assembly code. - virtual void beginAssembly(AsmPrinter &AP); - - virtual void finishAssembly(AsmPrinter &AP); + /// Called before the assembly for the module is generated by + /// the AsmPrinter (but after target specific hooks.) + virtual void beginAssembly(Module &M, GCModuleInfo &Info, + AsmPrinter &AP) {} + /// Called after the assembly for the module is generated by + /// the AsmPrinter (but before target specific hooks) + virtual void finishAssembly(Module &M, GCModuleInfo &Info, + AsmPrinter &AP) {} virtual ~GCMetadataPrinter(); }; diff --git a/contrib/llvm/include/llvm/CodeGen/GCStrategy.h b/contrib/llvm/include/llvm/CodeGen/GCStrategy.h index 81e1f85286e1..0b0c3124c537 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCStrategy.h +++ b/contrib/llvm/include/llvm/CodeGen/GCStrategy.h @@ -12,9 +12,14 @@ // specified in a function's 'gc' attribute. Algorithms are enabled by setting // flags in a subclass's constructor, and some virtual methods can be // overridden. +// +// GCStrategy is relevant for implementations using either gc.root or +// gc.statepoint based lowering strategies, but is currently focused mostly on +// options for gc.root. This will change over time. // -// When requested, the GCStrategy will be populated with data about each -// function which uses it. Specifically: +// When requested by a subclass of GCStrategy, the gc.root implementation will +// populate GCModuleInfo and GCFunctionInfo with that about each Function in +// the Module that opts in to garbage collection. Specifically: // // - Safe points // Garbage collection is generally only possible at certain points in code. @@ -31,40 +36,42 @@ // This information can used to emit the metadata tables which are required by // the target garbage collector runtime. // +// When used with gc.statepoint, information about safepoint and roots can be +// found in the binary StackMap section after code generation. Safepoint +// placement is currently the responsibility of the frontend, though late +// insertion support is planned. gc.statepoint does not currently support +// custom stack map formats; such can be generated by parsing the standard +// stack map section if desired. +// +// The read and write barrier support can be used with either implementation. +// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_GCSTRATEGY_H #define LLVM_CODEGEN_GCSTRATEGY_H +#include "llvm/ADT/Optional.h" #include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/Support/Registry.h" #include namespace llvm { - - class GCStrategy; - - /// The GC strategy registry uses all the defaults from Registry. - /// - typedef Registry GCRegistry; - /// GCStrategy describes a garbage collector algorithm's code generation /// requirements, and provides overridable hooks for those needs which cannot - /// be abstractly described. + /// be abstractly described. GCStrategy objects currently must be looked up + /// through the GCModuleInfo analysis pass. They are owned by the analysis + /// pass and recreated every time that pass is invalidated. class GCStrategy { - public: - typedef std::vector> list_type; - typedef list_type::iterator iterator; - private: - friend class GCModuleInfo; - const Module *M; std::string Name; - - list_type Functions; + friend class GCModuleInfo; protected: + bool UseStatepoints; /// Uses gc.statepoints as opposed to gc.roots, + /// if set, none of the other options can be + /// anything but their default values. + unsigned NeededSafePoints; ///< Bitmask of required safe points. bool CustomReadBarriers; ///< Default is to insert loads. bool CustomWriteBarriers; ///< Default is to insert stores. @@ -76,78 +83,114 @@ namespace llvm { public: GCStrategy(); - virtual ~GCStrategy() {} - - /// getName - The name of the GC strategy, for debugging. - /// + /// Return the name of the GC strategy. This is the value of the collector + /// name string specified on functions which use this strategy. const std::string &getName() const { return Name; } - /// getModule - The module within which the GC strategy is operating. - /// - const Module &getModule() const { return *M; } + /// By default, write barriers are replaced with simple store + /// instructions. If true, then performCustomLowering must instead lower + /// them. + bool customWriteBarrier() const { return CustomWriteBarriers; } + + /// By default, read barriers are replaced with simple load + /// instructions. If true, then performCustomLowering must instead lower + /// them. + bool customReadBarrier() const { return CustomReadBarriers; } - /// needsSafePoitns - True if safe points of any kind are required. By - // default, none are recorded. + /// Returns true if this strategy is expecting the use of gc.statepoints, + /// and false otherwise. + bool useStatepoints() const { return UseStatepoints; } + + /** @name Statepoint Specific Properties */ + ///@{ + + /// If the value specified can be reliably distinguished, returns true for + /// pointers to GC managed locations and false for pointers to non-GC + /// managed locations. Note a GCStrategy can always return 'None' (i.e. an + /// empty optional indicating it can't reliably distinguish. + virtual Optional isGCManagedPointer(const Value *V) const { + return None; + } + ///@} + + /** @name GCRoot Specific Properties + * These properties and overrides only apply to collector strategies using + * GCRoot. + */ + ///@{ + + /// True if safe points of any kind are required. By default, none are + /// recorded. bool needsSafePoints() const { return CustomSafePoints || NeededSafePoints != 0; } - /// needsSafePoint(Kind) - True if the given kind of safe point is - // required. By default, none are recorded. + /// True if the given kind of safe point is required. By default, none are + /// recorded. bool needsSafePoint(GC::PointKind Kind) const { return (NeededSafePoints & 1 << Kind) != 0; } - - /// customWriteBarrier - By default, write barriers are replaced with simple - /// store instructions. If true, then - /// performCustomLowering must instead lower them. - bool customWriteBarrier() const { return CustomWriteBarriers; } - - /// customReadBarrier - By default, read barriers are replaced with simple - /// load instructions. If true, then - /// performCustomLowering must instead lower them. - bool customReadBarrier() const { return CustomReadBarriers; } - - /// customRoots - By default, roots are left for the code generator so it - /// can generate a stack map. If true, then - // performCustomLowering must delete them. + + /// By default, roots are left for the code generator so it can generate a + /// stack map. If true, then performCustomLowering must delete them. bool customRoots() const { return CustomRoots; } - /// customSafePoints - By default, the GC analysis will find safe - /// points according to NeededSafePoints. If true, - /// then findCustomSafePoints must create them. + /// By default, the GC analysis will find safe points according to + /// NeededSafePoints. If true, then findCustomSafePoints must create them. bool customSafePoints() const { return CustomSafePoints; } - /// initializeRoots - If set, gcroot intrinsics should initialize their - // allocas to null before the first use. This is - // necessary for most GCs and is enabled by default. + /// If set, gcroot intrinsics should initialize their allocas to null + /// before the first use. This is necessary for most GCs and is enabled by + /// default. bool initializeRoots() const { return InitRoots; } - /// usesMetadata - If set, appropriate metadata tables must be emitted by - /// the back-end (assembler, JIT, or otherwise). + /// If set, appropriate metadata tables must be emitted by the back-end + /// (assembler, JIT, or otherwise). For statepoint, this method is + /// currently unsupported. The stackmap information can be found in the + /// StackMap section as described in the documentation. bool usesMetadata() const { return UsesMetadata; } + + ///@} - /// begin/end - Iterators for function metadata. - /// - iterator begin() { return Functions.begin(); } - iterator end() { return Functions.end(); } - - /// insertFunctionMetadata - Creates metadata for a function. - /// - GCFunctionInfo *insertFunctionInfo(const Function &F); - /// initializeCustomLowering/performCustomLowering - If any of the actions /// are set to custom, performCustomLowering must be overriden to transform /// the corresponding actions to LLVM IR. initializeCustomLowering is /// optional to override. These are the only GCStrategy methods through - /// which the LLVM IR can be modified. - virtual bool initializeCustomLowering(Module &F); - virtual bool performCustomLowering(Function &F); - virtual bool findCustomSafePoints(GCFunctionInfo& FI, MachineFunction& MF); + /// which the LLVM IR can be modified. These methods apply mostly to + /// gc.root based implementations, but can be overriden to provide custom + /// barrier lowerings with gc.statepoint as well. + ///@{ + virtual bool initializeCustomLowering(Module &F) { + // No changes made + return false; + } + virtual bool performCustomLowering(Function &F) { + llvm_unreachable("GCStrategy subclass specified a configuration which" + "requires a custom lowering without providing one"); + } + ///@} + /// Called if customSafepoints returns true, used only by gc.root + /// implementations. + virtual bool findCustomSafePoints(GCFunctionInfo& FI, MachineFunction& MF) { + llvm_unreachable("GCStrategy subclass specified a configuration which" + "requests custom safepoint identification without" + "providing an implementation for such"); + } }; - + + /// Subclasses of GCStrategy are made available for use during compilation by + /// adding them to the global GCRegistry. This can done either within the + /// LLVM source tree or via a loadable plugin. An example registeration + /// would be: + /// static GCRegistry::Add X("custom-name", + /// "my custom supper fancy gc strategy"); + /// + /// Note that to use a custom GCMetadataPrinter w/gc.roots, you must also + /// register your GCMetadataPrinter subclass with the + /// GCMetadataPrinterRegistery as well. + typedef Registry GCRegistry; } #endif diff --git a/contrib/llvm/include/llvm/CodeGen/GCs.h b/contrib/llvm/include/llvm/CodeGen/GCs.h index bb170c85cbf8..51a31842a5c0 100644 --- a/contrib/llvm/include/llvm/CodeGen/GCs.h +++ b/contrib/llvm/include/llvm/CodeGen/GCs.h @@ -36,6 +36,8 @@ namespace llvm { /// Creates a shadow stack garbage collector. This collector requires no code /// generator support. void linkShadowStackGC(); + + void linkStatepointExampleGC(); } #endif diff --git a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h index 84447616c989..952362ed6ce3 100644 --- a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -72,6 +72,11 @@ namespace ISD { /// the parent's frame or return address, and so on. FRAMEADDR, RETURNADDR, + /// FRAME_ALLOC_RECOVER - Represents the llvm.framerecover + /// intrinsic. Materializes the offset from the frame pointer of another + /// function to the result of llvm.frameallocate. + FRAME_ALLOC_RECOVER, + /// READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on /// the DAG, which implements the named register global variables extension. READ_REGISTER, @@ -485,7 +490,8 @@ namespace ISD { FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR, - + FMINNUM, FMAXNUM, + /// FSINCOS - Compute both fsin and fcos as a single operation. FSINCOS, @@ -674,6 +680,9 @@ namespace ISD { ATOMIC_LOAD_UMIN, ATOMIC_LOAD_UMAX, + // Masked load and store + MLOAD, MSTORE, + /// This corresponds to the llvm.lifetime.* intrinsics. The first operand /// is the chain and the second operand is the alloca pointer. LIFETIME_START, LIFETIME_END, @@ -744,7 +753,7 @@ namespace ISD { LAST_LOADEXT_TYPE }; - NodeType getExtForLoadExtType(LoadExtType); + NodeType getExtForLoadExtType(bool IsFP, LoadExtType); //===--------------------------------------------------------------------===// /// ISD::CondCode enum - These are ordered carefully to make the bitfields diff --git a/contrib/llvm/include/llvm/CodeGen/JITCodeEmitter.h b/contrib/llvm/include/llvm/CodeGen/JITCodeEmitter.h deleted file mode 100644 index dc2a0272db4e..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/JITCodeEmitter.h +++ /dev/null @@ -1,344 +0,0 @@ -//===-- llvm/CodeGen/JITCodeEmitter.h - Code emission ----------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines an abstract interface that is used by the machine code -// emission framework to output the code. This allows machine code emission to -// be separated from concerns such as resolution of call targets, and where the -// machine code will be written (memory or disk, f.e.). -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_JITCODEEMITTER_H -#define LLVM_CODEGEN_JITCODEEMITTER_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/CodeGen/MachineCodeEmitter.h" -#include "llvm/Support/DataTypes.h" -#include "llvm/Support/MathExtras.h" -#include - -namespace llvm { - -class MachineBasicBlock; -class MachineConstantPool; -class MachineJumpTableInfo; -class MachineFunction; -class MachineModuleInfo; -class MachineRelocation; -class Value; -class GlobalValue; -class Function; - -/// JITCodeEmitter - This class defines two sorts of methods: those for -/// emitting the actual bytes of machine code, and those for emitting auxiliary -/// structures, such as jump tables, relocations, etc. -/// -/// Emission of machine code is complicated by the fact that we don't (in -/// general) know the size of the machine code that we're about to emit before -/// we emit it. As such, we preallocate a certain amount of memory, and set the -/// BufferBegin/BufferEnd pointers to the start and end of the buffer. As we -/// emit machine instructions, we advance the CurBufferPtr to indicate the -/// location of the next byte to emit. In the case of a buffer overflow (we -/// need to emit more machine code than we have allocated space for), the -/// CurBufferPtr will saturate to BufferEnd and ignore stores. Once the entire -/// function has been emitted, the overflow condition is checked, and if it has -/// occurred, more memory is allocated, and we reemit the code into it. -/// -class JITCodeEmitter : public MachineCodeEmitter { - void anchor() override; -public: - virtual ~JITCodeEmitter() {} - - /// startFunction - This callback is invoked when the specified function is - /// about to be code generated. This initializes the BufferBegin/End/Ptr - /// fields. - /// - void startFunction(MachineFunction &F) override = 0; - - /// finishFunction - This callback is invoked when the specified function has - /// finished code generation. If a buffer overflow has occurred, this method - /// returns true (the callee is required to try again), otherwise it returns - /// false. - /// - bool finishFunction(MachineFunction &F) override = 0; - - /// allocIndirectGV - Allocates and fills storage for an indirect - /// GlobalValue, and returns the address. - virtual void *allocIndirectGV(const GlobalValue *GV, - const uint8_t *Buffer, size_t Size, - unsigned Alignment) = 0; - - /// emitByte - This callback is invoked when a byte needs to be written to the - /// output stream. - /// - void emitByte(uint8_t B) { - if (CurBufferPtr != BufferEnd) - *CurBufferPtr++ = B; - } - - /// emitWordLE - This callback is invoked when a 32-bit word needs to be - /// written to the output stream in little-endian format. - /// - void emitWordLE(uint32_t W) { - if (4 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 0); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 24); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitWordBE - This callback is invoked when a 32-bit word needs to be - /// written to the output stream in big-endian format. - /// - void emitWordBE(uint32_t W) { - if (4 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 24); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 0); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitDWordLE - This callback is invoked when a 64-bit word needs to be - /// written to the output stream in little-endian format. - /// - void emitDWordLE(uint64_t W) { - if (8 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 0); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 24); - *CurBufferPtr++ = (uint8_t)(W >> 32); - *CurBufferPtr++ = (uint8_t)(W >> 40); - *CurBufferPtr++ = (uint8_t)(W >> 48); - *CurBufferPtr++ = (uint8_t)(W >> 56); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitDWordBE - This callback is invoked when a 64-bit word needs to be - /// written to the output stream in big-endian format. - /// - void emitDWordBE(uint64_t W) { - if (8 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 56); - *CurBufferPtr++ = (uint8_t)(W >> 48); - *CurBufferPtr++ = (uint8_t)(W >> 40); - *CurBufferPtr++ = (uint8_t)(W >> 32); - *CurBufferPtr++ = (uint8_t)(W >> 24); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 0); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitAlignment - Move the CurBufferPtr pointer up to the specified - /// alignment (saturated to BufferEnd of course). - void emitAlignment(unsigned Alignment) { - if (Alignment == 0) Alignment = 1; - uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr, - Alignment); - CurBufferPtr = std::min(NewPtr, BufferEnd); - } - - /// emitAlignmentWithFill - Similar to emitAlignment, except that the - /// extra bytes are filled with the provided byte. - void emitAlignmentWithFill(unsigned Alignment, uint8_t Fill) { - if (Alignment == 0) Alignment = 1; - uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr, - Alignment); - // Fail if we don't have room. - if (NewPtr > BufferEnd) { - CurBufferPtr = BufferEnd; - return; - } - while (CurBufferPtr < NewPtr) { - *CurBufferPtr++ = Fill; - } - } - - /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be - /// written to the output stream. - void emitULEB128Bytes(uint64_t Value, unsigned PadTo = 0) { - do { - uint8_t Byte = Value & 0x7f; - Value >>= 7; - if (Value || PadTo != 0) Byte |= 0x80; - emitByte(Byte); - } while (Value); - - if (PadTo) { - do { - uint8_t Byte = (PadTo > 1) ? 0x80 : 0x0; - emitByte(Byte); - } while (--PadTo); - } - } - - /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be - /// written to the output stream. - void emitSLEB128Bytes(int64_t Value) { - int32_t Sign = Value >> (8 * sizeof(Value) - 1); - bool IsMore; - - do { - uint8_t Byte = Value & 0x7f; - Value >>= 7; - IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0; - if (IsMore) Byte |= 0x80; - emitByte(Byte); - } while (IsMore); - } - - /// emitString - This callback is invoked when a String needs to be - /// written to the output stream. - void emitString(const std::string &String) { - for (size_t i = 0, N = String.size(); i < N; ++i) { - uint8_t C = String[i]; - emitByte(C); - } - emitByte(0); - } - - /// emitInt32 - Emit a int32 directive. - void emitInt32(uint32_t Value) { - if (4 <= BufferEnd-CurBufferPtr) { - *((uint32_t*)CurBufferPtr) = Value; - CurBufferPtr += 4; - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitInt64 - Emit a int64 directive. - void emitInt64(uint64_t Value) { - if (8 <= BufferEnd-CurBufferPtr) { - *((uint64_t*)CurBufferPtr) = Value; - CurBufferPtr += 8; - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitInt32At - Emit the Int32 Value in Addr. - void emitInt32At(uintptr_t *Addr, uintptr_t Value) { - if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd) - (*(uint32_t*)Addr) = (uint32_t)Value; - } - - /// emitInt64At - Emit the Int64 Value in Addr. - void emitInt64At(uintptr_t *Addr, uintptr_t Value) { - if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd) - (*(uint64_t*)Addr) = (uint64_t)Value; - } - - - /// emitLabel - Emits a label - void emitLabel(MCSymbol *Label) override = 0; - - /// allocateSpace - Allocate a block of space in the current output buffer, - /// returning null (and setting conditions to indicate buffer overflow) on - /// failure. Alignment is the alignment in bytes of the buffer desired. - void *allocateSpace(uintptr_t Size, unsigned Alignment) override { - emitAlignment(Alignment); - void *Result; - - // Check for buffer overflow. - if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) { - CurBufferPtr = BufferEnd; - Result = nullptr; - } else { - // Allocate the space. - Result = CurBufferPtr; - CurBufferPtr += Size; - } - - return Result; - } - - /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, - /// this method does not allocate memory in the current output buffer, - /// because a global may live longer than the current function. - virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment) = 0; - - /// StartMachineBasicBlock - This should be called by the target when a new - /// basic block is about to be emitted. This way the MCE knows where the - /// start of the block is, and can implement getMachineBasicBlockAddress. - void StartMachineBasicBlock(MachineBasicBlock *MBB) override = 0; - - /// getCurrentPCValue - This returns the address that the next emitted byte - /// will be output to. - /// - uintptr_t getCurrentPCValue() const override { - return (uintptr_t)CurBufferPtr; - } - - /// getCurrentPCOffset - Return the offset from the start of the emitted - /// buffer that we are currently writing to. - uintptr_t getCurrentPCOffset() const override { - return CurBufferPtr-BufferBegin; - } - - /// earlyResolveAddresses - True if the code emitter can use symbol addresses - /// during code emission time. The JIT is capable of doing this because it - /// creates jump tables or constant pools in memory on the fly while the - /// object code emitters rely on a linker to have real addresses and should - /// use relocations instead. - bool earlyResolveAddresses() const override { return true; } - - /// addRelocation - Whenever a relocatable address is needed, it should be - /// noted with this interface. - void addRelocation(const MachineRelocation &MR) override = 0; - - /// FIXME: These should all be handled with relocations! - - /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in - /// the constant pool that was last emitted with the emitConstantPool method. - /// - uintptr_t getConstantPoolEntryAddress(unsigned Index) const override = 0; - - /// getJumpTableEntryAddress - Return the address of the jump table with index - /// 'Index' in the function that last called initJumpTableInfo. - /// - uintptr_t getJumpTableEntryAddress(unsigned Index) const override = 0; - - /// getMachineBasicBlockAddress - Return the address of the specified - /// MachineBasicBlock, only usable after the label for the MBB has been - /// emitted. - /// - uintptr_t - getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override = 0; - - /// getLabelAddress - Return the address of the specified Label, only usable - /// after the Label has been emitted. - /// - uintptr_t getLabelAddress(MCSymbol *Label) const override = 0; - - /// Specifies the MachineModuleInfo object. This is used for exception handling - /// purposes. - void setModuleInfo(MachineModuleInfo* Info) override = 0; - - /// getLabelLocations - Return the label locations map of the label IDs to - /// their address. - virtual DenseMap *getLabelLocations() { - return nullptr; - } -}; - -} // End llvm namespace - -#endif diff --git a/contrib/llvm/include/llvm/CodeGen/JumpInstrTables.h b/contrib/llvm/include/llvm/CodeGen/JumpInstrTables.h index 6ca3d7d1765f..005bc1eb2b2d 100644 --- a/contrib/llvm/include/llvm/CodeGen/JumpInstrTables.h +++ b/contrib/llvm/include/llvm/CodeGen/JumpInstrTables.h @@ -39,13 +39,14 @@ class Module; /// jmp f_orig@PLT /// \endverbatim /// -/// Support for an architecture depends on two functions in TargetInstrInfo: -/// getUnconditionalBranch, and getTrap. AsmPrinter uses these to generate the -/// appropriate instructions for the jump statement (an unconditional branch) -/// and for padding to make the table have a size that is a power of two. This -/// padding uses a trap instruction to ensure that calls to this area halt the -/// program. The default implementations of these functions call -/// llvm_unreachable. +/// Support for an architecture depends on three functions in TargetInstrInfo: +/// getUnconditionalBranch, getTrap, and getJumpInstrTableEntryBound. AsmPrinter +/// uses these to generate the appropriate instructions for the jump statement +/// (an unconditional branch) and for padding to make the table have a size that +/// is a power of two. This padding uses a trap instruction to ensure that calls +/// to this area halt the program. The default implementations of these +/// functions call llvm_unreachable, except for getJumpInstrTableEntryBound, +/// which returns 0 by default. class JumpInstrTables : public ModulePass { public: static char ID; @@ -64,6 +65,14 @@ public: /// Checks to see if there is already a table for the given FunctionType. bool hasTable(FunctionType *FunTy); + /// Maps the function into a subset of function types, depending on the + /// jump-instruction table style selected from JumpTableTypes in + /// JumpInstrTables.cpp. The choice of mapping determines the number of + /// jump-instruction tables generated by this pass. E.g., the simplest mapping + /// converts every function type into void f(); so, all functions end up in a + /// single table. + static FunctionType *transformType(JumpTable::JumpTableType JTT, + FunctionType *FunTy); private: /// The metadata used while a jump table is being built struct TableMeta { @@ -76,14 +85,6 @@ private: typedef DenseMap JumpMap; - /// Maps the function into a subset of function types, depending on the - /// jump-instruction table style selected from JumpTableTypes in - /// JumpInstrTables.cpp. The choice of mapping determines the number of - /// jump-instruction tables generated by this pass. E.g., the simplest mapping - /// converts every function type into void f(); so, all functions end up in a - /// single table. - FunctionType *transformType(FunctionType *FunTy); - /// The current state of functions and jump entries in the table(s). JumpMap Metadata; diff --git a/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h b/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h index 036aea30a510..11a360a491a7 100644 --- a/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h +++ b/contrib/llvm/include/llvm/CodeGen/LexicalScopes.h @@ -19,14 +19,14 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" -#include #include +#include namespace llvm { class MachineInstr; @@ -48,6 +48,8 @@ public: LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A) : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A), LastInsn(nullptr), FirstInsn(nullptr), DFSIn(0), DFSOut(0) { + assert((!D || D->isResolved()) && "Expected resolved node"); + assert((!I || I->isResolved()) && "Expected resolved node"); if (Parent) Parent->addChild(this); } @@ -116,8 +118,8 @@ public: private: LexicalScope *Parent; // Parent to this scope. - AssertingVH Desc; // Debug info descriptor. - AssertingVH InlinedAtLocation; // Location at which this + const MDNode *Desc; // Debug info descriptor. + const MDNode *InlinedAtLocation; // Location at which this // scope is inlined. bool AbstractScope; // Abstract Scope SmallVector Children; // Scopes defined in scope. @@ -148,12 +150,6 @@ public: /// empty - Return true if there is any lexical scope information available. bool empty() { return CurrentFnLexicalScope == nullptr; } - /// isCurrentFunctionScope - Return true if given lexical scope represents - /// current function. - bool isCurrentFunctionScope(const LexicalScope *LS) { - return LS == CurrentFnLexicalScope; - } - /// getCurrentFunctionScope - Return lexical scope for the current function. LexicalScope *getCurrentFunctionScope() const { return CurrentFnLexicalScope; @@ -163,7 +159,7 @@ public: /// which have machine instructions that belong to lexical scope identified by /// DebugLoc. void getMachineBasicBlocks(DebugLoc DL, - SmallPtrSet &MBBs); + SmallPtrSetImpl &MBBs); /// dominates - Return true if DebugLoc's lexical scope dominates at least one /// machine instruction's lexical scope in a given machine basic block. diff --git a/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h b/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h index 372c294da306..e7ccbfa617e5 100644 --- a/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/contrib/llvm/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -39,6 +39,7 @@ namespace { llvm::linkOcamlGC(); llvm::linkErlangGC(); llvm::linkShadowStackGC(); + llvm::linkStatepointExampleGC(); (void) llvm::createBURRListDAGScheduler(nullptr, llvm::CodeGenOpt::Default); diff --git a/contrib/llvm/include/llvm/CodeGen/LiveInterval.h b/contrib/llvm/include/llvm/CodeGen/LiveInterval.h index 6629e6046532..ce9845ee1673 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveInterval.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveInterval.h @@ -119,6 +119,12 @@ namespace llvm { return isDeadDef() ? nullptr : LateVal; } + /// Returns the value alive at the end of the instruction, if any. This can + /// be a live-through value, a live def or a dead def. + VNInfo *valueOutOrDead() const { + return LateVal; + } + /// Return the value defined by this instruction, if any. This includes /// dead defs, it is the value created by the instruction's def operands. VNInfo *valueDefined() const { @@ -204,6 +210,23 @@ namespace llvm { const_vni_iterator vni_begin() const { return valnos.begin(); } const_vni_iterator vni_end() const { return valnos.end(); } + /// Constructs a new LiveRange object. + LiveRange() { + } + + /// Constructs a new LiveRange object by copying segments and valnos from + /// another LiveRange. + LiveRange(const LiveRange &Other, BumpPtrAllocator &Allocator) { + // Duplicate valnos. + for (const VNInfo *VNI : Other.valnos) { + createValueCopy(VNI, Allocator); + } + // Now we can copy segments and remap their valnos. + for (const Segment &S : Other.segments) { + segments.push_back(Segment(S.start, S.end, valnos[S.valno->id])); + } + } + /// advanceTo - Advance the specified iterator to point to the Segment /// containing the specified position, or end() if the position is past the /// end of the range. If no Segment contains this position, but the @@ -217,6 +240,14 @@ namespace llvm { return I; } + const_iterator advanceTo(const_iterator I, SlotIndex Pos) const { + assert(I != end()); + if (Pos >= endIndex()) + return end(); + while (I->end <= Pos) ++I; + return I; + } + /// find - Return an iterator pointing to the first segment that ends after /// Pos, or end(). This is the same as advanceTo(begin(), Pos), but faster /// when searching large ranges. @@ -397,6 +428,12 @@ namespace llvm { /// scanning the Other range starting at I. bool overlapsFrom(const LiveRange &Other, const_iterator I) const; + /// Returns true if all segments of the @p Other live range are completely + /// covered by this live range. + /// Adjacent live ranges do not affect the covering:the liverange + /// [1,5](5,10] covers (3,7]. + bool covers(const LiveRange &Other) const; + /// Add the specified Segment to this range, merging segments as /// appropriate. This returns an iterator to the inserted segment (which /// may have grown since it was inserted). @@ -435,6 +472,12 @@ namespace llvm { removeSegment(S.start, S.end, RemoveDeadValNo); } + /// Remove segment pointed to by iterator @p I from this range. This does + /// not remove dead value numbers. + iterator removeSegment(iterator I) { + return segments.erase(I); + } + /// Query Liveness at Idx. /// The sub-instruction slot of Idx doesn't matter, only the instruction /// it refers to is considered. @@ -484,9 +527,9 @@ namespace llvm { /// Returns true if the live range is zero length, i.e. no live segments /// span instructions. It doesn't pay to spill such a range. bool isZeroLength(SlotIndexes *Indexes) const { - for (const_iterator i = begin(), e = end(); i != e; ++i) - if (Indexes->getNextNonNullIndex(i->start).getBaseIndex() < - i->end.getBaseIndex()) + for (const Segment &S : segments) + if (Indexes->getNextNonNullIndex(S.start).getBaseIndex() < + S.end.getBaseIndex()) return false; return true; } @@ -509,6 +552,10 @@ namespace llvm { void verify() const; #endif + protected: + /// Append a segment to the list of segments. + void append(const LiveRange::Segment S); + private: iterator addSegmentFrom(Segment S, iterator From); @@ -529,11 +576,122 @@ namespace llvm { public: typedef LiveRange super; + /// A live range for subregisters. The LaneMask specifies which parts of the + /// super register are covered by the interval. + /// (@sa TargetRegisterInfo::getSubRegIndexLaneMask()). + class SubRange : public LiveRange { + public: + SubRange *Next; + unsigned LaneMask; + + /// Constructs a new SubRange object. + SubRange(unsigned LaneMask) + : Next(nullptr), LaneMask(LaneMask) { + } + + /// Constructs a new SubRange object by copying liveness from @p Other. + SubRange(unsigned LaneMask, const LiveRange &Other, + BumpPtrAllocator &Allocator) + : LiveRange(Other, Allocator), Next(nullptr), LaneMask(LaneMask) { + } + }; + + private: + SubRange *SubRanges; ///< Single linked list of subregister live ranges. + + public: const unsigned reg; // the register or stack slot of this interval. float weight; // weight of this interval LiveInterval(unsigned Reg, float Weight) - : reg(Reg), weight(Weight) {} + : SubRanges(nullptr), reg(Reg), weight(Weight) {} + + template + class SingleLinkedListIterator { + T *P; + public: + SingleLinkedListIterator(T *P) : P(P) {} + SingleLinkedListIterator &operator++() { + P = P->Next; + return *this; + } + SingleLinkedListIterator &operator++(int) { + SingleLinkedListIterator res = *this; + ++*this; + return res; + } + bool operator!=(const SingleLinkedListIterator &Other) { + return P != Other.operator->(); + } + bool operator==(const SingleLinkedListIterator &Other) { + return P == Other.operator->(); + } + T &operator*() const { + return *P; + } + T *operator->() const { + return P; + } + }; + + typedef SingleLinkedListIterator subrange_iterator; + subrange_iterator subrange_begin() { + return subrange_iterator(SubRanges); + } + subrange_iterator subrange_end() { + return subrange_iterator(nullptr); + } + + typedef SingleLinkedListIterator const_subrange_iterator; + const_subrange_iterator subrange_begin() const { + return const_subrange_iterator(SubRanges); + } + const_subrange_iterator subrange_end() const { + return const_subrange_iterator(nullptr); + } + + iterator_range subranges() { + return make_range(subrange_begin(), subrange_end()); + } + + iterator_range subranges() const { + return make_range(subrange_begin(), subrange_end()); + } + + /// Creates a new empty subregister live range. The range is added at the + /// beginning of the subrange list; subrange iterators stay valid. + SubRange *createSubRange(BumpPtrAllocator &Allocator, unsigned LaneMask) { + SubRange *Range = new (Allocator) SubRange(LaneMask); + appendSubRange(Range); + return Range; + } + + /// Like createSubRange() but the new range is filled with a copy of the + /// liveness information in @p CopyFrom. + SubRange *createSubRangeFrom(BumpPtrAllocator &Allocator, unsigned LaneMask, + const LiveRange &CopyFrom) { + SubRange *Range = new (Allocator) SubRange(LaneMask, CopyFrom, Allocator); + appendSubRange(Range); + return Range; + } + + /// Returns true if subregister liveness information is available. + bool hasSubRanges() const { + return SubRanges != nullptr; + } + + /// Removes all subregister liveness information. + void clearSubRanges() { + SubRanges = nullptr; + } + + /// Removes all subranges without any segments (subranges without segments + /// are not considered valid and should only exist temporarily). + void removeEmptySubRanges(); + + /// Construct main live range by merging the SubRanges of @p LI. + void constructMainRangeFromSubranges(const SlotIndexes &Indexes, + VNInfo::Allocator &VNIAllocator); /// getSize - Returns the sum of sizes of all the LiveRange's. /// @@ -558,9 +716,23 @@ namespace llvm { void print(raw_ostream &OS) const; void dump() const; + /// \brief Walks the interval and assert if any invariants fail to hold. + /// + /// Note that this is a no-op when asserts are disabled. +#ifdef NDEBUG + void verify(const MachineRegisterInfo *MRI = nullptr) const {} +#else + void verify(const MachineRegisterInfo *MRI = nullptr) const; +#endif + private: LiveInterval& operator=(const LiveInterval& rhs) LLVM_DELETED_FUNCTION; + /// Appends @p Range to SubRanges list. + void appendSubRange(SubRange *Range) { + Range->Next = SubRanges; + SubRanges = Range; + } }; inline raw_ostream &operator<<(raw_ostream &OS, const LiveInterval &LI) { diff --git a/contrib/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h b/contrib/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h index 176665bc2566..d8c921fce313 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h @@ -17,8 +17,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_LIVEINTERVAL_ANALYSIS_H -#define LLVM_CODEGEN_LIVEINTERVAL_ANALYSIS_H +#ifndef LLVM_CODEGEN_LIVEINTERVALANALYSIS_H +#define LLVM_CODEGEN_LIVEINTERVALANALYSIS_H #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" @@ -50,7 +50,6 @@ namespace llvm { class LiveIntervals : public MachineFunctionPass { MachineFunction* MF; MachineRegisterInfo* MRI; - const TargetMachine* TM; const TargetRegisterInfo* TRI; const TargetInstrInfo* TII; AliasAnalysis *AA; @@ -155,16 +154,11 @@ namespace llvm { bool shrinkToUses(LiveInterval *li, SmallVectorImpl *dead = nullptr); - /// \brief Walk the values in the given interval and compute which ones - /// are dead. Dead values are not deleted, however: - /// - Dead PHIDef values are marked as unused. - /// - New dead machine instructions are added to the dead vector. - /// - CanSeparate is set to true if the interval may have been separated - /// into multiple connected components. - void computeDeadValues(LiveInterval *li, - LiveRange &LR, - bool *CanSeparate, - SmallVectorImpl *dead); + /// Specialized version of + /// shrinkToUses(LiveInterval *li, SmallVectorImpl *dead) + /// that works on a subregister live range and only looks at uses matching + /// the lane mask of the subregister range. + void shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg); /// extendToIndices - Extend the live range of LI to reach all points in /// Indices. The points in the Indices array must be jointly dominated by @@ -176,14 +170,21 @@ namespace llvm { /// See also LiveRangeCalc::extend(). void extendToIndices(LiveRange &LR, ArrayRef Indices); - /// pruneValue - If an LI value is live at Kill, prune its live range by - /// removing any liveness reachable from Kill. Add live range end points to + + /// If @p LR has a live value at @p Kill, prune its live range by removing + /// any liveness reachable from Kill. Add live range end points to /// EndPoints such that extendToIndices(LI, EndPoints) will reconstruct the /// value's live range. /// /// Calling pruneValue() and extendToIndices() can be used to reconstruct /// SSA form after adding defs to a virtual register. - void pruneValue(LiveInterval *LI, SlotIndex Kill, + void pruneValue(LiveRange &LR, SlotIndex Kill, + SmallVectorImpl *EndPoints); + + /// Subregister aware variant of pruneValue(LiveRange &LR, SlotIndex Kill, + /// SmallVectorImpl &EndPoints). Prunes the value in the main + /// range and all sub ranges. + void pruneValue(LiveInterval &LI, SlotIndex Kill, SmallVectorImpl *EndPoints); SlotIndexes *getSlotIndexes() const { @@ -405,6 +406,16 @@ namespace llvm { /// Compute RegMaskSlots and RegMaskBits. void computeRegMasks(); + /// Walk the values in @p LI and check for dead values: + /// - Dead PHIDef values are marked as unused. + /// - Dead operands are marked as such. + /// - Completely dead machine instructions are added to the @p dead vector + /// if it is not nullptr. + /// Returns true if any PHI value numbers have been removed which may + /// have separated the interval into multiple connected components. + bool computeDeadValues(LiveInterval &LI, + SmallVectorImpl *dead); + static LiveInterval* createInterval(unsigned Reg); void printInstrs(raw_ostream &O) const; @@ -414,6 +425,16 @@ namespace llvm { void computeRegUnitRange(LiveRange&, unsigned Unit); void computeVirtRegInterval(LiveInterval&); + + /// Helper function for repairIntervalsInRange(), walks backwards and + /// creates/modifies live segments in @p LR to match the operands found. + /// Only full operands or operands with subregisters matching @p LaneMask + /// are considered. + void repairOldRegInRange(MachineBasicBlock::iterator Begin, + MachineBasicBlock::iterator End, + const SlotIndex endIdx, LiveRange &LR, + unsigned Reg, unsigned LaneMask = ~0u); + class HMEditor; }; } // End llvm namespace diff --git a/contrib/llvm/include/llvm/CodeGen/LiveIntervalUnion.h b/contrib/llvm/include/llvm/CodeGen/LiveIntervalUnion.h index 2f40509a1111..1381c46a2750 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveIntervalUnion.h @@ -84,10 +84,16 @@ public: bool changedSince(unsigned tag) const { return tag != Tag; } // Add a live virtual register to this union and merge its segments. - void unify(LiveInterval &VirtReg); + void unify(LiveInterval &VirtReg, const LiveRange &Range); + void unify(LiveInterval &VirtReg) { + unify(VirtReg, VirtReg); + } // Remove a live virtual register's segments from this union. - void extract(LiveInterval &VirtReg); + void extract(LiveInterval &VirtReg, const LiveRange &Range); + void extract(LiveInterval &VirtReg) { + extract(VirtReg, VirtReg); + } // Remove all inserted virtual registers. void clear() { Segments.clear(); ++Tag; } diff --git a/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h b/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h index 847092b1d824..91e4ddcde170 100644 --- a/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -26,8 +26,8 @@ // %XMM0 = ..., %YMM0 (%YMM0 and all its sub-registers are alive) //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_LIVE_PHYS_REGS_H -#define LLVM_CODEGEN_LIVE_PHYS_REGS_H +#ifndef LLVM_CODEGEN_LIVEPHYSREGS_H +#define LLVM_CODEGEN_LIVEPHYSREGS_H #include "llvm/ADT/SparseSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -143,4 +143,4 @@ inline raw_ostream &operator<<(raw_ostream &OS, const LivePhysRegs& LR) { } // namespace llvm -#endif // LLVM_CODEGEN_LIVE_PHYS_REGS_H +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h b/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h index 5767cab1a4db..44c3c4eaf7b1 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h @@ -24,6 +24,7 @@ #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetSubtargetInfo.h" namespace llvm { @@ -111,18 +112,15 @@ public: /// @param vrm Map of virtual registers to physical registers for this /// function. If NULL, no virtual register map updates will /// be done. This could be the case if called before Regalloc. - LiveRangeEdit(LiveInterval *parent, - SmallVectorImpl &newRegs, - MachineFunction &MF, - LiveIntervals &lis, - VirtRegMap *vrm, + LiveRangeEdit(LiveInterval *parent, SmallVectorImpl &newRegs, + MachineFunction &MF, LiveIntervals &lis, VirtRegMap *vrm, Delegate *delegate = nullptr) - : Parent(parent), NewRegs(newRegs), - MRI(MF.getRegInfo()), LIS(lis), VRM(vrm), - TII(*MF.getTarget().getInstrInfo()), - TheDelegate(delegate), - FirstNew(newRegs.size()), - ScannedRemattable(false) { MRI.setDelegate(this); } + : Parent(parent), NewRegs(newRegs), MRI(MF.getRegInfo()), LIS(lis), + VRM(vrm), TII(*MF.getSubtarget().getInstrInfo()), + TheDelegate(delegate), FirstNew(newRegs.size()), + ScannedRemattable(false) { + MRI.setDelegate(this); + } ~LiveRangeEdit() { MRI.resetDelegate(this); } diff --git a/contrib/llvm/include/llvm/CodeGen/LiveVariables.h b/contrib/llvm/include/llvm/CodeGen/LiveVariables.h index a4a5fcc31e12..55b97dc3e71d 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveVariables.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveVariables.h @@ -134,14 +134,14 @@ private: // Intermediate data structures // PhysRegInfo - Keep track of which instruction was the last def of a // physical register. This is a purely local property, because all physical // register references are presumed dead across basic blocks. - MachineInstr **PhysRegDef; + std::vector PhysRegDef; // PhysRegInfo - Keep track of which instruction was the last use of a // physical register. This is a purely local property, because all physical // register references are presumed dead across basic blocks. - MachineInstr **PhysRegUse; + std::vector PhysRegUse; - SmallVector *PHIVarInfo; + std::vector> PHIVarInfo; // DistanceMap - Keep track the distance of a MI from the start of the // current basic block. @@ -175,6 +175,10 @@ private: // Intermediate data structures /// register which is used in a PHI node. We map that to the BB the vreg /// is coming from. void analyzePHINodes(const MachineFunction& Fn); + + void runOnInstr(MachineInstr *MI, SmallVectorImpl &Defs); + + void runOnBlock(MachineBasicBlock *MBB, unsigned NumRegs); public: bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h index a08cc2eb508a..1440b967aea2 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -486,11 +486,15 @@ public: /// Insert a range of instructions into the instruction list before I. template void insert(iterator I, IT S, IT E) { + assert((I == end() || I->getParent() == this) && + "iterator points outside of basic block"); Insts.insert(I.getInstrIterator(), S, E); } /// Insert MI into the instruction list before I. iterator insert(iterator I, MachineInstr *MI) { + assert((I == end() || I->getParent() == this) && + "iterator points outside of basic block"); assert(!MI->isBundledWithPred() && !MI->isBundledWithSucc() && "Cannot insert instruction with bundle flags"); return Insts.insert(I.getInstrIterator(), MI); @@ -498,6 +502,8 @@ public: /// Insert MI into the instruction list after I. iterator insertAfter(iterator I, MachineInstr *MI) { + assert((I == end() || I->getParent() == this) && + "iterator points outside of basic block"); assert(!MI->isBundledWithPred() && !MI->isBundledWithSucc() && "Cannot insert instruction with bundle flags"); return Insts.insertAfter(I.getInstrIterator(), MI); diff --git a/contrib/llvm/include/llvm/CodeGen/MachineCodeEmitter.h b/contrib/llvm/include/llvm/CodeGen/MachineCodeEmitter.h deleted file mode 100644 index 81b0ba1e7c71..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/MachineCodeEmitter.h +++ /dev/null @@ -1,334 +0,0 @@ -//===-- llvm/CodeGen/MachineCodeEmitter.h - Code emission -------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines an abstract interface that is used by the machine code -// emission framework to output the code. This allows machine code emission to -// be separated from concerns such as resolution of call targets, and where the -// machine code will be written (memory or disk, f.e.). -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_MACHINECODEEMITTER_H -#define LLVM_CODEGEN_MACHINECODEEMITTER_H - -#include "llvm/IR/DebugLoc.h" -#include "llvm/Support/DataTypes.h" -#include - -namespace llvm { - -class MachineBasicBlock; -class MachineConstantPool; -class MachineJumpTableInfo; -class MachineFunction; -class MachineModuleInfo; -class MachineRelocation; -class Value; -class GlobalValue; -class Function; -class MCSymbol; - -/// MachineCodeEmitter - This class defines two sorts of methods: those for -/// emitting the actual bytes of machine code, and those for emitting auxiliary -/// structures, such as jump tables, relocations, etc. -/// -/// Emission of machine code is complicated by the fact that we don't (in -/// general) know the size of the machine code that we're about to emit before -/// we emit it. As such, we preallocate a certain amount of memory, and set the -/// BufferBegin/BufferEnd pointers to the start and end of the buffer. As we -/// emit machine instructions, we advance the CurBufferPtr to indicate the -/// location of the next byte to emit. In the case of a buffer overflow (we -/// need to emit more machine code than we have allocated space for), the -/// CurBufferPtr will saturate to BufferEnd and ignore stores. Once the entire -/// function has been emitted, the overflow condition is checked, and if it has -/// occurred, more memory is allocated, and we reemit the code into it. -/// -class MachineCodeEmitter { - virtual void anchor(); -protected: - /// BufferBegin/BufferEnd - Pointers to the start and end of the memory - /// allocated for this code buffer. - uint8_t *BufferBegin, *BufferEnd; - /// CurBufferPtr - Pointer to the next byte of memory to fill when emitting - /// code. This is guaranteed to be in the range [BufferBegin,BufferEnd]. If - /// this pointer is at BufferEnd, it will never move due to code emission, and - /// all code emission requests will be ignored (this is the buffer overflow - /// condition). - uint8_t *CurBufferPtr; - -public: - virtual ~MachineCodeEmitter() {} - - /// startFunction - This callback is invoked when the specified function is - /// about to be code generated. This initializes the BufferBegin/End/Ptr - /// fields. - /// - virtual void startFunction(MachineFunction &F) = 0; - - /// finishFunction - This callback is invoked when the specified function has - /// finished code generation. If a buffer overflow has occurred, this method - /// returns true (the callee is required to try again), otherwise it returns - /// false. - /// - virtual bool finishFunction(MachineFunction &F) = 0; - - /// emitByte - This callback is invoked when a byte needs to be written to the - /// output stream. - /// - void emitByte(uint8_t B) { - if (CurBufferPtr != BufferEnd) - *CurBufferPtr++ = B; - } - - /// emitWordLE - This callback is invoked when a 32-bit word needs to be - /// written to the output stream in little-endian format. - /// - void emitWordLE(uint32_t W) { - if (4 <= BufferEnd-CurBufferPtr) { - emitWordLEInto(CurBufferPtr, W); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitWordLEInto - This callback is invoked when a 32-bit word needs to be - /// written to an arbitrary buffer in little-endian format. Buf must have at - /// least 4 bytes of available space. - /// - static void emitWordLEInto(uint8_t *&Buf, uint32_t W) { - *Buf++ = (uint8_t)(W >> 0); - *Buf++ = (uint8_t)(W >> 8); - *Buf++ = (uint8_t)(W >> 16); - *Buf++ = (uint8_t)(W >> 24); - } - - /// emitWordBE - This callback is invoked when a 32-bit word needs to be - /// written to the output stream in big-endian format. - /// - void emitWordBE(uint32_t W) { - if (4 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 24); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 0); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitDWordLE - This callback is invoked when a 64-bit word needs to be - /// written to the output stream in little-endian format. - /// - void emitDWordLE(uint64_t W) { - if (8 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 0); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 24); - *CurBufferPtr++ = (uint8_t)(W >> 32); - *CurBufferPtr++ = (uint8_t)(W >> 40); - *CurBufferPtr++ = (uint8_t)(W >> 48); - *CurBufferPtr++ = (uint8_t)(W >> 56); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitDWordBE - This callback is invoked when a 64-bit word needs to be - /// written to the output stream in big-endian format. - /// - void emitDWordBE(uint64_t W) { - if (8 <= BufferEnd-CurBufferPtr) { - *CurBufferPtr++ = (uint8_t)(W >> 56); - *CurBufferPtr++ = (uint8_t)(W >> 48); - *CurBufferPtr++ = (uint8_t)(W >> 40); - *CurBufferPtr++ = (uint8_t)(W >> 32); - *CurBufferPtr++ = (uint8_t)(W >> 24); - *CurBufferPtr++ = (uint8_t)(W >> 16); - *CurBufferPtr++ = (uint8_t)(W >> 8); - *CurBufferPtr++ = (uint8_t)(W >> 0); - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitAlignment - Move the CurBufferPtr pointer up to the specified - /// alignment (saturated to BufferEnd of course). - void emitAlignment(unsigned Alignment) { - if (Alignment == 0) Alignment = 1; - - if(Alignment <= (uintptr_t)(BufferEnd-CurBufferPtr)) { - // Move the current buffer ptr up to the specified alignment. - CurBufferPtr = - (uint8_t*)(((uintptr_t)CurBufferPtr+Alignment-1) & - ~(uintptr_t)(Alignment-1)); - } else { - CurBufferPtr = BufferEnd; - } - } - - - /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be - /// written to the output stream. - void emitULEB128Bytes(uint64_t Value) { - do { - uint8_t Byte = Value & 0x7f; - Value >>= 7; - if (Value) Byte |= 0x80; - emitByte(Byte); - } while (Value); - } - - /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be - /// written to the output stream. - void emitSLEB128Bytes(uint64_t Value) { - uint64_t Sign = Value >> (8 * sizeof(Value) - 1); - bool IsMore; - - do { - uint8_t Byte = Value & 0x7f; - Value >>= 7; - IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0; - if (IsMore) Byte |= 0x80; - emitByte(Byte); - } while (IsMore); - } - - /// emitString - This callback is invoked when a String needs to be - /// written to the output stream. - void emitString(const std::string &String) { - for (unsigned i = 0, N = static_cast(String.size()); - i < N; ++i) { - uint8_t C = String[i]; - emitByte(C); - } - emitByte(0); - } - - /// emitInt32 - Emit a int32 directive. - void emitInt32(int32_t Value) { - if (4 <= BufferEnd-CurBufferPtr) { - *((uint32_t*)CurBufferPtr) = Value; - CurBufferPtr += 4; - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitInt64 - Emit a int64 directive. - void emitInt64(uint64_t Value) { - if (8 <= BufferEnd-CurBufferPtr) { - *((uint64_t*)CurBufferPtr) = Value; - CurBufferPtr += 8; - } else { - CurBufferPtr = BufferEnd; - } - } - - /// emitInt32At - Emit the Int32 Value in Addr. - void emitInt32At(uintptr_t *Addr, uintptr_t Value) { - if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd) - (*(uint32_t*)Addr) = (uint32_t)Value; - } - - /// emitInt64At - Emit the Int64 Value in Addr. - void emitInt64At(uintptr_t *Addr, uintptr_t Value) { - if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd) - (*(uint64_t*)Addr) = (uint64_t)Value; - } - - /// processDebugLoc - Records debug location information about a - /// MachineInstruction. This is called before emitting any bytes associated - /// with the instruction. Even if successive instructions have the same debug - /// location, this method will be called for each one. - virtual void processDebugLoc(DebugLoc DL, bool BeforePrintintInsn) {} - - /// emitLabel - Emits a label - virtual void emitLabel(MCSymbol *Label) = 0; - - /// allocateSpace - Allocate a block of space in the current output buffer, - /// returning null (and setting conditions to indicate buffer overflow) on - /// failure. Alignment is the alignment in bytes of the buffer desired. - virtual void *allocateSpace(uintptr_t Size, unsigned Alignment) { - emitAlignment(Alignment); - void *Result; - - // Check for buffer overflow. - if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) { - CurBufferPtr = BufferEnd; - Result = nullptr; - } else { - // Allocate the space. - Result = CurBufferPtr; - CurBufferPtr += Size; - } - - return Result; - } - - /// StartMachineBasicBlock - This should be called by the target when a new - /// basic block is about to be emitted. This way the MCE knows where the - /// start of the block is, and can implement getMachineBasicBlockAddress. - virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) = 0; - - /// getCurrentPCValue - This returns the address that the next emitted byte - /// will be output to. - /// - virtual uintptr_t getCurrentPCValue() const { - return (uintptr_t)CurBufferPtr; - } - - /// getCurrentPCOffset - Return the offset from the start of the emitted - /// buffer that we are currently writing to. - virtual uintptr_t getCurrentPCOffset() const { - return CurBufferPtr-BufferBegin; - } - - /// earlyResolveAddresses - True if the code emitter can use symbol addresses - /// during code emission time. The JIT is capable of doing this because it - /// creates jump tables or constant pools in memory on the fly while the - /// object code emitters rely on a linker to have real addresses and should - /// use relocations instead. - virtual bool earlyResolveAddresses() const = 0; - - /// addRelocation - Whenever a relocatable address is needed, it should be - /// noted with this interface. - virtual void addRelocation(const MachineRelocation &MR) = 0; - - /// FIXME: These should all be handled with relocations! - - /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in - /// the constant pool that was last emitted with the emitConstantPool method. - /// - virtual uintptr_t getConstantPoolEntryAddress(unsigned Index) const = 0; - - /// getJumpTableEntryAddress - Return the address of the jump table with index - /// 'Index' in the function that last called initJumpTableInfo. - /// - virtual uintptr_t getJumpTableEntryAddress(unsigned Index) const = 0; - - /// getMachineBasicBlockAddress - Return the address of the specified - /// MachineBasicBlock, only usable after the label for the MBB has been - /// emitted. - /// - virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const= 0; - - /// getLabelAddress - Return the address of the specified Label, only usable - /// after the LabelID has been emitted. - /// - virtual uintptr_t getLabelAddress(MCSymbol *Label) const = 0; - - /// Specifies the MachineModuleInfo object. This is used for exception handling - /// purposes. - virtual void setModuleInfo(MachineModuleInfo* Info) = 0; -}; - -} // End llvm namespace - -#endif diff --git a/contrib/llvm/include/llvm/CodeGen/MachineCodeInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineCodeInfo.h deleted file mode 100644 index 820bc87425b9..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/MachineCodeInfo.h +++ /dev/null @@ -1,53 +0,0 @@ -//===-- MachineCodeInfo.h - Class used to report JIT info -------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines MachineCodeInfo, a class used by the JIT ExecutionEngine -// to report information about the generated machine code. -// -// See JIT::runJITOnFunction for usage. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_MACHINECODEINFO_H -#define LLVM_CODEGEN_MACHINECODEINFO_H - -#include "llvm/Support/DataTypes.h" - -namespace llvm { - -class MachineCodeInfo { -private: - size_t Size; // Number of bytes in memory used - void *Address; // The address of the function in memory - -public: - MachineCodeInfo() : Size(0), Address(nullptr) {} - - void setSize(size_t s) { - Size = s; - } - - void setAddress(void *a) { - Address = a; - } - - size_t size() const { - return Size; - } - - void *address() const { - return Address; - } - -}; - -} - -#endif - diff --git a/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h new file mode 100644 index 000000000000..176af14dc317 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h @@ -0,0 +1,29 @@ +//===-- llvm/CodeGen/MachineCombinerPattern.h - Instruction pattern supported by +// combiner ------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines instruction pattern supported by combiner +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINECOMBINERPATTERN_H +#define LLVM_CODEGEN_MACHINECOMBINERPATTERN_H + +namespace llvm { + +/// Enumeration of instruction pattern supported by machine combiner +/// +/// +namespace MachineCombinerPattern { +// Forward declaration +enum MC_PATTERN : int; +} // end namespace MachineCombinerPattern +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/MachineDominators.h b/contrib/llvm/include/llvm/CodeGen/MachineDominators.h index f1ae0bf5f9cf..a6980a6daeac 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineDominators.h @@ -15,6 +15,7 @@ #ifndef LLVM_CODEGEN_MACHINEDOMINATORS_H #define LLVM_CODEGEN_MACHINEDOMINATORS_H +#include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -38,6 +39,103 @@ typedef DomTreeNodeBase MachineDomTreeNode; /// compute a normal dominator tree. /// class MachineDominatorTree : public MachineFunctionPass { + /// \brief Helper structure used to hold all the basic blocks + /// involved in the split of a critical edge. + struct CriticalEdge { + MachineBasicBlock *FromBB; + MachineBasicBlock *ToBB; + MachineBasicBlock *NewBB; + CriticalEdge(MachineBasicBlock *FromBB, MachineBasicBlock *ToBB, + MachineBasicBlock *NewBB) + : FromBB(FromBB), ToBB(ToBB), NewBB(NewBB) {} + }; + + /// \brief Pile up all the critical edges to be split. + /// The splitting of a critical edge is local and thus, it is possible + /// to apply several of those changes at the same time. + mutable SmallVector CriticalEdgesToSplit; + /// \brief Remember all the basic blocks that are inserted during + /// edge splitting. + /// Invariant: NewBBs == all the basic blocks contained in the NewBB + /// field of all the elements of CriticalEdgesToSplit. + /// I.e., forall elt in CriticalEdgesToSplit, it exists BB in NewBBs + /// such as BB == elt.NewBB. + mutable SmallSet NewBBs; + + /// \brief Apply all the recorded critical edges to the DT. + /// This updates the underlying DT information in a way that uses + /// the fast query path of DT as much as possible. + /// + /// \post CriticalEdgesToSplit.empty(). + void applySplitCriticalEdges() const { + // Bail out early if there is nothing to do. + if (CriticalEdgesToSplit.empty()) + return; + + // For each element in CriticalEdgesToSplit, remember whether or + // not element is the new immediate domminator of its successor. + // The mapping is done by index, i.e., the information for the ith + // element of CriticalEdgesToSplit is the ith element of IsNewIDom. + SmallVector IsNewIDom; + IsNewIDom.resize(CriticalEdgesToSplit.size()); + size_t Idx = 0; + + // Collect all the dominance properties info, before invalidating + // the underlying DT. + for (CriticalEdge &Edge : CriticalEdgesToSplit) { + // Update dominator information. + MachineBasicBlock *Succ = Edge.ToBB; + MachineDomTreeNode *SucccDTNode = DT->getNode(Succ); + + IsNewIDom[Idx] = true; + for (MachineBasicBlock *PredBB : Succ->predecessors()) { + if (PredBB == Edge.NewBB) + continue; + // If we are in this situation: + // FromBB1 FromBB2 + // + + + // + + + + + // + + + + + // ... Split1 Split2 ... + // + + + // + + + // + + // Succ + // Instead of checking the domiance property with Split2, we + // check it with FromBB2 since Split2 is still unknown of the + // underlying DT structure. + if (NewBBs.count(PredBB)) { + assert(PredBB->pred_size() == 1 && "A basic block resulting from a " + "critical edge split has more " + "than one predecessor!"); + PredBB = *PredBB->pred_begin(); + } + if (!DT->dominates(SucccDTNode, DT->getNode(PredBB))) { + IsNewIDom[Idx] = false; + break; + } + } + ++Idx; + } + + // Now, update DT with the collected dominance properties info. + Idx = 0; + for (CriticalEdge &Edge : CriticalEdgesToSplit) { + // We know FromBB dominates NewBB. + MachineDomTreeNode *NewDTNode = DT->addNewBlock(Edge.NewBB, Edge.FromBB); + MachineDomTreeNode *SucccDTNode = DT->getNode(Edge.ToBB); + + // If all the other predecessors of "Succ" are dominated by "Succ" itself + // then the new block is the new immediate dominator of "Succ". Otherwise, + // the new block doesn't dominate anything. + if (IsNewIDom[Idx]) + DT->changeImmediateDominator(SucccDTNode, NewDTNode); + ++Idx; + } + NewBBs.clear(); + CriticalEdgesToSplit.clear(); + } + public: static char ID; // Pass ID, replacement for typeid DominatorTreeBase* DT; @@ -46,7 +144,10 @@ public: ~MachineDominatorTree(); - DominatorTreeBase& getBase() { return *DT; } + DominatorTreeBase &getBase() { + applySplitCriticalEdges(); + return *DT; + } void getAnalysisUsage(AnalysisUsage &AU) const override; @@ -55,14 +156,17 @@ public: /// dominators, this will always be a single block (the entry node). /// inline const std::vector &getRoots() const { + applySplitCriticalEdges(); return DT->getRoots(); } inline MachineBasicBlock *getRoot() const { + applySplitCriticalEdges(); return DT->getRoot(); } inline MachineDomTreeNode *getRootNode() const { + applySplitCriticalEdges(); return DT->getRootNode(); } @@ -70,17 +174,20 @@ public: inline bool dominates(const MachineDomTreeNode* A, const MachineDomTreeNode* B) const { + applySplitCriticalEdges(); return DT->dominates(A, B); } inline bool dominates(const MachineBasicBlock* A, const MachineBasicBlock* B) const { + applySplitCriticalEdges(); return DT->dominates(A, B); } // dominates - Return true if A dominates B. This performs the // special checks necessary if A and B are in the same basic block. bool dominates(const MachineInstr *A, const MachineInstr *B) const { + applySplitCriticalEdges(); const MachineBasicBlock *BBA = A->getParent(), *BBB = B->getParent(); if (BBA != BBB) return DT->dominates(BBA, BBB); @@ -100,11 +207,13 @@ public: inline bool properlyDominates(const MachineDomTreeNode* A, const MachineDomTreeNode* B) const { + applySplitCriticalEdges(); return DT->properlyDominates(A, B); } inline bool properlyDominates(const MachineBasicBlock* A, const MachineBasicBlock* B) const { + applySplitCriticalEdges(); return DT->properlyDominates(A, B); } @@ -112,10 +221,12 @@ public: /// for basic block A and B. If there is no such block then return NULL. inline MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B) { + applySplitCriticalEdges(); return DT->findNearestCommonDominator(A, B); } inline MachineDomTreeNode *operator[](MachineBasicBlock *BB) const { + applySplitCriticalEdges(); return DT->getNode(BB); } @@ -123,6 +234,7 @@ public: /// block. This is the same as using operator[] on this class. /// inline MachineDomTreeNode *getNode(MachineBasicBlock *BB) const { + applySplitCriticalEdges(); return DT->getNode(BB); } @@ -131,6 +243,7 @@ public: /// the children list of the immediate dominator. inline MachineDomTreeNode *addNewBlock(MachineBasicBlock *BB, MachineBasicBlock *DomBB) { + applySplitCriticalEdges(); return DT->addNewBlock(BB, DomBB); } @@ -139,11 +252,13 @@ public: /// inline void changeImmediateDominator(MachineBasicBlock *N, MachineBasicBlock* NewIDom) { + applySplitCriticalEdges(); DT->changeImmediateDominator(N, NewIDom); } inline void changeImmediateDominator(MachineDomTreeNode *N, MachineDomTreeNode* NewIDom) { + applySplitCriticalEdges(); DT->changeImmediateDominator(N, NewIDom); } @@ -151,24 +266,49 @@ public: /// dominate any other blocks. Removes node from its immediate dominator's /// children list. Deletes dominator node associated with basic block BB. inline void eraseNode(MachineBasicBlock *BB) { + applySplitCriticalEdges(); DT->eraseNode(BB); } /// splitBlock - BB is split and now it has one successor. Update dominator /// tree to reflect this change. inline void splitBlock(MachineBasicBlock* NewBB) { + applySplitCriticalEdges(); DT->splitBlock(NewBB); } /// isReachableFromEntry - Return true if A is dominated by the entry /// block of the function containing it. bool isReachableFromEntry(const MachineBasicBlock *A) { + applySplitCriticalEdges(); return DT->isReachableFromEntry(A); } void releaseMemory() override; void print(raw_ostream &OS, const Module*) const override; + + /// \brief Record that the critical edge (FromBB, ToBB) has been + /// split with NewBB. + /// This is best to use this method instead of directly update the + /// underlying information, because this helps mitigating the + /// number of time the DT information is invalidated. + /// + /// \note Do not use this method with regular edges. + /// + /// \note To benefit from the compile time improvement incurred by this + /// method, the users of this method have to limit the queries to the DT + /// interface between two edges splitting. In other words, they have to + /// pack the splitting of critical edges as much as possible. + void recordSplitCriticalEdge(MachineBasicBlock *FromBB, + MachineBasicBlock *ToBB, + MachineBasicBlock *NewBB) { + bool Inserted = NewBBs.insert(NewBB).second; + (void)Inserted; + assert(Inserted && + "A basic block inserted via edge splitting cannot appear twice"); + CriticalEdgesToSplit.push_back(CriticalEdge(FromBB, ToBB, NewBB)); + } }; //===------------------------------------- diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h index c51f8fe03bbf..667736021f92 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -109,13 +109,23 @@ class MachineFrameInfo { // block and doesn't need additional handling for allocation beyond that. bool PreAllocated; + // If true, an LLVM IR value might point to this object. + // Normally, spill slots and fixed-offset objects don't alias IR-accessible + // objects, but there are exceptions (on PowerPC, for example, some byval + // arguments have ABI-prescribed offsets). + bool isAliased; + StackObject(uint64_t Sz, unsigned Al, int64_t SP, bool IM, - bool isSS, const AllocaInst *Val) + bool isSS, const AllocaInst *Val, bool A) : SPOffset(SP), Size(Sz), Alignment(Al), isImmutable(IM), - isSpillSlot(isSS), Alloca(Val), PreAllocated(false) {} + isSpillSlot(isSS), Alloca(Val), PreAllocated(false), isAliased(A) {} }; - const TargetMachine &TM; + /// StackAlignment - The alignment of the stack. + unsigned StackAlignment; + + /// StackRealignable - Can the stack be realigned. + bool StackRealignable; /// Objects - The list of stack objects allocated... /// @@ -230,10 +240,17 @@ class MachineFrameInfo { /// pointer. bool HasInlineAsmWithSPAdjust; - const TargetFrameLowering *getFrameLowering() const; + /// True if the function contains a call to the llvm.vastart intrinsic. + bool HasVAStart; + + /// True if this is a varargs function that contains a musttail call. + bool HasMustTailInVarArgFunc; + public: - explicit MachineFrameInfo(const TargetMachine &TM, bool RealignOpt) - : TM(TM), RealignOption(RealignOpt) { + explicit MachineFrameInfo(unsigned StackAlign, bool isStackRealign, + bool RealignOpt) + : StackAlignment(StackAlign), StackRealignable(isStackRealign), + RealignOption(RealignOpt) { StackSize = NumFixedObjects = OffsetAdjustment = MaxAlignment = 0; HasVarSizedObjects = false; FrameAddressTaken = false; @@ -250,6 +267,8 @@ public: LocalFrameMaxAlign = 0; UseLocalStackAllocationBlock = false; HasInlineAsmWithSPAdjust = false; + HasVAStart = false; + HasMustTailInVarArgFunc = false; } /// hasStackObjects - Return true if there are any stack objects in this @@ -469,6 +488,14 @@ public: bool hasInlineAsmWithSPAdjust() const { return HasInlineAsmWithSPAdjust; } void setHasInlineAsmWithSPAdjust(bool B) { HasInlineAsmWithSPAdjust = B; } + /// Returns true if the function calls the llvm.va_start intrinsic. + bool hasVAStart() const { return HasVAStart; } + void setHasVAStart(bool B) { HasVAStart = B; } + + /// Returns true if the function is variadic and contains a musttail call. + bool hasMustTailInVarArgFunc() const { return HasMustTailInVarArgFunc; } + void setHasMustTailInVarArgFunc(bool B) { HasMustTailInVarArgFunc = B; } + /// getMaxCallFrameSize - Return the maximum size of a call frame that must be /// allocated for an outgoing function call. This is only available if /// CallFrameSetup/Destroy pseudo instructions are used by the target, and @@ -479,21 +506,34 @@ public: /// CreateFixedObject - Create a new object at a fixed location on the stack. /// All fixed objects should be created before other objects are created for - /// efficiency. By default, fixed objects are immutable. This returns an - /// index with a negative value. + /// efficiency. By default, fixed objects are not pointed to by LLVM IR + /// values. This returns an index with a negative value. /// - int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool Immutable); + int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool Immutable, + bool isAliased = false); /// CreateFixedSpillStackObject - Create a spill slot at a fixed location /// on the stack. Returns an index with a negative value. int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset); + /// Allocates memory at a fixed, target-specific offset from the frame + /// pointer. Marks the function as having its frame address taken. + int CreateFrameAllocation(uint64_t Size); + /// isFixedObjectIndex - Returns true if the specified index corresponds to a /// fixed stack object. bool isFixedObjectIndex(int ObjectIdx) const { return ObjectIdx < 0 && (ObjectIdx >= -(int)NumFixedObjects); } + /// isAliasedObjectIndex - Returns true if the specified index corresponds + /// to an object that might be pointed to by an LLVM IR value. + bool isAliasedObjectIndex(int ObjectIdx) const { + assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && + "Invalid Object Idx!"); + return Objects[ObjectIdx+NumFixedObjects].isAliased; + } + /// isImmutableObjectIndex - Returns true if the specified index corresponds /// to an immutable object. bool isImmutableObjectIndex(int ObjectIdx) const { diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h index 042c62b4a887..4e9ff9ebb4fe 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h @@ -21,6 +21,7 @@ #include "llvm/ADT/ilist.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Metadata.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/Recycler.h" @@ -38,6 +39,7 @@ class MachineModuleInfo; class MCContext; class Pass; class TargetMachine; +class TargetSubtargetInfo; class TargetRegisterClass; struct MachinePointerInfo; @@ -70,15 +72,24 @@ private: /// MachineFunction is destroyed. struct MachineFunctionInfo { virtual ~MachineFunctionInfo(); + + /// \brief Factory function: default behavior is to call new using the + /// supplied allocator. + /// + /// This function can be overridden in a derive class. + template + static Ty *create(BumpPtrAllocator &Allocator, MachineFunction &MF) { + return new (Allocator.Allocate()) Ty(MF); + } }; class MachineFunction { const Function *Fn; const TargetMachine &Target; + const TargetSubtargetInfo *STI; MCContext &Ctx; MachineModuleInfo &MMI; - GCModuleInfo *GMI; - + // RegInfo - Information about each register in use in the function. MachineRegisterInfo *RegInfo; @@ -138,12 +149,10 @@ class MachineFunction { void operator=(const MachineFunction&) LLVM_DELETED_FUNCTION; public: MachineFunction(const Function *Fn, const TargetMachine &TM, - unsigned FunctionNum, MachineModuleInfo &MMI, - GCModuleInfo* GMI); + unsigned FunctionNum, MachineModuleInfo &MMI); ~MachineFunction(); MachineModuleInfo &getMMI() const { return MMI; } - GCModuleInfo *getGMI() const { return GMI; } MCContext &getContext() const { return Ctx; } /// getFunction - Return the LLVM function that this machine code represents @@ -162,6 +171,11 @@ public: /// const TargetMachine &getTarget() const { return Target; } + /// getSubtarget - Return the subtarget for which this machine code is being + /// compiled. + const TargetSubtargetInfo &getSubtarget() const { return *STI; } + void setSubtarget(const TargetSubtargetInfo *ST) { STI = ST; } + /// getRegInfo - Return information about the registers currently in use. /// MachineRegisterInfo &getRegInfo() { return *RegInfo; } @@ -234,7 +248,7 @@ public: template Ty *getInfo() { if (!MFInfo) - MFInfo = new (Allocator.Allocate()) Ty(*this); + MFInfo = Ty::template create(Allocator, *this); return static_cast(MFInfo); } @@ -399,7 +413,7 @@ public: MachineMemOperand *getMachineMemOperand(MachinePointerInfo PtrInfo, unsigned f, uint64_t s, unsigned base_alignment, - const MDNode *TBAAInfo = nullptr, + const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr); /// getMachineMemOperand - Allocate a new MachineMemOperand by copying diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h index 3c828116411e..bcf1f5caaa8c 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h @@ -244,12 +244,22 @@ public: /// DebugLoc getDebugLoc() const { return debugLoc; } - /// getDebugVariable() - Return the debug variable referenced by + /// \brief Return the debug variable referenced by /// this DBG_VALUE instruction. DIVariable getDebugVariable() const { assert(isDebugValue() && "not a DBG_VALUE"); - const MDNode *Var = getOperand(getNumOperands() - 1).getMetadata(); - return DIVariable(Var); + DIVariable Var(getOperand(2).getMetadata()); + assert(Var.Verify() && "not a DIVariable"); + return Var; + } + + /// \brief Return the complex address expression referenced by + /// this DBG_VALUE instruction. + DIExpression getDebugExpression() const { + assert(isDebugValue() && "not a DBG_VALUE"); + DIExpression Expr(getOperand(3).getMetadata()); + assert(Expr.Verify() && "not a DIExpression"); + return Expr; } /// emitError - Emit an error referring to the source location of this @@ -510,6 +520,49 @@ public: return hasProperty(MCID::FoldableAsLoad, Type); } + /// \brief Return true if this instruction behaves + /// the same way as the generic REG_SEQUENCE instructions. + /// E.g., on ARM, + /// dX VMOVDRR rY, rZ + /// is equivalent to + /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. + /// + /// Note that for the optimizers to be able to take advantage of + /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be + /// override accordingly. + bool isRegSequenceLike(QueryType Type = IgnoreBundle) const { + return hasProperty(MCID::RegSequence, Type); + } + + /// \brief Return true if this instruction behaves + /// the same way as the generic EXTRACT_SUBREG instructions. + /// E.g., on ARM, + /// rX, rY VMOVRRD dZ + /// is equivalent to two EXTRACT_SUBREG: + /// rX = EXTRACT_SUBREG dZ, ssub_0 + /// rY = EXTRACT_SUBREG dZ, ssub_1 + /// + /// Note that for the optimizers to be able to take advantage of + /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be + /// override accordingly. + bool isExtractSubregLike(QueryType Type = IgnoreBundle) const { + return hasProperty(MCID::ExtractSubreg, Type); + } + + /// \brief Return true if this instruction behaves + /// the same way as the generic INSERT_SUBREG instructions. + /// E.g., on ARM, + /// dX = VSETLNi32 dY, rZ, Imm + /// is equivalent to a INSERT_SUBREG: + /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) + /// + /// Note that for the optimizers to be able to take advantage of + /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be + /// override accordingly. + bool isInsertSubregLike(QueryType Type = IgnoreBundle) const { + return hasProperty(MCID::InsertSubreg, Type); + } + //===--------------------------------------------------------------------===// // Side Effect Analysis //===--------------------------------------------------------------------===// @@ -614,7 +667,6 @@ public: /// are not marking copies from and to the same register class with this flag. bool isAsCheapAsAMove(QueryType Type = AllInBundle) const { // Only returns true for a bundle if all bundled instructions are cheap. - // FIXME: This probably requires a target hook. return hasProperty(MCID::CheapAsAMove, Type); } @@ -672,6 +724,12 @@ public: /// eraseFromBundle() to erase individual bundled instructions. void eraseFromParent(); + /// Unlink 'this' from the containing basic block and delete it. + /// + /// For all definitions mark their uses in DBG_VALUE nodes + /// as undefined. Otherwise like eraseFromParent(). + void eraseFromParentAndMarkDBGValuesForRemoval(); + /// Unlink 'this' form its basic block and delete it. /// /// If the instruction is part of a bundle, the other instructions in the @@ -1081,7 +1139,10 @@ public: /// setDebugLoc - Replace current source information with new such. /// Avoid using this, the constructor argument is preferable. /// - void setDebugLoc(const DebugLoc dl) { debugLoc = dl; } + void setDebugLoc(const DebugLoc dl) { + debugLoc = dl; + assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); + } /// RemoveOperand - Erase an operand from an instruction, leaving it with one /// fewer operand than it started with. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 21a482cdbd4c..8859b6a019ea 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -58,6 +58,10 @@ public: MachineInstr *operator->() const { return MI; } operator MachineBasicBlock::iterator() const { return MI; } + /// If conversion operators fail, use this method to get the MachineInstr + /// explicitly. + MachineInstr *getInstr() const { return MI; } + /// addReg - Add a new virtual register operand... /// const @@ -170,6 +174,8 @@ public: const MachineInstrBuilder &addMetadata(const MDNode *MD) const { MI->addOperand(*MF, MachineOperand::CreateMetadata(MD)); + assert((MI->isDebugValue() ? MI->getDebugVariable().Verify() : true) && + "first MDNode argument of a DBG_VALUE not a DIVariable"); return *this; } @@ -345,24 +351,25 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, /// address. The convention is that a DBG_VALUE is indirect iff the /// second operand is an immediate. /// -inline MachineInstrBuilder BuildMI(MachineFunction &MF, - DebugLoc DL, - const MCInstrDesc &MCID, - bool IsIndirect, - unsigned Reg, - unsigned Offset, - const MDNode *MD) { +inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, + const MCInstrDesc &MCID, bool IsIndirect, + unsigned Reg, unsigned Offset, + const MDNode *Variable, const MDNode *Expr) { + assert(DIVariable(Variable).Verify() && "not a DIVariable"); + assert(DIExpression(Expr).Verify() && "not a DIExpression"); if (IsIndirect) return BuildMI(MF, DL, MCID) - .addReg(Reg, RegState::Debug) - .addImm(Offset) - .addMetadata(MD); + .addReg(Reg, RegState::Debug) + .addImm(Offset) + .addMetadata(Variable) + .addMetadata(Expr); else { assert(Offset == 0 && "A direct address cannot have an offset."); return BuildMI(MF, DL, MCID) - .addReg(Reg, RegState::Debug) - .addReg(0U, RegState::Debug) - .addMetadata(MD); + .addReg(Reg, RegState::Debug) + .addReg(0U, RegState::Debug) + .addMetadata(Variable) + .addMetadata(Expr); } } @@ -371,15 +378,15 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, /// address and inserts it at position I. /// inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, - MachineBasicBlock::iterator I, - DebugLoc DL, - const MCInstrDesc &MCID, - bool IsIndirect, - unsigned Reg, - unsigned Offset, - const MDNode *MD) { + MachineBasicBlock::iterator I, DebugLoc DL, + const MCInstrDesc &MCID, bool IsIndirect, + unsigned Reg, unsigned Offset, + const MDNode *Variable, const MDNode *Expr) { + assert(DIVariable(Variable).Verify() && "not a DIVariable"); + assert(DIExpression(Expr).Verify() && "not a DIExpression"); MachineFunction &MF = *BB.getParent(); - MachineInstr *MI = BuildMI(MF, DL, MCID, IsIndirect, Reg, Offset, MD); + MachineInstr *MI = + BuildMI(MF, DL, MCID, IsIndirect, Reg, Offset, Variable, Expr); BB.insert(I, MI); return MachineInstrBuilder(MF, MI); } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h b/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h index 2532c16271f0..eb5086cbe5a5 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -18,6 +18,7 @@ #include "llvm/ADT/PointerUnion.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Value.h" // PointerLikeTypeTraits #include "llvm/Support/DataTypes.h" @@ -91,7 +92,7 @@ class MachineMemOperand { MachinePointerInfo PtrInfo; uint64_t Size; unsigned Flags; - const MDNode *TBAAInfo; + AAMDNodes AAInfo; const MDNode *Ranges; public: @@ -117,7 +118,8 @@ public: /// MachineMemOperand - Construct an MachineMemOperand object with the /// specified PtrInfo, flags, size, and base alignment. MachineMemOperand(MachinePointerInfo PtrInfo, unsigned flags, uint64_t s, - unsigned base_alignment, const MDNode *TBAAInfo = nullptr, + unsigned base_alignment, + const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr); const MachinePointerInfo &getPointerInfo() const { return PtrInfo; } @@ -161,8 +163,8 @@ public: /// base address, without the offset. uint64_t getBaseAlignment() const { return (1u << (Flags >> MOMaxBits)) >> 1; } - /// getTBAAInfo - Return the TBAA tag for the memory reference. - const MDNode *getTBAAInfo() const { return TBAAInfo; } + /// getAAInfo - Return the AA tags for the memory reference. + AAMDNodes getAAInfo() const { return AAInfo; } /// getRanges - Return the range tag for the memory reference. const MDNode *getRanges() const { return Ranges; } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h index 6d8d05684c56..584ce65c9d17 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h @@ -110,10 +110,6 @@ class MachineModuleInfo : public ImmutablePass { /// by debug and exception handling consumers. std::vector FrameInstructions; - /// CompactUnwindEncoding - If the target supports it, this is the compact - /// unwind encoding. It replaces a function's CIE and FDE. - uint32_t CompactUnwindEncoding; - /// LandingPads - List of LandingPadInfo describing the landing pad /// information in the current function. std::vector LandingPads; @@ -131,7 +127,7 @@ class MachineModuleInfo : public ImmutablePass { unsigned CurCallSite; /// TypeInfos - List of C++ TypeInfo used in the current function. - std::vector TypeInfos; + std::vector TypeInfos; /// FilterIds - List of typeids encoding filters used in the current function. std::vector FilterIds; @@ -165,13 +161,24 @@ class MachineModuleInfo : public ImmutablePass { /// to _fltused on Windows targets. bool UsesVAFloatArgument; + /// UsesMorestackAddr - True if the module calls the __morestack function + /// indirectly, as is required under the large code model on x86. This is used + /// to emit a definition of a symbol, __morestack_addr, containing the + /// address. See comments in lib/Target/X86/X86FrameLowering.cpp for more + /// details. + bool UsesMorestackAddr; + public: static char ID; // Pass identification, replacement for typeid struct VariableDbgInfo { - TrackingVH Var; + TrackingMDNodeRef Var; + TrackingMDNodeRef Expr; unsigned Slot; DebugLoc Loc; + + VariableDbgInfo(MDNode *Var, MDNode *Expr, unsigned Slot, DebugLoc Loc) + : Var(Var), Expr(Expr), Slot(Slot), Loc(Loc) {} }; typedef SmallVector VariableDbgInfoMapTy; VariableDbgInfoMapTy VariableDbgInfos; @@ -234,6 +241,14 @@ public: UsesVAFloatArgument = b; } + bool usesMorestackAddr() const { + return UsesMorestackAddr; + } + + void setUsesMorestackAddr(bool b) { + UsesMorestackAddr = b; + } + /// \brief Returns a reference to a list of cfi instructions in the current /// function's prologue. Used to construct frame maps for debug and exception /// handling comsumers. @@ -247,15 +262,6 @@ public: return FrameInstructions.size() - 1; } - /// getCompactUnwindEncoding - Returns the compact unwind encoding for a - /// function if the target supports the encoding. This encoding replaces a - /// function's CIE and FDE. - uint32_t getCompactUnwindEncoding() const { return CompactUnwindEncoding; } - - /// setCompactUnwindEncoding - Set the compact unwind encoding for a function - /// if the target supports the encoding. - void setCompactUnwindEncoding(uint32_t Enc) { CompactUnwindEncoding = Enc; } - /// getAddrLabelSymbol - Return the symbol to be used for the specified basic /// block when its address is taken. This cannot be its normal LBB label /// because the block may be accessed outside its containing function. @@ -313,12 +319,12 @@ public: /// addCatchTypeInfo - Provide the catch typeinfo for a landing pad. /// void addCatchTypeInfo(MachineBasicBlock *LandingPad, - ArrayRef TyInfo); + ArrayRef TyInfo); /// addFilterTypeInfo - Provide the filter typeinfo for a landing pad. /// void addFilterTypeInfo(MachineBasicBlock *LandingPad, - ArrayRef TyInfo); + ArrayRef TyInfo); /// addCleanup - Add a cleanup action for a landing pad. /// @@ -326,7 +332,7 @@ public: /// getTypeIDFor - Return the type id for the specified typeinfo. This is /// function wide. - unsigned getTypeIDFor(const GlobalVariable *TI); + unsigned getTypeIDFor(const GlobalValue *TI); /// getFilterIDFor - Return the id of the filter encoded by TyIds. This is /// function wide. @@ -387,7 +393,7 @@ public: /// getTypeInfos - Return a reference to the C++ typeinfo for the current /// function. - const std::vector &getTypeInfos() const { + const std::vector &getTypeInfos() const { return TypeInfos; } @@ -403,9 +409,9 @@ public: /// setVariableDbgInfo - Collect information used to emit debugging /// information of a variable. - void setVariableDbgInfo(MDNode *N, unsigned Slot, DebugLoc Loc) { - VariableDbgInfo Info = { N, Slot, Loc }; - VariableDbgInfos.push_back(std::move(Info)); + void setVariableDbgInfo(MDNode *Var, MDNode *Expr, unsigned Slot, + DebugLoc Loc) { + VariableDbgInfos.emplace_back(Var, Expr, Slot, Loc); } VariableDbgInfoMapTy &getVariableDbgInfo() { return VariableDbgInfos; } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineOperand.h b/contrib/llvm/include/llvm/CodeGen/MachineOperand.h index 22969bc80776..eed1e575f93b 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineOperand.h @@ -506,6 +506,11 @@ public: Contents.ImmVal = immVal; } + void setFPImm(const ConstantFP *CFP) { + assert(isFPImm() && "Wrong MachineOperand mutator"); + Contents.CFP = CFP; + } + void setOffset(int64_t Offset) { assert((isGlobal() || isSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && "Wrong MachineOperand accessor"); @@ -544,6 +549,11 @@ public: /// the setImm method should be used. void ChangeToImmediate(int64_t ImmVal); + /// ChangeToFPImmediate - Replace this operand with a new FP immediate operand + /// of the specified value. If an operand is known to be an FP immediate + /// already, the setFPImm method should be used. + void ChangeToFPImmediate(const ConstantFP *FPImm); + /// ChangeToRegister - Replace this operand with a new register operand of /// the specified value. If an operand is known to be an register already, /// the setReg method should be used. @@ -702,6 +712,8 @@ public: friend class MachineInstr; friend class MachineRegisterInfo; private: + void removeRegFromUses(); + //===--------------------------------------------------------------------===// // Methods for handling register use/def lists. //===--------------------------------------------------------------------===// diff --git a/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h b/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h index beb2c4f0c5c0..aab5c407629f 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h +++ b/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h @@ -22,7 +22,7 @@ namespace llvm { /// /// PostDominatorTree Class - Concrete subclass of DominatorTree that is used -/// to compute the a post-dominator tree. +/// to compute the post-dominator tree. /// struct MachinePostDominatorTree : public MachineFunctionPass { private: diff --git a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h index 51139f72ba22..caa48a5cf0cf 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h @@ -17,9 +17,10 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBundle.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" #include namespace llvm { @@ -39,7 +40,7 @@ public: }; private: - const TargetMachine &TM; + const MachineFunction *MF; Delegate *TheDelegate; /// IsSSA - True when the machine function is in SSA form and virtual @@ -51,6 +52,9 @@ private: /// accurate when after this flag is cleared. bool TracksLiveness; + /// True if subregister liveness is tracked. + bool TracksSubRegLiveness; + /// VRegInfo - Information we keep for each virtual register. /// /// Each element in this list contains the register class of the vreg and the @@ -69,7 +73,7 @@ private: /// PhysRegUseDefLists - This is an array of the head of the use/def list for /// physical registers. - MachineOperand **PhysRegUseDefLists; + std::vector PhysRegUseDefLists; /// getRegUseDefListHead - Return the head pointer for the register use/def /// list for the specified virtual or physical register. @@ -122,11 +126,10 @@ private: MachineRegisterInfo(const MachineRegisterInfo&) LLVM_DELETED_FUNCTION; void operator=(const MachineRegisterInfo&) LLVM_DELETED_FUNCTION; public: - explicit MachineRegisterInfo(const TargetMachine &TM); - ~MachineRegisterInfo(); + explicit MachineRegisterInfo(const MachineFunction *MF); const TargetRegisterInfo *getTargetRegisterInfo() const { - return TM.getRegisterInfo(); + return MF->getSubtarget().getRegisterInfo(); } void resetDelegate(Delegate *delegate) { @@ -179,6 +182,12 @@ public: /// information. void invalidateLiveness() { TracksLiveness = false; } + bool tracksSubRegLiveness() const { return TracksSubRegLiveness; } + + void enableSubRegLiveness(bool Enable = true) { + TracksSubRegLiveness = Enable; + } + //===--------------------------------------------------------------------===// // Register Info //===--------------------------------------------------------------------===// @@ -515,8 +524,12 @@ public: /// /// That function will return NULL if the virtual registers have incompatible /// constraints. + /// + /// Note that if ToReg is a physical register the function will replace and + /// apply sub registers to ToReg in order to obtain a final/proper physical + /// register. void replaceRegWith(unsigned FromReg, unsigned ToReg); - + /// getVRegDef - Return the machine instr that defines the specified virtual /// register or null if none is found. This assumes that the code is in SSA /// form, so there should only be one definition. @@ -764,6 +777,10 @@ public: const TargetRegisterInfo &TRI, const TargetInstrInfo &TII); + /// Returns a mask covering all bits that can appear in lane masks of + /// subregisters of the virtual register @p Reg. + unsigned getMaxLaneMaskForVReg(unsigned Reg) const; + /// defusechain_iterator - This class provides iterator support for machine /// operands in the function that use or define a specific register. If /// ReturnUses is true it returns uses of registers, if ReturnDefs is true it diff --git a/contrib/llvm/include/llvm/CodeGen/MachineRelocation.h b/contrib/llvm/include/llvm/CodeGen/MachineRelocation.h deleted file mode 100644 index e77845745165..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/MachineRelocation.h +++ /dev/null @@ -1,342 +0,0 @@ -//===-- llvm/CodeGen/MachineRelocation.h - Target Relocation ----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the MachineRelocation class. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_MACHINERELOCATION_H -#define LLVM_CODEGEN_MACHINERELOCATION_H - -#include "llvm/Support/DataTypes.h" -#include - -namespace llvm { -class GlobalValue; -class MachineBasicBlock; - -/// MachineRelocation - This represents a target-specific relocation value, -/// produced by the code emitter. This relocation is resolved after the has -/// been emitted, either to an object file or to memory, when the target of the -/// relocation can be resolved. -/// -/// A relocation is made up of the following logical portions: -/// 1. An offset in the machine code buffer, the location to modify. -/// 2. A target specific relocation type (a number from 0 to 63). -/// 3. A symbol being referenced, either as a GlobalValue* or as a string. -/// 4. An optional constant value to be added to the reference. -/// 5. A bit, CanRewrite, which indicates to the JIT that a function stub is -/// not needed for the relocation. -/// 6. An index into the GOT, if the target uses a GOT -/// -class MachineRelocation { - enum AddressType { - isResult, // Relocation has be transformed into its result pointer. - isGV, // The Target.GV field is valid. - isIndirectSym, // Relocation of an indirect symbol. - isBB, // Relocation of BB address. - isExtSym, // The Target.ExtSym field is valid. - isConstPool, // Relocation of constant pool address. - isJumpTable, // Relocation of jump table address. - isGOTIndex // The Target.GOTIndex field is valid. - }; - - /// Offset - This is the offset from the start of the code buffer of the - /// relocation to perform. - uintptr_t Offset; - - /// ConstantVal - A field that may be used by the target relocation type. - intptr_t ConstantVal; - - union { - void *Result; // If this has been resolved to a resolved pointer - GlobalValue *GV; // If this is a pointer to a GV or an indirect ref. - MachineBasicBlock *MBB; // If this is a pointer to an LLVM BB - const char *ExtSym; // If this is a pointer to a named symbol - unsigned Index; // Constant pool / jump table index - unsigned GOTIndex; // Index in the GOT of this symbol/global - } Target; - - unsigned TargetReloType : 6; // The target relocation ID - AddressType AddrType : 4; // The field of Target to use - bool MayNeedFarStub : 1; // True if this relocation may require a far-stub - bool GOTRelative : 1; // Should this relocation be relative to the GOT? - bool TargetResolve : 1; // True if target should resolve the address - -public: - // Relocation types used in a generic implementation. Currently, relocation - // entries for all things use the generic VANILLA type until they are refined - // into target relocation types. - enum RelocationType { - VANILLA - }; - - /// MachineRelocation::getGV - Return a relocation entry for a GlobalValue. - /// - static MachineRelocation getGV(uintptr_t offset, unsigned RelocationType, - GlobalValue *GV, intptr_t cst = 0, - bool MayNeedFarStub = 0, - bool GOTrelative = 0) { - assert((RelocationType & ~63) == 0 && "Relocation type too large!"); - MachineRelocation Result; - Result.Offset = offset; - Result.ConstantVal = cst; - Result.TargetReloType = RelocationType; - Result.AddrType = isGV; - Result.MayNeedFarStub = MayNeedFarStub; - Result.GOTRelative = GOTrelative; - Result.TargetResolve = false; - Result.Target.GV = GV; - return Result; - } - - /// MachineRelocation::getIndirectSymbol - Return a relocation entry for an - /// indirect symbol. - static MachineRelocation getIndirectSymbol(uintptr_t offset, - unsigned RelocationType, - GlobalValue *GV, intptr_t cst = 0, - bool MayNeedFarStub = 0, - bool GOTrelative = 0) { - assert((RelocationType & ~63) == 0 && "Relocation type too large!"); - MachineRelocation Result; - Result.Offset = offset; - Result.ConstantVal = cst; - Result.TargetReloType = RelocationType; - Result.AddrType = isIndirectSym; - Result.MayNeedFarStub = MayNeedFarStub; - Result.GOTRelative = GOTrelative; - Result.TargetResolve = false; - Result.Target.GV = GV; - return Result; - } - - /// MachineRelocation::getBB - Return a relocation entry for a BB. - /// - static MachineRelocation getBB(uintptr_t offset,unsigned RelocationType, - MachineBasicBlock *MBB, intptr_t cst = 0) { - assert((RelocationType & ~63) == 0 && "Relocation type too large!"); - MachineRelocation Result; - Result.Offset = offset; - Result.ConstantVal = cst; - Result.TargetReloType = RelocationType; - Result.AddrType = isBB; - Result.MayNeedFarStub = false; - Result.GOTRelative = false; - Result.TargetResolve = false; - Result.Target.MBB = MBB; - return Result; - } - - /// MachineRelocation::getExtSym - Return a relocation entry for an external - /// symbol, like "free". - /// - static MachineRelocation getExtSym(uintptr_t offset, unsigned RelocationType, - const char *ES, intptr_t cst = 0, - bool GOTrelative = 0, - bool NeedStub = true) { - assert((RelocationType & ~63) == 0 && "Relocation type too large!"); - MachineRelocation Result; - Result.Offset = offset; - Result.ConstantVal = cst; - Result.TargetReloType = RelocationType; - Result.AddrType = isExtSym; - Result.MayNeedFarStub = NeedStub; - Result.GOTRelative = GOTrelative; - Result.TargetResolve = false; - Result.Target.ExtSym = ES; - return Result; - } - - /// MachineRelocation::getConstPool - Return a relocation entry for a constant - /// pool entry. - /// - static MachineRelocation getConstPool(uintptr_t offset,unsigned RelocationType, - unsigned CPI, intptr_t cst = 0, - bool letTargetResolve = false) { - assert((RelocationType & ~63) == 0 && "Relocation type too large!"); - MachineRelocation Result; - Result.Offset = offset; - Result.ConstantVal = cst; - Result.TargetReloType = RelocationType; - Result.AddrType = isConstPool; - Result.MayNeedFarStub = false; - Result.GOTRelative = false; - Result.TargetResolve = letTargetResolve; - Result.Target.Index = CPI; - return Result; - } - - /// MachineRelocation::getJumpTable - Return a relocation entry for a jump - /// table entry. - /// - static MachineRelocation getJumpTable(uintptr_t offset,unsigned RelocationType, - unsigned JTI, intptr_t cst = 0, - bool letTargetResolve = false) { - assert((RelocationType & ~63) == 0 && "Relocation type too large!"); - MachineRelocation Result; - Result.Offset = offset; - Result.ConstantVal = cst; - Result.TargetReloType = RelocationType; - Result.AddrType = isJumpTable; - Result.MayNeedFarStub = false; - Result.GOTRelative = false; - Result.TargetResolve = letTargetResolve; - Result.Target.Index = JTI; - return Result; - } - - /// getMachineCodeOffset - Return the offset into the code buffer that the - /// relocation should be performed. - intptr_t getMachineCodeOffset() const { - return Offset; - } - - /// getRelocationType - Return the target-specific relocation ID for this - /// relocation. - unsigned getRelocationType() const { - return TargetReloType; - } - - /// getConstantVal - Get the constant value associated with this relocation. - /// This is often an offset from the symbol. - /// - intptr_t getConstantVal() const { - return ConstantVal; - } - - /// setConstantVal - Set the constant value associated with this relocation. - /// This is often an offset from the symbol. - /// - void setConstantVal(intptr_t val) { - ConstantVal = val; - } - - /// isGlobalValue - Return true if this relocation is a GlobalValue, as - /// opposed to a constant string. - bool isGlobalValue() const { - return AddrType == isGV; - } - - /// isIndirectSymbol - Return true if this relocation is the address an - /// indirect symbol - bool isIndirectSymbol() const { - return AddrType == isIndirectSym; - } - - /// isBasicBlock - Return true if this relocation is a basic block reference. - /// - bool isBasicBlock() const { - return AddrType == isBB; - } - - /// isExternalSymbol - Return true if this is a constant string. - /// - bool isExternalSymbol() const { - return AddrType == isExtSym; - } - - /// isConstantPoolIndex - Return true if this is a constant pool reference. - /// - bool isConstantPoolIndex() const { - return AddrType == isConstPool; - } - - /// isJumpTableIndex - Return true if this is a jump table reference. - /// - bool isJumpTableIndex() const { - return AddrType == isJumpTable; - } - - /// isGOTRelative - Return true the target wants the index into the GOT of - /// the symbol rather than the address of the symbol. - bool isGOTRelative() const { - return GOTRelative; - } - - /// mayNeedFarStub - This function returns true if the JIT for this target may - /// need either a stub function or an indirect global-variable load to handle - /// the relocated GlobalValue reference. For example, the x86-64 call - /// instruction can only call functions within +/-2GB of the call site. - /// Anything farther away needs a longer mov+call sequence, which can't just - /// be written on top of the existing call. - bool mayNeedFarStub() const { - return MayNeedFarStub; - } - - /// letTargetResolve - Return true if the target JITInfo is usually - /// responsible for resolving the address of this relocation. - bool letTargetResolve() const { - return TargetResolve; - } - - /// getGlobalValue - If this is a global value reference, return the - /// referenced global. - GlobalValue *getGlobalValue() const { - assert((isGlobalValue() || isIndirectSymbol()) && - "This is not a global value reference!"); - return Target.GV; - } - - MachineBasicBlock *getBasicBlock() const { - assert(isBasicBlock() && "This is not a basic block reference!"); - return Target.MBB; - } - - /// getString - If this is a string value, return the string reference. - /// - const char *getExternalSymbol() const { - assert(isExternalSymbol() && "This is not an external symbol reference!"); - return Target.ExtSym; - } - - /// getConstantPoolIndex - If this is a const pool reference, return - /// the index into the constant pool. - unsigned getConstantPoolIndex() const { - assert(isConstantPoolIndex() && "This is not a constant pool reference!"); - return Target.Index; - } - - /// getJumpTableIndex - If this is a jump table reference, return - /// the index into the jump table. - unsigned getJumpTableIndex() const { - assert(isJumpTableIndex() && "This is not a jump table reference!"); - return Target.Index; - } - - /// getResultPointer - Once this has been resolved to point to an actual - /// address, this returns the pointer. - void *getResultPointer() const { - assert(AddrType == isResult && "Result pointer isn't set yet!"); - return Target.Result; - } - - /// setResultPointer - Set the result to the specified pointer value. - /// - void setResultPointer(void *Ptr) { - Target.Result = Ptr; - AddrType = isResult; - } - - /// setGOTIndex - Set the GOT index to a specific value. - void setGOTIndex(unsigned idx) { - AddrType = isGOTIndex; - Target.GOTIndex = idx; - } - - /// getGOTIndex - Once this has been resolved to an entry in the GOT, - /// this returns that index. The index is from the lowest address entry - /// in the GOT. - unsigned getGOTIndex() const { - assert(AddrType == isGOTIndex); - return Target.GOTIndex; - } -}; -} - -#endif diff --git a/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h b/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h index 7d85432101b5..a31940161ca5 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -80,7 +80,6 @@ #include "llvm/CodeGen/MachinePassRegistry.h" #include "llvm/CodeGen/RegisterPressure.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" - #include namespace llvm { @@ -250,7 +249,7 @@ protected: public: ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr S, bool IsPostRA) - : ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, IsPostRA, + : ScheduleDAGInstrs(*C->MF, C->MLI, IsPostRA, /*RemoveKillFlags=*/IsPostRA, C->LIS), AA(C->AA), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU), CurrentTop(), CurrentBottom(), NextClusterPred(nullptr), NextClusterSucc(nullptr) { diff --git a/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h b/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h index 323b694f3933..bfe6e945b6da 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h @@ -44,8 +44,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_MACHINE_TRACE_METRICS_H -#define LLVM_CODEGEN_MACHINE_TRACE_METRICS_H +#ifndef LLVM_CODEGEN_MACHINETRACEMETRICS_H +#define LLVM_CODEGEN_MACHINETRACEMETRICS_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -264,8 +264,9 @@ public: /// classes are included. For the caller to account for extra machine /// instructions, it must first resolve each instruction's scheduling class. unsigned getResourceLength( - ArrayRef Extrablocks = None, - ArrayRef ExtraInstrs = None) const; + ArrayRef Extrablocks = None, + ArrayRef ExtraInstrs = None, + ArrayRef RemoveInstrs = None) const; /// Return the length of the (data dependency) critical path through the /// trace. @@ -286,6 +287,12 @@ public: /// Return the Depth of a PHI instruction in a trace center block successor. /// The PHI does not have to be part of the trace. unsigned getPHIDepth(const MachineInstr *PHI) const; + + /// A dependence is useful if the basic block of the defining instruction + /// is part of the trace of the user instruction. It is assumed that DefMI + /// dominates UseMI (see also isUsefulDominator). + bool isDepInTrace(const MachineInstr *DefMI, + const MachineInstr *UseMI) const; }; /// A trace ensemble is a collection of traces selected using the same diff --git a/contrib/llvm/include/llvm/CodeGen/MachineValueType.h b/contrib/llvm/include/llvm/CodeGen/MachineValueType.h index ad215ec09843..e3fbfe89c203 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineValueType.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineValueType.h @@ -15,6 +15,7 @@ #ifndef LLVM_CODEGEN_MACHINEVALUETYPE_H #define LLVM_CODEGEN_MACHINEVALUETYPE_H +#include "llvm/ADT/iterator_range.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -118,6 +119,7 @@ namespace llvm { // unspecified type. The register class // will be determined by the opcode. + FIRST_VALUETYPE = 0, // This is always the beginning of the list. LAST_VALUETYPE = 58, // This always remains at the end of the list. // This is the current maximum for LAST_VALUETYPE. @@ -165,6 +167,12 @@ namespace llvm { bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; } bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; } + /// isValid - Return true if this is a valid simple valuetype. + bool isValid() const { + return (SimpleTy >= MVT::FIRST_VALUETYPE && + SimpleTy < MVT::LAST_VALUETYPE); + } + /// isFloatingPoint - Return true if this is a FP, or a vector FP type. bool isFloatingPoint() const { return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE && @@ -196,21 +204,24 @@ namespace llvm { /// is32BitVector - Return true if this is a 32-bit vector type. bool is32BitVector() const { return (SimpleTy == MVT::v4i8 || SimpleTy == MVT::v2i16 || - SimpleTy == MVT::v1i32); + SimpleTy == MVT::v1i32 || SimpleTy == MVT::v2f16 || + SimpleTy == MVT::v1f32); } /// is64BitVector - Return true if this is a 64-bit vector type. bool is64BitVector() const { return (SimpleTy == MVT::v8i8 || SimpleTy == MVT::v4i16 || SimpleTy == MVT::v2i32 || SimpleTy == MVT::v1i64 || - SimpleTy == MVT::v1f64 || SimpleTy == MVT::v2f32); + SimpleTy == MVT::v4f16 || SimpleTy == MVT::v2f32 || + SimpleTy == MVT::v1f64); } /// is128BitVector - Return true if this is a 128-bit vector type. bool is128BitVector() const { return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 || SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 || - SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64); + SimpleTy == MVT::v8f16 || SimpleTy == MVT::v4f32 || + SimpleTy == MVT::v2f64); } /// is256BitVector - Return true if this is a 256-bit vector type. @@ -572,6 +583,52 @@ namespace llvm { /// returned as Other, otherwise they are invalid. static MVT getVT(Type *Ty, bool HandleUnknown = false); + private: + /// A simple iterator over the MVT::SimpleValueType enum. + struct mvt_iterator { + SimpleValueType VT; + mvt_iterator(SimpleValueType VT) : VT(VT) {} + MVT operator*() const { return VT; } + bool operator!=(const mvt_iterator &LHS) const { return VT != LHS.VT; } + mvt_iterator& operator++() { + VT = (MVT::SimpleValueType)((int)VT + 1); + assert((int)VT <= MVT::MAX_ALLOWED_VALUETYPE && + "MVT iterator overflowed."); + return *this; + } + }; + /// A range of the MVT::SimpleValueType enum. + typedef iterator_range mvt_range; + + public: + /// SimpleValueType Iteration + /// @{ + static mvt_range all_valuetypes() { + return mvt_range(MVT::FIRST_VALUETYPE, MVT::LAST_VALUETYPE); + } + static mvt_range integer_valuetypes() { + return mvt_range(MVT::FIRST_INTEGER_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_INTEGER_VALUETYPE + 1)); + } + static mvt_range fp_valuetypes() { + return mvt_range(MVT::FIRST_FP_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_FP_VALUETYPE + 1)); + } + static mvt_range vector_valuetypes() { + return mvt_range(MVT::FIRST_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_VECTOR_VALUETYPE + 1)); + } + static mvt_range integer_vector_valuetypes() { + return mvt_range( + MVT::FIRST_INTEGER_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_INTEGER_VECTOR_VALUETYPE + 1)); + } + static mvt_range fp_vector_valuetypes() { + return mvt_range( + MVT::FIRST_FP_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_FP_VECTOR_VALUETYPE + 1)); + } + /// @} }; } // End llvm namespace diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/CostAllocator.h b/contrib/llvm/include/llvm/CodeGen/PBQP/CostAllocator.h index ff62c0959344..02d39fe383f1 100644 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/CostAllocator.h +++ b/contrib/llvm/include/llvm/CodeGen/PBQP/CostAllocator.h @@ -15,117 +15,101 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_COSTALLOCATOR_H -#define LLVM_COSTALLOCATOR_H +#ifndef LLVM_CODEGEN_PBQP_COSTALLOCATOR_H +#define LLVM_CODEGEN_PBQP_COSTALLOCATOR_H -#include +#include "llvm/ADT/DenseSet.h" +#include #include +namespace llvm { namespace PBQP { -template -class CostPool { +template +class ValuePool { public: - - class PoolEntry { - public: - template - PoolEntry(CostPool &pool, CostKeyT cost) - : pool(pool), cost(std::move(cost)), refCount(0) {} - ~PoolEntry() { pool.removeEntry(this); } - void incRef() { ++refCount; } - bool decRef() { --refCount; return (refCount == 0); } - CostT& getCost() { return cost; } - const CostT& getCost() const { return cost; } - private: - CostPool &pool; - CostT cost; - std::size_t refCount; - }; - - class PoolRef { - public: - PoolRef(PoolEntry *entry) : entry(entry) { - this->entry->incRef(); - } - PoolRef(const PoolRef &r) { - entry = r.entry; - entry->incRef(); - } - PoolRef& operator=(const PoolRef &r) { - assert(entry != nullptr && "entry should not be null."); - PoolEntry *temp = r.entry; - temp->incRef(); - entry->decRef(); - entry = temp; - return *this; - } - - ~PoolRef() { - if (entry->decRef()) - delete entry; - } - void reset(PoolEntry *entry) { - entry->incRef(); - this->entry->decRef(); - this->entry = entry; - } - CostT& operator*() { return entry->getCost(); } - const CostT& operator*() const { return entry->getCost(); } - CostT* operator->() { return &entry->getCost(); } - const CostT* operator->() const { return &entry->getCost(); } - private: - PoolEntry *entry; - }; + typedef std::shared_ptr PoolRef; private: - class EntryComparator { + + class PoolEntry : public std::enable_shared_from_this { public: - template - typename std::enable_if< - !std::is_same::type>::value, - bool>::type - operator()(const PoolEntry* a, const CostKeyT &b) { - return compare(a->getCost(), b); - } - bool operator()(const PoolEntry* a, const PoolEntry* b) { - return compare(a->getCost(), b->getCost()); - } + template + PoolEntry(ValuePool &Pool, ValueKeyT Value) + : Pool(Pool), Value(std::move(Value)) {} + ~PoolEntry() { Pool.removeEntry(this); } + const ValueT& getValue() const { return Value; } private: - CostKeyTComparator compare; + ValuePool &Pool; + ValueT Value; }; - typedef std::set EntrySet; + class PoolEntryDSInfo { + public: + static inline PoolEntry* getEmptyKey() { return nullptr; } - EntrySet entrySet; + static inline PoolEntry* getTombstoneKey() { + return reinterpret_cast(static_cast(1)); + } - void removeEntry(PoolEntry *p) { entrySet.erase(p); } + template + static unsigned getHashValue(const ValueKeyT &C) { + return hash_value(C); + } + + static unsigned getHashValue(PoolEntry *P) { + return getHashValue(P->getValue()); + } + + static unsigned getHashValue(const PoolEntry *P) { + return getHashValue(P->getValue()); + } + + template + static + bool isEqual(const ValueKeyT1 &C1, const ValueKeyT2 &C2) { + return C1 == C2; + } + + template + static bool isEqual(const ValueKeyT &C, PoolEntry *P) { + if (P == getEmptyKey() || P == getTombstoneKey()) + return false; + return isEqual(C, P->getValue()); + } + + static bool isEqual(PoolEntry *P1, PoolEntry *P2) { + if (P1 == getEmptyKey() || P1 == getTombstoneKey()) + return P1 == P2; + return isEqual(P1->getValue(), P2); + } + + }; + + typedef DenseSet EntrySetT; + + EntrySetT EntrySet; + + void removeEntry(PoolEntry *P) { EntrySet.erase(P); } public: + template PoolRef getValue(ValueKeyT ValueKey) { + typename EntrySetT::iterator I = EntrySet.find_as(ValueKey); - template - PoolRef getCost(CostKeyT costKey) { - typename EntrySet::iterator itr = - std::lower_bound(entrySet.begin(), entrySet.end(), costKey, - EntryComparator()); + if (I != EntrySet.end()) + return PoolRef((*I)->shared_from_this(), &(*I)->getValue()); - if (itr != entrySet.end() && costKey == (*itr)->getCost()) - return PoolRef(*itr); - - PoolEntry *p = new PoolEntry(*this, std::move(costKey)); - entrySet.insert(itr, p); - return PoolRef(p); + auto P = std::make_shared(*this, std::move(ValueKey)); + EntrySet.insert(P.get()); + return PoolRef(std::move(P), &P->getValue()); } }; -template +template class PoolCostAllocator { private: - typedef CostPool VectorCostPool; - typedef CostPool MatrixCostPool; + typedef ValuePool VectorCostPool; + typedef ValuePool MatrixCostPool; public: typedef VectorT Vector; typedef MatrixT Matrix; @@ -133,15 +117,16 @@ public: typedef typename MatrixCostPool::PoolRef MatrixPtr; template - VectorPtr getVector(VectorKeyT v) { return vectorPool.getCost(std::move(v)); } + VectorPtr getVector(VectorKeyT v) { return VectorPool.getValue(std::move(v)); } template - MatrixPtr getMatrix(MatrixKeyT m) { return matrixPool.getCost(std::move(m)); } + MatrixPtr getMatrix(MatrixKeyT m) { return MatrixPool.getValue(std::move(m)); } private: - VectorCostPool vectorPool; - MatrixCostPool matrixPool; + VectorCostPool VectorPool; + MatrixCostPool MatrixPool; }; -} +} // namespace PBQP +} // namespace llvm -#endif // LLVM_COSTALLOCATOR_H +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/Graph.h b/contrib/llvm/include/llvm/CodeGen/PBQP/Graph.h index a55f0ea96c0a..4dc5674ae134 100644 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/Graph.h +++ b/contrib/llvm/include/llvm/CodeGen/PBQP/Graph.h @@ -17,11 +17,12 @@ #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" -#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" #include #include #include +namespace llvm { namespace PBQP { class GraphBase { @@ -29,12 +30,12 @@ namespace PBQP { typedef unsigned NodeId; typedef unsigned EdgeId; - /// \brief Returns a value representing an invalid (non-existent) node. + /// @brief Returns a value representing an invalid (non-existent) node. static NodeId invalidNodeId() { return std::numeric_limits::max(); } - /// \brief Returns a value representing an invalid (non-existent) edge. + /// @brief Returns a value representing an invalid (non-existent) edge. static EdgeId invalidEdgeId() { return std::numeric_limits::max(); } @@ -56,6 +57,7 @@ namespace PBQP { typedef typename CostAllocator::MatrixPtr MatrixPtr; typedef typename SolverT::NodeMetadata NodeMetadata; typedef typename SolverT::EdgeMetadata EdgeMetadata; + typedef typename SolverT::GraphMetadata GraphMetadata; private: @@ -172,6 +174,7 @@ namespace PBQP { // ----- MEMBERS ----- + GraphMetadata Metadata; CostAllocator CostAlloc; SolverT *Solver; @@ -187,13 +190,19 @@ namespace PBQP { // ----- INTERNAL METHODS ----- - NodeEntry& getNode(NodeId NId) { return Nodes[NId]; } - const NodeEntry& getNode(NodeId NId) const { return Nodes[NId]; } + NodeEntry &getNode(NodeId NId) { + assert(NId < Nodes.size() && "Out of bound NodeId"); + return Nodes[NId]; + } + const NodeEntry &getNode(NodeId NId) const { + assert(NId < Nodes.size() && "Out of bound NodeId"); + return Nodes[NId]; + } EdgeEntry& getEdge(EdgeId EId) { return Edges[EId]; } const EdgeEntry& getEdge(EdgeId EId) const { return Edges[EId]; } - NodeId addConstructedNode(const NodeEntry &N) { + NodeId addConstructedNode(NodeEntry N) { NodeId NId = 0; if (!FreeNodeIds.empty()) { NId = FreeNodeIds.back(); @@ -206,7 +215,7 @@ namespace PBQP { return NId; } - EdgeId addConstructedEdge(const EdgeEntry &E) { + EdgeId addConstructedEdge(EdgeEntry E) { assert(findEdge(E.getN1Id(), E.getN2Id()) == invalidEdgeId() && "Attempt to add duplicate edge."); EdgeId EId = 0; @@ -235,6 +244,12 @@ namespace PBQP { class NodeItr { public: + typedef std::forward_iterator_tag iterator_category; + typedef NodeId value_type; + typedef int difference_type; + typedef NodeId* pointer; + typedef NodeId& reference; + NodeItr(NodeId CurNId, const Graph &G) : CurNId(CurNId), EndNId(G.Nodes.size()), FreeNodeIds(G.FreeNodeIds) { this->CurNId = findNextInUse(CurNId); // Move to first in-use node id @@ -249,7 +264,7 @@ namespace PBQP { NodeId findNextInUse(NodeId NId) const { while (NId < EndNId && std::find(FreeNodeIds.begin(), FreeNodeIds.end(), NId) != - FreeNodeIds.end()) { + FreeNodeIds.end()) { ++NId; } return NId; @@ -328,10 +343,19 @@ namespace PBQP { const NodeEntry &NE; }; - /// \brief Construct an empty PBQP graph. - Graph() : Solver(nullptr) { } + /// @brief Construct an empty PBQP graph. + Graph() : Solver(nullptr) {} - /// \brief Lock this graph to the given solver instance in preparation + /// @brief Construct an empty PBQP graph with the given graph metadata. + Graph(GraphMetadata Metadata) : Metadata(Metadata), Solver(nullptr) {} + + /// @brief Get a reference to the graph metadata. + GraphMetadata& getMetadata() { return Metadata; } + + /// @brief Get a const-reference to the graph metadata. + const GraphMetadata& getMetadata() const { return Metadata; } + + /// @brief Lock this graph to the given solver instance in preparation /// for running the solver. This method will call solver.handleAddNode for /// each node in the graph, and handleAddEdge for each edge, to give the /// solver an opportunity to set up any requried metadata. @@ -344,13 +368,13 @@ namespace PBQP { Solver->handleAddEdge(EId); } - /// \brief Release from solver instance. + /// @brief Release from solver instance. void unsetSolver() { assert(Solver && "Solver not set."); Solver = nullptr; } - /// \brief Add a node with the given costs. + /// @brief Add a node with the given costs. /// @param Costs Cost vector for the new node. /// @return Node iterator for the added node. template @@ -363,9 +387,29 @@ namespace PBQP { return NId; } - /// \brief Add an edge between the given nodes with the given costs. + /// @brief Add a node bypassing the cost allocator. + /// @param Costs Cost vector ptr for the new node (must be convertible to + /// VectorPtr). + /// @return Node iterator for the added node. + /// + /// This method allows for fast addition of a node whose costs don't need + /// to be passed through the cost allocator. The most common use case for + /// this is when duplicating costs from an existing node (when using a + /// pooling allocator). These have already been uniqued, so we can avoid + /// re-constructing and re-uniquing them by attaching them directly to the + /// new node. + template + NodeId addNodeBypassingCostAllocator(OtherVectorPtrT Costs) { + NodeId NId = addConstructedNode(NodeEntry(Costs)); + if (Solver) + Solver->handleAddNode(NId); + return NId; + } + + /// @brief Add an edge between the given nodes with the given costs. /// @param N1Id First node. /// @param N2Id Second node. + /// @param Costs Cost matrix for new edge. /// @return Edge iterator for the added edge. template EdgeId addEdge(NodeId N1Id, NodeId N2Id, OtherVectorT Costs) { @@ -380,7 +424,32 @@ namespace PBQP { return EId; } - /// \brief Returns true if the graph is empty. + /// @brief Add an edge bypassing the cost allocator. + /// @param N1Id First node. + /// @param N2Id Second node. + /// @param Costs Cost matrix for new edge. + /// @return Edge iterator for the added edge. + /// + /// This method allows for fast addition of an edge whose costs don't need + /// to be passed through the cost allocator. The most common use case for + /// this is when duplicating costs from an existing edge (when using a + /// pooling allocator). These have already been uniqued, so we can avoid + /// re-constructing and re-uniquing them by attaching them directly to the + /// new edge. + template + NodeId addEdgeBypassingCostAllocator(NodeId N1Id, NodeId N2Id, + OtherMatrixPtrT Costs) { + assert(getNodeCosts(N1Id).getLength() == Costs->getRows() && + getNodeCosts(N2Id).getLength() == Costs->getCols() && + "Matrix dimensions mismatch."); + // Get cost matrix from the problem domain. + EdgeId EId = addConstructedEdge(EdgeEntry(N1Id, N2Id, Costs)); + if (Solver) + Solver->handleAddEdge(EId); + return EId; + } + + /// @brief Returns true if the graph is empty. bool empty() const { return NodeIdSet(*this).empty(); } NodeIdSet nodeIds() const { return NodeIdSet(*this); } @@ -388,15 +457,15 @@ namespace PBQP { AdjEdgeIdSet adjEdgeIds(NodeId NId) { return AdjEdgeIdSet(getNode(NId)); } - /// \brief Get the number of nodes in the graph. + /// @brief Get the number of nodes in the graph. /// @return Number of nodes in the graph. unsigned getNumNodes() const { return NodeIdSet(*this).size(); } - /// \brief Get the number of edges in the graph. + /// @brief Get the number of edges in the graph. /// @return Number of edges in the graph. unsigned getNumEdges() const { return EdgeIdSet(*this).size(); } - /// \brief Set a node's cost vector. + /// @brief Set a node's cost vector. /// @param NId Node to update. /// @param Costs New costs to set. template @@ -407,11 +476,23 @@ namespace PBQP { getNode(NId).Costs = AllocatedCosts; } - /// \brief Get a node's cost vector (const version). + /// @brief Get a VectorPtr to a node's cost vector. Rarely useful - use + /// getNodeCosts where possible. + /// @param NId Node id. + /// @return VectorPtr to node cost vector. + /// + /// This method is primarily useful for duplicating costs quickly by + /// bypassing the cost allocator. See addNodeBypassingCostAllocator. Prefer + /// getNodeCosts when dealing with node cost values. + const VectorPtr& getNodeCostsPtr(NodeId NId) const { + return getNode(NId).Costs; + } + + /// @brief Get a node's cost vector. /// @param NId Node id. /// @return Node cost vector. const Vector& getNodeCosts(NodeId NId) const { - return *getNode(NId).Costs; + return *getNodeCostsPtr(NId); } NodeMetadata& getNodeMetadata(NodeId NId) { @@ -426,7 +507,7 @@ namespace PBQP { return getNode(NId).getAdjEdgeIds().size(); } - /// \brief Set an edge's cost matrix. + /// @brief Set an edge's cost matrix. /// @param EId Edge id. /// @param Costs New cost matrix. template @@ -437,34 +518,48 @@ namespace PBQP { getEdge(EId).Costs = AllocatedCosts; } - /// \brief Get an edge's cost matrix (const version). + /// @brief Get a MatrixPtr to a node's cost matrix. Rarely useful - use + /// getEdgeCosts where possible. + /// @param EId Edge id. + /// @return MatrixPtr to edge cost matrix. + /// + /// This method is primarily useful for duplicating costs quickly by + /// bypassing the cost allocator. See addNodeBypassingCostAllocator. Prefer + /// getEdgeCosts when dealing with edge cost values. + const MatrixPtr& getEdgeCostsPtr(EdgeId EId) const { + return getEdge(EId).Costs; + } + + /// @brief Get an edge's cost matrix. /// @param EId Edge id. /// @return Edge cost matrix. - const Matrix& getEdgeCosts(EdgeId EId) const { return *getEdge(EId).Costs; } - - EdgeMetadata& getEdgeMetadata(EdgeId NId) { - return getEdge(NId).Metadata; + const Matrix& getEdgeCosts(EdgeId EId) const { + return *getEdge(EId).Costs; } - const EdgeMetadata& getEdgeMetadata(EdgeId NId) const { - return getEdge(NId).Metadata; + EdgeMetadata& getEdgeMetadata(EdgeId EId) { + return getEdge(EId).Metadata; } - /// \brief Get the first node connected to this edge. + const EdgeMetadata& getEdgeMetadata(EdgeId EId) const { + return getEdge(EId).Metadata; + } + + /// @brief Get the first node connected to this edge. /// @param EId Edge id. /// @return The first node connected to the given edge. NodeId getEdgeNode1Id(EdgeId EId) { return getEdge(EId).getN1Id(); } - /// \brief Get the second node connected to this edge. + /// @brief Get the second node connected to this edge. /// @param EId Edge id. /// @return The second node connected to the given edge. NodeId getEdgeNode2Id(EdgeId EId) { return getEdge(EId).getN2Id(); } - /// \brief Get the "other" node connected to this edge. + /// @brief Get the "other" node connected to this edge. /// @param EId Edge id. /// @param NId Node id for the "given" node. /// @return The iterator for the "other" node connected to this edge. @@ -476,7 +571,7 @@ namespace PBQP { return E.getN1Id(); } - /// \brief Get the edge connecting two nodes. + /// @brief Get the edge connecting two nodes. /// @param N1Id First node id. /// @param N2Id Second node id. /// @return An id for edge (N1Id, N2Id) if such an edge exists, @@ -491,7 +586,7 @@ namespace PBQP { return invalidEdgeId(); } - /// \brief Remove a node from the graph. + /// @brief Remove a node from the graph. /// @param NId Node id. void removeNode(NodeId NId) { if (Solver) @@ -499,7 +594,7 @@ namespace PBQP { NodeEntry &N = getNode(NId); // TODO: Can this be for-each'd? for (AdjEdgeItr AEItr = N.adjEdgesBegin(), - AEEnd = N.adjEdgesEnd(); + AEEnd = N.adjEdgesEnd(); AEItr != AEEnd;) { EdgeId EId = *AEItr; ++AEItr; @@ -508,7 +603,7 @@ namespace PBQP { FreeNodeIds.push_back(NId); } - /// \brief Disconnect an edge from the given node. + /// @brief Disconnect an edge from the given node. /// /// Removes the given edge from the adjacency list of the given node. /// This operation leaves the edge in an 'asymmetric' state: It will no @@ -541,14 +636,14 @@ namespace PBQP { E.disconnectFrom(*this, NId); } - /// \brief Convenience method to disconnect all neighbours from the given + /// @brief Convenience method to disconnect all neighbours from the given /// node. void disconnectAllNeighborsFromNode(NodeId NId) { for (auto AEId : adjEdgeIds(NId)) disconnectEdge(AEId, getEdgeOtherNodeId(AEId, NId)); } - /// \brief Re-attach an edge to its nodes. + /// @brief Re-attach an edge to its nodes. /// /// Adds an edge that had been previously disconnected back into the /// adjacency set of the nodes that the edge connects. @@ -559,7 +654,7 @@ namespace PBQP { Solver->handleReconnectEdge(EId, NId); } - /// \brief Remove an edge from the graph. + /// @brief Remove an edge from the graph. /// @param EId Edge id. void removeEdge(EdgeId EId) { if (Solver) @@ -570,7 +665,7 @@ namespace PBQP { Edges[EId].invalidate(); } - /// \brief Remove all nodes and edges from the graph. + /// @brief Remove all nodes and edges from the graph. void clear() { Nodes.clear(); FreeNodeIds.clear(); @@ -578,9 +673,9 @@ namespace PBQP { FreeEdgeIds.clear(); } - /// \brief Dump a graph to an output stream. + /// @brief Dump a graph to an output stream. template - void dump(OStream &OS) { + void dumpToStream(OStream &OS) { OS << nodeIds().size() << " " << edgeIds().size() << "\n"; for (auto NId : nodeIds()) { @@ -613,7 +708,12 @@ namespace PBQP { } } - /// \brief Print a representation of this graph in DOT format. + /// @brief Dump this graph to dbgs(). + void dump() { + dumpToStream(dbgs()); + } + + /// @brief Print a representation of this graph in DOT format. /// @param OS Output stream to print on. template void printDot(OStream &OS) { @@ -637,6 +737,7 @@ namespace PBQP { } }; -} +} // namespace PBQP +} // namespace llvm #endif // LLVM_CODEGEN_PBQP_GRAPH_HPP diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/Math.h b/contrib/llvm/include/llvm/CodeGen/PBQP/Math.h index 69a9d83cc092..2792608e29cc 100644 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/Math.h +++ b/contrib/llvm/include/llvm/CodeGen/PBQP/Math.h @@ -10,17 +10,19 @@ #ifndef LLVM_CODEGEN_PBQP_MATH_H #define LLVM_CODEGEN_PBQP_MATH_H +#include "llvm/ADT/Hashing.h" #include #include #include +namespace llvm { namespace PBQP { typedef float PBQPNum; /// \brief PBQP Vector class. class Vector { - friend class VectorComparator; + friend hash_code hash_value(const Vector &); public: /// \brief Construct a PBQP vector of the given size. @@ -136,21 +138,12 @@ private: PBQPNum *Data; }; -class VectorComparator { -public: - bool operator()(const Vector &A, const Vector &B) { - if (A.Length < B.Length) - return true; - if (B.Length < A.Length) - return false; - char *AData = reinterpret_cast(A.Data); - char *BData = reinterpret_cast(B.Data); - return std::lexicographical_compare(AData, - AData + A.Length * sizeof(PBQPNum), - BData, - BData + A.Length * sizeof(PBQPNum)); - } -}; +/// \brief Return a hash_value for the given vector. +inline hash_code hash_value(const Vector &V) { + unsigned *VBegin = reinterpret_cast(V.Data); + unsigned *VEnd = reinterpret_cast(V.Data + V.Length); + return hash_combine(V.Length, hash_combine_range(VBegin, VEnd)); +} /// \brief Output a textual representation of the given vector on the given /// output stream. @@ -166,11 +159,10 @@ OStream& operator<<(OStream &OS, const Vector &V) { return OS; } - /// \brief PBQP Matrix class class Matrix { private: - friend class MatrixComparator; + friend hash_code hash_value(const Matrix &); public: /// \brief Construct a PBQP Matrix with the given dimensions. @@ -384,24 +376,12 @@ private: PBQPNum *Data; }; -class MatrixComparator { -public: - bool operator()(const Matrix &A, const Matrix &B) { - if (A.Rows < B.Rows) - return true; - if (B.Rows < A.Rows) - return false; - if (A.Cols < B.Cols) - return true; - if (B.Cols < A.Cols) - return false; - char *AData = reinterpret_cast(A.Data); - char *BData = reinterpret_cast(B.Data); - return std::lexicographical_compare( - AData, AData + (A.Rows * A.Cols * sizeof(PBQPNum)), - BData, BData + (A.Rows * A.Cols * sizeof(PBQPNum))); - } -}; +/// \brief Return a hash_code for the given matrix. +inline hash_code hash_value(const Matrix &M) { + unsigned *MBegin = reinterpret_cast(M.Data); + unsigned *MEnd = reinterpret_cast(M.Data + (M.Rows * M.Cols)); + return hash_combine(M.Rows, M.Cols, hash_combine_range(MBegin, MEnd)); +} /// \brief Output a textual representation of the given matrix on the given /// output stream. @@ -409,7 +389,7 @@ template OStream& operator<<(OStream &OS, const Matrix &M) { assert((M.getRows() != 0) && "Zero-row matrix badness."); for (unsigned i = 0; i < M.getRows(); ++i) - OS << M.getRowAsVector(i); + OS << M.getRowAsVector(i) << "\n"; return OS; } @@ -423,6 +403,11 @@ private: Metadata md; }; +template +inline hash_code hash_value(const MDVector &V) { + return hash_value(static_cast(V)); +} + template class MDMatrix : public Matrix { public: @@ -433,6 +418,12 @@ private: Metadata md; }; +template +inline hash_code hash_value(const MDMatrix &M) { + return hash_value(static_cast(M)); } +} // namespace PBQP +} // namespace llvm + #endif // LLVM_CODEGEN_PBQP_MATH_H diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h b/contrib/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h index a55a06033c4e..21fde4d8a5cd 100644 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h +++ b/contrib/llvm/include/llvm/CodeGen/PBQP/ReductionRules.h @@ -11,13 +11,14 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_REDUCTIONRULES_H -#define LLVM_REDUCTIONRULES_H +#ifndef LLVM_CODEGEN_PBQP_REDUCTIONRULES_H +#define LLVM_CODEGEN_PBQP_REDUCTIONRULES_H #include "Graph.h" #include "Math.h" #include "Solution.h" +namespace llvm { namespace PBQP { /// \brief Reduce a node of degree one. @@ -186,6 +187,7 @@ namespace PBQP { return s; } -} +} // namespace PBQP +} // namespace llvm -#endif // LLVM_REDUCTIONRULES_H +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/RegAllocSolver.h b/contrib/llvm/include/llvm/CodeGen/PBQP/RegAllocSolver.h deleted file mode 100644 index 977c34843bbd..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/RegAllocSolver.h +++ /dev/null @@ -1,359 +0,0 @@ -//===-- RegAllocSolver.h - Heuristic PBQP Solver for reg alloc --*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Heuristic PBQP solver for register allocation problems. This solver uses a -// graph reduction approach. Nodes of degree 0, 1 and 2 are eliminated with -// optimality-preserving rules (see ReductionRules.h). When no low-degree (<3) -// nodes are present, a heuristic derived from Brigg's graph coloring approach -// is used. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H -#define LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H - -#include "CostAllocator.h" -#include "Graph.h" -#include "ReductionRules.h" -#include "Solution.h" -#include "llvm/Support/ErrorHandling.h" -#include -#include - -namespace PBQP { - - namespace RegAlloc { - - /// \brief Metadata to speed allocatability test. - /// - /// Keeps track of the number of infinities in each row and column. - class MatrixMetadata { - private: - MatrixMetadata(const MatrixMetadata&); - void operator=(const MatrixMetadata&); - public: - MatrixMetadata(const PBQP::Matrix& M) - : WorstRow(0), WorstCol(0), - UnsafeRows(new bool[M.getRows() - 1]()), - UnsafeCols(new bool[M.getCols() - 1]()) { - - unsigned* ColCounts = new unsigned[M.getCols() - 1](); - - for (unsigned i = 1; i < M.getRows(); ++i) { - unsigned RowCount = 0; - for (unsigned j = 1; j < M.getCols(); ++j) { - if (M[i][j] == std::numeric_limits::infinity()) { - ++RowCount; - ++ColCounts[j - 1]; - UnsafeRows[i - 1] = true; - UnsafeCols[j - 1] = true; - } - } - WorstRow = std::max(WorstRow, RowCount); - } - unsigned WorstColCountForCurRow = - *std::max_element(ColCounts, ColCounts + M.getCols() - 1); - WorstCol = std::max(WorstCol, WorstColCountForCurRow); - delete[] ColCounts; - } - - ~MatrixMetadata() { - delete[] UnsafeRows; - delete[] UnsafeCols; - } - - unsigned getWorstRow() const { return WorstRow; } - unsigned getWorstCol() const { return WorstCol; } - const bool* getUnsafeRows() const { return UnsafeRows; } - const bool* getUnsafeCols() const { return UnsafeCols; } - - private: - unsigned WorstRow, WorstCol; - bool* UnsafeRows; - bool* UnsafeCols; - }; - - class NodeMetadata { - public: - typedef enum { Unprocessed, - OptimallyReducible, - ConservativelyAllocatable, - NotProvablyAllocatable } ReductionState; - - NodeMetadata() : RS(Unprocessed), DeniedOpts(0), OptUnsafeEdges(nullptr){} - ~NodeMetadata() { delete[] OptUnsafeEdges; } - - void setup(const Vector& Costs) { - NumOpts = Costs.getLength() - 1; - OptUnsafeEdges = new unsigned[NumOpts](); - } - - ReductionState getReductionState() const { return RS; } - void setReductionState(ReductionState RS) { this->RS = RS; } - - void handleAddEdge(const MatrixMetadata& MD, bool Transpose) { - DeniedOpts += Transpose ? MD.getWorstCol() : MD.getWorstRow(); - const bool* UnsafeOpts = - Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows(); - for (unsigned i = 0; i < NumOpts; ++i) - OptUnsafeEdges[i] += UnsafeOpts[i]; - } - - void handleRemoveEdge(const MatrixMetadata& MD, bool Transpose) { - DeniedOpts -= Transpose ? MD.getWorstCol() : MD.getWorstRow(); - const bool* UnsafeOpts = - Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows(); - for (unsigned i = 0; i < NumOpts; ++i) - OptUnsafeEdges[i] -= UnsafeOpts[i]; - } - - bool isConservativelyAllocatable() const { - return (DeniedOpts < NumOpts) || - (std::find(OptUnsafeEdges, OptUnsafeEdges + NumOpts, 0) != - OptUnsafeEdges + NumOpts); - } - - private: - ReductionState RS; - unsigned NumOpts; - unsigned DeniedOpts; - unsigned* OptUnsafeEdges; - }; - - class RegAllocSolverImpl { - private: - typedef PBQP::MDMatrix RAMatrix; - public: - typedef PBQP::Vector RawVector; - typedef PBQP::Matrix RawMatrix; - typedef PBQP::Vector Vector; - typedef RAMatrix Matrix; - typedef PBQP::PoolCostAllocator< - Vector, PBQP::VectorComparator, - Matrix, PBQP::MatrixComparator> CostAllocator; - - typedef PBQP::GraphBase::NodeId NodeId; - typedef PBQP::GraphBase::EdgeId EdgeId; - - typedef RegAlloc::NodeMetadata NodeMetadata; - - struct EdgeMetadata { }; - - typedef PBQP::Graph Graph; - - RegAllocSolverImpl(Graph &G) : G(G) {} - - Solution solve() { - G.setSolver(*this); - Solution S; - setup(); - S = backpropagate(G, reduce()); - G.unsetSolver(); - return S; - } - - void handleAddNode(NodeId NId) { - G.getNodeMetadata(NId).setup(G.getNodeCosts(NId)); - } - void handleRemoveNode(NodeId NId) {} - void handleSetNodeCosts(NodeId NId, const Vector& newCosts) {} - - void handleAddEdge(EdgeId EId) { - handleReconnectEdge(EId, G.getEdgeNode1Id(EId)); - handleReconnectEdge(EId, G.getEdgeNode2Id(EId)); - } - - void handleRemoveEdge(EdgeId EId) { - handleDisconnectEdge(EId, G.getEdgeNode1Id(EId)); - handleDisconnectEdge(EId, G.getEdgeNode2Id(EId)); - } - - void handleDisconnectEdge(EdgeId EId, NodeId NId) { - NodeMetadata& NMd = G.getNodeMetadata(NId); - const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); - NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId)); - if (G.getNodeDegree(NId) == 3) { - // This node is becoming optimally reducible. - moveToOptimallyReducibleNodes(NId); - } else if (NMd.getReductionState() == - NodeMetadata::NotProvablyAllocatable && - NMd.isConservativelyAllocatable()) { - // This node just became conservatively allocatable. - moveToConservativelyAllocatableNodes(NId); - } - } - - void handleReconnectEdge(EdgeId EId, NodeId NId) { - NodeMetadata& NMd = G.getNodeMetadata(NId); - const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); - NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId)); - } - - void handleSetEdgeCosts(EdgeId EId, const Matrix& NewCosts) { - handleRemoveEdge(EId); - - NodeId N1Id = G.getEdgeNode1Id(EId); - NodeId N2Id = G.getEdgeNode2Id(EId); - NodeMetadata& N1Md = G.getNodeMetadata(N1Id); - NodeMetadata& N2Md = G.getNodeMetadata(N2Id); - const MatrixMetadata& MMd = NewCosts.getMetadata(); - N1Md.handleAddEdge(MMd, N1Id != G.getEdgeNode1Id(EId)); - N2Md.handleAddEdge(MMd, N2Id != G.getEdgeNode1Id(EId)); - } - - private: - - void removeFromCurrentSet(NodeId NId) { - switch (G.getNodeMetadata(NId).getReductionState()) { - case NodeMetadata::Unprocessed: break; - case NodeMetadata::OptimallyReducible: - assert(OptimallyReducibleNodes.find(NId) != - OptimallyReducibleNodes.end() && - "Node not in optimally reducible set."); - OptimallyReducibleNodes.erase(NId); - break; - case NodeMetadata::ConservativelyAllocatable: - assert(ConservativelyAllocatableNodes.find(NId) != - ConservativelyAllocatableNodes.end() && - "Node not in conservatively allocatable set."); - ConservativelyAllocatableNodes.erase(NId); - break; - case NodeMetadata::NotProvablyAllocatable: - assert(NotProvablyAllocatableNodes.find(NId) != - NotProvablyAllocatableNodes.end() && - "Node not in not-provably-allocatable set."); - NotProvablyAllocatableNodes.erase(NId); - break; - } - } - - void moveToOptimallyReducibleNodes(NodeId NId) { - removeFromCurrentSet(NId); - OptimallyReducibleNodes.insert(NId); - G.getNodeMetadata(NId).setReductionState( - NodeMetadata::OptimallyReducible); - } - - void moveToConservativelyAllocatableNodes(NodeId NId) { - removeFromCurrentSet(NId); - ConservativelyAllocatableNodes.insert(NId); - G.getNodeMetadata(NId).setReductionState( - NodeMetadata::ConservativelyAllocatable); - } - - void moveToNotProvablyAllocatableNodes(NodeId NId) { - removeFromCurrentSet(NId); - NotProvablyAllocatableNodes.insert(NId); - G.getNodeMetadata(NId).setReductionState( - NodeMetadata::NotProvablyAllocatable); - } - - void setup() { - // Set up worklists. - for (auto NId : G.nodeIds()) { - if (G.getNodeDegree(NId) < 3) - moveToOptimallyReducibleNodes(NId); - else if (G.getNodeMetadata(NId).isConservativelyAllocatable()) - moveToConservativelyAllocatableNodes(NId); - else - moveToNotProvablyAllocatableNodes(NId); - } - } - - // Compute a reduction order for the graph by iteratively applying PBQP - // reduction rules. Locally optimal rules are applied whenever possible (R0, - // R1, R2). If no locally-optimal rules apply then any conservatively - // allocatable node is reduced. Finally, if no conservatively allocatable - // node exists then the node with the lowest spill-cost:degree ratio is - // selected. - std::vector reduce() { - assert(!G.empty() && "Cannot reduce empty graph."); - - typedef GraphBase::NodeId NodeId; - std::vector NodeStack; - - // Consume worklists. - while (true) { - if (!OptimallyReducibleNodes.empty()) { - NodeSet::iterator NItr = OptimallyReducibleNodes.begin(); - NodeId NId = *NItr; - OptimallyReducibleNodes.erase(NItr); - NodeStack.push_back(NId); - switch (G.getNodeDegree(NId)) { - case 0: - break; - case 1: - applyR1(G, NId); - break; - case 2: - applyR2(G, NId); - break; - default: llvm_unreachable("Not an optimally reducible node."); - } - } else if (!ConservativelyAllocatableNodes.empty()) { - // Conservatively allocatable nodes will never spill. For now just - // take the first node in the set and push it on the stack. When we - // start optimizing more heavily for register preferencing, it may - // would be better to push nodes with lower 'expected' or worst-case - // register costs first (since early nodes are the most - // constrained). - NodeSet::iterator NItr = ConservativelyAllocatableNodes.begin(); - NodeId NId = *NItr; - ConservativelyAllocatableNodes.erase(NItr); - NodeStack.push_back(NId); - G.disconnectAllNeighborsFromNode(NId); - - } else if (!NotProvablyAllocatableNodes.empty()) { - NodeSet::iterator NItr = - std::min_element(NotProvablyAllocatableNodes.begin(), - NotProvablyAllocatableNodes.end(), - SpillCostComparator(G)); - NodeId NId = *NItr; - NotProvablyAllocatableNodes.erase(NItr); - NodeStack.push_back(NId); - G.disconnectAllNeighborsFromNode(NId); - } else - break; - } - - return NodeStack; - } - - class SpillCostComparator { - public: - SpillCostComparator(const Graph& G) : G(G) {} - bool operator()(NodeId N1Id, NodeId N2Id) { - PBQPNum N1SC = G.getNodeCosts(N1Id)[0] / G.getNodeDegree(N1Id); - PBQPNum N2SC = G.getNodeCosts(N2Id)[0] / G.getNodeDegree(N2Id); - return N1SC < N2SC; - } - private: - const Graph& G; - }; - - Graph& G; - typedef std::set NodeSet; - NodeSet OptimallyReducibleNodes; - NodeSet ConservativelyAllocatableNodes; - NodeSet NotProvablyAllocatableNodes; - }; - - typedef Graph Graph; - - inline Solution solve(Graph& G) { - if (G.empty()) - return Solution(); - RegAllocSolverImpl RegAllocSolver(G); - return RegAllocSolver.solve(); - } - - } -} - -#endif // LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h b/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h index 3556e60f3967..a3bfaeb7e6c7 100644 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h +++ b/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h @@ -18,6 +18,7 @@ #include "Math.h" #include +namespace llvm { namespace PBQP { /// \brief Represents a solution to a PBQP problem. @@ -87,6 +88,7 @@ namespace PBQP { }; -} +} // namespace PBQP +} // namespace llvm #endif // LLVM_CODEGEN_PBQP_SOLUTION_H diff --git a/contrib/llvm/include/llvm/CodeGen/PBQPRAConstraint.h b/contrib/llvm/include/llvm/CodeGen/PBQPRAConstraint.h new file mode 100644 index 000000000000..833b9bad613f --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/PBQPRAConstraint.h @@ -0,0 +1,69 @@ +//===-- RegAllocPBQP.h ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the PBQPBuilder interface, for classes which build PBQP +// instances to represent register allocation problems, and the RegAllocPBQP +// interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_PBQPRACONSTRAINT_H +#define LLVM_CODEGEN_PBQPRACONSTRAINT_H + +#include +#include + +namespace llvm { +namespace PBQP { +namespace RegAlloc { +// Forward declare PBQP graph class. +class PBQPRAGraph; +} +} + +class LiveIntervals; +class MachineBlockFrequencyInfo; +class MachineFunction; +class TargetRegisterInfo; + +typedef PBQP::RegAlloc::PBQPRAGraph PBQPRAGraph; + +/// @brief Abstract base for classes implementing PBQP register allocation +/// constraints (e.g. Spill-costs, interference, coalescing). +class PBQPRAConstraint { +public: + virtual ~PBQPRAConstraint() = 0; + virtual void apply(PBQPRAGraph &G) = 0; +private: + virtual void anchor(); +}; + +/// @brief PBQP register allocation constraint composer. +/// +/// Constraints added to this list will be applied, in the order that they are +/// added, to the PBQP graph. +class PBQPRAConstraintList : public PBQPRAConstraint { +public: + void apply(PBQPRAGraph &G) override { + for (auto &C : Constraints) + C->apply(G); + } + + void addConstraint(std::unique_ptr C) { + if (C) + Constraints.push_back(std::move(C)); + } +private: + std::vector> Constraints; + void anchor() override; +}; + +} + +#endif /* LLVM_CODEGEN_PBQPRACONSTRAINT_H */ diff --git a/contrib/llvm/include/llvm/CodeGen/Passes.h b/contrib/llvm/include/llvm/CodeGen/Passes.h index 87f55e8572fe..8ed32b8a8dd5 100644 --- a/contrib/llvm/include/llvm/CodeGen/Passes.h +++ b/contrib/llvm/include/llvm/CodeGen/Passes.h @@ -105,6 +105,7 @@ private: AnalysisID StopAfter; bool Started; bool Stopped; + bool AddingMachinePasses; protected: TargetMachine *TM; @@ -178,6 +179,10 @@ public: /// Return true if the optimized regalloc pipeline is enabled. bool getOptimizeRegAlloc() const; + /// Return true if the default global register allocator is in use and + /// has not be overriden on the command line with '-regalloc=...' + bool usingDefaultRegAlloc() const; + /// Add common target configurable passes that perform LLVM IR to IR /// transforms following machine independent optimization. virtual void addIRPasses(); @@ -255,12 +260,9 @@ protected: return false; } - /// addPreRegAlloc - This method may be implemented by targets that want to - /// run passes immediately before register allocation. This should return - /// true if -print-machineinstrs should print after these passes. - virtual bool addPreRegAlloc() { - return false; - } + /// This method may be implemented by targets that want to run passes + /// immediately before register allocation. + virtual void addPreRegAlloc() { } /// createTargetRegisterAllocator - Create the register allocator pass for /// this target at the current optimization level. @@ -286,24 +288,16 @@ protected: return false; } - /// addPostRegAlloc - This method may be implemented by targets that want to - /// run passes after register allocation pass pipeline but before - /// prolog-epilog insertion. This should return true if -print-machineinstrs - /// should print after these passes. - virtual bool addPostRegAlloc() { - return false; - } + /// This method may be implemented by targets that want to run passes after + /// register allocation pass pipeline but before prolog-epilog insertion. + virtual void addPostRegAlloc() { } /// Add passes that optimize machine instructions after register allocation. virtual void addMachineLateOptimization(); - /// addPreSched2 - This method may be implemented by targets that want to - /// run passes after prolog-epilog insertion and before the second instruction - /// scheduling pass. This should return true if -print-machineinstrs should - /// print after these passes. - virtual bool addPreSched2() { - return false; - } + /// This method may be implemented by targets that want to run passes after + /// prolog-epilog insertion and before the second instruction scheduling pass. + virtual void addPreSched2() { } /// addGCPasses - Add late codegen passes that analyze code for garbage /// collection. This should return true if GC info should be printed after @@ -313,24 +307,30 @@ protected: /// Add standard basic block placement passes. virtual void addBlockPlacement(); - /// addPreEmitPass - This pass may be implemented by targets that want to run - /// passes immediately before machine code is emitted. This should return - /// true if -print-machineinstrs should print out the code after the passes. - virtual bool addPreEmitPass() { - return false; - } + /// This pass may be implemented by targets that want to run passes + /// immediately before machine code is emitted. + virtual void addPreEmitPass() { } /// Utilities for targets to add passes to the pass manager. /// /// Add a CodeGen pass at this point in the pipeline after checking overrides. /// Return the pass that was added, or zero if no pass was added. - AnalysisID addPass(AnalysisID PassID); + /// @p printAfter if true and adding a machine function pass add an extra + /// machine printer pass afterwards + /// @p verifyAfter if true and adding a machine function pass add an extra + /// machine verification pass afterwards. + AnalysisID addPass(AnalysisID PassID, bool verifyAfter = true, + bool printAfter = true); /// Add a pass to the PassManager if that pass is supposed to be run, as /// determined by the StartAfter and StopAfter options. Takes ownership of the /// pass. - void addPass(Pass *P); + /// @p printAfter if true and adding a machine function pass add an extra + /// machine printer pass afterwards + /// @p verifyAfter if true and adding a machine function pass add an extra + /// machine verification pass afterwards. + void addPass(Pass *P, bool verifyAfter = true, bool printAfter = true); /// addMachinePasses helper to create the target-selected or overriden /// regalloc pass. @@ -339,13 +339,20 @@ protected: /// printAndVerify - Add a pass to dump then verify the machine function, if /// those steps are enabled. /// - void printAndVerify(const char *Banner); + void printAndVerify(const std::string &Banner); + + /// Add a pass to print the machine function if printing is enabled. + void addPrintPass(const std::string &Banner); + + /// Add a pass to perform basic verification of the machine function if + /// verification is enabled. + void addVerifyPass(const std::string &Banner); }; } // namespace llvm /// List of target independent CodeGen pass IDs. namespace llvm { - FunctionPass *createAtomicExpandLoadLinkedPass(const TargetMachine *TM); + FunctionPass *createAtomicExpandPass(const TargetMachine *TM); /// \brief Create a basic TargetTransformInfo analysis pass. /// @@ -372,8 +379,9 @@ namespace llvm { /// matching during instruction selection. FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = nullptr); - /// AtomicExpandLoadLinkedID -- FIXME - extern char &AtomicExpandLoadLinkedID; + /// AtomicExpandID -- Lowers atomic operations in terms of either cmpxchg + /// load-linked/store-conditional loops. + extern char &AtomicExpandID; /// MachineLoopInfo - This pass is a loop analysis pass. extern char &MachineLoopInfoID; @@ -489,6 +497,10 @@ namespace llvm { /// inserting cmov instructions. extern char &EarlyIfConverterID; + /// This pass performs instruction combining using trace metrics to estimate + /// critical-path and resource depth. + extern char &MachineCombinerID; + /// StackSlotColoring - This pass performs stack coloring and merging. /// It merges disjoint allocas to reduce the stack size. extern char &StackColoringID; @@ -551,7 +563,7 @@ namespace llvm { /// createMachineVerifierPass - This pass verifies cenerated machine code /// instructions for correctness. /// - FunctionPass *createMachineVerifierPass(const char *Banner = nullptr); + FunctionPass *createMachineVerifierPass(const std::string& Banner); /// createDwarfEHPass - This pass mulches exception handling code into a form /// adapted to code generation. Required if using dwarf exception handling. @@ -593,6 +605,10 @@ namespace llvm { /// createJumpInstrTables - This pass creates jump-instruction tables. ModulePass *createJumpInstrTablesPass(); + + /// createForwardControlFlowIntegrityPass - This pass adds control-flow + /// integrity. + ModulePass *createForwardControlFlowIntegrityPass(); } // End llvm namespace /// This initializer registers TargetMachine constructor, so the pass being diff --git a/contrib/llvm/include/llvm/CodeGen/RegAllocPBQP.h b/contrib/llvm/include/llvm/CodeGen/RegAllocPBQP.h index 441b0f084e69..5c0e9859915f 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegAllocPBQP.h +++ b/contrib/llvm/include/llvm/CodeGen/RegAllocPBQP.h @@ -16,150 +16,505 @@ #ifndef LLVM_CODEGEN_REGALLOCPBQP_H #define LLVM_CODEGEN_REGALLOCPBQP_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/PBQP/RegAllocSolver.h" -#include -#include +#include "llvm/CodeGen/PBQP/CostAllocator.h" +#include "llvm/CodeGen/PBQP/ReductionRules.h" +#include "llvm/CodeGen/PBQPRAConstraint.h" +#include "llvm/Support/ErrorHandling.h" namespace llvm { +namespace PBQP { +namespace RegAlloc { - class LiveIntervals; - class MachineBlockFrequencyInfo; - class MachineFunction; - class TargetRegisterInfo; +/// @brief Spill option index. +inline unsigned getSpillOptionIdx() { return 0; } - typedef PBQP::RegAlloc::Graph PBQPRAGraph; +/// \brief Metadata to speed allocatability test. +/// +/// Keeps track of the number of infinities in each row and column. +class MatrixMetadata { +private: + MatrixMetadata(const MatrixMetadata&); + void operator=(const MatrixMetadata&); +public: + MatrixMetadata(const Matrix& M) + : WorstRow(0), WorstCol(0), + UnsafeRows(new bool[M.getRows() - 1]()), + UnsafeCols(new bool[M.getCols() - 1]()) { - /// This class wraps up a PBQP instance representing a register allocation - /// problem, plus the structures necessary to map back from the PBQP solution - /// to a register allocation solution. (i.e. The PBQP-node <--> vreg map, - /// and the PBQP option <--> storage location map). - class PBQPRAProblem { - public: + unsigned* ColCounts = new unsigned[M.getCols() - 1](); - typedef SmallVector AllowedSet; - - PBQPRAGraph& getGraph() { return graph; } - - const PBQPRAGraph& getGraph() const { return graph; } - - /// Record the mapping between the given virtual register and PBQP node, - /// and the set of allowed pregs for the vreg. - /// - /// If you are extending - /// PBQPBuilder you are unlikely to need this: Nodes and options for all - /// vregs will already have been set up for you by the base class. - template - void recordVReg(unsigned vreg, PBQPRAGraph::NodeId nodeId, - AllowedRegsItr arBegin, AllowedRegsItr arEnd) { - assert(node2VReg.find(nodeId) == node2VReg.end() && "Re-mapping node."); - assert(vreg2Node.find(vreg) == vreg2Node.end() && "Re-mapping vreg."); - assert(allowedSets[vreg].empty() && "vreg already has pregs."); - - node2VReg[nodeId] = vreg; - vreg2Node[vreg] = nodeId; - std::copy(arBegin, arEnd, std::back_inserter(allowedSets[vreg])); + for (unsigned i = 1; i < M.getRows(); ++i) { + unsigned RowCount = 0; + for (unsigned j = 1; j < M.getCols(); ++j) { + if (M[i][j] == std::numeric_limits::infinity()) { + ++RowCount; + ++ColCounts[j - 1]; + UnsafeRows[i - 1] = true; + UnsafeCols[j - 1] = true; + } + } + WorstRow = std::max(WorstRow, RowCount); } + unsigned WorstColCountForCurRow = + *std::max_element(ColCounts, ColCounts + M.getCols() - 1); + WorstCol = std::max(WorstCol, WorstColCountForCurRow); + delete[] ColCounts; + } - /// Get the virtual register corresponding to the given PBQP node. - unsigned getVRegForNode(PBQPRAGraph::NodeId nodeId) const; + unsigned getWorstRow() const { return WorstRow; } + unsigned getWorstCol() const { return WorstCol; } + const bool* getUnsafeRows() const { return UnsafeRows.get(); } + const bool* getUnsafeCols() const { return UnsafeCols.get(); } - /// Get the PBQP node corresponding to the given virtual register. - PBQPRAGraph::NodeId getNodeForVReg(unsigned vreg) const; +private: + unsigned WorstRow, WorstCol; + std::unique_ptr UnsafeRows; + std::unique_ptr UnsafeCols; +}; - /// Returns true if the given PBQP option represents a physical register, - /// false otherwise. - bool isPRegOption(unsigned vreg, unsigned option) const { - // At present we only have spills or pregs, so anything that's not a - // spill is a preg. (This might be extended one day to support remat). - return !isSpillOption(vreg, option); - } +/// \brief Holds a vector of the allowed physical regs for a vreg. +class AllowedRegVector { + friend hash_code hash_value(const AllowedRegVector &); +public: - /// Returns true if the given PBQP option represents spilling, false - /// otherwise. - bool isSpillOption(unsigned vreg, unsigned option) const { - // We hardcode option zero as the spill option. - return option == 0; - } + AllowedRegVector() : NumOpts(0), Opts(nullptr) {} - /// Returns the allowed set for the given virtual register. - const AllowedSet& getAllowedSet(unsigned vreg) const; + AllowedRegVector(const std::vector &OptVec) + : NumOpts(OptVec.size()), Opts(new unsigned[NumOpts]) { + std::copy(OptVec.begin(), OptVec.end(), Opts.get()); + } - /// Get PReg for option. - unsigned getPRegForOption(unsigned vreg, unsigned option) const; + AllowedRegVector(const AllowedRegVector &Other) + : NumOpts(Other.NumOpts), Opts(new unsigned[NumOpts]) { + std::copy(Other.Opts.get(), Other.Opts.get() + NumOpts, Opts.get()); + } - private: + AllowedRegVector(AllowedRegVector &&Other) + : NumOpts(std::move(Other.NumOpts)), Opts(std::move(Other.Opts)) {} - typedef std::map Node2VReg; - typedef DenseMap VReg2Node; - typedef DenseMap AllowedSetMap; + AllowedRegVector& operator=(const AllowedRegVector &Other) { + NumOpts = Other.NumOpts; + Opts.reset(new unsigned[NumOpts]); + std::copy(Other.Opts.get(), Other.Opts.get() + NumOpts, Opts.get()); + return *this; + } - PBQPRAGraph graph; - Node2VReg node2VReg; - VReg2Node vreg2Node; + AllowedRegVector& operator=(AllowedRegVector &&Other) { + NumOpts = std::move(Other.NumOpts); + Opts = std::move(Other.Opts); + return *this; + } - AllowedSetMap allowedSets; + unsigned size() const { return NumOpts; } + unsigned operator[](size_t I) const { return Opts[I]; } - }; + bool operator==(const AllowedRegVector &Other) const { + if (NumOpts != Other.NumOpts) + return false; + return std::equal(Opts.get(), Opts.get() + NumOpts, Other.Opts.get()); + } - /// Builds PBQP instances to represent register allocation problems. Includes - /// spill, interference and coalescing costs by default. You can extend this - /// class to support additional constraints for your architecture. - class PBQPBuilder { - private: - PBQPBuilder(const PBQPBuilder&) LLVM_DELETED_FUNCTION; - void operator=(const PBQPBuilder&) LLVM_DELETED_FUNCTION; - public: + bool operator!=(const AllowedRegVector &Other) const { + return !(*this == Other); + } - typedef std::set RegSet; +private: + unsigned NumOpts; + std::unique_ptr Opts; +}; - /// Default constructor. - PBQPBuilder() {} - - /// Clean up a PBQPBuilder. - virtual ~PBQPBuilder() {} - - /// Build a PBQP instance to represent the register allocation problem for - /// the given MachineFunction. - virtual PBQPRAProblem *build(MachineFunction *mf, const LiveIntervals *lis, - const MachineBlockFrequencyInfo *mbfi, - const RegSet &vregs); - private: - - void addSpillCosts(PBQP::Vector &costVec, PBQP::PBQPNum spillCost); - - void addInterferenceCosts(PBQP::Matrix &costMat, - const PBQPRAProblem::AllowedSet &vr1Allowed, - const PBQPRAProblem::AllowedSet &vr2Allowed, - const TargetRegisterInfo *tri); - }; - - /// Extended builder which adds coalescing constraints to a problem. - class PBQPBuilderWithCoalescing : public PBQPBuilder { - public: - - /// Build a PBQP instance to represent the register allocation problem for - /// the given MachineFunction. - PBQPRAProblem *build(MachineFunction *mf, const LiveIntervals *lis, - const MachineBlockFrequencyInfo *mbfi, - const RegSet &vregs) override; - - private: - - void addPhysRegCoalesce(PBQP::Vector &costVec, unsigned pregOption, - PBQP::PBQPNum benefit); - - void addVirtRegCoalesce(PBQP::Matrix &costMat, - const PBQPRAProblem::AllowedSet &vr1Allowed, - const PBQPRAProblem::AllowedSet &vr2Allowed, - PBQP::PBQPNum benefit); - }; - - FunctionPass * - createPBQPRegisterAllocator(std::unique_ptr builder, - char *customPassID = nullptr); +inline hash_code hash_value(const AllowedRegVector &OptRegs) { + unsigned *OStart = OptRegs.Opts.get(); + unsigned *OEnd = OptRegs.Opts.get() + OptRegs.NumOpts; + return hash_combine(OptRegs.NumOpts, + hash_combine_range(OStart, OEnd)); } +/// \brief Holds graph-level metadata relevent to PBQP RA problems. +class GraphMetadata { +private: + typedef ValuePool AllowedRegVecPool; +public: + + typedef AllowedRegVecPool::PoolRef AllowedRegVecRef; + + GraphMetadata(MachineFunction &MF, + LiveIntervals &LIS, + MachineBlockFrequencyInfo &MBFI) + : MF(MF), LIS(LIS), MBFI(MBFI) {} + + MachineFunction &MF; + LiveIntervals &LIS; + MachineBlockFrequencyInfo &MBFI; + + void setNodeIdForVReg(unsigned VReg, GraphBase::NodeId NId) { + VRegToNodeId[VReg] = NId; + } + + GraphBase::NodeId getNodeIdForVReg(unsigned VReg) const { + auto VRegItr = VRegToNodeId.find(VReg); + if (VRegItr == VRegToNodeId.end()) + return GraphBase::invalidNodeId(); + return VRegItr->second; + } + + void eraseNodeIdForVReg(unsigned VReg) { + VRegToNodeId.erase(VReg); + } + + AllowedRegVecRef getAllowedRegs(AllowedRegVector Allowed) { + return AllowedRegVecs.getValue(std::move(Allowed)); + } + +private: + DenseMap VRegToNodeId; + AllowedRegVecPool AllowedRegVecs; +}; + +/// \brief Holds solver state and other metadata relevant to each PBQP RA node. +class NodeMetadata { +public: + typedef RegAlloc::AllowedRegVector AllowedRegVector; + + typedef enum { Unprocessed, + OptimallyReducible, + ConservativelyAllocatable, + NotProvablyAllocatable } ReductionState; + + NodeMetadata() + : RS(Unprocessed), NumOpts(0), DeniedOpts(0), OptUnsafeEdges(nullptr), + VReg(0) {} + + // FIXME: Re-implementing default behavior to work around MSVC. Remove once + // MSVC synthesizes move constructors properly. + NodeMetadata(const NodeMetadata &Other) + : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts), + OptUnsafeEdges(new unsigned[NumOpts]), VReg(Other.VReg), + AllowedRegs(Other.AllowedRegs) { + if (NumOpts > 0) { + std::copy(&Other.OptUnsafeEdges[0], &Other.OptUnsafeEdges[NumOpts], + &OptUnsafeEdges[0]); + } + } + + // FIXME: Re-implementing default behavior to work around MSVC. Remove once + // MSVC synthesizes move constructors properly. + NodeMetadata(NodeMetadata &&Other) + : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts), + OptUnsafeEdges(std::move(Other.OptUnsafeEdges)), VReg(Other.VReg), + AllowedRegs(std::move(Other.AllowedRegs)) {} + + // FIXME: Re-implementing default behavior to work around MSVC. Remove once + // MSVC synthesizes move constructors properly. + NodeMetadata& operator=(const NodeMetadata &Other) { + RS = Other.RS; + NumOpts = Other.NumOpts; + DeniedOpts = Other.DeniedOpts; + OptUnsafeEdges.reset(new unsigned[NumOpts]); + std::copy(Other.OptUnsafeEdges.get(), Other.OptUnsafeEdges.get() + NumOpts, + OptUnsafeEdges.get()); + VReg = Other.VReg; + AllowedRegs = Other.AllowedRegs; + return *this; + } + + // FIXME: Re-implementing default behavior to work around MSVC. Remove once + // MSVC synthesizes move constructors properly. + NodeMetadata& operator=(NodeMetadata &&Other) { + RS = Other.RS; + NumOpts = Other.NumOpts; + DeniedOpts = Other.DeniedOpts; + OptUnsafeEdges = std::move(Other.OptUnsafeEdges); + VReg = Other.VReg; + AllowedRegs = std::move(Other.AllowedRegs); + return *this; + } + + void setVReg(unsigned VReg) { this->VReg = VReg; } + unsigned getVReg() const { return VReg; } + + void setAllowedRegs(GraphMetadata::AllowedRegVecRef AllowedRegs) { + this->AllowedRegs = std::move(AllowedRegs); + } + const AllowedRegVector& getAllowedRegs() const { return *AllowedRegs; } + + void setup(const Vector& Costs) { + NumOpts = Costs.getLength() - 1; + OptUnsafeEdges = std::unique_ptr(new unsigned[NumOpts]()); + } + + ReductionState getReductionState() const { return RS; } + void setReductionState(ReductionState RS) { this->RS = RS; } + + void handleAddEdge(const MatrixMetadata& MD, bool Transpose) { + DeniedOpts += Transpose ? MD.getWorstRow() : MD.getWorstCol(); + const bool* UnsafeOpts = + Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows(); + for (unsigned i = 0; i < NumOpts; ++i) + OptUnsafeEdges[i] += UnsafeOpts[i]; + } + + void handleRemoveEdge(const MatrixMetadata& MD, bool Transpose) { + DeniedOpts -= Transpose ? MD.getWorstRow() : MD.getWorstCol(); + const bool* UnsafeOpts = + Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows(); + for (unsigned i = 0; i < NumOpts; ++i) + OptUnsafeEdges[i] -= UnsafeOpts[i]; + } + + bool isConservativelyAllocatable() const { + return (DeniedOpts < NumOpts) || + (std::find(&OptUnsafeEdges[0], &OptUnsafeEdges[NumOpts], 0) != + &OptUnsafeEdges[NumOpts]); + } + +private: + ReductionState RS; + unsigned NumOpts; + unsigned DeniedOpts; + std::unique_ptr OptUnsafeEdges; + unsigned VReg; + GraphMetadata::AllowedRegVecRef AllowedRegs; +}; + +class RegAllocSolverImpl { +private: + typedef MDMatrix RAMatrix; +public: + typedef PBQP::Vector RawVector; + typedef PBQP::Matrix RawMatrix; + typedef PBQP::Vector Vector; + typedef RAMatrix Matrix; + typedef PBQP::PoolCostAllocator CostAllocator; + + typedef GraphBase::NodeId NodeId; + typedef GraphBase::EdgeId EdgeId; + + typedef RegAlloc::NodeMetadata NodeMetadata; + struct EdgeMetadata { }; + typedef RegAlloc::GraphMetadata GraphMetadata; + + typedef PBQP::Graph Graph; + + RegAllocSolverImpl(Graph &G) : G(G) {} + + Solution solve() { + G.setSolver(*this); + Solution S; + setup(); + S = backpropagate(G, reduce()); + G.unsetSolver(); + return S; + } + + void handleAddNode(NodeId NId) { + G.getNodeMetadata(NId).setup(G.getNodeCosts(NId)); + } + void handleRemoveNode(NodeId NId) {} + void handleSetNodeCosts(NodeId NId, const Vector& newCosts) {} + + void handleAddEdge(EdgeId EId) { + handleReconnectEdge(EId, G.getEdgeNode1Id(EId)); + handleReconnectEdge(EId, G.getEdgeNode2Id(EId)); + } + + void handleRemoveEdge(EdgeId EId) { + handleDisconnectEdge(EId, G.getEdgeNode1Id(EId)); + handleDisconnectEdge(EId, G.getEdgeNode2Id(EId)); + } + + void handleDisconnectEdge(EdgeId EId, NodeId NId) { + NodeMetadata& NMd = G.getNodeMetadata(NId); + const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); + NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId)); + if (G.getNodeDegree(NId) == 3) { + // This node is becoming optimally reducible. + moveToOptimallyReducibleNodes(NId); + } else if (NMd.getReductionState() == + NodeMetadata::NotProvablyAllocatable && + NMd.isConservativelyAllocatable()) { + // This node just became conservatively allocatable. + moveToConservativelyAllocatableNodes(NId); + } + } + + void handleReconnectEdge(EdgeId EId, NodeId NId) { + NodeMetadata& NMd = G.getNodeMetadata(NId); + const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); + NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId)); + } + + void handleSetEdgeCosts(EdgeId EId, const Matrix& NewCosts) { + handleRemoveEdge(EId); + + NodeId N1Id = G.getEdgeNode1Id(EId); + NodeId N2Id = G.getEdgeNode2Id(EId); + NodeMetadata& N1Md = G.getNodeMetadata(N1Id); + NodeMetadata& N2Md = G.getNodeMetadata(N2Id); + const MatrixMetadata& MMd = NewCosts.getMetadata(); + N1Md.handleAddEdge(MMd, N1Id != G.getEdgeNode1Id(EId)); + N2Md.handleAddEdge(MMd, N2Id != G.getEdgeNode1Id(EId)); + } + +private: + + void removeFromCurrentSet(NodeId NId) { + switch (G.getNodeMetadata(NId).getReductionState()) { + case NodeMetadata::Unprocessed: break; + case NodeMetadata::OptimallyReducible: + assert(OptimallyReducibleNodes.find(NId) != + OptimallyReducibleNodes.end() && + "Node not in optimally reducible set."); + OptimallyReducibleNodes.erase(NId); + break; + case NodeMetadata::ConservativelyAllocatable: + assert(ConservativelyAllocatableNodes.find(NId) != + ConservativelyAllocatableNodes.end() && + "Node not in conservatively allocatable set."); + ConservativelyAllocatableNodes.erase(NId); + break; + case NodeMetadata::NotProvablyAllocatable: + assert(NotProvablyAllocatableNodes.find(NId) != + NotProvablyAllocatableNodes.end() && + "Node not in not-provably-allocatable set."); + NotProvablyAllocatableNodes.erase(NId); + break; + } + } + + void moveToOptimallyReducibleNodes(NodeId NId) { + removeFromCurrentSet(NId); + OptimallyReducibleNodes.insert(NId); + G.getNodeMetadata(NId).setReductionState( + NodeMetadata::OptimallyReducible); + } + + void moveToConservativelyAllocatableNodes(NodeId NId) { + removeFromCurrentSet(NId); + ConservativelyAllocatableNodes.insert(NId); + G.getNodeMetadata(NId).setReductionState( + NodeMetadata::ConservativelyAllocatable); + } + + void moveToNotProvablyAllocatableNodes(NodeId NId) { + removeFromCurrentSet(NId); + NotProvablyAllocatableNodes.insert(NId); + G.getNodeMetadata(NId).setReductionState( + NodeMetadata::NotProvablyAllocatable); + } + + void setup() { + // Set up worklists. + for (auto NId : G.nodeIds()) { + if (G.getNodeDegree(NId) < 3) + moveToOptimallyReducibleNodes(NId); + else if (G.getNodeMetadata(NId).isConservativelyAllocatable()) + moveToConservativelyAllocatableNodes(NId); + else + moveToNotProvablyAllocatableNodes(NId); + } + } + + // Compute a reduction order for the graph by iteratively applying PBQP + // reduction rules. Locally optimal rules are applied whenever possible (R0, + // R1, R2). If no locally-optimal rules apply then any conservatively + // allocatable node is reduced. Finally, if no conservatively allocatable + // node exists then the node with the lowest spill-cost:degree ratio is + // selected. + std::vector reduce() { + assert(!G.empty() && "Cannot reduce empty graph."); + + typedef GraphBase::NodeId NodeId; + std::vector NodeStack; + + // Consume worklists. + while (true) { + if (!OptimallyReducibleNodes.empty()) { + NodeSet::iterator NItr = OptimallyReducibleNodes.begin(); + NodeId NId = *NItr; + OptimallyReducibleNodes.erase(NItr); + NodeStack.push_back(NId); + switch (G.getNodeDegree(NId)) { + case 0: + break; + case 1: + applyR1(G, NId); + break; + case 2: + applyR2(G, NId); + break; + default: llvm_unreachable("Not an optimally reducible node."); + } + } else if (!ConservativelyAllocatableNodes.empty()) { + // Conservatively allocatable nodes will never spill. For now just + // take the first node in the set and push it on the stack. When we + // start optimizing more heavily for register preferencing, it may + // would be better to push nodes with lower 'expected' or worst-case + // register costs first (since early nodes are the most + // constrained). + NodeSet::iterator NItr = ConservativelyAllocatableNodes.begin(); + NodeId NId = *NItr; + ConservativelyAllocatableNodes.erase(NItr); + NodeStack.push_back(NId); + G.disconnectAllNeighborsFromNode(NId); + + } else if (!NotProvablyAllocatableNodes.empty()) { + NodeSet::iterator NItr = + std::min_element(NotProvablyAllocatableNodes.begin(), + NotProvablyAllocatableNodes.end(), + SpillCostComparator(G)); + NodeId NId = *NItr; + NotProvablyAllocatableNodes.erase(NItr); + NodeStack.push_back(NId); + G.disconnectAllNeighborsFromNode(NId); + } else + break; + } + + return NodeStack; + } + + class SpillCostComparator { + public: + SpillCostComparator(const Graph& G) : G(G) {} + bool operator()(NodeId N1Id, NodeId N2Id) { + PBQPNum N1SC = G.getNodeCosts(N1Id)[0] / G.getNodeDegree(N1Id); + PBQPNum N2SC = G.getNodeCosts(N2Id)[0] / G.getNodeDegree(N2Id); + return N1SC < N2SC; + } + private: + const Graph& G; + }; + + Graph& G; + typedef std::set NodeSet; + NodeSet OptimallyReducibleNodes; + NodeSet ConservativelyAllocatableNodes; + NodeSet NotProvablyAllocatableNodes; +}; + +class PBQPRAGraph : public PBQP::Graph { +private: + typedef PBQP::Graph BaseT; +public: + PBQPRAGraph(GraphMetadata Metadata) : BaseT(Metadata) {} +}; + +inline Solution solve(PBQPRAGraph& G) { + if (G.empty()) + return Solution(); + RegAllocSolverImpl RegAllocSolver(G); + return RegAllocSolver.solve(); +} + +} // namespace RegAlloc +} // namespace PBQP + +/// @brief Create a PBQP register allocator instance. +FunctionPass * +createPBQPRegisterAllocator(char *customPassID = nullptr); + +} // namespace llvm + #endif /* LLVM_CODEGEN_REGALLOCPBQP_H */ diff --git a/contrib/llvm/include/llvm/CodeGen/RegisterScavenging.h b/contrib/llvm/include/llvm/CodeGen/RegisterScavenging.h index 335dd7f084c1..474861e45df1 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegisterScavenging.h +++ b/contrib/llvm/include/llvm/CodeGen/RegisterScavenging.h @@ -34,7 +34,7 @@ class RegScavenger { MachineRegisterInfo* MRI; MachineBasicBlock *MBB; MachineBasicBlock::iterator MBBI; - unsigned NumPhysRegs; + unsigned NumRegUnits; /// Tracking - True if RegScavenger is currently tracking the liveness of /// registers. @@ -58,22 +58,19 @@ class RegScavenger { /// A vector of information on scavenged registers. SmallVector Scavenged; - /// CalleeSavedrRegs - A bitvector of callee saved registers for the target. - /// - BitVector CalleeSavedRegs; - - /// RegsAvailable - The current state of all the physical registers immediately - /// before MBBI. One bit per physical register. If bit is set that means it's - /// available, unset means the register is currently being used. - BitVector RegsAvailable; + /// RegUnitsAvailable - The current state of each reg unit immediatelly + /// before MBBI. One bit per register unit. If bit is not set it means any + /// register containing that register unit is currently being used. + BitVector RegUnitsAvailable; // These BitVectors are only used internally to forward(). They are members // to avoid frequent reallocations. - BitVector KillRegs, DefRegs; + BitVector KillRegUnits, DefRegUnits; + BitVector TmpRegUnits; public: RegScavenger() - : MBB(nullptr), NumPhysRegs(0), Tracking(false) {} + : MBB(nullptr), NumRegUnits(0), Tracking(false) {} /// enterBasicBlock - Start tracking liveness from the begin of the specific /// basic block. @@ -112,9 +109,9 @@ public: MachineBasicBlock::iterator getCurrentPosition() const { return MBBI; } - - /// getRegsUsed - return all registers currently in use in used. - void getRegsUsed(BitVector &used, bool includeReserved); + + /// isRegUsed - return if a specific register is currently used. + bool isRegUsed(unsigned Reg, bool includeReserved = true) const; /// getRegsAvailable - Return all available registers in the register class /// in Mask. @@ -157,40 +154,29 @@ public: return scavengeRegister(RegClass, MBBI, SPAdj); } - /// setUsed - Tell the scavenger a register is used. + /// setRegUsed - Tell the scavenger a register is used. /// - void setUsed(unsigned Reg); + void setRegUsed(unsigned Reg); private: /// isReserved - Returns true if a register is reserved. It is never "unused". bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); } - /// isUsed - Test if a register is currently being used. When called by the - /// isAliasUsed function, we only check isReserved if this is the original - /// register, not an alias register. + /// setUsed / setUnused - Mark the state of one or a number of register units. /// - bool isUsed(unsigned Reg, bool CheckReserved = true) const { - return !RegsAvailable.test(Reg) || (CheckReserved && isReserved(Reg)); + void setUsed(BitVector &RegUnits) { + RegUnitsAvailable.reset(RegUnits); + } + void setUnused(BitVector &RegUnits) { + RegUnitsAvailable |= RegUnits; } - /// isAliasUsed - Is Reg or an alias currently in use? - bool isAliasUsed(unsigned Reg) const; - - /// setUsed / setUnused - Mark the state of one or a number of registers. - /// - void setUsed(BitVector &Regs) { - RegsAvailable.reset(Regs); - } - void setUnused(BitVector &Regs) { - RegsAvailable |= Regs; - } - - /// Processes the current instruction and fill the KillRegs and DefRegs bit - /// vectors. + /// Processes the current instruction and fill the KillRegUnits and + /// DefRegUnits bit vectors. void determineKillsAndDefs(); - - /// Add Reg and all its sub-registers to BV. - void addRegWithSubRegs(BitVector &BV, unsigned Reg); - + + /// Add all Reg Units that Reg contains to BV. + void addRegUnits(BitVector &BV, unsigned Reg); + /// findSurvivorReg - Return the candidate register that is unused for the /// longest after StartMI. UseMI is set to the instruction where the search /// stopped. diff --git a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h index 81db8a2f79b5..64c9c4729e92 100644 --- a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h @@ -203,6 +203,16 @@ namespace RTLIB { COPYSIGN_F80, COPYSIGN_F128, COPYSIGN_PPCF128, + FMIN_F32, + FMIN_F64, + FMIN_F80, + FMIN_F128, + FMIN_PPCF128, + FMAX_F32, + FMAX_F64, + FMAX_F80, + FMAX_F128, + FMAX_PPCF128, // CONVERSION FPEXT_F64_F128, diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h index 5a65d590802a..80aee8c62880 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h @@ -190,6 +190,12 @@ namespace llvm { return getKind() == Order && Contents.OrdKind == Barrier; } + /// isNormalMemoryOrBarrier - Test if this is could be any kind of memory + /// dependence. + bool isNormalMemoryOrBarrier() const { + return (isNormalMemory() || isBarrier()); + } + /// isMustAlias - Test if this is an Order dependence that is marked /// as "must alias", meaning that the SUnits at either end of the edge /// have a memory dependence on a known memory location. diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index e6754a2c0342..00dd8f9a633e 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -75,8 +75,7 @@ namespace llvm { /// MachineInstrs. class ScheduleDAGInstrs : public ScheduleDAG { protected: - const MachineLoopInfo &MLI; - const MachineDominatorTree &MDT; + const MachineLoopInfo *MLI; const MachineFrameInfo *MFI; /// Live Intervals provides reaching defs in preRA scheduling. @@ -154,8 +153,7 @@ namespace llvm { public: explicit ScheduleDAGInstrs(MachineFunction &mf, - const MachineLoopInfo &mli, - const MachineDominatorTree &mdt, + const MachineLoopInfo *mli, bool IsPostRAFlag, bool RemoveKillFlags = false, LiveIntervals *LIS = nullptr); diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h index 1d6698c80021..ee4bc08495ec 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -16,9 +16,11 @@ #define LLVM_CODEGEN_SELECTIONDAG_H #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/ilist.h" #include "llvm/CodeGen/DAGCombine.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/Support/RecyclingAllocator.h" #include "llvm/Target/TargetMachine.h" @@ -170,7 +172,7 @@ void checkForCycles(const SelectionDAG *DAG, bool force = false); /// class SelectionDAG { const TargetMachine &TM; - const TargetSelectionDAGInfo &TSI; + const TargetSelectionDAGInfo *TSI; const TargetLowering *TLI; MachineFunction *MF; LLVMContext *Context; @@ -270,7 +272,7 @@ public: /// init - Prepare this SelectionDAG to process code in the given /// MachineFunction. /// - void init(MachineFunction &mf, const TargetLowering *TLI); + void init(MachineFunction &mf); /// clear - Clear state and free memory necessary to make this /// SelectionDAG ready to process a new block. @@ -279,8 +281,9 @@ public: MachineFunction &getMachineFunction() const { return *MF; } const TargetMachine &getTarget() const { return TM; } + const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } const TargetLowering &getTargetLoweringInfo() const { return *TLI; } - const TargetSelectionDAGInfo &getSelectionDAGInfo() const { return TSI; } + const TargetSelectionDAGInfo &getSelectionDAGInfo() const { return *TSI; } LLVMContext *getContext() const {return Context; } /// viewGraph - Pop up a GraphViz/gv window with the DAG rendered using 'dot'. @@ -368,6 +371,27 @@ public: /// the graph. void Legalize(); + /// \brief Transforms a SelectionDAG node and any operands to it into a node + /// that is compatible with the target instruction selector, as indicated by + /// the TargetLowering object. + /// + /// \returns true if \c N is a valid, legal node after calling this. + /// + /// This essentially runs a single recursive walk of the \c Legalize process + /// over the given node (and its operands). This can be used to incrementally + /// legalize the DAG. All of the nodes which are directly replaced, + /// potentially including N, are added to the output parameter \c + /// UpdatedNodes so that the delta to the DAG can be understood by the + /// caller. + /// + /// When this returns false, N has been legalized in a way that make the + /// pointer passed in no longer valid. It may have even been deleted from the + /// DAG, and so it shouldn't be used further. When this returns true, the + /// N passed in is a legal node, and can be immediately processed as such. + /// This may still have done some work on the DAG, and will still populate + /// UpdatedNodes with any new nodes replacing those originally in the DAG. + bool LegalizeOp(SDNode *N, SmallSetVector &UpdatedNodes); + /// LegalizeVectors - This transforms the SelectionDAG into a SelectionDAG /// that only uses vector math operations supported by the target. This is /// necessary as a separate step from Legalize because unrolling a vector @@ -729,7 +753,7 @@ public: SDValue SV, unsigned Align); /// getAtomicCmpSwap - Gets a node for an atomic cmpxchg op. There are two - /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces a the value loaded and a + /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, /// a success flag (initially i1), and a chain. SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, @@ -782,7 +806,8 @@ public: ArrayRef Ops, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + unsigned Size = 0); SDValue getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, ArrayRef Ops, @@ -797,15 +822,15 @@ public: SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, - const MDNode *TBAAInfo = nullptr, + const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr); SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO); SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, - bool isNonTemporal, unsigned Alignment, - const MDNode *TBAAInfo = nullptr); + bool isNonTemporal, bool isInvariant, unsigned Alignment, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO); @@ -816,7 +841,7 @@ public: SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, bool isInvariant, - unsigned Alignment, const MDNode *TBAAInfo = nullptr, + unsigned Alignment, const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr); SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, SDLoc dl, @@ -828,19 +853,25 @@ public: SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, - const MDNode *TBAAInfo = nullptr); + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachineMemOperand *MMO); SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, - const MDNode *TBAAInfo = nullptr); + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, EVT TVT, MachineMemOperand *MMO); SDValue getIndexedStore(SDValue OrigStoe, SDLoc dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM); + SDValue getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, + SDValue Mask, SDValue Src0, EVT MemVT, + MachineMemOperand *MMO, ISD::LoadExtType); + SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, + SDValue Ptr, SDValue Mask, EVT MemVT, + MachineMemOperand *MMO, bool IsTrunc); /// getSrcValue - Construct a node to track a Value* through the backend. SDValue getSrcValue(const Value *v); @@ -963,15 +994,18 @@ public: /// getDbgValue - Creates a SDDbgValue node. /// - SDDbgValue *getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, - bool IsIndirect, uint64_t Off, - DebugLoc DL, unsigned O); - /// Constant. - SDDbgValue *getConstantDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, - DebugLoc DL, unsigned O); - /// Frame index. - SDDbgValue *getFrameIndexDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, - DebugLoc DL, unsigned O); + /// SDNode + SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, + bool IsIndirect, uint64_t Off, DebugLoc DL, + unsigned O); + + /// Constant + SDDbgValue *getConstantDbgValue(MDNode *Var, MDNode *Expr, const Value *C, + uint64_t Off, DebugLoc DL, unsigned O); + + /// FrameIndex + SDDbgValue *getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, + uint64_t Off, DebugLoc DL, unsigned O); /// RemoveDeadNode - Remove the specified node from the system. If any of its /// operands then becomes dead, remove them as well. Inform UpdateListener @@ -1043,7 +1077,10 @@ public: case ISD::SADDO: case ISD::UADDO: case ISD::ADDC: - case ISD::ADDE: return true; + case ISD::ADDE: + case ISD::FMINNUM: + case ISD::FMAXNUM: + return true; default: return false; } } @@ -1202,6 +1239,7 @@ public: unsigned getEVTAlignment(EVT MemoryVT) const; private: + void InsertNode(SDNode *N); bool RemoveNodeFromCSEMaps(SDNode *N); void AddModifiedNodeToCSEMaps(SDNode *N); SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos); diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 520be402cfc5..d53e66da5a71 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/IR/BasicBlock.h" #include "llvm/Pass.h" +#include "llvm/Target/TargetSubtargetInfo.h" namespace llvm { class FastISel; @@ -50,15 +51,16 @@ public: AliasAnalysis *AA; GCFunctionInfo *GFI; CodeGenOpt::Level OptLevel; + const TargetInstrInfo *TII; + const TargetLowering *TLI; + static char ID; explicit SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL = CodeGenOpt::Default); virtual ~SelectionDAGISel(); - const TargetLowering *getTargetLowering() const { - return TM.getTargetLowering(); - } + const TargetLowering *getTargetLowering() const { return TLI; } void getAnalysisUsage(AnalysisUsage &AU) const override; @@ -238,6 +240,12 @@ public: const unsigned char *MatcherTable, unsigned TableSize); + /// \brief Return true if complex patterns for this target can mutate the + /// DAG. + virtual bool ComplexPatternFuncMutatesDAG() const { + return false; + } + private: // Calls to these functions are generated by tblgen. diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 223151105b0d..933ec7fff483 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -19,7 +19,6 @@ #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H #define LLVM_CODEGEN_SELECTIONDAGNODES_H -#include "llvm/ADT/iterator_range.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/GraphTraits.h" @@ -27,6 +26,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/ValueTypes.h" @@ -117,11 +117,13 @@ namespace ISD { /// of information is represented with the SDValue value type. /// class SDValue { + friend struct DenseMapInfo; + SDNode *Node; // The node defining the value we are using. unsigned ResNo; // Which return value of the node we are using. public: SDValue() : Node(nullptr), ResNo(0) {} - SDValue(SDNode *node, unsigned resno) : Node(node), ResNo(resno) {} + SDValue(SDNode *node, unsigned resno); /// get the index which selects a specific result in the SDNode unsigned getResNo() const { return ResNo; } @@ -208,10 +210,14 @@ public: template<> struct DenseMapInfo { static inline SDValue getEmptyKey() { - return SDValue((SDNode*)-1, -1U); + SDValue V; + V.ResNo = -1U; + return V; } static inline SDValue getTombstoneKey() { - return SDValue((SDNode*)-1, 0); + SDValue V; + V.ResNo = -2U; + return V; } static unsigned getHashValue(const SDValue &Val) { return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ @@ -411,6 +417,16 @@ public: return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; } + /// Test if this node is a memory intrinsic (with valid pointer information). + /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for + /// non-memory intrinsics (with chains) that are not really instances of + /// MemSDNode. For such nodes, we need some extra state to determine the + /// proper classof relationship. + bool isMemIntrinsic() const { + return (NodeType == ISD::INTRINSIC_W_CHAIN || + NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1); + } + /// isMachineOpcode - Test if this node has a post-isel opcode, directly /// corresponding to a MachineInstr opcode. bool isMachineOpcode() const { return NodeType < 0; } @@ -578,7 +594,7 @@ public: /// changes. /// NOTE: This is still very expensive. Use carefully. bool hasPredecessorHelper(const SDNode *N, - SmallPtrSet &Visited, + SmallPtrSetImpl &Visited, SmallVectorImpl &Worklist) const; /// getNumOperands - Return the number of values used by this operation. @@ -746,7 +762,13 @@ protected: ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()), NumValues(VTs.NumVTs), debugLoc(dl), IROrder(Order) { + assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); + assert(NumOperands == Ops.size() && + "NumOperands wasn't wide enough for its operands!"); + assert(NumValues == VTs.NumVTs && + "NumValues wasn't wide enough for its operands!"); for (unsigned i = 0; i != Ops.size(); ++i) { + assert(OperandList && "no operands available"); OperandList[i].setUser(this); OperandList[i].setInitial(Ops[i]); } @@ -759,7 +781,11 @@ protected: : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), debugLoc(dl), - IROrder(Order) {} + IROrder(Order) { + assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); + assert(NumValues == VTs.NumVTs && + "NumValues wasn't wide enough for its operands!"); + } /// InitOperands - Initialize the operands list of this with 1 operand. void InitOperands(SDUse *Ops, const SDValue &Op0) { @@ -818,6 +844,8 @@ protected: Ops[i].setInitial(Vals[i]); } NumOperands = N; + assert(NumOperands == N && + "NumOperands wasn't wide enough for its operands!"); OperandList = Ops; checkForCycles(this); } @@ -877,6 +905,13 @@ public: // Define inline functions from the SDValue class. +inline SDValue::SDValue(SDNode *node, unsigned resno) + : Node(node), ResNo(resno) { + assert((!Node || ResNo < Node->getNumValues()) && + "Invalid result number for the given node!"); + assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."); +} + inline unsigned SDValue::getOpcode() const { return Node->getOpcode(); } @@ -1088,8 +1123,8 @@ public: // Returns the offset from the location of the access. int64_t getSrcValueOffset() const { return MMO->getOffset(); } - /// Returns the TBAAInfo that describes the dereference. - const MDNode *getTBAAInfo() const { return MMO->getTBAAInfo(); } + /// Returns the AA info that describes the dereference. + AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } /// Returns the Ranges that describes the dereference. const MDNode *getRanges() const { return MMO->getRanges(); } @@ -1145,6 +1180,9 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || N->getOpcode() == ISD::ATOMIC_LOAD || N->getOpcode() == ISD::ATOMIC_STORE || + N->getOpcode() == ISD::MLOAD || + N->getOpcode() == ISD::MSTORE || + N->isMemIntrinsic() || N->isTargetMemoryOpcode(); } }; @@ -1273,14 +1311,14 @@ public: ArrayRef Ops, EVT MemoryVT, MachineMemOperand *MMO) : MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) { + SubclassData |= 1u << 13; } // Methods to support isa and dyn_cast static bool classof(const SDNode *N) { // We lower some target intrinsics to their target opcode // early a node with a target opcode can be of this class - return N->getOpcode() == ISD::INTRINSIC_W_CHAIN || - N->getOpcode() == ISD::INTRINSIC_VOID || + return N->isMemIntrinsic() || N->getOpcode() == ISD::PREFETCH || N->isTargetMemoryOpcode(); } @@ -1380,6 +1418,12 @@ public: /// isNaN - Return true if the value is a NaN. bool isNaN() const { return Value->isNaN(); } + /// isInfinity - Return true if the value is an infinity + bool isInfinity() const { return Value->isInfinity(); } + + /// isNegative - Return true if the value is negative. + bool isNegative() const { return Value->isNegative(); } + /// isExactlyValue - We don't rely on operator== working on double values, as /// it returns true for things that are clearly not equal, like -0.0 and 0.0. /// As such, this method can be used to do an exact bit-for-bit comparison of @@ -1893,6 +1937,81 @@ public: } }; +/// MaskedLoadStoreSDNode - This is a base class is used to represent MLOAD and +/// MSTORE nodes +/// +class MaskedLoadStoreSDNode : public MemSDNode { + // Operands + SDUse Ops[4]; +public: + friend class SelectionDAG; + MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, + SDValue *Operands, unsigned numOperands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + InitOperands(Ops, Operands, numOperands); + } + + // In the both nodes address is Op1, mask is Op2: + // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value + // MaskedStoreSDNode (Chain, ptr, mask, data) + // Mask is a vector of i1 elements + const SDValue &getBasePtr() const { return getOperand(1); } + const SDValue &getMask() const { return getOperand(2); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MLOAD || + N->getOpcode() == ISD::MSTORE; + } +}; + +/// MaskedLoadSDNode - This class is used to represent an MLOAD node +/// +class MaskedLoadSDNode : public MaskedLoadStoreSDNode { +public: + friend class SelectionDAG; + MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, + unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy, + EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands, + VTs, MemVT, MMO) { + SubclassData |= (unsigned short)ETy; + } + + ISD::LoadExtType getExtensionType() const { + return ISD::LoadExtType(SubclassData & 3); + } + const SDValue &getSrc0() const { return getOperand(3); } + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MLOAD; + } +}; + +/// MaskedStoreSDNode - This class is used to represent an MSTORE node +/// +class MaskedStoreSDNode : public MaskedLoadStoreSDNode { + +public: + friend class SelectionDAG; + MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, + unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT, + MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands, + VTs, MemVT, MMO) { + SubclassData |= (unsigned short)isTrunc; + } + /// isTruncatingStore - Return true if the op does a truncation before store. + /// For integers this is the same as doing a TRUNCATE and storing the result. + /// For floats, it is the same as doing an FP_ROUND and storing the result. + bool isTruncatingStore() const { return SubclassData & 1; } + + const SDValue &getValue() const { return getOperand(3); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MSTORE; + } +}; + /// MachineSDNode - An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. diff --git a/contrib/llvm/include/llvm/CodeGen/StackMapLivenessAnalysis.h b/contrib/llvm/include/llvm/CodeGen/StackMapLivenessAnalysis.h index 6f0754616206..f67a6e95191d 100644 --- a/contrib/llvm/include/llvm/CodeGen/StackMapLivenessAnalysis.h +++ b/contrib/llvm/include/llvm/CodeGen/StackMapLivenessAnalysis.h @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H -#define LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H +#ifndef LLVM_CODEGEN_STACKMAPLIVENESSANALYSIS_H +#define LLVM_CODEGEN_STACKMAPLIVENESSANALYSIS_H #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -61,4 +61,4 @@ private: } // llvm namespace -#endif // LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/StackMaps.h b/contrib/llvm/include/llvm/CodeGen/StackMaps.h index 5eddbb65259e..4e48afe14004 100644 --- a/contrib/llvm/include/llvm/CodeGen/StackMaps.h +++ b/contrib/llvm/include/llvm/CodeGen/StackMaps.h @@ -8,8 +8,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_STACKMAPS -#define LLVM_STACKMAPS +#ifndef LLVM_CODEGEN_STACKMAPS_H +#define LLVM_CODEGEN_STACKMAPS_H #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" @@ -81,6 +81,52 @@ public: unsigned getNextScratchIdx(unsigned StartIdx = 0) const; }; +/// MI-level Statepoint operands +/// +/// Statepoint operands take the form: +/// , , [call arguments], +/// , , +/// , , [other args], +/// [gc values] +class StatepointOpers { +private: + enum { + NCallArgsPos = 0, + CallTargetPos = 1 + }; + +public: + explicit StatepointOpers(const MachineInstr *MI): + MI(MI) { } + + /// Get starting index of non call related arguments + /// (statepoint flags, vm state and gc state). + unsigned getVarIdx() const { + return MI->getOperand(NCallArgsPos).getImm() + 2; + } + + /// Returns the index of the operand containing the number of non-gc non-call + /// arguments. + unsigned getNumVMSArgsIdx() const { + return getVarIdx() + 3; + } + + /// Returns the number of non-gc non-call arguments attached to the + /// statepoint. Note that this is the number of arguments, not the number of + /// operands required to represent those arguments. + unsigned getNumVMSArgs() const { + return MI->getOperand(getNumVMSArgsIdx()).getImm(); + } + + /// Returns the target of the underlying call. + const MachineOperand &getCallTarget() const { + return MI->getOperand(CallTargetPos); + } + +private: + const MachineInstr *MI; +}; + class StackMaps { public: struct Location { @@ -118,6 +164,12 @@ public: StackMaps(AsmPrinter &AP); + void reset() { + CSInfos.clear(); + ConstPool.clear(); + FnStackSize.clear(); + } + /// \brief Generate a stackmap record for a stackmap instruction. /// /// MI must be a raw STACKMAP, not a PATCHPOINT. @@ -126,6 +178,9 @@ public: /// \brief Generate a stackmap record for a patchpoint instruction. void recordPatchPoint(const MachineInstr &MI); + /// \brief Generate a stackmap record for a statepoint instruction. + void recordStatepoint(const MachineInstr &MI); + /// If there is any stack map data, create a stack map section and serialize /// the map info into it. This clears the stack map data structures /// afterwards. @@ -133,10 +188,9 @@ public: private: static const char *WSMP; - typedef SmallVector LocationVec; typedef SmallVector LiveOutVec; - typedef MapVector ConstantPool; + typedef MapVector ConstantPool; typedef MapVector FnStackSizeMap; struct CallsiteInfo { @@ -146,9 +200,9 @@ private: LiveOutVec LiveOuts; CallsiteInfo() : CSOffsetExpr(nullptr), ID(0) {} CallsiteInfo(const MCExpr *CSOffsetExpr, uint64_t ID, - LocationVec &Locations, LiveOutVec &LiveOuts) - : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations), - LiveOuts(LiveOuts) {} + LocationVec &&Locations, LiveOutVec &&LiveOuts) + : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(std::move(Locations)), + LiveOuts(std::move(LiveOuts)) {} }; typedef std::vector CallsiteInfoList; @@ -196,4 +250,4 @@ private: } -#endif // LLVM_STACKMAPS +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 87f140190a75..9209e1c67c1b 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -89,8 +89,6 @@ public: ArrayRef ModuleFlags, Mangler &Mang, const TargetMachine &TM) const override; - bool isSectionAtomizableBySymbols(const MCSection &Section) const override; - const MCSection * SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler &Mang, diff --git a/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h b/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h index 690b70fad89b..b6136665b968 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h @@ -41,7 +41,7 @@ class TargetSchedModel { unsigned MicroOpFactor; // Multiply to normalize microops to resource units. unsigned ResourceLCM; // Resource units per cycle. Latency normalization factor. public: - TargetSchedModel(): STI(nullptr), TII(nullptr) {} + TargetSchedModel(): SchedModel(MCSchedModel::GetDefaultSchedModel()), STI(nullptr), TII(nullptr) {} /// \brief Initialize the machine model for instruction scheduling. /// @@ -167,6 +167,7 @@ public: /// if converter after moving it to TargetSchedModel). unsigned computeInstrLatency(const MachineInstr *MI, bool UseDefaultDefLatency = true) const; + unsigned computeInstrLatency(unsigned Opcode) const; /// \brief Output dependency latency of a pair of defs of the same register. /// diff --git a/contrib/llvm/include/llvm/DebugInfo/DIContext.h b/contrib/llvm/include/llvm/DebugInfo/DIContext.h index c1aba01fbf75..622aa699c435 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DIContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DIContext.h @@ -21,7 +21,6 @@ #include "llvm/Object/RelocVisitor.h" #include "llvm/Support/Casting.h" #include "llvm/Support/DataTypes.h" - #include namespace llvm { @@ -66,11 +65,15 @@ class DIInliningInfo { } }; +/// A DINameKind is passed to name search methods to specify a +/// preference regarding the type of name resolution the caller wants. +enum class DINameKind { None, ShortName, LinkageName }; + /// DILineInfoSpecifier - controls which fields of DILineInfo container /// should be filled with data. struct DILineInfoSpecifier { enum class FileLineInfoKind { None, Default, AbsoluteFilePath }; - enum class FunctionNameKind { None, ShortName, LinkageName }; + typedef DINameKind FunctionNameKind; FileLineInfoKind FLIKind; FunctionNameKind FNKind; @@ -103,7 +106,11 @@ enum DIDumpType { DIDT_GnuPubtypes, DIDT_Str, DIDT_StrDwo, - DIDT_StrOffsetsDwo + DIDT_StrOffsetsDwo, + DIDT_AppleNames, + DIDT_AppleTypes, + DIDT_AppleNamespaces, + DIDT_AppleObjC }; // In place of applying the relocations to the data we've read from disk we use @@ -124,7 +131,7 @@ public: virtual ~DIContext(); /// getDWARFContext - get a context for binary DWARF data. - static DIContext *getDWARFContext(object::ObjectFile *); + static DIContext *getDWARFContext(const object::ObjectFile &Obj); virtual void dump(raw_ostream &OS, DIDumpType DumpType = DIDT_All) = 0; diff --git a/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.h b/contrib/llvm/include/llvm/DebugInfo/DWARFAbbreviationDeclaration.h similarity index 92% rename from contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFAbbreviationDeclaration.h index b86b9ecbe4b3..bb05c302be9a 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFAbbreviationDeclaration.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFAbbreviationDeclaration.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFABBREVIATIONDECLARATION_H -#define LLVM_DEBUGINFO_DWARFABBREVIATIONDECLARATION_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFABBREVIATIONDECLARATION_H +#define LLVM_LIB_DEBUGINFO_DWARFABBREVIATIONDECLARATION_H #include "llvm/ADT/SmallVector.h" #include "llvm/Support/DataExtractor.h" diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARFAcceleratorTable.h b/contrib/llvm/include/llvm/DebugInfo/DWARFAcceleratorTable.h new file mode 100644 index 000000000000..3fbc1c3b3c79 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFAcceleratorTable.h @@ -0,0 +1,49 @@ +//===--- DWARFAcceleratorTable.h --------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/DWARFFormValue.h" +#include "llvm/DebugInfo/DWARFRelocMap.h" +#include + +namespace llvm { + +class DWARFAcceleratorTable { + + struct Header { + uint32_t Magic; + uint16_t Version; + uint16_t HashFunction; + uint32_t NumBuckets; + uint32_t NumHashes; + uint32_t HeaderDataLength; + }; + + struct HeaderData { + typedef uint16_t AtomType; + typedef uint16_t Form; + uint32_t DIEOffsetBase; + SmallVector, 3> Atoms; + }; + + struct Header Hdr; + struct HeaderData HdrData; + DataExtractor AccelSection; + DataExtractor StringSection; + const RelocAddrMap& Relocs; +public: + DWARFAcceleratorTable(DataExtractor AccelSection, DataExtractor StringSection, + const RelocAddrMap &Relocs) + : AccelSection(AccelSection), StringSection(StringSection), Relocs(Relocs) {} + + bool extract(); + void dump(raw_ostream &OS) const; +}; + +} diff --git a/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARFCompileUnit.h similarity index 51% rename from contrib/llvm/lib/DebugInfo/DWARFCompileUnit.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFCompileUnit.h index 2ed188e70c1f..dbf777cc71aa 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFCompileUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFCompileUnit.h @@ -7,19 +7,20 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFCOMPILEUNIT_H -#define LLVM_DEBUGINFO_DWARFCOMPILEUNIT_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFCOMPILEUNIT_H +#define LLVM_LIB_DEBUGINFO_DWARFCOMPILEUNIT_H -#include "DWARFUnit.h" +#include "llvm/DebugInfo/DWARFUnit.h" namespace llvm { class DWARFCompileUnit : public DWARFUnit { public: - DWARFCompileUnit(const DWARFDebugAbbrev *DA, StringRef IS, StringRef RS, - StringRef SS, StringRef SOS, StringRef AOS, - const RelocAddrMap *M, bool LE) - : DWARFUnit(DA, IS, RS, SS, SOS, AOS, M, LE) {} + DWARFCompileUnit(DWARFContext &Context, const DWARFSection &Section, + const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, + StringRef SOS, StringRef AOS, bool LE, + const DWARFUnitSectionBase &UnitSection) + : DWARFUnit(Context, Section, DA, RS, SS, SOS, AOS, LE, UnitSection) {} void dump(raw_ostream &OS); // VTable anchor. ~DWARFCompileUnit() override; diff --git a/contrib/llvm/lib/DebugInfo/DWARFContext.h b/contrib/llvm/include/llvm/DebugInfo/DWARFContext.h similarity index 72% rename from contrib/llvm/lib/DebugInfo/DWARFContext.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFContext.h index 6d1ae921cec5..82b5bb088477 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFContext.h @@ -7,19 +7,21 @@ // //===----------------------------------------------------------------------===/ -#ifndef LLVM_DEBUGINFO_DWARFCONTEXT_H -#define LLVM_DEBUGINFO_DWARFCONTEXT_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFCONTEXT_H +#define LLVM_LIB_DEBUGINFO_DWARFCONTEXT_H -#include "DWARFCompileUnit.h" -#include "DWARFDebugAranges.h" -#include "DWARFDebugFrame.h" -#include "DWARFDebugLine.h" -#include "DWARFDebugLoc.h" -#include "DWARFDebugRangeList.h" -#include "DWARFTypeUnit.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARFCompileUnit.h" +#include "llvm/DebugInfo/DWARFDebugAranges.h" +#include "llvm/DebugInfo/DWARFDebugFrame.h" +#include "llvm/DebugInfo/DWARFDebugLine.h" +#include "llvm/DebugInfo/DWARFDebugLoc.h" +#include "llvm/DebugInfo/DWARFDebugRangeList.h" +#include "llvm/DebugInfo/DWARFSection.h" +#include "llvm/DebugInfo/DWARFTypeUnit.h" +#include namespace llvm { @@ -28,19 +30,17 @@ namespace llvm { /// information parsing. The actual data is supplied through pure virtual /// methods that a concrete implementation provides. class DWARFContext : public DIContext { - typedef SmallVector, 1> CUVector; - typedef SmallVector, 1> TUVector; - CUVector CUs; - TUVector TUs; + DWARFUnitSection CUs; + std::vector> TUs; std::unique_ptr Abbrev; std::unique_ptr Loc; std::unique_ptr Aranges; std::unique_ptr Line; std::unique_ptr DebugFrame; - CUVector DWOCUs; - TUVector DWOTUs; + DWARFUnitSection DWOCUs; + std::vector> DWOTUs; std::unique_ptr AbbrevDWO; std::unique_ptr LocDWO; @@ -64,11 +64,6 @@ class DWARFContext : public DIContext { void parseDWOTypeUnits(); public: - struct Section { - StringRef Data; - RelocAddrMap Relocs; - }; - DWARFContext() : DIContext(CK_DWARF) {} static bool classof(const DIContext *DICtx) { @@ -77,8 +72,9 @@ public: void dump(raw_ostream &OS, DIDumpType DumpType = DIDT_All) override; - typedef iterator_range cu_iterator_range; - typedef iterator_range tu_iterator_range; + typedef DWARFUnitSection::iterator_range cu_iterator_range; + typedef DWARFUnitSection::iterator_range tu_iterator_range; + typedef iterator_range>::iterator> tu_section_iterator_range; /// Get compile units in this context. cu_iterator_range compile_units() { @@ -87,9 +83,9 @@ public: } /// Get type units in this context. - tu_iterator_range type_units() { + tu_section_iterator_range type_unit_sections() { parseTypeUnits(); - return tu_iterator_range(TUs.begin(), TUs.end()); + return tu_section_iterator_range(TUs.begin(), TUs.end()); } /// Get compile units in the DWO context. @@ -99,9 +95,9 @@ public: } /// Get type units in the DWO context. - tu_iterator_range dwo_type_units() { + tu_section_iterator_range dwo_type_unit_sections() { parseDWOTypeUnits(); - return tu_iterator_range(DWOTUs.begin(), DWOTUs.end()); + return tu_section_iterator_range(DWOTUs.begin(), DWOTUs.end()); } /// Get the number of compile units in this context. @@ -159,8 +155,7 @@ public: const DWARFDebugFrame *getDebugFrame(); /// Get a pointer to a parsed line table corresponding to a compile unit. - const DWARFDebugLine::LineTable * - getLineTableForCompileUnit(DWARFCompileUnit *cu); + const DWARFDebugLine::LineTable *getLineTableForUnit(DWARFUnit *cu); DILineInfo getLineInfoForAddress(uint64_t Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; @@ -171,17 +166,15 @@ public: virtual bool isLittleEndian() const = 0; virtual uint8_t getAddressSize() const = 0; - virtual const Section &getInfoSection() = 0; - typedef MapVector > TypeSectionMap; + virtual const DWARFSection &getInfoSection() = 0; + typedef MapVector> TypeSectionMap; virtual const TypeSectionMap &getTypesSections() = 0; virtual StringRef getAbbrevSection() = 0; - virtual const Section &getLocSection() = 0; - virtual const Section &getLocDWOSection() = 0; + virtual const DWARFSection &getLocSection() = 0; virtual StringRef getARangeSection() = 0; virtual StringRef getDebugFrameSection() = 0; - virtual const Section &getLineSection() = 0; - virtual const Section &getLineDWOSection() = 0; + virtual const DWARFSection &getLineSection() = 0; virtual StringRef getStringSection() = 0; virtual StringRef getRangeSection() = 0; virtual StringRef getPubNamesSection() = 0; @@ -190,13 +183,19 @@ public: virtual StringRef getGnuPubTypesSection() = 0; // Sections for DWARF5 split dwarf proposal. - virtual const Section &getInfoDWOSection() = 0; + virtual const DWARFSection &getInfoDWOSection() = 0; virtual const TypeSectionMap &getTypesDWOSections() = 0; virtual StringRef getAbbrevDWOSection() = 0; + virtual const DWARFSection &getLineDWOSection() = 0; + virtual const DWARFSection &getLocDWOSection() = 0; virtual StringRef getStringDWOSection() = 0; virtual StringRef getStringOffsetDWOSection() = 0; virtual StringRef getRangeDWOSection() = 0; virtual StringRef getAddrSection() = 0; + virtual const DWARFSection& getAppleNamesSection() = 0; + virtual const DWARFSection& getAppleTypesSection() = 0; + virtual const DWARFSection& getAppleNamespacesSection() = 0; + virtual const DWARFSection& getAppleObjCSection() = 0; static bool isSupportedVersion(unsigned version) { return version == 2 || version == 3 || version == 4; @@ -217,15 +216,13 @@ class DWARFContextInMemory : public DWARFContext { virtual void anchor(); bool IsLittleEndian; uint8_t AddressSize; - Section InfoSection; + DWARFSection InfoSection; TypeSectionMap TypesSections; StringRef AbbrevSection; - Section LocSection; - Section LocDWOSection; + DWARFSection LocSection; StringRef ARangeSection; StringRef DebugFrameSection; - Section LineSection; - Section LineDWOSection; + DWARFSection LineSection; StringRef StringSection; StringRef RangeSection; StringRef PubNamesSection; @@ -234,42 +231,52 @@ class DWARFContextInMemory : public DWARFContext { StringRef GnuPubTypesSection; // Sections for DWARF5 split dwarf proposal. - Section InfoDWOSection; + DWARFSection InfoDWOSection; TypeSectionMap TypesDWOSections; StringRef AbbrevDWOSection; + DWARFSection LineDWOSection; + DWARFSection LocDWOSection; StringRef StringDWOSection; StringRef StringOffsetDWOSection; StringRef RangeDWOSection; StringRef AddrSection; + DWARFSection AppleNamesSection; + DWARFSection AppleTypesSection; + DWARFSection AppleNamespacesSection; + DWARFSection AppleObjCSection; SmallVector, 4> UncompressedSections; public: - DWARFContextInMemory(object::ObjectFile *); + DWARFContextInMemory(const object::ObjectFile &Obj); bool isLittleEndian() const override { return IsLittleEndian; } uint8_t getAddressSize() const override { return AddressSize; } - const Section &getInfoSection() override { return InfoSection; } + const DWARFSection &getInfoSection() override { return InfoSection; } const TypeSectionMap &getTypesSections() override { return TypesSections; } StringRef getAbbrevSection() override { return AbbrevSection; } - const Section &getLocSection() override { return LocSection; } - const Section &getLocDWOSection() override { return LocDWOSection; } + const DWARFSection &getLocSection() override { return LocSection; } StringRef getARangeSection() override { return ARangeSection; } StringRef getDebugFrameSection() override { return DebugFrameSection; } - const Section &getLineSection() override { return LineSection; } - const Section &getLineDWOSection() override { return LineDWOSection; } + const DWARFSection &getLineSection() override { return LineSection; } StringRef getStringSection() override { return StringSection; } StringRef getRangeSection() override { return RangeSection; } StringRef getPubNamesSection() override { return PubNamesSection; } StringRef getPubTypesSection() override { return PubTypesSection; } StringRef getGnuPubNamesSection() override { return GnuPubNamesSection; } StringRef getGnuPubTypesSection() override { return GnuPubTypesSection; } + const DWARFSection& getAppleNamesSection() override { return AppleNamesSection; } + const DWARFSection& getAppleTypesSection() override { return AppleTypesSection; } + const DWARFSection& getAppleNamespacesSection() override { return AppleNamespacesSection; } + const DWARFSection& getAppleObjCSection() override { return AppleObjCSection; } // Sections for DWARF5 split dwarf proposal. - const Section &getInfoDWOSection() override { return InfoDWOSection; } + const DWARFSection &getInfoDWOSection() override { return InfoDWOSection; } const TypeSectionMap &getTypesDWOSections() override { return TypesDWOSections; } StringRef getAbbrevDWOSection() override { return AbbrevDWOSection; } + const DWARFSection &getLineDWOSection() override { return LineDWOSection; } + const DWARFSection &getLocDWOSection() override { return LocDWOSection; } StringRef getStringDWOSection() override { return StringDWOSection; } StringRef getStringOffsetDWOSection() override { return StringOffsetDWOSection; diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugAbbrev.h similarity index 90% rename from contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugAbbrev.h index 3a9adba246f0..6752df9cd728 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugAbbrev.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugAbbrev.h @@ -7,10 +7,10 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGABBREV_H -#define LLVM_DEBUGINFO_DWARFDEBUGABBREV_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGABBREV_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGABBREV_H -#include "DWARFAbbreviationDeclaration.h" +#include "llvm/DebugInfo/DWARFAbbreviationDeclaration.h" #include #include #include diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugArangeSet.h similarity index 95% rename from contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugArangeSet.h index d6c2d8b27c52..837a8e63469e 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugArangeSet.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugArangeSet.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGARANGESET_H -#define LLVM_DEBUGINFO_DWARFDEBUGARANGESET_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGARANGESET_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGARANGESET_H #include "llvm/ADT/iterator_range.h" #include "llvm/Support/DataExtractor.h" diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugAranges.h similarity index 95% rename from contrib/llvm/lib/DebugInfo/DWARFDebugAranges.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugAranges.h index a9f37fe772c7..791f010a8892 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugAranges.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugAranges.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGARANGES_H -#define LLVM_DEBUGINFO_DWARFDEBUGARANGES_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGARANGES_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGARANGES_H #include "llvm/ADT/DenseSet.h" #include "llvm/Support/DataExtractor.h" diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugFrame.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugFrame.h similarity index 91% rename from contrib/llvm/lib/DebugInfo/DWARFDebugFrame.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugFrame.h index bd4ef45e4cfe..be925cbe7519 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugFrame.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugFrame.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGFRAME_H -#define LLVM_DEBUGINFO_DWARFDEBUGFRAME_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGFRAME_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGFRAME_H #include "llvm/Support/DataExtractor.h" #include "llvm/Support/raw_ostream.h" diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugInfoEntry.h similarity index 89% rename from contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugInfoEntry.h index cc58eb652adc..f05d64b6f34b 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugInfoEntry.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugInfoEntry.h @@ -7,13 +7,13 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGINFOENTRY_H -#define LLVM_DEBUGINFO_DWARFDEBUGINFOENTRY_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGINFOENTRY_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGINFOENTRY_H -#include "DWARFAbbreviationDeclaration.h" -#include "DWARFDebugRangeList.h" #include "llvm/ADT/SmallVector.h" #include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARFAbbreviationDeclaration.h" +#include "llvm/DebugInfo/DWARFDebugRangeList.h" #include "llvm/Support/DataTypes.h" namespace llvm { @@ -38,9 +38,9 @@ public: DWARFDebugInfoEntryMinimal() : Offset(0), SiblingIdx(0), AbbrevDecl(nullptr) {} - void dump(raw_ostream &OS, const DWARFUnit *u, unsigned recurseDepth, + void dump(raw_ostream &OS, DWARFUnit *u, unsigned recurseDepth, unsigned indent = 0) const; - void dumpAttribute(raw_ostream &OS, const DWARFUnit *u, uint32_t *offset_ptr, + void dumpAttribute(raw_ostream &OS, DWARFUnit *u, uint32_t *offset_ptr, uint16_t attr, uint16_t form, unsigned indent = 0) const; /// Extracts a debug info entry, which is a child of a given unit, @@ -125,9 +125,12 @@ public: /// returns its mangled name (or short name, if mangled is missing). /// This name may be fetched from specification or abstract origin /// for this subprogram. Returns null if no name is found. - const char * - getSubroutineName(const DWARFUnit *U, - DILineInfoSpecifier::FunctionNameKind Kind) const; + const char *getSubroutineName(const DWARFUnit *U, DINameKind Kind) const; + + /// Return the DIE name resolving DW_AT_sepcification or + /// DW_AT_abstract_origin references if necessary. + /// Returns null if no name is found. + const char *getName(const DWARFUnit *U, DINameKind Kind) const; /// Retrieves values of DW_AT_call_file, DW_AT_call_line and /// DW_AT_call_column from DIE (or zeroes if they are missing). diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugLine.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugLine.h similarity index 93% rename from contrib/llvm/lib/DebugInfo/DWARFDebugLine.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugLine.h index c7b7ec2c0e70..c5ee76e39682 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugLine.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugLine.h @@ -7,11 +7,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGLINE_H -#define LLVM_DEBUGINFO_DWARFDEBUGLINE_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGLINE_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGLINE_H -#include "DWARFRelocMap.h" #include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARFRelocMap.h" #include "llvm/Support/DataExtractor.h" #include #include @@ -179,10 +179,16 @@ public: // Extracts filename by its index in filename table in prologue. // Returns true on success. - bool getFileNameByIndex(uint64_t FileIndex, + bool getFileNameByIndex(uint64_t FileIndex, const char *CompDir, DILineInfoSpecifier::FileLineInfoKind Kind, std::string &Result) const; + // Fills the Result argument with the file and line information + // corresponding to Address. Returns true on success. + bool getFileLineInfoForAddress(uint64_t Address, const char *CompDir, + DILineInfoSpecifier::FileLineInfoKind Kind, + DILineInfo &Result) const; + void dump(raw_ostream &OS) const; void clear(); diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugLoc.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugLoc.h similarity index 94% rename from contrib/llvm/lib/DebugInfo/DWARFDebugLoc.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugLoc.h index 663acbb42f8d..a6135a0c9291 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugLoc.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugLoc.h @@ -7,11 +7,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGLOC_H -#define LLVM_DEBUGINFO_DWARFDEBUGLOC_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGLOC_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGLOC_H -#include "DWARFRelocMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/DWARFRelocMap.h" #include "llvm/Support/DataExtractor.h" namespace llvm { diff --git a/contrib/llvm/lib/DebugInfo/DWARFDebugRangeList.h b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugRangeList.h similarity index 96% rename from contrib/llvm/lib/DebugInfo/DWARFDebugRangeList.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFDebugRangeList.h index 587b550a6688..4ee3bdad3299 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFDebugRangeList.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFDebugRangeList.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFDEBUGRANGELIST_H -#define LLVM_DEBUGINFO_DWARFDEBUGRANGELIST_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFDEBUGRANGELIST_H +#define LLVM_LIB_DEBUGINFO_DWARFDEBUGRANGELIST_H #include "llvm/Support/DataExtractor.h" #include diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARFFormValue.h b/contrib/llvm/include/llvm/DebugInfo/DWARFFormValue.h index d517a72d62e0..5bb6d1b9ddb0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARFFormValue.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFFormValue.h @@ -57,6 +57,13 @@ public: bool isFormClass(FormClass FC) const; void dump(raw_ostream &OS, const DWARFUnit *U) const; + + /// \brief extracts a value in data at offset *offset_ptr. + /// + /// The passed DWARFUnit is allowed to be nullptr, in which + /// case no relocation processing will be performed and some + /// kind of forms that depend on Unit information are disallowed. + /// \returns wether the extraction succeeded. bool extractValue(DataExtractor data, uint32_t *offset_ptr, const DWARFUnit *u); bool isInlinedCStr() const { @@ -70,6 +77,7 @@ public: Optional getAsCString(const DWARFUnit *U) const; Optional getAsAddress(const DWARFUnit *U) const; Optional getAsSectionOffset() const; + Optional> getAsBlock() const; bool skipValue(DataExtractor debug_info_data, uint32_t *offset_ptr, const DWARFUnit *u) const; diff --git a/contrib/llvm/lib/DebugInfo/DWARFRelocMap.h b/contrib/llvm/include/llvm/DebugInfo/DWARFRelocMap.h similarity index 80% rename from contrib/llvm/lib/DebugInfo/DWARFRelocMap.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFRelocMap.h index 6929e367b84c..d7fe3032e505 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFRelocMap.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFRelocMap.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFRELOCMAP_H -#define LLVM_DEBUGINFO_DWARFRELOCMAP_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFRELOCMAP_H +#define LLVM_LIB_DEBUGINFO_DWARFRELOCMAP_H #include "llvm/ADT/DenseMap.h" @@ -18,5 +18,5 @@ typedef DenseMap > RelocAddrMap; } // namespace llvm -#endif // LLVM_DEBUGINFO_DWARFRELOCMAP_H +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARFSection.h b/contrib/llvm/include/llvm/DebugInfo/DWARFSection.h new file mode 100644 index 000000000000..5f09d9e37d81 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFSection.h @@ -0,0 +1,24 @@ +//===-- DWARFSection.h ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_DEBUGINFO_DWARFSECTION_H +#define LLVM_LIB_DEBUGINFO_DWARFSECTION_H + +#include "llvm/DebugInfo/DWARFRelocMap.h" + +namespace llvm { + +struct DWARFSection { + StringRef Data; + RelocAddrMap Relocs; +}; + +} + +#endif diff --git a/contrib/llvm/lib/DebugInfo/DWARFTypeUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARFTypeUnit.h similarity index 60% rename from contrib/llvm/lib/DebugInfo/DWARFTypeUnit.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFTypeUnit.h index cf773b8d8ef3..213b54139e04 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFTypeUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFTypeUnit.h @@ -7,10 +7,10 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFTYPEUNIT_H -#define LLVM_DEBUGINFO_DWARFTYPEUNIT_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFTYPEUNIT_H +#define LLVM_LIB_DEBUGINFO_DWARFTYPEUNIT_H -#include "DWARFUnit.h" +#include "llvm/DebugInfo/DWARFUnit.h" namespace llvm { @@ -19,10 +19,11 @@ private: uint64_t TypeHash; uint32_t TypeOffset; public: - DWARFTypeUnit(const DWARFDebugAbbrev *DA, StringRef IS, StringRef RS, - StringRef SS, StringRef SOS, StringRef AOS, - const RelocAddrMap *M, bool LE) - : DWARFUnit(DA, IS, RS, SS, SOS, AOS, M, LE) {} + DWARFTypeUnit(DWARFContext &Context, const DWARFSection &Section, + const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, + StringRef SOS, StringRef AOS, bool LE, + const DWARFUnitSectionBase &UnitSection) + : DWARFUnit(Context, Section, DA, RS, SS, SOS, AOS, LE, UnitSection) {} uint32_t getHeaderSize() const override { return DWARFUnit::getHeaderSize() + 12; } diff --git a/contrib/llvm/lib/DebugInfo/DWARFUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARFUnit.h similarity index 57% rename from contrib/llvm/lib/DebugInfo/DWARFUnit.h rename to contrib/llvm/include/llvm/DebugInfo/DWARFUnit.h index 471da36af1cf..d71a1b60b731 100644 --- a/contrib/llvm/lib/DebugInfo/DWARFUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARFUnit.h @@ -7,13 +7,14 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_DWARFUNIT_H -#define LLVM_DEBUGINFO_DWARFUNIT_H +#ifndef LLVM_LIB_DEBUGINFO_DWARFUNIT_H +#define LLVM_LIB_DEBUGINFO_DWARFUNIT_H -#include "DWARFDebugAbbrev.h" -#include "DWARFDebugInfoEntry.h" -#include "DWARFDebugRangeList.h" -#include "DWARFRelocMap.h" +#include "llvm/DebugInfo/DWARFDebugAbbrev.h" +#include "llvm/DebugInfo/DWARFDebugInfoEntry.h" +#include "llvm/DebugInfo/DWARFDebugRangeList.h" +#include "llvm/DebugInfo/DWARFRelocMap.h" +#include "llvm/DebugInfo/DWARFSection.h" #include namespace llvm { @@ -22,21 +23,96 @@ namespace object { class ObjectFile; } +class DWARFContext; class DWARFDebugAbbrev; +class DWARFUnit; class StringRef; class raw_ostream; +/// Base class for all DWARFUnitSection classes. This provides the +/// functionality common to all unit types. +class DWARFUnitSectionBase { +public: + /// Returns the Unit that contains the given section offset in the + /// same section this Unit originated from. + virtual DWARFUnit *getUnitForOffset(uint32_t Offset) const = 0; + + void parse(DWARFContext &C, const DWARFSection &Section); + void parseDWO(DWARFContext &C, const DWARFSection &DWOSection); + +protected: + virtual void parseImpl(DWARFContext &Context, const DWARFSection &Section, + const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, + StringRef SOS, StringRef AOS, bool isLittleEndian) = 0; + + ~DWARFUnitSectionBase() {} +}; + +/// Concrete instance of DWARFUnitSection, specialized for one Unit type. +template +class DWARFUnitSection final : public SmallVector, 1>, + public DWARFUnitSectionBase { + + struct UnitOffsetComparator { + bool operator()(uint32_t LHS, + const std::unique_ptr &RHS) const { + return LHS < RHS->getNextUnitOffset(); + } + }; + + bool Parsed; + +public: + DWARFUnitSection() : Parsed(false) {} + DWARFUnitSection(DWARFUnitSection &&DUS) : + SmallVector, 1>(std::move(DUS)), Parsed(DUS.Parsed) {} + + typedef llvm::SmallVectorImpl> UnitVector; + typedef typename UnitVector::iterator iterator; + typedef llvm::iterator_range iterator_range; + + UnitType *getUnitForOffset(uint32_t Offset) const override { + auto *CU = std::upper_bound(this->begin(), this->end(), Offset, + UnitOffsetComparator()); + if (CU != this->end()) + return CU->get(); + return nullptr; + } + +private: + void parseImpl(DWARFContext &Context, const DWARFSection &Section, + const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, + StringRef SOS, StringRef AOS, bool LE) override { + if (Parsed) + return; + DataExtractor Data(Section.Data, LE, 0); + uint32_t Offset = 0; + while (Data.isValidOffset(Offset)) { + auto U = llvm::make_unique(Context, Section, DA, RS, SS, SOS, + AOS, LE, *this); + if (!U->extract(Data, &Offset)) + break; + this->push_back(std::move(U)); + Offset = this->back()->getNextUnitOffset(); + } + Parsed = true; + } +}; + class DWARFUnit { + DWARFContext &Context; + // Section containing this DWARFUnit. + const DWARFSection &InfoSection; + const DWARFDebugAbbrev *Abbrev; - StringRef InfoSection; StringRef RangeSection; uint32_t RangeSectionBase; StringRef StringSection; StringRef StringOffsetSection; StringRef AddrOffsetSection; uint32_t AddrOffsetSectionBase; - const RelocAddrMap *RelocMap; bool isLittleEndian; + const DWARFUnitSectionBase &UnitSection; uint32_t Offset; uint32_t Length; @@ -48,11 +124,11 @@ class DWARFUnit { std::vector DieArray; class DWOHolder { - std::unique_ptr DWOFile; + object::OwningBinary DWOFile; std::unique_ptr DWOContext; DWARFUnit *DWOU; public: - DWOHolder(object::ObjectFile *DWOFile); + DWOHolder(StringRef DWOPath); DWARFUnit *getUnit() const { return DWOU; } }; std::unique_ptr DWO; @@ -63,12 +139,15 @@ protected: virtual uint32_t getHeaderSize() const { return 11; } public: - DWARFUnit(const DWARFDebugAbbrev *DA, StringRef IS, StringRef RS, - StringRef SS, StringRef SOS, StringRef AOS, const RelocAddrMap *M, - bool LE); + DWARFUnit(DWARFContext &Context, const DWARFSection &Section, + const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, + StringRef SOS, StringRef AOS, bool LE, + const DWARFUnitSectionBase &UnitSection); virtual ~DWARFUnit(); + DWARFContext& getContext() const { return Context; } + StringRef getStringSection() const { return StringSection; } StringRef getStringOffsetSection() const { return StringOffsetSection; } void setAddrOffsetSection(StringRef AOS, uint32_t Base) { @@ -85,13 +164,13 @@ public: bool getStringOffsetSectionItem(uint32_t Index, uint32_t &Result) const; DataExtractor getDebugInfoExtractor() const { - return DataExtractor(InfoSection, isLittleEndian, AddrSize); + return DataExtractor(InfoSection.Data, isLittleEndian, AddrSize); } DataExtractor getStringExtractor() const { return DataExtractor(StringSection, false, 0); } - const RelocAddrMap *getRelocMap() const { return RelocMap; } + const RelocAddrMap *getRelocMap() const { return &InfoSection.Relocs; } bool extract(DataExtractor debug_info, uint32_t* offset_ptr); @@ -131,6 +210,9 @@ public: /// chain is valid as long as parsed compile unit DIEs are not cleared. DWARFDebugInfoEntryInlinedChain getInlinedChainForAddress(uint64_t Address); + /// getUnitSection - Return the DWARFUnitSection containing this unit. + const DWARFUnitSectionBase &getUnitSection() const { return UnitSection; } + private: /// Size in bytes of the .debug_info data associated with this compile unit. size_t getDebugInfoSize() const { return Length + 4 - getHeaderSize(); } diff --git a/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h b/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h index e5dab6191ab6..d79bd3c4dfc8 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h @@ -18,9 +18,11 @@ #include "llvm-c/ExecutionEngine.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/Module.h" #include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueMap.h" #include "llvm/MC/MCCodeGenInfo.h" +#include "llvm/Object/Binary.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Mutex.h" #include "llvm/Target/TargetMachine.h" @@ -39,9 +41,7 @@ class Function; class GlobalVariable; class GlobalValue; class JITEventListener; -class JITMemoryManager; class MachineCodeInfo; -class Module; class MutexGuard; class ObjectCache; class RTDyldMemoryManager; @@ -131,29 +131,20 @@ class ExecutionEngine { protected: /// The list of Modules that we are JIT'ing from. We use a SmallVector to /// optimize for the case where there is only one module. - SmallVector Modules; + SmallVector, 1> Modules; void setDataLayout(const DataLayout *Val) { DL = Val; } /// getMemoryforGV - Allocate memory for a global variable. virtual char *getMemoryForGV(const GlobalVariable *GV); - // To avoid having libexecutionengine depend on the JIT and interpreter - // libraries, the execution engine implementations set these functions to ctor - // pointers at startup time if they are linked in. - static ExecutionEngine *(*JITCtor)( - Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM); static ExecutionEngine *(*MCJITCtor)( - Module *M, - std::string *ErrorStr, - RTDyldMemoryManager *MCJMM, - bool GVsWithCode, - TargetMachine *TM); - static ExecutionEngine *(*InterpCtor)(Module *M, std::string *ErrorStr); + std::unique_ptr M, + std::string *ErrorStr, + std::unique_ptr MCJMM, + std::unique_ptr TM); + static ExecutionEngine *(*InterpCtor)(std::unique_ptr M, + std::string *ErrorStr); /// LazyFunctionCreator - If an unknown function is needed, this function /// pointer is invoked to create it. If this returns null, the JIT will @@ -161,9 +152,8 @@ protected: void *(*LazyFunctionCreator)(const std::string &); public: - /// lock - This lock protects the ExecutionEngine, MCJIT, JIT, JITResolver and - /// JITEmitter classes. It must be held while changing the internal state of - /// any of those classes. + /// lock - This lock protects the ExecutionEngine and MCJIT classes. It must + /// be held while changing the internal state of any of those classes. sys::Mutex lock; //===--------------------------------------------------------------------===// @@ -172,44 +162,9 @@ public: virtual ~ExecutionEngine(); - /// create - This is the factory method for creating an execution engine which - /// is appropriate for the current machine. This takes ownership of the - /// module. - /// - /// \param GVsWithCode - Allocating globals with code breaks - /// freeMachineCodeForFunction and is probably unsafe and bad for performance. - /// However, we have clients who depend on this behavior, so we must support - /// it. Eventually, when we're willing to break some backwards compatibility, - /// this flag should be flipped to false, so that by default - /// freeMachineCodeForFunction works. - static ExecutionEngine *create(Module *M, - bool ForceInterpreter = false, - std::string *ErrorStr = nullptr, - CodeGenOpt::Level OptLevel = - CodeGenOpt::Default, - bool GVsWithCode = true); - - /// createJIT - This is the factory method for creating a JIT for the current - /// machine, it does not fall back to the interpreter. This takes ownership - /// of the Module and JITMemoryManager if successful. - /// - /// Clients should make sure to initialize targets prior to calling this - /// function. - static ExecutionEngine *createJIT(Module *M, - std::string *ErrorStr = nullptr, - JITMemoryManager *JMM = nullptr, - CodeGenOpt::Level OptLevel = - CodeGenOpt::Default, - bool GVsWithCode = true, - Reloc::Model RM = Reloc::Default, - CodeModel::Model CMM = - CodeModel::JITDefault); - - /// addModule - Add a Module to the list of modules that we can JIT from. - /// Note that this takes ownership of the Module: when the ExecutionEngine is - /// destroyed, it destroys the Module as well. - virtual void addModule(Module *M) { - Modules.push_back(M); + /// Add a Module to the list of modules that we can JIT from. + virtual void addModule(std::unique_ptr M) { + Modules.push_back(std::move(M)); } /// addObjectFile - Add an ObjectFile to the execution engine. @@ -223,6 +178,7 @@ public: /// /// MCJIT will take ownership of the ObjectFile. virtual void addObjectFile(std::unique_ptr O); + virtual void addObjectFile(object::OwningBinary O); /// addArchive - Add an Archive to the execution engine. /// @@ -230,11 +186,7 @@ public: /// resolve external symbols in objects it is loading. If a symbol is found /// in the Archive the contained object file will be extracted (in memory) /// and loaded for possible execution. - /// - /// MCJIT will take ownership of the Archive. - virtual void addArchive(object::Archive *A) { - llvm_unreachable("ExecutionEngine subclass doesn't implement addArchive."); - } + virtual void addArchive(object::OwningBinary A); //===--------------------------------------------------------------------===// @@ -263,11 +215,7 @@ public: /// it prints a message to stderr and aborts. /// /// This function is deprecated for the MCJIT execution engine. - /// - /// FIXME: the JIT and MCJIT interfaces should be disentangled or united - /// again, if possible. - /// - virtual void *getPointerToNamedFunction(const std::string &Name, + virtual void *getPointerToNamedFunction(StringRef Name, bool AbortOnFailure = true) = 0; /// mapSectionAddress - map a section to its target address space value. @@ -279,7 +227,7 @@ public: "EE!"); } - /// generateCodeForModule - Run code generationen for the specified module and + /// generateCodeForModule - Run code generation for the specified module and /// load it into memory. /// /// When this function has completed, all code and data for the specified @@ -293,7 +241,7 @@ public: /// locally can use the getFunctionAddress call, which will generate code /// and apply final preparations all in one step. /// - /// This method has no effect for the legacy JIT engine or the interpeter. + /// This method has no effect for the interpeter. virtual void generateCodeForModule(Module *M) {} /// finalizeObject - ensure the module is fully processed and is usable. @@ -302,8 +250,7 @@ public: /// object usable for execution. It should be called after sections within an /// object have been relocated using mapSectionAddress. When this method is /// called the MCJIT execution engine will reapply relocations for a loaded - /// object. This method has no effect for the legacy JIT engine or the - /// interpeter. + /// object. This method has no effect for the interpeter. virtual void finalizeObject() {} /// runStaticConstructorsDestructors - This method is used to execute all of @@ -312,11 +259,11 @@ public: /// \param isDtors - Run the destructors instead of constructors. virtual void runStaticConstructorsDestructors(bool isDtors); - /// runStaticConstructorsDestructors - This method is used to execute all of - /// the static constructors or destructors for a particular module. + /// This method is used to execute all of the static constructors or + /// destructors for a particular module. /// /// \param isDtors - Run the destructors instead of constructors. - void runStaticConstructorsDestructors(Module *module, bool isDtors); + void runStaticConstructorsDestructors(Module &module, bool isDtors); /// runFunctionAsMain - This is a helper function which wraps runFunction to @@ -373,13 +320,6 @@ public: /// getFunctionAddress instead. virtual void *getPointerToFunction(Function *F) = 0; - /// getPointerToBasicBlock - The different EE's represent basic blocks in - /// different ways. Return the representation for a blockaddress of the - /// specified block. - /// - /// This function will not be implemented for the MCJIT execution engine. - virtual void *getPointerToBasicBlock(BasicBlock *BB) = 0; - /// getPointerToFunctionOrStub - If the specified function has been /// code-gen'd, return a pointer to the function. If not, compile it, or use /// a stub to implement lazy compilation if available. See @@ -395,9 +335,9 @@ public: /// getGlobalValueAddress - Return the address of the specified global /// value. This may involve code generation. /// - /// This function should not be called with the JIT or interpreter engines. + /// This function should not be called with the interpreter engine. virtual uint64_t getGlobalValueAddress(const std::string &Name) { - // Default implementation for JIT and interpreter. MCJIT will override this. + // Default implementation for the interpreter. MCJIT will override this. // JIT and interpreter clients should use getPointerToGlobal instead. return 0; } @@ -405,14 +345,11 @@ public: /// getFunctionAddress - Return the address of the specified function. /// This may involve code generation. virtual uint64_t getFunctionAddress(const std::string &Name) { - // Default implementation for JIT and interpreter. MCJIT will override this. - // JIT and interpreter clients should use getPointerToFunction instead. + // Default implementation for the interpreter. MCJIT will override this. + // Interpreter clients should use getPointerToFunction instead. return 0; } - // The JIT overrides a version that actually does this. - virtual void runJITOnFunction(Function *, MachineCodeInfo * = nullptr) { } - /// getGlobalValueAtAddress - Return the LLVM global value object that starts /// at the specified address. /// @@ -427,18 +364,6 @@ public: void InitializeMemory(const Constant *Init, void *Addr); - /// recompileAndRelinkFunction - This method is used to force a function which - /// has already been compiled to be compiled again, possibly after it has been - /// modified. Then the entry to the old copy is overwritten with a branch to - /// the new copy. If there was no old copy, this acts just like - /// VM::getPointerToFunction(). - virtual void *recompileAndRelinkFunction(Function *F) = 0; - - /// freeMachineCodeForFunction - Release memory in the ExecutionEngine - /// corresponding to the machine code emitted to execute this function, useful - /// for garbage-collecting generated code. - virtual void freeMachineCodeForFunction(Function *F) = 0; - /// getOrEmitGlobalVariable - Return the address of the specified global /// variable, possibly emitting it to memory if needed. This is used by the /// Emitter. @@ -457,7 +382,7 @@ public: virtual void UnregisterJITEventListener(JITEventListener *) {} /// Sets the pre-compiled object cache. The ownership of the ObjectCache is - /// not changed. Supported by MCJIT but not JIT. + /// not changed. Supported by MCJIT but not the interpreter. virtual void setObjectCache(ObjectCache *) { llvm_unreachable("No support for an object cache"); } @@ -499,11 +424,6 @@ public: bool isCompilingLazily() const { return CompilingLazily; } - // Deprecated in favor of isCompilingLazily (to reduce double-negatives). - // Remove this in LLVM 2.8. - bool isLazyCompilationDisabled() const { - return !CompilingLazily; - } /// DisableGVCompilation - If called, the JIT will abort if it's asked to /// allocate space and populate a GlobalVariable that is not internal to @@ -544,7 +464,7 @@ public: } protected: - explicit ExecutionEngine(Module *M); + explicit ExecutionEngine(std::unique_ptr M); void emitGlobals(); @@ -564,36 +484,33 @@ namespace EngineKind { const static Kind Either = (Kind)(JIT | Interpreter); } -/// EngineBuilder - Builder class for ExecutionEngines. Use this by -/// stack-allocating a builder, chaining the various set* methods, and -/// terminating it with a .create() call. +/// Builder class for ExecutionEngines. Use this by stack-allocating a builder, +/// chaining the various set* methods, and terminating it with a .create() +/// call. class EngineBuilder { private: - Module *M; + std::unique_ptr M; EngineKind::Kind WhichEngine; std::string *ErrorStr; CodeGenOpt::Level OptLevel; - RTDyldMemoryManager *MCJMM; - JITMemoryManager *JMM; - bool AllocateGVsWithCode; + std::unique_ptr MCJMM; TargetOptions Options; Reloc::Model RelocModel; CodeModel::Model CMModel; std::string MArch; std::string MCPU; SmallVector MAttrs; - bool UseMCJIT; bool VerifyModules; /// InitEngine - Does the common initialization of default options. void InitEngine(); public: - /// EngineBuilder - Constructor for EngineBuilder. If create() is called and - /// is successful, the created engine takes ownership of the module. - EngineBuilder(Module *m) : M(m) { - InitEngine(); - } + /// Constructor for EngineBuilder. + EngineBuilder(std::unique_ptr M); + + // Out-of-line since we don't have the def'n of RTDyldMemoryManager here. + ~EngineBuilder(); /// setEngineKind - Controls whether the user wants the interpreter, the JIT, /// or whichever engine works. This option defaults to EngineKind::Either. @@ -607,26 +524,8 @@ public: /// is only appropriate for the MCJIT; setting this and configuring the builder /// to create anything other than MCJIT will cause a runtime error. If create() /// is called and is successful, the created engine takes ownership of the - /// memory manager. This option defaults to NULL. Using this option nullifies - /// the setJITMemoryManager() option. - EngineBuilder &setMCJITMemoryManager(RTDyldMemoryManager *mcjmm) { - MCJMM = mcjmm; - JMM = nullptr; - return *this; - } - - /// setJITMemoryManager - Sets the JIT memory manager to use. This allows - /// clients to customize their memory allocation policies. This is only - /// appropriate for either JIT or MCJIT; setting this and configuring the - /// builder to create an interpreter will cause a runtime error. If create() - /// is called and is successful, the created engine takes ownership of the - /// memory manager. This option defaults to NULL. This option overrides - /// setMCJITMemoryManager() as well. - EngineBuilder &setJITMemoryManager(JITMemoryManager *jmm) { - MCJMM = nullptr; - JMM = jmm; - return *this; - } + /// memory manager. This option defaults to NULL. + EngineBuilder &setMCJITMemoryManager(std::unique_ptr mcjmm); /// setErrorStr - Set the error string to write to on error. This option /// defaults to NULL. @@ -664,18 +563,6 @@ public: return *this; } - /// setAllocateGVsWithCode - Sets whether global values should be allocated - /// into the same buffer as code. For most applications this should be set - /// to false. Allocating globals with code breaks freeMachineCodeForFunction - /// and is probably unsafe and bad for performance. However, we have clients - /// who depend on this behavior, so we must support it. This option defaults - /// to false so that users of the new API can safely use the new memory - /// manager and free machine code. - EngineBuilder &setAllocateGVsWithCode(bool a) { - AllocateGVsWithCode = a; - return *this; - } - /// setMArch - Override the architecture set by the Module's triple. EngineBuilder &setMArch(StringRef march) { MArch.assign(march.begin(), march.end()); @@ -688,13 +575,6 @@ public: return *this; } - /// setUseMCJIT - Set whether the MC-JIT implementation should be used - /// (experimental). - EngineBuilder &setUseMCJIT(bool Value) { - UseMCJIT = Value; - return *this; - } - /// setVerifyModules - Set whether the JIT implementation should verify /// IR modules during compilation. EngineBuilder &setVerifyModules(bool Verify) { diff --git a/contrib/llvm/include/llvm/ExecutionEngine/JIT.h b/contrib/llvm/include/llvm/ExecutionEngine/JIT.h deleted file mode 100644 index 581d6e6c35eb..000000000000 --- a/contrib/llvm/include/llvm/ExecutionEngine/JIT.h +++ /dev/null @@ -1,38 +0,0 @@ -//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file forces the JIT to link in on certain operating systems. -// (Windows). -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_JIT_H -#define LLVM_EXECUTIONENGINE_JIT_H - -#include "llvm/ExecutionEngine/ExecutionEngine.h" -#include - -extern "C" void LLVMLinkInJIT(); - -namespace { - struct ForceJITLinking { - ForceJITLinking() { - // We must reference JIT in such a way that compilers will not - // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. - if (std::getenv("bar") != (char*) -1) - return; - - LLVMLinkInJIT(); - } - } ForceJITLinking; -} - -#endif diff --git a/contrib/llvm/include/llvm/ExecutionEngine/JITEventListener.h b/contrib/llvm/include/llvm/ExecutionEngine/JITEventListener.h index 99fe36c6b5f6..c3edec868783 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/JITEventListener.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/JITEventListener.h @@ -15,6 +15,7 @@ #ifndef LLVM_EXECUTIONENGINE_JITEVENTLISTENER_H #define LLVM_EXECUTIONENGINE_JITEVENTLISTENER_H +#include "RuntimeDyld.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/DebugLoc.h" #include "llvm/Support/DataTypes.h" @@ -25,7 +26,10 @@ class Function; class MachineFunction; class OProfileWrapper; class IntelJITEventsWrapper; -class ObjectImage; + +namespace object { + class ObjectFile; +} /// JITEvent_EmittedFunctionDetails - Helper struct for containing information /// about a generated machine code function. @@ -57,24 +61,7 @@ public: public: JITEventListener() {} - virtual ~JITEventListener(); - - /// NotifyFunctionEmitted - Called after a function has been successfully - /// emitted to memory. The function still has its MachineFunction attached, - /// if you should happen to need that. - virtual void NotifyFunctionEmitted(const Function &, - void *, size_t, - const EmittedFunctionDetails &) {} - - /// NotifyFreeingMachineCode - Called from freeMachineCodeForFunction(), after - /// the global mapping is removed, but before the machine code is returned to - /// the allocator. - /// - /// OldPtr is the address of the machine code and will be the same as the Code - /// parameter to a previous NotifyFunctionEmitted call. The Function passed - /// to NotifyFunctionEmitted may have been destroyed by the time of the - /// matching NotifyFreeingMachineCode call. - virtual void NotifyFreeingMachineCode(void *) {} + virtual ~JITEventListener() {} /// NotifyObjectEmitted - Called after an object has been successfully /// emitted to memory. NotifyFunctionEmitted will not be called for @@ -84,11 +71,15 @@ public: /// The ObjectImage contains the generated object image /// with section headers updated to reflect the address at which sections /// were loaded and with relocations performed in-place on debug sections. - virtual void NotifyObjectEmitted(const ObjectImage &Obj) {} + virtual void NotifyObjectEmitted(const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) {} /// NotifyFreeingObject - Called just before the memory associated with /// a previously emitted object is released. - virtual void NotifyFreeingObject(const ObjectImage &Obj) {} + virtual void NotifyFreeingObject(const object::ObjectFile &Obj) {} + + // Get a pointe to the GDB debugger registration listener. + static JITEventListener *createGDBRegistrationListener(); #if LLVM_USE_INTEL_JITEVENTS // Construct an IntelJITEventListener @@ -122,7 +113,8 @@ public: return nullptr; } #endif // USE_OPROFILE - +private: + virtual void anchor(); }; } // end namespace llvm. diff --git a/contrib/llvm/include/llvm/ExecutionEngine/JITMemoryManager.h b/contrib/llvm/include/llvm/ExecutionEngine/JITMemoryManager.h deleted file mode 100644 index b22d899c9fec..000000000000 --- a/contrib/llvm/include/llvm/ExecutionEngine/JITMemoryManager.h +++ /dev/null @@ -1,164 +0,0 @@ -//===-- JITMemoryManager.h - Interface JIT uses to Allocate Mem -*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_JITMEMORYMANAGER_H -#define LLVM_EXECUTIONENGINE_JITMEMORYMANAGER_H - -#include "llvm/ExecutionEngine/RuntimeDyld.h" -#include "llvm/Support/DataTypes.h" -#include - -namespace llvm { - - class Function; - class GlobalValue; - -/// JITMemoryManager - This interface is used by the JIT to allocate and manage -/// memory for the code generated by the JIT. This can be reimplemented by -/// clients that have a strong desire to control how the layout of JIT'd memory -/// works. -class JITMemoryManager : public RTDyldMemoryManager { -protected: - bool HasGOT; - -public: - JITMemoryManager() : HasGOT(false) {} - virtual ~JITMemoryManager(); - - /// CreateDefaultMemManager - This is used to create the default - /// JIT Memory Manager if the client does not provide one to the JIT. - static JITMemoryManager *CreateDefaultMemManager(); - - /// setMemoryWritable - When code generation is in progress, - /// the code pages may need permissions changed. - virtual void setMemoryWritable() = 0; - - /// setMemoryExecutable - When code generation is done and we're ready to - /// start execution, the code pages may need permissions changed. - virtual void setMemoryExecutable() = 0; - - /// setPoisonMemory - Setting this flag to true makes the memory manager - /// garbage values over freed memory. This is useful for testing and - /// debugging, and may be turned on by default in debug mode. - virtual void setPoisonMemory(bool poison) = 0; - - //===--------------------------------------------------------------------===// - // Global Offset Table Management - //===--------------------------------------------------------------------===// - - /// AllocateGOT - If the current table requires a Global Offset Table, this - /// method is invoked to allocate it. This method is required to set HasGOT - /// to true. - virtual void AllocateGOT() = 0; - - /// isManagingGOT - Return true if the AllocateGOT method is called. - bool isManagingGOT() const { - return HasGOT; - } - - /// getGOTBase - If this is managing a Global Offset Table, this method should - /// return a pointer to its base. - virtual uint8_t *getGOTBase() const = 0; - - //===--------------------------------------------------------------------===// - // Main Allocation Functions - //===--------------------------------------------------------------------===// - - /// startFunctionBody - When we start JITing a function, the JIT calls this - /// method to allocate a block of free RWX memory, which returns a pointer to - /// it. If the JIT wants to request a block of memory of at least a certain - /// size, it passes that value as ActualSize, and this method returns a block - /// with at least that much space. If the JIT doesn't know ahead of time how - /// much space it will need to emit the function, it passes 0 for the - /// ActualSize. In either case, this method is required to pass back the size - /// of the allocated block through ActualSize. The JIT will be careful to - /// not write more than the returned ActualSize bytes of memory. - virtual uint8_t *startFunctionBody(const Function *F, - uintptr_t &ActualSize) = 0; - - /// allocateStub - This method is called by the JIT to allocate space for a - /// function stub (used to handle limited branch displacements) while it is - /// JIT compiling a function. For example, if foo calls bar, and if bar - /// either needs to be lazily compiled or is a native function that exists too - /// far away from the call site to work, this method will be used to make a - /// thunk for it. The stub should be "close" to the current function body, - /// but should not be included in the 'actualsize' returned by - /// startFunctionBody. - virtual uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, - unsigned Alignment) = 0; - - /// endFunctionBody - This method is called when the JIT is done codegen'ing - /// the specified function. At this point we know the size of the JIT - /// compiled function. This passes in FunctionStart (which was returned by - /// the startFunctionBody method) and FunctionEnd which is a pointer to the - /// actual end of the function. This method should mark the space allocated - /// and remember where it is in case the client wants to deallocate it. - virtual void endFunctionBody(const Function *F, uint8_t *FunctionStart, - uint8_t *FunctionEnd) = 0; - - /// allocateSpace - Allocate a memory block of the given size. This method - /// cannot be called between calls to startFunctionBody and endFunctionBody. - virtual uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) = 0; - - /// allocateGlobal - Allocate memory for a global. - virtual uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) = 0; - - /// deallocateFunctionBody - Free the specified function body. The argument - /// must be the return value from a call to startFunctionBody() that hasn't - /// been deallocated yet. This is never called when the JIT is currently - /// emitting a function. - virtual void deallocateFunctionBody(void *Body) = 0; - - /// CheckInvariants - For testing only. Return true if all internal - /// invariants are preserved, or return false and set ErrorStr to a helpful - /// error message. - virtual bool CheckInvariants(std::string &) { - return true; - } - - /// GetDefaultCodeSlabSize - For testing only. Returns DefaultCodeSlabSize - /// from DefaultJITMemoryManager. - virtual size_t GetDefaultCodeSlabSize() { - return 0; - } - - /// GetDefaultDataSlabSize - For testing only. Returns DefaultCodeSlabSize - /// from DefaultJITMemoryManager. - virtual size_t GetDefaultDataSlabSize() { - return 0; - } - - /// GetDefaultStubSlabSize - For testing only. Returns DefaultCodeSlabSize - /// from DefaultJITMemoryManager. - virtual size_t GetDefaultStubSlabSize() { - return 0; - } - - /// GetNumCodeSlabs - For testing only. Returns the number of MemoryBlocks - /// allocated for code. - virtual unsigned GetNumCodeSlabs() { - return 0; - } - - /// GetNumDataSlabs - For testing only. Returns the number of MemoryBlocks - /// allocated for data. - virtual unsigned GetNumDataSlabs() { - return 0; - } - - /// GetNumStubSlabs - For testing only. Returns the number of MemoryBlocks - /// allocated for function stubs. - virtual unsigned GetNumStubSlabs() { - return 0; - } -}; - -} // end namespace llvm. - -#endif diff --git a/contrib/llvm/include/llvm/ExecutionEngine/ObjectBuffer.h b/contrib/llvm/include/llvm/ExecutionEngine/ObjectBuffer.h deleted file mode 100644 index 6221d3b335df..000000000000 --- a/contrib/llvm/include/llvm/ExecutionEngine/ObjectBuffer.h +++ /dev/null @@ -1,83 +0,0 @@ -//===---- ObjectBuffer.h - Utility class to wrap object image memory -----===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares a wrapper class to hold the memory into which an -// object will be generated. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_OBJECTBUFFER_H -#define LLVM_EXECUTIONENGINE_OBJECTBUFFER_H - -#include "llvm/ADT/SmallVector.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" - -namespace llvm { - -/// ObjectBuffer - This class acts as a container for the memory buffer used during -/// generation and loading of executable objects using MCJIT and RuntimeDyld. The -/// underlying memory for the object will be owned by the ObjectBuffer instance -/// throughout its lifetime. The getMemBuffer() method provides a way to create a -/// MemoryBuffer wrapper object instance to be owned by other classes (such as -/// ObjectFile) as needed, but the MemoryBuffer instance returned does not own the -/// actual memory it points to. -class ObjectBuffer { - virtual void anchor(); -public: - ObjectBuffer() {} - ObjectBuffer(MemoryBuffer* Buf) : Buffer(Buf) {} - virtual ~ObjectBuffer() {} - - /// getMemBuffer - Like MemoryBuffer::getMemBuffer() this function - /// returns a pointer to an object that is owned by the caller. However, - /// the caller does not take ownership of the underlying memory. - MemoryBuffer *getMemBuffer() const { - return MemoryBuffer::getMemBuffer(Buffer->getBuffer(), - Buffer->getBufferIdentifier(), false); - } - - const char *getBufferStart() const { return Buffer->getBufferStart(); } - size_t getBufferSize() const { return Buffer->getBufferSize(); } - StringRef getBuffer() const { return Buffer->getBuffer(); } - -protected: - // The memory contained in an ObjectBuffer - std::unique_ptr Buffer; -}; - -/// ObjectBufferStream - This class encapsulates the SmallVector and -/// raw_svector_ostream needed to generate an object using MC code emission -/// while providing a common ObjectBuffer interface for access to the -/// memory once the object has been generated. -class ObjectBufferStream : public ObjectBuffer { - void anchor() override; -public: - ObjectBufferStream() : OS(SV) {} - virtual ~ObjectBufferStream() {} - - raw_ostream &getOStream() { return OS; } - void flush() - { - OS.flush(); - - // Make the data accessible via the ObjectBuffer::Buffer - Buffer.reset(MemoryBuffer::getMemBuffer(StringRef(SV.data(), SV.size()), - "", - false)); - } - -protected: - SmallVector SV; // Working buffer into which we JIT. - raw_svector_ostream OS; // streaming wrapper -}; - -} // namespace llvm - -#endif diff --git a/contrib/llvm/include/llvm/ExecutionEngine/ObjectCache.h b/contrib/llvm/include/llvm/ExecutionEngine/ObjectCache.h index d1849dfc3bf6..cc01a4e58999 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/ObjectCache.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/ObjectCache.h @@ -27,13 +27,12 @@ public: virtual ~ObjectCache() { } /// notifyObjectCompiled - Provides a pointer to compiled code for Module M. - virtual void notifyObjectCompiled(const Module *M, const MemoryBuffer *Obj) = 0; + virtual void notifyObjectCompiled(const Module *M, MemoryBufferRef Obj) = 0; - /// getObjectCopy - Returns a pointer to a newly allocated MemoryBuffer that - /// contains the object which corresponds with Module M, or 0 if an object is - /// not available. The caller owns both the MemoryBuffer returned by this - /// and the memory it references. - virtual MemoryBuffer* getObject(const Module* M) = 0; + /// Returns a pointer to a newly allocated MemoryBuffer that contains the + /// object which corresponds with Module M, or 0 if an object is not + /// available. + virtual std::unique_ptr getObject(const Module* M) = 0; }; } diff --git a/contrib/llvm/include/llvm/ExecutionEngine/ObjectImage.h b/contrib/llvm/include/llvm/ExecutionEngine/ObjectImage.h deleted file mode 100644 index 1fcedd8d6a92..000000000000 --- a/contrib/llvm/include/llvm/ExecutionEngine/ObjectImage.h +++ /dev/null @@ -1,71 +0,0 @@ -//===---- ObjectImage.h - Format independent executuable object image -----===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares a file format independent ObjectImage class. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_OBJECTIMAGE_H -#define LLVM_EXECUTIONENGINE_OBJECTIMAGE_H - -#include "llvm/ExecutionEngine/ObjectBuffer.h" -#include "llvm/Object/ObjectFile.h" - -namespace llvm { - - -/// ObjectImage - A container class that represents an ObjectFile that has been -/// or is in the process of being loaded into memory for execution. -class ObjectImage { - ObjectImage() LLVM_DELETED_FUNCTION; - ObjectImage(const ObjectImage &other) LLVM_DELETED_FUNCTION; - virtual void anchor(); - -protected: - std::unique_ptr Buffer; - -public: - ObjectImage(ObjectBuffer *Input) : Buffer(Input) {} - virtual ~ObjectImage() {} - - virtual object::symbol_iterator begin_symbols() const = 0; - virtual object::symbol_iterator end_symbols() const = 0; - iterator_range symbols() const { - return iterator_range(begin_symbols(), - end_symbols()); - } - - virtual object::section_iterator begin_sections() const = 0; - virtual object::section_iterator end_sections() const = 0; - iterator_range sections() const { - return iterator_range(begin_sections(), - end_sections()); - } - - virtual /* Triple::ArchType */ unsigned getArch() const = 0; - - // Subclasses can override these methods to update the image with loaded - // addresses for sections and common symbols - virtual void updateSectionAddress(const object::SectionRef &Sec, - uint64_t Addr) = 0; - virtual void updateSymbolAddress(const object::SymbolRef &Sym, - uint64_t Addr) = 0; - - virtual StringRef getData() const = 0; - - virtual object::ObjectFile* getObjectFile() const = 0; - - // Subclasses can override these methods to provide JIT debugging support - virtual void registerWithDebugger() = 0; - virtual void deregisterWithDebugger() = 0; -}; - -} // end namespace llvm - -#endif // LLVM_EXECUTIONENGINE_OBJECTIMAGE_H diff --git a/contrib/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h b/contrib/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h index b1d6810f374b..ef81cd328bdb 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_EXECUTIONENGINE_RT_DYLD_MEMORY_MANAGER_H -#define LLVM_EXECUTIONENGINE_RT_DYLD_MEMORY_MANAGER_H +#ifndef LLVM_EXECUTIONENGINE_RTDYLDMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_RTDYLDMEMORYMANAGER_H #include "llvm-c/ExecutionEngine.h" #include "llvm/ADT/StringRef.h" @@ -22,7 +22,10 @@ namespace llvm { class ExecutionEngine; -class ObjectImage; + + namespace object { + class ObjectFile; + } // RuntimeDyld clients often want to handle the memory management of // what gets placed where. For JIT clients, this is the subset of @@ -76,9 +79,15 @@ public: virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size); + /// This method returns the address of the specified function or variable in + /// the current process. + static uint64_t getSymbolAddressInProcess(const std::string &Name); + /// This method returns the address of the specified function or variable. /// It is used to resolve symbols during module linking. - virtual uint64_t getSymbolAddress(const std::string &Name); + virtual uint64_t getSymbolAddress(const std::string &Name) { + return getSymbolAddressInProcess(Name); + } /// This method returns the address of the specified function. As such it is /// only useful for resolving library symbols, not code generated symbols. @@ -103,7 +112,7 @@ public: /// address space can use this call to remap the section addresses for the /// newly loaded object. virtual void notifyObjectLoaded(ExecutionEngine *EE, - const ObjectImage *) {} + const object::ObjectFile &) {} /// This method is called when object loading is complete and section page /// permissions can be applied. It is up to the memory manager implementation @@ -123,4 +132,4 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS( } // namespace llvm -#endif // LLVM_EXECUTIONENGINE_RT_DYLD_MEMORY_MANAGER_H +#endif diff --git a/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h b/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h index f123ffb803bd..799fc34eb659 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -15,58 +15,75 @@ #define LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H #include "llvm/ADT/StringRef.h" -#include "llvm/ExecutionEngine/ObjectBuffer.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/Support/Memory.h" +#include namespace llvm { namespace object { class ObjectFile; + template class OwningBinary; } class RuntimeDyldImpl; -class ObjectImage; +class RuntimeDyldCheckerImpl; class RuntimeDyld { - friend class RuntimeDyldChecker; + friend class RuntimeDyldCheckerImpl; RuntimeDyld(const RuntimeDyld &) LLVM_DELETED_FUNCTION; void operator=(const RuntimeDyld &) LLVM_DELETED_FUNCTION; // RuntimeDyldImpl is the actual class. RuntimeDyld is just the public // interface. - RuntimeDyldImpl *Dyld; + std::unique_ptr Dyld; RTDyldMemoryManager *MM; bool ProcessAllSections; + RuntimeDyldCheckerImpl *Checker; protected: // Change the address associated with a section when resolving relocations. // Any relocations already associated with the symbol will be re-resolved. void reassignSectionAddress(unsigned SectionID, uint64_t Addr); public: + + /// \brief Information about the loaded object. + class LoadedObjectInfo { + friend class RuntimeDyldImpl; + public: + LoadedObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, + unsigned EndIdx) + : RTDyld(RTDyld), BeginIdx(BeginIdx), EndIdx(EndIdx) { } + + virtual ~LoadedObjectInfo() {} + + virtual object::OwningBinary + getObjectForDebug(const object::ObjectFile &Obj) const = 0; + + uint64_t getSectionLoadAddress(StringRef Name) const; + + protected: + virtual void anchor(); + + RuntimeDyldImpl &RTDyld; + unsigned BeginIdx, EndIdx; + }; + RuntimeDyld(RTDyldMemoryManager *); ~RuntimeDyld(); - /// Prepare the object contained in the input buffer for execution. - /// Ownership of the input buffer is transferred to the ObjectImage - /// instance returned from this function if successful. In the case of load - /// failure, the input buffer will be deleted. - ObjectImage *loadObject(ObjectBuffer *InputBuffer); - - /// Prepare the referenced object file for execution. - /// Ownership of the input object is transferred to the ObjectImage - /// instance returned from this function if successful. In the case of load - /// failure, the input object will be deleted. - ObjectImage *loadObject(std::unique_ptr InputObject); + /// Add the referenced object file to the list of objects to be loaded and + /// relocated. + std::unique_ptr loadObject(const object::ObjectFile &O); /// Get the address of our local copy of the symbol. This may or may not /// be the address used for relocation (clients can copy the data around /// and resolve relocatons based on where they put it). - void *getSymbolAddress(StringRef Name); + void *getSymbolAddress(StringRef Name) const; /// Get the address of the target copy of the symbol. This is the address /// used for relocation. - uint64_t getSymbolLoadAddress(StringRef Name); + uint64_t getSymbolLoadAddress(StringRef Name) const; /// Resolve the relocations for all symbols we currently know about. void resolveRelocations(); diff --git a/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h index 8dd891e83648..35ceba27596c 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h @@ -7,18 +7,19 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIMEDYLDCHECKER_H -#define LLVM_RUNTIMEDYLDCHECKER_H +#ifndef LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H +#define LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H -#include "RuntimeDyld.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include +#include "llvm/ADT/StringRef.h" namespace llvm { class MCDisassembler; +class MemoryBuffer; class MCInstPrinter; +class RuntimeDyld; +class RuntimeDyldCheckerImpl; +class raw_ostream; /// \brief RuntimeDyld invariant checker for verifying that RuntimeDyld has /// correctly applied relocations. @@ -61,14 +62,16 @@ class MCInstPrinter; /// | expr '>>' expr /// class RuntimeDyldChecker { - friend class RuntimeDyldCheckerExprEval; public: - RuntimeDyldChecker(RuntimeDyld &RTDyld, - MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - llvm::raw_ostream &ErrStream) - : RTDyld(*RTDyld.Dyld), Disassembler(Disassembler), - InstPrinter(InstPrinter), ErrStream(ErrStream) {} + RuntimeDyldChecker(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, raw_ostream &ErrStream); + ~RuntimeDyldChecker(); + + // \brief Get the associated RTDyld instance. + RuntimeDyld& getRTDyld(); + + // \brief Get the associated RTDyld instance. + const RuntimeDyld& getRTDyld() const; /// \brief Check a single expression against the attached RuntimeDyld /// instance. @@ -79,20 +82,20 @@ public: /// method to be evaluated as an expression. bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; + /// \brief Returns the address of the requested section (or an error message + /// in the second element of the pair if the address cannot be found). + /// + /// if 'LinkerAddress' is true, this returns the address of the section + /// within the linker's memory. If 'LinkerAddress' is false it returns the + /// address within the target process (i.e. the load address). + std::pair getSectionAddr(StringRef FileName, + StringRef SectionName, + bool LinkerAddress); + private: - - bool isSymbolValid(StringRef Symbol) const; - uint64_t getSymbolAddress(StringRef Symbol) const; - uint64_t readMemoryAtSymbol(StringRef Symbol, int64_t Offset, - unsigned Size) const; - StringRef getSubsectionStartingAt(StringRef Name) const; - - RuntimeDyldImpl &RTDyld; - MCDisassembler *Disassembler; - MCInstPrinter *InstPrinter; - llvm::raw_ostream &ErrStream; + std::unique_ptr Impl; }; } // end namespace llvm -#endif // LLVM_RUNTIMEDYLDCHECKER_H +#endif diff --git a/contrib/llvm/include/llvm/IR/Argument.h b/contrib/llvm/include/llvm/IR/Argument.h index 7c398a5e5530..dd76a90aa5ea 100644 --- a/contrib/llvm/include/llvm/IR/Argument.h +++ b/contrib/llvm/include/llvm/IR/Argument.h @@ -105,6 +105,14 @@ public: /// its containing function. bool hasInAllocaAttr() const; + /// \brief Return true if this argument has the zext attribute on it in its + /// containing function. + bool hasZExtAttr() const; + + /// \brief Return true if this argument has the sext attribute on it in its + /// containing function. + bool hasSExtAttr() const; + /// \brief Add a Attribute to an argument. void addAttr(AttributeSet AS); diff --git a/contrib/llvm/include/llvm/IR/AssemblyAnnotationWriter.h b/contrib/llvm/include/llvm/IR/AssemblyAnnotationWriter.h index a8d52f68176c..19e32a2dcdcc 100644 --- a/contrib/llvm/include/llvm/IR/AssemblyAnnotationWriter.h +++ b/contrib/llvm/include/llvm/IR/AssemblyAnnotationWriter.h @@ -14,8 +14,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_IR_ASMANNOTATIONWRITER_H -#define LLVM_IR_ASMANNOTATIONWRITER_H +#ifndef LLVM_IR_ASSEMBLYANNOTATIONWRITER_H +#define LLVM_IR_ASSEMBLYANNOTATIONWRITER_H namespace llvm { diff --git a/contrib/llvm/include/llvm/IR/BasicBlock.h b/contrib/llvm/include/llvm/IR/BasicBlock.h index a19489aa49b1..7c7dd2ca5643 100644 --- a/contrib/llvm/include/llvm/IR/BasicBlock.h +++ b/contrib/llvm/include/llvm/IR/BasicBlock.h @@ -23,6 +23,7 @@ namespace llvm { +class CallInst; class LandingPadInst; class TerminatorInst; class LLVMContext; @@ -125,6 +126,14 @@ public: TerminatorInst *getTerminator(); const TerminatorInst *getTerminator() const; + /// \brief Returns the call instruction marked 'musttail' prior to the + /// terminating return instruction of this basic block, if such a call is + /// present. Otherwise, returns null. + CallInst *getTerminatingMustTailCall(); + const CallInst *getTerminatingMustTailCall() const { + return const_cast(this)->getTerminatingMustTailCall(); + } + /// \brief Returns a pointer to the first instruction in this block that is /// not a PHINode instruction. /// @@ -173,6 +182,13 @@ public: /// right after \p MovePos in the function \p MovePos lives in. void moveAfter(BasicBlock *MovePos); + /// \brief Insert unlinked basic block into a function. + /// + /// Inserts an unlinked basic block into \c Parent. If \c InsertBefore is + /// provided, inserts before that basic block, otherwise inserts at the end. + /// + /// \pre \a getParent() is \c nullptr. + void insertInto(Function *Parent, BasicBlock *InsertBefore = nullptr); /// \brief Return the predecessor of this block if it has a single predecessor /// block. Otherwise return a null pointer. diff --git a/contrib/llvm/include/llvm/IR/CFG.h b/contrib/llvm/include/llvm/IR/CFG.h index c8be8bd1f2a7..5400d2384868 100644 --- a/contrib/llvm/include/llvm/IR/CFG.h +++ b/contrib/llvm/include/llvm/IR/CFG.h @@ -93,6 +93,9 @@ inline pred_iterator pred_end(BasicBlock *BB) { return pred_iterator(BB, true);} inline const_pred_iterator pred_end(const BasicBlock *BB) { return const_pred_iterator(BB, true); } +inline bool pred_empty(const BasicBlock *BB) { + return pred_begin(BB) == pred_end(BB); +} @@ -257,6 +260,9 @@ inline succ_iterator succ_end(BasicBlock *BB) { inline succ_const_iterator succ_end(const BasicBlock *BB) { return succ_const_iterator(BB->getTerminator(), true); } +inline bool succ_empty(const BasicBlock *BB) { + return succ_begin(BB) == succ_end(BB); +} template struct isPodLike > { static const bool value = isPodLike::value; diff --git a/contrib/llvm/include/llvm/IR/CallingConv.h b/contrib/llvm/include/llvm/IR/CallingConv.h index 1eaf4f7f469f..9872e6ec794d 100644 --- a/contrib/llvm/include/llvm/IR/CallingConv.h +++ b/contrib/llvm/include/llvm/IR/CallingConv.h @@ -20,10 +20,13 @@ namespace llvm { /// the well-known calling conventions. /// namespace CallingConv { + /// LLVM IR allows to use arbitrary numbers as calling convention identifiers. + typedef unsigned ID; + /// A set of enums which specify the assigned numeric values for known llvm /// calling conventions. /// @brief LLVM Calling Convention Representation - enum ID { + enum { /// C - The default llvm calling convention, compatible with C. This /// convention is the only calling convention that supports varargs calls. /// As with typical C calling conventions, the callee/caller have to @@ -137,7 +140,11 @@ namespace CallingConv { /// convention differs from the more common \c X86_64_SysV convention /// in a number of ways, most notably in that XMM registers used to pass /// arguments are shadowed by GPRs, and vice versa. - X86_64_Win64 = 79 + X86_64_Win64 = 79, + + /// \brief MSVC calling convention that passes vectors and vector aggregates + /// in SSE registers. + X86_VectorCall = 80 }; } // End CallingConv namespace diff --git a/contrib/llvm/include/llvm/IR/Constant.h b/contrib/llvm/include/llvm/IR/Constant.h index 82ad9fc2f407..d26991eaab75 100644 --- a/contrib/llvm/include/llvm/IR/Constant.h +++ b/contrib/llvm/include/llvm/IR/Constant.h @@ -48,11 +48,16 @@ protected: : User(ty, vty, Ops, NumOps) {} void destroyConstantImpl(); + void replaceUsesOfWithOnConstantImpl(Constant *Replacement); + public: /// isNullValue - Return true if this is the value that would be returned by /// getNullValue. bool isNullValue() const; + /// \brief Returns true if the value is one. + bool isOneValue() const; + /// isAllOnesValue - Return true if this is the value that would be returned by /// getAllOnesValue. bool isAllOnesValue() const; @@ -64,6 +69,9 @@ public: /// Return true if the value is negative zero or null value. bool isZeroValue() const; + /// \brief Return true if the value is not the smallest signed value. + bool isNotMinSignedValue() const; + /// \brief Return true if the value is the smallest signed value. bool isMinSignedValue() const; diff --git a/contrib/llvm/include/llvm/IR/ConstantRange.h b/contrib/llvm/include/llvm/IR/ConstantRange.h index 342422cbe25f..5e8cd34c4d69 100644 --- a/contrib/llvm/include/llvm/IR/ConstantRange.h +++ b/contrib/llvm/include/llvm/IR/ConstantRange.h @@ -29,15 +29,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_CONSTANTRANGE_H -#define LLVM_SUPPORT_CONSTANTRANGE_H +#ifndef LLVM_IR_CONSTANTRANGE_H +#define LLVM_IR_CONSTANTRANGE_H #include "llvm/ADT/APInt.h" #include "llvm/Support/DataTypes.h" namespace llvm { -/// ConstantRange - This class represents an range of values. +/// This class represents a range of values. /// class ConstantRange { APInt Lower, Upper; @@ -59,7 +59,7 @@ public: /// assert out if the two APInt's are not the same bit width. ConstantRange(APIntMoveTy Lower, APIntMoveTy Upper); - /// makeICmpRegion - Produce the smallest range that contains all values that + /// Produce the smallest range that contains all values that /// might satisfy the comparison specified by Pred when compared to any value /// contained within Other. /// @@ -69,47 +69,46 @@ public: static ConstantRange makeICmpRegion(unsigned Pred, const ConstantRange &Other); - /// getLower - Return the lower value for this range... + /// Return the lower value for this range. /// const APInt &getLower() const { return Lower; } - /// getUpper - Return the upper value for this range... + /// Return the upper value for this range. /// const APInt &getUpper() const { return Upper; } - /// getBitWidth - get the bit width of this ConstantRange + /// Get the bit width of this ConstantRange. /// uint32_t getBitWidth() const { return Lower.getBitWidth(); } - /// isFullSet - Return true if this set contains all of the elements possible - /// for this data-type + /// Return true if this set contains all of the elements possible + /// for this data-type. /// bool isFullSet() const; - /// isEmptySet - Return true if this set contains no members. + /// Return true if this set contains no members. /// bool isEmptySet() const; - /// isWrappedSet - Return true if this set wraps around the top of the range, - /// for example: [100, 8) + /// Return true if this set wraps around the top of the range. + /// For example: [100, 8). /// bool isWrappedSet() const; - /// isSignWrappedSet - Return true if this set wraps around the INT_MIN of - /// its bitwidth, for example: i8 [120, 140). + /// Return true if this set wraps around the INT_MIN of + /// its bitwidth. For example: i8 [120, 140). /// bool isSignWrappedSet() const; - /// contains - Return true if the specified value is in the set. + /// Return true if the specified value is in the set. /// bool contains(const APInt &Val) const; - /// contains - Return true if the other range is a subset of this one. + /// Return true if the other range is a subset of this one. /// bool contains(const ConstantRange &CR) const; - /// getSingleElement - If this set contains a single element, return it, - /// otherwise return null. + /// If this set contains a single element, return it, otherwise return null. /// const APInt *getSingleElement() const { if (Upper == Lower + 1) @@ -117,35 +116,31 @@ public: return nullptr; } - /// isSingleElement - Return true if this set contains exactly one member. + /// Return true if this set contains exactly one member. /// bool isSingleElement() const { return getSingleElement() != nullptr; } - /// getSetSize - Return the number of elements in this set. + /// Return the number of elements in this set. /// APInt getSetSize() const; - /// getUnsignedMax - Return the largest unsigned value contained in the - /// ConstantRange. + /// Return the largest unsigned value contained in the ConstantRange. /// APInt getUnsignedMax() const; - /// getUnsignedMin - Return the smallest unsigned value contained in the - /// ConstantRange. + /// Return the smallest unsigned value contained in the ConstantRange. /// APInt getUnsignedMin() const; - /// getSignedMax - Return the largest signed value contained in the - /// ConstantRange. + /// Return the largest signed value contained in the ConstantRange. /// APInt getSignedMax() const; - /// getSignedMin - Return the smallest signed value contained in the - /// ConstantRange. + /// Return the smallest signed value contained in the ConstantRange. /// APInt getSignedMin() const; - /// operator== - Return true if this range is equal to another range. + /// Return true if this range is equal to another range. /// bool operator==(const ConstantRange &CR) const { return Lower == CR.Lower && Upper == CR.Upper; @@ -154,15 +149,14 @@ public: return !operator==(CR); } - /// subtract - Subtract the specified constant from the endpoints of this - /// constant range. + /// Subtract the specified constant from the endpoints of this constant range. ConstantRange subtract(const APInt &CI) const; /// \brief Subtract the specified range from this range (aka relative /// complement of the sets). ConstantRange difference(const ConstantRange &CR) const; - /// intersectWith - Return the range that results from the intersection of + /// Return the range that results from the intersection of /// this range with another range. The resultant range is guaranteed to /// include all elements contained in both input ranges, and to have the /// smallest possible set size that does so. Because there may be two @@ -171,7 +165,7 @@ public: /// ConstantRange intersectWith(const ConstantRange &CR) const; - /// unionWith - Return the range that results from the union of this range + /// Return the range that results from the union of this range /// with another range. The resultant range is guaranteed to include the /// elements of both sets, but may contain more. For example, [3, 9) union /// [12,15) is [3, 15), which includes 9, 10, and 11, which were not included @@ -179,85 +173,84 @@ public: /// ConstantRange unionWith(const ConstantRange &CR) const; - /// zeroExtend - Return a new range in the specified integer type, which must + /// Return a new range in the specified integer type, which must /// be strictly larger than the current type. The returned range will /// correspond to the possible range of values if the source range had been /// zero extended to BitWidth. ConstantRange zeroExtend(uint32_t BitWidth) const; - /// signExtend - Return a new range in the specified integer type, which must + /// Return a new range in the specified integer type, which must /// be strictly larger than the current type. The returned range will /// correspond to the possible range of values if the source range had been /// sign extended to BitWidth. ConstantRange signExtend(uint32_t BitWidth) const; - /// truncate - Return a new range in the specified integer type, which must be + /// Return a new range in the specified integer type, which must be /// strictly smaller than the current type. The returned range will /// correspond to the possible range of values if the source range had been /// truncated to the specified type. ConstantRange truncate(uint32_t BitWidth) const; - /// zextOrTrunc - make this range have the bit width given by \p BitWidth. The + /// Make this range have the bit width given by \p BitWidth. The /// value is zero extended, truncated, or left alone to make it that width. ConstantRange zextOrTrunc(uint32_t BitWidth) const; - /// sextOrTrunc - make this range have the bit width given by \p BitWidth. The + /// Make this range have the bit width given by \p BitWidth. The /// value is sign extended, truncated, or left alone to make it that width. ConstantRange sextOrTrunc(uint32_t BitWidth) const; - /// add - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from an addition of a value in this range and a value in \p Other. ConstantRange add(const ConstantRange &Other) const; - /// sub - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from a subtraction of a value in this range and a value in \p Other. ConstantRange sub(const ConstantRange &Other) const; - /// multiply - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from a multiplication of a value in this range and a value in \p Other. /// TODO: This isn't fully implemented yet. ConstantRange multiply(const ConstantRange &Other) const; - /// smax - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from a signed maximum of a value in this range and a value in \p Other. ConstantRange smax(const ConstantRange &Other) const; - /// umax - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from an unsigned maximum of a value in this range and a value in \p Other. ConstantRange umax(const ConstantRange &Other) const; - /// udiv - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from an unsigned division of a value in this range and a value in /// \p Other. ConstantRange udiv(const ConstantRange &Other) const; - /// binaryAnd - return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from a binary-and of a value in this range by a value in \p Other. ConstantRange binaryAnd(const ConstantRange &Other) const; - /// binaryOr - return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from a binary-or of a value in this range by a value in \p Other. ConstantRange binaryOr(const ConstantRange &Other) const; - /// shl - Return a new range representing the possible values resulting + /// Return a new range representing the possible values resulting /// from a left shift of a value in this range by a value in \p Other. /// TODO: This isn't fully implemented yet. ConstantRange shl(const ConstantRange &Other) const; - /// lshr - Return a new range representing the possible values resulting - /// from a logical right shift of a value in this range and a value in - /// \p Other. + /// Return a new range representing the possible values resulting from a + /// logical right shift of a value in this range and a value in \p Other. ConstantRange lshr(const ConstantRange &Other) const; - /// inverse - Return a new range that is the logical not of the current set. + /// Return a new range that is the logical not of the current set. /// ConstantRange inverse() const; - /// print - Print out the bounds to a stream... + /// Print out the bounds to a stream. /// void print(raw_ostream &OS) const; - /// dump - Allow printing from a debugger easily... + /// Allow printing from a debugger easily. /// void dump() const; }; diff --git a/contrib/llvm/include/llvm/IR/Constants.h b/contrib/llvm/include/llvm/IR/Constants.h index 0e72f040d3e0..5b098b4f51d6 100644 --- a/contrib/llvm/include/llvm/IR/Constants.h +++ b/contrib/llvm/include/llvm/IR/Constants.h @@ -37,12 +37,8 @@ class PointerType; class VectorType; class SequentialType; -template -struct ConstantCreator; -template -struct ConstantArrayCreator; -template -struct ConvertConstantType; +struct ConstantExprKeyType; +template struct ConstantAggrKeyType; //===----------------------------------------------------------------------===// /// This is the shared class of boolean and integer constants. This class @@ -268,6 +264,9 @@ public: /// isNegative - Return true if the sign bit is set. bool isNegative() const { return Val.isNegative(); } + /// isInfinity - Return true if the value is infinity + bool isInfinity() const { return Val.isInfinity(); } + /// isNaN - Return true if the value is a NaN. bool isNaN() const { return Val.isNaN(); } @@ -326,6 +325,9 @@ public: /// index. Constant *getElementValue(unsigned Idx) const; + /// \brief Return the number of elements in the array, vector, or struct. + unsigned getNumElements() const; + /// Methods for support type inquiry through isa, cast, and dyn_cast: /// static bool classof(const Value *V) { @@ -338,7 +340,7 @@ public: /// ConstantArray - Constant Array Declarations /// class ConstantArray : public Constant { - friend struct ConstantArrayCreator; + friend struct ConstantAggrKeyType; ConstantArray(const ConstantArray &) LLVM_DELETED_FUNCTION; protected: ConstantArray(ArrayType *T, ArrayRef Val); @@ -346,6 +348,10 @@ public: // ConstantArray accessors static Constant *get(ArrayType *T, ArrayRef V); +private: + static Constant *getImpl(ArrayType *T, ArrayRef V); + +public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); @@ -376,14 +382,14 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantArray, Constant) // ConstantStruct - Constant Struct Declarations // class ConstantStruct : public Constant { - friend struct ConstantArrayCreator; + friend struct ConstantAggrKeyType; ConstantStruct(const ConstantStruct &) LLVM_DELETED_FUNCTION; protected: ConstantStruct(StructType *T, ArrayRef Val); public: // ConstantStruct accessors static Constant *get(StructType *T, ArrayRef V); - static Constant *get(StructType *T, ...) END_WITH_NULL; + static Constant *get(StructType *T, ...) LLVM_END_WITH_NULL; /// getAnon - Return an anonymous struct that has the specified /// elements. If the struct is possibly empty, then you must specify a @@ -435,7 +441,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantStruct, Constant) /// ConstantVector - Constant Vector Declarations /// class ConstantVector : public Constant { - friend struct ConstantArrayCreator; + friend struct ConstantAggrKeyType; ConstantVector(const ConstantVector &) LLVM_DELETED_FUNCTION; protected: ConstantVector(VectorType *T, ArrayRef Val); @@ -443,6 +449,10 @@ public: // ConstantVector accessors static Constant *get(ArrayRef V); +private: + static Constant *getImpl(ArrayRef V); + +public: /// getSplat - Return a ConstantVector with the specified constant in each /// element. static Constant *getSplat(unsigned NumElts, Constant *Elt); @@ -794,9 +804,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value) /// constant expressions. The Opcode field for the ConstantExpr class is /// maintained in the Value::SubclassData field. class ConstantExpr : public Constant { - friend struct ConstantCreator > >; - friend struct ConvertConstantType; + friend struct ConstantExprKeyType; protected: ConstantExpr(Type *ty, unsigned Opcode, Use *Ops, unsigned NumOps) @@ -856,19 +864,25 @@ public: bool HasNUW = false, bool HasNSW = false); static Constant *getLShr(Constant *C1, Constant *C2, bool isExact = false); static Constant *getAShr(Constant *C1, Constant *C2, bool isExact = false); - static Constant *getTrunc (Constant *C, Type *Ty); - static Constant *getSExt (Constant *C, Type *Ty); - static Constant *getZExt (Constant *C, Type *Ty); - static Constant *getFPTrunc (Constant *C, Type *Ty); - static Constant *getFPExtend(Constant *C, Type *Ty); - static Constant *getUIToFP (Constant *C, Type *Ty); - static Constant *getSIToFP (Constant *C, Type *Ty); - static Constant *getFPToUI (Constant *C, Type *Ty); - static Constant *getFPToSI (Constant *C, Type *Ty); - static Constant *getPtrToInt(Constant *C, Type *Ty); - static Constant *getIntToPtr(Constant *C, Type *Ty); - static Constant *getBitCast (Constant *C, Type *Ty); - static Constant *getAddrSpaceCast(Constant *C, Type *Ty); + static Constant *getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getSExt(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getZExt(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getFPTrunc(Constant *C, Type *Ty, + bool OnlyIfReduced = false); + static Constant *getFPExtend(Constant *C, Type *Ty, + bool OnlyIfReduced = false); + static Constant *getUIToFP(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getSIToFP(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getFPToUI(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getFPToSI(Constant *C, Type *Ty, bool OnlyIfReduced = false); + static Constant *getPtrToInt(Constant *C, Type *Ty, + bool OnlyIfReduced = false); + static Constant *getIntToPtr(Constant *C, Type *Ty, + bool OnlyIfReduced = false); + static Constant *getBitCast(Constant *C, Type *Ty, + bool OnlyIfReduced = false); + static Constant *getAddrSpaceCast(Constant *C, Type *Ty, + bool OnlyIfReduced = false); static Constant *getNSWNeg(Constant *C) { return getNeg(C, false, true); } static Constant *getNUWNeg(Constant *C) { return getNeg(C, true, false); } @@ -923,13 +937,14 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); - // @brief Convenience function for getting one of the casting operations - // using a CastOps opcode. - static Constant *getCast( - unsigned ops, ///< The opcode for the conversion - Constant *C, ///< The constant to be converted - Type *Ty ///< The type to which the constant is converted - ); + /// \brief Convenience function for getting a Cast operation. + /// + /// \param ops The opcode for the conversion + /// \param C The constant to be converted + /// \param Ty The type to which the constant is converted + /// \param OnlyIfReduced see \a getWithOperands() docs. + static Constant *getCast(unsigned ops, Constant *C, Type *Ty, + bool OnlyIfReduced = false); // @brief Create a ZExt or BitCast cast constant expression static Constant *getZExtOrBitCast( @@ -995,44 +1010,53 @@ public: /// Select constant expr /// - static Constant *getSelect(Constant *C, Constant *V1, Constant *V2); + /// \param OnlyIfReducedTy see \a getWithOperands() docs. + static Constant *getSelect(Constant *C, Constant *V1, Constant *V2, + Type *OnlyIfReducedTy = nullptr); /// get - Return a binary or shift operator constant expression, /// folding if possible. /// + /// \param OnlyIfReducedTy see \a getWithOperands() docs. static Constant *get(unsigned Opcode, Constant *C1, Constant *C2, - unsigned Flags = 0); + unsigned Flags = 0, Type *OnlyIfReducedTy = nullptr); - /// @brief Return an ICmp or FCmp comparison operator constant expression. - static Constant *getCompare(unsigned short pred, Constant *C1, Constant *C2); + /// \brief Return an ICmp or FCmp comparison operator constant expression. + /// + /// \param OnlyIfReduced see \a getWithOperands() docs. + static Constant *getCompare(unsigned short pred, Constant *C1, Constant *C2, + bool OnlyIfReduced = false); /// get* - Return some common constants without having to /// specify the full Instruction::OPCODE identifier. /// - static Constant *getICmp(unsigned short pred, Constant *LHS, Constant *RHS); - static Constant *getFCmp(unsigned short pred, Constant *LHS, Constant *RHS); + static Constant *getICmp(unsigned short pred, Constant *LHS, Constant *RHS, + bool OnlyIfReduced = false); + static Constant *getFCmp(unsigned short pred, Constant *LHS, Constant *RHS, + bool OnlyIfReduced = false); /// Getelementptr form. Value* is only accepted for convenience; /// all elements must be Constant's. /// - static Constant *getGetElementPtr(Constant *C, - ArrayRef IdxList, - bool InBounds = false) { - return getGetElementPtr(C, makeArrayRef((Value * const *)IdxList.data(), - IdxList.size()), - InBounds); + /// \param OnlyIfReducedTy see \a getWithOperands() docs. + static Constant *getGetElementPtr(Constant *C, ArrayRef IdxList, + bool InBounds = false, + Type *OnlyIfReducedTy = nullptr) { + return getGetElementPtr( + C, makeArrayRef((Value * const *)IdxList.data(), IdxList.size()), + InBounds, OnlyIfReducedTy); } - static Constant *getGetElementPtr(Constant *C, - Constant *Idx, - bool InBounds = false) { + static Constant *getGetElementPtr(Constant *C, Constant *Idx, + bool InBounds = false, + Type *OnlyIfReducedTy = nullptr) { // This form of the function only exists to avoid ambiguous overload // warnings about whether to convert Idx to ArrayRef or // ArrayRef. - return getGetElementPtr(C, cast(Idx), InBounds); + return getGetElementPtr(C, cast(Idx), InBounds, OnlyIfReducedTy); } - static Constant *getGetElementPtr(Constant *C, - ArrayRef IdxList, - bool InBounds = false); + static Constant *getGetElementPtr(Constant *C, ArrayRef IdxList, + bool InBounds = false, + Type *OnlyIfReducedTy = nullptr); /// Create an "inbounds" getelementptr. See the documentation for the /// "inbounds" flag in LangRef.html for details. @@ -1052,12 +1076,17 @@ public: return getGetElementPtr(C, IdxList, true); } - static Constant *getExtractElement(Constant *Vec, Constant *Idx); - static Constant *getInsertElement(Constant *Vec, Constant *Elt,Constant *Idx); - static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask); - static Constant *getExtractValue(Constant *Agg, ArrayRef Idxs); + static Constant *getExtractElement(Constant *Vec, Constant *Idx, + Type *OnlyIfReducedTy = nullptr); + static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx, + Type *OnlyIfReducedTy = nullptr); + static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask, + Type *OnlyIfReducedTy = nullptr); + static Constant *getExtractValue(Constant *Agg, ArrayRef Idxs, + Type *OnlyIfReducedTy = nullptr); static Constant *getInsertValue(Constant *Agg, Constant *Val, - ArrayRef Idxs); + ArrayRef Idxs, + Type *OnlyIfReducedTy = nullptr); /// getOpcode - Return the opcode at the root of this constant expression unsigned getOpcode() const { return getSubclassDataFromValue(); } @@ -1084,11 +1113,17 @@ public: return getWithOperands(Ops, getType()); } - /// getWithOperands - This returns the current constant expression with the - /// operands replaced with the specified values and with the specified result - /// type. The specified array must have the same number of operands as our - /// current one. - Constant *getWithOperands(ArrayRef Ops, Type *Ty) const; + /// \brief Get the current expression with the operands replaced. + /// + /// Return the current constant expression with the operands replaced with \c + /// Ops and the type with \c Ty. The new operands must have the same number + /// as the current ones. + /// + /// If \c OnlyIfReduced is \c true, nullptr will be returned unless something + /// gets constant-folded, the type changes, or the expression is otherwise + /// canonicalized. This parameter should almost always be \c false. + Constant *getWithOperands(ArrayRef Ops, Type *Ty, + bool OnlyIfReduced = false) const; /// getAsInstruction - Returns an Instruction which implements the same operation /// as this ConstantExpr. The instruction is not linked to any basic block. @@ -1164,6 +1199,9 @@ public: /// index. UndefValue *getElementValue(unsigned Idx) const; + /// \brief Return the number of elements in the array, vector, or struct. + unsigned getNumElements() const; + void destroyConstant() override; /// Methods for support type inquiry through isa, cast, and dyn_cast: diff --git a/contrib/llvm/include/llvm/IR/DIBuilder.h b/contrib/llvm/include/llvm/IR/DIBuilder.h index 267350409604..ae1ac650a9ec 100644 --- a/contrib/llvm/include/llvm/IR/DIBuilder.h +++ b/contrib/llvm/include/llvm/IR/DIBuilder.h @@ -18,6 +18,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/DebugInfo.h" +#include "llvm/IR/TrackingMDRef.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/DataTypes.h" @@ -27,6 +28,7 @@ namespace llvm { class Function; class Module; class Value; + class Constant; class LLVMContext; class MDNode; class StringRef; @@ -38,7 +40,6 @@ namespace llvm { class DIFile; class DIEnumerator; class DIType; - class DIArray; class DIGlobalVariable; class DIImportedEntity; class DINameSpace; @@ -53,7 +54,6 @@ namespace llvm { class DIObjCProperty; class DIBuilder { - private: Module &M; LLVMContext &VMContext; @@ -66,27 +66,34 @@ namespace llvm { Function *DeclareFn; // llvm.dbg.declare Function *ValueFn; // llvm.dbg.value - SmallVector AllEnumTypes; - /// Use TrackingVH to collect RetainTypes, since they can be updated - /// later on. - SmallVector, 4> AllRetainTypes; - SmallVector AllSubprograms; - SmallVector AllGVs; - SmallVector, 4> AllImportedModules; + SmallVector AllEnumTypes; + /// Track the RetainTypes, since they can be updated later on. + SmallVector AllRetainTypes; + SmallVector AllSubprograms; + SmallVector AllGVs; + SmallVector AllImportedModules; - // Private use for multiple types of template parameters. - DITemplateValueParameter - createTemplateValueParameter(unsigned Tag, DIDescriptor Scope, - StringRef Name, DIType Ty, Value *Val, - MDNode *File = nullptr, unsigned LineNo = 0, - unsigned ColumnNo = 0); + /// \brief Track nodes that may be unresolved. + SmallVector UnresolvedNodes; + bool AllowUnresolvedNodes; + + /// Each subprogram's preserved local variables. + DenseMap> PreservedVariables; DIBuilder(const DIBuilder &) LLVM_DELETED_FUNCTION; void operator=(const DIBuilder &) LLVM_DELETED_FUNCTION; - public: - explicit DIBuilder(Module &M); - enum ComplexAddrKind { OpPlus=1, OpDeref }; + /// \brief Create a temporary. + /// + /// Create an \a MDNodeFwdDecl and track it in \a UnresolvedNodes. + void trackIfUnresolved(MDNode *N); + + public: + /// \brief Construct a builder for a module. + /// + /// If \c AllowUnresolved, collect unresolved nodes attached to the module + /// in order to resolve cycles during \a finalize(). + explicit DIBuilder(Module &M, bool AllowUnresolved = true); enum DebugEmissionKind { FullDebug=1, LineTablesOnly }; /// finalize - Construct any deferred debug info descriptors. @@ -165,8 +172,12 @@ namespace llvm { /// \brief Create debugging information entry for a pointer to member. /// @param PointeeTy Type pointed to by this pointer. + /// @param SizeInBits Size. + /// @param AlignInBits Alignment. (optional) /// @param Class Type for which this pointer points to members of. - DIDerivedType createMemberPointerType(DIType PointeeTy, DIType Class); + DIDerivedType createMemberPointerType(DIType PointeeTy, DIType Class, + uint64_t SizeInBits, + uint64_t AlignInBits = 0); /// createReferenceType - Create debugging information entry for a c++ /// style reference or rvalue reference type. @@ -218,36 +229,10 @@ namespace llvm { /// @param Ty Type of the static member. /// @param Flags Flags to encode member attribute, e.g. private. /// @param Val Const initializer of the member. - DIDerivedType - createStaticMemberType(DIDescriptor Scope, StringRef Name, - DIFile File, unsigned LineNo, DIType Ty, - unsigned Flags, llvm::Value *Val); - - /// createObjCIVar - Create debugging information entry for Objective-C - /// instance variable. - /// @param Name Member name. - /// @param File File where this member is defined. - /// @param LineNo Line number. - /// @param SizeInBits Member size. - /// @param AlignInBits Member alignment. - /// @param OffsetInBits Member offset. - /// @param Flags Flags to encode member attribute, e.g. private - /// @param Ty Parent type. - /// @param PropertyName Name of the Objective C property associated with - /// this ivar. - /// @param PropertyGetterName Name of the Objective C property getter - /// selector. - /// @param PropertySetterName Name of the Objective C property setter - /// selector. - /// @param PropertyAttributes Objective C property attributes. - DIDerivedType createObjCIVar(StringRef Name, DIFile File, - unsigned LineNo, uint64_t SizeInBits, - uint64_t AlignInBits, uint64_t OffsetInBits, - unsigned Flags, DIType Ty, - StringRef PropertyName = StringRef(), - StringRef PropertyGetterName = StringRef(), - StringRef PropertySetterName = StringRef(), - unsigned PropertyAttributes = 0); + DIDerivedType createStaticMemberType(DIDescriptor Scope, StringRef Name, + DIFile File, unsigned LineNo, + DIType Ty, unsigned Flags, + llvm::Constant *Val); /// createObjCIVar - Create debugging information entry for Objective-C /// instance variable. @@ -366,8 +351,8 @@ namespace llvm { /// @param LineNo Line number. /// @param ColumnNo Column Number. DITemplateValueParameter - createTemplateValueParameter(DIDescriptor Scope, StringRef Name, - DIType Ty, Value *Val, MDNode *File = nullptr, + createTemplateValueParameter(DIDescriptor Scope, StringRef Name, DIType Ty, + Constant *Val, MDNode *File = nullptr, unsigned LineNo = 0, unsigned ColumnNo = 0); /// \brief Create debugging information for a template template parameter. @@ -435,8 +420,9 @@ namespace llvm { /// includes return type at 0th index. /// @param Flags E.g.: LValueReference. /// These flags are used to emit dwarf attributes. - DICompositeType createSubroutineType(DIFile File, DIArray ParameterTypes, - unsigned Flags = 0); + DISubroutineType createSubroutineType(DIFile File, + DITypeArray ParameterTypes, + unsigned Flags = 0); /// createArtificialType - Create a new DIType with "artificial" flag set. DIType createArtificialType(DIType Ty); @@ -463,44 +449,22 @@ namespace llvm { /// through debug info anchors. void retainType(DIType T); - /// createUnspecifiedParameter - Create unspecified type descriptor + /// createUnspecifiedParameter - Create unspecified parameter type /// for a subroutine type. - DIDescriptor createUnspecifiedParameter(); + DIBasicType createUnspecifiedParameter(); /// getOrCreateArray - Get a DIArray, create one if required. - DIArray getOrCreateArray(ArrayRef Elements); + DIArray getOrCreateArray(ArrayRef Elements); + + /// getOrCreateTypeArray - Get a DITypeArray, create one if required. + DITypeArray getOrCreateTypeArray(ArrayRef Elements); /// getOrCreateSubrange - Create a descriptor for a value range. This /// implicitly uniques the values returned. DISubrange getOrCreateSubrange(int64_t Lo, int64_t Count); - /// createGlobalVariable - Create a new descriptor for the specified global. - /// @param Name Name of the variable. - /// @param File File where this variable is defined. - /// @param LineNo Line number. - /// @param Ty Variable Type. - /// @param isLocalToUnit Boolean flag indicate whether this variable is - /// externally visible or not. - /// @param Val llvm::Value of the variable. - DIGlobalVariable - createGlobalVariable(StringRef Name, DIFile File, unsigned LineNo, - DITypeRef Ty, bool isLocalToUnit, llvm::Value *Val); - /// \brief Create a new descriptor for the specified global. - /// @param Name Name of the variable. - /// @param LinkageName Mangled variable name. - /// @param File File where this variable is defined. - /// @param LineNo Line number. - /// @param Ty Variable Type. - /// @param isLocalToUnit Boolean flag indicate whether this variable is - /// externally visible or not. - /// @param Val llvm::Value of the variable. - DIGlobalVariable - createGlobalVariable(StringRef Name, StringRef LinkageName, DIFile File, - unsigned LineNo, DITypeRef Ty, bool isLocalToUnit, - llvm::Value *Val); - - /// createStaticVariable - Create a new descriptor for the specified + /// createGlobalVariable - Create a new descriptor for the specified /// variable. /// @param Context Variable scope. /// @param Name Name of the variable. @@ -512,12 +476,19 @@ namespace llvm { /// externally visible or not. /// @param Val llvm::Value of the variable. /// @param Decl Reference to the corresponding declaration. - DIGlobalVariable - createStaticVariable(DIDescriptor Context, StringRef Name, - StringRef LinkageName, DIFile File, unsigned LineNo, - DITypeRef Ty, bool isLocalToUnit, llvm::Value *Val, - MDNode *Decl = nullptr); + DIGlobalVariable createGlobalVariable(DIDescriptor Context, StringRef Name, + StringRef LinkageName, DIFile File, + unsigned LineNo, DITypeRef Ty, + bool isLocalToUnit, + llvm::Constant *Val, + MDNode *Decl = nullptr); + /// createTempGlobalVariableFwdDecl - Identical to createGlobalVariable + /// except that the resulting DbgNode is temporary and meant to be RAUWed. + DIGlobalVariable createTempGlobalVariableFwdDecl( + DIDescriptor Context, StringRef Name, StringRef LinkageName, + DIFile File, unsigned LineNo, DITypeRef Ty, bool isLocalToUnit, + llvm::Constant *Val, MDNode *Decl = nullptr); /// createLocalVariable - Create a new descriptor for the specified /// local variable. @@ -540,23 +511,18 @@ namespace llvm { unsigned Flags = 0, unsigned ArgNo = 0); - - /// createComplexVariable - Create a new descriptor for the specified + /// createExpression - Create a new descriptor for the specified /// variable which has a complex address expression for its address. - /// @param Tag Dwarf TAG. Usually DW_TAG_auto_variable or - /// DW_TAG_arg_variable. - /// @param Scope Variable scope. - /// @param Name Variable name. - /// @param F File where this variable is defined. - /// @param LineNo Line number. - /// @param Ty Variable Type /// @param Addr An array of complex address operations. - /// @param ArgNo If this variable is an argument then this argument's - /// number. 1 indicates 1st argument. - DIVariable createComplexVariable(unsigned Tag, DIDescriptor Scope, - StringRef Name, DIFile F, unsigned LineNo, - DITypeRef Ty, ArrayRef Addr, - unsigned ArgNo = 0); + DIExpression createExpression(ArrayRef Addr = None); + + /// createPieceExpression - Create a descriptor to describe one part + /// of aggregate variable that is fragmented across multiple Values. + /// + /// @param OffsetInBytes Offset of the piece in bytes. + /// @param SizeInBytes Size of the piece in bytes. + DIExpression createPieceExpression(unsigned OffsetInBytes, + unsigned SizeInBytes); /// createFunction - Create a new descriptor for the specified subprogram. /// See comments in DISubprogram for descriptions of these fields. @@ -586,6 +552,21 @@ namespace llvm { MDNode *TParam = nullptr, MDNode *Decl = nullptr); + /// createTempFunctionFwdDecl - Identical to createFunction, + /// except that the resulting DbgNode is meant to be RAUWed. + DISubprogram createTempFunctionFwdDecl(DIDescriptor Scope, StringRef Name, + StringRef LinkageName, + DIFile File, unsigned LineNo, + DICompositeType Ty, bool isLocalToUnit, + bool isDefinition, + unsigned ScopeLine, + unsigned Flags = 0, + bool isOptimized = false, + Function *Fn = nullptr, + MDNode *TParam = nullptr, + MDNode *Decl = nullptr); + + /// FIXME: this is added for dragonegg. Once we update dragonegg /// to call resolve function, this will be removed. DISubprogram createFunction(DIScopeRef Scope, StringRef Name, @@ -646,8 +627,9 @@ namespace llvm { /// lexical block as it crosses a file. /// @param Scope Lexical block. /// @param File Source file. - DILexicalBlockFile createLexicalBlockFile(DIDescriptor Scope, - DIFile File); + /// @param Discriminator DWARF path discriminator value. + DILexicalBlockFile createLexicalBlockFile(DIDescriptor Scope, DIFile File, + unsigned Discriminator = 0); /// createLexicalBlock - This creates a descriptor for a lexical block /// with the specified parent context. @@ -655,10 +637,8 @@ namespace llvm { /// @param File Source file. /// @param Line Line number. /// @param Col Column number. - /// @param Discriminator DWARF path discriminator value. DILexicalBlock createLexicalBlock(DIDescriptor Scope, DIFile File, - unsigned Line, unsigned Col, - unsigned Discriminator); + unsigned Line, unsigned Col); /// \brief Create a descriptor for an imported module. /// @param Context The scope this module is imported into @@ -679,7 +659,7 @@ namespace llvm { /// @param Decl The declaration (or definition) of a function, type, or /// variable /// @param Line Line number - DIImportedEntity createImportedDeclaration(DIScope Context, DIScope Decl, + DIImportedEntity createImportedDeclaration(DIScope Context, DIDescriptor Decl, unsigned Line, StringRef Name = StringRef()); DIImportedEntity createImportedDeclaration(DIScope Context, @@ -690,36 +670,52 @@ namespace llvm { /// insertDeclare - Insert a new llvm.dbg.declare intrinsic call. /// @param Storage llvm::Value of the variable /// @param VarInfo Variable's debug info descriptor. + /// @param Expr A complex location expression. /// @param InsertAtEnd Location for the new intrinsic. Instruction *insertDeclare(llvm::Value *Storage, DIVariable VarInfo, - BasicBlock *InsertAtEnd); + DIExpression Expr, BasicBlock *InsertAtEnd); /// insertDeclare - Insert a new llvm.dbg.declare intrinsic call. /// @param Storage llvm::Value of the variable /// @param VarInfo Variable's debug info descriptor. + /// @param Expr A complex location expression. /// @param InsertBefore Location for the new intrinsic. Instruction *insertDeclare(llvm::Value *Storage, DIVariable VarInfo, - Instruction *InsertBefore); - + DIExpression Expr, Instruction *InsertBefore); /// insertDbgValueIntrinsic - Insert a new llvm.dbg.value intrinsic call. /// @param Val llvm::Value of the variable /// @param Offset Offset /// @param VarInfo Variable's debug info descriptor. + /// @param Expr A complex location expression. /// @param InsertAtEnd Location for the new intrinsic. Instruction *insertDbgValueIntrinsic(llvm::Value *Val, uint64_t Offset, - DIVariable VarInfo, + DIVariable VarInfo, DIExpression Expr, BasicBlock *InsertAtEnd); /// insertDbgValueIntrinsic - Insert a new llvm.dbg.value intrinsic call. /// @param Val llvm::Value of the variable /// @param Offset Offset /// @param VarInfo Variable's debug info descriptor. + /// @param Expr A complex location expression. /// @param InsertBefore Location for the new intrinsic. Instruction *insertDbgValueIntrinsic(llvm::Value *Val, uint64_t Offset, - DIVariable VarInfo, + DIVariable VarInfo, DIExpression Expr, Instruction *InsertBefore); + /// \brief Replace the vtable holder in the given composite type. + /// + /// If this creates a self reference, it may orphan some unresolved cycles + /// in the operands of \c T, so \a DIBuilder needs to track that. + void replaceVTableHolder(DICompositeType &T, DICompositeType VTableHolder); + + /// \brief Replace arrays on a composite type. + /// + /// If \c T is resolved, but the arrays aren't -- which can happen if \c T + /// has a self-reference -- \a DIBuilder needs to track the array to + /// resolve cycles. + void replaceArrays(DICompositeType &T, DIArray Elements, + DIArray TParems = DIArray()); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/IR/DataLayout.h b/contrib/llvm/include/llvm/IR/DataLayout.h index 877029f92f0b..a9e75955ce74 100644 --- a/contrib/llvm/include/llvm/IR/DataLayout.h +++ b/contrib/llvm/include/llvm/IR/DataLayout.h @@ -27,7 +27,8 @@ #include "llvm/Pass.h" #include "llvm/Support/DataTypes.h" -// this needs to be outside of the namespace, to avoid conflict with llvm-c decl +// This needs to be outside of the namespace, to avoid conflict with llvm-c +// decl. typedef struct LLVMOpaqueTargetData *LLVMTargetDataRef; namespace llvm { @@ -45,79 +46,71 @@ class ArrayRef; /// Enum used to categorize the alignment types stored by LayoutAlignElem enum AlignTypeEnum { - INVALID_ALIGN = 0, ///< An invalid alignment - INTEGER_ALIGN = 'i', ///< Integer type alignment - VECTOR_ALIGN = 'v', ///< Vector type alignment - FLOAT_ALIGN = 'f', ///< Floating point type alignment - AGGREGATE_ALIGN = 'a' ///< Aggregate alignment + INVALID_ALIGN = 0, + INTEGER_ALIGN = 'i', + VECTOR_ALIGN = 'v', + FLOAT_ALIGN = 'f', + AGGREGATE_ALIGN = 'a' }; -/// Layout alignment element. +/// \brief Layout alignment element. /// /// Stores the alignment data associated with a given alignment type (integer, /// vector, float) and type bit width. /// -/// @note The unusual order of elements in the structure attempts to reduce +/// \note The unusual order of elements in the structure attempts to reduce /// padding and make the structure slightly more cache friendly. struct LayoutAlignElem { - unsigned AlignType : 8; ///< Alignment type (AlignTypeEnum) - unsigned TypeBitWidth : 24; ///< Type bit width - unsigned ABIAlign : 16; ///< ABI alignment for this type/bitw - unsigned PrefAlign : 16; ///< Pref. alignment for this type/bitw + /// \brief Alignment type from \c AlignTypeEnum + unsigned AlignType : 8; + unsigned TypeBitWidth : 24; + unsigned ABIAlign : 16; + unsigned PrefAlign : 16; - /// Initializer static LayoutAlignElem get(AlignTypeEnum align_type, unsigned abi_align, unsigned pref_align, uint32_t bit_width); - /// Equality predicate bool operator==(const LayoutAlignElem &rhs) const; }; -/// Layout pointer alignment element. +/// \brief Layout pointer alignment element. /// /// Stores the alignment data associated with a given pointer and address space. /// -/// @note The unusual order of elements in the structure attempts to reduce +/// \note The unusual order of elements in the structure attempts to reduce /// padding and make the structure slightly more cache friendly. struct PointerAlignElem { - unsigned ABIAlign; ///< ABI alignment for this type/bitw - unsigned PrefAlign; ///< Pref. alignment for this type/bitw - uint32_t TypeByteWidth; ///< Type byte width - uint32_t AddressSpace; ///< Address space for the pointer type + unsigned ABIAlign; + unsigned PrefAlign; + uint32_t TypeByteWidth; + uint32_t AddressSpace; /// Initializer static PointerAlignElem get(uint32_t AddressSpace, unsigned ABIAlign, - unsigned PrefAlign, uint32_t TypeByteWidth); - /// Equality predicate + unsigned PrefAlign, uint32_t TypeByteWidth); bool operator==(const PointerAlignElem &rhs) const; }; -/// This class holds a parsed version of the target data layout string in a -/// module and provides methods for querying it. The target data layout string -/// is specified *by the target* - a frontend generating LLVM IR is required to -/// generate the right target data for the target being codegen'd to. +/// \brief A parsed version of the target data layout string in and methods for +/// querying it. +/// +/// The target data layout string is specified *by the target* - a frontend +/// generating LLVM IR is required to generate the right target data for the +/// target being codegen'd to. class DataLayout { private: - bool LittleEndian; ///< Defaults to false - unsigned StackNaturalAlign; ///< Stack natural alignment + /// Defaults to false. + bool BigEndian; - enum ManglingModeT { - MM_None, - MM_ELF, - MM_MachO, - MM_WINCOFF, - MM_Mips - }; + unsigned StackNaturalAlign; + + enum ManglingModeT { MM_None, MM_ELF, MM_MachO, MM_WINCOFF, MM_Mips }; ManglingModeT ManglingMode; - SmallVector LegalIntWidths; ///< Legal Integers. + SmallVector LegalIntWidths; - /// Alignments - Where the primitive type alignment data is stored. - /// - /// @sa reset(). - /// @note Could support multiple size pointer alignments, e.g., 32-bit - /// pointers vs. 64-bit pointers by extending LayoutAlignment, but for now, - /// we don't. + /// \brief Primitive type alignment data. SmallVector Alignments; + typedef SmallVector PointersTy; PointersTy Pointers; @@ -128,31 +121,28 @@ private: PointersTy::iterator findPointerLowerBound(uint32_t AddressSpace); - /// InvalidAlignmentElem - This member is a signal that a requested alignment - /// type and bit width were not found in the SmallVector. + /// This member is a signal that a requested alignment type and bit width were + /// not found in the SmallVector. static const LayoutAlignElem InvalidAlignmentElem; - /// InvalidPointerElem - This member is a signal that a requested pointer - /// type and bit width were not found in the DenseSet. + /// This member is a signal that a requested pointer type and bit width were + /// not found in the DenseSet. static const PointerAlignElem InvalidPointerElem; // The StructType -> StructLayout map. mutable void *LayoutMap; - //! Set/initialize target alignments void setAlignment(AlignTypeEnum align_type, unsigned abi_align, unsigned pref_align, uint32_t bit_width); unsigned getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width, bool ABIAlign, Type *Ty) const; - - //! Set/initialize pointer alignments void setPointerAlignment(uint32_t AddrSpace, unsigned ABIAlign, unsigned PrefAlign, uint32_t TypeByteWidth); - //! Internal helper method that returns requested alignment for type. + /// Internal helper method that returns requested alignment for type. unsigned getAlignment(Type *Ty, bool abi_or_pref) const; - /// Valid alignment predicate. + /// \brief Valid alignment predicate. /// /// Predicate that tests a LayoutAlignElem reference returned by get() against /// InvalidAlignmentElem. @@ -160,10 +150,10 @@ private: return &align != &InvalidAlignmentElem; } - /// Valid pointer predicate. + /// \brief Valid pointer predicate. /// - /// Predicate that tests a PointerAlignElem reference returned by get() against - /// InvalidPointerElem. + /// Predicate that tests a PointerAlignElem reference returned by get() + /// against \c InvalidPointerElem. bool validPointer(const PointerAlignElem &align) const { return &align != &InvalidPointerElem; } @@ -184,11 +174,13 @@ public: /// Initialize target data from properties stored in the module. explicit DataLayout(const Module *M); + void init(const Module *M); + DataLayout(const DataLayout &DL) : LayoutMap(nullptr) { *this = DL; } DataLayout &operator=(const DataLayout &DL) { clear(); - LittleEndian = DL.isLittleEndian(); + BigEndian = DL.isBigEndian(); StackNaturalAlign = DL.StackNaturalAlign; ManglingMode = DL.ManglingMode; LegalIntWidths = DL.LegalIntWidths; @@ -200,27 +192,28 @@ public: bool operator==(const DataLayout &Other) const; bool operator!=(const DataLayout &Other) const { return !(*this == Other); } - ~DataLayout(); // Not virtual, do not subclass this class + ~DataLayout(); // Not virtual, do not subclass this class /// Parse a data layout string (with fallback to default values). void reset(StringRef LayoutDescription); /// Layout endianness... - bool isLittleEndian() const { return LittleEndian; } - bool isBigEndian() const { return !LittleEndian; } + bool isLittleEndian() const { return !BigEndian; } + bool isBigEndian() const { return BigEndian; } - /// getStringRepresentation - Return the string representation of the - /// DataLayout. This representation is in the same format accepted by the - /// string constructor above. + /// \brief Returns the string representation of the DataLayout. + /// + /// This representation is in the same format accepted by the string + /// constructor above. std::string getStringRepresentation() const; - /// isLegalInteger - This function returns true if the specified type is - /// known to be a native integer type supported by the CPU. For example, - /// i64 is not native on most 32-bit CPUs and i37 is not native on any known - /// one. This returns false if the integer width is not legal. + /// \brief Returns true if the specified type is known to be a native integer + /// type supported by the CPU. + /// + /// For example, i64 is not native on most 32-bit CPUs and i37 is not native + /// on any known one. This returns false if the integer width is not legal. /// /// The width is specified in bits. - /// bool isLegalInteger(unsigned Width) const { for (unsigned LegalIntWidth : LegalIntWidths) if (LegalIntWidth == Width) @@ -228,22 +221,20 @@ public: return false; } - bool isIllegalInteger(unsigned Width) const { - return !isLegalInteger(Width); - } + bool isIllegalInteger(unsigned Width) const { return !isLegalInteger(Width); } /// Returns true if the given alignment exceeds the natural stack alignment. bool exceedsNaturalStackAlignment(unsigned Align) const { return (StackNaturalAlign != 0) && (Align > StackNaturalAlign); } + unsigned getStackAlignment() const { return StackNaturalAlign; } + bool hasMicrosoftFastStdCallMangling() const { return ManglingMode == MM_WINCOFF; } - bool hasLinkerPrivateGlobalPrefix() const { - return ManglingMode == MM_MachO; - } + bool hasLinkerPrivateGlobalPrefix() const { return ManglingMode == MM_MachO; } const char *getLinkerPrivateGlobalPrefix() const { if (ManglingMode == MM_MachO) @@ -281,10 +272,11 @@ public: static const char *getManglingComponent(const Triple &T); - /// fitsInLegalInteger - This function returns true if the specified type fits - /// in a native integer type supported by the CPU. For example, if the CPU - /// only supports i32 as a native integer type, then i27 fits in a legal - /// integer type but i45 does not. + /// \brief Returns true if the specified type fits in a native integer type + /// supported by the CPU. + /// + /// For example, if the CPU only supports i32 as a native integer type, then + /// i27 fits in a legal integer type but i45 does not. bool fitsInLegalInteger(unsigned Width) const { for (unsigned LegalIntWidth : LegalIntWidths) if (Width <= LegalIntWidth) @@ -342,118 +334,116 @@ public: /// [*] The alloc size depends on the alignment, and thus on the target. /// These values are for x86-32 linux. - /// getTypeSizeInBits - Return the number of bits necessary to hold the - /// specified type. For example, returns 36 for i36 and 80 for x86_fp80. - /// The type passed must have a size (Type::isSized() must return true). + /// \brief Returns the number of bits necessary to hold the specified type. + /// + /// For example, returns 36 for i36 and 80 for x86_fp80. The type passed must + /// have a size (Type::isSized() must return true). uint64_t getTypeSizeInBits(Type *Ty) const; - /// getTypeStoreSize - Return the maximum number of bytes that may be - /// overwritten by storing the specified type. For example, returns 5 - /// for i36 and 10 for x86_fp80. + /// \brief Returns the maximum number of bytes that may be overwritten by + /// storing the specified type. + /// + /// For example, returns 5 for i36 and 10 for x86_fp80. uint64_t getTypeStoreSize(Type *Ty) const { - return (getTypeSizeInBits(Ty)+7)/8; + return (getTypeSizeInBits(Ty) + 7) / 8; } - /// getTypeStoreSizeInBits - Return the maximum number of bits that may be - /// overwritten by storing the specified type; always a multiple of 8. For - /// example, returns 40 for i36 and 80 for x86_fp80. + /// \brief Returns the maximum number of bits that may be overwritten by + /// storing the specified type; always a multiple of 8. + /// + /// For example, returns 40 for i36 and 80 for x86_fp80. uint64_t getTypeStoreSizeInBits(Type *Ty) const { - return 8*getTypeStoreSize(Ty); + return 8 * getTypeStoreSize(Ty); } - /// getTypeAllocSize - Return the offset in bytes between successive objects - /// of the specified type, including alignment padding. This is the amount - /// that alloca reserves for this type. For example, returns 12 or 16 for - /// x86_fp80, depending on alignment. + /// \brief Returns the offset in bytes between successive objects of the + /// specified type, including alignment padding. + /// + /// This is the amount that alloca reserves for this type. For example, + /// returns 12 or 16 for x86_fp80, depending on alignment. uint64_t getTypeAllocSize(Type *Ty) const { // Round up to the next alignment boundary. - return RoundUpAlignment(getTypeStoreSize(Ty), getABITypeAlignment(Ty)); + return RoundUpToAlignment(getTypeStoreSize(Ty), getABITypeAlignment(Ty)); } - /// getTypeAllocSizeInBits - Return the offset in bits between successive - /// objects of the specified type, including alignment padding; always a - /// multiple of 8. This is the amount that alloca reserves for this type. - /// For example, returns 96 or 128 for x86_fp80, depending on alignment. + /// \brief Returns the offset in bits between successive objects of the + /// specified type, including alignment padding; always a multiple of 8. + /// + /// This is the amount that alloca reserves for this type. For example, + /// returns 96 or 128 for x86_fp80, depending on alignment. uint64_t getTypeAllocSizeInBits(Type *Ty) const { - return 8*getTypeAllocSize(Ty); + return 8 * getTypeAllocSize(Ty); } - /// getABITypeAlignment - Return the minimum ABI-required alignment for the - /// specified type. + /// \brief Returns the minimum ABI-required alignment for the specified type. unsigned getABITypeAlignment(Type *Ty) const; - /// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for - /// an integer type of the specified bitwidth. + /// \brief Returns the minimum ABI-required alignment for an integer type of + /// the specified bitwidth. unsigned getABIIntegerTypeAlignment(unsigned BitWidth) const; - /// getPrefTypeAlignment - Return the preferred stack/global alignment for - /// the specified type. This is always at least as good as the ABI alignment. + /// \brief Returns the preferred stack/global alignment for the specified + /// type. + /// + /// This is always at least as good as the ABI alignment. unsigned getPrefTypeAlignment(Type *Ty) const; - /// getPreferredTypeAlignmentShift - Return the preferred alignment for the - /// specified type, returned as log2 of the value (a shift amount). + /// \brief Returns the preferred alignment for the specified type, returned as + /// log2 of the value (a shift amount). unsigned getPreferredTypeAlignmentShift(Type *Ty) const; - /// getIntPtrType - Return an integer type with size at least as big as that - /// of a pointer in the given address space. + /// \brief Returns an integer type with size at least as big as that of a + /// pointer in the given address space. IntegerType *getIntPtrType(LLVMContext &C, unsigned AddressSpace = 0) const; - /// getIntPtrType - Return an integer (vector of integer) type with size at - /// least as big as that of a pointer of the given pointer (vector of pointer) - /// type. + /// \brief Returns an integer (vector of integer) type with size at least as + /// big as that of a pointer of the given pointer (vector of pointer) type. Type *getIntPtrType(Type *) const; - /// getSmallestLegalIntType - Return the smallest integer type with size at - /// least as big as Width bits. + /// \brief Returns the smallest integer type with size at least as big as + /// Width bits. Type *getSmallestLegalIntType(LLVMContext &C, unsigned Width = 0) const; - /// getLargestLegalIntType - Return the largest legal integer type, or null if - /// none are set. + /// \brief Returns the largest legal integer type, or null if none are set. Type *getLargestLegalIntType(LLVMContext &C) const { unsigned LargestSize = getLargestLegalIntTypeSize(); return (LargestSize == 0) ? nullptr : Type::getIntNTy(C, LargestSize); } - /// getLargestLegalIntTypeSize - Return the size of largest legal integer - /// type size, or 0 if none are set. + /// \brief Returns the size of largest legal integer type size, or 0 if none + /// are set. unsigned getLargestLegalIntTypeSize() const; - /// getIndexedOffset - return the offset from the beginning of the type for - /// the specified indices. This is used to implement getelementptr. + /// \brief Returns the offset from the beginning of the type for the specified + /// indices. + /// + /// This is used to implement getelementptr. uint64_t getIndexedOffset(Type *Ty, ArrayRef Indices) const; - /// getStructLayout - Return a StructLayout object, indicating the alignment - /// of the struct, its size, and the offsets of its fields. Note that this - /// information is lazily cached. + /// \brief Returns a StructLayout object, indicating the alignment of the + /// struct, its size, and the offsets of its fields. + /// + /// Note that this information is lazily cached. const StructLayout *getStructLayout(StructType *Ty) const; - /// getPreferredAlignment - Return the preferred alignment of the specified - /// global. This includes an explicitly requested alignment (if the global - /// has one). + /// \brief Returns the preferred alignment of the specified global. + /// + /// This includes an explicitly requested alignment (if the global has one). unsigned getPreferredAlignment(const GlobalVariable *GV) const; - /// getPreferredAlignmentLog - Return the preferred alignment of the - /// specified global, returned in log form. This includes an explicitly - /// requested alignment (if the global has one). + /// \brief Returns the preferred alignment of the specified global, returned + /// in log form. + /// + /// This includes an explicitly requested alignment (if the global has one). unsigned getPreferredAlignmentLog(const GlobalVariable *GV) const; - - /// RoundUpAlignment - Round the specified value up to the next alignment - /// boundary specified by Alignment. For example, 7 rounded up to an - /// alignment boundary of 4 is 8. 8 rounded up to the alignment boundary of 4 - /// is 8 because it is already aligned. - template - static UIntTy RoundUpAlignment(UIntTy Val, unsigned Alignment) { - assert((Alignment & (Alignment-1)) == 0 && "Alignment must be power of 2!"); - return (Val + (Alignment-1)) & ~UIntTy(Alignment-1); - } }; inline DataLayout *unwrap(LLVMTargetDataRef P) { - return reinterpret_cast(P); + return reinterpret_cast(P); } inline LLVMTargetDataRef wrap(const DataLayout *P) { - return reinterpret_cast(const_cast(P)); + return reinterpret_cast(const_cast(P)); } class DataLayoutPass : public ImmutablePass { @@ -466,40 +456,28 @@ public: const DataLayout &getDataLayout() const { return DL; } - // For use with the C API. C++ code should always use the constructor that - // takes a module. - explicit DataLayoutPass(const DataLayout &DL); - - explicit DataLayoutPass(const Module *M); - static char ID; // Pass identification, replacement for typeid + + bool doFinalization(Module &M) override; + bool doInitialization(Module &M) override; }; -/// StructLayout - used to lazily calculate structure layout information for a -/// target machine, based on the DataLayout structure. -/// +/// Used to lazily calculate structure layout information for a target machine, +/// based on the DataLayout structure. class StructLayout { uint64_t StructSize; unsigned StructAlignment; unsigned NumElements; - uint64_t MemberOffsets[1]; // variable sized array! + uint64_t MemberOffsets[1]; // variable sized array! public: + uint64_t getSizeInBytes() const { return StructSize; } - uint64_t getSizeInBytes() const { - return StructSize; - } + uint64_t getSizeInBits() const { return 8 * StructSize; } - uint64_t getSizeInBits() const { - return 8*StructSize; - } + unsigned getAlignment() const { return StructAlignment; } - unsigned getAlignment() const { - return StructAlignment; - } - - /// getElementContainingOffset - Given a valid byte offset into the structure, - /// return the structure index that contains it. - /// + /// \brief Given a valid byte offset into the structure, returns the structure + /// index that contains it. unsigned getElementContainingOffset(uint64_t Offset) const; uint64_t getElementOffset(unsigned Idx) const { @@ -508,15 +486,14 @@ public: } uint64_t getElementOffsetInBits(unsigned Idx) const { - return getElementOffset(Idx)*8; + return getElementOffset(Idx) * 8; } private: - friend class DataLayout; // Only DataLayout can create this class + friend class DataLayout; // Only DataLayout can create this class StructLayout(StructType *ST, const DataLayout &DL); }; - // The implementation of this method is provided inline as it is particularly // well suited to constant folding when called on a specific Type subclass. inline uint64_t DataLayout::getTypeSizeInBits(Type *Ty) const { @@ -546,7 +523,7 @@ inline uint64_t DataLayout::getTypeSizeInBits(Type *Ty) const { case Type::PPC_FP128TyID: case Type::FP128TyID: return 128; - // In memory objects this is always aligned to a higher boundary, but + // In memory objects this is always aligned to a higher boundary, but // only 80 bits contain information. case Type::X86_FP80TyID: return 80; diff --git a/contrib/llvm/include/llvm/IR/DebugInfo.h b/contrib/llvm/include/llvm/IR/DebugInfo.h index 088eb9f01049..5c85d6d52824 100644 --- a/contrib/llvm/include/llvm/IR/DebugInfo.h +++ b/contrib/llvm/include/llvm/IR/DebugInfo.h @@ -18,13 +18,15 @@ #define LLVM_IR_DEBUGINFO_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Dwarf.h" +#include "llvm/Support/ErrorHandling.h" +#include namespace llvm { class BasicBlock; @@ -37,6 +39,7 @@ class Value; class DbgDeclareInst; class DbgValueInst; class Instruction; +class Metadata; class MDNode; class MDString; class NamedMDNode; @@ -52,21 +55,78 @@ class DIType; class DIScope; class DIObjCProperty; -/// Maps from type identifier to the actual MDNode. +/// \brief Maps from type identifier to the actual MDNode. typedef DenseMap DITypeIdentifierMap; -/// DIDescriptor - A thin wraper around MDNode to access encoded debug info. -/// This should not be stored in a container, because the underlying MDNode -/// may change in certain situations. +class DIHeaderFieldIterator + : public std::iterator { + StringRef Header; + StringRef Current; + +public: + DIHeaderFieldIterator() {} + DIHeaderFieldIterator(StringRef Header) + : Header(Header), Current(Header.slice(0, Header.find('\0'))) {} + StringRef operator*() const { return Current; } + const StringRef * operator->() const { return &Current; } + DIHeaderFieldIterator &operator++() { + increment(); + return *this; + } + DIHeaderFieldIterator operator++(int) { + DIHeaderFieldIterator X(*this); + increment(); + return X; + } + bool operator==(const DIHeaderFieldIterator &X) const { + return Current.data() == X.Current.data(); + } + bool operator!=(const DIHeaderFieldIterator &X) const { + return !(*this == X); + } + + StringRef getHeader() const { return Header; } + StringRef getCurrent() const { return Current; } + StringRef getPrefix() const { + if (Current.begin() == Header.begin()) + return StringRef(); + return Header.slice(0, Current.begin() - Header.begin() - 1); + } + StringRef getSuffix() const { + if (Current.end() == Header.end()) + return StringRef(); + return Header.slice(Current.end() - Header.begin() + 1, StringRef::npos); + } + +private: + void increment() { + assert(Current.data() != nullptr && "Cannot increment past the end"); + StringRef Suffix = getSuffix(); + Current = Suffix.slice(0, Suffix.find('\0')); + } +}; + +/// \brief A thin wraper around MDNode to access encoded debug info. +/// +/// This should not be stored in a container, because the underlying MDNode may +/// change in certain situations. class DIDescriptor { // Befriends DIRef so DIRef can befriend the protected member // function: getFieldAs. template friend class DIRef; public: + /// \brief Accessibility flags. + /// + /// The three accessibility flags are mutually exclusive and rolled together + /// in the first two bits. enum { - FlagPrivate = 1 << 0, - FlagProtected = 1 << 1, + FlagAccessibility = 1 << 0 | 1 << 1, + FlagPrivate = 1, + FlagProtected = 2, + FlagPublic = 3, + FlagFwdDecl = 1 << 2, FlagAppleBlock = 1 << 3, FlagBlockByrefStruct = 1 << 4, @@ -108,8 +168,9 @@ public: bool Verify() const; - operator MDNode *() const { return const_cast(DbgNode); } - MDNode *operator->() const { return const_cast(DbgNode); } + MDNode *get() const { return const_cast(DbgNode); } + operator MDNode *() const { return get(); } + MDNode *operator->() const { return get(); } // An explicit operator bool so that we can do testing of DI values // easily. @@ -121,12 +182,36 @@ public: bool operator==(DIDescriptor Other) const { return DbgNode == Other.DbgNode; } bool operator!=(DIDescriptor Other) const { return !operator==(Other); } - uint16_t getTag() const { - return getUnsignedField(0) & ~LLVMDebugVersionMask; + StringRef getHeader() const { + return getStringField(0); } + size_t getNumHeaderFields() const { + return std::distance(DIHeaderFieldIterator(getHeader()), + DIHeaderFieldIterator()); + } + + StringRef getHeaderField(unsigned Index) const { + // Since callers expect an empty string for out-of-range accesses, we can't + // use std::advance() here. + for (DIHeaderFieldIterator I(getHeader()), E; I != E; ++I, --Index) + if (!Index) + return *I; + return StringRef(); + } + + template T getHeaderFieldAs(unsigned Index) const { + T Int; + if (getHeaderField(Index).getAsInteger(0, Int)) + return 0; + return Int; + } + + uint16_t getTag() const { return getHeaderFieldAs(0); } + bool isDerivedType() const; bool isCompositeType() const; + bool isSubroutineType() const; bool isBasicType() const; bool isVariable() const; bool isSubprogram() const; @@ -140,20 +225,21 @@ public: bool isSubrange() const; bool isEnumerator() const; bool isType() const; - bool isUnspecifiedParameter() const; bool isTemplateTypeParameter() const; bool isTemplateValueParameter() const; bool isObjCProperty() const; bool isImportedEntity() const; + bool isExpression() const; - /// print - print descriptor. void print(raw_ostream &OS) const; - - /// dump - print descriptor to dbgs() with a newline. void dump() const; + + /// \brief Replace all uses of debug info referenced by this descriptor. + void replaceAllUsesWith(LLVMContext &VMContext, DIDescriptor D); + void replaceAllUsesWith(MDNode *D); }; -/// DISubrange - This is used to represent ranges, for array bounds. +/// \brief This is used to represent ranges, for array bounds. class DISubrange : public DIDescriptor { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; @@ -161,23 +247,27 @@ class DISubrange : public DIDescriptor { public: explicit DISubrange(const MDNode *N = nullptr) : DIDescriptor(N) {} - int64_t getLo() const { return getInt64Field(1); } - int64_t getCount() const { return getInt64Field(2); } + int64_t getLo() const { return getHeaderFieldAs(1); } + int64_t getCount() const { return getHeaderFieldAs(2); } bool Verify() const; }; -/// DIArray - This descriptor holds an array of descriptors. -class DIArray : public DIDescriptor { +/// \brief This descriptor holds an array of nodes with type T. +template class DITypedArray : public DIDescriptor { public: - explicit DIArray(const MDNode *N = nullptr) : DIDescriptor(N) {} - - unsigned getNumElements() const; - DIDescriptor getElement(unsigned Idx) const { - return getDescriptorField(Idx); + explicit DITypedArray(const MDNode *N = nullptr) : DIDescriptor(N) {} + unsigned getNumElements() const { + return DbgNode ? DbgNode->getNumOperands() : 0; + } + T getElement(unsigned Idx) const { + return getFieldAs(Idx); } }; -/// DIEnumerator - A wrapper for an enumerator (e.g. X and Y in 'enum {X,Y}'). +typedef DITypedArray DIArray; + +/// \brief A wrapper for an enumerator (e.g. X and Y in 'enum {X,Y}'). +/// /// FIXME: it seems strange that this doesn't have either a reference to the /// type/precision or a file/line pair for location info. class DIEnumerator : public DIDescriptor { @@ -187,16 +277,17 @@ class DIEnumerator : public DIDescriptor { public: explicit DIEnumerator(const MDNode *N = nullptr) : DIDescriptor(N) {} - StringRef getName() const { return getStringField(1); } - int64_t getEnumValue() const { return getInt64Field(2); } + StringRef getName() const { return getHeaderField(1); } + int64_t getEnumValue() const { return getHeaderFieldAs(2); } bool Verify() const; }; template class DIRef; typedef DIRef DIScopeRef; typedef DIRef DITypeRef; +typedef DITypedArray DITypeArray; -/// DIScope - A base class for various scopes. +/// \brief A base class for various scopes. /// /// Although, implementation-wise, DIScope is the parent class of most /// other DIxxx classes, including DIType and its descendants, most of @@ -212,21 +303,28 @@ protected: public: explicit DIScope(const MDNode *N = nullptr) : DIDescriptor(N) {} - /// Gets the parent scope for this scope node or returns a - /// default constructed scope. + /// \brief Get the parent scope. + /// + /// Gets the parent scope for this scope node or returns a default + /// constructed scope. DIScopeRef getContext() const; + /// \brief Get the scope name. + /// /// If the scope node has a name, return that, else return an empty string. StringRef getName() const; StringRef getFilename() const; StringRef getDirectory() const; - /// Generate a reference to this DIScope. Uses the type identifier instead - /// of the actual MDNode if possible, to help type uniquing. + /// \brief Generate a reference to this DIScope. + /// + /// Uses the type identifier instead of the actual MDNode if possible, to + /// help type uniquing. DIScopeRef getRef() const; }; -/// Represents reference to a DIDescriptor, abstracts over direct and -/// identifier-based metadata references. +/// \brief Represents reference to a DIDescriptor. +/// +/// Abstracts over direct and identifier-based metadata references. template class DIRef { template friend DescTy DIDescriptor::getFieldAs(unsigned Elt) const; @@ -234,15 +332,16 @@ template class DIRef { friend DIScopeRef DIScope::getRef() const; friend class DIType; - /// Val can be either a MDNode or a MDString, in the latter, - /// MDString specifies the type identifier. - const Value *Val; - explicit DIRef(const Value *V); + /// \brief Val can be either a MDNode or a MDString. + /// + /// In the latter, MDString specifies the type identifier. + const Metadata *Val; + explicit DIRef(const Metadata *V); public: T resolve(const DITypeIdentifierMap &Map) const; StringRef getName() const; - operator Value *() const { return const_cast(Val); } + operator Metadata *() const { return const_cast(Val); } }; template @@ -273,17 +372,18 @@ template StringRef DIRef::getName() const { return MS->getString(); } -/// Specialize getFieldAs to handle fields that are references to DIScopes. +/// \brief Handle fields that are references to DIScopes. template <> DIScopeRef DIDescriptor::getFieldAs(unsigned Elt) const; -/// Specialize DIRef constructor for DIScopeRef. -template <> DIRef::DIRef(const Value *V); +/// \brief Specialize DIRef constructor for DIScopeRef. +template <> DIRef::DIRef(const Metadata *V); -/// Specialize getFieldAs to handle fields that are references to DITypes. +/// \brief Handle fields that are references to DITypes. template <> DITypeRef DIDescriptor::getFieldAs(unsigned Elt) const; -/// Specialize DIRef constructor for DITypeRef. -template <> DIRef::DIRef(const Value *V); +/// \brief Specialize DIRef constructor for DITypeRef. +template <> DIRef::DIRef(const Metadata *V); -/// DIType - This is a wrapper for a type. +/// \briefThis is a wrapper for a type. +/// /// FIXME: Types should be factored much better so that CV qualifiers and /// others do not require a huge and empty descriptor full of zeros. class DIType : public DIScope { @@ -299,22 +399,35 @@ public: return DITypeRef(&*getRef()); } - /// Verify - Verify that a type descriptor is well formed. bool Verify() const; DIScopeRef getContext() const { return getFieldAs(2); } - StringRef getName() const { return getStringField(3); } - unsigned getLineNumber() const { return getUnsignedField(4); } - uint64_t getSizeInBits() const { return getUInt64Field(5); } - uint64_t getAlignInBits() const { return getUInt64Field(6); } + StringRef getName() const { return getHeaderField(1); } + unsigned getLineNumber() const { + return getHeaderFieldAs(2); + } + uint64_t getSizeInBits() const { + return getHeaderFieldAs(3); + } + uint64_t getAlignInBits() const { + return getHeaderFieldAs(4); + } // FIXME: Offset is only used for DW_TAG_member nodes. Making every type // carry this is just plain insane. - uint64_t getOffsetInBits() const { return getUInt64Field(7); } - unsigned getFlags() const { return getUnsignedField(8); } - bool isPrivate() const { return (getFlags() & FlagPrivate) != 0; } - bool isProtected() const { return (getFlags() & FlagProtected) != 0; } + uint64_t getOffsetInBits() const { + return getHeaderFieldAs(5); + } + unsigned getFlags() const { return getHeaderFieldAs(6); } + bool isPrivate() const { + return (getFlags() & FlagAccessibility) == FlagPrivate; + } + bool isProtected() const { + return (getFlags() & FlagAccessibility) == FlagProtected; + } + bool isPublic() const { + return (getFlags() & FlagAccessibility) == FlagPublic; + } bool isForwardDecl() const { return (getFlags() & FlagFwdDecl) != 0; } - // isAppleBlock - Return true if this is the Apple Blocks extension. bool isAppleBlockExtension() const { return (getFlags() & FlagAppleBlock) != 0; } @@ -336,27 +449,22 @@ public: return (getFlags() & FlagRValueReference) != 0; } bool isValid() const { return DbgNode && isType(); } - - /// replaceAllUsesWith - Replace all uses of debug info referenced by - /// this descriptor. - void replaceAllUsesWith(LLVMContext &VMContext, DIDescriptor D); - void replaceAllUsesWith(MDNode *D); }; -/// DIBasicType - A basic type, like 'int' or 'float'. +/// \brief A basic type, like 'int' or 'float'. class DIBasicType : public DIType { public: explicit DIBasicType(const MDNode *N = nullptr) : DIType(N) {} - unsigned getEncoding() const { return getUnsignedField(9); } + unsigned getEncoding() const { return getHeaderFieldAs(7); } - /// Verify - Verify that a basic type descriptor is well formed. bool Verify() const; }; -/// DIDerivedType - A simple derived type, like a const qualified type, -/// a typedef, a pointer or reference, et cetera. Or, a data member of -/// a class/struct/union. +/// \brief A simple derived type +/// +/// Like a const qualified type, a typedef, a pointer or reference, et cetera. +/// Or, a data member of a class/struct/union. class DIDerivedType : public DIType { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; @@ -364,62 +472,96 @@ class DIDerivedType : public DIType { public: explicit DIDerivedType(const MDNode *N = nullptr) : DIType(N) {} - DITypeRef getTypeDerivedFrom() const { return getFieldAs(9); } + DITypeRef getTypeDerivedFrom() const { return getFieldAs(3); } - /// getObjCProperty - Return property node, if this ivar is - /// associated with one. + /// \brief Return property node, if this ivar is associated with one. MDNode *getObjCProperty() const; DITypeRef getClassType() const { assert(getTag() == dwarf::DW_TAG_ptr_to_member_type); - return getFieldAs(10); + return getFieldAs(4); } Constant *getConstant() const { assert((getTag() == dwarf::DW_TAG_member) && isStaticMember()); - return getConstantField(10); + return getConstantField(4); } - /// Verify - Verify that a derived type descriptor is well formed. bool Verify() const; }; -/// DICompositeType - This descriptor holds a type that can refer to multiple -/// other types, like a function or struct. +/// \brief Types that refer to multiple other types. +/// +/// This descriptor holds a type that can refer to multiple other types, like a +/// function or struct. +/// /// DICompositeType is derived from DIDerivedType because some /// composite types (such as enums) can be derived from basic types // FIXME: Make this derive from DIType directly & just store the // base type in a single DIType field. class DICompositeType : public DIDerivedType { + friend class DIBuilder; friend class DIDescriptor; void printInternal(raw_ostream &OS) const; + /// \brief Set the array of member DITypes. + void setArraysHelper(MDNode *Elements, MDNode *TParams); + public: explicit DICompositeType(const MDNode *N = nullptr) : DIDerivedType(N) {} - DIArray getTypeArray() const { return getFieldAs(10); } - void setTypeArray(DIArray Elements, DIArray TParams = DIArray()); - unsigned getRunTimeLang() const { return getUnsignedField(11); } - DITypeRef getContainingType() const { return getFieldAs(12); } + DIArray getElements() const { + assert(!isSubroutineType() && "no elements for DISubroutineType"); + return getFieldAs(4); + } + +private: + template + void setArrays(DITypedArray Elements, DIArray TParams = DIArray()) { + assert((!TParams || DbgNode->getNumOperands() == 8) && + "If you're setting the template parameters this should include a slot " + "for that!"); + setArraysHelper(Elements, TParams); + } + +public: + unsigned getRunTimeLang() const { + return getHeaderFieldAs(7); + } + DITypeRef getContainingType() const { return getFieldAs(5); } + +private: + /// \brief Set the containing type. void setContainingType(DICompositeType ContainingType); - DIArray getTemplateParams() const { return getFieldAs(13); } + +public: + DIArray getTemplateParams() const { return getFieldAs(6); } MDString *getIdentifier() const; - /// Verify - Verify that a composite type descriptor is well formed. bool Verify() const; }; -/// DIFile - This is a wrapper for a file. +class DISubroutineType : public DICompositeType { +public: + explicit DISubroutineType(const MDNode *N = nullptr) : DICompositeType(N) {} + DITypedArray getTypeArray() const { + return getFieldAs>(4); + } +}; + +/// \brief This is a wrapper for a file. class DIFile : public DIScope { friend class DIDescriptor; public: explicit DIFile(const MDNode *N = nullptr) : DIScope(N) {} + + /// \brief Retrieve the MDNode for the directory/file pair. MDNode *getFileNode() const; bool Verify() const; }; -/// DICompileUnit - A wrapper for a compile unit. +/// \brief A wrapper for a compile unit. class DICompileUnit : public DIScope { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; @@ -428,13 +570,13 @@ public: explicit DICompileUnit(const MDNode *N = nullptr) : DIScope(N) {} dwarf::SourceLanguage getLanguage() const { - return static_cast(getUnsignedField(2)); + return static_cast(getHeaderFieldAs(1)); } - StringRef getProducer() const { return getStringField(3); } + StringRef getProducer() const { return getHeaderField(2); } - bool isOptimized() const { return getUnsignedField(4) != 0; } - StringRef getFlags() const { return getStringField(5); } - unsigned getRunTimeVersion() const { return getUnsignedField(6); } + bool isOptimized() const { return getHeaderFieldAs(3) != 0; } + StringRef getFlags() const { return getHeaderField(4); } + unsigned getRunTimeVersion() const { return getHeaderFieldAs(5); } DIArray getEnumTypes() const; DIArray getRetainedTypes() const; @@ -442,14 +584,16 @@ public: DIArray getGlobalVariables() const; DIArray getImportedEntities() const; - StringRef getSplitDebugFilename() const { return getStringField(12); } - unsigned getEmissionKind() const { return getUnsignedField(13); } + void replaceSubprograms(DIArray Subprograms); + void replaceGlobalVariables(DIArray GlobalVariables); + + StringRef getSplitDebugFilename() const { return getHeaderField(6); } + unsigned getEmissionKind() const { return getHeaderFieldAs(7); } - /// Verify - Verify that a compile unit is well formed. bool Verify() const; }; -/// DISubprogram - This is a wrapper for a subprogram (e.g. a function). +/// \brief This is a wrapper for a subprogram (e.g. a function). class DISubprogram : public DIScope { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; @@ -457,93 +601,95 @@ class DISubprogram : public DIScope { public: explicit DISubprogram(const MDNode *N = nullptr) : DIScope(N) {} + StringRef getName() const { return getHeaderField(1); } + StringRef getDisplayName() const { return getHeaderField(2); } + StringRef getLinkageName() const { return getHeaderField(3); } + unsigned getLineNumber() const { return getHeaderFieldAs(4); } + + /// \brief Check if this is local (like 'static' in C). + unsigned isLocalToUnit() const { return getHeaderFieldAs(5); } + unsigned isDefinition() const { return getHeaderFieldAs(6); } + + unsigned getVirtuality() const { return getHeaderFieldAs(7); } + unsigned getVirtualIndex() const { return getHeaderFieldAs(8); } + + unsigned getFlags() const { return getHeaderFieldAs(9); } + + unsigned isOptimized() const { return getHeaderFieldAs(10); } + + /// \brief Get the beginning of the scope of the function (not the name). + unsigned getScopeLineNumber() const { return getHeaderFieldAs(11); } + DIScopeRef getContext() const { return getFieldAs(2); } - StringRef getName() const { return getStringField(3); } - StringRef getDisplayName() const { return getStringField(4); } - StringRef getLinkageName() const { return getStringField(5); } - unsigned getLineNumber() const { return getUnsignedField(6); } - DICompositeType getType() const { return getFieldAs(7); } + DISubroutineType getType() const { return getFieldAs(3); } - /// isLocalToUnit - Return true if this subprogram is local to the current - /// compile unit, like 'static' in C. - unsigned isLocalToUnit() const { return getUnsignedField(8); } - unsigned isDefinition() const { return getUnsignedField(9); } + DITypeRef getContainingType() const { return getFieldAs(4); } - unsigned getVirtuality() const { return getUnsignedField(10); } - unsigned getVirtualIndex() const { return getUnsignedField(11); } - - DITypeRef getContainingType() const { return getFieldAs(12); } - - unsigned getFlags() const { return getUnsignedField(13); } - - unsigned isArtificial() const { - return (getUnsignedField(13) & FlagArtificial) != 0; - } - /// isPrivate - Return true if this subprogram has "private" - /// access specifier. - bool isPrivate() const { return (getUnsignedField(13) & FlagPrivate) != 0; } - /// isProtected - Return true if this subprogram has "protected" - /// access specifier. - bool isProtected() const { - return (getUnsignedField(13) & FlagProtected) != 0; - } - /// isExplicit - Return true if this subprogram is marked as explicit. - bool isExplicit() const { return (getUnsignedField(13) & FlagExplicit) != 0; } - /// isPrototyped - Return true if this subprogram is prototyped. - bool isPrototyped() const { - return (getUnsignedField(13) & FlagPrototyped) != 0; - } - - /// Return true if this subprogram is a C++11 reference-qualified - /// non-static member function (void foo() &). - unsigned isLValueReference() const { - return (getUnsignedField(13) & FlagLValueReference) != 0; - } - - /// Return true if this subprogram is a C++11 - /// rvalue-reference-qualified non-static member function - /// (void foo() &&). - unsigned isRValueReference() const { - return (getUnsignedField(13) & FlagRValueReference) != 0; - } - - unsigned isOptimized() const; - - /// Verify - Verify that a subprogram descriptor is well formed. bool Verify() const; - /// describes - Return true if this subprogram provides debugging - /// information for the function F. + /// \brief Check if this provides debugging information for the function F. bool describes(const Function *F); - Function *getFunction() const { return getFunctionField(15); } - void replaceFunction(Function *F) { replaceFunctionField(15, F); } - DIArray getTemplateParams() const { return getFieldAs(16); } + Function *getFunction() const { return getFunctionField(5); } + void replaceFunction(Function *F) { replaceFunctionField(5, F); } + DIArray getTemplateParams() const { return getFieldAs(6); } DISubprogram getFunctionDeclaration() const { - return getFieldAs(17); + return getFieldAs(7); } MDNode *getVariablesNodes() const; DIArray getVariables() const; - /// getScopeLineNumber - Get the beginning of the scope of the - /// function, not necessarily where the name of the program - /// starts. - unsigned getScopeLineNumber() const { return getUnsignedField(19); } + unsigned isArtificial() const { return (getFlags() & FlagArtificial) != 0; } + /// \brief Check for the "private" access specifier. + bool isPrivate() const { + return (getFlags() & FlagAccessibility) == FlagPrivate; + } + /// \brief Check for the "protected" access specifier. + bool isProtected() const { + return (getFlags() & FlagAccessibility) == FlagProtected; + } + /// \brief Check for the "public" access specifier. + bool isPublic() const { + return (getFlags() & FlagAccessibility) == FlagPublic; + } + /// \brief Check for "explicit". + bool isExplicit() const { return (getFlags() & FlagExplicit) != 0; } + /// \brief Check if this is prototyped. + bool isPrototyped() const { return (getFlags() & FlagPrototyped) != 0; } + + /// \brief Check if this is reference-qualified. + /// + /// Return true if this subprogram is a C++11 reference-qualified non-static + /// member function (void foo() &). + unsigned isLValueReference() const { + return (getFlags() & FlagLValueReference) != 0; + } + + /// \brief Check if this is rvalue-reference-qualified. + /// + /// Return true if this subprogram is a C++11 rvalue-reference-qualified + /// non-static member function (void foo() &&). + unsigned isRValueReference() const { + return (getFlags() & FlagRValueReference) != 0; + } + }; -/// DILexicalBlock - This is a wrapper for a lexical block. +/// \brief This is a wrapper for a lexical block. class DILexicalBlock : public DIScope { public: explicit DILexicalBlock(const MDNode *N = nullptr) : DIScope(N) {} DIScope getContext() const { return getFieldAs(2); } - unsigned getLineNumber() const { return getUnsignedField(3); } - unsigned getColumnNumber() const { return getUnsignedField(4); } - unsigned getDiscriminator() const { return getUnsignedField(5); } + unsigned getLineNumber() const { + return getHeaderFieldAs(1); + } + unsigned getColumnNumber() const { + return getHeaderFieldAs(2); + } bool Verify() const; }; -/// DILexicalBlockFile - This is a wrapper for a lexical block with -/// a filename change. +/// \brief This is a wrapper for a lexical block with a filename change. class DILexicalBlockFile : public DIScope { public: explicit DILexicalBlockFile(const MDNode *N = nullptr) : DIScope(N) {} @@ -555,68 +701,63 @@ public: unsigned getLineNumber() const { return getScope().getLineNumber(); } unsigned getColumnNumber() const { return getScope().getColumnNumber(); } DILexicalBlock getScope() const { return getFieldAs(2); } + unsigned getDiscriminator() const { return getHeaderFieldAs(1); } bool Verify() const; }; -/// DINameSpace - A wrapper for a C++ style name space. +/// \brief A wrapper for a C++ style name space. class DINameSpace : public DIScope { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; public: explicit DINameSpace(const MDNode *N = nullptr) : DIScope(N) {} + StringRef getName() const { return getHeaderField(1); } + unsigned getLineNumber() const { return getHeaderFieldAs(2); } DIScope getContext() const { return getFieldAs(2); } - StringRef getName() const { return getStringField(3); } - unsigned getLineNumber() const { return getUnsignedField(4); } bool Verify() const; }; -/// DIUnspecifiedParameter - This is a wrapper for unspecified parameters. -class DIUnspecifiedParameter : public DIDescriptor { -public: - explicit DIUnspecifiedParameter(const MDNode *N = nullptr) - : DIDescriptor(N) {} - bool Verify() const; -}; - -/// DITemplateTypeParameter - This is a wrapper for template type parameter. +/// \brief This is a wrapper for template type parameter. class DITemplateTypeParameter : public DIDescriptor { public: explicit DITemplateTypeParameter(const MDNode *N = nullptr) : DIDescriptor(N) {} + StringRef getName() const { return getHeaderField(1); } + unsigned getLineNumber() const { return getHeaderFieldAs(2); } + unsigned getColumnNumber() const { return getHeaderFieldAs(3); } + DIScopeRef getContext() const { return getFieldAs(1); } - StringRef getName() const { return getStringField(2); } - DITypeRef getType() const { return getFieldAs(3); } - StringRef getFilename() const { return getFieldAs(4).getFilename(); } + DITypeRef getType() const { return getFieldAs(2); } + StringRef getFilename() const { return getFieldAs(3).getFilename(); } StringRef getDirectory() const { - return getFieldAs(4).getDirectory(); + return getFieldAs(3).getDirectory(); } - unsigned getLineNumber() const { return getUnsignedField(5); } - unsigned getColumnNumber() const { return getUnsignedField(6); } bool Verify() const; }; -/// DITemplateValueParameter - This is a wrapper for template value parameter. +/// \brief This is a wrapper for template value parameter. class DITemplateValueParameter : public DIDescriptor { public: explicit DITemplateValueParameter(const MDNode *N = nullptr) : DIDescriptor(N) {} + StringRef getName() const { return getHeaderField(1); } + unsigned getLineNumber() const { return getHeaderFieldAs(2); } + unsigned getColumnNumber() const { return getHeaderFieldAs(3); } + DIScopeRef getContext() const { return getFieldAs(1); } - StringRef getName() const { return getStringField(2); } - DITypeRef getType() const { return getFieldAs(3); } - Value *getValue() const; - StringRef getFilename() const { return getFieldAs(5).getFilename(); } + DITypeRef getType() const { return getFieldAs(2); } + Metadata *getValue() const; + StringRef getFilename() const { return getFieldAs(4).getFilename(); } StringRef getDirectory() const { - return getFieldAs(5).getDirectory(); + return getFieldAs(4).getDirectory(); } - unsigned getLineNumber() const { return getUnsignedField(6); } - unsigned getColumnNumber() const { return getUnsignedField(7); } bool Verify() const; }; -/// DIGlobalVariable - This is a wrapper for a global variable. +/// \brief This is a wrapper for a global variable. class DIGlobalVariable : public DIDescriptor { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; @@ -624,32 +765,30 @@ class DIGlobalVariable : public DIDescriptor { public: explicit DIGlobalVariable(const MDNode *N = nullptr) : DIDescriptor(N) {} - DIScope getContext() const { return getFieldAs(2); } - StringRef getName() const { return getStringField(3); } - StringRef getDisplayName() const { return getStringField(4); } - StringRef getLinkageName() const { return getStringField(5); } - StringRef getFilename() const { return getFieldAs(6).getFilename(); } + StringRef getName() const { return getHeaderField(1); } + StringRef getDisplayName() const { return getHeaderField(2); } + StringRef getLinkageName() const { return getHeaderField(3); } + unsigned getLineNumber() const { return getHeaderFieldAs(4); } + unsigned isLocalToUnit() const { return getHeaderFieldAs(5); } + unsigned isDefinition() const { return getHeaderFieldAs(6); } + + DIScope getContext() const { return getFieldAs(1); } + StringRef getFilename() const { return getFieldAs(2).getFilename(); } StringRef getDirectory() const { - return getFieldAs(6).getDirectory(); + return getFieldAs(2).getDirectory(); } + DITypeRef getType() const { return getFieldAs(3); } - unsigned getLineNumber() const { return getUnsignedField(7); } - DITypeRef getType() const { return getFieldAs(8); } - unsigned isLocalToUnit() const { return getUnsignedField(9); } - unsigned isDefinition() const { return getUnsignedField(10); } - - GlobalVariable *getGlobal() const { return getGlobalVariableField(11); } - Constant *getConstant() const { return getConstantField(11); } + GlobalVariable *getGlobal() const { return getGlobalVariableField(4); } + Constant *getConstant() const { return getConstantField(4); } DIDerivedType getStaticDataMemberDeclaration() const { - return getFieldAs(12); + return getFieldAs(5); } - /// Verify - Verify that a global variable descriptor is well formed. bool Verify() const; }; -/// DIVariable - This is a wrapper for a variable (e.g. parameter, local, -/// global etc). +/// \brief This is a wrapper for a variable (e.g. parameter, local, global etc). class DIVariable : public DIDescriptor { friend class DIDescriptor; void printInternal(raw_ostream &OS) const; @@ -657,73 +796,107 @@ class DIVariable : public DIDescriptor { public: explicit DIVariable(const MDNode *N = nullptr) : DIDescriptor(N) {} - DIScope getContext() const { return getFieldAs(1); } - StringRef getName() const { return getStringField(2); } - DIFile getFile() const { return getFieldAs(3); } - unsigned getLineNumber() const { return (getUnsignedField(4) << 8) >> 8; } - unsigned getArgNumber() const { - unsigned L = getUnsignedField(4); - return L >> 24; + StringRef getName() const { return getHeaderField(1); } + unsigned getLineNumber() const { + // FIXME: Line number and arg number shouldn't be merged together like this. + return (getHeaderFieldAs(2) << 8) >> 8; } - DITypeRef getType() const { return getFieldAs(5); } + unsigned getArgNumber() const { return getHeaderFieldAs(2) >> 24; } - /// isArtificial - Return true if this variable is marked as "artificial". + DIScope getContext() const { return getFieldAs(1); } + DIFile getFile() const { return getFieldAs(2); } + DITypeRef getType() const { return getFieldAs(3); } + + /// \brief Return true if this variable is marked as "artificial". bool isArtificial() const { - return (getUnsignedField(6) & FlagArtificial) != 0; + return (getHeaderFieldAs(3) & FlagArtificial) != 0; } bool isObjectPointer() const { - return (getUnsignedField(6) & FlagObjectPointer) != 0; + return (getHeaderFieldAs(3) & FlagObjectPointer) != 0; } /// \brief Return true if this variable is represented as a pointer. bool isIndirect() const { - return (getUnsignedField(6) & FlagIndirectVariable) != 0; + return (getHeaderFieldAs(3) & FlagIndirectVariable) != 0; } - /// getInlinedAt - If this variable is inlined then return inline location. + /// \brief If this variable is inlined then return inline location. MDNode *getInlinedAt() const; - /// Verify - Verify that a variable descriptor is well formed. bool Verify() const; - /// HasComplexAddr - Return true if the variable has a complex address. - bool hasComplexAddress() const { return getNumAddrElements() > 0; } - - /// \brief Return the size of this variable's complex address or - /// zero if there is none. - unsigned getNumAddrElements() const { - if (DbgNode->getNumOperands() < 9) - return 0; - return getDescriptorField(8)->getNumOperands(); - } - - /// \brief return the Idx'th complex address element. - uint64_t getAddrElement(unsigned Idx) const; - - /// isBlockByrefVariable - Return true if the variable was declared as - /// a "__block" variable (Apple Blocks). + /// \brief Check if this is a "__block" variable (Apple Blocks). bool isBlockByrefVariable(const DITypeIdentifierMap &Map) const { return (getType().resolve(Map)).isBlockByrefStruct(); } - /// isInlinedFnArgument - Return true if this variable provides debugging - /// information for an inlined function arguments. + /// \brief Check if this is an inlined function argument. bool isInlinedFnArgument(const Function *CurFn); + /// \brief Return the size reported by the variable's type. + unsigned getSizeInBits(const DITypeIdentifierMap &Map); + void printExtendedName(raw_ostream &OS) const; }; -/// DILocation - This object holds location information. This object -/// is not associated with any DWARF tag. +/// \brief A complex location expression. +class DIExpression : public DIDescriptor { + friend class DIDescriptor; + void printInternal(raw_ostream &OS) const; + +public: + explicit DIExpression(const MDNode *N = nullptr) : DIDescriptor(N) {} + + bool Verify() const; + + /// \brief Return the number of elements in the complex expression. + unsigned getNumElements() const { + if (!DbgNode) + return 0; + unsigned N = getNumHeaderFields(); + assert(N > 0 && "missing tag"); + return N - 1; + } + + /// \brief return the Idx'th complex address element. + uint64_t getElement(unsigned Idx) const; + + /// \brief Return whether this is a piece of an aggregate variable. + bool isVariablePiece() const; + /// \brief Return the offset of this piece in bytes. + uint64_t getPieceOffset() const; + /// \brief Return the size of this piece in bytes. + uint64_t getPieceSize() const; +}; + +/// \brief This object holds location information. +/// +/// This object is not associated with any DWARF tag. class DILocation : public DIDescriptor { public: explicit DILocation(const MDNode *N) : DIDescriptor(N) {} - unsigned getLineNumber() const { return getUnsignedField(0); } - unsigned getColumnNumber() const { return getUnsignedField(1); } - DIScope getScope() const { return getFieldAs(2); } - DILocation getOrigLocation() const { return getFieldAs(3); } + unsigned getLineNumber() const { + if (auto *L = dyn_cast_or_null(DbgNode)) + return L->getLine(); + return 0; + } + unsigned getColumnNumber() const { + if (auto *L = dyn_cast_or_null(DbgNode)) + return L->getColumn(); + return 0; + } + DIScope getScope() const { + if (auto *L = dyn_cast_or_null(DbgNode)) + return DIScope(dyn_cast_or_null(L->getScope())); + return DIScope(nullptr); + } + DILocation getOrigLocation() const { + if (auto *L = dyn_cast_or_null(DbgNode)) + return DILocation(dyn_cast_or_null(L->getInlinedAt())); + return DILocation(nullptr); + } StringRef getFilename() const { return getScope().getFilename(); } StringRef getDirectory() const { return getScope().getDirectory(); } bool Verify() const; @@ -731,23 +904,30 @@ public: return (getLineNumber() == Other.getLineNumber() && getFilename() == Other.getFilename()); } - /// getDiscriminator - DWARF discriminators are used to distinguish - /// identical file locations for instructions that are on different - /// basic blocks. If two instructions are inside the same lexical block - /// and are in different basic blocks, we create a new lexical block - /// with identical location as the original but with a different - /// discriminator value (lib/Transforms/Util/AddDiscriminators.cpp - /// for details). + /// \brief Get the DWAF discriminator. + /// + /// DWARF discriminators are used to distinguish identical file locations for + /// instructions that are on different basic blocks. If two instructions are + /// inside the same lexical block and are in different basic blocks, we + /// create a new lexical block with identical location as the original but + /// with a different discriminator value + /// (lib/Transforms/Util/AddDiscriminators.cpp for details). unsigned getDiscriminator() const { // Since discriminators are associated with lexical blocks, make // sure this location is a lexical block before retrieving its // value. - return getScope().isLexicalBlock() - ? getFieldAs(2).getDiscriminator() + return getScope().isLexicalBlockFile() + ? DILexicalBlockFile( + cast(cast(DbgNode)->getScope())) + .getDiscriminator() : 0; } + + /// \brief Generate a new discriminator value for this location. unsigned computeNewDiscriminator(LLVMContext &Ctx); - DILocation copyWithNewScope(LLVMContext &Ctx, DILexicalBlock NewScope); + + /// \brief Return a copy of this location with a different scope. + DILocation copyWithNewScope(LLVMContext &Ctx, DILexicalBlockFile NewScope); }; class DIObjCProperty : public DIDescriptor { @@ -757,36 +937,38 @@ class DIObjCProperty : public DIDescriptor { public: explicit DIObjCProperty(const MDNode *N) : DIDescriptor(N) {} - StringRef getObjCPropertyName() const { return getStringField(1); } - DIFile getFile() const { return getFieldAs(2); } - unsigned getLineNumber() const { return getUnsignedField(3); } + StringRef getObjCPropertyName() const { return getHeaderField(1); } + DIFile getFile() const { return getFieldAs(1); } + unsigned getLineNumber() const { return getHeaderFieldAs(2); } - StringRef getObjCPropertyGetterName() const { return getStringField(4); } - StringRef getObjCPropertySetterName() const { return getStringField(5); } + StringRef getObjCPropertyGetterName() const { return getHeaderField(3); } + StringRef getObjCPropertySetterName() const { return getHeaderField(4); } + unsigned getAttributes() const { return getHeaderFieldAs(5); } bool isReadOnlyObjCProperty() const { - return (getUnsignedField(6) & dwarf::DW_APPLE_PROPERTY_readonly) != 0; + return (getAttributes() & dwarf::DW_APPLE_PROPERTY_readonly) != 0; } bool isReadWriteObjCProperty() const { - return (getUnsignedField(6) & dwarf::DW_APPLE_PROPERTY_readwrite) != 0; + return (getAttributes() & dwarf::DW_APPLE_PROPERTY_readwrite) != 0; } bool isAssignObjCProperty() const { - return (getUnsignedField(6) & dwarf::DW_APPLE_PROPERTY_assign) != 0; + return (getAttributes() & dwarf::DW_APPLE_PROPERTY_assign) != 0; } bool isRetainObjCProperty() const { - return (getUnsignedField(6) & dwarf::DW_APPLE_PROPERTY_retain) != 0; + return (getAttributes() & dwarf::DW_APPLE_PROPERTY_retain) != 0; } bool isCopyObjCProperty() const { - return (getUnsignedField(6) & dwarf::DW_APPLE_PROPERTY_copy) != 0; + return (getAttributes() & dwarf::DW_APPLE_PROPERTY_copy) != 0; } bool isNonAtomicObjCProperty() const { - return (getUnsignedField(6) & dwarf::DW_APPLE_PROPERTY_nonatomic) != 0; + return (getAttributes() & dwarf::DW_APPLE_PROPERTY_nonatomic) != 0; } - /// Objective-C doesn't have an ODR, so there is no benefit in storing + /// \brief Get the type. + /// + /// \note Objective-C doesn't have an ODR, so there is no benefit in storing /// the type as a DITypeRef here. - DIType getType() const { return getFieldAs(7); } + DIType getType() const { return getFieldAs(2); } - /// Verify - Verify that a derived type descriptor is well formed. bool Verify() const; }; @@ -799,47 +981,47 @@ public: explicit DIImportedEntity(const MDNode *N) : DIDescriptor(N) {} DIScope getContext() const { return getFieldAs(1); } DIScopeRef getEntity() const { return getFieldAs(2); } - unsigned getLineNumber() const { return getUnsignedField(3); } - StringRef getName() const { return getStringField(4); } + unsigned getLineNumber() const { return getHeaderFieldAs(1); } + StringRef getName() const { return getHeaderField(2); } bool Verify() const; }; -/// getDISubprogram - Find subprogram that is enclosing this scope. +/// \brief Find subprogram that is enclosing this scope. DISubprogram getDISubprogram(const MDNode *Scope); -/// getDICompositeType - Find underlying composite type. +/// \brief Find debug info for a given function. +/// \returns a valid DISubprogram, if found. Otherwise, it returns an empty +/// DISubprogram. +DISubprogram getDISubprogram(const Function *F); + +/// \brief Find underlying composite type. DICompositeType getDICompositeType(DIType T); -/// getOrInsertFnSpecificMDNode - Return a NameMDNode that is suitable -/// to hold function specific information. -NamedMDNode *getOrInsertFnSpecificMDNode(Module &M, DISubprogram SP); - -/// getFnSpecificMDNode - Return a NameMDNode, if available, that is -/// suitable to hold function specific information. -NamedMDNode *getFnSpecificMDNode(const Module &M, DISubprogram SP); - -/// createInlinedVariable - Create a new inlined variable based on current -/// variable. +/// \brief Create a new inlined variable based on current variable. +/// /// @param DV Current Variable. /// @param InlinedScope Location at current variable is inlined. DIVariable createInlinedVariable(MDNode *DV, MDNode *InlinedScope, LLVMContext &VMContext); -/// cleanseInlinedVariable - Remove inlined scope from the variable. +/// \brief Remove inlined scope from the variable. DIVariable cleanseInlinedVariable(MDNode *DV, LLVMContext &VMContext); -/// Construct DITypeIdentifierMap by going through retained types of each CU. +/// \brief Generate map by visiting all retained types. DITypeIdentifierMap generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes); -/// Strip debug info in the module if it exists. +/// \brief Strip debug info in the module if it exists. +/// /// To do this, we remove all calls to the debugger intrinsics and any named /// metadata for debugging. We also remove debug locations for instructions. /// Return true if module is modified. bool StripDebugInfo(Module &M); -/// Return Debug Info Metadata Version by checking module flags. +/// \brief Return Debug Info Metadata Version by checking module flags. unsigned getDebugMetadataVersionFromModule(const Module &M); +/// \brief Utility to find all debug info in a module. +/// /// DebugInfoFinder tries to list all debug info MDNodes used in a module. To /// list debug info MDNodes used by an instruction, DebugInfoFinder uses /// processDeclare, processValue and processLocation to handle DbgDeclareInst, @@ -850,44 +1032,29 @@ class DebugInfoFinder { public: DebugInfoFinder() : TypeMapInitialized(false) {} - /// processModule - Process entire module and collect debug info - /// anchors. + /// \brief Process entire module and collect debug info anchors. void processModule(const Module &M); - /// processDeclare - Process DbgDeclareInst. + /// \brief Process DbgDeclareInst. void processDeclare(const Module &M, const DbgDeclareInst *DDI); - /// Process DbgValueInst. + /// \brief Process DbgValueInst. void processValue(const Module &M, const DbgValueInst *DVI); - /// processLocation - Process DILocation. + /// \brief Process DILocation. void processLocation(const Module &M, DILocation Loc); - /// Clear all lists. + /// \brief Clear all lists. void reset(); private: - /// Initialize TypeIdentifierMap. void InitializeTypeMap(const Module &M); - /// processType - Process DIType. void processType(DIType DT); - - /// processSubprogram - Process DISubprogram. void processSubprogram(DISubprogram SP); - void processScope(DIScope Scope); - - /// addCompileUnit - Add compile unit into CUs. bool addCompileUnit(DICompileUnit CU); - - /// addGlobalVariable - Add global variable into GVs. bool addGlobalVariable(DIGlobalVariable DIG); - - // addSubprogram - Add subprogram into SPs. bool addSubprogram(DISubprogram SP); - - /// addType - Add type into Tys. bool addType(DIType DT); - bool addScope(DIScope Scope); public: @@ -924,14 +1091,15 @@ public: unsigned scope_count() const { return Scopes.size(); } private: - SmallVector CUs; // Compile Units - SmallVector SPs; // Subprograms - SmallVector GVs; // Global Variables; - SmallVector TYs; // Types - SmallVector Scopes; // Scopes + SmallVector CUs; + SmallVector SPs; + SmallVector GVs; + SmallVector TYs; + SmallVector Scopes; SmallPtrSet NodesSeen; DITypeIdentifierMap TypeIdentifierMap; - /// Specify if TypeIdentifierMap is initialized. + + /// \brief Specify if TypeIdentifierMap is initialized. bool TypeMapInitialized; }; diff --git a/contrib/llvm/include/llvm/IR/DebugLoc.h b/contrib/llvm/include/llvm/IR/DebugLoc.h index 3d969a8b7532..86e64417099d 100644 --- a/contrib/llvm/include/llvm/IR/DebugLoc.h +++ b/contrib/llvm/include/llvm/IR/DebugLoc.h @@ -15,51 +15,41 @@ #ifndef LLVM_IR_DEBUGLOC_H #define LLVM_IR_DEBUGLOC_H +#include "llvm/IR/TrackingMDRef.h" #include "llvm/Support/DataTypes.h" namespace llvm { - template struct DenseMapInfo; - class MDNode; + class LLVMContext; class raw_ostream; + class MDNode; /// DebugLoc - Debug location id. This is carried by Instruction, SDNode, /// and MachineInstr to compactly encode file/line/scope information for an /// operation. class DebugLoc { - friend struct DenseMapInfo; + TrackingMDNodeRef Loc; - /// getEmptyKey() - A private constructor that returns an unknown that is - /// not equal to the tombstone key or DebugLoc(). - static DebugLoc getEmptyKey() { - DebugLoc DL; - DL.LineCol = 1; - return DL; - } - - /// getTombstoneKey() - A private constructor that returns an unknown that - /// is not equal to the empty key or DebugLoc(). - static DebugLoc getTombstoneKey() { - DebugLoc DL; - DL.LineCol = 2; - return DL; - } - - /// LineCol - This 32-bit value encodes the line and column number for the - /// location, encoded as 24-bits for line and 8 bits for col. A value of 0 - /// for either means unknown. - uint32_t LineCol; - - /// ScopeIdx - This is an opaque ID# for Scope/InlinedAt information, - /// decoded by LLVMContext. 0 is unknown. - int ScopeIdx; public: - DebugLoc() : LineCol(0), ScopeIdx(0) {} // Defaults to unknown. + DebugLoc() {} + DebugLoc(DebugLoc &&X) : Loc(std::move(X.Loc)) {} + DebugLoc(const DebugLoc &X) : Loc(X.Loc) {} + DebugLoc &operator=(DebugLoc &&X) { + Loc = std::move(X.Loc); + return *this; + } + DebugLoc &operator=(const DebugLoc &X) { + Loc = X.Loc; + return *this; + } + + /// \brief Check whether this has a trivial destructor. + bool hasTrivialDestructor() const { return Loc.hasTrivialDestructor(); } /// get - Get a new DebugLoc that corresponds to the specified line/col /// scope/inline location. - static DebugLoc get(unsigned Line, unsigned Col, - MDNode *Scope, MDNode *InlinedAt = nullptr); + static DebugLoc get(unsigned Line, unsigned Col, MDNode *Scope, + MDNode *InlinedAt = nullptr); /// getFromDILocation - Translate the DILocation quad into a DebugLoc. static DebugLoc getFromDILocation(MDNode *N); @@ -68,56 +58,54 @@ namespace llvm { static DebugLoc getFromDILexicalBlock(MDNode *N); /// isUnknown - Return true if this is an unknown location. - bool isUnknown() const { return ScopeIdx == 0; } + bool isUnknown() const { return !Loc; } - unsigned getLine() const { - return (LineCol << 8) >> 8; // Mask out column. - } - - unsigned getCol() const { - return LineCol >> 24; - } + unsigned getLine() const; + unsigned getCol() const; /// getScope - This returns the scope pointer for this DebugLoc, or null if /// invalid. - MDNode *getScope(const LLVMContext &Ctx) const; + MDNode *getScope() const; + MDNode *getScope(const LLVMContext &) const { return getScope(); } /// getInlinedAt - This returns the InlinedAt pointer for this DebugLoc, or /// null if invalid or not present. - MDNode *getInlinedAt(const LLVMContext &Ctx) const; + MDNode *getInlinedAt() const; + MDNode *getInlinedAt(const LLVMContext &) const { return getInlinedAt(); } /// getScopeAndInlinedAt - Return both the Scope and the InlinedAt values. + void getScopeAndInlinedAt(MDNode *&Scope, MDNode *&IA) const; void getScopeAndInlinedAt(MDNode *&Scope, MDNode *&IA, - const LLVMContext &Ctx) const; + const LLVMContext &) const { + return getScopeAndInlinedAt(Scope, IA); + } /// getScopeNode - Get MDNode for DebugLoc's scope, or null if invalid. - MDNode *getScopeNode(const LLVMContext &Ctx) const; + MDNode *getScopeNode() const; + MDNode *getScopeNode(const LLVMContext &) const { return getScopeNode(); } // getFnDebugLoc - Walk up the scope chain of given debug loc and find line // number info for the function. - DebugLoc getFnDebugLoc(const LLVMContext &Ctx) const; + DebugLoc getFnDebugLoc() const; + DebugLoc getFnDebugLoc(const LLVMContext &) const { + return getFnDebugLoc(); + } /// getAsMDNode - This method converts the compressed DebugLoc node into a /// DILocation compatible MDNode. - MDNode *getAsMDNode(const LLVMContext &Ctx) const; + MDNode *getAsMDNode() const; + MDNode *getAsMDNode(LLVMContext &) const { return getAsMDNode(); } - bool operator==(const DebugLoc &DL) const { - return LineCol == DL.LineCol && ScopeIdx == DL.ScopeIdx; - } + bool operator==(const DebugLoc &DL) const { return Loc == DL.Loc; } bool operator!=(const DebugLoc &DL) const { return !(*this == DL); } - void dump(const LLVMContext &Ctx) const; + void dump() const; + void dump(const LLVMContext &) const { dump(); } /// \brief prints source location /path/to/file.exe:line:col @[inlined at] - void print(const LLVMContext &Ctx, raw_ostream &OS) const; + void print(raw_ostream &OS) const; + void print(const LLVMContext &, raw_ostream &OS) const { print(OS); } }; - template <> - struct DenseMapInfo { - static DebugLoc getEmptyKey() { return DebugLoc::getEmptyKey(); } - static DebugLoc getTombstoneKey() { return DebugLoc::getTombstoneKey(); } - static unsigned getHashValue(const DebugLoc &Key); - static bool isEqual(DebugLoc LHS, DebugLoc RHS) { return LHS == RHS; } - }; } // end namespace llvm #endif /* LLVM_SUPPORT_DEBUGLOC_H */ diff --git a/contrib/llvm/include/llvm/IR/DerivedTypes.h b/contrib/llvm/include/llvm/IR/DerivedTypes.h index ff150872a4e1..182015c98aa2 100644 --- a/contrib/llvm/include/llvm/IR/DerivedTypes.h +++ b/contrib/llvm/include/llvm/IR/DerivedTypes.h @@ -123,6 +123,9 @@ public: typedef Type::subtype_iterator param_iterator; param_iterator param_begin() const { return ContainedTys + 1; } param_iterator param_end() const { return &ContainedTys[NumContainedTys]; } + ArrayRef params() const { + return makeArrayRef(param_begin(), param_end()); + } /// Parameter type accessors. Type *getParamType(unsigned i) const { return ContainedTys[i+1]; } @@ -204,9 +207,6 @@ class StructType : public CompositeType { /// void *SymbolTableEntry; public: - ~StructType() { - delete [] ContainedTys; // Delete the body. - } /// StructType::create - This creates an identified struct. static StructType *create(LLVMContext &Context, StringRef Name); @@ -221,7 +221,7 @@ public: StringRef Name, bool isPacked = false); static StructType *create(LLVMContext &Context, ArrayRef Elements); - static StructType *create(StringRef Name, Type *elt1, ...) END_WITH_NULL; + static StructType *create(StringRef Name, Type *elt1, ...) LLVM_END_WITH_NULL; /// StructType::get - This static method is the primary way to create a /// literal StructType. @@ -236,7 +236,7 @@ public: /// structure types by specifying the elements as arguments. Note that this /// method always returns a non-packed struct, and requires at least one /// element type. - static StructType *get(Type *elt1, ...) END_WITH_NULL; + static StructType *get(Type *elt1, ...) LLVM_END_WITH_NULL; bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; } @@ -249,7 +249,7 @@ public: bool isOpaque() const { return (getSubclassData() & SCDB_HasBody) == 0; } /// isSized - Return true if this is a sized type. - bool isSized(SmallPtrSet *Visited = nullptr) const; + bool isSized(SmallPtrSetImpl *Visited = nullptr) const; /// hasName - Return true if this is a named struct that has a non-empty name. bool hasName() const { return SymbolTableEntry != nullptr; } @@ -266,7 +266,7 @@ public: /// setBody - Specify a body for an opaque identified type. void setBody(ArrayRef Elements, bool isPacked = false); - void setBody(Type *elt1, ...) END_WITH_NULL; + void setBody(Type *elt1, ...) LLVM_END_WITH_NULL; /// isValidElementType - Return true if the specified type is valid as a /// element type. @@ -277,6 +277,9 @@ public: typedef Type::subtype_iterator element_iterator; element_iterator element_begin() const { return ContainedTys; } element_iterator element_end() const { return &ContainedTys[NumContainedTys];} + ArrayRef const elements() const { + return makeArrayRef(element_begin(), element_end()); + } /// isLayoutIdentical - Return true if this is layout identical to the /// specified struct. diff --git a/contrib/llvm/include/llvm/IR/DiagnosticInfo.h b/contrib/llvm/include/llvm/IR/DiagnosticInfo.h index 9c9f236180ce..c6a8854e0774 100644 --- a/contrib/llvm/include/llvm/IR/DiagnosticInfo.h +++ b/contrib/llvm/include/llvm/IR/DiagnosticInfo.h @@ -12,12 +12,13 @@ // Diagnostics reporting is still done as part of the LLVMContext. //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_DIAGNOSTICINFO_H -#define LLVM_SUPPORT_DIAGNOSTICINFO_H +#ifndef LLVM_IR_DIAGNOSTICINFO_H +#define LLVM_IR_DIAGNOSTICINFO_H #include "llvm-c/Core.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Module.h" #include "llvm/Support/Casting.h" namespace llvm { @@ -44,8 +45,10 @@ enum DiagnosticSeverity { /// \brief Defines the different supported kind of a diagnostic. /// This enum should be extended with a new ID for each added concrete subclass. enum DiagnosticKind { + DK_Bitcode, DK_InlineAsm, DK_StackSize, + DK_Linker, DK_DebugMetadataVersion, DK_SampleProfile, DK_OptimizationRemark, @@ -95,6 +98,8 @@ public: virtual void print(DiagnosticPrinter &DP) const = 0; }; +typedef std::function DiagnosticHandlerFunction; + /// Diagnostic information for inline asm reporting. /// This is basically a message and an optional location. class DiagnosticInfoInlineAsm : public DiagnosticInfo { @@ -324,7 +329,7 @@ public: } /// \see DiagnosticInfoOptimizationBase::isEnabled. - virtual bool isEnabled() const override; + bool isEnabled() const override; }; /// Diagnostic information for missed-optimization remarks. @@ -350,7 +355,7 @@ public: } /// \see DiagnosticInfoOptimizationBase::isEnabled. - virtual bool isEnabled() const override; + bool isEnabled() const override; }; /// Diagnostic information for optimization analysis remarks. @@ -377,7 +382,7 @@ public: } /// \see DiagnosticInfoOptimizationBase::isEnabled. - virtual bool isEnabled() const override; + bool isEnabled() const override; }; // Create wrappers for C Binding types (see CBindingWrapping.h). @@ -432,7 +437,7 @@ public: } /// \see DiagnosticInfoOptimizationBase::isEnabled. - virtual bool isEnabled() const override; + bool isEnabled() const override; }; /// Emit a warning when loop vectorization is specified but fails. \p Fn is the diff --git a/contrib/llvm/include/llvm/IR/DiagnosticPrinter.h b/contrib/llvm/include/llvm/IR/DiagnosticPrinter.h index 411c781e01c5..db5779a8a8a5 100644 --- a/contrib/llvm/include/llvm/IR/DiagnosticPrinter.h +++ b/contrib/llvm/include/llvm/IR/DiagnosticPrinter.h @@ -13,8 +13,8 @@ // on their needs. //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_DIAGNOSTICPRINTER_H -#define LLVM_SUPPORT_DIAGNOSTICPRINTER_H +#ifndef LLVM_IR_DIAGNOSTICPRINTER_H +#define LLVM_IR_DIAGNOSTICPRINTER_H #include diff --git a/contrib/llvm/include/llvm/IR/Dominators.h b/contrib/llvm/include/llvm/IR/Dominators.h index e2d1ccc8a3ca..c1f208e3d72f 100644 --- a/contrib/llvm/include/llvm/IR/Dominators.h +++ b/contrib/llvm/include/llvm/IR/Dominators.h @@ -31,6 +31,11 @@ namespace llvm { +// FIXME: Replace this brittle forward declaration with the include of the new +// PassManager.h when doing so doesn't break the PassManagerBuilder. +template class AnalysisManager; +class PreservedAnalyses; + EXTERN_TEMPLATE_INSTANTIATION(class DomTreeNodeBase); EXTERN_TEMPLATE_INSTANTIATION(class DominatorTreeBase); @@ -69,6 +74,13 @@ public: DominatorTree() : DominatorTreeBase(false) {} + DominatorTree(DominatorTree &&Arg) + : Base(std::move(static_cast(Arg))) {} + DominatorTree &operator=(DominatorTree &&RHS) { + Base::operator=(std::move(static_cast(RHS))); + return *this; + } + /// \brief Returns *false* if the other dominator tree matches this dominator /// tree. inline bool compare(const DominatorTree &Other) const { @@ -155,6 +167,43 @@ template <> struct GraphTraits }; /// \brief Analysis pass which computes a \c DominatorTree. +class DominatorTreeAnalysis { +public: + /// \brief Provide the result typedef for this analysis pass. + typedef DominatorTree Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + /// \brief Run the analysis pass over a function and produce a dominator tree. + DominatorTree run(Function &F); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "DominatorTreeAnalysis"; } + +private: + static char PassID; +}; + +/// \brief Printer pass for the \c DominatorTree. +class DominatorTreePrinterPass { + raw_ostream &OS; + +public: + explicit DominatorTreePrinterPass(raw_ostream &OS); + PreservedAnalyses run(Function &F, AnalysisManager *AM); + + static StringRef name() { return "DominatorTreePrinterPass"; } +}; + +/// \brief Verifier pass for the \c DominatorTree. +struct DominatorTreeVerifierPass { + PreservedAnalyses run(Function &F, AnalysisManager *AM); + + static StringRef name() { return "DominatorTreeVerifierPass"; } +}; + +/// \brief Legacy analysis pass which computes a \c DominatorTree. class DominatorTreeWrapperPass : public FunctionPass { DominatorTree DT; diff --git a/contrib/llvm/include/llvm/IR/Function.h b/contrib/llvm/include/llvm/IR/Function.h index ad4b1395f0cb..51403281e964 100644 --- a/contrib/llvm/include/llvm/IR/Function.h +++ b/contrib/llvm/include/llvm/IR/Function.h @@ -87,11 +87,14 @@ private: ValueSymbolTable *SymTab; ///< Symbol table of args/instructions AttributeSet AttributeSets; ///< Parameter attributes - // HasLazyArguments is stored in Value::SubclassData. - /*bool HasLazyArguments;*/ - - // The Calling Convention is stored in Value::SubclassData. - /*CallingConv::ID CallingConvention;*/ + /* + * Value::SubclassData + * + * bit 0 : HasLazyArguments + * bit 1 : HasPrefixData + * bit 2 : HasPrologueData + * bit 3-6: CallingConvention + */ friend class SymbolTableListTraits; @@ -102,7 +105,7 @@ private: /// needs it. The hasLazyArguments predicate returns true if the arg list /// hasn't been set up yet. bool hasLazyArguments() const { - return getSubclassDataFromValue() & 1; + return getSubclassDataFromValue() & (1<<0); } void CheckLazyArguments() const { if (hasLazyArguments()) @@ -143,6 +146,9 @@ public: /// arguments. bool isVarArg() const; + bool isMaterializable() const; + void setIsMaterializable(bool V); + /// getIntrinsicID - This method returns the ID number of the specified /// function, or Intrinsic::not_intrinsic if the function is not an /// intrinsic, or if the pointer is null. This value is always defined to be @@ -159,11 +165,11 @@ public: /// calling convention of this function. The enum values for the known /// calling conventions are defined in CallingConv.h. CallingConv::ID getCallingConv() const { - return static_cast(getSubclassDataFromValue() >> 2); + return static_cast(getSubclassDataFromValue() >> 3); } void setCallingConv(CallingConv::ID CC) { - setValueSubclassData((getSubclassDataFromValue() & 3) | - (static_cast(CC) << 2)); + setValueSubclassData((getSubclassDataFromValue() & 7) | + (static_cast(CC) << 3)); } /// @brief Return the attribute list for this Function. @@ -445,12 +451,19 @@ public: bool arg_empty() const; bool hasPrefixData() const { - return getSubclassDataFromValue() & 2; + return getSubclassDataFromValue() & (1<<1); } Constant *getPrefixData() const; void setPrefixData(Constant *PrefixData); + bool hasPrologueData() const { + return getSubclassDataFromValue() & (1<<2); + } + + Constant *getPrologueData() const; + void setPrologueData(Constant *PrologueData); + /// viewCFG - This function is meant for use from the debugger. You can just /// say 'call F->viewCFG()' and a ghostview window should pop up from the /// program, displaying the CFG of the current function with the code for each diff --git a/contrib/llvm/include/llvm/IR/GVMaterializer.h b/contrib/llvm/include/llvm/IR/GVMaterializer.h index a1216a174282..6f57dc2a98a6 100644 --- a/contrib/llvm/include/llvm/IR/GVMaterializer.h +++ b/contrib/llvm/include/llvm/IR/GVMaterializer.h @@ -19,11 +19,13 @@ #define LLVM_IR_GVMATERIALIZER_H #include +#include namespace llvm { class Function; class GlobalValue; class Module; +class StructType; class GVMaterializer { protected: @@ -32,17 +34,13 @@ protected: public: virtual ~GVMaterializer(); - /// True if GV can be materialized from whatever backing store this - /// GVMaterializer uses and has not been materialized yet. - virtual bool isMaterializable(const GlobalValue *GV) const = 0; - /// True if GV has been materialized and can be dematerialized back to /// whatever backing store this GVMaterializer uses. virtual bool isDematerializable(const GlobalValue *GV) const = 0; /// Make sure the given GlobalValue is fully read. /// - virtual std::error_code Materialize(GlobalValue *GV) = 0; + virtual std::error_code materialize(GlobalValue *GV) = 0; /// If the given GlobalValue is read in, and if the GVMaterializer supports /// it, release the memory for the GV, and set it up to be materialized @@ -55,7 +53,7 @@ public: /// virtual std::error_code MaterializeModule(Module *M) = 0; - virtual void releaseBuffer() = 0; + virtual std::vector getIdentifiedStructTypes() const = 0; }; } // End llvm namespace diff --git a/contrib/llvm/include/llvm/IR/GlobalObject.h b/contrib/llvm/include/llvm/IR/GlobalObject.h index 2e042f489749..546fea2dfa9b 100644 --- a/contrib/llvm/include/llvm/IR/GlobalObject.h +++ b/contrib/llvm/include/llvm/IR/GlobalObject.h @@ -1,4 +1,4 @@ -//===-- llvm/GlobalObject.h - Class to represent a global object *- C++ -*-===// +//===-- llvm/GlobalObject.h - Class to represent global objects -*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -35,12 +35,24 @@ protected: std::string Section; // Section to emit this into, empty means default Comdat *ObjComdat; + static const unsigned AlignmentBits = 5; + static const unsigned GlobalObjectSubClassDataBits = + GlobalValueSubClassDataBits - AlignmentBits; + +private: + static const unsigned AlignmentMask = (1 << AlignmentBits) - 1; + public: unsigned getAlignment() const { - return (1u << getGlobalValueSubClassData()) >> 1; + unsigned Data = getGlobalValueSubClassData(); + unsigned AlignmentData = Data & AlignmentMask; + return (1u << AlignmentData) >> 1; } void setAlignment(unsigned Align); + unsigned getGlobalObjectSubClassData() const; + void setGlobalObjectSubClassData(unsigned Val); + bool hasSection() const { return !StringRef(getSection()).empty(); } const char *getSection() const { return Section.c_str(); } void setSection(StringRef S); diff --git a/contrib/llvm/include/llvm/IR/GlobalValue.h b/contrib/llvm/include/llvm/IR/GlobalValue.h index 68e410ba4b8b..d0f7e9a11790 100644 --- a/contrib/llvm/include/llvm/IR/GlobalValue.h +++ b/contrib/llvm/include/llvm/IR/GlobalValue.h @@ -20,6 +20,7 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/DerivedTypes.h" +#include namespace llvm { @@ -84,6 +85,7 @@ private: // (19 + 3 + 2 + 1 + 2 + 5) == 32. unsigned SubClassData : 19; protected: + static const unsigned GlobalValueSubClassDataBits = 19; unsigned getGlobalValueSubClassData() const { return SubClassData; } @@ -246,6 +248,7 @@ public: bool hasLinkOnceLinkage() const { return isLinkOnceLinkage(Linkage); } + bool hasLinkOnceODRLinkage() const { return isLinkOnceODRLinkage(Linkage); } bool hasWeakLinkage() const { return isWeakLinkage(Linkage); } @@ -309,7 +312,7 @@ public: /// Make sure this GlobalValue is fully read. If the module is corrupt, this /// returns true and fills in the optional string with information about the /// problem. If successful, this returns false. - bool Materialize(std::string *ErrInfo = nullptr); + std::error_code materialize(); /// If this GlobalValue is read in, and if the GVMaterializer supports it, /// release the memory for the function, and set it up to be materialized @@ -325,6 +328,13 @@ public: /// the current translation unit. bool isDeclaration() const; + bool isDeclarationForLinker() const { + if (hasAvailableExternallyLinkage()) + return true; + + return isDeclaration(); + } + /// This method unlinks 'this' from the containing module, but does not delete /// it. virtual void removeFromParent() = 0; diff --git a/contrib/llvm/include/llvm/IR/IRBuilder.h b/contrib/llvm/include/llvm/IR/IRBuilder.h index aed2463d42d8..e5f62fb9625e 100644 --- a/contrib/llvm/include/llvm/IR/IRBuilder.h +++ b/contrib/llvm/include/llvm/IR/IRBuilder.h @@ -28,7 +28,7 @@ #include "llvm/Support/CBindingWrapping.h" namespace llvm { - class MDNode; +class MDNode; /// \brief This provides the default implementation of the IRBuilder /// 'InsertHelper' method that is called whenever an instruction is created by @@ -364,43 +364,60 @@ public: /// \brief Create and insert a memset to the specified pointer and the /// specified value. /// - /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is - /// specified, it will be added to the instruction. + /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is + /// specified, it will be added to the instruction. Likewise with alias.scope + /// and noalias tags. CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, unsigned Align, - bool isVolatile = false, MDNode *TBAATag = nullptr) { - return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, TBAATag); + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr) { + return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, + TBAATag, ScopeTag, NoAliasTag); } CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align, - bool isVolatile = false, MDNode *TBAATag = nullptr); + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); /// \brief Create and insert a memcpy between the specified pointers. /// /// If the pointers aren't i8*, they will be converted. If a TBAA tag is - /// specified, it will be added to the instruction. + /// specified, it will be added to the instruction. Likewise with alias.scope + /// and noalias tags. CallInst *CreateMemCpy(Value *Dst, Value *Src, uint64_t Size, unsigned Align, bool isVolatile = false, MDNode *TBAATag = nullptr, - MDNode *TBAAStructTag = nullptr) { + MDNode *TBAAStructTag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr) { return CreateMemCpy(Dst, Src, getInt64(Size), Align, isVolatile, TBAATag, - TBAAStructTag); + TBAAStructTag, ScopeTag, NoAliasTag); } CallInst *CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align, bool isVolatile = false, MDNode *TBAATag = nullptr, - MDNode *TBAAStructTag = nullptr); + MDNode *TBAAStructTag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); /// \brief Create and insert a memmove between the specified /// pointers. /// /// If the pointers aren't i8*, they will be converted. If a TBAA tag is - /// specified, it will be added to the instruction. + /// specified, it will be added to the instruction. Likewise with alias.scope + /// and noalias tags. CallInst *CreateMemMove(Value *Dst, Value *Src, uint64_t Size, unsigned Align, - bool isVolatile = false, MDNode *TBAATag = nullptr) { - return CreateMemMove(Dst, Src, getInt64(Size), Align, isVolatile, TBAATag); + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr) { + return CreateMemMove(Dst, Src, getInt64(Size), Align, isVolatile, + TBAATag, ScopeTag, NoAliasTag); } CallInst *CreateMemMove(Value *Dst, Value *Src, Value *Size, unsigned Align, - bool isVolatile = false, MDNode *TBAATag = nullptr); + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); /// \brief Create a lifetime.start intrinsic. /// @@ -412,7 +429,46 @@ public: /// If the pointer isn't i8* it will be converted. CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr); + /// \brief Create a call to Masked Load intrinsic + CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask, + Value *PassThru = 0, const Twine &Name = ""); + + /// \brief Create a call to Masked Store intrinsic + CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, + Value *Mask); + + /// \brief Create an assume intrinsic call that allows the optimizer to + /// assume that the provided condition will be true. + CallInst *CreateAssumption(Value *Cond); + + /// \brief Create a call to the experimental.gc.statepoint intrinsic to + /// start a new statepoint sequence. + CallInst *CreateGCStatepoint(Value *ActualCallee, + ArrayRef CallArgs, + ArrayRef DeoptArgs, + ArrayRef GCArgs, + const Twine &Name = ""); + + /// \brief Create a call to the experimental.gc.result intrinsic to extract + /// the result from a call wrapped in a statepoint. + CallInst *CreateGCResult(Instruction *Statepoint, + Type *ResultType, + const Twine &Name = ""); + + /// \brief Create a call to the experimental.gc.relocate intrinsics to + /// project the relocated value of one pointer from the statepoint. + CallInst *CreateGCRelocate(Instruction *Statepoint, + int BaseOffset, + int DerivedOffset, + Type *ResultType, + const Twine &Name = ""); + private: + /// \brief Create a call to a masked intrinsic with given Id. + /// Masked intrinsic has only one overloaded type - data type. + CallInst *CreateMaskedIntrinsic(unsigned Id, ArrayRef Ops, + Type *DataTy, const Twine &Name = ""); + Value *getCastedInt8PtrValue(Value *Ptr); }; @@ -429,7 +485,7 @@ private: /// The first template argument handles whether or not to preserve names in the /// final instruction output. This defaults to on. The second template argument /// specifies a class to use for creating constants. This defaults to creating -/// minimally folded constants. The fourth template argument allows clients to +/// minimally folded constants. The third template argument allows clients to /// specify custom insertion hooks that are called on every newly created /// insertion. template()), + return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, None), Name); } InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, @@ -1226,6 +1281,18 @@ public: return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name); return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name); } + + Value *CreateBitOrPointerCast(Value *V, Type *DestTy, + const Twine &Name = "") { + if (V->getType() == DestTy) + return V; + if (V->getType()->isPointerTy() && DestTy->isIntegerTy()) + return CreatePtrToInt(V, DestTy, Name); + if (V->getType()->isIntegerTy() && DestTy->isPointerTy()) + return CreateIntToPtr(V, DestTy, Name); + + return CreateBitCast(V, DestTy, Name); + } private: // \brief Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a // compile time error, instead of converting the string to bool for the @@ -1508,6 +1575,44 @@ public: } return V; } + + /// \brief Create an assume intrinsic call that represents an alignment + /// assumption on the provided pointer. + /// + /// An optional offset can be provided, and if it is provided, the offset + /// must be subtracted from the provided pointer to get the pointer with the + /// specified alignment. + CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, + unsigned Alignment, + Value *OffsetValue = nullptr) { + assert(isa(PtrValue->getType()) && + "trying to create an alignment assumption on a non-pointer?"); + + PointerType *PtrTy = cast(PtrValue->getType()); + Type *IntPtrTy = getIntPtrTy(&DL, PtrTy->getAddressSpace()); + Value *PtrIntValue = CreatePtrToInt(PtrValue, IntPtrTy, "ptrint"); + + Value *Mask = ConstantInt::get(IntPtrTy, + Alignment > 0 ? Alignment - 1 : 0); + if (OffsetValue) { + bool IsOffsetZero = false; + if (ConstantInt *CI = dyn_cast(OffsetValue)) + IsOffsetZero = CI->isZero(); + + if (!IsOffsetZero) { + if (OffsetValue->getType() != IntPtrTy) + OffsetValue = CreateIntCast(OffsetValue, IntPtrTy, /*isSigned*/ true, + "offsetcast"); + PtrIntValue = CreateSub(PtrIntValue, OffsetValue, "offsetptr"); + } + } + + Value *Zero = ConstantInt::get(IntPtrTy, 0); + Value *MaskedPtr = CreateAnd(PtrIntValue, Mask, "maskedptr"); + Value *InvCond = CreateICmpEQ(MaskedPtr, Zero, "maskcond"); + + return CreateAssumption(InvCond); + } }; // Create wrappers for C Binding types (see CBindingWrapping.h). diff --git a/contrib/llvm/include/llvm/IR/IRPrintingPasses.h b/contrib/llvm/include/llvm/IR/IRPrintingPasses.h index 2f78c83165eb..7f2027b6e297 100644 --- a/contrib/llvm/include/llvm/IR/IRPrintingPasses.h +++ b/contrib/llvm/include/llvm/IR/IRPrintingPasses.h @@ -16,8 +16,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_IR_IR_PRINTING_PASSES_H -#define LLVM_IR_IR_PRINTING_PASSES_H +#ifndef LLVM_IR_IRPRINTINGPASSES_H +#define LLVM_IR_IRPRINTINGPASSES_H #include "llvm/ADT/StringRef.h" #include @@ -58,7 +58,7 @@ public: PrintModulePass(); PrintModulePass(raw_ostream &OS, const std::string &Banner = ""); - PreservedAnalyses run(Module *M); + PreservedAnalyses run(Module &M); static StringRef name() { return "PrintModulePass"; } }; @@ -75,7 +75,7 @@ public: PrintFunctionPass(); PrintFunctionPass(raw_ostream &OS, const std::string &Banner = ""); - PreservedAnalyses run(Function *F); + PreservedAnalyses run(Function &F); static StringRef name() { return "PrintFunctionPass"; } }; diff --git a/contrib/llvm/include/llvm/IR/InlineAsm.h b/contrib/llvm/include/llvm/IR/InlineAsm.h index ac190892bab0..b2d79d0f0bfc 100644 --- a/contrib/llvm/include/llvm/IR/InlineAsm.h +++ b/contrib/llvm/include/llvm/IR/InlineAsm.h @@ -25,12 +25,9 @@ namespace llvm { class PointerType; class FunctionType; class Module; + struct InlineAsmKeyType; -template -class ConstantUniqueMap; -template -struct ConstantCreator; +template class ConstantUniqueMap; class InlineAsm : public Value { public: @@ -40,9 +37,8 @@ public: }; private: - friend struct ConstantCreator; - friend class ConstantUniqueMap; + friend struct InlineAsmKeyType; + friend class ConstantUniqueMap; InlineAsm(const InlineAsm &) LLVM_DELETED_FUNCTION; void operator=(const InlineAsm&) LLVM_DELETED_FUNCTION; diff --git a/contrib/llvm/include/llvm/IR/InstrTypes.h b/contrib/llvm/include/llvm/IR/InstrTypes.h index 981aad852b29..030f5d690a0c 100644 --- a/contrib/llvm/include/llvm/IR/InstrTypes.h +++ b/contrib/llvm/include/llvm/IR/InstrTypes.h @@ -29,8 +29,8 @@ class LLVMContext; // TerminatorInst Class //===----------------------------------------------------------------------===// -/// TerminatorInst - Subclasses of this class are all able to terminate a basic -/// block. Thus, these are all the flow control type of operations. +/// Subclasses of this class are all able to terminate a basic +/// block. Thus, these are all the flow control type of operations. /// class TerminatorInst : public Instruction { protected: @@ -51,23 +51,19 @@ protected: virtual BasicBlock *getSuccessorV(unsigned idx) const = 0; virtual unsigned getNumSuccessorsV() const = 0; virtual void setSuccessorV(unsigned idx, BasicBlock *B) = 0; - TerminatorInst *clone_impl() const override = 0; public: - /// getNumSuccessors - Return the number of successors that this terminator - /// has. + /// Return the number of successors that this terminator has. unsigned getNumSuccessors() const { return getNumSuccessorsV(); } - /// getSuccessor - Return the specified successor. - /// + /// Return the specified successor. BasicBlock *getSuccessor(unsigned idx) const { return getSuccessorV(idx); } - /// setSuccessor - Update the specified successor to point at the provided - /// block. + /// Update the specified successor to point at the provided block. void setSuccessor(unsigned idx, BasicBlock *B) { setSuccessorV(idx, B); } @@ -153,7 +149,7 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - /// Create() - Construct a binary instruction, given the opcode and the two + /// Construct a binary instruction, given the opcode and the two /// operands. Optionally (if InstBefore is specified) insert the instruction /// into a BasicBlock right before the specified instruction. The specified /// Instruction is allowed to be a dereferenced end iterator. @@ -162,14 +158,14 @@ public: const Twine &Name = Twine(), Instruction *InsertBefore = nullptr); - /// Create() - Construct a binary instruction, given the opcode and the two + /// Construct a binary instruction, given the opcode and the two /// operands. Also automatically insert this instruction to the end of the /// BasicBlock specified. /// static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name, BasicBlock *InsertAtEnd); - /// Create* - These methods just forward to Create, and are useful when you + /// These methods just forward to Create, and are useful when you /// statically know what type of instruction you're going to create. These /// helpers just save some typing. #define HANDLE_BINARY_INST(N, OPC, CLASS) \ @@ -281,8 +277,7 @@ public: /// Helper functions to construct and inspect unary operations (NEG and NOT) /// via binary operators SUB and XOR: /// - /// CreateNeg, CreateNot - Create the NEG and NOT - /// instructions out of SUB and XOR instructions. + /// Create the NEG and NOT instructions out of SUB and XOR instructions. /// static BinaryOperator *CreateNeg(Value *Op, const Twine &Name = "", Instruction *InsertBefore = nullptr); @@ -305,16 +300,14 @@ public: static BinaryOperator *CreateNot(Value *Op, const Twine &Name, BasicBlock *InsertAtEnd); - /// isNeg, isFNeg, isNot - Check if the given Value is a - /// NEG, FNeg, or NOT instruction. + /// Check if the given Value is a NEG, FNeg, or NOT instruction. /// static bool isNeg(const Value *V); static bool isFNeg(const Value *V, bool IgnoreZeroSign=false); static bool isNot(const Value *V); - /// getNegArgument, getNotArgument - Helper functions to extract the - /// unary argument of a NEG, FNEG or NOT operation implemented via - /// Sub, FSub, or Xor. + /// Helper functions to extract the unary argument of a NEG, FNEG or NOT + /// operation implemented via Sub, FSub, or Xor. /// static const Value *getNegArgument(const Value *BinOp); static Value *getNegArgument( Value *BinOp); @@ -327,37 +320,42 @@ public: return static_cast(Instruction::getOpcode()); } - /// swapOperands - Exchange the two operands to this instruction. + /// Exchange the two operands to this instruction. /// This instruction is safe to use on any binary instruction and /// does not modify the semantics of the instruction. If the instruction /// cannot be reversed (ie, it's a Div), then return true. /// bool swapOperands(); - /// setHasNoUnsignedWrap - Set or clear the nsw flag on this instruction, - /// which must be an operator which supports this flag. See LangRef.html - /// for the meaning of this flag. + /// Set or clear the nsw flag on this instruction, which must be an operator + /// which supports this flag. See LangRef.html for the meaning of this flag. void setHasNoUnsignedWrap(bool b = true); - /// setHasNoSignedWrap - Set or clear the nsw flag on this instruction, - /// which must be an operator which supports this flag. See LangRef.html - /// for the meaning of this flag. + /// Set or clear the nsw flag on this instruction, which must be an operator + /// which supports this flag. See LangRef.html for the meaning of this flag. void setHasNoSignedWrap(bool b = true); - /// setIsExact - Set or clear the exact flag on this instruction, - /// which must be an operator which supports this flag. See LangRef.html - /// for the meaning of this flag. + /// Set or clear the exact flag on this instruction, which must be an operator + /// which supports this flag. See LangRef.html for the meaning of this flag. void setIsExact(bool b = true); - /// hasNoUnsignedWrap - Determine whether the no unsigned wrap flag is set. + /// Determine whether the no unsigned wrap flag is set. bool hasNoUnsignedWrap() const; - /// hasNoSignedWrap - Determine whether the no signed wrap flag is set. + /// Determine whether the no signed wrap flag is set. bool hasNoSignedWrap() const; - /// isExact - Determine whether the exact flag is set. + /// Determine whether the exact flag is set. bool isExact() const; + /// Convenience method to copy supported wrapping, exact, and fast-math flags + /// from V to this instruction. + void copyIRFlags(const Value *V); + + /// Logical 'and' of any supported wrapping, exact, and fast-math flags of + /// V and this instruction. + void andIRFlags(const Value *V); + // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return I->isBinaryOp(); @@ -378,7 +376,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryOperator, Value) // CastInst Class //===----------------------------------------------------------------------===// -/// CastInst - This is the base class for all instructions that perform data +/// This is the base class for all instructions that perform data /// casts. It is simply provided so that instruction category testing /// can be performed with code like: /// @@ -491,6 +489,19 @@ public: Instruction *InsertBefore = 0 ///< Place to insert the instruction ); + /// @brief Create a BitCast, a PtrToInt, or an IntToPTr cast instruction. + /// + /// If the value is a pointer type and the destination an integer type, + /// creates a PtrToInt cast. If the value is an integer type and the + /// destination a pointer type, creates an IntToPtr cast. Otherwise, creates + /// a bitcast. + static CastInst *CreateBitOrPointerCast( + Value *S, ///< The pointer value to be casted (operand 0) + Type *Ty, ///< The type to which cast should be made + const Twine &Name = "", ///< Name for the instruction + Instruction *InsertBefore = 0 ///< Place to insert the instruction + ); + /// @brief Create a ZExt, BitCast, or Trunc for int -> int casts. static CastInst *CreateIntegerCast( Value *S, ///< The pointer value to be casted (operand 0) @@ -553,6 +564,17 @@ public: Type *DestTy ///< The Type to which the value should be cast. ); + /// @brief Check whether a bitcast, inttoptr, or ptrtoint cast between these + /// types is valid and a no-op. + /// + /// This ensures that any pointer<->integer cast has enough bits in the + /// integer and any other cast is a bitcast. + static bool isBitOrNoopPointerCastable( + Type *SrcTy, ///< The Type from which the value should be cast. + Type *DestTy, ///< The Type to which the value should be cast. + const DataLayout *Layout = 0 ///< Optional DataLayout. + ); + /// Returns the opcode necessary to cast Val into Ty using usual casting /// rules. /// @brief Infer the opcode for cast operand and type diff --git a/contrib/llvm/include/llvm/IR/Instruction.h b/contrib/llvm/include/llvm/IR/Instruction.h index bac6a95b1b71..ba7791c99b6a 100644 --- a/contrib/llvm/include/llvm/IR/Instruction.h +++ b/contrib/llvm/include/llvm/IR/Instruction.h @@ -25,6 +25,7 @@ namespace llvm { class FastMathFlags; class LLVMContext; class MDNode; +struct AAMDNodes; template class SymbolTableListTraits; @@ -155,19 +156,25 @@ public: /// getAllMetadata - Get all metadata attached to this Instruction. The first /// element of each pair returned is the KindID, the second element is the /// metadata value. This list is returned sorted by the KindID. - void getAllMetadata(SmallVectorImpl > &MDs)const{ + void + getAllMetadata(SmallVectorImpl> &MDs) const { if (hasMetadata()) getAllMetadataImpl(MDs); } /// getAllMetadataOtherThanDebugLoc - This does the same thing as /// getAllMetadata, except that it filters out the debug location. - void getAllMetadataOtherThanDebugLoc(SmallVectorImpl > &MDs) const { + void getAllMetadataOtherThanDebugLoc( + SmallVectorImpl> &MDs) const { if (hasMetadataOtherThanDebugLoc()) getAllMetadataOtherThanDebugLocImpl(MDs); } + /// getAAMetadata - Fills the AAMDNodes structure with AA metadata from + /// this instruction. When Merge is true, the existing AA metadata is + /// merged with that from this instruction providing the most-general result. + void getAAMetadata(AAMDNodes &N, bool Merge = false) const; + /// setMetadata - Set the metadata of the specified kind to the specified /// node. This updates/replaces metadata if already present, or removes it if /// Node is null. @@ -179,7 +186,7 @@ public: /// convenience method for passes to do so. void dropUnknownMetadata(ArrayRef KnownIDs); void dropUnknownMetadata() { - return dropUnknownMetadata(ArrayRef()); + return dropUnknownMetadata(None); } void dropUnknownMetadata(unsigned ID1) { return dropUnknownMetadata(makeArrayRef(ID1)); @@ -189,6 +196,10 @@ public: return dropUnknownMetadata(IDs); } + /// setAAMetadata - Sets the metadata on this instruction from the + /// AAMDNodes structure. + void setAAMetadata(const AAMDNodes &N); + /// setDebugLoc - Set the debug location information for this instruction. void setDebugLoc(const DebugLoc &Loc) { DbgLoc = Loc; } @@ -220,11 +231,16 @@ public: /// this flag. void setHasAllowReciprocal(bool B); - /// Convenience function for setting all the fast-math flags on this + /// Convenience function for setting multiple fast-math flags on this /// instruction, which must be an operator which supports these flags. See - /// LangRef.html for the meaning of these flats. + /// LangRef.html for the meaning of these flags. void setFastMathFlags(FastMathFlags FMF); + /// Convenience function for transferring all fast-math flag values to this + /// instruction, which must be an operator which supports these flags. See + /// LangRef.html for the meaning of these flags. + void copyFastMathFlags(FastMathFlags FMF); + /// Determine whether the unsafe-algebra flag is set. bool hasUnsafeAlgebra() const; @@ -242,7 +258,7 @@ public: /// Convenience function for getting all the fast-math flags, which must be an /// operator which supports these flags. See LangRef.html for the meaning of - /// these flats. + /// these flags. FastMathFlags getFastMathFlags() const; /// Copy I's fast-math flags @@ -258,9 +274,10 @@ private: // These are all implemented in Metadata.cpp. MDNode *getMetadataImpl(unsigned KindID) const; MDNode *getMetadataImpl(StringRef Kind) const; - void getAllMetadataImpl(SmallVectorImpl > &)const; - void getAllMetadataOtherThanDebugLocImpl(SmallVectorImpl > &) const; + void + getAllMetadataImpl(SmallVectorImpl> &) const; + void getAllMetadataOtherThanDebugLocImpl( + SmallVectorImpl> &) const; void clearMetadataHashEntries(); public: //===--------------------------------------------------------------------===// @@ -323,6 +340,11 @@ public: return mayReadFromMemory() || mayWriteToMemory(); } + /// isAtomic - Return true if this instruction has an + /// AtomicOrdering of unordered or higher. + /// + bool isAtomic() const; + /// mayThrow - Return true if this instruction may throw an exception. /// bool mayThrow() const; diff --git a/contrib/llvm/include/llvm/IR/Instructions.h b/contrib/llvm/include/llvm/IR/Instructions.h index 308467f7aa17..045e51eb1baa 100644 --- a/contrib/llvm/include/llvm/IR/Instructions.h +++ b/contrib/llvm/include/llvm/IR/Instructions.h @@ -17,8 +17,8 @@ #define LLVM_IR_INSTRUCTIONS_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/DerivedTypes.h" @@ -50,6 +50,22 @@ enum SynchronizationScope { CrossThread = 1 }; +/// Returns true if the ordering is at least as strong as acquire +/// (i.e. acquire, acq_rel or seq_cst) +inline bool isAtLeastAcquire(AtomicOrdering Ord) { + return (Ord == Acquire || + Ord == AcquireRelease || + Ord == SequentiallyConsistent); +} + +/// Returns true if the ordering is at least as strong as release +/// (i.e. release, acq_rel or seq_cst) +inline bool isAtLeastRelease(AtomicOrdering Ord) { +return (Ord == Release || + Ord == AcquireRelease || + Ord == SequentiallyConsistent); +} + //===----------------------------------------------------------------------===// // AllocaInst Class //===----------------------------------------------------------------------===// @@ -119,7 +135,7 @@ public: return getSubclassDataFromInstruction() & 32; } - /// \brief Specify whether this alloca is used to represent a the arguments to + /// \brief Specify whether this alloca is used to represent the arguments to /// a call. void setUsedWithInAlloca(bool V) { setInstructionSubclassData((getSubclassDataFromInstruction() & ~32) | @@ -225,7 +241,6 @@ public: (xthread << 6)); } - bool isAtomic() const { return getOrdering() != NotAtomic; } void setAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope = CrossThread) { setOrdering(Ordering); @@ -345,7 +360,6 @@ public: (xthread << 6)); } - bool isAtomic() const { return getOrdering() != NotAtomic; } void setAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope = CrossThread) { setOrdering(Ordering); @@ -637,7 +651,7 @@ public: Sub, /// *p = old & v And, - /// *p = ~old & v + /// *p = ~(old & v) Nand, /// *p = old | v Or, diff --git a/contrib/llvm/include/llvm/IR/IntrinsicInst.h b/contrib/llvm/include/llvm/IR/IntrinsicInst.h index e053f7867c53..c227ea080167 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicInst.h +++ b/contrib/llvm/include/llvm/IR/IntrinsicInst.h @@ -28,6 +28,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Metadata.h" namespace llvm { /// IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic @@ -81,7 +82,14 @@ namespace llvm { class DbgDeclareInst : public DbgInfoIntrinsic { public: Value *getAddress() const; - MDNode *getVariable() const { return cast(getArgOperand(1)); } + MDNode *getVariable() const { + return cast( + cast(getArgOperand(1))->getMetadata()); + } + MDNode *getExpression() const { + return cast( + cast(getArgOperand(2))->getMetadata()); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const IntrinsicInst *I) { @@ -102,7 +110,14 @@ namespace llvm { return cast( const_cast(getArgOperand(1)))->getZExtValue(); } - MDNode *getVariable() const { return cast(getArgOperand(2)); } + MDNode *getVariable() const { + return cast( + cast(getArgOperand(2))->getMetadata()); + } + MDNode *getExpression() const { + return cast( + cast(getArgOperand(3))->getMetadata()); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const IntrinsicInst *I) { @@ -320,6 +335,33 @@ namespace llvm { Value *getSrc() const { return const_cast(getArgOperand(1)); } }; + /// This represents the llvm.instrprof_increment intrinsic. + class InstrProfIncrementInst : public IntrinsicInst { + public: + static inline bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_increment; + } + static inline bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + GlobalVariable *getName() const { + return cast( + const_cast(getArgOperand(0))->stripPointerCasts()); + } + + ConstantInt *getHash() const { + return cast(const_cast(getArgOperand(1))); + } + + ConstantInt *getNumCounters() const { + return cast(const_cast(getArgOperand(2))); + } + + ConstantInt *getIndex() const { + return cast(const_cast(getArgOperand(3))); + } + }; } #endif diff --git a/contrib/llvm/include/llvm/IR/Intrinsics.h b/contrib/llvm/include/llvm/IR/Intrinsics.h index b0d746bd4127..56d1e4af26aa 100644 --- a/contrib/llvm/include/llvm/IR/Intrinsics.h +++ b/contrib/llvm/include/llvm/IR/Intrinsics.h @@ -28,10 +28,9 @@ class LLVMContext; class Module; class AttributeSet; -/// Intrinsic Namespace - This namespace contains an enum with a value for -/// every intrinsic/builtin function known by LLVM. These enum values are -/// returned by Function::getIntrinsicID(). -/// +/// This namespace contains an enum with a value for every intrinsic/builtin +/// function known by LLVM. The enum values are returned by +/// Function::getIntrinsicID(). namespace Intrinsic { enum ID { not_intrinsic = 0, // Must be zero @@ -43,25 +42,21 @@ namespace Intrinsic { , num_intrinsics }; - /// Intrinsic::getName(ID) - Return the LLVM name for an intrinsic, such as - /// "llvm.ppc.altivec.lvx". + /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx". std::string getName(ID id, ArrayRef Tys = None); - /// Intrinsic::getType(ID) - Return the function type for an intrinsic. - /// + /// Return the function type for an intrinsic. FunctionType *getType(LLVMContext &Context, ID id, ArrayRef Tys = None); - /// Intrinsic::isOverloaded(ID) - Returns true if the intrinsic can be - /// overloaded. + /// Returns true if the intrinsic can be overloaded. bool isOverloaded(ID id); - /// Intrinsic::getAttributes(ID) - Return the attributes for an intrinsic. - /// + /// Return the attributes for an intrinsic. AttributeSet getAttributes(LLVMContext &C, ID id); - /// Intrinsic::getDeclaration(M, ID) - Create or insert an LLVM Function - /// declaration for an intrinsic, and return it. + /// Create or insert an LLVM Function declaration for an intrinsic, and return + /// it. /// /// The Tys parameter is for intrinsics with overloaded types (e.g., those /// using iAny, fAny, vAny, or iPTRAny). For a declaration of an overloaded @@ -75,14 +70,14 @@ namespace Intrinsic { /// Map a MS builtin name to an intrinsic ID. ID getIntrinsicForMSBuiltin(const char *Prefix, const char *BuiltinName); - /// IITDescriptor - This is a type descriptor which explains the type - /// requirements of an intrinsic. This is returned by - /// getIntrinsicInfoTableEntries. + /// This is a type descriptor which explains the type requirements of an + /// intrinsic. This is returned by getIntrinsicInfoTableEntries. struct IITDescriptor { enum IITDescriptorKind { Void, VarArg, MMX, Metadata, Half, Float, Double, Integer, Vector, Pointer, Struct, - Argument, ExtendArgument, TruncArgument, HalfVecArgument + Argument, ExtendArgument, TruncArgument, HalfVecArgument, + SameVecWidthArgument, PtrToArgument } Kind; union { @@ -102,13 +97,15 @@ namespace Intrinsic { }; unsigned getArgumentNumber() const { assert(Kind == Argument || Kind == ExtendArgument || - Kind == TruncArgument || Kind == HalfVecArgument); + Kind == TruncArgument || Kind == HalfVecArgument || + Kind == SameVecWidthArgument || Kind == PtrToArgument); return Argument_Info >> 2; } ArgKind getArgumentKind() const { assert(Kind == Argument || Kind == ExtendArgument || - Kind == TruncArgument || Kind == HalfVecArgument); - return (ArgKind)(Argument_Info&3); + Kind == TruncArgument || Kind == HalfVecArgument || + Kind == SameVecWidthArgument || Kind == PtrToArgument); + return (ArgKind)(Argument_Info & 3); } static IITDescriptor get(IITDescriptorKind K, unsigned Field) { @@ -117,9 +114,8 @@ namespace Intrinsic { } }; - /// getIntrinsicInfoTableEntries - Return the IIT table descriptor for the - /// specified intrinsic into an array of IITDescriptors. - /// + /// Return the IIT table descriptor for the specified intrinsic into an array + /// of IITDescriptors. void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl &T); } // End Intrinsic namespace diff --git a/contrib/llvm/include/llvm/IR/Intrinsics.td b/contrib/llvm/include/llvm/IR/Intrinsics.td index 0b8f64fc7984..a1188bccdc25 100644 --- a/contrib/llvm/include/llvm/IR/Intrinsics.td +++ b/contrib/llvm/include/llvm/IR/Intrinsics.td @@ -112,6 +112,11 @@ class LLVMMatchType // the intrinsic is overloaded, so the matched type should be declared as iAny. class LLVMExtendedType : LLVMMatchType; class LLVMTruncatedType : LLVMMatchType; +class LLVMVectorSameWidth + : LLVMMatchType { + ValueType ElTy = elty.VT; +} +class LLVMPointerTo : LLVMMatchType; // Match the type of another intrinsic parameter that is expected to be a // vector type, but change the element count to be half as many @@ -254,6 +259,10 @@ def int_gcwrite : Intrinsic<[], // def int_returnaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>; def int_frameaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_frameallocate : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty]>; +def int_framerecover : Intrinsic<[llvm_ptr_ty], + [llvm_ptr_ty, llvm_ptr_ty], + [IntrNoMem]>; def int_read_register : Intrinsic<[llvm_anyint_ty], [llvm_metadata_ty], [IntrNoMem], "llvm.read_register">; def int_write_register : Intrinsic<[], [llvm_metadata_ty, llvm_anyint_ty], @@ -277,12 +286,22 @@ def int_pcmarker : Intrinsic<[], [llvm_i32_ty]>; def int_readcyclecounter : Intrinsic<[llvm_i64_ty]>; +// The assume intrinsic is marked as arbitrarily writing so that proper +// control dependencies will be maintained. +def int_assume : Intrinsic<[], [llvm_i1_ty], []>; + // Stack Protector Intrinsic - The stackprotector intrinsic writes the stack // guard to the correct place on the stack frame. def int_stackprotector : Intrinsic<[], [llvm_ptr_ty, llvm_ptrptr_ty], []>; def int_stackprotectorcheck : Intrinsic<[], [llvm_ptrptr_ty], [IntrReadWriteArgMem]>; +// A counter increment for instrumentation based profiling. +def int_instrprof_increment : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_i32_ty], + []>; + //===------------------- Standard C Library Intrinsics --------------------===// // @@ -324,6 +343,8 @@ let Properties = [IntrNoMem] in { def int_exp : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; def int_exp2 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; def int_fabs : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; + def int_minnum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; + def int_maxnum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; def int_copysign : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; def int_floor : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; @@ -369,9 +390,12 @@ let Properties = [IntrNoMem] in { // places. let Properties = [IntrNoMem] in { def int_dbg_declare : Intrinsic<[], - [llvm_metadata_ty, llvm_metadata_ty]>; + [llvm_metadata_ty, + llvm_metadata_ty, + llvm_metadata_ty]>; def int_dbg_value : Intrinsic<[], [llvm_metadata_ty, llvm_i64_ty, + llvm_metadata_ty, llvm_metadata_ty]>; } @@ -476,11 +500,29 @@ def int_experimental_stackmap : Intrinsic<[], def int_experimental_patchpoint_void : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, - llvm_vararg_ty]>; + llvm_vararg_ty], + [Throws]>; def int_experimental_patchpoint_i64 : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, - llvm_vararg_ty]>; + llvm_vararg_ty], + [Throws]>; + + +//===------------------------ Garbage Collection Intrinsics ---------------===// +// These are documented in docs/Statepoint.rst + +def int_experimental_gc_statepoint : Intrinsic<[llvm_i32_ty], + [llvm_anyptr_ty, llvm_i32_ty, + llvm_i32_ty, llvm_vararg_ty]>; + +def int_experimental_gc_result_int : Intrinsic<[llvm_anyint_ty], [llvm_i32_ty]>; +def int_experimental_gc_result_float : Intrinsic<[llvm_anyfloat_ty], + [llvm_i32_ty]>; +def int_experimental_gc_result_ptr : Intrinsic<[llvm_anyptr_ty], [llvm_i32_ty]>; + +def int_experimental_gc_relocate : Intrinsic<[llvm_anyptr_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>; //===-------------------------- Other Intrinsics --------------------------===// // @@ -528,6 +570,17 @@ def int_convertuu : Intrinsic<[llvm_anyint_ty], def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], [], "llvm.clear_cache">; +//===-------------------------- Masked Intrinsics -------------------------===// +// +def int_masked_store : Intrinsic<[], [llvm_anyvector_ty, LLVMPointerTo<0>, + llvm_i32_ty, + LLVMVectorSameWidth<0, llvm_i1_ty>], + [IntrReadWriteArgMem]>; + +def int_masked_load : Intrinsic<[llvm_anyvector_ty], + [LLVMPointerTo<0>, llvm_i32_ty, + LLVMVectorSameWidth<0, llvm_i1_ty>, LLVMMatchType<0>], + [IntrReadArgMem]>; //===----------------------------------------------------------------------===// // Target-specific intrinsics //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsARM.td b/contrib/llvm/include/llvm/IR/IntrinsicsARM.td index a02d7072d720..ce758e257312 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsARM.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsARM.td @@ -20,8 +20,13 @@ let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". def int_arm_thread_pointer : GCCBuiltin<"__builtin_thread_pointer">, Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; +// A space-consuming intrinsic primarily for testing ARMConstantIslands. The +// first argument is the number of bytes this "instruction" takes up, the second +// and return value are essentially chains, used to force ordering during ISel. +def int_arm_space : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; + //===----------------------------------------------------------------------===// -// Saturating Arithmentic +// Saturating Arithmetic def int_arm_qadd : GCCBuiltin<"__builtin_arm_qadd">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], @@ -132,6 +137,7 @@ def int_arm_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], // HINT def int_arm_hint : Intrinsic<[], [llvm_i32_ty]>; +def int_arm_dbg : Intrinsic<[], [llvm_i32_ty]>; //===----------------------------------------------------------------------===// // RBIT @@ -340,10 +346,6 @@ def int_arm_neon_vqneg : Neon_1Arg_Intrinsic; // Vector Count Leading Sign/Zero Bits. def int_arm_neon_vcls : Neon_1Arg_Intrinsic; -def int_arm_neon_vclz : Neon_1Arg_Intrinsic; - -// Vector Count One Bits. -def int_arm_neon_vcnt : Neon_1Arg_Intrinsic; // Vector Reciprocal Estimate. def int_arm_neon_vrecpe : Neon_1Arg_Intrinsic; diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td b/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td index cd512841a1af..9deed414b50a 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -797,24 +797,30 @@ def llvm_anyi64ptr_ty : LLVMAnyPointerType; // (space)i64* // Generated within nvvm. Use for ldu on sm_20 or later def int_nvvm_ldu_global_i : Intrinsic<[llvm_anyint_ty], - [LLVMAnyPointerType>], [IntrReadMem, NoCapture<0>], + [LLVMAnyPointerType>, llvm_i32_ty], + [IntrReadMem, NoCapture<0>], "llvm.nvvm.ldu.global.i">; def int_nvvm_ldu_global_f : Intrinsic<[llvm_anyfloat_ty], - [LLVMAnyPointerType>], [IntrReadMem, NoCapture<0>], + [LLVMAnyPointerType>, llvm_i32_ty], + [IntrReadMem, NoCapture<0>], "llvm.nvvm.ldu.global.f">; def int_nvvm_ldu_global_p : Intrinsic<[llvm_anyptr_ty], - [LLVMAnyPointerType>], [IntrReadMem, NoCapture<0>], + [LLVMAnyPointerType>, llvm_i32_ty], + [IntrReadMem, NoCapture<0>], "llvm.nvvm.ldu.global.p">; // Generated within nvvm. Use for ldg on sm_35 or later def int_nvvm_ldg_global_i : Intrinsic<[llvm_anyint_ty], - [LLVMAnyPointerType>], [IntrReadMem, NoCapture<0>], + [LLVMAnyPointerType>, llvm_i32_ty], + [IntrReadMem, NoCapture<0>], "llvm.nvvm.ldg.global.i">; def int_nvvm_ldg_global_f : Intrinsic<[llvm_anyfloat_ty], - [LLVMAnyPointerType>], [IntrReadMem, NoCapture<0>], + [LLVMAnyPointerType>, llvm_i32_ty], + [IntrReadMem, NoCapture<0>], "llvm.nvvm.ldg.global.f">; def int_nvvm_ldg_global_p : Intrinsic<[llvm_anyptr_ty], - [LLVMAnyPointerType>], [IntrReadMem, NoCapture<0>], + [LLVMAnyPointerType>, llvm_i32_ty], + [IntrReadMem, NoCapture<0>], "llvm.nvvm.ldg.global.p">; // Use for generic pointers diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/contrib/llvm/include/llvm/IR/IntrinsicsPowerPC.td index 49ddfb8b613b..5cdabdeadaea 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsPowerPC.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsPowerPC.td @@ -28,8 +28,10 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". def int_ppc_dcbz : Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbzl : Intrinsic<[], [llvm_ptr_ty], []>; - // sync instruction + // sync instruction (i.e. sync 0, a.k.a hwsync) def int_ppc_sync : Intrinsic<[], [], []>; + // lwsync is sync 1 + def int_ppc_lwsync : Intrinsic<[], [], []>; // Intrinsics used to generate ctr-based loops. These should only be // generated by the PowerPC backend! @@ -45,6 +47,13 @@ let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". list properties> : GCCBuiltin, Intrinsic; + + /// PowerPC_VSX_Intrinsic - Base class for all VSX intrinsics. + class PowerPC_VSX_Intrinsic ret_types, + list param_types, + list properties> + : GCCBuiltin, + Intrinsic; } //===----------------------------------------------------------------------===// @@ -86,6 +95,32 @@ class PowerPC_Vec_WWW_Intrinsic [IntrNoMem]>; +//===----------------------------------------------------------------------===// +// PowerPC VSX Intrinsic Class Definitions. +// + +/// PowerPC_VSX_Vec_DDD_Intrinsic - A PowerPC intrinsic that takes two v2f64 +/// vectors and returns one. These intrinsics have no side effects. +class PowerPC_VSX_Vec_DDD_Intrinsic + : PowerPC_VSX_Intrinsic; + +/// PowerPC_VSX_Vec_FFF_Intrinsic - A PowerPC intrinsic that takes two v4f32 +/// vectors and returns one. These intrinsics have no side effects. +class PowerPC_VSX_Vec_FFF_Intrinsic + : PowerPC_VSX_Intrinsic; + +/// PowerPC_VSX_Sca_DDD_Intrinsic - A PowerPC intrinsic that takes two f64 +/// scalars and returns one. These intrinsics have no side effects. +class PowerPC_VSX_Sca_DDD_Intrinsic + : PowerPC_VSX_Intrinsic; + + //===----------------------------------------------------------------------===// // PowerPC Altivec Intrinsic Definitions. @@ -474,3 +509,36 @@ def int_ppc_altivec_vexptefp : PowerPC_Vec_FF_Intrinsic<"vexptefp">; def int_ppc_altivec_vlogefp : PowerPC_Vec_FF_Intrinsic<"vlogefp">; def int_ppc_altivec_vrefp : PowerPC_Vec_FF_Intrinsic<"vrefp">; def int_ppc_altivec_vrsqrtefp : PowerPC_Vec_FF_Intrinsic<"vrsqrtefp">; + + +//===----------------------------------------------------------------------===// +// PowerPC VSX Intrinsic Definitions. + +let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". + +// Vector load. +def int_ppc_vsx_lxvw4x : + Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; +def int_ppc_vsx_lxvd2x : + Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + +// Vector store. +def int_ppc_vsx_stxvw4x : + Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty], [IntrReadWriteArgMem]>; +def int_ppc_vsx_stxvd2x : + Intrinsic<[], [llvm_v2f64_ty, llvm_ptr_ty], [IntrReadWriteArgMem]>; + +// Vector and scalar maximum. +def int_ppc_vsx_xvmaxdp : PowerPC_VSX_Vec_DDD_Intrinsic<"xvmaxdp">; +def int_ppc_vsx_xvmaxsp : PowerPC_VSX_Vec_FFF_Intrinsic<"xvmaxsp">; +def int_ppc_vsx_xsmaxdp : PowerPC_VSX_Sca_DDD_Intrinsic<"xsmaxdp">; + +// Vector and scalar minimum. +def int_ppc_vsx_xvmindp : PowerPC_VSX_Vec_DDD_Intrinsic<"xvmindp">; +def int_ppc_vsx_xvminsp : PowerPC_VSX_Vec_FFF_Intrinsic<"xvminsp">; +def int_ppc_vsx_xsmindp : PowerPC_VSX_Sca_DDD_Intrinsic<"xsmindp">; + +// Vector divide. +def int_ppc_vsx_xvdivdp : PowerPC_VSX_Vec_DDD_Intrinsic<"xvdivdp">; +def int_ppc_vsx_xvdivsp : PowerPC_VSX_Vec_FFF_Intrinsic<"xvdivsp">; +} diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsR600.td b/contrib/llvm/include/llvm/IR/IntrinsicsR600.td index ba69eaae089f..505566738221 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsR600.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsR600.td @@ -33,10 +33,14 @@ defm int_r600_read_tgid : R600ReadPreloadRegisterIntrinsic_xyz < "__builtin_r600_read_tgid">; defm int_r600_read_tidig : R600ReadPreloadRegisterIntrinsic_xyz < "__builtin_r600_read_tidig">; - } // End TargetPrefix = "r600" let TargetPrefix = "AMDGPU" in { + +class AMDGPUReadPreloadRegisterIntrinsic + : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, + GCCBuiltin; + def int_AMDGPU_div_scale : GCCBuiltin<"__builtin_amdgpu_div_scale">, // 1st parameter: Numerator // 2nd parameter: Denominator @@ -48,7 +52,7 @@ def int_AMDGPU_div_scale : GCCBuiltin<"__builtin_amdgpu_div_scale">, def int_AMDGPU_div_fmas : GCCBuiltin<"__builtin_amdgpu_div_fmas">, Intrinsic<[llvm_anyfloat_ty], - [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], [IntrNoMem]>; def int_AMDGPU_div_fixup : GCCBuiltin<"__builtin_amdgpu_div_fixup">, @@ -69,4 +73,13 @@ def int_AMDGPU_rsq : GCCBuiltin<"__builtin_amdgpu_rsq">, def int_AMDGPU_rsq_clamped : GCCBuiltin<"__builtin_amdgpu_rsq_clamped">, Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; +def int_AMDGPU_ldexp : GCCBuiltin<"__builtin_amdgpu_ldexp">, + Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem]>; + +def int_AMDGPU_class : GCCBuiltin<"__builtin_amdgpu_class">, + Intrinsic<[llvm_i1_ty], [llvm_anyfloat_ty, llvm_i32_ty], [IntrNoMem]>; + +def int_AMDGPU_read_workdim : AMDGPUReadPreloadRegisterIntrinsic < + "__builtin_amdgpu_read_workdim">; + } // End TargetPrefix = "AMDGPU" diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsX86.td b/contrib/llvm/include/llvm/IR/IntrinsicsX86.td index 5de950813cd4..81c729065505 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsX86.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsX86.td @@ -886,7 +886,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector insert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_insertps : GCCBuiltin<"__builtin_ia32_insertps128">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,llvm_i32_ty], + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; } @@ -896,13 +896,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_v16i8_ty], [IntrNoMem]>; def int_x86_sse41_pblendw : GCCBuiltin<"__builtin_ia32_pblendw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_sse41_blendpd : GCCBuiltin<"__builtin_ia32_blendpd">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty], + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_sse41_blendps : GCCBuiltin<"__builtin_ia32_blendps">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty], + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_sse41_blendvpd : GCCBuiltin<"__builtin_ia32_blendvpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,llvm_v2f64_ty], @@ -915,17 +915,17 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector dot product let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_dppd : GCCBuiltin<"__builtin_ia32_dppd">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,llvm_i32_ty], + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, Commutative]>; def int_x86_sse41_dpps : GCCBuiltin<"__builtin_ia32_dpps">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,llvm_i32_ty], + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, Commutative]>; } // Vector sum of absolute differences let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_i32_ty], + Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_i8_ty], [IntrNoMem, Commutative]>; } @@ -1171,10 +1171,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_blend_pd_256 : GCCBuiltin<"__builtin_ia32_blendpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, - llvm_v4f64_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx_blend_ps_256 : GCCBuiltin<"__builtin_ia32_blendps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, - llvm_v8f32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx_blendv_pd_256 : GCCBuiltin<"__builtin_ia32_blendvpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; @@ -1187,7 +1187,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_dp_ps_256 : GCCBuiltin<"__builtin_ia32_dpps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, - llvm_v8f32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; } // Vector compare @@ -1389,6 +1389,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_storeupd512_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_store_ss : + GCCBuiltin<"__builtin_ia32_storess_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; } //===----------------------------------------------------------------------===// @@ -1580,6 +1584,44 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_psrl_dq_bs : GCCBuiltin<"__builtin_ia32_psrldqi256_byteshift">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi512">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi512">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrai_d : GCCBuiltin<"__builtin_ia32_psradi512">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrai_q : GCCBuiltin<"__builtin_ia32_psraqi512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_psll_d : GCCBuiltin<"__builtin_ia32_pslld512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_q : GCCBuiltin<"__builtin_ia32_psllq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_d : GCCBuiltin<"__builtin_ia32_psrld512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_q : GCCBuiltin<"__builtin_ia32_psrlq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psra_d : GCCBuiltin<"__builtin_ia32_psrad512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psra_q : GCCBuiltin<"__builtin_ia32_psraq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } // Pack ops. @@ -1706,13 +1748,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_pblendw : GCCBuiltin<"__builtin_ia32_pblendw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx2_pblendd_128 : GCCBuiltin<"__builtin_ia32_pblendd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx2_pblendd_256 : GCCBuiltin<"__builtin_ia32_pblendd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty], [IntrNoMem]>; } // Vector load with broadcast @@ -1787,6 +1829,23 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_vinserti128 : GCCBuiltin<"__builtin_ia32_insert128i256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vextractf32x4_512 : + GCCBuiltin<"__builtin_ia32_extractf32x4_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v16f32_ty, llvm_i8_ty, + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextracti32x4_512 : + GCCBuiltin<"__builtin_ia32_extracti32x4_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v16i32_ty, llvm_i8_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextractf64x4_512 : + GCCBuiltin<"__builtin_ia32_extractf64x4_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v8f64_ty, llvm_i8_ty, + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextracti64x4_512 : + GCCBuiltin<"__builtin_ia32_extracti64x4_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v8i64_ty, llvm_i8_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; } // Conditional load ops @@ -1871,6 +1930,31 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_psrav_d_256 : GCCBuiltin<"__builtin_ia32_psrav8si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_psllv_d : GCCBuiltin<"__builtin_ia32_psllv16si_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_psllv_q : GCCBuiltin<"__builtin_ia32_psllv8di_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_psrav_d : GCCBuiltin<"__builtin_ia32_psrav16si_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_psrav_q : GCCBuiltin<"__builtin_ia32_psrav8di_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_psrlv_d : GCCBuiltin<"__builtin_ia32_psrlv16si_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_psrlv_q : GCCBuiltin<"__builtin_ia32_psrlv8di_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; } // Gather ops @@ -1951,11 +2035,9 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty, llvm_v32i8_ty, - llvm_i32_ty], [IntrNoMem, Commutative]>; + llvm_i8_ty], [IntrNoMem, Commutative]>; def int_x86_avx2_movntdqa : GCCBuiltin<"__builtin_ia32_movntdqa256">, Intrinsic<[llvm_v4i64_ty], [llvm_ptr_ty], [IntrReadMem]>; - def int_x86_avx512_movntdqa : GCCBuiltin<"__builtin_ia32_movntdqa512">, - Intrinsic<[llvm_v8i64_ty], [llvm_ptr_ty], [IntrReadMem]>; } //===----------------------------------------------------------------------===// @@ -1986,13 +2068,35 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfmaddps512">, + def int_x86_fma_mask_vfmadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfmaddps512_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_fma_vfmadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfmaddpd512">, + def int_x86_fma_mask_vfmadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfmaddps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmadd_ps_128 : GCCBuiltin<"__builtin_ia32_vfmaddps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfmaddpd512_mask">, Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfmaddpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmadd_pd_128 : GCCBuiltin<"__builtin_ia32_vfmaddpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_fma_vfmsub_ss : GCCBuiltin<"__builtin_ia32_vfmsubss">, Intrinsic<[llvm_v4f32_ty], @@ -2018,13 +2122,35 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfmsubps512">, + def int_x86_fma_mask_vfmsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfmsubps512_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfmsubpd512">, + def int_x86_fma_mask_vfmsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfmsubps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsub_ps_128 : GCCBuiltin<"__builtin_ia32_vfmsubps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfmsubpd512_mask">, Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfmsubpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfmsubpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_fma_vfnmadd_ss : GCCBuiltin<"__builtin_ia32_vfnmaddss">, Intrinsic<[llvm_v4f32_ty], @@ -2050,13 +2176,35 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfnmaddps512">, + def int_x86_fma_mask_vfnmadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfnmaddps512_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfnmaddpd512">, + def int_x86_fma_mask_vfnmadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfnmaddps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmadd_ps_128 : GCCBuiltin<"__builtin_ia32_vfnmaddps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfnmaddpd512_mask">, Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfnmaddpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmadd_pd_128 : GCCBuiltin<"__builtin_ia32_vfnmaddpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_fma_vfnmsub_ss : GCCBuiltin<"__builtin_ia32_vfnmsubss">, Intrinsic<[llvm_v4f32_ty], @@ -2082,13 +2230,35 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfnmsubps512">, + def int_x86_fma_mask_vfnmsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfnmsubps512_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfnmsubpd512">, + def int_x86_fma_mask_vfnmsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfnmsubps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmsub_ps_128 : GCCBuiltin<"__builtin_ia32_vfnmsubps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfnmsubpd512_mask">, Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfnmsubpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfnmsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfnmsubpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_fma_vfmaddsub_ps : GCCBuiltin<"__builtin_ia32_vfmaddsubps">, Intrinsic<[llvm_v4f32_ty], @@ -2108,13 +2278,35 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmaddsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfmaddsubps512">, + def int_x86_fma_mask_vfmaddsub_ps_512 : GCCBuiltin<"__builtin_ia32_vfmaddsubps512_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_fma_vfmaddsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd512">, + def int_x86_fma_mask_vfmaddsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfmaddsubps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmaddsub_ps_128 : GCCBuiltin<"__builtin_ia32_vfmaddsubps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmaddsub_pd_512 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd512_mask">, Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmaddsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmaddsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfmaddsubpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_fma_vfmsubadd_ps : GCCBuiltin<"__builtin_ia32_vfmsubaddps">, Intrinsic<[llvm_v4f32_ty], @@ -2134,13 +2326,35 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsubadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfmsubaddps512">, + def int_x86_fma_mask_vfmsubadd_ps_512 : GCCBuiltin<"__builtin_ia32_vfmsubaddps512_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsubadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd512">, + def int_x86_fma_mask_vfmsubadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfmsubaddps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsubadd_ps_128 : GCCBuiltin<"__builtin_ia32_vfmsubaddps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsubadd_pd_512 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd512_mask">, Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsubadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], + [IntrNoMem]>; + def int_x86_fma_mask_vfmsubadd_pd_128 : GCCBuiltin<"__builtin_ia32_vfmsubaddpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; } @@ -2748,6 +2962,30 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_rdseed_64 : Intrinsic<[llvm_i64_ty, llvm_i32_ty], [], []>; } +//===----------------------------------------------------------------------===// +// ADX + +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_addcarryx_u32: GCCBuiltin<"__builtin_ia32_addcarryx_u32">, + Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; + def int_x86_addcarryx_u64: GCCBuiltin<"__builtin_ia32_addcarryx_u64">, + Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; + def int_x86_addcarry_u32: GCCBuiltin<"__builtin_ia32_addcarry_u32">, + Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; + def int_x86_addcarry_u64: GCCBuiltin<"__builtin_ia32_addcarry_u64">, + Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; + def int_x86_subborrow_u32: GCCBuiltin<"__builtin_ia32_subborrow_u32">, + Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; + def int_x86_subborrow_u64: GCCBuiltin<"__builtin_ia32_subborrow_u64">, + Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; +} + //===----------------------------------------------------------------------===// // RTM intrinsics. Transactional Memory support. @@ -2955,10 +3193,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx512_sqrt_pd_512 : GCCBuiltin<"__builtin_ia32_sqrtpd512">, - Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty], [IntrNoMem]>; - def int_x86_avx512_sqrt_ps_512 : GCCBuiltin<"__builtin_ia32_sqrtps512">, - Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty], [IntrNoMem]>; + def int_x86_avx512_sqrt_pd_512 : GCCBuiltin<"__builtin_ia32_sqrtpd512_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_sqrt_ps_512 : GCCBuiltin<"__builtin_ia32_sqrtps512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, @@ -2993,6 +3233,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_rcp28_pd : GCCBuiltin<"__builtin_ia32_rcp28pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_exp2_ps : GCCBuiltin<"__builtin_ia32_exp2ps_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_exp2_pd : GCCBuiltin<"__builtin_ia32_exp2pd_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], @@ -3167,10 +3414,26 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_blend_ps_256 : GCCBuiltin<"__builtin_ia32_blendmps_256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_ps_128 : GCCBuiltin<"__builtin_ia32_blendmps_128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; def int_x86_avx512_mask_blend_pd_512 : GCCBuiltin<"__builtin_ia32_blendmpd_512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_blend_pd_256 : GCCBuiltin<"__builtin_ia32_blendmpd_256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_pd_128 : GCCBuiltin<"__builtin_ia32_blendmpd_128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; def int_x86_avx512_mask_blend_d_512 : GCCBuiltin<"__builtin_ia32_blendmd_512_mask">, Intrinsic<[llvm_v16i32_ty], @@ -3180,8 +3443,428 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_blend_d_256 : GCCBuiltin<"__builtin_ia32_blendmd_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_q_256 : GCCBuiltin<"__builtin_ia32_blendmq_256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_d_128 : GCCBuiltin<"__builtin_ia32_blendmd_128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_q_128 : GCCBuiltin<"__builtin_ia32_blendmq_128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_blend_w_512 : GCCBuiltin<"__builtin_ia32_blendmw_512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_w_256 : GCCBuiltin<"__builtin_ia32_blendmw_256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_w_128 : GCCBuiltin<"__builtin_ia32_blendmw_128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_b_512 : GCCBuiltin<"__builtin_ia32_blendmb_512_mask">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_b_256 : GCCBuiltin<"__builtin_ia32_blendmb_256_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_blend_b_128 : GCCBuiltin<"__builtin_ia32_blendmb_128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + } +let TargetPrefix = "x86" in { + def int_x86_avx512_mask_valign_q_512 : GCCBuiltin<"__builtin_ia32_alignq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_valign_d_512 : GCCBuiltin<"__builtin_ia32_alignd512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; +} + +// Compares +let TargetPrefix = "x86" in { + // 512-bit + def int_x86_avx512_mask_pcmpeq_b_512 : GCCBuiltin<"__builtin_ia32_pcmpeqb512_mask">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_w_512 : GCCBuiltin<"__builtin_ia32_pcmpeqw512_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_d_512 : GCCBuiltin<"__builtin_ia32_pcmpeqd512_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_q_512 : GCCBuiltin<"__builtin_ia32_pcmpeqq512_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pcmpgt_b_512: GCCBuiltin<"__builtin_ia32_pcmpgtb512_mask">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_w_512: GCCBuiltin<"__builtin_ia32_pcmpgtw512_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_d_512: GCCBuiltin<"__builtin_ia32_pcmpgtd512_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_q_512: GCCBuiltin<"__builtin_ia32_pcmpgtq512_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cmp_b_512: GCCBuiltin<"__builtin_ia32_cmpb512_mask">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, + llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_w_512: GCCBuiltin<"__builtin_ia32_cmpw512_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_d_512: GCCBuiltin<"__builtin_ia32_cmpd512_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem ]>; + def int_x86_avx512_mask_cmp_q_512: GCCBuiltin<"__builtin_ia32_cmpq512_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_ucmp_b_512: GCCBuiltin<"__builtin_ia32_ucmpb512_mask">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, + llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_w_512: GCCBuiltin<"__builtin_ia32_ucmpw512_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_d_512: GCCBuiltin<"__builtin_ia32_ucmpd512_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_q_512: GCCBuiltin<"__builtin_ia32_ucmpq512_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + // 256-bit + def int_x86_avx512_mask_pcmpeq_b_256 : GCCBuiltin<"__builtin_ia32_pcmpeqb256_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_w_256 : GCCBuiltin<"__builtin_ia32_pcmpeqw256_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_d_256 : GCCBuiltin<"__builtin_ia32_pcmpeqd256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_q_256 : GCCBuiltin<"__builtin_ia32_pcmpeqq256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pcmpgt_b_256: GCCBuiltin<"__builtin_ia32_pcmpgtb256_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_w_256: GCCBuiltin<"__builtin_ia32_pcmpgtw256_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_d_256: GCCBuiltin<"__builtin_ia32_pcmpgtd256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_q_256: GCCBuiltin<"__builtin_ia32_pcmpgtq256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cmp_b_256: GCCBuiltin<"__builtin_ia32_cmpb256_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_w_256: GCCBuiltin<"__builtin_ia32_cmpw256_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_d_256: GCCBuiltin<"__builtin_ia32_cmpd256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_q_256: GCCBuiltin<"__builtin_ia32_cmpq256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_ucmp_b_256: GCCBuiltin<"__builtin_ia32_ucmpb256_mask">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_w_256: GCCBuiltin<"__builtin_ia32_ucmpw256_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_d_256: GCCBuiltin<"__builtin_ia32_ucmpd256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_q_256: GCCBuiltin<"__builtin_ia32_ucmpq256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + // 128-bit + def int_x86_avx512_mask_pcmpeq_b_128 : GCCBuiltin<"__builtin_ia32_pcmpeqb128_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_w_128 : GCCBuiltin<"__builtin_ia32_pcmpeqw128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_d_128 : GCCBuiltin<"__builtin_ia32_pcmpeqd128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpeq_q_128 : GCCBuiltin<"__builtin_ia32_pcmpeqq128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pcmpgt_b_128: GCCBuiltin<"__builtin_ia32_pcmpgtb128_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_w_128: GCCBuiltin<"__builtin_ia32_pcmpgtw128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_d_128: GCCBuiltin<"__builtin_ia32_pcmpgtd128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pcmpgt_q_128: GCCBuiltin<"__builtin_ia32_pcmpgtq128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cmp_b_128: GCCBuiltin<"__builtin_ia32_cmpb128_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_w_128: GCCBuiltin<"__builtin_ia32_cmpw128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_d_128: GCCBuiltin<"__builtin_ia32_cmpd128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_q_128: GCCBuiltin<"__builtin_ia32_cmpq128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_ucmp_b_128: GCCBuiltin<"__builtin_ia32_ucmpb128_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_w_128: GCCBuiltin<"__builtin_ia32_ucmpw128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_d_128: GCCBuiltin<"__builtin_ia32_ucmpd128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_ucmp_q_128: GCCBuiltin<"__builtin_ia32_ucmpq128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; +} + +// Compress, Expand +let TargetPrefix = "x86" in { + def int_x86_avx512_mask_compress_ps_512 : + GCCBuiltin<"__builtin_ia32_compresssf512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_pd_512 : + GCCBuiltin<"__builtin_ia32_compressdf512_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_ps_256 : + GCCBuiltin<"__builtin_ia32_compresssf256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_pd_256 : + GCCBuiltin<"__builtin_ia32_compressdf256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_ps_128 : + GCCBuiltin<"__builtin_ia32_compresssf128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_pd_128 : + GCCBuiltin<"__builtin_ia32_compressdf128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_compress_store_ps_512 : + GCCBuiltin<"__builtin_ia32_compressstoresf512_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v16f32_ty, + llvm_i16_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_pd_512 : + GCCBuiltin<"__builtin_ia32_compressstoredf512_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v8f64_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_ps_256 : + GCCBuiltin<"__builtin_ia32_compressstoresf256_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_pd_256 : + GCCBuiltin<"__builtin_ia32_compressstoredf256_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v4f64_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_ps_128 : + GCCBuiltin<"__builtin_ia32_compressstoresf128_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_pd_128 : + GCCBuiltin<"__builtin_ia32_compressstoredf128_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + + def int_x86_avx512_mask_compress_d_512 : + GCCBuiltin<"__builtin_ia32_compresssi512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_q_512 : + GCCBuiltin<"__builtin_ia32_compressdi512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_d_256 : + GCCBuiltin<"__builtin_ia32_compresssi256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_q_256 : + GCCBuiltin<"__builtin_ia32_compressdi256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_d_128 : + GCCBuiltin<"__builtin_ia32_compresssi128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_q_128 : + GCCBuiltin<"__builtin_ia32_compressdi128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_compress_store_d_512 : + GCCBuiltin<"__builtin_ia32_compressstoresi512_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_q_512 : + GCCBuiltin<"__builtin_ia32_compressstoredi512_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_d_256 : + GCCBuiltin<"__builtin_ia32_compressstoresi256_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_q_256 : + GCCBuiltin<"__builtin_ia32_compressstoredi256_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_d_128 : + GCCBuiltin<"__builtin_ia32_compressstoresi128_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_compress_store_q_128 : + GCCBuiltin<"__builtin_ia32_compressstoredi128_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrReadWriteArgMem]>; + +// expand + def int_x86_avx512_mask_expand_ps_512 : + GCCBuiltin<"__builtin_ia32_expandsf512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_pd_512 : + GCCBuiltin<"__builtin_ia32_expanddf512_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_ps_256 : + GCCBuiltin<"__builtin_ia32_expandsf256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_pd_256 : + GCCBuiltin<"__builtin_ia32_expanddf256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_ps_128 : + GCCBuiltin<"__builtin_ia32_expandsf128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_pd_128 : + GCCBuiltin<"__builtin_ia32_expanddf128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_expand_load_ps_512 : + GCCBuiltin<"__builtin_ia32_expandloadsf512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty, llvm_v16f32_ty, + llvm_i16_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_pd_512 : + GCCBuiltin<"__builtin_ia32_expandloaddf512_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty, llvm_v8f64_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_ps_256 : + GCCBuiltin<"__builtin_ia32_expandloadsf256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_pd_256 : + GCCBuiltin<"__builtin_ia32_expandloaddf256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4f64_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_ps_128 : + GCCBuiltin<"__builtin_ia32_expandloadsf128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_pd_128 : + GCCBuiltin<"__builtin_ia32_expandloaddf128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrReadArgMem]>; + + def int_x86_avx512_mask_expand_d_512 : + GCCBuiltin<"__builtin_ia32_expandsi512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_q_512 : + GCCBuiltin<"__builtin_ia32_expanddi512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_d_256 : + GCCBuiltin<"__builtin_ia32_expandsi256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_q_256 : + GCCBuiltin<"__builtin_ia32_expanddi256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_d_128 : + GCCBuiltin<"__builtin_ia32_expandsi128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_q_128 : + GCCBuiltin<"__builtin_ia32_expanddi128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_expand_load_d_512 : + GCCBuiltin<"__builtin_ia32_expandloadsi512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_ptr_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_q_512 : + GCCBuiltin<"__builtin_ia32_expandloaddi512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_ptr_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_d_256 : + GCCBuiltin<"__builtin_ia32_expandloadsi256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_q_256 : + GCCBuiltin<"__builtin_ia32_expandloaddi256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_d_128 : + GCCBuiltin<"__builtin_ia32_expandloadsi128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_expand_load_q_128 : + GCCBuiltin<"__builtin_ia32_expandloaddi128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrReadArgMem]>; + +} // Misc. let TargetPrefix = "x86" in { def int_x86_avx512_mask_cmp_ps_512 : GCCBuiltin<"__builtin_ia32_cmpps512_mask">, @@ -3190,13 +3873,6 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_cmp_pd_512 : GCCBuiltin<"__builtin_ia32_cmppd512_mask">, Intrinsic<[llvm_i8_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_pcmpeq_d_512 : GCCBuiltin<"__builtin_ia32_pcmpeqd512_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_q_512 : GCCBuiltin<"__builtin_ia32_pcmpeqq512_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; def int_x86_avx512_mask_pand_d_512 : GCCBuiltin<"__builtin_ia32_pandd512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], @@ -3205,6 +3881,8 @@ let TargetPrefix = "x86" in { Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_movntdqa : GCCBuiltin<"__builtin_ia32_movntdqa512">, + Intrinsic<[llvm_v8i64_ty], [llvm_ptr_ty], [IntrReadMem]>; } //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/include/llvm/IR/LLVMContext.h b/contrib/llvm/include/llvm/IR/LLVMContext.h index 4d940d599b9a..2f18782a0730 100644 --- a/contrib/llvm/include/llvm/IR/LLVMContext.h +++ b/contrib/llvm/include/llvm/IR/LLVMContext.h @@ -18,6 +18,7 @@ #include "llvm-c/Core.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Options.h" namespace llvm { @@ -52,7 +53,12 @@ public: MD_fpmath = 3, // "fpmath" MD_range = 4, // "range" MD_tbaa_struct = 5, // "tbaa.struct" - MD_invariant_load = 6 // "invariant.load" + MD_invariant_load = 6, // "invariant.load" + MD_alias_scope = 7, // "alias.scope" + MD_noalias = 8, // "noalias", + MD_nontemporal = 9, // "nontemporal" + MD_mem_parallel_loop_access = 10, // "llvm.mem.parallel_loop_access" + MD_nonnull = 11 // "nonnull" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. @@ -97,12 +103,14 @@ public: /// setDiagnosticHandler - This method sets a handler that is invoked /// when the backend needs to report anything to the user. The first /// argument is a function pointer and the second is a context pointer that - /// gets passed into the DiagHandler. + /// gets passed into the DiagHandler. The third argument should be set to + /// true if the handler only expects enabled diagnostics. /// /// LLVMContext doesn't take ownership or interpret either of these /// pointers. void setDiagnosticHandler(DiagnosticHandlerTy DiagHandler, - void *DiagContext = nullptr); + void *DiagContext = nullptr, + bool RespectFilters = false); /// getDiagnosticHandler - Return the diagnostic handler set by /// setDiagnosticHandler. @@ -112,14 +120,16 @@ public: /// setDiagnosticContext. void *getDiagnosticContext() const; - /// diagnose - Report a message to the currently installed diagnostic handler. + /// \brief Report a message to the currently installed diagnostic handler. + /// /// This function returns, in particular in the case of error reporting - /// (DI.Severity == RS_Error), so the caller should leave the compilation + /// (DI.Severity == \a DS_Error), so the caller should leave the compilation /// process in a self-consistent state, even though the generated code /// need not be correct. - /// The diagnostic message will be implicitly prefixed with a severity - /// keyword according to \p DI.getSeverity(), i.e., "error: " - /// for RS_Error, "warning: " for RS_Warning, and "note: " for RS_Note. + /// + /// The diagnostic message will be implicitly prefixed with a severity keyword + /// according to \p DI.getSeverity(), i.e., "error: " for \a DS_Error, + /// "warning: " for \a DS_Warning, and "note: " for \a DS_Note. void diagnose(const DiagnosticInfo &DI); /// \brief Registers a yield callback with the given context. @@ -157,6 +167,14 @@ public: void emitError(const Instruction *I, const Twine &ErrorStr); void emitError(const Twine &ErrorStr); + /// \brief Query for a debug option's value. + /// + /// This function returns typed data populated from command line parsing. + template + ValT getOption() const { + return OptionRegistry::instance().template get(); + } + private: LLVMContext(LLVMContext&) LLVM_DELETED_FUNCTION; void operator=(LLVMContext&) LLVM_DELETED_FUNCTION; diff --git a/contrib/llvm/include/llvm/IR/LeakDetector.h b/contrib/llvm/include/llvm/IR/LeakDetector.h deleted file mode 100644 index cb18df875867..000000000000 --- a/contrib/llvm/include/llvm/IR/LeakDetector.h +++ /dev/null @@ -1,92 +0,0 @@ -//===- LeakDetector.h - Provide leak detection ------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines a class that can be used to provide very simple memory leak -// checks for an API. Basically LLVM uses this to make sure that Instructions, -// for example, are deleted when they are supposed to be, and not leaked away. -// -// When compiling with NDEBUG (Release build), this class does nothing, thus -// adding no checking overhead to release builds. Note that this class is -// implemented in a very simple way, requiring completely manual manipulation -// and checking for garbage, but this is intentional: users should not be using -// this API, only other APIs should. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_IR_LEAKDETECTOR_H -#define LLVM_IR_LEAKDETECTOR_H - -#include - -namespace llvm { - -class LLVMContext; -class Value; - -struct LeakDetector { - /// addGarbageObject - Add a pointer to the internal set of "garbage" object - /// pointers. This should be called when objects are created, or if they are - /// taken out of an owning collection. - /// - static void addGarbageObject(void *Object) { -#ifndef NDEBUG - addGarbageObjectImpl(Object); -#endif - } - - /// removeGarbageObject - Remove a pointer from our internal representation of - /// our "garbage" objects. This should be called when an object is added to - /// an "owning" collection. - /// - static void removeGarbageObject(void *Object) { -#ifndef NDEBUG - removeGarbageObjectImpl(Object); -#endif - } - - /// checkForGarbage - Traverse the internal representation of garbage - /// pointers. If there are any pointers that have been add'ed, but not - /// remove'd, big obnoxious warnings about memory leaks are issued. - /// - /// The specified message will be printed indicating when the check was - /// performed. - /// - static void checkForGarbage(LLVMContext &C, const std::string &Message) { -#ifndef NDEBUG - checkForGarbageImpl(C, Message); -#endif - } - - /// Overload the normal methods to work better with Value*'s because they are - /// by far the most common in LLVM. This does not affect the actual - /// functioning of this class, it just makes the warning messages nicer. - /// - static void addGarbageObject(const Value *Object) { -#ifndef NDEBUG - addGarbageObjectImpl(Object); -#endif - } - static void removeGarbageObject(const Value *Object) { -#ifndef NDEBUG - removeGarbageObjectImpl(Object); -#endif - } - -private: - // If we are debugging, the actual implementations will be called... - static void addGarbageObjectImpl(const Value *Object); - static void removeGarbageObjectImpl(const Value *Object); - static void addGarbageObjectImpl(void *Object); - static void removeGarbageObjectImpl(void *Object); - static void checkForGarbageImpl(LLVMContext &C, const std::string &Message); -}; - -} // End llvm namespace - -#endif diff --git a/contrib/llvm/include/llvm/IR/LegacyPassManager.h b/contrib/llvm/include/llvm/IR/LegacyPassManager.h index c967a6bca8b2..6c04e9dc3d3f 100644 --- a/contrib/llvm/include/llvm/IR/LegacyPassManager.h +++ b/contrib/llvm/include/llvm/IR/LegacyPassManager.h @@ -37,9 +37,10 @@ class PassManagerBase { public: virtual ~PassManagerBase(); - /// add - Add a pass to the queue of passes to run. This passes ownership of + /// Add a pass to the queue of passes to run. This passes ownership of /// the Pass to the PassManager. When the PassManager is destroyed, the pass /// will be destroyed as well, so there is no need to delete the pass. This + /// may even destroy the pass right away if it is found to be redundant. This /// implies that all passes MUST be allocated with 'new'. virtual void add(Pass *P) = 0; }; @@ -51,10 +52,6 @@ public: PassManager(); ~PassManager(); - /// add - Add a pass to the queue of passes to run. This passes ownership of - /// the Pass to the PassManager. When the PassManager is destroyed, the pass - /// will be destroyed as well, so there is no need to delete the pass. This - /// implies that all passes MUST be allocated with 'new'. void add(Pass *P) override; /// run - Execute all of the passes scheduled for execution. Keep track of @@ -75,11 +72,6 @@ public: explicit FunctionPassManager(Module *M); ~FunctionPassManager(); - /// add - Add a pass to the queue of passes to run. This passes - /// ownership of the Pass to the PassManager. When the - /// PassManager_X is destroyed, the pass will be destroyed as well, so - /// there is no need to delete the pass. - /// This implies that all passes MUST be allocated with 'new'. void add(Pass *P) override; /// run - Execute all of the passes scheduled for execution. Keep diff --git a/contrib/llvm/include/llvm/IR/LegacyPassManagers.h b/contrib/llvm/include/llvm/IR/LegacyPassManagers.h index f6065a4e21a6..7f7889ad5fb3 100644 --- a/contrib/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/contrib/llvm/include/llvm/IR/LegacyPassManagers.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_PASSMANAGERS_H -#define LLVM_PASSMANAGERS_H +#ifndef LLVM_IR_LEGACYPASSMANAGERS_H +#define LLVM_IR_LEGACYPASSMANAGERS_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -61,7 +61,7 @@ // // [o] class FunctionPassManager; // -// This is a external interface used by JIT to manage FunctionPasses. This +// This is a external interface used to manage FunctionPasses. This // interface relies on FunctionPassManagerImpl to do all the tasks. // // [o] class FunctionPassManagerImpl : public ModulePass, PMDataManager, @@ -195,6 +195,9 @@ public: /// then return NULL. Pass *findAnalysisPass(AnalysisID AID); + /// Retrieve the PassInfo for an analysis. + const PassInfo *findAnalysisPassInfo(AnalysisID AID) const; + /// Find analysis usage information for the pass P. AnalysisUsage *findAnalysisUsage(Pass *P); @@ -248,9 +251,15 @@ private: DenseMap > InversedLastUser; /// Immutable passes are managed by top level manager. - SmallVector ImmutablePasses; + SmallVector ImmutablePasses; DenseMap AnUsageMap; + + /// Collection of PassInfo objects found via analysis IDs and in this top + /// level manager. This is used to memoize queries to the pass registry. + /// FIXME: This is an egregious hack because querying the pass registry is + /// either slow or racy. + mutable DenseMap AnalysisPassInfos; }; @@ -393,7 +402,7 @@ private: // Collection of higher level analysis used by the pass managed by // this manager. - SmallVector HigherLevelAnalysis; + SmallVector HigherLevelAnalysis; unsigned Depth; }; diff --git a/contrib/llvm/include/llvm/IR/MDBuilder.h b/contrib/llvm/include/llvm/IR/MDBuilder.h index 37d263bf52a8..91a6685f6125 100644 --- a/contrib/llvm/include/llvm/IR/MDBuilder.h +++ b/contrib/llvm/include/llvm/IR/MDBuilder.h @@ -15,6 +15,7 @@ #ifndef LLVM_IR_MDBUILDER_H #define LLVM_IR_MDBUILDER_H +#include "llvm/ADT/StringRef.h" #include "llvm/Support/DataTypes.h" #include @@ -23,9 +24,10 @@ namespace llvm { class APInt; template class ArrayRef; class LLVMContext; +class Constant; +class ConstantAsMetadata; class MDNode; class MDString; -class StringRef; class MDBuilder { LLVMContext &Context; @@ -36,6 +38,9 @@ public: /// \brief Return the given string as metadata. MDString *createString(StringRef Str); + /// \brief Return the given constant as metadata. + ConstantAsMetadata *createConstant(Constant *C); + //===------------------------------------------------------------------===// // FPMath metadata. //===------------------------------------------------------------------===// @@ -63,19 +68,54 @@ public: MDNode *createRange(const APInt &Lo, const APInt &Hi); //===------------------------------------------------------------------===// - // TBAA metadata. + // AA metadata. //===------------------------------------------------------------------===// - /// \brief Return metadata appropriate for a TBAA root node. Each returned +protected: + /// \brief Return metadata appropriate for a AA root node (scope or TBAA). + /// Each returned node is distinct from all other metadata and will never + /// be identified (uniqued) with anything else. + MDNode *createAnonymousAARoot(StringRef Name = StringRef(), + MDNode *Extra = nullptr); + +public: + /// \brief Return metadata appropriate for a TBAA root node. Each returned /// node is distinct from all other metadata and will never be identified /// (uniqued) with anything else. - MDNode *createAnonymousTBAARoot(); + MDNode *createAnonymousTBAARoot() { + return createAnonymousAARoot(); + } + + /// \brief Return metadata appropriate for an alias scope domain node. + /// Each returned node is distinct from all other metadata and will never + /// be identified (uniqued) with anything else. + MDNode *createAnonymousAliasScopeDomain(StringRef Name = StringRef()) { + return createAnonymousAARoot(Name); + } + + /// \brief Return metadata appropriate for an alias scope root node. + /// Each returned node is distinct from all other metadata and will never + /// be identified (uniqued) with anything else. + MDNode *createAnonymousAliasScope(MDNode *Domain, + StringRef Name = StringRef()) { + return createAnonymousAARoot(Name, Domain); + } /// \brief Return metadata appropriate for a TBAA root node with the given /// name. This may be identified (uniqued) with other roots with the same /// name. MDNode *createTBAARoot(StringRef Name); + /// \brief Return metadata appropriate for an alias scope domain node with + /// the given name. This may be identified (uniqued) with other roots with + /// the same name. + MDNode *createAliasScopeDomain(StringRef Name); + + /// \brief Return metadata appropriate for an alias scope node with + /// the given name. This may be identified (uniqued) with other scopes with + /// the same name and domain. + MDNode *createAliasScope(StringRef Name, MDNode *Domain); + /// \brief Return metadata for a non-root TBAA node with the given name, /// parent in the TBAA tree, and value for 'pointsToConstantMemory'. MDNode *createTBAANode(StringRef Name, MDNode *Parent, diff --git a/contrib/llvm/include/llvm/IR/Mangler.h b/contrib/llvm/include/llvm/IR/Mangler.h index c1ba5858a612..1e6b5b1dca00 100644 --- a/contrib/llvm/include/llvm/IR/Mangler.h +++ b/contrib/llvm/include/llvm/IR/Mangler.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_MANGLER_H -#define LLVM_TARGET_MANGLER_H +#ifndef LLVM_IR_MANGLER_H +#define LLVM_IR_MANGLER_H #include "llvm/ADT/DenseMap.h" #include "llvm/Support/raw_ostream.h" @@ -66,4 +66,4 @@ public: } // End llvm namespace -#endif // LLVM_TARGET_MANGLER_H +#endif diff --git a/contrib/llvm/include/llvm/IR/Metadata.def b/contrib/llvm/include/llvm/IR/Metadata.def new file mode 100644 index 000000000000..2098bb57eb56 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/Metadata.def @@ -0,0 +1,59 @@ +//===- llvm/Metadata.def - Metadata definitions -----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Macros for running through all types of metadata. +// +//===----------------------------------------------------------------------===// + +#if !(defined HANDLE_METADATA || defined HANDLE_METADATA_LEAF || \ + defined HANDLE_METADATA_BRANCH || defined HANDLE_UNIQUABLE_LEAF || \ + defined HANDLE_UNIQUABLE_BRANCH) +#error "Missing macro definition of HANDLE_METADATA*" +#endif + +// Handler for all types of metadata. +#ifndef HANDLE_METADATA +#define HANDLE_METADATA(CLASS) +#endif + +// Handler for leaf nodes in the class hierarchy. +#ifndef HANDLE_METADATA_LEAF +#define HANDLE_METADATA_LEAF(CLASS) HANDLE_METADATA(CLASS) +#endif + +// Handler for non-leaf nodes in the class hierarchy. +#ifndef HANDLE_METADATA_BRANCH +#define HANDLE_METADATA_BRANCH(CLASS) HANDLE_METADATA(CLASS) +#endif + +// Handler for leaf nodes under UniquableMDNode. +#ifndef HANDLE_UNIQUABLE_LEAF +#define HANDLE_UNIQUABLE_LEAF(CLASS) HANDLE_METADATA_LEAF(CLASS) +#endif + +// Handler for non-leaf nodes under UniquableMDNode. +#ifndef HANDLE_UNIQUABLE_BRANCH +#define HANDLE_UNIQUABLE_BRANCH(CLASS) HANDLE_METADATA_BRANCH(CLASS) +#endif + +HANDLE_METADATA_LEAF(MDString) +HANDLE_METADATA_BRANCH(ValueAsMetadata) +HANDLE_METADATA_LEAF(ConstantAsMetadata) +HANDLE_METADATA_LEAF(LocalAsMetadata) +HANDLE_METADATA_BRANCH(MDNode) +HANDLE_METADATA_LEAF(MDNodeFwdDecl) +HANDLE_UNIQUABLE_BRANCH(UniquableMDNode) +HANDLE_UNIQUABLE_LEAF(MDTuple) +HANDLE_UNIQUABLE_LEAF(MDLocation) + +#undef HANDLE_METADATA +#undef HANDLE_METADATA_LEAF +#undef HANDLE_METADATA_BRANCH +#undef HANDLE_UNIQUABLE_LEAF +#undef HANDLE_UNIQUABLE_BRANCH diff --git a/contrib/llvm/include/llvm/IR/Metadata.h b/contrib/llvm/include/llvm/IR/Metadata.h index 7a0ca887201b..27a5d6f80041 100644 --- a/contrib/llvm/include/llvm/IR/Metadata.h +++ b/contrib/llvm/include/llvm/IR/Metadata.h @@ -17,10 +17,14 @@ #define LLVM_IR_METADATA_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/MetadataTracking.h" #include "llvm/IR/Value.h" +#include "llvm/Support/ErrorHandling.h" +#include namespace llvm { class LLVMContext; @@ -30,169 +34,888 @@ template enum LLVMConstants : uint32_t { - DEBUG_METADATA_VERSION = 1 // Current debug info version number. + DEBUG_METADATA_VERSION = 2 // Current debug info version number. }; +/// \brief Root of the metadata hierarchy. +/// +/// This is a root class for typeless data in the IR. +class Metadata { + friend class ReplaceableMetadataImpl; + + /// \brief RTTI. + const unsigned char SubclassID; + +protected: + /// \brief Storage flag for non-uniqued, otherwise unowned, metadata. + bool IsDistinctInContext : 1; + // TODO: expose remaining bits to subclasses. + + unsigned short SubclassData16; + unsigned SubclassData32; + +public: + enum MetadataKind { + MDTupleKind, + MDLocationKind, + MDNodeFwdDeclKind, + ConstantAsMetadataKind, + LocalAsMetadataKind, + MDStringKind + }; + +protected: + Metadata(unsigned ID) + : SubclassID(ID), IsDistinctInContext(false), SubclassData16(0), + SubclassData32(0) {} + ~Metadata() {} + + /// \brief Store this in a big non-uniqued untyped bucket. + bool isStoredDistinctInContext() const { return IsDistinctInContext; } + + /// \brief Default handling of a changed operand, which asserts. + /// + /// If subclasses pass themselves in as owners to a tracking node reference, + /// they must provide an implementation of this method. + void handleChangedOperand(void *, Metadata *) { + llvm_unreachable("Unimplemented in Metadata subclass"); + } + +public: + unsigned getMetadataID() const { return SubclassID; } + + /// \brief User-friendly dump. + void dump() const; + void print(raw_ostream &OS) const; + void printAsOperand(raw_ostream &OS, bool PrintType = true, + const Module *M = nullptr) const; +}; + +#define HANDLE_METADATA(CLASS) class CLASS; +#include "llvm/IR/Metadata.def" + +inline raw_ostream &operator<<(raw_ostream &OS, const Metadata &MD) { + MD.print(OS); + return OS; +} + +/// \brief Metadata wrapper in the Value hierarchy. +/// +/// A member of the \a Value hierarchy to represent a reference to metadata. +/// This allows, e.g., instrinsics to have metadata as operands. +/// +/// Notably, this is the only thing in either hierarchy that is allowed to +/// reference \a LocalAsMetadata. +class MetadataAsValue : public Value { + friend class ReplaceableMetadataImpl; + friend class LLVMContextImpl; + + Metadata *MD; + + MetadataAsValue(Type *Ty, Metadata *MD); + ~MetadataAsValue(); + + /// \brief Drop use of metadata (during teardown). + void dropUse() { MD = nullptr; } + +public: + static MetadataAsValue *get(LLVMContext &Context, Metadata *MD); + static MetadataAsValue *getIfExists(LLVMContext &Context, Metadata *MD); + Metadata *getMetadata() const { return MD; } + + static bool classof(const Value *V) { + return V->getValueID() == MetadataAsValueVal; + } + +private: + void handleChangedMetadata(Metadata *MD); + void track(); + void untrack(); +}; + +/// \brief Shared implementation of use-lists for replaceable metadata. +/// +/// Most metadata cannot be RAUW'ed. This is a shared implementation of +/// use-lists and associated API for the two that support it (\a ValueAsMetadata +/// and \a TempMDNode). +class ReplaceableMetadataImpl { + friend class MetadataTracking; + +public: + typedef MetadataTracking::OwnerTy OwnerTy; + +private: + uint64_t NextIndex; + SmallDenseMap, 4> UseMap; + +public: + ReplaceableMetadataImpl() : NextIndex(0) {} + ~ReplaceableMetadataImpl() { + assert(UseMap.empty() && "Cannot destroy in-use replaceable metadata"); + } + + /// \brief Replace all uses of this with MD. + /// + /// Replace all uses of this with \c MD, which is allowed to be null. + void replaceAllUsesWith(Metadata *MD); + + /// \brief Resolve all uses of this. + /// + /// Resolve all uses of this, turning off RAUW permanently. If \c + /// ResolveUsers, call \a UniquableMDNode::resolve() on any users whose last + /// operand is resolved. + void resolveAllUses(bool ResolveUsers = true); + +private: + void addRef(void *Ref, OwnerTy Owner); + void dropRef(void *Ref); + void moveRef(void *Ref, void *New, const Metadata &MD); + + static ReplaceableMetadataImpl *get(Metadata &MD); +}; + +/// \brief Value wrapper in the Metadata hierarchy. +/// +/// This is a custom value handle that allows other metadata to refer to +/// classes in the Value hierarchy. +/// +/// Because of full uniquing support, each value is only wrapped by a single \a +/// ValueAsMetadata object, so the lookup maps are far more efficient than +/// those using ValueHandleBase. +class ValueAsMetadata : public Metadata, ReplaceableMetadataImpl { + friend class ReplaceableMetadataImpl; + friend class LLVMContextImpl; + + Value *V; + + /// \brief Drop users without RAUW (during teardown). + void dropUsers() { + ReplaceableMetadataImpl::resolveAllUses(/* ResolveUsers */ false); + } + +protected: + ValueAsMetadata(unsigned ID, Value *V) + : Metadata(ID), V(V) { + assert(V && "Expected valid value"); + } + ~ValueAsMetadata() {} + +public: + static ValueAsMetadata *get(Value *V); + static ConstantAsMetadata *getConstant(Value *C) { + return cast(get(C)); + } + static LocalAsMetadata *getLocal(Value *Local) { + return cast(get(Local)); + } + + static ValueAsMetadata *getIfExists(Value *V); + static ConstantAsMetadata *getConstantIfExists(Value *C) { + return cast_or_null(getIfExists(C)); + } + static LocalAsMetadata *getLocalIfExists(Value *Local) { + return cast_or_null(getIfExists(Local)); + } + + Value *getValue() const { return V; } + Type *getType() const { return V->getType(); } + LLVMContext &getContext() const { return V->getContext(); } + + static void handleDeletion(Value *V); + static void handleRAUW(Value *From, Value *To); + +protected: + /// \brief Handle collisions after \a Value::replaceAllUsesWith(). + /// + /// RAUW isn't supported directly for \a ValueAsMetadata, but if the wrapped + /// \a Value gets RAUW'ed and the target already exists, this is used to + /// merge the two metadata nodes. + void replaceAllUsesWith(Metadata *MD) { + ReplaceableMetadataImpl::replaceAllUsesWith(MD); + } + +public: + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == LocalAsMetadataKind || + MD->getMetadataID() == ConstantAsMetadataKind; + } +}; + +class ConstantAsMetadata : public ValueAsMetadata { + friend class ValueAsMetadata; + + ConstantAsMetadata(Constant *C) + : ValueAsMetadata(ConstantAsMetadataKind, C) {} + +public: + static ConstantAsMetadata *get(Constant *C) { + return ValueAsMetadata::getConstant(C); + } + static ConstantAsMetadata *getIfExists(Constant *C) { + return ValueAsMetadata::getConstantIfExists(C); + } + + Constant *getValue() const { + return cast(ValueAsMetadata::getValue()); + } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == ConstantAsMetadataKind; + } +}; + +class LocalAsMetadata : public ValueAsMetadata { + friend class ValueAsMetadata; + + LocalAsMetadata(Value *Local) + : ValueAsMetadata(LocalAsMetadataKind, Local) { + assert(!isa(Local) && "Expected local value"); + } + +public: + static LocalAsMetadata *get(Value *Local) { + return ValueAsMetadata::getLocal(Local); + } + static LocalAsMetadata *getIfExists(Value *Local) { + return ValueAsMetadata::getLocalIfExists(Local); + } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == LocalAsMetadataKind; + } +}; + +/// \brief Transitional API for extracting constants from Metadata. +/// +/// This namespace contains transitional functions for metadata that points to +/// \a Constants. +/// +/// In prehistory -- when metadata was a subclass of \a Value -- \a MDNode +/// operands could refer to any \a Value. There's was a lot of code like this: +/// +/// \code +/// MDNode *N = ...; +/// auto *CI = dyn_cast(N->getOperand(2)); +/// \endcode +/// +/// Now that \a Value and \a Metadata are in separate hierarchies, maintaining +/// the semantics for \a isa(), \a cast(), \a dyn_cast() (etc.) requires three +/// steps: cast in the \a Metadata hierarchy, extraction of the \a Value, and +/// cast in the \a Value hierarchy. Besides creating boiler-plate, this +/// requires subtle control flow changes. +/// +/// The end-goal is to create a new type of metadata, called (e.g.) \a MDInt, +/// so that metadata can refer to numbers without traversing a bridge to the \a +/// Value hierarchy. In this final state, the code above would look like this: +/// +/// \code +/// MDNode *N = ...; +/// auto *MI = dyn_cast(N->getOperand(2)); +/// \endcode +/// +/// The API in this namespace supports the transition. \a MDInt doesn't exist +/// yet, and even once it does, changing each metadata schema to use it is its +/// own mini-project. In the meantime this API prevents us from introducing +/// complex and bug-prone control flow that will disappear in the end. In +/// particular, the above code looks like this: +/// +/// \code +/// MDNode *N = ...; +/// auto *CI = mdconst::dyn_extract(N->getOperand(2)); +/// \endcode +/// +/// The full set of provided functions includes: +/// +/// mdconst::hasa <=> isa +/// mdconst::extract <=> cast +/// mdconst::extract_or_null <=> cast_or_null +/// mdconst::dyn_extract <=> dyn_cast +/// mdconst::dyn_extract_or_null <=> dyn_cast_or_null +/// +/// The target of the cast must be a subclass of \a Constant. +namespace mdconst { + +namespace detail { +template T &make(); +template struct HasDereference { + typedef char Yes[1]; + typedef char No[2]; + template struct SFINAE {}; + + template + static Yes &hasDereference(SFINAE(*make()))> * = 0); + template static No &hasDereference(...); + + static const bool value = + sizeof(hasDereference(nullptr)) == sizeof(Yes); +}; +template struct IsValidPointer { + static const bool value = std::is_base_of::value && + HasDereference::value; +}; +template struct IsValidReference { + static const bool value = std::is_base_of::value && + std::is_convertible::value; +}; +} // end namespace detail + +/// \brief Check whether Metadata has a Value. +/// +/// As an analogue to \a isa(), check whether \c MD has an \a Value inside of +/// type \c X. +template +inline typename std::enable_if::value, bool>::type +hasa(Y &&MD) { + assert(MD && "Null pointer sent into hasa"); + if (auto *V = dyn_cast(MD)) + return isa(V->getValue()); + return false; +} +template +inline + typename std::enable_if::value, bool>::type + hasa(Y &MD) { + return hasa(&MD); +} + +/// \brief Extract a Value from Metadata. +/// +/// As an analogue to \a cast(), extract the \a Value subclass \c X from \c MD. +template +inline typename std::enable_if::value, X *>::type +extract(Y &&MD) { + return cast(cast(MD)->getValue()); +} +template +inline + typename std::enable_if::value, X *>::type + extract(Y &MD) { + return extract(&MD); +} + +/// \brief Extract a Value from Metadata, allowing null. +/// +/// As an analogue to \a cast_or_null(), extract the \a Value subclass \c X +/// from \c MD, allowing \c MD to be null. +template +inline typename std::enable_if::value, X *>::type +extract_or_null(Y &&MD) { + if (auto *V = cast_or_null(MD)) + return cast(V->getValue()); + return nullptr; +} + +/// \brief Extract a Value from Metadata, if any. +/// +/// As an analogue to \a dyn_cast_or_null(), extract the \a Value subclass \c X +/// from \c MD, return null if \c MD doesn't contain a \a Value or if the \a +/// Value it does contain is of the wrong subclass. +template +inline typename std::enable_if::value, X *>::type +dyn_extract(Y &&MD) { + if (auto *V = dyn_cast(MD)) + return dyn_cast(V->getValue()); + return nullptr; +} + +/// \brief Extract a Value from Metadata, if any, allowing null. +/// +/// As an analogue to \a dyn_cast_or_null(), extract the \a Value subclass \c X +/// from \c MD, return null if \c MD doesn't contain a \a Value or if the \a +/// Value it does contain is of the wrong subclass, allowing \c MD to be null. +template +inline typename std::enable_if::value, X *>::type +dyn_extract_or_null(Y &&MD) { + if (auto *V = dyn_cast_or_null(MD)) + return dyn_cast(V->getValue()); + return nullptr; +} + +} // end namespace mdconst + //===----------------------------------------------------------------------===// -/// MDString - a single uniqued string. +/// \brief A single uniqued string. +/// /// These are used to efficiently contain a byte sequence for metadata. /// MDString is always unnamed. -class MDString : public Value { - virtual void anchor(); - MDString(const MDString &) LLVM_DELETED_FUNCTION; +class MDString : public Metadata { + friend class StringMapEntry; + + MDString(const MDString &) LLVM_DELETED_FUNCTION; + MDString &operator=(MDString &&) LLVM_DELETED_FUNCTION; + MDString &operator=(const MDString &) LLVM_DELETED_FUNCTION; + + StringMapEntry *Entry; + MDString() : Metadata(MDStringKind), Entry(nullptr) {} + MDString(MDString &&) : Metadata(MDStringKind) {} - explicit MDString(LLVMContext &C); public: static MDString *get(LLVMContext &Context, StringRef Str); static MDString *get(LLVMContext &Context, const char *Str) { return get(Context, Str ? StringRef(Str) : StringRef()); } - StringRef getString() const { return getName(); } + StringRef getString() const; - unsigned getLength() const { return (unsigned)getName().size(); } + unsigned getLength() const { return (unsigned)getString().size(); } typedef StringRef::iterator iterator; - /// begin() - Pointer to the first byte of the string. - iterator begin() const { return getName().begin(); } + /// \brief Pointer to the first byte of the string. + iterator begin() const { return getString().begin(); } - /// end() - Pointer to one byte past the end of the string. - iterator end() const { return getName().end(); } + /// \brief Pointer to one byte past the end of the string. + iterator end() const { return getString().end(); } - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const Value *V) { - return V->getValueID() == MDStringVal; + const unsigned char *bytes_begin() const { return getString().bytes_begin(); } + const unsigned char *bytes_end() const { return getString().bytes_end(); } + + /// \brief Methods for support type inquiry through isa, cast, and dyn_cast. + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == MDStringKind; } }; +/// \brief A collection of metadata nodes that might be associated with a +/// memory access used by the alias-analysis infrastructure. +struct AAMDNodes { + explicit AAMDNodes(MDNode *T = nullptr, MDNode *S = nullptr, + MDNode *N = nullptr) + : TBAA(T), Scope(S), NoAlias(N) {} -class MDNodeOperand; + bool operator==(const AAMDNodes &A) const { + return TBAA == A.TBAA && Scope == A.Scope && NoAlias == A.NoAlias; + } + + bool operator!=(const AAMDNodes &A) const { return !(*this == A); } + + LLVM_EXPLICIT operator bool() const { return TBAA || Scope || NoAlias; } + + /// \brief The tag for type-based alias analysis. + MDNode *TBAA; + + /// \brief The tag for alias scope specification (used with noalias). + MDNode *Scope; + + /// \brief The tag specifying the noalias scope. + MDNode *NoAlias; +}; + +// Specialize DenseMapInfo for AAMDNodes. +template<> +struct DenseMapInfo { + static inline AAMDNodes getEmptyKey() { + return AAMDNodes(DenseMapInfo::getEmptyKey(), 0, 0); + } + static inline AAMDNodes getTombstoneKey() { + return AAMDNodes(DenseMapInfo::getTombstoneKey(), 0, 0); + } + static unsigned getHashValue(const AAMDNodes &Val) { + return DenseMapInfo::getHashValue(Val.TBAA) ^ + DenseMapInfo::getHashValue(Val.Scope) ^ + DenseMapInfo::getHashValue(Val.NoAlias); + } + static bool isEqual(const AAMDNodes &LHS, const AAMDNodes &RHS) { + return LHS == RHS; + } +}; + +/// \brief Tracking metadata reference owned by Metadata. +/// +/// Similar to \a TrackingMDRef, but it's expected to be owned by an instance +/// of \a Metadata, which has the option of registering itself for callbacks to +/// re-unique itself. +/// +/// In particular, this is used by \a MDNode. +class MDOperand { + MDOperand(MDOperand &&) LLVM_DELETED_FUNCTION; + MDOperand(const MDOperand &) LLVM_DELETED_FUNCTION; + MDOperand &operator=(MDOperand &&) LLVM_DELETED_FUNCTION; + MDOperand &operator=(const MDOperand &) LLVM_DELETED_FUNCTION; + + Metadata *MD; + +public: + MDOperand() : MD(nullptr) {} + ~MDOperand() { untrack(); } + + Metadata *get() const { return MD; } + operator Metadata *() const { return get(); } + Metadata *operator->() const { return get(); } + Metadata &operator*() const { return *get(); } + + void reset() { + untrack(); + MD = nullptr; + } + void reset(Metadata *MD, Metadata *Owner) { + untrack(); + this->MD = MD; + track(Owner); + } + +private: + void track(Metadata *Owner) { + if (MD) { + if (Owner) + MetadataTracking::track(this, *MD, *Owner); + else + MetadataTracking::track(MD); + } + } + void untrack() { + assert(static_cast(this) == &MD && "Expected same address"); + if (MD) + MetadataTracking::untrack(MD); + } +}; + +template <> struct simplify_type { + typedef Metadata *SimpleType; + static SimpleType getSimplifiedValue(MDOperand &MD) { return MD.get(); } +}; + +template <> struct simplify_type { + typedef Metadata *SimpleType; + static SimpleType getSimplifiedValue(const MDOperand &MD) { return MD.get(); } +}; //===----------------------------------------------------------------------===// -/// MDNode - a tuple of other values. -class MDNode : public Value, public FoldingSetNode { +/// \brief Tuple of metadata. +class MDNode : public Metadata { MDNode(const MDNode &) LLVM_DELETED_FUNCTION; void operator=(const MDNode &) LLVM_DELETED_FUNCTION; - friend class MDNodeOperand; - friend class LLVMContextImpl; - friend struct FoldingSetTrait; + void *operator new(size_t) LLVM_DELETED_FUNCTION; - /// Hash - If the MDNode is uniqued cache the hash to speed up lookup. - unsigned Hash; - - /// NumOperands - This many 'MDNodeOperand' items are co-allocated onto the - /// end of this MDNode. + LLVMContext &Context; unsigned NumOperands; - // Subclass data enums. - enum { - /// FunctionLocalBit - This bit is set if this MDNode is function local. - /// This is true when it (potentially transitively) contains a reference to - /// something in a function, like an argument, basicblock, or instruction. - FunctionLocalBit = 1 << 0, +protected: + unsigned MDNodeSubclassData; - /// NotUniquedBit - This is set on MDNodes that are not uniqued because they - /// have a null operand. - NotUniquedBit = 1 << 1, + void *operator new(size_t Size, unsigned NumOps); + void operator delete(void *Mem); - /// DestroyFlag - This bit is set by destroy() so the destructor can assert - /// that the node isn't being destroyed with a plain 'delete'. - DestroyFlag = 1 << 2 - }; + /// \brief Required by std, but never called. + void operator delete(void *, unsigned) { + llvm_unreachable("Constructor throws?"); + } - // FunctionLocal enums. - enum FunctionLocalness { - FL_Unknown = -1, - FL_No = 0, - FL_Yes = 1 - }; + /// \brief Required by std, but never called. + void operator delete(void *, unsigned, bool) { + llvm_unreachable("Constructor throws?"); + } - /// replaceOperand - Replace each instance of F from the operand list of this - /// node with T. - void replaceOperand(MDNodeOperand *Op, Value *NewVal); - ~MDNode(); + MDNode(LLVMContext &Context, unsigned ID, ArrayRef MDs); + ~MDNode() {} - MDNode(LLVMContext &C, ArrayRef Vals, bool isFunctionLocal); + void dropAllReferences(); + + MDOperand *mutable_begin() { return mutable_end() - NumOperands; } + MDOperand *mutable_end() { return reinterpret_cast(this); } - static MDNode *getMDNode(LLVMContext &C, ArrayRef Vals, - FunctionLocalness FL, bool Insert = true); public: - // Constructors and destructors. - static MDNode *get(LLVMContext &Context, ArrayRef Vals); - // getWhenValsUnresolved - Construct MDNode determining function-localness - // from isFunctionLocal argument, not by analyzing Vals. - static MDNode *getWhenValsUnresolved(LLVMContext &Context, - ArrayRef Vals, - bool isFunctionLocal); + static inline MDNode *get(LLVMContext &Context, ArrayRef MDs); + static inline MDNode *getIfExists(LLVMContext &Context, + ArrayRef MDs); + static inline MDNode *getDistinct(LLVMContext &Context, + ArrayRef MDs); - static MDNode *getIfExists(LLVMContext &Context, ArrayRef Vals); + /// \brief Return a temporary MDNode + /// + /// For use in constructing cyclic MDNode structures. A temporary MDNode is + /// not uniqued, may be RAUW'd, and must be manually deleted with + /// deleteTemporary. + static MDNodeFwdDecl *getTemporary(LLVMContext &Context, + ArrayRef MDs); - /// getTemporary - Return a temporary MDNode, for use in constructing - /// cyclic MDNode structures. A temporary MDNode is not uniqued, - /// may be RAUW'd, and must be manually deleted with deleteTemporary. - static MDNode *getTemporary(LLVMContext &Context, ArrayRef Vals); - - /// deleteTemporary - Deallocate a node created by getTemporary. The - /// node must not have any users. + /// \brief Deallocate a node created by getTemporary. + /// + /// The node must not have any users. static void deleteTemporary(MDNode *N); - /// replaceOperandWith - Replace a specific operand. - void replaceOperandWith(unsigned i, Value *NewVal); + LLVMContext &getContext() const { return Context; } - /// getOperand - Return specified operand. - Value *getOperand(unsigned i) const LLVM_READONLY; + /// \brief Replace a specific operand. + void replaceOperandWith(unsigned I, Metadata *New); - /// getNumOperands - Return number of MDNode operands. + /// \brief Check if node is fully resolved. + bool isResolved() const; + + /// \brief Check if node is distinct. + /// + /// Distinct nodes are not uniqued, and will not be returned by \a + /// MDNode::get(). + bool isDistinct() const { + return isStoredDistinctInContext() || isa(this); + } + +protected: + /// \brief Set an operand. + /// + /// Sets the operand directly, without worrying about uniquing. + void setOperand(unsigned I, Metadata *New); + +public: + typedef const MDOperand *op_iterator; + typedef iterator_range op_range; + + op_iterator op_begin() const { + return const_cast(this)->mutable_begin(); + } + op_iterator op_end() const { + return const_cast(this)->mutable_end(); + } + op_range operands() const { return op_range(op_begin(), op_end()); } + + const MDOperand &getOperand(unsigned I) const { + assert(I < NumOperands && "Out of range"); + return op_begin()[I]; + } + + /// \brief Return number of MDNode operands. unsigned getNumOperands() const { return NumOperands; } - /// isFunctionLocal - Return whether MDNode is local to a function. - bool isFunctionLocal() const { - return (getSubclassDataFromValue() & FunctionLocalBit) != 0; + /// \brief Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == MDTupleKind || + MD->getMetadataID() == MDLocationKind || + MD->getMetadataID() == MDNodeFwdDeclKind; } - // getFunction - If this metadata is function-local and recursively has a - // function-local operand, return the first such operand's parent function. - // Otherwise, return null. getFunction() should not be used for performance- - // critical code because it recursively visits all the MDNode's operands. - const Function *getFunction() const; - - /// Profile - calculate a unique identifier for this MDNode to collapse - /// duplicates - void Profile(FoldingSetNodeID &ID) const; - - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const Value *V) { - return V->getValueID() == MDNodeVal; - } - - /// Check whether MDNode is a vtable access. + /// \brief Check whether MDNode is a vtable access. bool isTBAAVtableAccess() const; - /// Methods for metadata merging. + /// \brief Methods for metadata merging. + static MDNode *concatenate(MDNode *A, MDNode *B); + static MDNode *intersect(MDNode *A, MDNode *B); static MDNode *getMostGenericTBAA(MDNode *A, MDNode *B); + static AAMDNodes getMostGenericAA(const AAMDNodes &A, const AAMDNodes &B); static MDNode *getMostGenericFPMath(MDNode *A, MDNode *B); static MDNode *getMostGenericRange(MDNode *A, MDNode *B); + static MDNode *getMostGenericAliasScope(MDNode *A, MDNode *B); +}; + +/// \brief Uniquable metadata node. +/// +/// A uniquable metadata node. This contains the basic functionality +/// for implementing sub-types of \a MDNode that can be uniqued like +/// constants. +/// +/// There is limited support for RAUW at construction time. At +/// construction time, if any operands are an instance of \a +/// MDNodeFwdDecl (or another unresolved \a UniquableMDNode, which +/// indicates an \a MDNodeFwdDecl in its path), the node itself will be +/// unresolved. As soon as all operands become resolved, it will drop +/// RAUW support permanently. +/// +/// If an unresolved node is part of a cycle, \a resolveCycles() needs +/// to be called on some member of the cycle when each \a MDNodeFwdDecl +/// has been removed. +class UniquableMDNode : public MDNode { + friend class ReplaceableMetadataImpl; + friend class MDNode; + friend class LLVMContextImpl; + + /// \brief Support RAUW as long as one of its arguments is replaceable. + /// + /// FIXME: Save memory by storing this in a pointer union with the + /// LLVMContext, and adding an LLVMContext reference to RMI. + std::unique_ptr ReplaceableUses; + +protected: + /// \brief Create a new node. + /// + /// If \c AllowRAUW, then if any operands are unresolved support RAUW. RAUW + /// will be dropped once all operands have been resolved (or if \a + /// resolveCycles() is called). + UniquableMDNode(LLVMContext &C, unsigned ID, ArrayRef Vals, + bool AllowRAUW); + ~UniquableMDNode() {} + + void storeDistinctInContext(); + +public: + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == MDTupleKind || + MD->getMetadataID() == MDLocationKind; + } + + /// \brief Check whether any operands are forward declarations. + /// + /// Returns \c true as long as any operands (or their operands, etc.) are \a + /// MDNodeFwdDecl. + /// + /// As forward declarations are resolved, their containers should get + /// resolved automatically. However, if this (or one of its operands) is + /// involved in a cycle, \a resolveCycles() needs to be called explicitly. + bool isResolved() const { return !ReplaceableUses; } + + /// \brief Resolve cycles. + /// + /// Once all forward declarations have been resolved, force cycles to be + /// resolved. + /// + /// \pre No operands (or operands' operands, etc.) are \a MDNodeFwdDecl. + void resolveCycles(); + private: - // destroy - Delete this node. Only when there are no uses. - void destroy(); + void handleChangedOperand(void *Ref, Metadata *New); - bool isNotUniqued() const { - return (getSubclassDataFromValue() & NotUniquedBit) != 0; - } - void setIsNotUniqued(); + void resolve(); + void resolveAfterOperandChange(Metadata *Old, Metadata *New); + void decrementUnresolvedOperandCount(); - // Shadow Value::setValueSubclassData with a private forwarding method so that - // any future subclasses cannot accidentally use it. - void setValueSubclassData(unsigned short D) { - Value::setValueSubclassData(D); + void deleteAsSubclass(); + UniquableMDNode *uniquify(); + void eraseFromStore(); +}; + +/// \brief Tuple of metadata. +/// +/// This is the simple \a MDNode arbitrary tuple. Nodes are uniqued by +/// default based on their operands. +class MDTuple : public UniquableMDNode { + friend class LLVMContextImpl; + friend class UniquableMDNode; + + MDTuple(LLVMContext &C, ArrayRef Vals, bool AllowRAUW) + : UniquableMDNode(C, MDTupleKind, Vals, AllowRAUW) {} + ~MDTuple() { dropAllReferences(); } + + void setHash(unsigned Hash) { MDNodeSubclassData = Hash; } + void recalculateHash(); + + static MDTuple *getImpl(LLVMContext &Context, ArrayRef MDs, + bool ShouldCreate); + +public: + /// \brief Get the hash, if any. + unsigned getHash() const { return MDNodeSubclassData; } + + static MDTuple *get(LLVMContext &Context, ArrayRef MDs) { + return getImpl(Context, MDs, /* ShouldCreate */ true); } + static MDTuple *getIfExists(LLVMContext &Context, ArrayRef MDs) { + return getImpl(Context, MDs, /* ShouldCreate */ false); + } + + /// \brief Return a distinct node. + /// + /// Return a distinct node -- i.e., a node that is not uniqued. + static MDTuple *getDistinct(LLVMContext &Context, ArrayRef MDs); + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == MDTupleKind; + } + +private: + MDTuple *uniquifyImpl(); + void eraseFromStoreImpl(); +}; + +MDNode *MDNode::get(LLVMContext &Context, ArrayRef MDs) { + return MDTuple::get(Context, MDs); +} +MDNode *MDNode::getIfExists(LLVMContext &Context, ArrayRef MDs) { + return MDTuple::getIfExists(Context, MDs); +} +MDNode *MDNode::getDistinct(LLVMContext &Context, ArrayRef MDs) { + return MDTuple::getDistinct(Context, MDs); +} + +/// \brief Debug location. +/// +/// A debug location in source code, used for debug info and otherwise. +class MDLocation : public UniquableMDNode { + friend class LLVMContextImpl; + friend class UniquableMDNode; + + MDLocation(LLVMContext &C, unsigned Line, unsigned Column, + ArrayRef MDs, bool AllowRAUW); + ~MDLocation() { dropAllReferences(); } + + static MDLocation *constructHelper(LLVMContext &Context, unsigned Line, + unsigned Column, Metadata *Scope, + Metadata *InlinedAt, bool AllowRAUW); + + static MDLocation *getImpl(LLVMContext &Context, unsigned Line, + unsigned Column, Metadata *Scope, + Metadata *InlinedAt, bool ShouldCreate); + + // Disallow replacing operands. + void replaceOperandWith(unsigned I, Metadata *New) LLVM_DELETED_FUNCTION; + +public: + static MDLocation *get(LLVMContext &Context, unsigned Line, unsigned Column, + Metadata *Scope, Metadata *InlinedAt = nullptr) { + return getImpl(Context, Line, Column, Scope, InlinedAt, + /* ShouldCreate */ true); + } + static MDLocation *getIfExists(LLVMContext &Context, unsigned Line, + unsigned Column, Metadata *Scope, + Metadata *InlinedAt = nullptr) { + return getImpl(Context, Line, Column, Scope, InlinedAt, + /* ShouldCreate */ false); + } + static MDLocation *getDistinct(LLVMContext &Context, unsigned Line, + unsigned Column, Metadata *Scope, + Metadata *InlinedAt = nullptr); + + unsigned getLine() const { return MDNodeSubclassData; } + unsigned getColumn() const { return SubclassData16; } + Metadata *getScope() const { return getOperand(0); } + Metadata *getInlinedAt() const { + if (getNumOperands() == 2) + return getOperand(1); + return nullptr; + } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == MDLocationKind; + } + +private: + MDLocation *uniquifyImpl(); + void eraseFromStoreImpl(); +}; + +/// \brief Forward declaration of metadata. +/// +/// Forward declaration of metadata, in the form of a basic tuple. Unlike \a +/// MDTuple, this class has full support for RAUW, is not owned, is not +/// uniqued, and is suitable for forward references. +class MDNodeFwdDecl : public MDNode, ReplaceableMetadataImpl { + friend class Metadata; + friend class ReplaceableMetadataImpl; + + MDNodeFwdDecl(LLVMContext &C, ArrayRef Vals) + : MDNode(C, MDNodeFwdDeclKind, Vals) {} + +public: + ~MDNodeFwdDecl() { dropAllReferences(); } + + // MSVC doesn't see the alternative: "using MDNode::operator delete". + void operator delete(void *Mem) { MDNode::operator delete(Mem); } + + static MDNodeFwdDecl *get(LLVMContext &Context, ArrayRef MDs) { + return new (MDs.size()) MDNodeFwdDecl(Context, MDs); + } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == MDNodeFwdDeclKind; + } + + using ReplaceableMetadataImpl::replaceAllUsesWith; }; //===----------------------------------------------------------------------===// -/// NamedMDNode - a tuple of MDNodes. Despite its name, a NamedMDNode isn't -/// itself an MDNode. NamedMDNodes belong to modules, have names, and contain -/// lists of MDNodes. +/// \brief A tuple of MDNodes. +/// +/// Despite its name, a NamedMDNode isn't itself an MDNode. NamedMDNodes belong +/// to modules, have names, and contain lists of MDNodes. +/// +/// TODO: Inherit from Metadata. class NamedMDNode : public ilist_node { friend class SymbolTableListTraits; friend struct ilist_traits; @@ -202,7 +925,7 @@ class NamedMDNode : public ilist_node { std::string Name; Module *Parent; - void *Operands; // SmallVector, 4> + void *Operands; // SmallVector void setParent(Module *M) { Parent = M; } @@ -245,46 +968,34 @@ class NamedMDNode : public ilist_node { }; public: - /// eraseFromParent - Drop all references and remove the node from parent - /// module. + /// \brief Drop all references and remove the node from parent module. void eraseFromParent(); - /// dropAllReferences - Remove all uses and clear node vector. + /// \brief Remove all uses and clear node vector. void dropAllReferences(); - /// ~NamedMDNode - Destroy NamedMDNode. ~NamedMDNode(); - /// getParent - Get the module that holds this named metadata collection. + /// \brief Get the module that holds this named metadata collection. inline Module *getParent() { return Parent; } inline const Module *getParent() const { return Parent; } - /// getOperand - Return specified operand. MDNode *getOperand(unsigned i) const; - - /// getNumOperands - Return the number of NamedMDNode operands. unsigned getNumOperands() const; - - /// addOperand - Add metadata operand. void addOperand(MDNode *M); - - /// getName - Return a constant reference to this named metadata's name. + void setOperand(unsigned I, MDNode *New); StringRef getName() const; - - /// print - Implement operator<< on NamedMDNode. void print(raw_ostream &ROS) const; - - /// dump() - Allow printing of NamedMDNodes from the debugger. void dump() const; // --------------------------------------------------------------------------- // Operand Iterator interface... // - typedef op_iterator_impl op_iterator; + typedef op_iterator_impl op_iterator; op_iterator op_begin() { return op_iterator(this, 0); } op_iterator op_end() { return op_iterator(this, getNumOperands()); } - typedef op_iterator_impl const_op_iterator; + typedef op_iterator_impl const_op_iterator; const_op_iterator op_begin() const { return const_op_iterator(this, 0); } const_op_iterator op_end() const { return const_op_iterator(this, getNumOperands()); } diff --git a/contrib/llvm/include/llvm/IR/MetadataTracking.h b/contrib/llvm/include/llvm/IR/MetadataTracking.h new file mode 100644 index 000000000000..541d9b3b1245 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/MetadataTracking.h @@ -0,0 +1,99 @@ +//===- llvm/IR/MetadataTracking.h - Metadata tracking ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Low-level functions to enable tracking of metadata that could RAUW. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_METADATATRACKING_H +#define LLVM_IR_METADATATRACKING_H + +#include "llvm/ADT/PointerUnion.h" +#include "llvm/Support/Casting.h" +#include + +namespace llvm { + +class Metadata; +class MetadataAsValue; + +/// \brief API for tracking metadata references through RAUW and deletion. +/// +/// Shared API for updating \a Metadata pointers in subclasses that support +/// RAUW. +/// +/// This API is not meant to be used directly. See \a TrackingMDRef for a +/// user-friendly tracking reference. +class MetadataTracking { +public: + /// \brief Track the reference to metadata. + /// + /// Register \c MD with \c *MD, if the subclass supports tracking. If \c *MD + /// gets RAUW'ed, \c MD will be updated to the new address. If \c *MD gets + /// deleted, \c MD will be set to \c nullptr. + /// + /// If tracking isn't supported, \c *MD will not change. + /// + /// \return true iff tracking is supported by \c MD. + static bool track(Metadata *&MD) { + return track(&MD, *MD, static_cast(nullptr)); + } + + /// \brief Track the reference to metadata for \a Metadata. + /// + /// As \a track(Metadata*&), but with support for calling back to \c Owner to + /// tell it that its operand changed. This could trigger \c Owner being + /// re-uniqued. + static bool track(void *Ref, Metadata &MD, Metadata &Owner) { + return track(Ref, MD, &Owner); + } + + /// \brief Track the reference to metadata for \a MetadataAsValue. + /// + /// As \a track(Metadata*&), but with support for calling back to \c Owner to + /// tell it that its operand changed. This could trigger \c Owner being + /// re-uniqued. + static bool track(void *Ref, Metadata &MD, MetadataAsValue &Owner) { + return track(Ref, MD, &Owner); + } + + /// \brief Stop tracking a reference to metadata. + /// + /// Stops \c *MD from tracking \c MD. + static void untrack(Metadata *&MD) { untrack(&MD, *MD); } + static void untrack(void *Ref, Metadata &MD); + + /// \brief Move tracking from one reference to another. + /// + /// Semantically equivalent to \c untrack(MD) followed by \c track(New), + /// except that ownership callbacks are maintained. + /// + /// Note: it is an error if \c *MD does not equal \c New. + /// + /// \return true iff tracking is supported by \c MD. + static bool retrack(Metadata *&MD, Metadata *&New) { + return retrack(&MD, *MD, &New); + } + static bool retrack(void *Ref, Metadata &MD, void *New); + + /// \brief Check whether metadata is replaceable. + static bool isReplaceable(const Metadata &MD); + + typedef PointerUnion OwnerTy; + +private: + /// \brief Track a reference to metadata for an owner. + /// + /// Generalized version of tracking. + static bool track(void *Ref, Metadata &MD, OwnerTy Owner); +}; + +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/IR/Module.h b/contrib/llvm/include/llvm/IR/Module.h index 0b0f4ea3185a..b24023b05e4f 100644 --- a/contrib/llvm/include/llvm/IR/Module.h +++ b/contrib/llvm/include/llvm/IR/Module.h @@ -33,8 +33,6 @@ class GVMaterializer; class LLVMContext; class RandomNumberGenerator; class StructType; -template struct DenseMapInfo; -template class DenseMap; template<> struct ilist_traits : public SymbolTableListTraits { @@ -138,6 +136,11 @@ public: /// The Function constant iterator typedef FunctionListType::const_iterator const_iterator; + /// The Function reverse iterator. + typedef FunctionListType::reverse_iterator reverse_iterator; + /// The Function constant reverse iterator. + typedef FunctionListType::const_reverse_iterator const_reverse_iterator; + /// The Global Alias iterators. typedef AliasListType::iterator alias_iterator; /// The Global Alias constant iterator @@ -145,7 +148,7 @@ public: /// The named metadata iterators. typedef NamedMDListType::iterator named_metadata_iterator; - /// The named metadata constant interators. + /// The named metadata constant iterators. typedef NamedMDListType::const_iterator const_named_metadata_iterator; /// This enumeration defines the supported behaviors of module flags. @@ -178,15 +181,23 @@ public: /// Appends the two values, which are required to be metadata /// nodes. However, duplicate entries in the second list are dropped /// during the append operation. - AppendUnique = 6 + AppendUnique = 6, + + // Markers: + ModFlagBehaviorFirstVal = Error, + ModFlagBehaviorLastVal = AppendUnique }; + /// Checks if Metadata represents a valid ModFlagBehavior, and stores the + /// converted result in MFB. + static bool isValidModFlagBehavior(Metadata *MD, ModFlagBehavior &MFB); + struct ModuleFlagEntry { ModFlagBehavior Behavior; MDString *Key; - Value *Val; - ModuleFlagEntry(ModFlagBehavior B, MDString *K, Value *V) - : Behavior(B), Key(K), Val(V) {} + Metadata *Val; + ModuleFlagEntry(ModFlagBehavior B, MDString *K, Metadata *V) + : Behavior(B), Key(K), Val(V) {} }; /// @} @@ -206,9 +217,8 @@ private: Materializer; ///< Used to materialize GlobalValues std::string ModuleID; ///< Human readable identifier for the module std::string TargetTriple; ///< Platform target triple Module compiled on + ///< Format: (arch)(sub)-(vendor)-(sys0-(abi) void *NamedMDSymTab; ///< NamedMDNode names. - // Allow lazy initialization in const method. - mutable RandomNumberGenerator *RNG; ///< The random number generator for this module. // We need to keep the string because the C API expects us to own the string // representation. @@ -238,6 +248,12 @@ public: /// @returns the module identifier as a string const std::string &getModuleIdentifier() const { return ModuleID; } + /// \brief Get a short "name" for the module. + /// + /// This is useful for debugging or logging. It is essentially a convenience + /// wrapper around getModuleIdentifier(). + StringRef getName() const { return ModuleID; } + /// Get the data layout string for the module's target platform. This is /// equivalent to getDataLayout()->getStringRepresentation(). const std::string &getDataLayoutStr() const { return DataLayoutStr; } @@ -257,10 +273,16 @@ public: /// @returns a string containing the module-scope inline assembly blocks. const std::string &getModuleInlineAsm() const { return GlobalScopeAsm; } - /// Get the RandomNumberGenerator for this module. The RNG can be - /// seeded via -rng-seed= and is salted with the ModuleID. - /// The returned RNG should not be shared across threads. - RandomNumberGenerator &getRNG() const; + /// Get a RandomNumberGenerator salted for use with this module. The + /// RNG can be seeded via -rng-seed= and is salted with the + /// ModuleID and the provided pass salt. The returned RNG should not + /// be shared across threads or passes. + /// + /// A unique RNG per pass ensures a reproducible random stream even + /// when other randomness consuming passes are added or removed. In + /// addition, the random stream will be reproducible across LLVM + /// versions when the pass does not change. + RandomNumberGenerator *createRNG(const Pass* P) const; /// @} /// @name Module Level Mutators @@ -314,6 +336,8 @@ public: /// name. StructType *getTypeByName(StringRef Name) const; + std::vector getIdentifiedStructTypes() const; + /// @} /// @name Function Accessors /// @{ @@ -340,11 +364,11 @@ public: /// function arguments, which makes it easier for clients to use. Constant *getOrInsertFunction(StringRef Name, AttributeSet AttributeList, - Type *RetTy, ...) END_WITH_NULL; + Type *RetTy, ...) LLVM_END_WITH_NULL; /// Same as above, but without the attributes. Constant *getOrInsertFunction(StringRef Name, Type *RetTy, ...) - END_WITH_NULL; + LLVM_END_WITH_NULL; /// Look up the specified function in the module symbol table. If it does not /// exist, return null. @@ -358,8 +382,11 @@ public: /// does not exist, return null. If AllowInternal is set to true, this /// function will return types that have InternalLinkage. By default, these /// types are not returned. - const GlobalVariable *getGlobalVariable(StringRef Name, - bool AllowInternal = false) const { + GlobalVariable *getGlobalVariable(StringRef Name) const { + return getGlobalVariable(Name, false); + } + + GlobalVariable *getGlobalVariable(StringRef Name, bool AllowInternal) const { return const_cast(this)->getGlobalVariable(Name, AllowInternal); } @@ -425,7 +452,7 @@ public: /// Return the corresponding value if Key appears in module flags, otherwise /// return null. - Value *getModuleFlag(StringRef Key) const; + Metadata *getModuleFlag(StringRef Key) const; /// Returns the NamedMDNode in the module that represents module-level flags. /// This method returns null if there are no module-level flags. @@ -438,7 +465,8 @@ public: /// Add a module-level flag to the module-level flags metadata. It will create /// the module-level flags named metadata if it doesn't already exist. - void addModuleFlag(ModFlagBehavior Behavior, StringRef Key, Value *Val); + void addModuleFlag(ModFlagBehavior Behavior, StringRef Key, Metadata *Val); + void addModuleFlag(ModFlagBehavior Behavior, StringRef Key, Constant *Val); void addModuleFlag(ModFlagBehavior Behavior, StringRef Key, uint32_t Val); void addModuleFlag(MDNode *Node); @@ -457,9 +485,6 @@ public: /// Retrieves the GVMaterializer, if any, for this Module. GVMaterializer *getMaterializer() const { return Materializer.get(); } - /// True if the definition of GV has yet to be materializedfrom the - /// GVMaterializer. - bool isMaterializable(const GlobalValue *GV) const; /// Returns true if this GV was loaded from this Module's GVMaterializer and /// the GVMaterializer knows how to dematerialize the GV. bool isDematerializable(const GlobalValue *GV) const; @@ -467,10 +492,10 @@ public: /// Make sure the GlobalValue is fully read. If the module is corrupt, this /// returns true and fills in the optional string with information about the /// problem. If successful, this returns false. - bool Materialize(GlobalValue *GV, std::string *ErrInfo = nullptr); + std::error_code materialize(GlobalValue *GV); /// If the GlobalValue is read in, and if the GVMaterializer supports it, /// release the memory for the function, and set it up to be materialized - /// lazily. If !isDematerializable(), this method is a noop. + /// lazily. If !isDematerializable(), this method is a no-op. void Dematerialize(GlobalValue *GV); /// Make sure all GlobalValues in this Module are fully read. @@ -479,7 +504,7 @@ public: /// Make sure all GlobalValues in this Module are fully read and clear the /// Materializer. If the module is corrupt, this DOES NOT clear the old /// Materializer. - std::error_code materializeAllPermanently(bool ReleaseBuffer = false); + std::error_code materializeAllPermanently(); /// @} /// @name Direct access to the globals list, functions list, and symbol table @@ -547,9 +572,20 @@ public: const_iterator begin() const { return FunctionList.begin(); } iterator end () { return FunctionList.end(); } const_iterator end () const { return FunctionList.end(); } + reverse_iterator rbegin() { return FunctionList.rbegin(); } + const_reverse_iterator rbegin() const{ return FunctionList.rbegin(); } + reverse_iterator rend() { return FunctionList.rend(); } + const_reverse_iterator rend() const { return FunctionList.rend(); } size_t size() const { return FunctionList.size(); } bool empty() const { return FunctionList.empty(); } + iterator_range functions() { + return iterator_range(begin(), end()); + } + iterator_range functions() const { + return iterator_range(begin(), end()); + } + /// @} /// @name Alias Iteration /// @{ diff --git a/contrib/llvm/include/llvm/IR/Operator.h b/contrib/llvm/include/llvm/IR/Operator.h index 888cabffe378..0933f2170236 100644 --- a/contrib/llvm/include/llvm/IR/Operator.h +++ b/contrib/llvm/include/llvm/IR/Operator.h @@ -28,9 +28,8 @@ class GetElementPtrInst; class BinaryOperator; class ConstantExpr; -/// Operator - This is a utility class that provides an abstraction for the -/// common functionality between Instructions and ConstantExprs. -/// +/// This is a utility class that provides an abstraction for the common +/// functionality between Instructions and ConstantExprs. class Operator : public User { private: // The Operator class is intended to be used as a utility, and is never itself @@ -46,17 +45,15 @@ protected: ~Operator(); public: - /// getOpcode - Return the opcode for this Instruction or ConstantExpr. - /// + /// Return the opcode for this Instruction or ConstantExpr. unsigned getOpcode() const { if (const Instruction *I = dyn_cast(this)) return I->getOpcode(); return cast(this)->getOpcode(); } - /// getOpcode - If V is an Instruction or ConstantExpr, return its - /// opcode. Otherwise return UserOp1. - /// + /// If V is an Instruction or ConstantExpr, return its opcode. + /// Otherwise return UserOp1. static unsigned getOpcode(const Value *V) { if (const Instruction *I = dyn_cast(V)) return I->getOpcode(); @@ -72,10 +69,9 @@ public: } }; -/// OverflowingBinaryOperator - Utility class for integer arithmetic operators -/// which may exhibit overflow - Add, Sub, and Mul. It does not include SDiv, -/// despite that operator having the potential for overflow. -/// +/// Utility class for integer arithmetic operators which may exhibit overflow - +/// Add, Sub, and Mul. It does not include SDiv, despite that operator having +/// the potential for overflow. class OverflowingBinaryOperator : public Operator { public: enum { @@ -96,13 +92,13 @@ private: } public: - /// hasNoUnsignedWrap - Test whether this operation is known to never + /// Test whether this operation is known to never /// undergo unsigned overflow, aka the nuw property. bool hasNoUnsignedWrap() const { return SubclassOptionalData & NoUnsignedWrap; } - /// hasNoSignedWrap - Test whether this operation is known to never + /// Test whether this operation is known to never /// undergo signed overflow, aka the nsw property. bool hasNoSignedWrap() const { return (SubclassOptionalData & NoSignedWrap) != 0; @@ -126,8 +122,8 @@ public: } }; -/// PossiblyExactOperator - A udiv or sdiv instruction, which can be marked as -/// "exact", indicating that no bits are destroyed. +/// A udiv or sdiv instruction, which can be marked as "exact", +/// indicating that no bits are destroyed. class PossiblyExactOperator : public Operator { public: enum { @@ -142,8 +138,7 @@ private: } public: - /// isExact - Test whether this division is known to be exact, with - /// zero remainder. + /// Test whether this division is known to be exact, with zero remainder. bool isExact() const { return SubclassOptionalData & IsExact; } @@ -217,7 +212,7 @@ public: }; -/// FPMathOperator - Utility class for floating point operations which can have +/// Utility class for floating point operations which can have /// information about relaxed accuracy requirements attached to them. class FPMathOperator : public Operator { private: @@ -257,11 +252,18 @@ private: (B * FastMathFlags::AllowReciprocal); } - /// Convenience function for setting all the fast-math flags + /// Convenience function for setting multiple fast-math flags. + /// FMF is a mask of the bits to set. void setFastMathFlags(FastMathFlags FMF) { SubclassOptionalData |= FMF.Flags; } + /// Convenience function for copying all fast-math flags. + /// All values in FMF are transferred to this operator. + void copyFastMathFlags(FastMathFlags FMF) { + SubclassOptionalData = FMF.Flags; + } + public: /// Test whether this operation is permitted to be /// algebraically transformed, aka the 'A' fast-math property. @@ -312,8 +314,7 @@ public: }; -/// ConcreteOperator - A helper template for defining operators for individual -/// opcodes. +/// A helper template for defining operators for individual opcodes. template class ConcreteOperator : public SuperClass { public: @@ -357,6 +358,8 @@ class LShrOperator }; +class ZExtOperator : public ConcreteOperator {}; + class GEPOperator : public ConcreteOperator { @@ -372,8 +375,7 @@ class GEPOperator } public: - /// isInBounds - Test whether this is an inbounds GEP, as defined - /// by LangRef.html. + /// Test whether this is an inbounds GEP, as defined by LangRef.html. bool isInBounds() const { return SubclassOptionalData & IsInBounds; } @@ -393,16 +395,14 @@ public: return 0U; // get index for modifying correct operand } - /// getPointerOperandType - Method to return the pointer operand as a - /// PointerType. + /// Method to return the pointer operand as a PointerType. Type *getPointerOperandType() const { return getPointerOperand()->getType(); } - /// getPointerAddressSpace - Method to return the address space of the - /// pointer operand. + /// Method to return the address space of the pointer operand. unsigned getPointerAddressSpace() const { - return cast(getPointerOperandType())->getAddressSpace(); + return getPointerOperandType()->getPointerAddressSpace(); } unsigned getNumIndices() const { // Note: always non-negative @@ -413,8 +413,8 @@ public: return getNumOperands() > 1; } - /// hasAllZeroIndices - Return true if all of the indices of this GEP are - /// zeros. If so, the result pointer and the first operand have the same + /// Return true if all of the indices of this GEP are zeros. + /// If so, the result pointer and the first operand have the same /// value, just potentially different types. bool hasAllZeroIndices() const { for (const_op_iterator I = idx_begin(), E = idx_end(); I != E; ++I) { @@ -426,8 +426,8 @@ public: return true; } - /// hasAllConstantIndices - Return true if all of the indices of this GEP are - /// constant integers. If so, the result pointer and the first operand have + /// Return true if all of the indices of this GEP are constant integers. + /// If so, the result pointer and the first operand have /// a constant offset between them. bool hasAllConstantIndices() const { for (const_op_iterator I = idx_begin(), E = idx_end(); I != E; ++I) { @@ -493,14 +493,12 @@ public: return 0U; // get index for modifying correct operand } - /// getPointerOperandType - Method to return the pointer operand as a - /// PointerType. + /// Method to return the pointer operand as a PointerType. Type *getPointerOperandType() const { return getPointerOperand()->getType(); } - /// getPointerAddressSpace - Method to return the address space of the - /// pointer operand. + /// Method to return the address space of the pointer operand. unsigned getPointerAddressSpace() const { return cast(getPointerOperandType())->getAddressSpace(); } diff --git a/contrib/llvm/include/llvm/IR/PassManager.h b/contrib/llvm/include/llvm/IR/PassManager.h index cc2a80b9ff7d..262576849264 100644 --- a/contrib/llvm/include/llvm/IR/PassManager.h +++ b/contrib/llvm/include/llvm/IR/PassManager.h @@ -35,14 +35,17 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_IR_PASS_MANAGER_H -#define LLVM_IR_PASS_MANAGER_H +#ifndef LLVM_IR_PASSMANAGER_H +#define LLVM_IR_PASSMANAGER_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManagerInternal.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/type_traits.h" #include #include @@ -91,9 +94,12 @@ public: } /// \brief Mark a particular pass as preserved, adding it to the set. - template void preserve() { + template void preserve() { preserve(PassT::ID()); } + + /// \brief Mark an abstract PassID as preserved, adding it to the set. + void preserve(void *PassID) { if (!areAllPreserved()) - PreservedPassIDs.insert(PassT::ID()); + PreservedPassIDs.insert(PassID); } /// \brief Intersect this set with another in place. @@ -107,11 +113,9 @@ public: PreservedPassIDs = Arg.PreservedPassIDs; return; } - for (SmallPtrSet::const_iterator I = PreservedPassIDs.begin(), - E = PreservedPassIDs.end(); - I != E; ++I) - if (!Arg.PreservedPassIDs.count(*I)) - PreservedPassIDs.erase(*I); + for (void *P : PreservedPassIDs) + if (!Arg.PreservedPassIDs.count(P)) + PreservedPassIDs.erase(P); } /// \brief Intersect this set with a temporary other set in place. @@ -125,11 +129,9 @@ public: PreservedPassIDs = std::move(Arg.PreservedPassIDs); return; } - for (SmallPtrSet::const_iterator I = PreservedPassIDs.begin(), - E = PreservedPassIDs.end(); - I != E; ++I) - if (!Arg.PreservedPassIDs.count(*I)) - PreservedPassIDs.erase(*I); + for (void *P : PreservedPassIDs) + if (!Arg.PreservedPassIDs.count(P)) + PreservedPassIDs.erase(P); } /// \brief Query whether a pass is marked as preserved by this set. @@ -144,408 +146,115 @@ public: PreservedPassIDs.count(PassID); } + /// \brief Test whether all passes are preserved. + /// + /// This is used primarily to optimize for the case of no changes which will + /// common in many scenarios. + bool areAllPreserved() const { + return PreservedPassIDs.count((void *)AllPassesID); + } + private: // Note that this must not be -1 or -2 as those are already used by the // SmallPtrSet. static const uintptr_t AllPassesID = (intptr_t)(-3); - bool areAllPreserved() const { - return PreservedPassIDs.count((void *)AllPassesID); - } - SmallPtrSet PreservedPassIDs; }; -/// \brief Implementation details of the pass manager interfaces. -namespace detail { - -/// \brief Template for the abstract base class used to dispatch -/// polymorphically over pass objects. -template struct PassConcept { - // Boiler plate necessary for the container of derived classes. - virtual ~PassConcept() {} - - /// \brief The polymorphic API which runs the pass over a given IR entity. - /// - /// Note that actual pass object can omit the analysis manager argument if - /// desired. Also that the analysis manager may be null if there is no - /// analysis manager in the pass pipeline. - virtual PreservedAnalyses run(IRUnitT IR, AnalysisManagerT *AM) = 0; - - /// \brief Polymorphic method to access the name of a pass. - virtual StringRef name() = 0; -}; - -/// \brief SFINAE metafunction for computing whether \c PassT has a run method -/// accepting an \c AnalysisManagerT. -template -class PassRunAcceptsAnalysisManager { - typedef char SmallType; - struct BigType { - char a, b; - }; - - template - struct Checker; - - template static SmallType f(Checker *); - template static BigType f(...); +// Forward declare the analysis manager template. +template class AnalysisManager; +/// \brief Manages a sequence of passes over units of IR. +/// +/// A pass manager contains a sequence of passes to run over units of IR. It is +/// itself a valid pass over that unit of IR, and when over some given IR will +/// run each pass in sequence. This is the primary and most basic building +/// block of a pass pipeline. +/// +/// If it is run with an \c AnalysisManager argument, it will propagate +/// that analysis manager to each pass it runs, as well as calling the analysis +/// manager's invalidation routine with the PreservedAnalyses of each pass it +/// runs. +template class PassManager { public: - enum { Value = sizeof(f(nullptr)) == sizeof(SmallType) }; -}; - -/// \brief A template wrapper used to implement the polymorphic API. -/// -/// Can be instantiated for any object which provides a \c run method accepting -/// an \c IRUnitT. It requires the pass to be a copyable object. When the -/// \c run method also accepts an \c AnalysisManagerT*, we pass it along. -template ::Value> -struct PassModel; - -/// \brief Specialization of \c PassModel for passes that accept an analyis -/// manager. -template -struct PassModel - : PassConcept { - explicit PassModel(PassT Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - PassModel(const PassModel &Arg) : Pass(Arg.Pass) {} - PassModel(PassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(PassModel &LHS, PassModel &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - } - PassModel &operator=(PassModel RHS) { - swap(*this, RHS); - return *this; - } - - PreservedAnalyses run(IRUnitT IR, AnalysisManagerT *AM) override { - return Pass.run(IR, AM); - } - StringRef name() override { return PassT::name(); } - PassT Pass; -}; - -/// \brief Specialization of \c PassModel for passes that accept an analyis -/// manager. -template -struct PassModel - : PassConcept { - explicit PassModel(PassT Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - PassModel(const PassModel &Arg) : Pass(Arg.Pass) {} - PassModel(PassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(PassModel &LHS, PassModel &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - } - PassModel &operator=(PassModel RHS) { - swap(*this, RHS); - return *this; - } - - PreservedAnalyses run(IRUnitT IR, AnalysisManagerT *AM) override { - return Pass.run(IR); - } - StringRef name() override { return PassT::name(); } - PassT Pass; -}; - -/// \brief Abstract concept of an analysis result. -/// -/// This concept is parameterized over the IR unit that this result pertains -/// to. -template struct AnalysisResultConcept { - virtual ~AnalysisResultConcept() {} - - /// \brief Method to try and mark a result as invalid. + /// \brief Construct a pass manager. /// - /// When the outer analysis manager detects a change in some underlying - /// unit of the IR, it will call this method on all of the results cached. - /// - /// This method also receives a set of preserved analyses which can be used - /// to avoid invalidation because the pass which changed the underlying IR - /// took care to update or preserve the analysis result in some way. - /// - /// \returns true if the result is indeed invalid (the default). - virtual bool invalidate(IRUnitT IR, const PreservedAnalyses &PA) = 0; -}; - -/// \brief SFINAE metafunction for computing whether \c ResultT provides an -/// \c invalidate member function. -template class ResultHasInvalidateMethod { - typedef char SmallType; - struct BigType { - char a, b; - }; - - template - struct Checker; - - template static SmallType f(Checker *); - template static BigType f(...); - -public: - enum { Value = sizeof(f(nullptr)) == sizeof(SmallType) }; -}; - -/// \brief Wrapper to model the analysis result concept. -/// -/// By default, this will implement the invalidate method with a trivial -/// implementation so that the actual analysis result doesn't need to provide -/// an invalidation handler. It is only selected when the invalidation handler -/// is not part of the ResultT's interface. -template ::Value> -struct AnalysisResultModel; - -/// \brief Specialization of \c AnalysisResultModel which provides the default -/// invalidate functionality. -template -struct AnalysisResultModel - : AnalysisResultConcept { - explicit AnalysisResultModel(ResultT Result) : Result(std::move(Result)) {} + /// It can be passed a flag to get debug logging as the passes are run. + PassManager(bool DebugLogging = false) : DebugLogging(DebugLogging) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. - AnalysisResultModel(const AnalysisResultModel &Arg) : Result(Arg.Result) {} - AnalysisResultModel(AnalysisResultModel &&Arg) - : Result(std::move(Arg.Result)) {} - friend void swap(AnalysisResultModel &LHS, AnalysisResultModel &RHS) { - using std::swap; - swap(LHS.Result, RHS.Result); - } - AnalysisResultModel &operator=(AnalysisResultModel RHS) { - swap(*this, RHS); - return *this; - } - - /// \brief The model bases invalidation solely on being in the preserved set. - // - // FIXME: We should actually use two different concepts for analysis results - // rather than two different models, and avoid the indirect function call for - // ones that use the trivial behavior. - bool invalidate(IRUnitT, const PreservedAnalyses &PA) override { - return !PA.preserved(PassT::ID()); - } - - ResultT Result; -}; - -/// \brief Specialization of \c AnalysisResultModel which delegates invalidate -/// handling to \c ResultT. -template -struct AnalysisResultModel - : AnalysisResultConcept { - explicit AnalysisResultModel(ResultT Result) : Result(std::move(Result)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - AnalysisResultModel(const AnalysisResultModel &Arg) : Result(Arg.Result) {} - AnalysisResultModel(AnalysisResultModel &&Arg) - : Result(std::move(Arg.Result)) {} - friend void swap(AnalysisResultModel &LHS, AnalysisResultModel &RHS) { - using std::swap; - swap(LHS.Result, RHS.Result); - } - AnalysisResultModel &operator=(AnalysisResultModel RHS) { - swap(*this, RHS); - return *this; - } - - /// \brief The model delegates to the \c ResultT method. - bool invalidate(IRUnitT IR, const PreservedAnalyses &PA) override { - return Result.invalidate(IR, PA); - } - - ResultT Result; -}; - -/// \brief Abstract concept of an analysis pass. -/// -/// This concept is parameterized over the IR unit that it can run over and -/// produce an analysis result. -template -struct AnalysisPassConcept { - virtual ~AnalysisPassConcept() {} - - /// \brief Method to run this analysis over a unit of IR. - /// \returns A unique_ptr to the analysis result object to be queried by - /// users. - virtual std::unique_ptr> - run(IRUnitT IR, AnalysisManagerT *AM) = 0; -}; - -/// \brief Wrapper to model the analysis pass concept. -/// -/// Can wrap any type which implements a suitable \c run method. The method -/// must accept the IRUnitT as an argument and produce an object which can be -/// wrapped in a \c AnalysisResultModel. -template ::Value> -struct AnalysisPassModel; - -/// \brief Specialization of \c AnalysisPassModel which passes an -/// \c AnalysisManager to PassT's run method. -template -struct AnalysisPassModel - : AnalysisPassConcept { - explicit AnalysisPassModel(PassT Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - AnalysisPassModel(const AnalysisPassModel &Arg) : Pass(Arg.Pass) {} - AnalysisPassModel(AnalysisPassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(AnalysisPassModel &LHS, AnalysisPassModel &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - } - AnalysisPassModel &operator=(AnalysisPassModel RHS) { - swap(*this, RHS); - return *this; - } - - // FIXME: Replace PassT::Result with type traits when we use C++11. - typedef AnalysisResultModel - ResultModelT; - - /// \brief The model delegates to the \c PassT::run method. - /// - /// The return is wrapped in an \c AnalysisResultModel. - std::unique_ptr> - run(IRUnitT IR, AnalysisManagerT *AM) override { - return make_unique(Pass.run(IR, AM)); - } - - PassT Pass; -}; - -/// \brief Specialization of \c AnalysisPassModel which does not pass an -/// \c AnalysisManager to PassT's run method. -template -struct AnalysisPassModel - : AnalysisPassConcept { - explicit AnalysisPassModel(PassT Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - AnalysisPassModel(const AnalysisPassModel &Arg) : Pass(Arg.Pass) {} - AnalysisPassModel(AnalysisPassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(AnalysisPassModel &LHS, AnalysisPassModel &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - } - AnalysisPassModel &operator=(AnalysisPassModel RHS) { - swap(*this, RHS); - return *this; - } - - // FIXME: Replace PassT::Result with type traits when we use C++11. - typedef AnalysisResultModel - ResultModelT; - - /// \brief The model delegates to the \c PassT::run method. - /// - /// The return is wrapped in an \c AnalysisResultModel. - std::unique_ptr> - run(IRUnitT IR, AnalysisManagerT *) override { - return make_unique(Pass.run(IR)); - } - - PassT Pass; -}; - -} // End namespace detail - -class ModuleAnalysisManager; - -class ModulePassManager { -public: - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - ModulePassManager() {} - ModulePassManager(ModulePassManager &&Arg) : Passes(std::move(Arg.Passes)) {} - ModulePassManager &operator=(ModulePassManager &&RHS) { + PassManager(PassManager &&Arg) + : Passes(std::move(Arg.Passes)), + DebugLogging(std::move(Arg.DebugLogging)) {} + PassManager &operator=(PassManager &&RHS) { Passes = std::move(RHS.Passes); + DebugLogging = std::move(RHS.DebugLogging); return *this; } - /// \brief Run all of the module passes in this module pass manager over - /// a module. - /// - /// This method should only be called for a single module as there is the - /// expectation that the lifetime of a pass is bounded to that of a module. - PreservedAnalyses run(Module *M, ModuleAnalysisManager *AM = nullptr); + /// \brief Run all of the passes in this manager over the IR. + PreservedAnalyses run(IRUnitT &IR, AnalysisManager *AM = nullptr) { + PreservedAnalyses PA = PreservedAnalyses::all(); - template void addPass(ModulePassT Pass) { - Passes.emplace_back(new ModulePassModel(std::move(Pass))); + if (DebugLogging) + dbgs() << "Starting pass manager run.\n"; + + for (unsigned Idx = 0, Size = Passes.size(); Idx != Size; ++Idx) { + if (DebugLogging) + dbgs() << "Running pass: " << Passes[Idx]->name() << "\n"; + + PreservedAnalyses PassPA = Passes[Idx]->run(IR, AM); + + // If we have an active analysis manager at this level we want to ensure + // we update it as each pass runs and potentially invalidates analyses. + // We also update the preserved set of analyses based on what analyses we + // have already handled the invalidation for here and don't need to + // invalidate when finished. + if (AM) + PassPA = AM->invalidate(IR, std::move(PassPA)); + + // Finally, we intersect the final preserved analyses to compute the + // aggregate preserved set for this pass manager. + PA.intersect(std::move(PassPA)); + + // FIXME: Historically, the pass managers all called the LLVM context's + // yield function here. We don't have a generic way to acquire the + // context and it isn't yet clear what the right pattern is for yielding + // in the new pass manager so it is currently omitted. + //IR.getContext().yield(); + } + + if (DebugLogging) + dbgs() << "Finished pass manager run.\n"; + + return PA; } - static StringRef name() { return "ModulePassManager"; } + template void addPass(PassT Pass) { + typedef detail::PassModel PassModelT; + Passes.emplace_back(new PassModelT(std::move(Pass))); + } + + static StringRef name() { return "PassManager"; } private: - // Pull in the concept type and model template specialized for modules. - typedef detail::PassConcept - ModulePassConcept; - template - struct ModulePassModel - : detail::PassModel { - ModulePassModel(PassT Pass) - : detail::PassModel( - std::move(Pass)) {} - }; + typedef detail::PassConcept PassConceptT; - ModulePassManager(const ModulePassManager &) LLVM_DELETED_FUNCTION; - ModulePassManager &operator=(const ModulePassManager &) LLVM_DELETED_FUNCTION; + PassManager(const PassManager &) LLVM_DELETED_FUNCTION; + PassManager &operator=(const PassManager &) LLVM_DELETED_FUNCTION; - std::vector> Passes; + std::vector> Passes; + + /// \brief Flag indicating whether we should do debug logging. + bool DebugLogging; }; -class FunctionAnalysisManager; +/// \brief Convenience typedef for a pass manager over modules. +typedef PassManager ModulePassManager; -class FunctionPassManager { -public: - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - FunctionPassManager() {} - FunctionPassManager(FunctionPassManager &&Arg) - : Passes(std::move(Arg.Passes)) {} - FunctionPassManager &operator=(FunctionPassManager &&RHS) { - Passes = std::move(RHS.Passes); - return *this; - } - - template void addPass(FunctionPassT Pass) { - Passes.emplace_back(new FunctionPassModel(std::move(Pass))); - } - - PreservedAnalyses run(Function *F, FunctionAnalysisManager *AM = nullptr); - - static StringRef name() { return "FunctionPassManager"; } - -private: - // Pull in the concept type and model template specialized for functions. - typedef detail::PassConcept - FunctionPassConcept; - template - struct FunctionPassModel - : detail::PassModel { - FunctionPassModel(PassT Pass) - : detail::PassModel( - std::move(Pass)) {} - }; - - FunctionPassManager(const FunctionPassManager &) LLVM_DELETED_FUNCTION; - FunctionPassManager & - operator=(const FunctionPassManager &) LLVM_DELETED_FUNCTION; - - std::vector> Passes; -}; +/// \brief Convenience typedef for a pass manager over functions. +typedef PassManager FunctionPassManager; namespace detail { @@ -560,6 +269,12 @@ namespace detail { /// - invalidateImpl /// /// The details of the call pattern are within. +/// +/// Note that there is also a generic analysis manager template which implements +/// the above required functions along with common datastructures used for +/// managing analyses. This base class is factored so that if you need to +/// customize the handling of a specific IR unit, you can do so without +/// replicating *all* of the boilerplate. template class AnalysisManagerBase { DerivedT *derived_this() { return static_cast(this); } const DerivedT *derived_this() const { @@ -572,7 +287,7 @@ template class AnalysisManagerBase { protected: typedef detail::AnalysisResultConcept ResultConceptT; - typedef detail::AnalysisPassConcept PassConceptT; + typedef detail::AnalysisPassConcept PassConceptT; // FIXME: Provide template aliases for the models when we're using C++11 in // a mode supporting them. @@ -592,7 +307,7 @@ public: /// /// If there is not a valid cached result in the manager already, this will /// re-run the analysis to produce a valid result. - template typename PassT::Result &getResult(IRUnitT IR) { + template typename PassT::Result &getResult(IRUnitT &IR) { assert(AnalysisPasses.count(PassT::ID()) && "This analysis pass was not registered prior to being queried"); @@ -609,7 +324,7 @@ public: /// /// \returns null if there is no cached result. template - typename PassT::Result *getCachedResult(IRUnitT IR) const { + typename PassT::Result *getCachedResult(IRUnitT &IR) const { assert(AnalysisPasses.count(PassT::ID()) && "This analysis pass was not registered prior to being queried"); @@ -631,25 +346,28 @@ public: template void registerPass(PassT Pass) { assert(!AnalysisPasses.count(PassT::ID()) && "Registered the same analysis pass twice!"); - typedef detail::AnalysisPassModel PassModelT; + typedef detail::AnalysisPassModel PassModelT; AnalysisPasses[PassT::ID()].reset(new PassModelT(std::move(Pass))); } /// \brief Invalidate a specific analysis pass for an IR module. /// /// Note that the analysis result can disregard invalidation. - template void invalidate(Module *M) { + template void invalidate(IRUnitT &IR) { assert(AnalysisPasses.count(PassT::ID()) && "This analysis pass was not registered prior to being invalidated"); - derived_this()->invalidateImpl(PassT::ID(), M); + derived_this()->invalidateImpl(PassT::ID(), IR); } /// \brief Invalidate analyses cached for an IR unit. /// /// Walk through all of the analyses pertaining to this unit of IR and /// invalidate them unless they are preserved by the PreservedAnalyses set. - void invalidate(IRUnitT IR, const PreservedAnalyses &PA) { - derived_this()->invalidateImpl(IR, PA); + /// We accept the PreservedAnalyses set by value and update it with each + /// analyis pass which has been successfully invalidated and thus can be + /// preserved going forward. The updated set is returned. + PreservedAnalyses invalidate(IRUnitT &IR, PreservedAnalyses PA) { + return derived_this()->invalidateImpl(IR, std::move(PA)); } protected: @@ -679,108 +397,153 @@ private: } // End namespace detail -/// \brief A module analysis pass manager with lazy running and caching of +/// \brief A generic analysis pass manager with lazy running and caching of /// results. -class ModuleAnalysisManager - : public detail::AnalysisManagerBase { - friend class detail::AnalysisManagerBase; - typedef detail::AnalysisManagerBase BaseT; - typedef BaseT::ResultConceptT ResultConceptT; - typedef BaseT::PassConceptT PassConceptT; - -public: - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - ModuleAnalysisManager() {} - ModuleAnalysisManager(ModuleAnalysisManager &&Arg) - : BaseT(std::move(static_cast(Arg))), - ModuleAnalysisResults(std::move(Arg.ModuleAnalysisResults)) {} - ModuleAnalysisManager &operator=(ModuleAnalysisManager &&RHS) { - BaseT::operator=(std::move(static_cast(RHS))); - ModuleAnalysisResults = std::move(RHS.ModuleAnalysisResults); - return *this; - } - -private: - ModuleAnalysisManager(const ModuleAnalysisManager &) LLVM_DELETED_FUNCTION; - ModuleAnalysisManager & - operator=(const ModuleAnalysisManager &) LLVM_DELETED_FUNCTION; - - /// \brief Get a module pass result, running the pass if necessary. - ResultConceptT &getResultImpl(void *PassID, Module *M); - - /// \brief Get a cached module pass result or return null. - ResultConceptT *getCachedResultImpl(void *PassID, Module *M) const; - - /// \brief Invalidate a module pass result. - void invalidateImpl(void *PassID, Module *M); - - /// \brief Invalidate results across a module. - void invalidateImpl(Module *M, const PreservedAnalyses &PA); - - /// \brief Map type from module analysis pass ID to pass result concept - /// pointer. - typedef DenseMap>> - ModuleAnalysisResultMapT; - - /// \brief Cache of computed module analysis results for this module. - ModuleAnalysisResultMapT ModuleAnalysisResults; -}; - -/// \brief A function analysis manager to coordinate and cache analyses run over -/// a module. -class FunctionAnalysisManager - : public detail::AnalysisManagerBase { - friend class detail::AnalysisManagerBase; - typedef detail::AnalysisManagerBase - BaseT; - typedef BaseT::ResultConceptT ResultConceptT; - typedef BaseT::PassConceptT PassConceptT; +/// +/// This analysis manager can be used for any IR unit where the address of the +/// IR unit sufficies as its identity. It manages the cache for a unit of IR via +/// the address of each unit of IR cached. +template +class AnalysisManager + : public detail::AnalysisManagerBase, IRUnitT> { + friend class detail::AnalysisManagerBase, IRUnitT>; + typedef detail::AnalysisManagerBase, IRUnitT> BaseT; + typedef typename BaseT::ResultConceptT ResultConceptT; + typedef typename BaseT::PassConceptT PassConceptT; public: // Most public APIs are inherited from the CRTP base class. + /// \brief Construct an empty analysis manager. + /// + /// A flag can be passed to indicate that the manager should perform debug + /// logging. + AnalysisManager(bool DebugLogging = false) : DebugLogging(DebugLogging) {} + // We have to explicitly define all the special member functions because MSVC // refuses to generate them. - FunctionAnalysisManager() {} - FunctionAnalysisManager(FunctionAnalysisManager &&Arg) + AnalysisManager(AnalysisManager &&Arg) : BaseT(std::move(static_cast(Arg))), - FunctionAnalysisResults(std::move(Arg.FunctionAnalysisResults)) {} - FunctionAnalysisManager &operator=(FunctionAnalysisManager &&RHS) { + AnalysisResults(std::move(Arg.AnalysisResults)), + DebugLogging(std::move(Arg.DebugLogging)) {} + AnalysisManager &operator=(AnalysisManager &&RHS) { BaseT::operator=(std::move(static_cast(RHS))); - FunctionAnalysisResults = std::move(RHS.FunctionAnalysisResults); + AnalysisResults = std::move(RHS.AnalysisResults); + DebugLogging = std::move(RHS.DebugLogging); return *this; } /// \brief Returns true if the analysis manager has an empty results cache. - bool empty() const; + bool empty() const { + assert(AnalysisResults.empty() == AnalysisResultLists.empty() && + "The storage and index of analysis results disagree on how many " + "there are!"); + return AnalysisResults.empty(); + } - /// \brief Clear the function analysis result cache. + /// \brief Clear the analysis result cache. /// - /// This routine allows cleaning up when the set of functions itself has + /// This routine allows cleaning up when the set of IR units itself has /// potentially changed, and thus we can't even look up a a result and - /// invalidate it directly. Notably, this does *not* call invalidate - /// functions as there is nothing to be done for them. - void clear(); + /// invalidate it directly. Notably, this does *not* call invalidate functions + /// as there is nothing to be done for them. + void clear() { + AnalysisResults.clear(); + AnalysisResultLists.clear(); + } private: - FunctionAnalysisManager(const FunctionAnalysisManager &) - LLVM_DELETED_FUNCTION; - FunctionAnalysisManager & - operator=(const FunctionAnalysisManager &) LLVM_DELETED_FUNCTION; + AnalysisManager(const AnalysisManager &) LLVM_DELETED_FUNCTION; + AnalysisManager &operator=(const AnalysisManager &) LLVM_DELETED_FUNCTION; - /// \brief Get a function pass result, running the pass if necessary. - ResultConceptT &getResultImpl(void *PassID, Function *F); + /// \brief Get an analysis result, running the pass if necessary. + ResultConceptT &getResultImpl(void *PassID, IRUnitT &IR) { + typename AnalysisResultMapT::iterator RI; + bool Inserted; + std::tie(RI, Inserted) = AnalysisResults.insert(std::make_pair( + std::make_pair(PassID, &IR), typename AnalysisResultListT::iterator())); - /// \brief Get a cached function pass result or return null. - ResultConceptT *getCachedResultImpl(void *PassID, Function *F) const; + // If we don't have a cached result for this function, look up the pass and + // run it to produce a result, which we then add to the cache. + if (Inserted) { + auto &P = this->lookupPass(PassID); + if (DebugLogging) + dbgs() << "Running analysis: " << P.name() << "\n"; + AnalysisResultListT &ResultList = AnalysisResultLists[&IR]; + ResultList.emplace_back(PassID, P.run(IR, this)); + RI->second = std::prev(ResultList.end()); + } + + return *RI->second->second; + } + + /// \brief Get a cached analysis result or return null. + ResultConceptT *getCachedResultImpl(void *PassID, IRUnitT &IR) const { + typename AnalysisResultMapT::const_iterator RI = + AnalysisResults.find(std::make_pair(PassID, &IR)); + return RI == AnalysisResults.end() ? nullptr : &*RI->second->second; + } /// \brief Invalidate a function pass result. - void invalidateImpl(void *PassID, Function *F); + void invalidateImpl(void *PassID, IRUnitT &IR) { + typename AnalysisResultMapT::iterator RI = + AnalysisResults.find(std::make_pair(PassID, &IR)); + if (RI == AnalysisResults.end()) + return; + + if (DebugLogging) + dbgs() << "Invalidating analysis: " << this->lookupPass(PassID).name() + << "\n"; + AnalysisResultLists[&IR].erase(RI->second); + AnalysisResults.erase(RI); + } /// \brief Invalidate the results for a function.. - void invalidateImpl(Function *F, const PreservedAnalyses &PA); + PreservedAnalyses invalidateImpl(IRUnitT &IR, PreservedAnalyses PA) { + // Short circuit for a common case of all analyses being preserved. + if (PA.areAllPreserved()) + return std::move(PA); + + if (DebugLogging) + dbgs() << "Invalidating all non-preserved analyses for: " + << IR.getName() << "\n"; + + // Clear all the invalidated results associated specifically with this + // function. + SmallVector InvalidatedPassIDs; + AnalysisResultListT &ResultsList = AnalysisResultLists[&IR]; + for (typename AnalysisResultListT::iterator I = ResultsList.begin(), + E = ResultsList.end(); + I != E;) { + void *PassID = I->first; + + // Pass the invalidation down to the pass itself to see if it thinks it is + // necessary. The analysis pass can return false if no action on the part + // of the analysis manager is required for this invalidation event. + if (I->second->invalidate(IR, PA)) { + if (DebugLogging) + dbgs() << "Invalidating analysis: " << this->lookupPass(PassID).name() + << "\n"; + + InvalidatedPassIDs.push_back(I->first); + I = ResultsList.erase(I); + } else { + ++I; + } + + // After handling each pass, we mark it as preserved. Once we've + // invalidated any stale results, the rest of the system is allowed to + // start preserving this analysis again. + PA.preserve(PassID); + } + while (!InvalidatedPassIDs.empty()) + AnalysisResults.erase( + std::make_pair(InvalidatedPassIDs.pop_back_val(), &IR)); + if (ResultsList.empty()) + AnalysisResultLists.erase(&IR); + + return std::move(PA); + } /// \brief List of function analysis pass IDs and associated concept pointers. /// @@ -788,30 +551,37 @@ private: /// erases. Provides both the pass ID and concept pointer such that it is /// half of a bijection and provides storage for the actual result concept. typedef std::list>>> - FunctionAnalysisResultListT; + void *, std::unique_ptr>>> + AnalysisResultListT; /// \brief Map type from function pointer to our custom list type. - typedef DenseMap - FunctionAnalysisResultListMapT; + typedef DenseMap AnalysisResultListMapT; /// \brief Map from function to a list of function analysis results. /// /// Provides linear time removal of all analysis results for a function and /// the ultimate storage for a particular cached analysis result. - FunctionAnalysisResultListMapT FunctionAnalysisResultLists; + AnalysisResultListMapT AnalysisResultLists; /// \brief Map type from a pair of analysis ID and function pointer to an /// iterator into a particular result list. - typedef DenseMap, - FunctionAnalysisResultListT::iterator> - FunctionAnalysisResultMapT; + typedef DenseMap, + typename AnalysisResultListT::iterator> AnalysisResultMapT; /// \brief Map from an analysis ID and function to a particular cached /// analysis result. - FunctionAnalysisResultMapT FunctionAnalysisResults; + AnalysisResultMapT AnalysisResults; + + /// \brief A flag indicating whether debug logging is enabled. + bool DebugLogging; }; +/// \brief Convenience typedef for the Module analysis manager. +typedef AnalysisManager ModuleAnalysisManager; + +/// \brief Convenience typedef for the Function analysis manager. +typedef AnalysisManager FunctionAnalysisManager; + /// \brief A module analysis which acts as a proxy for a function analysis /// manager. /// @@ -826,6 +596,8 @@ public: static void *ID() { return (void *)&PassID; } + static StringRef name() { return "FunctionAnalysisManagerModuleProxy"; } + explicit FunctionAnalysisManagerModuleProxy(FunctionAnalysisManager &FAM) : FAM(&FAM) {} // We have to explicitly define all the special member functions because MSVC @@ -850,7 +622,7 @@ public: /// In debug builds, it will also assert that the analysis manager is empty /// as no queries should arrive at the function analysis manager prior to /// this analysis being requested. - Result run(Module *M); + Result run(Module &M); private: static char PassID; @@ -888,7 +660,7 @@ public: /// Regardless of whether this analysis is marked as preserved, all of the /// analyses in the \c FunctionAnalysisManager are potentially invalidated /// based on the set of preserved analyses. - bool invalidate(Module *M, const PreservedAnalyses &PA); + bool invalidate(Module &M, const PreservedAnalyses &PA); private: FunctionAnalysisManager *FAM; @@ -924,7 +696,7 @@ public: const ModuleAnalysisManager &getManager() const { return *MAM; } /// \brief Handle invalidation by ignoring it, this pass is immutable. - bool invalidate(Function *) { return false; } + bool invalidate(Function &) { return false; } private: const ModuleAnalysisManager *MAM; @@ -932,6 +704,8 @@ public: static void *ID() { return (void *)&PassID; } + static StringRef name() { return "ModuleAnalysisManagerFunctionProxy"; } + ModuleAnalysisManagerFunctionProxy(const ModuleAnalysisManager &MAM) : MAM(&MAM) {} // We have to explicitly define all the special member functions because MSVC @@ -950,7 +724,7 @@ public: /// \brief Run the analysis pass and create our proxy result object. /// Nothing to see here, it just forwards the \c MAM reference into the /// result. - Result run(Function *) { return Result(*MAM); } + Result run(Function &) { return Result(*MAM); } private: static char PassID; @@ -966,6 +740,20 @@ private: /// \c FunctionAnalysisManagerModuleProxy analysis prior to running the function /// pass over the module to enable a \c FunctionAnalysisManager to be used /// within this run safely. +/// +/// Function passes run within this adaptor can rely on having exclusive access +/// to the function they are run over. They should not read or modify any other +/// functions! Other threads or systems may be manipulating other functions in +/// the module, and so their state should never be relied on. +/// FIXME: Make the above true for all of LLVM's actual passes, some still +/// violate this principle. +/// +/// Function passes can also read the module containing the function, but they +/// should not modify that module outside of the use lists of various globals. +/// For example, a function pass is not permitted to add functions to the +/// module. +/// FIXME: Make the above true for all of LLVM's actual passes, some still +/// violate this principle. template class ModuleToFunctionPassAdaptor { public: explicit ModuleToFunctionPassAdaptor(FunctionPassT Pass) @@ -976,7 +764,8 @@ public: : Pass(Arg.Pass) {} ModuleToFunctionPassAdaptor(ModuleToFunctionPassAdaptor &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(ModuleToFunctionPassAdaptor &LHS, ModuleToFunctionPassAdaptor &RHS) { + friend void swap(ModuleToFunctionPassAdaptor &LHS, + ModuleToFunctionPassAdaptor &RHS) { using std::swap; swap(LHS.Pass, RHS.Pass); } @@ -986,21 +775,23 @@ public: } /// \brief Runs the function pass across every function in the module. - PreservedAnalyses run(Module *M, ModuleAnalysisManager *AM) { + PreservedAnalyses run(Module &M, ModuleAnalysisManager *AM) { FunctionAnalysisManager *FAM = nullptr; if (AM) // Setup the function analysis manager from its proxy. FAM = &AM->getResult(M).getManager(); PreservedAnalyses PA = PreservedAnalyses::all(); - for (Module::iterator I = M->begin(), E = M->end(); I != E; ++I) { - PreservedAnalyses PassPA = Pass.run(I, FAM); + for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) { + PreservedAnalyses PassPA = Pass.run(*I, FAM); // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so - // directly handle the function analysis manager's invalidation here. + // directly handle the function analysis manager's invalidation here and + // update our preserved set to reflect that these have already been + // handled. if (FAM) - FAM->invalidate(I, PassPA); + PassPA = FAM->invalidate(*I, std::move(PassPA)); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -1029,6 +820,66 @@ createModuleToFunctionPassAdaptor(FunctionPassT Pass) { return std::move(ModuleToFunctionPassAdaptor(std::move(Pass))); } +/// \brief A template utility pass to force an analysis result to be available. +/// +/// This is a no-op pass which simply forces a specific analysis pass's result +/// to be available when it is run. +template struct RequireAnalysisPass { + /// \brief Run this pass over some unit of IR. + /// + /// This pass can be run over any unit of IR and use any analysis manager + /// provided they satisfy the basic API requirements. When this pass is + /// created, these methods can be instantiated to satisfy whatever the + /// context requires. + template + PreservedAnalyses run(IRUnitT &Arg, AnalysisManager *AM) { + if (AM) + (void)AM->template getResult(Arg); + + return PreservedAnalyses::all(); + } + + static StringRef name() { return "RequireAnalysisPass"; } +}; + +/// \brief A template utility pass to force an analysis result to be +/// invalidated. +/// +/// This is a no-op pass which simply forces a specific analysis result to be +/// invalidated when it is run. +template struct InvalidateAnalysisPass { + /// \brief Run this pass over some unit of IR. + /// + /// This pass can be run over any unit of IR and use any analysis manager + /// provided they satisfy the basic API requirements. When this pass is + /// created, these methods can be instantiated to satisfy whatever the + /// context requires. + template + PreservedAnalyses run(IRUnitT &Arg, AnalysisManager *AM) { + if (AM) + // We have to directly invalidate the analysis result as we can't + // enumerate all other analyses and use the preserved set to control it. + (void)AM->template invalidate(Arg); + + return PreservedAnalyses::all(); + } + + static StringRef name() { return "InvalidateAnalysisPass"; } +}; + +/// \brief A utility pass that does nothing but preserves no analyses. +/// +/// As a consequence fo not preserving any analyses, this pass will force all +/// analysis passes to be re-run to produce fresh results if any are needed. +struct InvalidateAllAnalysesPass { + /// \brief Run this pass over some unit of IR. + template PreservedAnalyses run(IRUnitT &Arg) { + return PreservedAnalyses::none(); + } + + static StringRef name() { return "InvalidateAllAnalysesPass"; } +}; + } #endif diff --git a/contrib/llvm/include/llvm/IR/PassManagerInternal.h b/contrib/llvm/include/llvm/IR/PassManagerInternal.h new file mode 100644 index 000000000000..297f5f4e07f4 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/PassManagerInternal.h @@ -0,0 +1,349 @@ +//===- PassManager internal APIs and implementation details -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This header provides internal APIs and implementation details used by the +/// pass management interfaces exposed in PassManager.h. To understand more +/// context of why these particular interfaces are needed, see that header +/// file. None of these APIs should be used elsewhere. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_PASSMANAGERINTERNAL_H +#define LLVM_IR_PASSMANAGERINTERNAL_H + +#include "llvm/ADT/StringRef.h" + +namespace llvm { + +template class AnalysisManager; +class PreservedAnalyses; + +/// \brief Implementation details of the pass manager interfaces. +namespace detail { + +/// \brief Template for the abstract base class used to dispatch +/// polymorphically over pass objects. +template struct PassConcept { + // Boiler plate necessary for the container of derived classes. + virtual ~PassConcept() {} + + /// \brief The polymorphic API which runs the pass over a given IR entity. + /// + /// Note that actual pass object can omit the analysis manager argument if + /// desired. Also that the analysis manager may be null if there is no + /// analysis manager in the pass pipeline. + virtual PreservedAnalyses run(IRUnitT &IR, AnalysisManager *AM) = 0; + + /// \brief Polymorphic method to access the name of a pass. + virtual StringRef name() = 0; +}; + +/// \brief SFINAE metafunction for computing whether \c PassT has a run method +/// accepting an \c AnalysisManager. +template +class PassRunAcceptsAnalysisManager { + typedef char SmallType; + struct BigType { + char a, b; + }; + + template *)> + struct Checker; + + template static SmallType f(Checker *); + template static BigType f(...); + +public: + enum { Value = sizeof(f(nullptr)) == sizeof(SmallType) }; +}; + +/// \brief A template wrapper used to implement the polymorphic API. +/// +/// Can be instantiated for any object which provides a \c run method accepting +/// an \c IRUnitT. It requires the pass to be a copyable object. When the +/// \c run method also accepts an \c AnalysisManager*, we pass it +/// along. +template ::Value> +struct PassModel; + +/// \brief Specialization of \c PassModel for passes that accept an analyis +/// manager. +template +struct PassModel + : PassConcept { + explicit PassModel(PassT Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + PassModel(const PassModel &Arg) : Pass(Arg.Pass) {} + PassModel(PassModel &&Arg) : Pass(std::move(Arg.Pass)) {} + friend void swap(PassModel &LHS, PassModel &RHS) { + using std::swap; + swap(LHS.Pass, RHS.Pass); + } + PassModel &operator=(PassModel RHS) { + swap(*this, RHS); + return *this; + } + + PreservedAnalysesT run(IRUnitT &IR, AnalysisManager *AM) override { + return Pass.run(IR, AM); + } + StringRef name() override { return PassT::name(); } + PassT Pass; +}; + +/// \brief Specialization of \c PassModel for passes that accept an analyis +/// manager. +template +struct PassModel + : PassConcept { + explicit PassModel(PassT Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + PassModel(const PassModel &Arg) : Pass(Arg.Pass) {} + PassModel(PassModel &&Arg) : Pass(std::move(Arg.Pass)) {} + friend void swap(PassModel &LHS, PassModel &RHS) { + using std::swap; + swap(LHS.Pass, RHS.Pass); + } + PassModel &operator=(PassModel RHS) { + swap(*this, RHS); + return *this; + } + + PreservedAnalysesT run(IRUnitT &IR, AnalysisManager *AM) override { + return Pass.run(IR); + } + StringRef name() override { return PassT::name(); } + PassT Pass; +}; + +/// \brief Abstract concept of an analysis result. +/// +/// This concept is parameterized over the IR unit that this result pertains +/// to. +template struct AnalysisResultConcept { + virtual ~AnalysisResultConcept() {} + + /// \brief Method to try and mark a result as invalid. + /// + /// When the outer analysis manager detects a change in some underlying + /// unit of the IR, it will call this method on all of the results cached. + /// + /// This method also receives a set of preserved analyses which can be used + /// to avoid invalidation because the pass which changed the underlying IR + /// took care to update or preserve the analysis result in some way. + /// + /// \returns true if the result is indeed invalid (the default). + virtual bool invalidate(IRUnitT &IR, const PreservedAnalyses &PA) = 0; +}; + +/// \brief SFINAE metafunction for computing whether \c ResultT provides an +/// \c invalidate member function. +template class ResultHasInvalidateMethod { + typedef char SmallType; + struct BigType { + char a, b; + }; + + template + struct Checker; + + template static SmallType f(Checker *); + template static BigType f(...); + +public: + enum { Value = sizeof(f(nullptr)) == sizeof(SmallType) }; +}; + +/// \brief Wrapper to model the analysis result concept. +/// +/// By default, this will implement the invalidate method with a trivial +/// implementation so that the actual analysis result doesn't need to provide +/// an invalidation handler. It is only selected when the invalidation handler +/// is not part of the ResultT's interface. +template ::Value> +struct AnalysisResultModel; + +/// \brief Specialization of \c AnalysisResultModel which provides the default +/// invalidate functionality. +template +struct AnalysisResultModel + : AnalysisResultConcept { + explicit AnalysisResultModel(ResultT Result) : Result(std::move(Result)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + AnalysisResultModel(const AnalysisResultModel &Arg) : Result(Arg.Result) {} + AnalysisResultModel(AnalysisResultModel &&Arg) + : Result(std::move(Arg.Result)) {} + friend void swap(AnalysisResultModel &LHS, AnalysisResultModel &RHS) { + using std::swap; + swap(LHS.Result, RHS.Result); + } + AnalysisResultModel &operator=(AnalysisResultModel RHS) { + swap(*this, RHS); + return *this; + } + + /// \brief The model bases invalidation solely on being in the preserved set. + // + // FIXME: We should actually use two different concepts for analysis results + // rather than two different models, and avoid the indirect function call for + // ones that use the trivial behavior. + bool invalidate(IRUnitT &, const PreservedAnalysesT &PA) override { + return !PA.preserved(PassT::ID()); + } + + ResultT Result; +}; + +/// \brief Specialization of \c AnalysisResultModel which delegates invalidate +/// handling to \c ResultT. +template +struct AnalysisResultModel + : AnalysisResultConcept { + explicit AnalysisResultModel(ResultT Result) : Result(std::move(Result)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + AnalysisResultModel(const AnalysisResultModel &Arg) : Result(Arg.Result) {} + AnalysisResultModel(AnalysisResultModel &&Arg) + : Result(std::move(Arg.Result)) {} + friend void swap(AnalysisResultModel &LHS, AnalysisResultModel &RHS) { + using std::swap; + swap(LHS.Result, RHS.Result); + } + AnalysisResultModel &operator=(AnalysisResultModel RHS) { + swap(*this, RHS); + return *this; + } + + /// \brief The model delegates to the \c ResultT method. + bool invalidate(IRUnitT &IR, const PreservedAnalysesT &PA) override { + return Result.invalidate(IR, PA); + } + + ResultT Result; +}; + +/// \brief Abstract concept of an analysis pass. +/// +/// This concept is parameterized over the IR unit that it can run over and +/// produce an analysis result. +template struct AnalysisPassConcept { + virtual ~AnalysisPassConcept() {} + + /// \brief Method to run this analysis over a unit of IR. + /// \returns A unique_ptr to the analysis result object to be queried by + /// users. + virtual std::unique_ptr> + run(IRUnitT &IR, AnalysisManager *AM) = 0; + + /// \brief Polymorphic method to access the name of a pass. + virtual StringRef name() = 0; +}; + +/// \brief Wrapper to model the analysis pass concept. +/// +/// Can wrap any type which implements a suitable \c run method. The method +/// must accept the IRUnitT as an argument and produce an object which can be +/// wrapped in a \c AnalysisResultModel. +template ::Value> +struct AnalysisPassModel; + +/// \brief Specialization of \c AnalysisPassModel which passes an +/// \c AnalysisManager to PassT's run method. +template +struct AnalysisPassModel : AnalysisPassConcept { + explicit AnalysisPassModel(PassT Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + AnalysisPassModel(const AnalysisPassModel &Arg) : Pass(Arg.Pass) {} + AnalysisPassModel(AnalysisPassModel &&Arg) : Pass(std::move(Arg.Pass)) {} + friend void swap(AnalysisPassModel &LHS, AnalysisPassModel &RHS) { + using std::swap; + swap(LHS.Pass, RHS.Pass); + } + AnalysisPassModel &operator=(AnalysisPassModel RHS) { + swap(*this, RHS); + return *this; + } + + // FIXME: Replace PassT::Result with type traits when we use C++11. + typedef AnalysisResultModel + ResultModelT; + + /// \brief The model delegates to the \c PassT::run method. + /// + /// The return is wrapped in an \c AnalysisResultModel. + std::unique_ptr> + run(IRUnitT &IR, AnalysisManager *AM) override { + return make_unique(Pass.run(IR, AM)); + } + + /// \brief The model delegates to a static \c PassT::name method. + /// + /// The returned string ref must point to constant immutable data! + StringRef name() override { return PassT::name(); } + + PassT Pass; +}; + +/// \brief Specialization of \c AnalysisPassModel which does not pass an +/// \c AnalysisManager to PassT's run method. +template +struct AnalysisPassModel : AnalysisPassConcept { + explicit AnalysisPassModel(PassT Pass) : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC + // refuses to generate them. + AnalysisPassModel(const AnalysisPassModel &Arg) : Pass(Arg.Pass) {} + AnalysisPassModel(AnalysisPassModel &&Arg) : Pass(std::move(Arg.Pass)) {} + friend void swap(AnalysisPassModel &LHS, AnalysisPassModel &RHS) { + using std::swap; + swap(LHS.Pass, RHS.Pass); + } + AnalysisPassModel &operator=(AnalysisPassModel RHS) { + swap(*this, RHS); + return *this; + } + + // FIXME: Replace PassT::Result with type traits when we use C++11. + typedef AnalysisResultModel + ResultModelT; + + /// \brief The model delegates to the \c PassT::run method. + /// + /// The return is wrapped in an \c AnalysisResultModel. + std::unique_ptr> + run(IRUnitT &IR, AnalysisManager *) override { + return make_unique(Pass.run(IR)); + } + + /// \brief The model delegates to a static \c PassT::name method. + /// + /// The returned string ref must point to constant immutable data! + StringRef name() override { return PassT::name(); } + + PassT Pass; +}; + +} // End namespace detail +} + +#endif diff --git a/contrib/llvm/include/llvm/IR/PatternMatch.h b/contrib/llvm/include/llvm/IR/PatternMatch.h index 2efb29489473..f94e10576893 100644 --- a/contrib/llvm/include/llvm/IR/PatternMatch.h +++ b/contrib/llvm/include/llvm/IR/PatternMatch.h @@ -32,61 +32,64 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Operator.h" namespace llvm { namespace PatternMatch { -template -bool match(Val *V, const Pattern &P) { - return const_cast(P).match(V); +template bool match(Val *V, const Pattern &P) { + return const_cast(P).match(V); } - -template -struct OneUse_match { +template struct OneUse_match { SubPattern_t SubPattern; OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { return V->hasOneUse() && SubPattern.match(V); } }; -template -inline OneUse_match m_OneUse(const T &SubPattern) { return SubPattern; } +template inline OneUse_match m_OneUse(const T &SubPattern) { + return SubPattern; +} - -template -struct class_match { - template - bool match(ITy *V) { return isa(V); } +template struct class_match { + template bool match(ITy *V) { return isa(V); } }; -/// m_Value() - Match an arbitrary value and ignore it. +/// \brief Match an arbitrary value and ignore it. inline class_match m_Value() { return class_match(); } -/// m_ConstantInt() - Match an arbitrary ConstantInt and ignore it. + +/// \brief Match an arbitrary binary operation and ignore it. +inline class_match m_BinOp() { + return class_match(); +} + +/// \brief Matches any compare instruction and ignore it. +inline class_match m_Cmp() { return class_match(); } + +/// \brief Match an arbitrary ConstantInt and ignore it. inline class_match m_ConstantInt() { return class_match(); } -/// m_Undef() - Match an arbitrary undef constant. + +/// \brief Match an arbitrary undef constant. inline class_match m_Undef() { return class_match(); } +/// \brief Match an arbitrary Constant and ignore it. inline class_match m_Constant() { return class_match(); } /// Matching combinators -template -struct match_combine_or { +template struct match_combine_or { LTy L; RTy R; - match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) { } + match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} - template - bool match(ITy *V) { + template bool match(ITy *V) { if (L.match(V)) return true; if (R.match(V)) @@ -95,15 +98,13 @@ struct match_combine_or { } }; -template -struct match_combine_and { +template struct match_combine_and { LTy L; RTy R; - match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) { } + match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} - template - bool match(ITy *V) { + template bool match(ITy *V) { if (L.match(V)) if (R.match(V)) return true; @@ -112,46 +113,44 @@ struct match_combine_and { }; /// Combine two pattern matchers matching L || R -template +template inline match_combine_or m_CombineOr(const LTy &L, const RTy &R) { return match_combine_or(L, R); } /// Combine two pattern matchers matching L && R -template +template inline match_combine_and m_CombineAnd(const LTy &L, const RTy &R) { return match_combine_and(L, R); } struct match_zero { - template - bool match(ITy *V) { - if (const Constant *C = dyn_cast(V)) + template bool match(ITy *V) { + if (const auto *C = dyn_cast(V)) return C->isNullValue(); return false; } }; -/// m_Zero() - Match an arbitrary zero/null constant. This includes +/// \brief Match an arbitrary zero/null constant. This includes /// zero_initializer for vectors and ConstantPointerNull for pointers. inline match_zero m_Zero() { return match_zero(); } struct match_neg_zero { - template - bool match(ITy *V) { - if (const Constant *C = dyn_cast(V)) + template bool match(ITy *V) { + if (const auto *C = dyn_cast(V)) return C->isNegativeZeroValue(); return false; } }; -/// m_NegZero() - Match an arbitrary zero/null constant. This includes +/// \brief Match an arbitrary zero/null constant. This includes /// zero_initializer for vectors and ConstantPointerNull for pointers. For /// floating point constants, this will match negative zero but not positive /// zero inline match_neg_zero m_NegZero() { return match_neg_zero(); } -/// m_AnyZero() - Match an arbitrary zero/null constant. This includes +/// \brief - Match an arbitrary zero/null constant. This includes /// zero_initializer for vectors and ConstantPointerNull for pointers. For /// floating point constants, this will match negative zero and positive zero inline match_combine_or m_AnyZero() { @@ -161,16 +160,14 @@ inline match_combine_or m_AnyZero() { struct apint_match { const APInt *&Res; apint_match(const APInt *&R) : Res(R) {} - template - bool match(ITy *V) { - if (ConstantInt *CI = dyn_cast(V)) { + template bool match(ITy *V) { + if (auto *CI = dyn_cast(V)) { Res = &CI->getValue(); return true; } if (V->getType()->isVectorTy()) - if (const Constant *C = dyn_cast(V)) - if (ConstantInt *CI = - dyn_cast_or_null(C->getSplatValue())) { + if (const auto *C = dyn_cast(V)) + if (auto *CI = dyn_cast_or_null(C->getSplatValue())) { Res = &CI->getValue(); return true; } @@ -178,16 +175,13 @@ struct apint_match { } }; -/// m_APInt - Match a ConstantInt or splatted ConstantVector, binding the +/// \brief Match a ConstantInt or splatted ConstantVector, binding the /// specified pointer to the contained APInt. inline apint_match m_APInt(const APInt *&Res) { return Res; } - -template -struct constantint_match { - template - bool match(ITy *V) { - if (const ConstantInt *CI = dyn_cast(V)) { +template struct constantint_match { + template bool match(ITy *V) { + if (const auto *CI = dyn_cast(V)) { const APInt &CIV = CI->getValue(); if (Val >= 0) return CIV == static_cast(Val); @@ -200,45 +194,39 @@ struct constantint_match { } }; -/// m_ConstantInt - Match a ConstantInt with a specific value. -template -inline constantint_match m_ConstantInt() { +/// \brief Match a ConstantInt with a specific value. +template inline constantint_match m_ConstantInt() { return constantint_match(); } -/// cst_pred_ty - This helper class is used to match scalar and vector constants -/// that satisfy a specified predicate. -template -struct cst_pred_ty : public Predicate { - template - bool match(ITy *V) { - if (const ConstantInt *CI = dyn_cast(V)) +/// \brief This helper class is used to match scalar and vector constants that +/// satisfy a specified predicate. +template struct cst_pred_ty : public Predicate { + template bool match(ITy *V) { + if (const auto *CI = dyn_cast(V)) return this->isValue(CI->getValue()); if (V->getType()->isVectorTy()) - if (const Constant *C = dyn_cast(V)) - if (const ConstantInt *CI = - dyn_cast_or_null(C->getSplatValue())) + if (const auto *C = dyn_cast(V)) + if (const auto *CI = dyn_cast_or_null(C->getSplatValue())) return this->isValue(CI->getValue()); return false; } }; -/// api_pred_ty - This helper class is used to match scalar and vector constants -/// that satisfy a specified predicate, and bind them to an APInt. -template -struct api_pred_ty : public Predicate { +/// \brief This helper class is used to match scalar and vector constants that +/// satisfy a specified predicate, and bind them to an APInt. +template struct api_pred_ty : public Predicate { const APInt *&Res; api_pred_ty(const APInt *&R) : Res(R) {} - template - bool match(ITy *V) { - if (const ConstantInt *CI = dyn_cast(V)) + template bool match(ITy *V) { + if (const auto *CI = dyn_cast(V)) if (this->isValue(CI->getValue())) { Res = &CI->getValue(); return true; } if (V->getType()->isVectorTy()) - if (const Constant *C = dyn_cast(V)) - if (ConstantInt *CI = dyn_cast_or_null(C->getSplatValue())) + if (const auto *C = dyn_cast(V)) + if (auto *CI = dyn_cast_or_null(C->getSplatValue())) if (this->isValue(CI->getValue())) { Res = &CI->getValue(); return true; @@ -248,12 +236,11 @@ struct api_pred_ty : public Predicate { } }; - struct is_one { bool isValue(const APInt &C) { return C == 1; } }; -/// m_One() - Match an integer 1 or a vector with all elements equal to 1. +/// \brief Match an integer 1 or a vector with all elements equal to 1. inline cst_pred_ty m_One() { return cst_pred_ty(); } inline api_pred_ty m_One(const APInt *&V) { return V; } @@ -261,34 +248,43 @@ struct is_all_ones { bool isValue(const APInt &C) { return C.isAllOnesValue(); } }; -/// m_AllOnes() - Match an integer or vector with all bits set to true. -inline cst_pred_ty m_AllOnes() {return cst_pred_ty();} +/// \brief Match an integer or vector with all bits set to true. +inline cst_pred_ty m_AllOnes() { + return cst_pred_ty(); +} inline api_pred_ty m_AllOnes(const APInt *&V) { return V; } struct is_sign_bit { bool isValue(const APInt &C) { return C.isSignBit(); } }; -/// m_SignBit() - Match an integer or vector with only the sign bit(s) set. -inline cst_pred_ty m_SignBit() {return cst_pred_ty();} +/// \brief Match an integer or vector with only the sign bit(s) set. +inline cst_pred_ty m_SignBit() { + return cst_pred_ty(); +} inline api_pred_ty m_SignBit(const APInt *&V) { return V; } struct is_power2 { bool isValue(const APInt &C) { return C.isPowerOf2(); } }; -/// m_Power2() - Match an integer or vector power of 2. +/// \brief Match an integer or vector power of 2. inline cst_pred_ty m_Power2() { return cst_pred_ty(); } inline api_pred_ty m_Power2(const APInt *&V) { return V; } -template -struct bind_ty { +struct is_maxsignedvalue { + bool isValue(const APInt &C) { return C.isMaxSignedValue(); } +}; + +inline cst_pred_ty m_MaxSignedValue() { return cst_pred_ty(); } +inline api_pred_ty m_MaxSignedValue(const APInt *&V) { return V; } + +template struct bind_ty { Class *&VR; bind_ty(Class *&V) : VR(V) {} - template - bool match(ITy *V) { - if (Class *CV = dyn_cast(V)) { + template bool match(ITy *V) { + if (auto *CV = dyn_cast(V)) { VR = CV; return true; } @@ -296,64 +292,62 @@ struct bind_ty { } }; -/// m_Value - Match a value, capturing it if we match. +/// \brief Match a value, capturing it if we match. inline bind_ty m_Value(Value *&V) { return V; } -/// m_ConstantInt - Match a ConstantInt, capturing the value if we match. +/// \brief Match a binary operator, capturing it if we match. +inline bind_ty m_BinOp(BinaryOperator *&I) { return I; } + +/// \brief Match a ConstantInt, capturing the value if we match. inline bind_ty m_ConstantInt(ConstantInt *&CI) { return CI; } -/// m_Constant - Match a Constant, capturing the value if we match. +/// \brief Match a Constant, capturing the value if we match. inline bind_ty m_Constant(Constant *&C) { return C; } -/// m_ConstantFP - Match a ConstantFP, capturing the value if we match. +/// \brief Match a ConstantFP, capturing the value if we match. inline bind_ty m_ConstantFP(ConstantFP *&C) { return C; } -/// specificval_ty - Match a specified Value*. +/// \brief Match a specified Value*. struct specificval_ty { const Value *Val; specificval_ty(const Value *V) : Val(V) {} - template - bool match(ITy *V) { - return V == Val; - } + template bool match(ITy *V) { return V == Val; } }; -/// m_Specific - Match if we have a specific specified value. +/// \brief Match if we have a specific specified value. inline specificval_ty m_Specific(const Value *V) { return V; } -/// Match a specified floating point value or vector of all elements of that -/// value. +/// \brief Match a specified floating point value or vector of all elements of +/// that value. struct specific_fpval { double Val; specific_fpval(double V) : Val(V) {} - template - bool match(ITy *V) { - if (const ConstantFP *CFP = dyn_cast(V)) + template bool match(ITy *V) { + if (const auto *CFP = dyn_cast(V)) return CFP->isExactlyValue(Val); if (V->getType()->isVectorTy()) - if (const Constant *C = dyn_cast(V)) - if (ConstantFP *CFP = dyn_cast_or_null(C->getSplatValue())) + if (const auto *C = dyn_cast(V)) + if (auto *CFP = dyn_cast_or_null(C->getSplatValue())) return CFP->isExactlyValue(Val); return false; } }; -/// Match a specific floating point value or vector with all elements equal to -/// the value. +/// \brief Match a specific floating point value or vector with all elements +/// equal to the value. inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } -/// Match a float 1.0 or vector with all elements equal to 1.0. +/// \brief Match a float 1.0 or vector with all elements equal to 1.0. inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } struct bind_const_intval_ty { uint64_t &VR; bind_const_intval_ty(uint64_t &V) : VR(V) {} - template - bool match(ITy *V) { - if (ConstantInt *CV = dyn_cast(V)) + template bool match(ITy *V) { + if (const auto *CV = dyn_cast(V)) if (CV->getBitWidth() <= 64) { VR = CV->getZExtValue(); return true; @@ -362,152 +356,196 @@ struct bind_const_intval_ty { } }; -/// m_ConstantInt - Match a ConstantInt and bind to its value. This does not -/// match ConstantInts wider than 64-bits. +/// \brief Match a specified integer value or vector of all elements of that +// value. +struct specific_intval { + uint64_t Val; + specific_intval(uint64_t V) : Val(V) {} + + template bool match(ITy *V) { + const auto *CI = dyn_cast(V); + if (!CI && V->getType()->isVectorTy()) + if (const auto *C = dyn_cast(V)) + CI = dyn_cast_or_null(C->getSplatValue()); + + if (CI && CI->getBitWidth() <= 64) + return CI->getZExtValue() == Val; + + return false; + } +}; + +/// \brief Match a specific integer value or vector with all elements equal to +/// the value. +inline specific_intval m_SpecificInt(uint64_t V) { return specific_intval(V); } + +/// \brief Match a ConstantInt and bind to its value. This does not match +/// ConstantInts wider than 64-bits. inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } +//===----------------------------------------------------------------------===// +// Matcher for any binary operator. +// +template struct AnyBinaryOp_match { + LHS_t L; + RHS_t R; + + AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} + + template bool match(OpTy *V) { + if (auto *I = dyn_cast(V)) + return L.match(I->getOperand(0)) && R.match(I->getOperand(1)); + return false; + } +}; + +template +inline AnyBinaryOp_match m_BinOp(const LHS &L, const RHS &R) { + return AnyBinaryOp_match(L, R); +} + //===----------------------------------------------------------------------===// // Matchers for specific binary operators. // -template +template struct BinaryOp_match { LHS_t L; RHS_t R; BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { if (V->getValueID() == Value::InstructionVal + Opcode) { - BinaryOperator *I = cast(V); + auto *I = cast(V); return L.match(I->getOperand(0)) && R.match(I->getOperand(1)); } - if (ConstantExpr *CE = dyn_cast(V)) + if (auto *CE = dyn_cast(V)) return CE->getOpcode() == Opcode && L.match(CE->getOperand(0)) && R.match(CE->getOperand(1)); return false; } }; -template -inline BinaryOp_match -m_Add(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_Add(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_FAdd(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_FAdd(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_Sub(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_Sub(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_FSub(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_FSub(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_Mul(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_Mul(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_FMul(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_FMul(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_UDiv(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_UDiv(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_SDiv(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_SDiv(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_FDiv(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_FDiv(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_URem(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_URem(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_SRem(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_SRem(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_FRem(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_FRem(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_And(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_And(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_Or(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_Or(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_Xor(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_Xor(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_Shl(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_Shl(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_LShr(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_LShr(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template -inline BinaryOp_match -m_AShr(const LHS &L, const RHS &R) { +template +inline BinaryOp_match m_AShr(const LHS &L, + const RHS &R) { return BinaryOp_match(L, R); } -template +template struct OverflowingBinaryOp_match { LHS_t L; RHS_t R; - OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} + OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) + : L(LHS), R(RHS) {} - template - bool match(OpTy *V) { - if (OverflowingBinaryOperator *Op = dyn_cast(V)) { + template bool match(OpTy *V) { + if (auto *Op = dyn_cast(V)) { if (Op->getOpcode() != Opcode) return false; if (WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap && @@ -591,43 +629,42 @@ m_NUWShl(const LHS &L, const RHS &R) { //===----------------------------------------------------------------------===// // Class that matches two different binary ops. // -template +template struct BinOp2_match { LHS_t L; RHS_t R; BinOp2_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { if (V->getValueID() == Value::InstructionVal + Opc1 || V->getValueID() == Value::InstructionVal + Opc2) { - BinaryOperator *I = cast(V); + auto *I = cast(V); return L.match(I->getOperand(0)) && R.match(I->getOperand(1)); } - if (ConstantExpr *CE = dyn_cast(V)) + if (auto *CE = dyn_cast(V)) return (CE->getOpcode() == Opc1 || CE->getOpcode() == Opc2) && L.match(CE->getOperand(0)) && R.match(CE->getOperand(1)); return false; } }; -/// m_Shr - Matches LShr or AShr. -template +/// \brief Matches LShr or AShr. +template inline BinOp2_match m_Shr(const LHS &L, const RHS &R) { return BinOp2_match(L, R); } -/// m_LogicalShift - Matches LShr or Shl. -template +/// \brief Matches LShr or Shl. +template inline BinOp2_match m_LogicalShift(const LHS &L, const RHS &R) { return BinOp2_match(L, R); } -/// m_IDiv - Matches UDiv and SDiv. -template +/// \brief Matches UDiv and SDiv. +template inline BinOp2_match m_IDiv(const LHS &L, const RHS &R) { return BinOp2_match(L, R); @@ -636,38 +673,36 @@ m_IDiv(const LHS &L, const RHS &R) { //===----------------------------------------------------------------------===// // Class that matches exact binary ops. // -template -struct Exact_match { +template struct Exact_match { SubPattern_t SubPattern; Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { if (PossiblyExactOperator *PEO = dyn_cast(V)) return PEO->isExact() && SubPattern.match(V); return false; } }; -template -inline Exact_match m_Exact(const T &SubPattern) { return SubPattern; } +template inline Exact_match m_Exact(const T &SubPattern) { + return SubPattern; +} //===----------------------------------------------------------------------===// // Matchers for CmpInst classes // -template +template struct CmpClass_match { PredicateTy &Predicate; LHS_t L; RHS_t R; CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) - : Predicate(Pred), L(LHS), R(RHS) {} + : Predicate(Pred), L(LHS), R(RHS) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { if (Class *I = dyn_cast(V)) if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { Predicate = I->getPredicate(); @@ -677,123 +712,114 @@ struct CmpClass_match { } }; -template -inline CmpClass_match -m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { - return CmpClass_match(Pred, L, R); +template +inline CmpClass_match +m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { + return CmpClass_match(Pred, L, R); } -template +template +inline CmpClass_match +m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { + return CmpClass_match(Pred, L, R); +} + +template inline CmpClass_match m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { - return CmpClass_match(Pred, L, R); + return CmpClass_match(Pred, L, R); } //===----------------------------------------------------------------------===// // Matchers for SelectInst classes // -template +template struct SelectClass_match { Cond_t C; LHS_t L; RHS_t R; - SelectClass_match(const Cond_t &Cond, const LHS_t &LHS, - const RHS_t &RHS) - : C(Cond), L(LHS), R(RHS) {} + SelectClass_match(const Cond_t &Cond, const LHS_t &LHS, const RHS_t &RHS) + : C(Cond), L(LHS), R(RHS) {} - template - bool match(OpTy *V) { - if (SelectInst *I = dyn_cast(V)) - return C.match(I->getOperand(0)) && - L.match(I->getOperand(1)) && + template bool match(OpTy *V) { + if (auto *I = dyn_cast(V)) + return C.match(I->getOperand(0)) && L.match(I->getOperand(1)) && R.match(I->getOperand(2)); return false; } }; -template -inline SelectClass_match -m_Select(const Cond &C, const LHS &L, const RHS &R) { +template +inline SelectClass_match m_Select(const Cond &C, const LHS &L, + const RHS &R) { return SelectClass_match(C, L, R); } -/// m_SelectCst - This matches a select of two constants, e.g.: -/// m_SelectCst<-1, 0>(m_Value(V)) -template -inline SelectClass_match, constantint_match > +/// \brief This matches a select of two constants, e.g.: +/// m_SelectCst<-1, 0>(m_Value(V)) +template +inline SelectClass_match, constantint_match> m_SelectCst(const Cond &C) { return m_Select(C, m_ConstantInt(), m_ConstantInt()); } - //===----------------------------------------------------------------------===// // Matchers for CastInst classes // -template -struct CastClass_match { +template struct CastClass_match { Op_t Op; CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} - template - bool match(OpTy *V) { - if (Operator *O = dyn_cast(V)) + template bool match(OpTy *V) { + if (auto *O = dyn_cast(V)) return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); return false; } }; -/// m_BitCast -template -inline CastClass_match -m_BitCast(const OpTy &Op) { +/// \brief Matches BitCast. +template +inline CastClass_match m_BitCast(const OpTy &Op) { return CastClass_match(Op); } -/// m_PtrToInt -template -inline CastClass_match -m_PtrToInt(const OpTy &Op) { +/// \brief Matches PtrToInt. +template +inline CastClass_match m_PtrToInt(const OpTy &Op) { return CastClass_match(Op); } -/// m_Trunc -template -inline CastClass_match -m_Trunc(const OpTy &Op) { +/// \brief Matches Trunc. +template +inline CastClass_match m_Trunc(const OpTy &Op) { return CastClass_match(Op); } -/// m_SExt -template -inline CastClass_match -m_SExt(const OpTy &Op) { +/// \brief Matches SExt. +template +inline CastClass_match m_SExt(const OpTy &Op) { return CastClass_match(Op); } -/// m_ZExt -template -inline CastClass_match -m_ZExt(const OpTy &Op) { +/// \brief Matches ZExt. +template +inline CastClass_match m_ZExt(const OpTy &Op) { return CastClass_match(Op); } -/// m_UIToFP -template -inline CastClass_match -m_UIToFP(const OpTy &Op) { +/// \brief Matches UIToFP. +template +inline CastClass_match m_UIToFP(const OpTy &Op) { return CastClass_match(Op); } -/// m_SIToFP -template -inline CastClass_match -m_SIToFP(const OpTy &Op) { +/// \brief Matches SIToFP. +template +inline CastClass_match m_SIToFP(const OpTy &Op) { return CastClass_match(Op); } @@ -801,46 +827,41 @@ m_SIToFP(const OpTy &Op) { // Matchers for unary operators // -template -struct not_match { +template struct not_match { LHS_t L; not_match(const LHS_t &LHS) : L(LHS) {} - template - bool match(OpTy *V) { - if (Operator *O = dyn_cast(V)) + template bool match(OpTy *V) { + if (auto *O = dyn_cast(V)) if (O->getOpcode() == Instruction::Xor) return matchIfNot(O->getOperand(0), O->getOperand(1)); return false; } + private: bool matchIfNot(Value *LHS, Value *RHS) { return (isa(RHS) || isa(RHS) || // FIXME: Remove CV. isa(RHS)) && - cast(RHS)->isAllOnesValue() && - L.match(LHS); + cast(RHS)->isAllOnesValue() && L.match(LHS); } }; -template -inline not_match m_Not(const LHS &L) { return L; } +template inline not_match m_Not(const LHS &L) { return L; } - -template -struct neg_match { +template struct neg_match { LHS_t L; neg_match(const LHS_t &LHS) : L(LHS) {} - template - bool match(OpTy *V) { - if (Operator *O = dyn_cast(V)) + template bool match(OpTy *V) { + if (auto *O = dyn_cast(V)) if (O->getOpcode() == Instruction::Sub) return matchIfNeg(O->getOperand(0), O->getOperand(1)); return false; } + private: bool matchIfNeg(Value *LHS, Value *RHS) { return ((isa(LHS) && cast(LHS)->isZero()) || @@ -849,36 +870,33 @@ private: } }; -/// m_Neg - Match an integer negate. -template -inline neg_match m_Neg(const LHS &L) { return L; } +/// \brief Match an integer negate. +template inline neg_match m_Neg(const LHS &L) { return L; } - -template -struct fneg_match { +template struct fneg_match { LHS_t L; fneg_match(const LHS_t &LHS) : L(LHS) {} - template - bool match(OpTy *V) { - if (Operator *O = dyn_cast(V)) + template bool match(OpTy *V) { + if (auto *O = dyn_cast(V)) if (O->getOpcode() == Instruction::FSub) return matchIfFNeg(O->getOperand(0), O->getOperand(1)); return false; } + private: bool matchIfFNeg(Value *LHS, Value *RHS) { - if (ConstantFP *C = dyn_cast(LHS)) + if (const auto *C = dyn_cast(LHS)) return C->isNegativeZeroValue() && L.match(RHS); return false; } }; -/// m_FNeg - Match a floating point negate. -template -inline fneg_match m_FNeg(const LHS &L) { return L; } - +/// \brief Match a floating point negate. +template inline fneg_match m_FNeg(const LHS &L) { + return L; +} //===----------------------------------------------------------------------===// // Matchers for control flow. @@ -886,13 +904,10 @@ inline fneg_match m_FNeg(const LHS &L) { return L; } struct br_match { BasicBlock *&Succ; - br_match(BasicBlock *&Succ) - : Succ(Succ) { - } + br_match(BasicBlock *&Succ) : Succ(Succ) {} - template - bool match(OpTy *V) { - if (BranchInst *BI = dyn_cast(V)) + template bool match(OpTy *V) { + if (auto *BI = dyn_cast(V)) if (BI->isUnconditional()) { Succ = BI->getSuccessor(0); return true; @@ -903,17 +918,14 @@ struct br_match { inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } -template -struct brc_match { +template struct brc_match { Cond_t Cond; BasicBlock *&T, *&F; brc_match(const Cond_t &C, BasicBlock *&t, BasicBlock *&f) - : Cond(C), T(t), F(f) { - } + : Cond(C), T(t), F(f) {} - template - bool match(OpTy *V) { - if (BranchInst *BI = dyn_cast(V)) + template bool match(OpTy *V) { + if (auto *BI = dyn_cast(V)) if (BI->isConditional() && Cond.match(BI->getCondition())) { T = BI->getSuccessor(0); F = BI->getSuccessor(1); @@ -923,31 +935,28 @@ struct brc_match { } }; -template +template inline brc_match m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { return brc_match(C, T, F); } - //===----------------------------------------------------------------------===// // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). // -template +template struct MaxMin_match { LHS_t L; RHS_t R; - MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) - : L(LHS), R(RHS) {} + MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". - SelectInst *SI = dyn_cast(V); + auto *SI = dyn_cast(V); if (!SI) return false; - CmpInst_t *Cmp = dyn_cast(SI->getCondition()); + auto *Cmp = dyn_cast(SI->getCondition()); if (!Cmp) return false; // At this point we have a select conditioned on a comparison. Check that @@ -959,8 +968,8 @@ struct MaxMin_match { if ((TrueVal != LHS || FalseVal != RHS) && (TrueVal != RHS || FalseVal != LHS)) return false; - typename CmpInst_t::Predicate Pred = LHS == TrueVal ? - Cmp->getPredicate() : Cmp->getSwappedPredicate(); + typename CmpInst_t::Predicate Pred = + LHS == TrueVal ? Cmp->getPredicate() : Cmp->getSwappedPredicate(); // Does "(x pred y) ? x : y" represent the desired max/min operation? if (!Pred_t::match(Pred)) return false; @@ -969,83 +978,83 @@ struct MaxMin_match { } }; -/// smax_pred_ty - Helper class for identifying signed max predicates. +/// \brief Helper class for identifying signed max predicates. struct smax_pred_ty { static bool match(ICmpInst::Predicate Pred) { return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; } }; -/// smin_pred_ty - Helper class for identifying signed min predicates. +/// \brief Helper class for identifying signed min predicates. struct smin_pred_ty { static bool match(ICmpInst::Predicate Pred) { return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; } }; -/// umax_pred_ty - Helper class for identifying unsigned max predicates. +/// \brief Helper class for identifying unsigned max predicates. struct umax_pred_ty { static bool match(ICmpInst::Predicate Pred) { return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; } }; -/// umin_pred_ty - Helper class for identifying unsigned min predicates. +/// \brief Helper class for identifying unsigned min predicates. struct umin_pred_ty { static bool match(ICmpInst::Predicate Pred) { return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; } }; -/// ofmax_pred_ty - Helper class for identifying ordered max predicates. +/// \brief Helper class for identifying ordered max predicates. struct ofmax_pred_ty { static bool match(FCmpInst::Predicate Pred) { return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; } }; -/// ofmin_pred_ty - Helper class for identifying ordered min predicates. +/// \brief Helper class for identifying ordered min predicates. struct ofmin_pred_ty { static bool match(FCmpInst::Predicate Pred) { return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; } }; -/// ufmax_pred_ty - Helper class for identifying unordered max predicates. +/// \brief Helper class for identifying unordered max predicates. struct ufmax_pred_ty { static bool match(FCmpInst::Predicate Pred) { return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; } }; -/// ufmin_pred_ty - Helper class for identifying unordered min predicates. +/// \brief Helper class for identifying unordered min predicates. struct ufmin_pred_ty { static bool match(FCmpInst::Predicate Pred) { return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; } }; -template -inline MaxMin_match -m_SMax(const LHS &L, const RHS &R) { +template +inline MaxMin_match m_SMax(const LHS &L, + const RHS &R) { return MaxMin_match(L, R); } -template -inline MaxMin_match -m_SMin(const LHS &L, const RHS &R) { +template +inline MaxMin_match m_SMin(const LHS &L, + const RHS &R) { return MaxMin_match(L, R); } -template -inline MaxMin_match -m_UMax(const LHS &L, const RHS &R) { +template +inline MaxMin_match m_UMax(const LHS &L, + const RHS &R) { return MaxMin_match(L, R); } -template -inline MaxMin_match -m_UMin(const LHS &L, const RHS &R) { +template +inline MaxMin_match m_UMin(const LHS &L, + const RHS &R) { return MaxMin_match(L, R); } @@ -1058,9 +1067,9 @@ m_UMin(const LHS &L, const RHS &R) { /// /// max(L, R) iff L and R are not NaN /// m_OrdFMax(L, R) = R iff L or R are NaN -template -inline MaxMin_match -m_OrdFMax(const LHS &L, const RHS &R) { +template +inline MaxMin_match m_OrdFMax(const LHS &L, + const RHS &R) { return MaxMin_match(L, R); } @@ -1073,9 +1082,9 @@ m_OrdFMax(const LHS &L, const RHS &R) { /// /// max(L, R) iff L and R are not NaN /// m_OrdFMin(L, R) = R iff L or R are NaN -template -inline MaxMin_match -m_OrdFMin(const LHS &L, const RHS &R) { +template +inline MaxMin_match m_OrdFMin(const LHS &L, + const RHS &R) { return MaxMin_match(L, R); } @@ -1088,7 +1097,7 @@ m_OrdFMin(const LHS &L, const RHS &R) { /// /// max(L, R) iff L and R are not NaN /// m_UnordFMin(L, R) = L iff L or R are NaN -template +template inline MaxMin_match m_UnordFMax(const LHS &L, const RHS &R) { return MaxMin_match(L, R); @@ -1103,40 +1112,39 @@ m_UnordFMax(const LHS &L, const RHS &R) { /// /// max(L, R) iff L and R are not NaN /// m_UnordFMin(L, R) = L iff L or R are NaN -template +template inline MaxMin_match m_UnordFMin(const LHS &L, const RHS &R) { return MaxMin_match(L, R); } -template -struct Argument_match { +template struct Argument_match { unsigned OpI; Opnd_t Val; - Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) { } + Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} - template - bool match(OpTy *V) { + template bool match(OpTy *V) { CallSite CS(V); return CS.isCall() && Val.match(CS.getArgument(OpI)); } }; -/// Match an argument -template +/// \brief Match an argument. +template inline Argument_match m_Argument(const Opnd_t &Op) { return Argument_match(OpI, Op); } -/// Intrinsic matchers. +/// \brief Intrinsic matchers. struct IntrinsicID_match { unsigned ID; - IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) { } + IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} - template - bool match(OpTy *V) { - IntrinsicInst *II = dyn_cast(V); - return II && II->getIntrinsicID() == ID; + template bool match(OpTy *V) { + if (const auto *CI = dyn_cast(V)) + if (const auto *F = CI->getCalledFunction()) + return F->getIntrinsicID() == ID; + return false; } }; @@ -1147,64 +1155,74 @@ struct IntrinsicID_match { template struct m_Intrinsic_Ty; -template -struct m_Intrinsic_Ty { - typedef match_combine_and > Ty; + typename T9 = void, typename T10 = void> +struct m_Intrinsic_Ty; +template struct m_Intrinsic_Ty { + typedef match_combine_and> Ty; }; -template -struct m_Intrinsic_Ty { - typedef match_combine_and::Ty, - Argument_match > Ty; +template struct m_Intrinsic_Ty { + typedef match_combine_and::Ty, Argument_match> + Ty; }; template struct m_Intrinsic_Ty { typedef match_combine_and::Ty, - Argument_match > Ty; + Argument_match> Ty; }; template struct m_Intrinsic_Ty { typedef match_combine_and::Ty, - Argument_match > Ty; + Argument_match> Ty; }; -/// Match intrinsic calls like this: -/// m_Intrinsic(m_Value(X)) -template -inline IntrinsicID_match -m_Intrinsic() { return IntrinsicID_match(IntrID); } +/// \brief Match intrinsic calls like this: +/// m_Intrinsic(m_Value(X)) +template inline IntrinsicID_match m_Intrinsic() { + return IntrinsicID_match(IntrID); +} -template -inline typename m_Intrinsic_Ty::Ty -m_Intrinsic(const T0 &Op0) { +template +inline typename m_Intrinsic_Ty::Ty m_Intrinsic(const T0 &Op0) { return m_CombineAnd(m_Intrinsic(), m_Argument<0>(Op0)); } -template -inline typename m_Intrinsic_Ty::Ty -m_Intrinsic(const T0 &Op0, const T1 &Op1) { +template +inline typename m_Intrinsic_Ty::Ty m_Intrinsic(const T0 &Op0, + const T1 &Op1) { return m_CombineAnd(m_Intrinsic(Op0), m_Argument<1>(Op1)); } -template +template inline typename m_Intrinsic_Ty::Ty m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { return m_CombineAnd(m_Intrinsic(Op0, Op1), m_Argument<2>(Op2)); } -template +template inline typename m_Intrinsic_Ty::Ty m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { return m_CombineAnd(m_Intrinsic(Op0, Op1, Op2), m_Argument<3>(Op3)); } -// Helper intrinsic matching specializations -template -inline typename m_Intrinsic_Ty::Ty -m_BSwap(const Opnd0 &Op0) { +// Helper intrinsic matching specializations. +template +inline typename m_Intrinsic_Ty::Ty m_BSwap(const Opnd0 &Op0) { return m_Intrinsic(Op0); } +template +inline typename m_Intrinsic_Ty::Ty m_FMin(const Opnd0 &Op0, + const Opnd1 &Op1) { + return m_Intrinsic(Op0, Op1); +} + +template +inline typename m_Intrinsic_Ty::Ty m_FMax(const Opnd0 &Op0, + const Opnd1 &Op1) { + return m_Intrinsic(Op0, Op1); +} + } // end namespace PatternMatch } // end namespace llvm diff --git a/contrib/llvm/include/llvm/IR/PredIteratorCache.h b/contrib/llvm/include/llvm/IR/PredIteratorCache.h index 02bc583a255a..5e1be37805ff 100644 --- a/contrib/llvm/include/llvm/IR/PredIteratorCache.h +++ b/contrib/llvm/include/llvm/IR/PredIteratorCache.h @@ -11,14 +11,14 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_IR_PREDITERATORCACHE_H +#define LLVM_IR_PREDITERATORCACHE_H + #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/CFG.h" #include "llvm/Support/Allocator.h" -#ifndef LLVM_IR_PREDITERATORCACHE_H -#define LLVM_IR_PREDITERATORCACHE_H - namespace llvm { /// PredIteratorCache - This class is an extremely trivial cache for diff --git a/contrib/llvm/include/llvm/IR/Statepoint.h b/contrib/llvm/include/llvm/IR/Statepoint.h new file mode 100644 index 000000000000..e3c4243e9d81 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/Statepoint.h @@ -0,0 +1,215 @@ +//===-- llvm/IR/Statepoint.h - gc.statepoint utilities ------ --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains utility functions and a wrapper class analogous to +// CallSite for accessing the fields of gc.statepoint, gc.relocate, and +// gc.result intrinsics +// +//===----------------------------------------------------------------------===// + +#ifndef __LLVM_IR_STATEPOINT_H +#define __LLVM_IR_STATEPOINT_H + +#include "llvm/ADT/iterator_range.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { + +bool isStatepoint(const ImmutableCallSite &CS); +bool isStatepoint(const Instruction *inst); +bool isStatepoint(const Instruction &inst); + +bool isGCRelocate(const Instruction *inst); +bool isGCRelocate(const ImmutableCallSite &CS); + +bool isGCResult(const Instruction *inst); +bool isGCResult(const ImmutableCallSite &CS); + +/// Analogous to CallSiteBase, this provides most of the actual +/// functionality for Statepoint and ImmutableStatepoint. It is +/// templatized to allow easily specializing of const and non-const +/// concrete subtypes. This is structured analogous to CallSite +/// rather than the IntrinsicInst.h helpers since we want to support +/// invokable statepoints in the near future. +/// TODO: This does not currently allow the if(Statepoint S = ...) +/// idiom used with CallSites. Consider refactoring to support. +template +class StatepointBase { + CallSiteTy StatepointCS; + void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION; + void *operator new(size_t s) LLVM_DELETED_FUNCTION; + + protected: + explicit StatepointBase(InstructionTy *I) : StatepointCS(I) { + assert(isStatepoint(I)); + } + explicit StatepointBase(CallSiteTy CS) : StatepointCS(CS) { + assert(isStatepoint(CS)); + } + + public: + typedef typename CallSiteTy::arg_iterator arg_iterator; + + /// Return the underlying CallSite. + CallSiteTy getCallSite() { + return StatepointCS; + } + + /// Return the value actually being called or invoked. + ValueTy *actualCallee() { + return StatepointCS.getArgument(0); + } + /// Number of arguments to be passed to the actual callee. + int numCallArgs() { + return cast(StatepointCS.getArgument(1))->getZExtValue(); + } + /// Number of additional arguments excluding those intended + /// for garbage collection. + int numTotalVMSArgs() { + return cast(StatepointCS.getArgument(3 + numCallArgs()))->getZExtValue(); + } + + typename CallSiteTy::arg_iterator call_args_begin() { + // 3 = callTarget, #callArgs, flag + int Offset = 3; + assert(Offset <= (int)StatepointCS.arg_size()); + return StatepointCS.arg_begin() + Offset; + } + typename CallSiteTy::arg_iterator call_args_end() { + int Offset = 3 + numCallArgs(); + assert(Offset <= (int)StatepointCS.arg_size()); + return StatepointCS.arg_begin() + Offset; + } + + /// range adapter for call arguments + iterator_range call_args() { + return iterator_range(call_args_begin(), call_args_end()); + } + + typename CallSiteTy::arg_iterator vm_state_begin() { + return call_args_end(); + } + typename CallSiteTy::arg_iterator vm_state_end() { + int Offset = 3 + numCallArgs() + 1 + numTotalVMSArgs(); + assert(Offset <= (int)StatepointCS.arg_size()); + return StatepointCS.arg_begin() + Offset; + } + + /// range adapter for vm state arguments + iterator_range vm_state_args() { + return iterator_range(vm_state_begin(), vm_state_end()); + } + + typename CallSiteTy::arg_iterator first_vm_state_stack_begin() { + // 6 = numTotalVMSArgs, 1st_objectID, 1st_bci, + // 1st_#stack, 1st_#local, 1st_#monitor + return vm_state_begin() + 6; + } + + typename CallSiteTy::arg_iterator gc_args_begin() { + return vm_state_end(); + } + typename CallSiteTy::arg_iterator gc_args_end() { + return StatepointCS.arg_end(); + } + + /// range adapter for gc arguments + iterator_range gc_args() { + return iterator_range(gc_args_begin(), gc_args_end()); + } + + +#ifndef NDEBUG + /// Asserts if this statepoint is malformed. Common cases for failure + /// include incorrect length prefixes for variable length sections or + /// illegal values for parameters. + void verify() { + assert(numCallArgs() >= 0 && + "number of arguments to actually callee can't be negative"); + + // The internal asserts in the iterator accessors do the rest. + (void)call_args_begin(); + (void)call_args_end(); + (void)vm_state_begin(); + (void)vm_state_end(); + (void)gc_args_begin(); + (void)gc_args_end(); + } +#endif +}; + +/// A specialization of it's base class for read only access +/// to a gc.statepoint. +class ImmutableStatepoint + : public StatepointBase { + typedef StatepointBase + Base; + +public: + explicit ImmutableStatepoint(const Instruction *I) : Base(I) {} + explicit ImmutableStatepoint(ImmutableCallSite CS) : Base(CS) {} +}; + +/// A specialization of it's base class for read-write access +/// to a gc.statepoint. +class Statepoint : public StatepointBase { + typedef StatepointBase Base; + +public: + explicit Statepoint(Instruction *I) : Base(I) {} + explicit Statepoint(CallSite CS) : Base(CS) {} +}; + +/// Wraps a call to a gc.relocate and provides access to it's operands. +/// TODO: This should likely be refactored to resememble the wrappers in +/// InstrinsicInst.h. +class GCRelocateOperands { + ImmutableCallSite RelocateCS; + + public: + GCRelocateOperands(const User* U) : RelocateCS(U) { + assert(isGCRelocate(U)); + } + GCRelocateOperands(const Instruction *inst) : RelocateCS(inst) { + assert(isGCRelocate(inst)); + } + GCRelocateOperands(CallSite CS) : RelocateCS(CS) { + assert(isGCRelocate(CS)); + } + + /// The statepoint with which this gc.relocate is associated. + const Instruction *statepoint() { + return cast(RelocateCS.getArgument(0)); + } + /// The index into the associate statepoint's argument list + /// which contains the base pointer of the pointer whose + /// relocation this gc.relocate describes. + int basePtrIndex() { + return cast(RelocateCS.getArgument(1))->getZExtValue(); + } + /// The index into the associate statepoint's argument list which + /// contains the pointer whose relocation this gc.relocate describes. + int derivedPtrIndex() { + return cast(RelocateCS.getArgument(2))->getZExtValue(); + } + Value *basePtr() { + ImmutableCallSite CS(statepoint()); + return *(CS.arg_begin() + basePtrIndex()); + } + Value *derivedPtr() { + ImmutableCallSite CS(statepoint()); + return *(CS.arg_begin() + derivedPtrIndex()); + } +}; +} +#endif diff --git a/contrib/llvm/include/llvm/IR/TrackingMDRef.h b/contrib/llvm/include/llvm/IR/TrackingMDRef.h new file mode 100644 index 000000000000..e24112154e16 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/TrackingMDRef.h @@ -0,0 +1,170 @@ +//===- llvm/IR/TrackingMDRef.h - Tracking Metadata references ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// References to metadata that track RAUW. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_TRACKINGMDREF_H +#define LLVM_IR_TRACKINGMDREF_H + +#include "llvm/IR/MetadataTracking.h" +#include "llvm/Support/Casting.h" + +namespace llvm { + +class Metadata; +class MDNode; +class ValueAsMetadata; + +/// \brief Tracking metadata reference. +/// +/// This class behaves like \a TrackingVH, but for metadata. +class TrackingMDRef { + Metadata *MD; + +public: + TrackingMDRef() : MD(nullptr) {} + explicit TrackingMDRef(Metadata *MD) : MD(MD) { track(); } + + TrackingMDRef(TrackingMDRef &&X) : MD(X.MD) { retrack(X); } + TrackingMDRef(const TrackingMDRef &X) : MD(X.MD) { track(); } + TrackingMDRef &operator=(TrackingMDRef &&X) { + if (&X == this) + return *this; + + untrack(); + MD = X.MD; + retrack(X); + return *this; + } + TrackingMDRef &operator=(const TrackingMDRef &X) { + if (&X == this) + return *this; + + untrack(); + MD = X.MD; + track(); + return *this; + } + ~TrackingMDRef() { untrack(); } + + Metadata *get() const { return MD; } + operator Metadata *() const { return get(); } + Metadata *operator->() const { return get(); } + Metadata &operator*() const { return *get(); } + + void reset() { + untrack(); + MD = nullptr; + } + void reset(Metadata *MD) { + untrack(); + this->MD = MD; + track(); + } + + /// \brief Check whether this has a trivial destructor. + /// + /// If \c MD isn't replaceable, the destructor will be a no-op. + bool hasTrivialDestructor() const { + return !MD || !MetadataTracking::isReplaceable(*MD); + } + + bool operator==(const TrackingMDRef &X) const { return MD == X.MD; } + bool operator!=(const TrackingMDRef &X) const { return MD != X.MD; } + +private: + void track() { + if (MD) + MetadataTracking::track(MD); + } + void untrack() { + if (MD) + MetadataTracking::untrack(MD); + } + void retrack(TrackingMDRef &X) { + assert(MD == X.MD && "Expected values to match"); + if (X.MD) { + MetadataTracking::retrack(X.MD, MD); + X.MD = nullptr; + } + } +}; + +/// \brief Typed tracking ref. +/// +/// Track refererences of a particular type. It's useful to use this for \a +/// MDNode and \a ValueAsMetadata. +template class TypedTrackingMDRef { + TrackingMDRef Ref; + +public: + TypedTrackingMDRef() {} + explicit TypedTrackingMDRef(T *MD) : Ref(static_cast(MD)) {} + + TypedTrackingMDRef(TypedTrackingMDRef &&X) : Ref(std::move(X.Ref)) {} + TypedTrackingMDRef(const TypedTrackingMDRef &X) : Ref(X.Ref) {} + TypedTrackingMDRef &operator=(TypedTrackingMDRef &&X) { + Ref = std::move(X.Ref); + return *this; + } + TypedTrackingMDRef &operator=(const TypedTrackingMDRef &X) { + Ref = X.Ref; + return *this; + } + + T *get() const { return (T *)Ref.get(); } + operator T *() const { return get(); } + T *operator->() const { return get(); } + T &operator*() const { return *get(); } + + bool operator==(const TypedTrackingMDRef &X) const { return Ref == X.Ref; } + bool operator!=(const TypedTrackingMDRef &X) const { return Ref != X.Ref; } + + void reset() { Ref.reset(); } + void reset(T *MD) { Ref.reset(static_cast(MD)); } + + /// \brief Check whether this has a trivial destructor. + bool hasTrivialDestructor() const { return Ref.hasTrivialDestructor(); } +}; + +typedef TypedTrackingMDRef TrackingMDNodeRef; +typedef TypedTrackingMDRef TrackingValueAsMetadataRef; + +// Expose the underlying metadata to casting. +template <> struct simplify_type { + typedef Metadata *SimpleType; + static SimpleType getSimplifiedValue(TrackingMDRef &MD) { return MD.get(); } +}; + +template <> struct simplify_type { + typedef Metadata *SimpleType; + static SimpleType getSimplifiedValue(const TrackingMDRef &MD) { + return MD.get(); + } +}; + +template struct simplify_type> { + typedef T *SimpleType; + static SimpleType getSimplifiedValue(TypedTrackingMDRef &MD) { + return MD.get(); + } +}; + +template struct simplify_type> { + typedef T *SimpleType; + static SimpleType getSimplifiedValue(const TypedTrackingMDRef &MD) { + return MD.get(); + } +}; + +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/IR/Type.h b/contrib/llvm/include/llvm/IR/Type.h index 7955587e3c76..c2073c7750b2 100644 --- a/contrib/llvm/include/llvm/IR/Type.h +++ b/contrib/llvm/include/llvm/IR/Type.h @@ -265,7 +265,7 @@ public: /// get the actual size for a particular target, it is reasonable to use the /// DataLayout subsystem to do this. /// - bool isSized(SmallPtrSet *Visited = nullptr) const { + bool isSized(SmallPtrSetImpl *Visited = nullptr) const { // If it's a primitive, it is always sized. if (getTypeID() == IntegerTyID || isFloatingPointTy() || getTypeID() == PointerTyID || @@ -313,6 +313,9 @@ public: typedef Type * const *subtype_iterator; subtype_iterator subtype_begin() const { return ContainedTys; } subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];} + ArrayRef subtypes() const { + return makeArrayRef(subtype_begin(), subtype_end()); + } typedef std::reverse_iterator subtype_reverse_iterator; subtype_reverse_iterator subtype_rbegin() const { @@ -323,7 +326,7 @@ public: } /// getContainedType - This method is used to implement the type iterator - /// (defined a the end of the file). For derived types, this returns the + /// (defined at the end of the file). For derived types, this returns the /// types 'contained' in the derived type. /// Type *getContainedType(unsigned i) const { @@ -419,7 +422,7 @@ private: /// isSizedDerivedType - Derived types like structures and arrays are sized /// iff all of the members of the type are sized as well. Since asking for /// their size is relatively uncommon, move this operation out of line. - bool isSizedDerivedType(SmallPtrSet *Visited = nullptr) const; + bool isSizedDerivedType(SmallPtrSetImpl *Visited = nullptr) const; }; // Printing of types. diff --git a/contrib/llvm/include/llvm/IR/TypeFinder.h b/contrib/llvm/include/llvm/IR/TypeFinder.h index cea66a4ab069..73a63ad0349e 100644 --- a/contrib/llvm/include/llvm/IR/TypeFinder.h +++ b/contrib/llvm/include/llvm/IR/TypeFinder.h @@ -31,6 +31,7 @@ class TypeFinder { // To avoid walking constant expressions multiple times and other IR // objects, we keep several helper maps. DenseSet VisitedConstants; + DenseSet VisitedMetadata; DenseSet VisitedTypes; std::vector StructTypes; diff --git a/contrib/llvm/include/llvm/IR/UseListOrder.h b/contrib/llvm/include/llvm/IR/UseListOrder.h new file mode 100644 index 000000000000..5df459b1bd30 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/UseListOrder.h @@ -0,0 +1,62 @@ +//===- llvm/IR/UseListOrder.h - LLVM Use List Order -------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file has structures and command-line options for preserving use-list +// order. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_USELISTORDER_H +#define LLVM_IR_USELISTORDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include + +namespace llvm { + +class Module; +class Function; +class Value; + +/// \brief Structure to hold a use-list order. +struct UseListOrder { + const Value *V; + const Function *F; + std::vector Shuffle; + + UseListOrder(const Value *V, const Function *F, size_t ShuffleSize) + : V(V), F(F), Shuffle(ShuffleSize) {} + + UseListOrder() : V(0), F(0) {} + UseListOrder(UseListOrder &&X) + : V(X.V), F(X.F), Shuffle(std::move(X.Shuffle)) {} + UseListOrder &operator=(UseListOrder &&X) { + V = X.V; + F = X.F; + Shuffle = std::move(X.Shuffle); + return *this; + } + +private: + UseListOrder(const UseListOrder &X) LLVM_DELETED_FUNCTION; + UseListOrder &operator=(const UseListOrder &X) LLVM_DELETED_FUNCTION; +}; + +typedef std::vector UseListOrderStack; + +/// \brief Whether to preserve use-list ordering. +bool shouldPreserveBitcodeUseListOrder(); +bool shouldPreserveAssemblyUseListOrder(); +void setPreserveBitcodeUseListOrder(bool ShouldPreserve); +void setPreserveAssemblyUseListOrder(bool ShouldPreserve); + +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/IR/User.h b/contrib/llvm/include/llvm/IR/User.h index 848adae9ceca..f578227d6cae 100644 --- a/contrib/llvm/include/llvm/IR/User.h +++ b/contrib/llvm/include/llvm/IR/User.h @@ -26,9 +26,9 @@ namespace llvm { -/// OperandTraits - Compile-time customization of -/// operand-related allocators and accessors -/// for use of the User class +/// \brief Compile-time customization of User operands. +/// +/// Customizes operand-related allocators and accessors. template struct OperandTraits; @@ -39,11 +39,8 @@ class User : public Value { friend struct HungoffOperandTraits; virtual void anchor(); protected: - /// NumOperands - The number of values used by this User. + /// \brief This is a pointer to the array of Uses for this User. /// - unsigned NumOperands; - - /// OperandList - This is a pointer to the array of Uses for this User. /// For nodes of fixed arity (e.g. a binary operator) this array will live /// prefixed to some derived class instance. For nodes of resizable variable /// arity (e.g. PHINodes, SwitchInst etc.), this memory will be dynamically @@ -52,7 +49,9 @@ protected: void *operator new(size_t s, unsigned Us); User(Type *ty, unsigned vty, Use *OpList, unsigned NumOps) - : Value(ty, vty), NumOperands(NumOps), OperandList(OpList) {} + : Value(ty, vty), OperandList(OpList) { + NumOperands = NumOps; + } Use *allocHungoffUses(unsigned) const; void dropHungoffUses() { Use::zap(OperandList, OperandList + NumOperands, true); @@ -64,13 +63,13 @@ public: ~User() { Use::zap(OperandList, OperandList + NumOperands); } - /// operator delete - free memory allocated for User and Use objects + /// \brief Free memory allocated for User and Use objects. void operator delete(void *Usr); - /// placement delete - required by std, but never called. + /// \brief Placement delete - required by std, but never called. void operator delete(void*, unsigned) { llvm_unreachable("Constructor throws?"); } - /// placement delete - required by std, but never called. + /// \brief Placement delete - required by std, but never called. void operator delete(void*, unsigned, bool) { llvm_unreachable("Constructor throws?"); } @@ -128,8 +127,7 @@ public: return const_op_range(op_begin(), op_end()); } - /// Convenience iterator for directly iterating over the Values in the - /// OperandList + /// \brief Iterator for directly iterating over the operand Values. struct value_op_iterator : iterator_adaptor_base(value_op_begin(), value_op_end()); } - // dropAllReferences() - This function is in charge of "letting go" of all - // objects that this User refers to. This allows one to - // 'delete' a whole class at a time, even though there may be circular - // references... First all references are dropped, and all use counts go to - // zero. Then everything is deleted for real. Note that no operations are - // valid on an object that has "dropped all references", except operator - // delete. - // + /// \brief Drop all references to operands. + /// + /// This function is in charge of "letting go" of all objects that this User + /// refers to. This allows one to 'delete' a whole class at a time, even + /// though there may be circular references... First all references are + /// dropped, and all use counts go to zero. Then everything is deleted for + /// real. Note that no operations are valid on an object that has "dropped + /// all references", except operator delete. void dropAllReferences() { for (Use &U : operands()) U.set(nullptr); } - /// replaceUsesOfWith - Replaces all references to the "From" definition with - /// references to the "To" definition. + /// \brief Replace uses of one Value with another. /// + /// Replaces all references to the "From" definition with references to the + /// "To" definition. void replaceUsesOfWith(Value *From, Value *To); // Methods for support type inquiry through isa, cast, and dyn_cast: diff --git a/contrib/llvm/include/llvm/IR/Value.h b/contrib/llvm/include/llvm/IR/Value.h index b5bbc96eac2a..705fc0f52d58 100644 --- a/contrib/llvm/include/llvm/IR/Value.h +++ b/contrib/llvm/include/llvm/IR/Value.h @@ -37,7 +37,6 @@ class GlobalVariable; class InlineAsm; class Instruction; class LLVMContext; -class MDNode; class Module; class StringRef; class Twine; @@ -53,6 +52,8 @@ typedef StringMapEntry ValueName; // Value Class //===----------------------------------------------------------------------===// +/// \brief LLVM Value Representation +/// /// This is a very important LLVM class. It is the base class of all values /// computed by a program that may be used as operands to other values. Value is /// the super class of other important classes such as Instruction and Function. @@ -64,31 +65,45 @@ typedef StringMapEntry ValueName; /// using this Value. A Value can also have an arbitrary number of ValueHandle /// objects that watch it and listen to RAUW and Destroy events. See /// llvm/IR/ValueHandle.h for details. -/// -/// @brief LLVM Value Representation class Value { Type *VTy; Use *UseList; - friend class ValueSymbolTable; // Allow ValueSymbolTable to directly mod Name. + friend class ValueAsMetadata; // Allow access to NameAndIsUsedByMD. friend class ValueHandleBase; - ValueName *Name; + PointerIntPair NameAndIsUsedByMD; const unsigned char SubclassID; // Subclass identifier (for isa/dyn_cast) unsigned char HasValueHandle : 1; // Has a ValueHandle pointing to this? protected: - /// SubclassOptionalData - This member is similar to SubclassData, however it - /// is for holding information which may be used to aid optimization, but - /// which may be cleared to zero without affecting conservative - /// interpretation. + /// \brief Hold subclass data that can be dropped. + /// + /// This member is similar to SubclassData, however it is for holding + /// information which may be used to aid optimization, but which may be + /// cleared to zero without affecting conservative interpretation. unsigned char SubclassOptionalData : 7; private: - /// SubclassData - This member is defined by this class, but is not used for - /// anything. Subclasses can use it to hold whatever state they find useful. - /// This field is initialized to zero by the ctor. + /// \brief Hold arbitrary subclass data. + /// + /// This member is defined by this class, but is not used for anything. + /// Subclasses can use it to hold whatever state they find useful. This + /// field is initialized to zero by the ctor. unsigned short SubclassData; +protected: + /// \brief The number of operands in the subclass. + /// + /// This member is defined by this class, but not used for anything. + /// Subclasses can use it to store their number of operands, if they have + /// any. + /// + /// This is stored here to save space in User on 64-bit hosts. Since most + /// instances of Value have operands, 32-bit hosts aren't significantly + /// affected. + unsigned NumOperands; + +private: template // UseT == 'Use' or 'const Use' class use_iterator_impl : public std::iterator { @@ -175,6 +190,7 @@ private: Use &getUse() const { return *UI; } /// \brief Return the operand # of this use in its User. + /// /// FIXME: Replace all callers with a direct call to Use::getOperandNo. unsigned getOperandNo() const { return UI->getOperandNo(); } }; @@ -187,15 +203,14 @@ protected: public: virtual ~Value(); - /// dump - Support for debugging, callable in GDB: V->dump() - // + /// \brief Support for debugging, callable in GDB: V->dump() void dump() const; - /// print - Implement operator<< on Value. - /// + /// \brief Implement operator<< on Value. void print(raw_ostream &O) const; /// \brief Print the name of this Value out to the specified raw_ostream. + /// /// This is useful when you just want to print 'int %reg126', not the /// instruction that generated it. If you specify a Module for context, then /// even constanst get pretty-printed; for example, the type of a null @@ -203,40 +218,56 @@ public: void printAsOperand(raw_ostream &O, bool PrintType = true, const Module *M = nullptr) const; - /// All values are typed, get the type of this value. - /// + /// \brief All values are typed, get the type of this value. Type *getType() const { return VTy; } - /// All values hold a context through their type. + /// \brief All values hold a context through their type. LLVMContext &getContext() const; - // All values can potentially be named. - bool hasName() const { return Name != nullptr && SubclassID != MDStringVal; } - ValueName *getValueName() const { return Name; } - void setValueName(ValueName *VN) { Name = VN; } + // \brief All values can potentially be named. + bool hasName() const { return getValueName() != nullptr; } + ValueName *getValueName() const { return NameAndIsUsedByMD.getPointer(); } + void setValueName(ValueName *VN) { NameAndIsUsedByMD.setPointer(VN); } - /// getName() - Return a constant reference to the value's name. This is cheap - /// and guaranteed to return the same reference as long as the value is not - /// modified. +private: + void destroyValueName(); + +public: + /// \brief Return a constant reference to the value's name. + /// + /// This is cheap and guaranteed to return the same reference as long as the + /// value is not modified. StringRef getName() const; - /// setName() - Change the name of the value, choosing a new unique name if - /// the provided name is taken. + /// \brief Change the name of the value. + /// + /// Choose a new unique name if the provided name is taken. /// /// \param Name The new name; or "" if the value's name should be removed. void setName(const Twine &Name); - /// takeName - transfer the name from V to this value, setting V's name to - /// empty. It is an error to call V->takeName(V). + /// \brief Transfer the name from V to this value. + /// + /// After taking V's name, sets V's name to empty. + /// + /// \note It is an error to call V->takeName(V). void takeName(Value *V); - /// replaceAllUsesWith - Go through the uses list for this definition and make - /// each use point to "V" instead of "this". After this completes, 'this's - /// use list is guaranteed to be empty. + /// \brief Change all uses of this to point to a new Value. /// + /// Go through the uses list for this definition and make each use point to + /// "V" instead of "this". After this completes, 'this's use list is + /// guaranteed to be empty. void replaceAllUsesWith(Value *V); + /// replaceUsesOutsideBlock - Go through the uses list for this definition and + /// make each use point to "V" instead of "this" when the use is outside the + /// block. 'This's use list is expected to have at least one element. + /// Unlike replaceAllUsesWith this function does not support basic block + /// values or constant users. + void replaceUsesOutsideBlock(Value *V, BasicBlock *BB); + //---------------------------------------------------------------------- // Methods for handling the chain of uses of this Value. // @@ -255,6 +286,8 @@ public: return iterator_range(use_begin(), use_end()); } + bool user_empty() const { return UseList == nullptr; } + typedef user_iterator_impl user_iterator; typedef user_iterator_impl const_user_iterator; user_iterator user_begin() { return user_iterator(UseList); } @@ -270,36 +303,38 @@ public: return iterator_range(user_begin(), user_end()); } - /// hasOneUse - Return true if there is exactly one user of this value. This - /// is specialized because it is a common request and does not require - /// traversing the whole use list. + /// \brief Return true if there is exactly one user of this value. /// + /// This is specialized because it is a common request and does not require + /// traversing the whole use list. bool hasOneUse() const { const_use_iterator I = use_begin(), E = use_end(); if (I == E) return false; return ++I == E; } - /// hasNUses - Return true if this Value has exactly N users. - /// + /// \brief Return true if this Value has exactly N users. bool hasNUses(unsigned N) const; - /// hasNUsesOrMore - Return true if this value has N users or more. This is - /// logically equivalent to getNumUses() >= N. + /// \brief Return true if this value has N users or more. /// + /// This is logically equivalent to getNumUses() >= N. bool hasNUsesOrMore(unsigned N) const; + /// \brief Check if this value is used in the specified basic block. bool isUsedInBasicBlock(const BasicBlock *BB) const; - /// getNumUses - This method computes the number of uses of this Value. This - /// is a linear time operation. Use hasOneUse, hasNUses, or hasNUsesOrMore - /// to check for specific values. + /// \brief This method computes the number of uses of this Value. + /// + /// This is a linear time operation. Use hasOneUse, hasNUses, or + /// hasNUsesOrMore to check for specific values. unsigned getNumUses() const; - /// addUse - This method should only be used by the Use class. - /// + /// \brief This method should only be used by the Use class. void addUse(Use &U) { U.addToList(&UseList); } + /// \brief Concrete subclass of this. + /// /// An enumeration for keeping track of the concrete subclass of Value that /// is actually instantiated. Values of this enumeration are kept in the /// Value classes SubclassID field. They are used for concrete type @@ -322,8 +357,7 @@ public: ConstantStructVal, // This is an instance of ConstantStruct ConstantVectorVal, // This is an instance of ConstantVector ConstantPointerNullVal, // This is an instance of ConstantPointerNull - MDNodeVal, // This is an instance of MDNode - MDStringVal, // This is an instance of MDString + MetadataAsValueVal, // This is an instance of MetadataAsValue InlineAsmVal, // This is an instance of InlineAsm InstructionVal, // This is an instance of Instruction // Enum values starting at InstructionVal are used for Instructions; @@ -334,11 +368,12 @@ public: ConstantLastVal = ConstantPointerNullVal }; - /// getValueID - Return an ID for the concrete type of this object. This is - /// used to implement the classof checks. This should not be used for any - /// other purpose, as the values may change as LLVM evolves. Also, note that - /// for instructions, the Instruction's opcode is added to InstructionVal. So - /// this means three things: + /// \brief Return an ID for the concrete type of this object. + /// + /// This is used to implement the classof checks. This should not be used + /// for any other purpose, as the values may change as LLVM evolves. Also, + /// note that for instructions, the Instruction's opcode is added to + /// InstructionVal. So this means three things: /// # there is no value with code InstructionVal (no opcode==0). /// # there are more possible values for the value type than in ValueTy enum. /// # the InstructionVal enumerator must be the highest valued enumerator in @@ -347,64 +382,62 @@ public: return SubclassID; } - /// getRawSubclassOptionalData - Return the raw optional flags value - /// contained in this value. This should only be used when testing two - /// Values for equivalence. + /// \brief Return the raw optional flags value contained in this value. + /// + /// This should only be used when testing two Values for equivalence. unsigned getRawSubclassOptionalData() const { return SubclassOptionalData; } - /// clearSubclassOptionalData - Clear the optional flags contained in - /// this value. + /// \brief Clear the optional flags contained in this value. void clearSubclassOptionalData() { SubclassOptionalData = 0; } - /// hasSameSubclassOptionalData - Test whether the optional flags contained - /// in this value are equal to the optional flags in the given value. + /// \brief Check the optional flags for equality. bool hasSameSubclassOptionalData(const Value *V) const { return SubclassOptionalData == V->SubclassOptionalData; } - /// intersectOptionalDataWith - Clear any optional flags in this value - /// that are not also set in the given value. + /// \brief Clear any optional flags not set in the given Value. void intersectOptionalDataWith(const Value *V) { SubclassOptionalData &= V->SubclassOptionalData; } - /// hasValueHandle - Return true if there is a value handle associated with - /// this value. + /// \brief Return true if there is a value handle associated with this value. bool hasValueHandle() const { return HasValueHandle; } - /// \brief Strips off any unneeded pointer casts, all-zero GEPs and aliases - /// from the specified value, returning the original uncasted value. + /// \brief Return true if there is metadata referencing this value. + bool isUsedByMetadata() const { return NameAndIsUsedByMD.getInt(); } + + /// \brief Strip off pointer casts, all-zero GEPs, and aliases. /// - /// If this is called on a non-pointer value, it returns 'this'. + /// Returns the original uncasted value. If this is called on a non-pointer + /// value, it returns 'this'. Value *stripPointerCasts(); const Value *stripPointerCasts() const { return const_cast(this)->stripPointerCasts(); } - /// \brief Strips off any unneeded pointer casts and all-zero GEPs from the - /// specified value, returning the original uncasted value. + /// \brief Strip off pointer casts and all-zero GEPs. /// - /// If this is called on a non-pointer value, it returns 'this'. + /// Returns the original uncasted value. If this is called on a non-pointer + /// value, it returns 'this'. Value *stripPointerCastsNoFollowAliases(); const Value *stripPointerCastsNoFollowAliases() const { return const_cast(this)->stripPointerCastsNoFollowAliases(); } - /// \brief Strips off unneeded pointer casts and all-constant GEPs from the - /// specified value, returning the original pointer value. + /// \brief Strip off pointer casts and all-constant inbounds GEPs. /// - /// If this is called on a non-pointer value, it returns 'this'. + /// Returns the original pointer value. If this is called on a non-pointer + /// value, it returns 'this'. Value *stripInBoundsConstantOffsets(); const Value *stripInBoundsConstantOffsets() const { return const_cast(this)->stripInBoundsConstantOffsets(); } - /// \brief Strips like \c stripInBoundsConstantOffsets but also accumulates - /// the constant offset stripped. + /// \brief Accumulate offsets from \a stripInBoundsConstantOffsets(). /// /// Stores the resulting constant offset stripped into the APInt provided. /// The provided APInt will be extended or truncated as needed to be the @@ -419,23 +452,27 @@ public: ->stripAndAccumulateInBoundsConstantOffsets(DL, Offset); } - /// \brief Strips off unneeded pointer casts and any in-bounds offsets from - /// the specified value, returning the original pointer value. + /// \brief Strip off pointer casts and inbounds GEPs. /// - /// If this is called on a non-pointer value, it returns 'this'. + /// Returns the original pointer value. If this is called on a non-pointer + /// value, it returns 'this'. Value *stripInBoundsOffsets(); const Value *stripInBoundsOffsets() const { return const_cast(this)->stripInBoundsOffsets(); } - /// isDereferenceablePointer - Test if this value is always a pointer to - /// allocated and suitably aligned memory for a simple load or store. + /// \brief Check if this is always a dereferenceable pointer. + /// + /// Test if this value is always a pointer to allocated and suitably aligned + /// memory for a simple load or store. bool isDereferenceablePointer(const DataLayout *DL = nullptr) const; - /// DoPHITranslation - If this value is a PHI node with CurBB as its parent, - /// return the value in the PHI node corresponding to PredBB. If not, return - /// ourself. This is useful if you want to know the value something has in a - /// predecessor block. + /// \brief Translate PHI node to its predecessor from the given basic block. + /// + /// If this value is a PHI node with CurBB as its parent, return the value in + /// the PHI node corresponding to PredBB. If not, return ourself. This is + /// useful if you want to know the value something has in a predecessor + /// block. Value *DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB); const Value *DoPHITranslation(const BasicBlock *CurBB, @@ -443,11 +480,14 @@ public: return const_cast(this)->DoPHITranslation(CurBB, PredBB); } - /// MaximumAlignment - This is the greatest alignment value supported by - /// load, store, and alloca instructions, and global values. + /// \brief The maximum alignment for instructions. + /// + /// This is the greatest alignment value supported by load, store, and alloca + /// instructions, and global values. static const unsigned MaximumAlignment = 1u << 29; - /// mutateType - Mutate the type of this Value to be of the specified type. + /// \brief Mutate the type of this Value to be of the specified type. + /// /// Note that this is an extremely dangerous operation which can create /// completely invalid IR very easily. It is strongly recommended that you /// recreate IR objects with the right types instead of mutating them in @@ -456,6 +496,37 @@ public: VTy = Ty; } + /// \brief Sort the use-list. + /// + /// Sorts the Value's use-list by Cmp using a stable mergesort. Cmp is + /// expected to compare two \a Use references. + template void sortUseList(Compare Cmp); + + /// \brief Reverse the use-list. + void reverseUseList(); + +private: + /// \brief Merge two lists together. + /// + /// Merges \c L and \c R using \c Cmp. To enable stable sorts, always pushes + /// "equal" items from L before items from R. + /// + /// \return the first element in the list. + /// + /// \note Completely ignores \a Use::Prev (doesn't read, doesn't update). + template + static Use *mergeUseLists(Use *L, Use *R, Compare Cmp) { + Use *Merged; + mergeUseListsImpl(L, R, &Merged, Cmp); + return Merged; + } + + /// \brief Tail-recursive helper for \a mergeUseLists(). + /// + /// \param[out] Next the first element in the list. + template + static void mergeUseListsImpl(Use *L, Use *R, Use **Next, Compare Cmp); + protected: unsigned short getSubclassDataFromValue() const { return SubclassData; } void setValueSubclassData(unsigned short D) { SubclassData = D; } @@ -472,6 +543,91 @@ void Use::set(Value *V) { if (V) V->addUse(*this); } +template void Value::sortUseList(Compare Cmp) { + if (!UseList || !UseList->Next) + // No need to sort 0 or 1 uses. + return; + + // Note: this function completely ignores Prev pointers until the end when + // they're fixed en masse. + + // Create a binomial vector of sorted lists, visiting uses one at a time and + // merging lists as necessary. + const unsigned MaxSlots = 32; + Use *Slots[MaxSlots]; + + // Collect the first use, turning it into a single-item list. + Use *Next = UseList->Next; + UseList->Next = nullptr; + unsigned NumSlots = 1; + Slots[0] = UseList; + + // Collect all but the last use. + while (Next->Next) { + Use *Current = Next; + Next = Current->Next; + + // Turn Current into a single-item list. + Current->Next = nullptr; + + // Save Current in the first available slot, merging on collisions. + unsigned I; + for (I = 0; I < NumSlots; ++I) { + if (!Slots[I]) + break; + + // Merge two lists, doubling the size of Current and emptying slot I. + // + // Since the uses in Slots[I] originally preceded those in Current, send + // Slots[I] in as the left parameter to maintain a stable sort. + Current = mergeUseLists(Slots[I], Current, Cmp); + Slots[I] = nullptr; + } + // Check if this is a new slot. + if (I == NumSlots) { + ++NumSlots; + assert(NumSlots <= MaxSlots && "Use list bigger than 2^32"); + } + + // Found an open slot. + Slots[I] = Current; + } + + // Merge all the lists together. + assert(Next && "Expected one more Use"); + assert(!Next->Next && "Expected only one Use"); + UseList = Next; + for (unsigned I = 0; I < NumSlots; ++I) + if (Slots[I]) + // Since the uses in Slots[I] originally preceded those in UseList, send + // Slots[I] in as the left parameter to maintain a stable sort. + UseList = mergeUseLists(Slots[I], UseList, Cmp); + + // Fix the Prev pointers. + for (Use *I = UseList, **Prev = &UseList; I; I = I->Next) { + I->setPrev(Prev); + Prev = &I->Next; + } +} + +template +void Value::mergeUseListsImpl(Use *L, Use *R, Use **Next, Compare Cmp) { + if (!L) { + *Next = R; + return; + } + if (!R) { + *Next = L; + return; + } + if (Cmp(*R, *L)) { + *Next = R; + mergeUseListsImpl(L, R->Next, &R->Next, Cmp); + return; + } + *Next = L; + mergeUseListsImpl(L->Next, R, &L->Next, Cmp); +} // isa - Provide some specializations of isa so that we don't have to include // the subtype header files to test to see if the value is a subclass... @@ -537,12 +693,6 @@ template <> struct isa_impl { } }; -template <> struct isa_impl { - static inline bool doit(const Value &Val) { - return Val.getValueID() == Value::MDNodeVal; - } -}; - // Value* is only 4-byte aligned. template<> class PointerLikeTypeTraits { diff --git a/contrib/llvm/include/llvm/IR/ValueHandle.h b/contrib/llvm/include/llvm/IR/ValueHandle.h index aa29b2ed6906..8fc7fdb03950 100644 --- a/contrib/llvm/include/llvm/IR/ValueHandle.h +++ b/contrib/llvm/include/llvm/IR/ValueHandle.h @@ -33,15 +33,16 @@ public: enum { NumLowBitsAvailable = 2 }; }; -/// ValueHandleBase - This is the common base class of value handles. +/// \brief This is the common base class of value handles. +/// /// ValueHandle's are smart pointers to Value's that have special behavior when /// the value is deleted or ReplaceAllUsesWith'd. See the specific handles /// below for details. -/// class ValueHandleBase { friend class Value; protected: - /// HandleBaseKind - This indicates what sub class the handle actually is. + /// \brief This indicates what sub class the handle actually is. + /// /// This is to avoid having a vtable for the light-weight handle pointers. The /// fully general Callback version does have a vtable. enum HandleBaseKind { @@ -55,55 +56,48 @@ private: PointerIntPair PrevPair; ValueHandleBase *Next; - // A subclass may want to store some information along with the value - // pointer. Allow them to do this by making the value pointer a pointer-int - // pair. The 'setValPtrInt' and 'getValPtrInt' methods below give them this - // access. - PointerIntPair VP; + Value* V; ValueHandleBase(const ValueHandleBase&) LLVM_DELETED_FUNCTION; public: explicit ValueHandleBase(HandleBaseKind Kind) - : PrevPair(nullptr, Kind), Next(nullptr), VP(nullptr, 0) {} + : PrevPair(nullptr, Kind), Next(nullptr), V(nullptr) {} ValueHandleBase(HandleBaseKind Kind, Value *V) - : PrevPair(nullptr, Kind), Next(nullptr), VP(V, 0) { - if (isValid(VP.getPointer())) + : PrevPair(nullptr, Kind), Next(nullptr), V(V) { + if (isValid(V)) AddToUseList(); } ValueHandleBase(HandleBaseKind Kind, const ValueHandleBase &RHS) - : PrevPair(nullptr, Kind), Next(nullptr), VP(RHS.VP) { - if (isValid(VP.getPointer())) + : PrevPair(nullptr, Kind), Next(nullptr), V(RHS.V) { + if (isValid(V)) AddToExistingUseList(RHS.getPrevPtr()); } ~ValueHandleBase() { - if (isValid(VP.getPointer())) + if (isValid(V)) RemoveFromUseList(); } Value *operator=(Value *RHS) { - if (VP.getPointer() == RHS) return RHS; - if (isValid(VP.getPointer())) RemoveFromUseList(); - VP.setPointer(RHS); - if (isValid(VP.getPointer())) AddToUseList(); + if (V == RHS) return RHS; + if (isValid(V)) RemoveFromUseList(); + V = RHS; + if (isValid(V)) AddToUseList(); return RHS; } Value *operator=(const ValueHandleBase &RHS) { - if (VP.getPointer() == RHS.VP.getPointer()) return RHS.VP.getPointer(); - if (isValid(VP.getPointer())) RemoveFromUseList(); - VP.setPointer(RHS.VP.getPointer()); - if (isValid(VP.getPointer())) AddToExistingUseList(RHS.getPrevPtr()); - return VP.getPointer(); + if (V == RHS.V) return RHS.V; + if (isValid(V)) RemoveFromUseList(); + V = RHS.V; + if (isValid(V)) AddToExistingUseList(RHS.getPrevPtr()); + return V; } - Value *operator->() const { return getValPtr(); } - Value &operator*() const { return *getValPtr(); } + Value *operator->() const { return V; } + Value &operator*() const { return *V; } protected: - Value *getValPtr() const { return VP.getPointer(); } - - void setValPtrInt(unsigned K) { VP.setInt(K); } - unsigned getValPtrInt() const { return VP.getInt(); } + Value *getValPtr() const { return V; } static bool isValid(Value *V) { return V && @@ -122,26 +116,28 @@ private: HandleBaseKind getKind() const { return PrevPair.getInt(); } void setPrevPtr(ValueHandleBase **Ptr) { PrevPair.setPointer(Ptr); } - /// AddToExistingUseList - Add this ValueHandle to the use list for VP, where + /// \brief Add this ValueHandle to the use list for V. + /// /// List is the address of either the head of the list or a Next node within /// the existing use list. void AddToExistingUseList(ValueHandleBase **List); - /// AddToExistingUseListAfter - Add this ValueHandle to the use list after - /// Node. + /// \brief Add this ValueHandle to the use list after Node. void AddToExistingUseListAfter(ValueHandleBase *Node); - /// AddToUseList - Add this ValueHandle to the use list for VP. + /// \brief Add this ValueHandle to the use list for V. void AddToUseList(); - /// RemoveFromUseList - Remove this ValueHandle from its current use list. + /// \brief Remove this ValueHandle from its current use list. void RemoveFromUseList(); }; -/// WeakVH - This is a value handle that tries hard to point to a Value, even -/// across RAUW operations, but will null itself out if the value is destroyed. -/// this is useful for advisory sorts of information, but should not be used as -/// the key of a map (since the map would have to rearrange itself when the -/// pointer changes). +/// \brief Value handle that is nullable, but tries to track the Value. +/// +/// This is a value handle that tries hard to point to a Value, even across +/// RAUW operations, but will null itself out if the value is destroyed. this +/// is useful for advisory sorts of information, but should not be used as the +/// key of a map (since the map would have to rearrange itself when the pointer +/// changes). class WeakVH : public ValueHandleBase { public: WeakVH() : ValueHandleBase(Weak) {} @@ -170,14 +166,16 @@ template<> struct simplify_type { } }; -/// AssertingVH - This is a Value Handle that points to a value and asserts out -/// if the value is destroyed while the handle is still live. This is very -/// useful for catching dangling pointer bugs and other things which can be -/// non-obvious. One particularly useful place to use this is as the Key of a -/// map. Dangling pointer bugs often lead to really subtle bugs that only occur -/// if another object happens to get allocated to the same address as the old -/// one. Using an AssertingVH ensures that an assert is triggered as soon as -/// the bad delete occurs. +/// \brief Value handle that asserts if the Value is deleted. +/// +/// This is a Value Handle that points to a value and asserts out if the value +/// is destroyed while the handle is still live. This is very useful for +/// catching dangling pointer bugs and other things which can be non-obvious. +/// One particularly useful place to use this is as the Key of a map. Dangling +/// pointer bugs often lead to really subtle bugs that only occur if another +/// object happens to get allocated to the same address as the old one. Using +/// an AssertingVH ensures that an assert is triggered as soon as the bad +/// delete occurs. /// /// Note that an AssertingVH handle does *not* follow values across RAUW /// operations. This means that RAUW's need to explicitly update the @@ -189,25 +187,23 @@ class AssertingVH : public ValueHandleBase #endif { + friend struct DenseMapInfo >; #ifndef NDEBUG - ValueTy *getValPtr() const { - return static_cast(ValueHandleBase::getValPtr()); - } - void setValPtr(ValueTy *P) { - ValueHandleBase::operator=(GetAsValue(P)); - } + Value *getRawValPtr() const { return ValueHandleBase::getValPtr(); } + void setRawValPtr(Value *P) { ValueHandleBase::operator=(P); } #else - ValueTy *ThePtr; - ValueTy *getValPtr() const { return ThePtr; } - void setValPtr(ValueTy *P) { ThePtr = P; } + Value *ThePtr; + Value *getRawValPtr() const { return ThePtr; } + void setRawValPtr(Value *P) { ThePtr = P; } #endif - - // Convert a ValueTy*, which may be const, to the type the base - // class expects. + // Convert a ValueTy*, which may be const, to the raw Value*. static Value *GetAsValue(Value *V) { return V; } static Value *GetAsValue(const Value *V) { return const_cast(V); } + ValueTy *getValPtr() const { return static_cast(getRawValPtr()); } + void setValPtr(ValueTy *P) { setRawValPtr(GetAsValue(P)); } + public: #ifndef NDEBUG AssertingVH() : ValueHandleBase(Assert) {} @@ -215,7 +211,7 @@ public: AssertingVH(const AssertingVH &RHS) : ValueHandleBase(Assert, RHS) {} #else AssertingVH() : ThePtr(nullptr) {} - AssertingVH(ValueTy *P) : ThePtr(P) {} + AssertingVH(ValueTy *P) : ThePtr(GetAsValue(P)) {} #endif operator ValueTy*() const { @@ -238,21 +234,25 @@ public: // Specialize DenseMapInfo to allow AssertingVH to participate in DenseMap. template struct DenseMapInfo > { - typedef DenseMapInfo PointerInfo; static inline AssertingVH getEmptyKey() { - return AssertingVH(PointerInfo::getEmptyKey()); + AssertingVH Res; + Res.setRawValPtr(DenseMapInfo::getEmptyKey()); + return Res; } - static inline T* getTombstoneKey() { - return AssertingVH(PointerInfo::getTombstoneKey()); + static inline AssertingVH getTombstoneKey() { + AssertingVH Res; + Res.setRawValPtr(DenseMapInfo::getTombstoneKey()); + return Res; } static unsigned getHashValue(const AssertingVH &Val) { - return PointerInfo::getHashValue(Val); + return DenseMapInfo::getHashValue(Val.getRawValPtr()); } static bool isEqual(const AssertingVH &LHS, const AssertingVH &RHS) { - return LHS == RHS; + return DenseMapInfo::isEqual(LHS.getRawValPtr(), + RHS.getRawValPtr()); } }; - + template struct isPodLike > { #ifdef NDEBUG @@ -263,8 +263,7 @@ struct isPodLike > { }; -/// TrackingVH - This is a value handle that tracks a Value (or Value subclass), -/// even across RAUW operations. +/// \brief Value handle that tracks a Value across RAUW. /// /// TrackingVH is designed for situations where a client needs to hold a handle /// to a Value (or subclass) across some operations which may move that value, @@ -332,12 +331,14 @@ public: ValueTy &operator*() const { return *getValPtr(); } }; -/// CallbackVH - This is a value handle that allows subclasses to define -/// callbacks that run when the underlying Value has RAUW called on it or is -/// destroyed. This class can be used as the key of a map, as long as the user -/// takes it out of the map before calling setValPtr() (since the map has to -/// rearrange itself when the pointer changes). Unlike ValueHandleBase, this -/// class has a vtable and a virtual destructor. +/// \brief Value handle with callbacks on RAUW and destruction. +/// +/// This is a value handle that allows subclasses to define callbacks that run +/// when the underlying Value has RAUW called on it or is destroyed. This +/// class can be used as the key of a map, as long as the user takes it out of +/// the map before calling setValPtr() (since the map has to rearrange itself +/// when the pointer changes). Unlike ValueHandleBase, this class has a vtable +/// and a virtual destructor. class CallbackVH : public ValueHandleBase { virtual void anchor(); protected: @@ -358,16 +359,20 @@ public: return getValPtr(); } - /// Called when this->getValPtr() is destroyed, inside ~Value(), so you may - /// call any non-virtual Value method on getValPtr(), but no subclass methods. - /// If WeakVH were implemented as a CallbackVH, it would use this method to - /// call setValPtr(NULL). AssertingVH would use this method to cause an - /// assertion failure. + /// \brief Callback for Value destruction. + /// + /// Called when this->getValPtr() is destroyed, inside ~Value(), so you + /// may call any non-virtual Value method on getValPtr(), but no subclass + /// methods. If WeakVH were implemented as a CallbackVH, it would use this + /// method to call setValPtr(NULL). AssertingVH would use this method to + /// cause an assertion failure. /// /// All implementations must remove the reference from this object to the /// Value that's being destroyed. virtual void deleted() { setValPtr(nullptr); } + /// \brief Callback for Value RAUW. + /// /// Called when this->getValPtr()->replaceAllUsesWith(new_value) is called, /// _before_ any of the uses have actually been replaced. If WeakVH were /// implemented as a CallbackVH, it would use this method to call diff --git a/contrib/llvm/include/llvm/IR/ValueMap.h b/contrib/llvm/include/llvm/IR/ValueMap.h index 43a79c7db2b9..f2ea405f1428 100644 --- a/contrib/llvm/include/llvm/IR/ValueMap.h +++ b/contrib/llvm/include/llvm/IR/ValueMap.h @@ -27,10 +27,13 @@ #define LLVM_IR_VALUEMAP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/IR/TrackingMDRef.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Mutex.h" +#include "llvm/Support/UniqueLock.h" #include "llvm/Support/type_traits.h" #include +#include namespace llvm { @@ -78,8 +81,10 @@ class ValueMap { friend class ValueMapCallbackVH; typedef ValueMapCallbackVH ValueMapCVH; typedef DenseMap > MapT; + typedef DenseMap MDMapT; typedef typename Config::ExtraData ExtraData; MapT Map; + std::unique_ptr MDMap; ExtraData Data; ValueMap(const ValueMap&) LLVM_DELETED_FUNCTION; ValueMap& operator=(const ValueMap&) LLVM_DELETED_FUNCTION; @@ -90,12 +95,19 @@ public: typedef unsigned size_type; explicit ValueMap(unsigned NumInitBuckets = 64) - : Map(NumInitBuckets), Data() {} + : Map(NumInitBuckets), Data() {} explicit ValueMap(const ExtraData &Data, unsigned NumInitBuckets = 64) - : Map(NumInitBuckets), Data(Data) {} + : Map(NumInitBuckets), Data(Data) {} ~ValueMap() {} + bool hasMD() const { return MDMap; } + MDMapT &MD() { + if (!MDMap) + MDMap.reset(new MDMapT); + return *MDMap; + } + typedef ValueMapIterator iterator; typedef ValueMapConstIterator const_iterator; inline iterator begin() { return iterator(Map.begin()); } @@ -109,9 +121,12 @@ public: /// Grow the map so that it has at least Size buckets. Does not shrink void resize(size_t Size) { Map.resize(Size); } - void clear() { Map.clear(); } + void clear() { + Map.clear(); + MDMap.reset(); + } - /// Return 1 if the specified key is in the map, 0 otherwise. + /// Return 1 if the specified key is in the map, 0 otherwise. size_type count(const KeyT &Val) const { return Map.find_as(Val) == Map.end() ? 0 : 1; } @@ -209,6 +224,9 @@ class ValueMapCallbackVH : public CallbackVH { : CallbackVH(const_cast(static_cast(Key))), Map(Map) {} + // Private constructor used to create empty/tombstone DenseMap keys. + ValueMapCallbackVH(Value *V) : CallbackVH(V), Map(nullptr) {} + public: KeyT Unwrap() const { return cast_or_null(getValPtr()); } @@ -216,12 +234,11 @@ public: // Make a copy that won't get changed even when *this is destroyed. ValueMapCallbackVH Copy(*this); typename Config::mutex_type *M = Config::getMutex(Copy.Map->Data); + unique_lock Guard; if (M) - M->acquire(); + Guard = unique_lock(*M); Config::onDelete(Copy.Map->Data, Copy.Unwrap()); // May destroy *this. Copy.Map->Map.erase(Copy); // Definitely destroys *this. - if (M) - M->release(); } void allUsesReplacedWith(Value *new_key) override { assert(isa(new_key) && @@ -229,8 +246,9 @@ public: // Make a copy that won't get changed even when *this is destroyed. ValueMapCallbackVH Copy(*this); typename Config::mutex_type *M = Config::getMutex(Copy.Map->Data); + unique_lock Guard; if (M) - M->acquire(); + Guard = unique_lock(*M); KeyT typed_new_key = cast(new_key); // Can destroy *this: @@ -245,27 +263,24 @@ public: Copy.Map->insert(std::make_pair(typed_new_key, Target)); } } - if (M) - M->release(); } }; template struct DenseMapInfo > { typedef ValueMapCallbackVH VH; - typedef DenseMapInfo PointerInfo; static inline VH getEmptyKey() { - return VH(PointerInfo::getEmptyKey(), nullptr); + return VH(DenseMapInfo::getEmptyKey()); } static inline VH getTombstoneKey() { - return VH(PointerInfo::getTombstoneKey(), nullptr); + return VH(DenseMapInfo::getTombstoneKey()); } static unsigned getHashValue(const VH &Val) { - return PointerInfo::getHashValue(Val.Unwrap()); + return DenseMapInfo::getHashValue(Val.Unwrap()); } static unsigned getHashValue(const KeyT &Val) { - return PointerInfo::getHashValue(Val); + return DenseMapInfo::getHashValue(Val); } static bool isEqual(const VH &LHS, const VH &RHS) { return LHS == RHS; diff --git a/contrib/llvm/include/llvm/IR/Verifier.h b/contrib/llvm/include/llvm/IR/Verifier.h index 0272e206f37a..43bd123e7f44 100644 --- a/contrib/llvm/include/llvm/IR/Verifier.h +++ b/contrib/llvm/include/llvm/IR/Verifier.h @@ -77,8 +77,8 @@ class VerifierPass { public: explicit VerifierPass(bool FatalErrors = true) : FatalErrors(FatalErrors) {} - PreservedAnalyses run(Module *M); - PreservedAnalyses run(Function *F); + PreservedAnalyses run(Module &M); + PreservedAnalyses run(Function &F); static StringRef name() { return "VerifierPass"; } }; diff --git a/contrib/llvm/include/llvm/IRReader/IRReader.h b/contrib/llvm/include/llvm/IRReader/IRReader.h index 59ffc095f47f..2d9ace0b62a0 100644 --- a/contrib/llvm/include/llvm/IRReader/IRReader.h +++ b/contrib/llvm/include/llvm/IRReader/IRReader.h @@ -15,12 +15,12 @@ #ifndef LLVM_IRREADER_IRREADER_H #define LLVM_IRREADER_IRREADER_H +#include "llvm/Support/MemoryBuffer.h" #include namespace llvm { class Module; -class MemoryBuffer; class SMDiagnostic; class LLVMContext; @@ -28,20 +28,21 @@ class LLVMContext; /// for it which does lazy deserialization of function bodies. Otherwise, /// attempt to parse it as LLVM Assembly and return a fully populated /// Module. -Module *getLazyIRFileModule(const std::string &Filename, SMDiagnostic &Err, - LLVMContext &Context); +std::unique_ptr getLazyIRFileModule(StringRef Filename, + SMDiagnostic &Err, + LLVMContext &Context); /// If the given MemoryBuffer holds a bitcode image, return a Module /// for it. Otherwise, attempt to parse it as LLVM Assembly and return -/// a Module for it. This function *never* takes ownership of Buffer. -Module *ParseIR(MemoryBuffer *Buffer, SMDiagnostic &Err, LLVMContext &Context); +/// a Module for it. +std::unique_ptr parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, + LLVMContext &Context); /// If the given file holds a bitcode image, return a Module for it. /// Otherwise, attempt to parse it as LLVM Assembly and return a Module /// for it. -Module *ParseIRFile(const std::string &Filename, SMDiagnostic &Err, - LLVMContext &Context); - +std::unique_ptr parseIRFile(StringRef Filename, SMDiagnostic &Err, + LLVMContext &Context); } #endif diff --git a/contrib/llvm/include/llvm/InitializePasses.h b/contrib/llvm/include/llvm/InitializePasses.h index 02f4259a51ac..30280033ee20 100644 --- a/contrib/llvm/include/llvm/InitializePasses.h +++ b/contrib/llvm/include/llvm/InitializePasses.h @@ -71,8 +71,9 @@ void initializeAliasDebuggerPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlwaysInlinerPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); -void initializeAtomicExpandLoadLinkedPass(PassRegistry&); +void initializeAtomicExpandPass(PassRegistry&); void initializeSampleProfileLoaderPass(PassRegistry&); +void initializeAlignmentFromAssumptionsPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAliasAnalysisPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry &); @@ -89,6 +90,8 @@ void initializeCFGOnlyPrinterPass(PassRegistry&); void initializeCFGOnlyViewerPass(PassRegistry&); void initializeCFGPrinterPass(PassRegistry&); void initializeCFGSimplifyPassPass(PassRegistry&); +void initializeCFLAliasAnalysisPass(PassRegistry&); +void initializeForwardControlFlowIntegrityPass(PassRegistry&); void initializeFlattenCFGPassPass(PassRegistry&); void initializeStructurizeCFGPass(PassRegistry&); void initializeCFGViewerPass(PassRegistry&); @@ -103,7 +106,6 @@ void initializeDAEPass(PassRegistry&); void initializeDAHPass(PassRegistry&); void initializeDCEPass(PassRegistry&); void initializeDSEPass(PassRegistry&); -void initializeDebugIRPass(PassRegistry&); void initializeDebugInfoVerifierLegacyPassPass(PassRegistry &); void initializeDeadInstEliminationPass(PassRegistry&); void initializeDeadMachineInstructionElimPass(PassRegistry&); @@ -119,15 +121,16 @@ void initializeEarlyIfConverterPass(PassRegistry&); void initializeEdgeBundlesPass(PassRegistry&); void initializeExpandPostRAPass(PassRegistry&); void initializeGCOVProfilerPass(PassRegistry&); +void initializeInstrProfilingPass(PassRegistry&); void initializeAddressSanitizerPass(PassRegistry&); void initializeAddressSanitizerModulePass(PassRegistry&); void initializeMemorySanitizerPass(PassRegistry&); void initializeThreadSanitizerPass(PassRegistry&); +void initializeSanitizerCoverageModulePass(PassRegistry&); void initializeDataFlowSanitizerPass(PassRegistry&); void initializeScalarizerPass(PassRegistry&); void initializeEarlyCSEPass(PassRegistry&); void initializeExpandISelPseudosPass(PassRegistry&); -void initializeFindUsedTypesPass(PassRegistry&); void initializeFunctionAttrsPass(PassRegistry&); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); @@ -207,6 +210,7 @@ void initializeObjCARCAPElimPass(PassRegistry&); void initializeObjCARCExpandPass(PassRegistry&); void initializeObjCARCContractPass(PassRegistry&); void initializeObjCARCOptPass(PassRegistry&); +void initializePAEvalPass(PassRegistry &); void initializeOptimizePHIsPass(PassRegistry&); void initializePartiallyInlineLibCallsPass(PassRegistry&); void initializePEIPass(PassRegistry&); @@ -259,10 +263,13 @@ void initializeTailDuplicatePassPass(PassRegistry&); void initializeTargetPassConfigPass(PassRegistry&); void initializeDataLayoutPassPass(PassRegistry &); void initializeTargetTransformInfoAnalysisGroup(PassRegistry&); +void initializeFunctionTargetTransformInfoPass(PassRegistry &); void initializeNoTTIPass(PassRegistry&); void initializeTargetLibraryInfoPass(PassRegistry&); +void initializeAssumptionCacheTrackerPass(PassRegistry &); void initializeTwoAddressInstructionPassPass(PassRegistry&); void initializeTypeBasedAliasAnalysisPass(PassRegistry&); +void initializeScopedNoAliasAAPass(PassRegistry&); void initializeUnifyFunctionExitNodesPass(PassRegistry&); void initializeUnreachableBlockElimPass(PassRegistry&); void initializeUnreachableMachineBlockElimPass(PassRegistry&); @@ -277,7 +284,9 @@ void initializeSLPVectorizerPass(PassRegistry&); void initializeBBVectorizePass(PassRegistry&); void initializeMachineFunctionPrinterPassPass(PassRegistry&); void initializeStackMapLivenessPass(PassRegistry&); +void initializeMachineCombinerPass(PassRegistry &); void initializeLoadCombinePass(PassRegistry&); +void initializeRewriteSymbolsPass(PassRegistry&); } #endif diff --git a/contrib/llvm/include/llvm/LTO/LTOCodeGenerator.h b/contrib/llvm/include/llvm/LTO/LTOCodeGenerator.h index b19b2325f73b..0c9ce4a54f02 100644 --- a/contrib/llvm/include/llvm/LTO/LTOCodeGenerator.h +++ b/contrib/llvm/include/llvm/LTO/LTOCodeGenerator.h @@ -32,8 +32,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LTO_CODE_GENERATOR_H -#define LTO_CODE_GENERATOR_H +#ifndef LLVM_LTO_LTOCODEGENERATOR_H +#define LLVM_LTO_LTOCODEGENERATOR_H #include "llvm-c/lto.h" #include "llvm/ADT/ArrayRef.h" @@ -61,10 +61,11 @@ struct LTOCodeGenerator { static const char *getVersionString(); LTOCodeGenerator(); + LTOCodeGenerator(std::unique_ptr Context); ~LTOCodeGenerator(); // Merge given module, return true on success. - bool addModule(struct LTOModule*, std::string &errMsg); + bool addModule(struct LTOModule *); void setTargetOptions(TargetOptions options); void setDebugInfo(lto_debug_model); @@ -101,6 +102,7 @@ struct LTOCodeGenerator { bool disableOpt, bool disableInline, bool disableGVNLoadPRE, + bool disableVectorization, std::string &errMsg); // As with compile_to_file(), this function compiles the merged module into @@ -112,19 +114,23 @@ struct LTOCodeGenerator { bool disableOpt, bool disableInline, bool disableGVNLoadPRE, + bool disableVectorization, std::string &errMsg); void setDiagnosticHandler(lto_diagnostic_handler_t, void *); + LLVMContext &getContext() { return Context; } + private: void initializeLTOPasses(); bool generateObjectFile(raw_ostream &out, bool disableOpt, bool disableInline, - bool disableGVNLoadPRE, std::string &errMsg); + bool disableGVNLoadPRE, bool disableVectorization, + std::string &errMsg); void applyScopeRestrictions(); - void applyRestriction(GlobalValue &GV, const ArrayRef &Libcalls, + void applyRestriction(GlobalValue &GV, ArrayRef Libcalls, std::vector &MustPreserveList, - SmallPtrSet &AsmUsed, + SmallPtrSetImpl &AsmUsed, Mangler &Mangler); bool determineTarget(std::string &errMsg); @@ -134,6 +140,8 @@ private: typedef StringMap StringSet; + void initialize(); + std::unique_ptr OwnedContext; LLVMContext &Context; Linker IRLinker; TargetMachine *TargetMach; @@ -142,7 +150,7 @@ private: lto_codegen_model CodeModel; StringSet MustPreserveSymbols; StringSet AsmUndefinedRefs; - MemoryBuffer *NativeObjectFile; + std::unique_ptr NativeObjectFile; std::vector CodegenOptions; std::string MCpu; std::string MAttr; @@ -152,4 +160,4 @@ private: void *DiagContext; }; } -#endif // LTO_CODE_GENERATOR_H +#endif diff --git a/contrib/llvm/include/llvm/LTO/LTOModule.h b/contrib/llvm/include/llvm/LTO/LTOModule.h index c43846a64050..53c2b8e521be 100644 --- a/contrib/llvm/include/llvm/LTO/LTOModule.h +++ b/contrib/llvm/include/llvm/LTO/LTOModule.h @@ -11,11 +11,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LTO_MODULE_H -#define LTO_MODULE_H +#ifndef LLVM_LTO_LTOMODULE_H +#define LLVM_LTO_LTOMODULE_H #include "llvm-c/lto.h" #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCObjectFileInfo.h" @@ -37,8 +38,6 @@ namespace llvm { /// struct LTOModule { private: - typedef StringMap StringSet; - struct NameAndAttributes { const char *name; uint32_t attributes; @@ -46,21 +45,27 @@ private: const GlobalValue *symbol; }; + std::unique_ptr OwnedContext; + std::unique_ptr IRFile; std::unique_ptr _target; - StringSet _linkeropt_strings; + StringSet<> _linkeropt_strings; std::vector _deplibs; std::vector _linkeropts; std::vector _symbols; // _defines and _undefines only needed to disambiguate tentative definitions - StringSet _defines; + StringSet<> _defines; StringMap _undefines; std::vector _asm_undefines; LTOModule(std::unique_ptr Obj, TargetMachine *TM); + LTOModule(std::unique_ptr Obj, TargetMachine *TM, + std::unique_ptr Context); public: + ~LTOModule(); + /// Returns 'true' if the file or memory contents is LLVM bitcode. static bool isBitcodeFile(const void *mem, size_t length); static bool isBitcodeFile(const char *path); @@ -71,8 +76,8 @@ public: StringRef triplePrefix); /// Create a MemoryBuffer from a memory range with an optional name. - static MemoryBuffer *makeBuffer(const void *mem, size_t length, - StringRef name = ""); + static std::unique_ptr + makeBuffer(const void *mem, size_t length, StringRef name = ""); /// Create an LTOModule. N.B. These methods take ownership of the buffer. The /// caller must have initialized the Targets, the TargetMCs, the AsmPrinters, @@ -95,6 +100,13 @@ public: TargetOptions options, std::string &errMsg, StringRef path = ""); + static LTOModule *createInLocalContext(const void *mem, size_t length, + TargetOptions options, + std::string &errMsg, StringRef path); + static LTOModule *createInContext(const void *mem, size_t length, + TargetOptions options, std::string &errMsg, + StringRef path, LLVMContext *Context); + const Module &getModule() const { return const_cast(this)->getModule(); } @@ -202,10 +214,9 @@ private: /// Get string that the data pointer points to. bool objcClassNameFromExpression(const Constant *c, std::string &name); - /// Create an LTOModule (private version). N.B. This method takes ownership of - /// the buffer. - static LTOModule *makeLTOModule(std::unique_ptr Buffer, - TargetOptions options, std::string &errMsg); + /// Create an LTOModule (private version). + static LTOModule *makeLTOModule(MemoryBufferRef Buffer, TargetOptions options, + std::string &errMsg, LLVMContext *Context); }; } -#endif // LTO_MODULE_H +#endif diff --git a/contrib/llvm/include/llvm/LinkAllPasses.h b/contrib/llvm/include/llvm/LinkAllPasses.h index e06560ca0b64..2e8feab6d29d 100644 --- a/contrib/llvm/include/llvm/LinkAllPasses.h +++ b/contrib/llvm/include/llvm/LinkAllPasses.h @@ -18,7 +18,6 @@ #include "llvm/Analysis/AliasSetTracker.h" #include "llvm/Analysis/CallPrinter.h" #include "llvm/Analysis/DomPrinter.h" -#include "llvm/Analysis/FindUsedTypes.h" #include "llvm/Analysis/IntervalPartition.h" #include "llvm/Analysis/Lint.h" #include "llvm/Analysis/Passes.h" @@ -33,6 +32,7 @@ #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Utils/SymbolRewriter.h" #include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h" #include "llvm/Transforms/Vectorize.h" #include @@ -52,15 +52,18 @@ namespace { (void) llvm::createAliasAnalysisCounterPass(); (void) llvm::createAliasDebugger(); (void) llvm::createArgumentPromotionPass(); + (void) llvm::createAlignmentFromAssumptionsPass(); (void) llvm::createBasicAliasAnalysisPass(); (void) llvm::createLibCallAliasAnalysisPass(nullptr); (void) llvm::createScalarEvolutionAliasAnalysisPass(); (void) llvm::createTypeBasedAliasAnalysisPass(); + (void) llvm::createScopedNoAliasAAPass(); (void) llvm::createBoundsCheckingPass(); (void) llvm::createBreakCriticalEdgesPass(); (void) llvm::createCallGraphPrinterPass(); (void) llvm::createCallGraphViewerPass(); (void) llvm::createCFGSimplificationPass(); + (void) llvm::createCFLAliasAnalysisPass(); (void) llvm::createStructurizeCFGPass(); (void) llvm::createConstantMergePass(); (void) llvm::createConstantPropagationPass(); @@ -75,6 +78,7 @@ namespace { (void) llvm::createDomOnlyViewerPass(); (void) llvm::createDomViewerPass(); (void) llvm::createGCOVProfilerPass(); + (void) llvm::createInstrProfilingPass(); (void) llvm::createFunctionInliningPass(); (void) llvm::createAlwaysInlinerPass(); (void) llvm::createGlobalDCEPass(); @@ -107,6 +111,7 @@ namespace { (void) llvm::createObjCARCExpandPass(); (void) llvm::createObjCARCContractPass(); (void) llvm::createObjCARCOptPass(); + (void) llvm::createPAEvalPass(); (void) llvm::createPromoteMemoryToRegisterPass(); (void) llvm::createDemoteRegisterToMemoryPass(); (void) llvm::createPruneEHPass(); @@ -134,7 +139,7 @@ namespace { (void) llvm::createConstantHoistingPass(); (void) llvm::createCodeGenPreparePass(); (void) llvm::createEarlyCSEPass(); - (void)llvm::createMergedLoadStoreMotionPass(); + (void) llvm::createMergedLoadStoreMotionPass(); (void) llvm::createGVNPass(); (void) llvm::createMemCpyOptPass(); (void) llvm::createLoopDeletionPass(); @@ -160,15 +165,15 @@ namespace { (void) llvm::createPartiallyInlineLibCallsPass(); (void) llvm::createScalarizerPass(); (void) llvm::createSeparateConstOffsetFromGEPPass(); + (void) llvm::createRewriteSymbolsPass(); (void)new llvm::IntervalPartition(); - (void)new llvm::FindUsedTypes(); (void)new llvm::ScalarEvolution(); ((llvm::Function*)nullptr)->viewCFGOnly(); llvm::RGPassManager RGM; ((llvm::RegionPass*)nullptr)->runOnRegion((llvm::Region*)nullptr, RGM); llvm::AliasSetTracker X(*(llvm::AliasAnalysis*)nullptr); - X.add((llvm::Value*)nullptr, 0, nullptr); // for -print-alias-sets + X.add(nullptr, 0, llvm::AAMDNodes()); // for -print-alias-sets } } ForcePassLinking; // Force link by creating a global definition. } diff --git a/contrib/llvm/include/llvm/Linker/Linker.h b/contrib/llvm/include/llvm/Linker/Linker.h index 6254bbb6d6d5..9c3ecea590bd 100644 --- a/contrib/llvm/include/llvm/Linker/Linker.h +++ b/contrib/llvm/include/llvm/Linker/Linker.h @@ -10,52 +10,78 @@ #ifndef LLVM_LINKER_LINKER_H #define LLVM_LINKER_LINKER_H -#include "llvm/ADT/SmallPtrSet.h" -#include +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/IR/DiagnosticInfo.h" namespace llvm { - -class Comdat; -class GlobalValue; class Module; -class StringRef; class StructType; +class Type; /// This class provides the core functionality of linking in LLVM. It keeps a /// pointer to the merged module so far. It doesn't take ownership of the /// module since it is assumed that the user of this class will want to do /// something with it after the linking. class Linker { - public: - enum LinkerMode { - DestroySource = 0, // Allow source module to be destroyed. - PreserveSource = 1 // Preserve the source module. +public: + struct StructTypeKeyInfo { + struct KeyTy { + ArrayRef ETypes; + bool IsPacked; + KeyTy(ArrayRef E, bool P); + KeyTy(const StructType *ST); + bool operator==(const KeyTy &that) const; + bool operator!=(const KeyTy &that) const; }; + static StructType *getEmptyKey(); + static StructType *getTombstoneKey(); + static unsigned getHashValue(const KeyTy &Key); + static unsigned getHashValue(const StructType *ST); + static bool isEqual(const KeyTy &LHS, const StructType *RHS); + static bool isEqual(const StructType *LHS, const StructType *RHS); + }; - Linker(Module *M, bool SuppressWarnings=false); - ~Linker(); + typedef DenseSet NonOpaqueStructTypeSet; + typedef DenseSet OpaqueStructTypeSet; - Module *getModule() const { return Composite; } - void deleteModule(); + struct IdentifiedStructTypeSet { + // The set of opaque types is the composite module. + OpaqueStructTypeSet OpaqueStructTypes; - /// \brief Link \p Src into the composite. The source is destroyed if - /// \p Mode is DestroySource and preserved if it is PreserveSource. - /// If \p ErrorMsg is not null, information about any error is written - /// to it. - /// Returns true on error. - bool linkInModule(Module *Src, unsigned Mode, std::string *ErrorMsg); - bool linkInModule(Module *Src, std::string *ErrorMsg) { - return linkInModule(Src, Linker::DestroySource, ErrorMsg); - } + // The set of identified but non opaque structures in the composite module. + NonOpaqueStructTypeSet NonOpaqueStructTypes; - static bool LinkModules(Module *Dest, Module *Src, unsigned Mode, - std::string *ErrorMsg); + void addNonOpaque(StructType *Ty); + void addOpaque(StructType *Ty); + StructType *findNonOpaque(ArrayRef ETypes, bool IsPacked); + bool hasType(StructType *Ty); + }; - private: - Module *Composite; - SmallPtrSet IdentifiedStructTypes; + Linker(Module *M, DiagnosticHandlerFunction DiagnosticHandler); + Linker(Module *M); + ~Linker(); - bool SuppressWarnings; + Module *getModule() const { return Composite; } + void deleteModule(); + + /// \brief Link \p Src into the composite. The source is destroyed. + /// Returns true on error. + bool linkInModule(Module *Src); + + static bool LinkModules(Module *Dest, Module *Src, + DiagnosticHandlerFunction DiagnosticHandler); + + static bool LinkModules(Module *Dest, Module *Src); + +private: + void init(Module *M, DiagnosticHandlerFunction DiagnosticHandler); + Module *Composite; + + IdentifiedStructTypeSet IdentifiedStructTypes; + + DiagnosticHandlerFunction DiagnosticHandler; }; } // End llvm namespace diff --git a/contrib/llvm/include/llvm/MC/ConstantPools.h b/contrib/llvm/include/llvm/MC/ConstantPools.h index 2e76074db774..1fc0332f63c2 100644 --- a/contrib/llvm/include/llvm/MC/ConstantPools.h +++ b/contrib/llvm/include/llvm/MC/ConstantPools.h @@ -12,10 +12,12 @@ //===----------------------------------------------------------------------===// -#ifndef LLVM_MC_CONSTANTPOOL_H -#define LLVM_MC_CONSTANTPOOL_H +#ifndef LLVM_MC_CONSTANTPOOLS_H +#define LLVM_MC_CONSTANTPOOLS_H +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" + namespace llvm { class MCContext; class MCExpr; diff --git a/contrib/llvm/include/llvm/MC/MCAnalysis/MCAtom.h b/contrib/llvm/include/llvm/MC/MCAnalysis/MCAtom.h deleted file mode 100644 index 33f3431a59a9..000000000000 --- a/contrib/llvm/include/llvm/MC/MCAnalysis/MCAtom.h +++ /dev/null @@ -1,199 +0,0 @@ -//===-- MCAtom.h ------------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains the declaration of the MCAtom class, which is used to -// represent a contiguous region in a decoded object that is uniformly data or -// instructions. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCANALYSIS_MCATOM_H -#define LLVM_MC_MCANALYSIS_MCATOM_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/MC/MCInst.h" -#include "llvm/Support/DataTypes.h" -#include - -namespace llvm { - -class MCModule; - -class MCAtom; -class MCTextAtom; -class MCDataAtom; - -/// \brief Represents a contiguous range of either instructions (a TextAtom) -/// or data (a DataAtom). Address ranges are expressed as _closed_ intervals. -class MCAtom { - virtual void anchor(); -public: - virtual ~MCAtom() {} - - enum AtomKind { TextAtom, DataAtom }; - AtomKind getKind() const { return Kind; } - - /// \brief Get the start address of the atom. - uint64_t getBeginAddr() const { return Begin; } - /// \brief Get the end address, i.e. the last one inside the atom. - uint64_t getEndAddr() const { return End; } - - /// \name Atom modification methods: - /// When modifying a TextAtom, keep instruction boundaries in mind. - /// For instance, split must me given the start address of an instruction. - /// @{ - - /// \brief Splits the atom in two at a given address. - /// \param SplitPt Address at which to start a new atom, splitting this one. - /// \returns The newly created atom starting at \p SplitPt. - virtual MCAtom *split(uint64_t SplitPt) = 0; - - /// \brief Truncates an atom, discarding everything after \p TruncPt. - /// \param TruncPt Last byte address to be contained in this atom. - virtual void truncate(uint64_t TruncPt) = 0; - /// @} - - /// \name Naming: - /// - /// This is mostly for display purposes, and may contain anything that hints - /// at what the atom contains: section or symbol name, BB start address, .. - /// @{ - StringRef getName() const { return Name; } - void setName(StringRef NewName) { Name = NewName.str(); } - /// @} - -protected: - const AtomKind Kind; - std::string Name; - MCModule *Parent; - uint64_t Begin, End; - - friend class MCModule; - MCAtom(AtomKind K, MCModule *P, uint64_t B, uint64_t E) - : Kind(K), Name("(unknown)"), Parent(P), Begin(B), End(E) { } - - /// \name Atom remapping helpers - /// @{ - - /// \brief Remap the atom, using the given range, updating Begin/End. - /// One or both of the bounds can remain the same, but overlapping with other - /// atoms in the module is still forbidden. - void remap(uint64_t NewBegin, uint64_t NewEnd); - - /// \brief Remap the atom to prepare for a truncation at TruncPt. - /// Equivalent to: - /// \code - /// // Bound checks - /// remap(Begin, TruncPt); - /// \endcode - void remapForTruncate(uint64_t TruncPt); - - /// \brief Remap the atom to prepare for a split at SplitPt. - /// The bounds for the resulting atoms are returned in {L,R}{Begin,End}. - /// The current atom is truncated to \p LEnd. - void remapForSplit(uint64_t SplitPt, - uint64_t &LBegin, uint64_t &LEnd, - uint64_t &RBegin, uint64_t &REnd); - /// @} -}; - -/// \name Text atom -/// @{ - -/// \brief An entry in an MCTextAtom: a disassembled instruction. -/// NOTE: Both the Address and Size field are actually redundant when taken in -/// the context of the text atom, and may better be exposed in an iterator -/// instead of stored in the atom, which would replace this class. -class MCDecodedInst { -public: - MCInst Inst; - uint64_t Address; - uint64_t Size; - MCDecodedInst(const MCInst &Inst, uint64_t Address, uint64_t Size) - : Inst(Inst), Address(Address), Size(Size) {} -}; - -/// \brief An atom consisting of disassembled instructions. -class MCTextAtom : public MCAtom { -private: - typedef std::vector InstListTy; - InstListTy Insts; - - /// \brief The address of the next appended instruction, i.e., the - /// address immediately after the last instruction in the atom. - uint64_t NextInstAddress; -public: - /// Append an instruction, expanding the atom if necessary. - void addInst(const MCInst &Inst, uint64_t Size); - - /// \name Instruction list access - /// @{ - typedef InstListTy::const_iterator const_iterator; - const_iterator begin() const { return Insts.begin(); } - const_iterator end() const { return Insts.end(); } - - const MCDecodedInst &back() const { return Insts.back(); } - const MCDecodedInst &at(size_t n) const { return Insts.at(n); } - size_t size() const { return Insts.size(); } - /// @} - - /// \name Atom type specific split/truncate logic. - /// @{ - MCTextAtom *split(uint64_t SplitPt) override; - void truncate(uint64_t TruncPt) override; - /// @} - - // Class hierarchy. - static bool classof(const MCAtom *A) { return A->getKind() == TextAtom; } -private: - friend class MCModule; - // Private constructor - only callable by MCModule - MCTextAtom(MCModule *P, uint64_t Begin, uint64_t End) - : MCAtom(TextAtom, P, Begin, End), NextInstAddress(Begin) {} -}; -/// @} - -/// \name Data atom -/// @{ - -/// \brief An entry in an MCDataAtom. -// NOTE: This may change to a more complex type in the future. -typedef uint8_t MCData; - -/// \brief An atom consising of a sequence of bytes. -class MCDataAtom : public MCAtom { - std::vector Data; - -public: - /// Append a data entry, expanding the atom if necessary. - void addData(const MCData &D); - - /// Get a reference to the data in this atom. - ArrayRef getData() const { return Data; } - - /// \name Atom type specific split/truncate logic. - /// @{ - MCDataAtom *split(uint64_t SplitPt) override; - void truncate(uint64_t TruncPt) override; - /// @} - - // Class hierarchy. - static bool classof(const MCAtom *A) { return A->getKind() == DataAtom; } -private: - friend class MCModule; - // Private constructor - only callable by MCModule - MCDataAtom(MCModule *P, uint64_t Begin, uint64_t End) - : MCAtom(DataAtom, P, Begin, End) { - Data.reserve(End + 1 - Begin); - } -}; - -} - -#endif diff --git a/contrib/llvm/include/llvm/MC/MCAnalysis/MCFunction.h b/contrib/llvm/include/llvm/MC/MCAnalysis/MCFunction.h deleted file mode 100644 index 44fa4503b8e0..000000000000 --- a/contrib/llvm/include/llvm/MC/MCAnalysis/MCFunction.h +++ /dev/null @@ -1,142 +0,0 @@ -//===-- MCFunction.h --------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the data structures to hold a CFG reconstructed from -// machine code. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCANALYSIS_MCFUNCTION_H -#define LLVM_MC_MCANALYSIS_MCFUNCTION_H - -#include "llvm/ADT/StringRef.h" -#include "llvm/MC/MCInst.h" -#include -#include -#include - -namespace llvm { - -class MCFunction; -class MCModule; -class MCTextAtom; - -/// \brief Basic block containing a sequence of disassembled instructions. -/// The basic block is backed by an MCTextAtom, which holds the instructions, -/// and the address range it covers. -/// Create a basic block using MCFunction::createBlock. -class MCBasicBlock { - const MCTextAtom *Insts; - - // MCFunction owns the basic block. - MCFunction *Parent; - friend class MCFunction; - MCBasicBlock(const MCTextAtom &Insts, MCFunction *Parent); - - /// \name Predecessors/Successors, to represent the CFG. - /// @{ - typedef std::vector BasicBlockListTy; - BasicBlockListTy Successors; - BasicBlockListTy Predecessors; - /// @} -public: - - /// \brief Get the backing MCTextAtom, containing the instruction sequence. - const MCTextAtom *getInsts() const { return Insts; } - - /// \name Get the owning MCFunction. - /// @{ - const MCFunction *getParent() const { return Parent; } - MCFunction *getParent() { return Parent; } - /// @} - - /// MC CFG access: Predecessors/Successors. - /// @{ - typedef BasicBlockListTy::const_iterator succ_const_iterator; - succ_const_iterator succ_begin() const { return Successors.begin(); } - succ_const_iterator succ_end() const { return Successors.end(); } - - typedef BasicBlockListTy::const_iterator pred_const_iterator; - pred_const_iterator pred_begin() const { return Predecessors.begin(); } - pred_const_iterator pred_end() const { return Predecessors.end(); } - - void addSuccessor(const MCBasicBlock *MCBB); - bool isSuccessor(const MCBasicBlock *MCBB) const; - - void addPredecessor(const MCBasicBlock *MCBB); - bool isPredecessor(const MCBasicBlock *MCBB) const; - - /// \brief Split block, mirrorring NewAtom = Insts->split(..). - /// This moves all successors to \p SplitBB, and - /// adds a fallthrough to it. - /// \p SplitBB The result of splitting Insts, a basic block directly following - /// this basic block. - void splitBasicBlock(MCBasicBlock *SplitBB); - /// @} -}; - -/// \brief Represents a function in machine code, containing MCBasicBlocks. -/// MCFunctions are created by MCModule. -class MCFunction { - MCFunction (const MCFunction&) LLVM_DELETED_FUNCTION; - MCFunction& operator=(const MCFunction&) LLVM_DELETED_FUNCTION; - - std::string Name; - MCModule *ParentModule; - typedef std::vector> BasicBlockListTy; - BasicBlockListTy Blocks; - - // MCModule owns the function. - friend class MCModule; - MCFunction(StringRef Name, MCModule *Parent); - -public: - /// \brief Create an MCBasicBlock backed by Insts and add it to this function. - /// \param Insts Sequence of straight-line code backing the basic block. - /// \returns The newly created basic block. - MCBasicBlock &createBlock(const MCTextAtom &Insts); - - StringRef getName() const { return Name; } - - /// \name Get the owning MC Module. - /// @{ - const MCModule *getParent() const { return ParentModule; } - MCModule *getParent() { return ParentModule; } - /// @} - - /// \name Access to the function's basic blocks. No ordering is enforced, - /// except that the first block is the entry block. - /// @{ - /// \brief Get the entry point basic block. - const MCBasicBlock *getEntryBlock() const { return front(); } - MCBasicBlock *getEntryBlock() { return front(); } - - bool empty() const { return Blocks.empty(); } - - typedef BasicBlockListTy::const_iterator const_iterator; - typedef BasicBlockListTy:: iterator iterator; - const_iterator begin() const { return Blocks.begin(); } - iterator begin() { return Blocks.begin(); } - const_iterator end() const { return Blocks.end(); } - iterator end() { return Blocks.end(); } - - const MCBasicBlock* front() const { return Blocks.front().get(); } - MCBasicBlock* front() { return Blocks.front().get(); } - const MCBasicBlock* back() const { return Blocks.back().get(); } - MCBasicBlock* back() { return Blocks.back().get(); } - - /// \brief Find the basic block, if any, that starts at \p StartAddr. - const MCBasicBlock *find(uint64_t StartAddr) const; - MCBasicBlock *find(uint64_t StartAddr); - /// @} -}; - -} - -#endif diff --git a/contrib/llvm/include/llvm/MC/MCAnalysis/MCModule.h b/contrib/llvm/include/llvm/MC/MCAnalysis/MCModule.h deleted file mode 100644 index cf7e2c0a645e..000000000000 --- a/contrib/llvm/include/llvm/MC/MCAnalysis/MCModule.h +++ /dev/null @@ -1,134 +0,0 @@ -//===-- MCModule.h - MCModule class -----------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains the declaration of the MCModule class, which is used to -// represent a complete, disassembled object file or executable. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCANALYSIS_MCMODULE_H -#define LLVM_MC_MCANALYSIS_MCMODULE_H - -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/DataTypes.h" -#include -#include - -namespace llvm { - -class MCAtom; -class MCBasicBlock; -class MCDataAtom; -class MCFunction; -class MCObjectDisassembler; -class MCTextAtom; - -/// \brief A completely disassembled object file or executable. -/// It comprises a list of MCAtom's, each representing a contiguous range of -/// either instructions or data. -/// An MCModule is created using MCObjectDisassembler::buildModule. -class MCModule { - /// \name Atom tracking - /// @{ - - /// \brief Atoms in this module, sorted by begin address. - /// FIXME: This doesn't handle overlapping atoms (which happen when a basic - /// block starts in the middle of an instruction of another basic block.) - typedef std::vector AtomListTy; - AtomListTy Atoms; - - // For access to map/remap. - friend class MCAtom; - - /// \brief Remap \p Atom to the given range, and update its Begin/End fields. - /// \param Atom An atom belonging to this module. - /// An atom should always use this method to update its bounds, because this - /// enables the owning MCModule to keep track of its atoms. - void remap(MCAtom *Atom, uint64_t NewBegin, uint64_t NewEnd); - - /// \brief Insert an atom in the module, using its Begin and End addresses. - void map(MCAtom *NewAtom); - /// @} - - /// \name Basic block tracking - /// @{ - typedef std::vector BBsByAtomTy; - BBsByAtomTy BBsByAtom; - - // For access to basic block > atom tracking. - friend class MCBasicBlock; - friend class MCTextAtom; - - /// \brief Keep track of \p BBBackedByAtom as being backed by \p Atom. - /// This is used to update succs/preds when \p Atom is split. - void trackBBForAtom(const MCTextAtom *Atom, MCBasicBlock *BBBackedByAtom); - void splitBasicBlocksForAtom(const MCTextAtom *TA, const MCTextAtom *NewTA); - /// @} - - /// \name Function tracking - /// @{ - typedef std::vector> FunctionListTy; - FunctionListTy Functions; - /// @} - - /// The address of the entrypoint function. - uint64_t Entrypoint; - - MCModule (const MCModule &) LLVM_DELETED_FUNCTION; - MCModule& operator=(const MCModule &) LLVM_DELETED_FUNCTION; - - // MCObjectDisassembler creates MCModules. - friend class MCObjectDisassembler; - -public: - MCModule(); - ~MCModule(); - - /// \name Create a new MCAtom covering the specified offset range. - /// @{ - MCTextAtom *createTextAtom(uint64_t Begin, uint64_t End); - MCDataAtom *createDataAtom(uint64_t Begin, uint64_t End); - /// @} - - /// \name Access to the owned atom list, ordered by begin address. - /// @{ - const MCAtom *findAtomContaining(uint64_t Addr) const; - MCAtom *findAtomContaining(uint64_t Addr); - const MCAtom *findFirstAtomAfter(uint64_t Addr) const; - MCAtom *findFirstAtomAfter(uint64_t Addr); - - typedef AtomListTy::const_iterator const_atom_iterator; - typedef AtomListTy:: iterator atom_iterator; - const_atom_iterator atom_begin() const { return Atoms.begin(); } - atom_iterator atom_begin() { return Atoms.begin(); } - const_atom_iterator atom_end() const { return Atoms.end(); } - atom_iterator atom_end() { return Atoms.end(); } - /// @} - - /// \brief Create a new MCFunction. - MCFunction *createFunction(StringRef Name); - - /// \name Access to the owned function list. - /// @{ - typedef FunctionListTy::const_iterator const_func_iterator; - typedef FunctionListTy:: iterator func_iterator; - const_func_iterator func_begin() const { return Functions.begin(); } - func_iterator func_begin() { return Functions.begin(); } - const_func_iterator func_end() const { return Functions.end(); } - func_iterator func_end() { return Functions.end(); } - /// @} - - /// \brief Get the address of the entrypoint function, or 0 if there is none. - uint64_t getEntrypoint() const { return Entrypoint; } -}; - -} - -#endif diff --git a/contrib/llvm/include/llvm/MC/MCAnalysis/MCModuleYAML.h b/contrib/llvm/include/llvm/MC/MCAnalysis/MCModuleYAML.h deleted file mode 100644 index 48562777677b..000000000000 --- a/contrib/llvm/include/llvm/MC/MCAnalysis/MCModuleYAML.h +++ /dev/null @@ -1,40 +0,0 @@ -//===- MCModuleYAML.h - MCModule YAMLIO implementation ----------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief This file declares classes for handling the YAML representation -/// of MCModule. -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCANALYSIS_MCMODULEYAML_H -#define LLVM_MC_MCANALYSIS_MCMODULEYAML_H - -#include "llvm/ADT/StringRef.h" -#include "llvm/MC/MCAnalysis/MCModule.h" -#include "llvm/Support/raw_ostream.h" - -namespace llvm { - -class MCInstrInfo; -class MCRegisterInfo; - -/// \brief Dump a YAML representation of the MCModule \p MCM to \p OS. -/// \returns The empty string on success, an error message on failure. -StringRef mcmodule2yaml(raw_ostream &OS, const MCModule &MCM, - const MCInstrInfo &MII, const MCRegisterInfo &MRI); - -/// \brief Creates a new module and returns it in \p MCM. -/// \returns The empty string on success, an error message on failure. -StringRef yaml2mcmodule(std::unique_ptr &MCM, StringRef YamlContent, - const MCInstrInfo &MII, const MCRegisterInfo &MRI); - -} // end namespace llvm - -#endif diff --git a/contrib/llvm/include/llvm/MC/MCAsmBackend.h b/contrib/llvm/include/llvm/MC/MCAsmBackend.h index 82b65fdaf5c8..a6d41392724e 100644 --- a/contrib/llvm/include/llvm/MC/MCAsmBackend.h +++ b/contrib/llvm/include/llvm/MC/MCAsmBackend.h @@ -1,4 +1,4 @@ -//===-- llvm/MC/MCAsmBack.h - MC Asm Backend --------------------*- C++ -*-===// +//===-- llvm/MC/MCAsmBackend.h - MC Asm Backend -----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -67,14 +67,6 @@ public: return false; } - /// isSectionAtomizable - Check whether the given section can be split into - /// atoms. - /// - /// \see MCAssembler::isSymbolLinkerVisible(). - virtual bool isSectionAtomizable(const MCSection &Section) const { - return true; - } - /// @name Target Fixup Interfaces /// @{ @@ -146,7 +138,7 @@ public: /// \brief Generate the compact unwind encoding for the CFI instructions. virtual uint32_t - generateCompactUnwindEncoding(ArrayRef) const { + generateCompactUnwindEncoding(ArrayRef) const { return 0; } }; diff --git a/contrib/llvm/include/llvm/MC/MCAsmInfo.h b/contrib/llvm/include/llvm/MC/MCAsmInfo.h index 06e473d6b625..a750a0fd5cef 100644 --- a/contrib/llvm/include/llvm/MC/MCAsmInfo.h +++ b/contrib/llvm/include/llvm/MC/MCAsmInfo.h @@ -31,22 +31,23 @@ class MCContext; namespace WinEH { enum class EncodingType { - ET_Invalid, /// Invalid - ET_Alpha, /// Windows Alpha - ET_Alpha64, /// Windows AXP64 - ET_ARM, /// Windows NT (Windows on ARM) - ET_CE, /// Windows CE ARM, PowerPC, SH3, SH4 - ET_Itanium, /// Windows x64, Windows Itanium (IA-64) - ET_MIPS = ET_Alpha, + Invalid, /// Invalid + Alpha, /// Windows Alpha + Alpha64, /// Windows AXP64 + ARM, /// Windows NT (Windows on ARM) + CE, /// Windows CE ARM, PowerPC, SH3, SH4 + Itanium, /// Windows x64, Windows Itanium (IA-64) + MIPS = Alpha, }; } enum class ExceptionHandling { - None, /// No exception support - DwarfCFI, /// DWARF-like instruction based exceptions - SjLj, /// setjmp/longjmp based exceptions - ARM, /// ARM EHABI - WinEH, /// Windows Exception Handling + None, /// No exception support + DwarfCFI, /// DWARF-like instruction based exceptions + SjLj, /// setjmp/longjmp based exceptions + ARM, /// ARM EHABI + ItaniumWinEH, /// Itanium EH built on Windows unwind info (.pdata and .xdata) + MSVC, /// MSVC compatible exception handling }; namespace LCOMM { @@ -87,16 +88,11 @@ protected: bool HasMachoTBSSDirective; /// True if the compiler should emit a ".reference .constructors_used" or - /// ".reference .destructors_used" directive after the a static ctor/dtor + /// ".reference .destructors_used" directive after the static ctor/dtor /// list. This directive is only emitted in Static relocation model. Default /// is false. bool HasStaticCtorDtorReferenceInStaticMode; - /// True if the linker has a bug and requires that the debug_line section be - /// of a minimum size. In practice such a linker requires a non-empty line - /// sequence if a file is present. Default to false. - bool LinkerRequiresNonEmptyDwarfLines; - /// This is the maximum possible length of an instruction, which is needed to /// compute the size of an inline asm. Defaults to 4. unsigned MaxInstLength; @@ -128,6 +124,10 @@ protected: /// file. Defaults to "L" const char *PrivateGlobalPrefix; + /// This prefix is used for labels for basic blocks. Defaults to the same as + /// PrivateGlobalPrefix. + const char *PrivateLabelPrefix; + /// This prefix is used for symbols that should be passed through the /// assembler but be removed by the linker. This is 'l' on Darwin, currently /// used for some ObjC metadata. The default of "" meast that for this system @@ -220,11 +220,16 @@ protected: //===--- Global Variable Emission Directives --------------------------===// - /// This is the directive used to declare a global entity. Defaults to NULL. + /// This is the directive used to declare a global entity. Defaults to + /// ".globl". const char *GlobalDirective; - /// True if the assembler supports the .set directive. Defaults to true. - bool HasSetDirective; + /// True if the expression + /// .long f - g + /// uses an relocation but it can be supressed by writting + /// a = f - g + /// .long a + bool SetDirectiveSuppressesReloc; /// False if the assembler requires that we use /// \code @@ -265,6 +270,9 @@ protected: /// to false. bool HasNoDeadStrip; + /// Used to declare a global as being a weak symbol. Defaults to ".weak". + const char *WeakDirective; + /// This directive, if non-null, is used to declare a global as being a weak /// undefined symbol. Defaults to NULL. const char *WeakRefDirective; @@ -295,9 +303,6 @@ protected: //===--- Dwarf Emission Directives -----------------------------------===// - /// True if target asm supports leb128 directives. Defaults to false. - bool HasLEB128; - /// True if target supports emission of debugging information. Defaults to /// false. bool SupportsDebugInformation; @@ -377,6 +382,12 @@ public: return nullptr; } + /// \brief True if the section is atomized using the symbols in it. + /// This is false if the section is not atomized at all (most ELF sections) or + /// if it is atomized based on its contents (MachO' __TEXT,__cstring for + /// example). + virtual bool isSectionAtomizableBySymbols(const MCSection &Section) const; + virtual const MCExpr *getExprForPersonalitySymbol(const MCSymbol *Sym, unsigned Encoding, MCStreamer &Streamer) const; @@ -404,9 +415,6 @@ public: bool hasStaticCtorDtorReferenceInStaticMode() const { return HasStaticCtorDtorReferenceInStaticMode; } - bool getLinkerRequiresNonEmptyDwarfLines() const { - return LinkerRequiresNonEmptyDwarfLines; - } unsigned getMaxInstLength() const { return MaxInstLength; } unsigned getMinInstAlignment() const { return MinInstAlignment; } bool getDollarIsPC() const { return DollarIsPC; } @@ -421,6 +429,7 @@ public: bool useAssignmentForEHBegin() const { return UseAssignmentForEHBegin; } const char *getPrivateGlobalPrefix() const { return PrivateGlobalPrefix; } + const char *getPrivateLabelPrefix() const { return PrivateLabelPrefix; } bool hasLinkerPrivateGlobalPrefix() const { return LinkerPrivateGlobalPrefix[0] != '\0'; } @@ -445,7 +454,9 @@ public: bool getAlignmentIsInBytes() const { return AlignmentIsInBytes; } unsigned getTextAlignFillValue() const { return TextAlignFillValue; } const char *getGlobalDirective() const { return GlobalDirective; } - bool hasSetDirective() const { return HasSetDirective; } + bool doesSetDirectiveSuppressesReloc() const { + return SetDirectiveSuppressesReloc; + } bool hasAggressiveSymbolFolding() const { return HasAggressiveSymbolFolding; } bool getCOMMDirectiveAlignmentIsInBytes() const { return COMMDirectiveAlignmentIsInBytes; @@ -457,6 +468,7 @@ public: bool hasSingleParameterDotFile() const { return HasSingleParameterDotFile; } bool hasIdentDirective() const { return HasIdentDirective; } bool hasNoDeadStrip() const { return HasNoDeadStrip; } + const char *getWeakDirective() const { return WeakDirective; } const char *getWeakRefDirective() const { return WeakRefDirective; } bool hasWeakDefDirective() const { return HasWeakDefDirective; } bool hasWeakDefCanBeHiddenDirective() const { @@ -471,19 +483,27 @@ public: MCSymbolAttr getProtectedVisibilityAttr() const { return ProtectedVisibilityAttr; } - bool hasLEB128() const { return HasLEB128; } bool doesSupportDebugInformation() const { return SupportsDebugInformation; } bool doesSupportExceptionHandling() const { return ExceptionsType != ExceptionHandling::None; } ExceptionHandling getExceptionHandlingType() const { return ExceptionsType; } WinEH::EncodingType getWinEHEncodingType() const { return WinEHEncodingType; } - bool isExceptionHandlingDwarf() const { + + /// Return true if the exception handling type uses the language-specific data + /// area (LSDA) format specified by the Itanium C++ ABI. + bool usesItaniumLSDAForExceptions() const { return (ExceptionsType == ExceptionHandling::DwarfCFI || ExceptionsType == ExceptionHandling::ARM || - // Windows handler data still uses DWARF LSDA encoding. - ExceptionsType == ExceptionHandling::WinEH); + // This Windows EH type uses the Itanium LSDA encoding. + ExceptionsType == ExceptionHandling::ItaniumWinEH); } + + bool usesWindowsCFI() const { + return ExceptionsType == ExceptionHandling::ItaniumWinEH || + ExceptionsType == ExceptionHandling::MSVC; + } + bool doesDwarfUseRelocationsAcrossSections() const { return DwarfUsesRelocationsAcrossSections; } diff --git a/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h b/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h index 3d249f93068d..d587c3ce9d54 100644 --- a/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h +++ b/contrib/llvm/include/llvm/MC/MCAsmInfoDarwin.h @@ -19,9 +19,9 @@ namespace llvm { class MCAsmInfoDarwin : public MCAsmInfo { - virtual void anchor(); public: explicit MCAsmInfoDarwin(); + bool isSectionAtomizableBySymbols(const MCSection &Section) const override; }; } diff --git a/contrib/llvm/include/llvm/MC/MCAsmInfoELF.h b/contrib/llvm/include/llvm/MC/MCAsmInfoELF.h index 27fea84e7a6d..7bd246056eca 100644 --- a/contrib/llvm/include/llvm/MC/MCAsmInfoELF.h +++ b/contrib/llvm/include/llvm/MC/MCAsmInfoELF.h @@ -15,6 +15,9 @@ namespace llvm { class MCAsmInfoELF : public MCAsmInfo { virtual void anchor(); + const MCSection * + getNonexecutableStackSection(MCContext &Ctx) const override final; + protected: MCAsmInfoELF(); }; diff --git a/contrib/llvm/include/llvm/MC/MCAssembler.h b/contrib/llvm/include/llvm/MC/MCAssembler.h index 1cb34c2fe33b..681a31728799 100644 --- a/contrib/llvm/include/llvm/MC/MCAssembler.h +++ b/contrib/llvm/include/llvm/MC/MCAssembler.h @@ -11,6 +11,7 @@ #define LLVM_MC_MCASSEMBLER_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/ilist.h" @@ -593,7 +594,10 @@ private: unsigned Alignment; /// \brief Keeping track of bundle-locked state. - BundleLockStateType BundleLockState; + BundleLockStateType BundleLockState; + + /// \brief Current nesting depth of bundle_lock directives. + unsigned BundleLockNestingDepth; /// \brief We've seen a bundle_lock directive but not its first instruction /// yet. @@ -665,9 +669,7 @@ public: return BundleLockState; } - void setBundleLockState(BundleLockStateType NewState) { - BundleLockState = NewState; - } + void setBundleLockState(BundleLockStateType NewState); bool isBundleGroupBeforeFirstInst() const { return BundleGroupBeforeFirstInst; @@ -684,34 +686,27 @@ public: // FIXME: Same concerns as with SectionData. class MCSymbolData : public ilist_node { -public: const MCSymbol *Symbol; - /// Fragment - The fragment this symbol's value is relative to, if any. - MCFragment *Fragment; + /// Fragment - The fragment this symbol's value is relative to, if any. Also + /// stores if this symbol is visible outside this translation unit (bit 0) or + /// if it is private extern (bit 1). + PointerIntPair Fragment; - /// Offset - The offset to apply to the fragment address to form this symbol's - /// value. - uint64_t Offset; + union { + /// Offset - The offset to apply to the fragment address to form this + /// symbol's value. + uint64_t Offset; - /// IsExternal - True if this symbol is visible outside this translation - /// unit. - unsigned IsExternal : 1; - - /// IsPrivateExtern - True if this symbol is private extern. - unsigned IsPrivateExtern : 1; - - /// CommonSize - The size of the symbol, if it is 'common', or 0. - // - // FIXME: Pack this in with other fields? We could put it in offset, since a - // common symbol can never get a definition. - uint64_t CommonSize; + /// CommonSize - The size of the symbol, if it is 'common'. + uint64_t CommonSize; + }; /// SymbolSize - An expression describing how to calculate the size of /// a symbol. If a symbol has no size this field will be NULL. const MCExpr *SymbolSize; - /// CommonAlign - The alignment of the symbol, if it is 'common'. + /// CommonAlign - The alignment of the symbol, if it is 'common', or -1. // // FIXME: Pack this in with other fields? unsigned CommonAlign; @@ -734,30 +729,41 @@ public: const MCSymbol &getSymbol() const { return *Symbol; } - MCFragment *getFragment() const { return Fragment; } - void setFragment(MCFragment *Value) { Fragment = Value; } + MCFragment *getFragment() const { return Fragment.getPointer(); } + void setFragment(MCFragment *Value) { Fragment.setPointer(Value); } - uint64_t getOffset() const { return Offset; } - void setOffset(uint64_t Value) { Offset = Value; } + uint64_t getOffset() const { + assert(!isCommon()); + return Offset; + } + void setOffset(uint64_t Value) { + assert(!isCommon()); + Offset = Value; + } /// @} /// @name Symbol Attributes /// @{ - bool isExternal() const { return IsExternal; } - void setExternal(bool Value) { IsExternal = Value; } + bool isExternal() const { return Fragment.getInt() & 1; } + void setExternal(bool Value) { + Fragment.setInt((Fragment.getInt() & ~1) | unsigned(Value)); + } - bool isPrivateExtern() const { return IsPrivateExtern; } - void setPrivateExtern(bool Value) { IsPrivateExtern = Value; } + bool isPrivateExtern() const { return Fragment.getInt() & 2; } + void setPrivateExtern(bool Value) { + Fragment.setInt((Fragment.getInt() & ~2) | (unsigned(Value) << 1)); + } /// isCommon - Is this a 'common' symbol. - bool isCommon() const { return CommonSize != 0; } + bool isCommon() const { return CommonAlign != -1U; } /// setCommon - Mark this symbol as being 'common'. /// /// \param Size - The size of the symbol. /// \param Align - The alignment of the symbol. void setCommon(uint64_t Size, unsigned Align) { + assert(getOffset() == 0); CommonSize = Size; CommonAlign = Align; } @@ -910,7 +916,6 @@ private: unsigned BundleAlignSize; unsigned RelaxAll : 1; - unsigned NoExecStack : 1; unsigned SubsectionsViaSymbols : 1; /// ELF specific e_header flags @@ -1056,9 +1061,6 @@ public: bool getRelaxAll() const { return RelaxAll; } void setRelaxAll(bool Value) { RelaxAll = Value; } - bool getNoExecStack() const { return NoExecStack; } - void setNoExecStack(bool Value) { NoExecStack = Value; } - bool isBundlingEnabled() const { return BundleAlignSize != 0; } diff --git a/contrib/llvm/include/llvm/MC/MCContext.h b/contrib/llvm/include/llvm/MC/MCContext.h index eb0340f7421a..e3163a7946a7 100644 --- a/contrib/llvm/include/llvm/MC/MCContext.h +++ b/contrib/llvm/include/llvm/MC/MCContext.h @@ -73,6 +73,10 @@ namespace llvm { /// Symbols - Bindings of names to symbols. SymbolTable Symbols; + /// ELF sections can have a corresponding symbol. This maps one to the + /// other. + DenseMap SectionSymbols; + /// A maping from a local label number and an instance count to a symbol. /// For example, in the assembly /// 1: @@ -231,6 +235,10 @@ namespace llvm { MCSymbol *GetOrCreateSymbol(StringRef Name); MCSymbol *GetOrCreateSymbol(const Twine &Name); + MCSymbol *getOrCreateSectionSymbol(const MCSectionELF &Section); + + MCSymbol *getOrCreateFrameAllocSymbol(StringRef FuncName); + /// LookupSymbol - Get the symbol for \p Name, or null. MCSymbol *LookupSymbol(StringRef Name) const; MCSymbol *LookupSymbol(const Twine &Name) const; @@ -284,6 +292,13 @@ namespace llvm { const MCSectionCOFF *getCOFFSection(StringRef Section); + /// Gets or creates a section equivalent to Sec that is associated with the + /// section containing KeySym. For example, to create a debug info section + /// associated with an inline function, pass the normal debug info section + /// as Sec and the function symbol as KeySym. + const MCSectionCOFF *getAssociativeCOFFSection(const MCSectionCOFF *Sec, + const MCSymbol *KeySym); + /// @} /// @name Dwarf Management diff --git a/contrib/llvm/include/llvm/MC/MCDisassembler.h b/contrib/llvm/include/llvm/MC/MCDisassembler.h index 9d441bbd88fb..d6b0a305b1da 100644 --- a/contrib/llvm/include/llvm/MC/MCDisassembler.h +++ b/contrib/llvm/include/llvm/MC/MCDisassembler.h @@ -10,6 +10,7 @@ #define LLVM_MC_MCDISASSEMBLER_H #include "llvm-c/Disassembler.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/MC/MCRelocationInfo.h" #include "llvm/MC/MCSymbolizer.h" #include "llvm/Support/DataTypes.h" @@ -18,12 +19,11 @@ namespace llvm { class MCInst; class MCSubtargetInfo; -class MemoryObject; class raw_ostream; class MCContext; -/// MCDisassembler - Superclass for all disassemblers. Consumes a memory region -/// and provides an array of assembly instructions. +/// Superclass for all disassemblers. Consumes a memory region and provides an +/// array of assembly instructions. class MCDisassembler { public: /// Ternary decode status. Most backends will just use Fail and @@ -54,34 +54,31 @@ public: Success = 3 }; - /// Constructor - Performs initial setup for the disassembler. MCDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx) : Ctx(Ctx), STI(STI), Symbolizer(), CommentStream(nullptr) {} virtual ~MCDisassembler(); - /// getInstruction - Returns the disassembly of a single instruction. + /// Returns the disassembly of a single instruction. /// - /// @param instr - An MCInst to populate with the contents of the + /// @param Instr - An MCInst to populate with the contents of the /// instruction. - /// @param size - A value to populate with the size of the instruction, or + /// @param Size - A value to populate with the size of the instruction, or /// the number of bytes consumed while attempting to decode /// an invalid instruction. - /// @param region - The memory object to use as a source for machine code. - /// @param address - The address, in the memory space of region, of the first + /// @param Address - The address, in the memory space of region, of the first /// byte of the instruction. - /// @param vStream - The stream to print warnings and diagnostic messages on. - /// @param cStream - The stream to print comments and annotations on. + /// @param VStream - The stream to print warnings and diagnostic messages on. + /// @param CStream - The stream to print comments and annotations on. /// @return - MCDisassembler::Success if the instruction is valid, /// MCDisassembler::SoftFail if the instruction was /// disassemblable but invalid, /// MCDisassembler::Fail if the instruction was invalid. - virtual DecodeStatus getInstruction(MCInst& instr, - uint64_t& size, - const MemoryObject ®ion, - uint64_t address, - raw_ostream &vStream, - raw_ostream &cStream) const = 0; + virtual DecodeStatus getInstruction(MCInst &Instr, uint64_t &Size, + ArrayRef Bytes, uint64_t Address, + raw_ostream &VStream, + raw_ostream &CStream) const = 0; + private: MCContext &Ctx; diff --git a/contrib/llvm/include/llvm/MC/MCDwarf.h b/contrib/llvm/include/llvm/MC/MCDwarf.h index 6cd9a9a21e21..c266acf2f09c 100644 --- a/contrib/llvm/include/llvm/MC/MCDwarf.h +++ b/contrib/llvm/include/llvm/MC/MCDwarf.h @@ -16,16 +16,16 @@ #define LLVM_MC_MCDWARF_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/raw_ostream.h" #include -#include #include #include +#include namespace llvm { class MCAsmBackend; @@ -457,7 +457,7 @@ public: return Offset; } - const StringRef getValues() const { + StringRef getValues() const { assert(Operation == OpEscape); return StringRef(&Values[0], Values.size()); } @@ -466,13 +466,15 @@ public: struct MCDwarfFrameInfo { MCDwarfFrameInfo() : Begin(nullptr), End(nullptr), Personality(nullptr), Lsda(nullptr), - Instructions(), PersonalityEncoding(), LsdaEncoding(0), - CompactUnwindEncoding(0), IsSignalFrame(false), IsSimple(false) {} + Instructions(), CurrentCfaRegister(0), PersonalityEncoding(), + LsdaEncoding(0), CompactUnwindEncoding(0), IsSignalFrame(false), + IsSimple(false) {} MCSymbol *Begin; MCSymbol *End; const MCSymbol *Personality; const MCSymbol *Lsda; std::vector Instructions; + unsigned CurrentCfaRegister; unsigned PersonalityEncoding; unsigned LsdaEncoding; uint32_t CompactUnwindEncoding; diff --git a/contrib/llvm/include/llvm/MC/MCELFStreamer.h b/contrib/llvm/include/llvm/MC/MCELFStreamer.h index 66729fe0147e..ab6c5e3d6124 100644 --- a/contrib/llvm/include/llvm/MC/MCELFStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCELFStreamer.h @@ -41,10 +41,18 @@ public: virtual ~MCELFStreamer(); + /// state management + void reset() override { + LocalCommons.clear(); + BindingExplicitlySet.clear(); + SeenIdent = false; + MCObjectStreamer::reset(); + } + /// @name MCStreamer Interface /// @{ - void InitSections() override; + void InitSections(bool NoExecStack) override; void ChangeSection(const MCSection *Section, const MCExpr *Subsection) override; void EmitLabel(MCSymbol *Symbol) override; @@ -107,8 +115,7 @@ private: MCELFStreamer *createARMELFStreamer(MCContext &Context, MCAsmBackend &TAB, raw_ostream &OS, MCCodeEmitter *Emitter, - bool RelaxAll, bool NoExecStack, - bool IsThumb); + bool RelaxAll, bool IsThumb); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCExpr.h b/contrib/llvm/include/llvm/MC/MCExpr.h index 163574eef55a..bd9b2bc4751a 100644 --- a/contrib/llvm/include/llvm/MC/MCExpr.h +++ b/contrib/llvm/include/llvm/MC/MCExpr.h @@ -19,6 +19,7 @@ class MCAsmInfo; class MCAsmLayout; class MCAssembler; class MCContext; +class MCFixup; class MCSection; class MCSectionData; class MCStreamer; @@ -49,11 +50,17 @@ private: bool EvaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, const SectionAddrMap *Addrs) const; + + bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, + const MCAsmLayout *Layout, + const SectionAddrMap *Addrs, bool InSet) const; + protected: explicit MCExpr(ExprKind _Kind) : Kind(_Kind) {} bool EvaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, + const MCFixup *Fixup, const SectionAddrMap *Addrs, bool InSet, bool ForceVarExpansion) const; @@ -87,13 +94,17 @@ public: bool EvaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const; bool EvaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout) const; + int64_t evaluateKnownAbsolute(const MCAsmLayout &Layout) const; + /// EvaluateAsRelocatable - Try to evaluate the expression to a relocatable /// value, i.e. an expression of the fixed form (a - b + constant). /// /// @param Res - The relocatable value, if evaluation succeeds. /// @param Layout - The assembler layout object to use for evaluating values. + /// @param Fixup - The Fixup object if available. /// @result - True on success. - bool EvaluateAsRelocatable(MCValue &Res, const MCAsmLayout *Layout) const; + bool EvaluateAsRelocatable(MCValue &Res, const MCAsmLayout *Layout, + const MCFixup *Fixup) const; /// \brief Try to evaluate the expression to the form (a - b + constant) where /// neither a nor b are variables. @@ -101,7 +112,8 @@ public: /// This is a more aggressive variant of EvaluateAsRelocatable. The intended /// use is for when relocations are not available, like the symbol value in /// the symbol table. - bool EvaluateAsValue(MCValue &Res, const MCAsmLayout *Layout) const; + bool EvaluateAsValue(MCValue &Res, const MCAsmLayout *Layout, + const MCFixup *Fixup) const; /// FindAssociatedSection - Find the "associated section" for this expression, /// which is currently defined as the absolute section for constants, or @@ -182,6 +194,7 @@ public: VK_ARM_TARGET1, VK_ARM_TARGET2, VK_ARM_PREL31, + VK_ARM_SBREL, // symbol(sbrel) VK_ARM_TLSLDO, // symbol(tlsldo) VK_ARM_TLSCALL, // symbol(tlscall) VK_ARM_TLSDESC, // symbol(tlsdesc) @@ -271,21 +284,20 @@ public: }; private: + /// The symbol reference modifier. + const unsigned Kind : 16; + + /// Specifies how the variant kind should be printed. + const unsigned UseParensForSymbolVariant : 1; + + // FIXME: Remove this bit. + const unsigned HasSubsectionsViaSymbols : 1; + /// The symbol being referenced. const MCSymbol *Symbol; - /// The symbol reference modifier. - const VariantKind Kind; - - /// MCAsmInfo that is used to print symbol variants correctly. - const MCAsmInfo *MAI; - - explicit MCSymbolRefExpr(const MCSymbol *_Symbol, VariantKind _Kind, - const MCAsmInfo *_MAI) - : MCExpr(MCExpr::SymbolRef), Symbol(_Symbol), Kind(_Kind), MAI(_MAI) { - assert(Symbol); - assert(MAI); - } + explicit MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind, + const MCAsmInfo *MAI); public: /// @name Construction @@ -305,9 +317,12 @@ public: /// @{ const MCSymbol &getSymbol() const { return *Symbol; } - const MCAsmInfo &getMCAsmInfo() const { return *MAI; } - VariantKind getKind() const { return Kind; } + VariantKind getKind() const { return static_cast(Kind); } + + void printVariantKind(raw_ostream &OS) const; + + bool hasSubsectionsViaSymbols() const { return HasSubsectionsViaSymbols; } /// @} /// @name Static Utility Functions @@ -525,7 +540,8 @@ public: virtual void PrintImpl(raw_ostream &OS) const = 0; virtual bool EvaluateAsRelocatableImpl(MCValue &Res, - const MCAsmLayout *Layout) const = 0; + const MCAsmLayout *Layout, + const MCFixup *Fixup) const = 0; virtual void visitUsedExpr(MCStreamer& Streamer) const = 0; virtual const MCSection *FindAssociatedSection() const = 0; diff --git a/contrib/llvm/include/llvm/MC/MCInst.h b/contrib/llvm/include/llvm/MC/MCInst.h index 6918280a4219..25cd5ccb08fd 100644 --- a/contrib/llvm/include/llvm/MC/MCInst.h +++ b/contrib/llvm/include/llvm/MC/MCInst.h @@ -31,7 +31,7 @@ class MCInst; /// MCOperand - Instances of this class represent operands of the MCInst class. /// This is a simple discriminated union. class MCOperand { - enum MachineOperandType { + enum MachineOperandType : unsigned char { kInvalid, ///< Uninitialized. kRegister, ///< Register operand. kImmediate, ///< Immediate operand. @@ -39,7 +39,7 @@ class MCOperand { kExpr, ///< Relocatable immediate operand. kInst ///< Sub-instruction operand. }; - unsigned char Kind; + MachineOperandType Kind; union { unsigned RegVal; @@ -172,8 +172,11 @@ public: size_t size() { return Operands.size(); } typedef SmallVectorImpl::iterator iterator; + typedef SmallVectorImpl::const_iterator const_iterator; iterator begin() { return Operands.begin(); } - iterator end() { return Operands.end(); } + const_iterator begin() const { return Operands.begin(); } + iterator end() { return Operands.end(); } + const_iterator end() const { return Operands.end(); } iterator insert(iterator I, const MCOperand &Op) { return Operands.insert(I, Op); } diff --git a/contrib/llvm/include/llvm/MC/MCInstPrinter.h b/contrib/llvm/include/llvm/MC/MCInstPrinter.h index 7f55b29f8ee0..95124c3021dd 100644 --- a/contrib/llvm/include/llvm/MC/MCInstPrinter.h +++ b/contrib/llvm/include/llvm/MC/MCInstPrinter.h @@ -1,4 +1,4 @@ -//===-- MCInstPrinter.h - Convert an MCInst to target assembly syntax -----===// +//===- MCInstPrinter.h - MCInst to target assembly syntax -------*- C++ -*-===// // // The LLVM Compiler Infrastructure // diff --git a/contrib/llvm/include/llvm/MC/MCInstrDesc.h b/contrib/llvm/include/llvm/MC/MCInstrDesc.h index 5896de7f76df..360989305d3a 100644 --- a/contrib/llvm/include/llvm/MC/MCInstrDesc.h +++ b/contrib/llvm/include/llvm/MC/MCInstrDesc.h @@ -44,11 +44,12 @@ namespace MCOI { /// Operand Type - Operands are tagged with one of the values of this enum. enum OperandType { - OPERAND_UNKNOWN, - OPERAND_IMMEDIATE, - OPERAND_REGISTER, - OPERAND_MEMORY, - OPERAND_PCREL + OPERAND_UNKNOWN = 0, + OPERAND_IMMEDIATE = 1, + OPERAND_REGISTER = 2, + OPERAND_MEMORY = 3, + OPERAND_PCREL = 4, + OPERAND_FIRST_TARGET = 5 }; } @@ -125,7 +126,10 @@ namespace MCID { Rematerializable, CheapAsAMove, ExtraSrcRegAllocReq, - ExtraDefRegAllocReq + ExtraDefRegAllocReq, + RegSequence, + ExtractSubreg, + InsertSubreg }; } @@ -357,6 +361,47 @@ public: return Flags & (1 << MCID::FoldableAsLoad); } + /// \brief Return true if this instruction behaves + /// the same way as the generic REG_SEQUENCE instructions. + /// E.g., on ARM, + /// dX VMOVDRR rY, rZ + /// is equivalent to + /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. + /// + /// Note that for the optimizers to be able to take advantage of + /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be + /// override accordingly. + bool isRegSequenceLike() const { return Flags & (1 << MCID::RegSequence); } + + /// \brief Return true if this instruction behaves + /// the same way as the generic EXTRACT_SUBREG instructions. + /// E.g., on ARM, + /// rX, rY VMOVRRD dZ + /// is equivalent to two EXTRACT_SUBREG: + /// rX = EXTRACT_SUBREG dZ, ssub_0 + /// rY = EXTRACT_SUBREG dZ, ssub_1 + /// + /// Note that for the optimizers to be able to take advantage of + /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be + /// override accordingly. + bool isExtractSubregLike() const { + return Flags & (1 << MCID::ExtractSubreg); + } + + /// \brief Return true if this instruction behaves + /// the same way as the generic INSERT_SUBREG instructions. + /// E.g., on ARM, + /// dX = VSETLNi32 dY, rZ, Imm + /// is equivalent to a INSERT_SUBREG: + /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) + /// + /// Note that for the optimizers to be able to take advantage of + /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be + /// override accordingly. + bool isInsertSubregLike() const { + return Flags & (1 << MCID::InsertSubreg); + } + //===--------------------------------------------------------------------===// // Side Effect Analysis //===--------------------------------------------------------------------===// @@ -451,9 +496,12 @@ public: } /// isRematerializable - Returns true if this instruction is a candidate for - /// remat. This flag is deprecated, please don't use it anymore. If this - /// flag is set, the isReallyTriviallyReMaterializable() method is called to - /// verify the instruction is really rematable. + /// remat. This flag is only used in TargetInstrInfo method + /// isTriviallyRematerializable. + /// + /// If this flag is set, the isReallyTriviallyReMaterializable() + /// or isReallyTriviallyReMaterializableGeneric methods are called to verify + /// the instruction is really rematable. bool isRematerializable() const { return Flags & (1 << MCID::Rematerializable); } @@ -464,6 +512,9 @@ public: /// where we would like to remat or hoist the instruction, but not if it costs /// more than moving the instruction into the appropriate register. Note, we /// are not marking copies from and to the same register class with this flag. + /// + /// This method could be called by interface TargetInstrInfo::isAsCheapAsAMove + /// for different subtargets. bool isAsCheapAsAMove() const { return Flags & (1 << MCID::CheapAsAMove); } diff --git a/contrib/llvm/include/llvm/MC/MCInstrItineraries.h b/contrib/llvm/include/llvm/MC/MCInstrItineraries.h index 5104345e1abb..94d599f672af 100644 --- a/contrib/llvm/include/llvm/MC/MCInstrItineraries.h +++ b/contrib/llvm/include/llvm/MC/MCInstrItineraries.h @@ -22,7 +22,7 @@ namespace llvm { //===----------------------------------------------------------------------===// -/// Instruction stage - These values represent a non-pipelined step in +/// These values represent a non-pipelined step in /// the execution of an instruction. Cycles represents the number of /// discrete time slots needed to complete the stage. Units represent /// the choice of functional units that can be used to complete the @@ -67,12 +67,12 @@ struct InstrStage { int NextCycles_; ///< Number of machine cycles to next stage ReservationKinds Kind_; ///< Kind of the FU reservation - /// getCycles - returns the number of cycles the stage is occupied + /// Returns the number of cycles the stage is occupied. unsigned getCycles() const { return Cycles_; } - /// getUnits - returns the choice of FUs + /// Returns the choice of FUs. unsigned getUnits() const { return Units_; } @@ -81,7 +81,7 @@ struct InstrStage { return Kind_; } - /// getNextCycles - returns the number of cycles from the start of + /// Returns the number of cycles from the start of /// this stage to the start of the next stage in the itinerary unsigned getNextCycles() const { return (NextCycles_ >= 0) ? (unsigned)NextCycles_ : Cycles_; @@ -90,10 +90,9 @@ struct InstrStage { //===----------------------------------------------------------------------===// -/// Instruction itinerary - An itinerary represents the scheduling -/// information for an instruction. This includes a set of stages -/// occupies by the instruction, and the pipeline cycle in which -/// operands are read and written. +/// An itinerary represents the scheduling information for an instruction. +/// This includes a set of stages occupied by the instruction and the pipeline +/// cycle in which operands are read and written. /// struct InstrItinerary { int NumMicroOps; ///< # of micro-ops, -1 means it's variable @@ -105,12 +104,11 @@ struct InstrItinerary { //===----------------------------------------------------------------------===// -/// Instruction itinerary Data - Itinerary data supplied by a subtarget to be -/// used by a target. +/// Itinerary data supplied by a subtarget to be used by a target. /// class InstrItineraryData { public: - const MCSchedModel *SchedModel; ///< Basic machine properties. + MCSchedModel SchedModel; ///< Basic machine properties. const InstrStage *Stages; ///< Array of stages selected const unsigned *OperandCycles; ///< Array of operand cycles selected const unsigned *Forwardings; ///< Array of pipeline forwarding pathes @@ -118,45 +116,38 @@ public: /// Ctors. /// - InstrItineraryData() : SchedModel(&MCSchedModel::DefaultSchedModel), + InstrItineraryData() : SchedModel(MCSchedModel::GetDefaultSchedModel()), Stages(nullptr), OperandCycles(nullptr), Forwardings(nullptr), Itineraries(nullptr) {} - InstrItineraryData(const MCSchedModel *SM, const InstrStage *S, + InstrItineraryData(const MCSchedModel &SM, const InstrStage *S, const unsigned *OS, const unsigned *F) : SchedModel(SM), Stages(S), OperandCycles(OS), Forwardings(F), - Itineraries(SchedModel->InstrItineraries) {} + Itineraries(SchedModel.InstrItineraries) {} - /// isEmpty - Returns true if there are no itineraries. - /// + /// Returns true if there are no itineraries. bool isEmpty() const { return Itineraries == nullptr; } - /// isEndMarker - Returns true if the index is for the end marker - /// itinerary. - /// + /// Returns true if the index is for the end marker itinerary. bool isEndMarker(unsigned ItinClassIndx) const { return ((Itineraries[ItinClassIndx].FirstStage == ~0U) && (Itineraries[ItinClassIndx].LastStage == ~0U)); } - /// beginStage - Return the first stage of the itinerary. - /// + /// Return the first stage of the itinerary. const InstrStage *beginStage(unsigned ItinClassIndx) const { unsigned StageIdx = Itineraries[ItinClassIndx].FirstStage; return Stages + StageIdx; } - /// endStage - Return the last+1 stage of the itinerary. - /// + /// Return the last+1 stage of the itinerary. const InstrStage *endStage(unsigned ItinClassIndx) const { unsigned StageIdx = Itineraries[ItinClassIndx].LastStage; return Stages + StageIdx; } - /// getStageLatency - Return the total stage latency of the given - /// class. The latency is the maximum completion time for any stage - /// in the itinerary. - /// + /// Return the total stage latency of the given class. + /// The latency is the maximum completion time for any stage in the itinerary. /// If no stages exist, it defaults to one cycle. unsigned getStageLatency(unsigned ItinClassIndx) const { // If the target doesn't provide itinerary information, use a simple @@ -174,9 +165,8 @@ public: return Latency; } - /// getOperandCycle - Return the cycle for the given class and - /// operand. Return -1 if no cycle is specified for the operand. - /// + /// Return the cycle for the given class and operand. + /// Return -1 if no cycle is specified for the operand. int getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const { if (isEmpty()) return -1; @@ -189,7 +179,7 @@ public: return (int)OperandCycles[FirstIdx + OperandIdx]; } - /// hasPipelineForwarding - Return true if there is a pipeline forwarding + /// Return true if there is a pipeline forwarding /// between instructions of itinerary classes DefClass and UseClasses so that /// value produced by an instruction of itinerary class DefClass, operand /// index DefIdx can be bypassed when it's read by an instruction of @@ -212,7 +202,7 @@ public: Forwardings[FirstUseIdx + UseIdx]; } - /// getOperandLatency - Compute and return the use operand latency of a given + /// Compute and return the use operand latency of a given /// itinerary class and operand index if the value is produced by an /// instruction of the specified itinerary class and def operand index. int getOperandLatency(unsigned DefClass, unsigned DefIdx, @@ -236,9 +226,8 @@ public: return UseCycle; } - /// getNumMicroOps - Return the number of micro-ops that the given class - /// decodes to. Return -1 for classes that require dynamic lookup via - /// TargetInstrInfo. + /// Return the number of micro-ops that the given class decodes to. + /// Return -1 for classes that require dynamic lookup via TargetInstrInfo. int getNumMicroOps(unsigned ItinClassIndx) const { if (isEmpty()) return 1; diff --git a/contrib/llvm/include/llvm/MC/MCLinkerOptimizationHint.h b/contrib/llvm/include/llvm/MC/MCLinkerOptimizationHint.h index 50fd527ffe2a..890d6385aacd 100644 --- a/contrib/llvm/include/llvm/MC/MCLinkerOptimizationHint.h +++ b/contrib/llvm/include/llvm/MC/MCLinkerOptimizationHint.h @@ -18,8 +18,8 @@ #define LLVM_MC_MCLINKEROPTIMIZATIONHINT_H #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/MC/MCMachObjectWriter.h" #include "llvm/Support/raw_ostream.h" @@ -45,7 +45,7 @@ static inline StringRef MCLOHDirectiveName() { return StringRef(".loh"); } -static inline bool isValidMCLOHType(MCLOHType Kind) { +static inline bool isValidMCLOHType(unsigned Kind) { return Kind >= MCLOH_AdrpAdrp && Kind <= MCLOH_AdrpLdrGot; } diff --git a/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h b/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h index 12a7f0ee7bb5..0c5aa8a18063 100644 --- a/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h @@ -14,6 +14,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/MachO.h" #include @@ -104,7 +105,7 @@ class MachObjectWriter : public MCObjectWriter { /// @name Symbol Table Data /// @{ - SmallString<256> StringTable; + StringTableBuilder StringTable; std::vector LocalSymbolData; std::vector ExternalSymbolData; std::vector UndefinedSymbolData; @@ -239,8 +240,7 @@ public: /// ComputeSymbolTable - Compute the symbol table data /// - /// \param StringTable [out] - The string table data. - void ComputeSymbolTable(MCAssembler &Asm, SmallString<256> &StringTable, + void ComputeSymbolTable(MCAssembler &Asm, std::vector &LocalSymbolData, std::vector &ExternalSymbolData, std::vector &UndefinedSymbolData); diff --git a/contrib/llvm/include/llvm/MC/MCObjectDisassembler.h b/contrib/llvm/include/llvm/MC/MCObjectDisassembler.h deleted file mode 100644 index 5b935db59585..000000000000 --- a/contrib/llvm/include/llvm/MC/MCObjectDisassembler.h +++ /dev/null @@ -1,174 +0,0 @@ -//===-- llvm/MC/MCObjectDisassembler.h --------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains the declaration of the MCObjectDisassembler class, which -// can be used to construct an MCModule and an MC CFG from an ObjectFile. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCOBJECTDISASSEMBLER_H -#define LLVM_MC_MCOBJECTDISASSEMBLER_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/DataTypes.h" -#include "llvm/Support/MemoryObject.h" -#include - -namespace llvm { - -namespace object { - class ObjectFile; - class MachOObjectFile; -} - -class MCBasicBlock; -class MCDisassembler; -class MCFunction; -class MCInstrAnalysis; -class MCModule; -class MCObjectSymbolizer; - -/// \brief Disassemble an ObjectFile to an MCModule and MCFunctions. -/// This class builds on MCDisassembler to disassemble whole sections, creating -/// MCAtom (MCTextAtom for disassembled sections and MCDataAtom for raw data). -/// It can also be used to create a control flow graph consisting of MCFunctions -/// and MCBasicBlocks. -class MCObjectDisassembler { -public: - MCObjectDisassembler(const object::ObjectFile &Obj, - const MCDisassembler &Dis, - const MCInstrAnalysis &MIA); - virtual ~MCObjectDisassembler() {} - - /// \brief Build an MCModule, creating atoms and optionally functions. - /// \param withCFG Also build a CFG by adding MCFunctions to the Module. - /// If withCFG is false, the MCModule built only contains atoms, representing - /// what was found in the object file. If withCFG is true, MCFunctions are - /// created, containing MCBasicBlocks. All text atoms are split to form basic - /// block atoms, which then each back an MCBasicBlock. - MCModule *buildModule(bool withCFG = false); - - MCModule *buildEmptyModule(); - - typedef std::vector AddressSetTy; - /// \name Create a new MCFunction. - MCFunction *createFunction(MCModule *Module, uint64_t BeginAddr, - AddressSetTy &CallTargets, - AddressSetTy &TailCallTargets); - - /// \brief Set the region on which to fallback if disassembly was requested - /// somewhere not accessible in the object file. - /// This is used for dynamic disassembly (see RawMemoryObject). - void setFallbackRegion(std::unique_ptr &Region) { - FallbackRegion.reset(Region.release()); - } - - /// \brief Set the symbolizer to use to get information on external functions. - /// Note that this isn't used to do instruction-level symbolization (that is, - /// plugged into MCDisassembler), but to symbolize function call targets. - void setSymbolizer(MCObjectSymbolizer *ObjectSymbolizer) { - MOS = ObjectSymbolizer; - } - - /// \brief Get the effective address of the entrypoint, or 0 if there is none. - virtual uint64_t getEntrypoint(); - - /// \name Get the addresses of static constructors/destructors in the object. - /// The caller is expected to know how to interpret the addresses; - /// for example, Mach-O init functions expect 5 arguments, not for ELF. - /// The addresses are original object file load addresses, not effective. - /// @{ - virtual ArrayRef getStaticInitFunctions(); - virtual ArrayRef getStaticExitFunctions(); - /// @} - - /// \name Translation between effective and objectfile load address. - /// @{ - /// \brief Compute the effective load address, from an objectfile virtual - /// address. This is implemented in a format-specific way, to take into - /// account things like PIE/ASLR when doing dynamic disassembly. - /// For example, on Mach-O this would be done by adding the VM addr slide, - /// on glibc ELF by keeping a map between segment load addresses, filled - /// using dl_iterate_phdr, etc.. - /// In most static situations and in the default impl., this returns \p Addr. - virtual uint64_t getEffectiveLoadAddr(uint64_t Addr); - - /// \brief Compute the original load address, as specified in the objectfile. - /// This is the inverse of getEffectiveLoadAddr. - virtual uint64_t getOriginalLoadAddr(uint64_t EffectiveAddr); - /// @} - -protected: - const object::ObjectFile &Obj; - const MCDisassembler &Dis; - const MCInstrAnalysis &MIA; - MCObjectSymbolizer *MOS; - - /// \brief The fallback memory region, outside the object file. - std::unique_ptr FallbackRegion; - - /// \brief Return a memory region suitable for reading starting at \p Addr. - /// In most cases, this returns a StringRefMemoryObject backed by the - /// containing section. When no section was found, this returns the - /// FallbackRegion, if it is suitable. - /// If it is not, or if there is no fallback region, this returns 0. - MemoryObject *getRegionFor(uint64_t Addr); - -private: - /// \brief Fill \p Module by creating an atom for each section. - /// This could be made much smarter, using information like symbols, but also - /// format-specific features, like mach-o function_start or data_in_code LCs. - void buildSectionAtoms(MCModule *Module); - - /// \brief Enrich \p Module with a CFG consisting of MCFunctions. - /// \param Module An MCModule returned by buildModule, with no CFG. - /// NOTE: Each MCBasicBlock in a MCFunction is backed by a single MCTextAtom. - /// When the CFG is built, contiguous instructions that were previously in a - /// single MCTextAtom will be split in multiple basic block atoms. - void buildCFG(MCModule *Module); - - MCBasicBlock *getBBAt(MCModule *Module, MCFunction *MCFN, uint64_t BeginAddr, - AddressSetTy &CallTargets, - AddressSetTy &TailCallTargets); -}; - -class MCMachOObjectDisassembler : public MCObjectDisassembler { - const object::MachOObjectFile &MOOF; - - uint64_t VMAddrSlide; - uint64_t HeaderLoadAddress; - - // __DATA;__mod_init_func support. - llvm::StringRef ModInitContents; - // __DATA;__mod_exit_func support. - llvm::StringRef ModExitContents; - -public: - /// \brief Construct a Mach-O specific object disassembler. - /// \param VMAddrSlide The virtual address slide applied by dyld. - /// \param HeaderLoadAddress The load address of the mach_header for this - /// object. - MCMachOObjectDisassembler(const object::MachOObjectFile &MOOF, - const MCDisassembler &Dis, - const MCInstrAnalysis &MIA, uint64_t VMAddrSlide, - uint64_t HeaderLoadAddress); - -protected: - uint64_t getEffectiveLoadAddr(uint64_t Addr) override; - uint64_t getOriginalLoadAddr(uint64_t EffectiveAddr) override; - uint64_t getEntrypoint() override; - - ArrayRef getStaticInitFunctions() override; - ArrayRef getStaticExitFunctions() override; -}; - -} - -#endif diff --git a/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h b/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h index 4d1715eccf5a..321043c522ee 100644 --- a/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_MC_MCBJECTFILEINFO_H -#define LLVM_MC_MCBJECTFILEINFO_H +#ifndef LLVM_MC_MCOBJECTFILEINFO_H +#define LLVM_MC_MCOBJECTFILEINFO_H #include "llvm/ADT/Triple.h" #include "llvm/Support/CodeGen.h" @@ -116,6 +116,7 @@ protected: /// These are used for the Fission separate debug information files. const MCSection *DwarfInfoDWOSection; + const MCSection *DwarfTypesDWOSection; const MCSection *DwarfAbbrevDWOSection; const MCSection *DwarfStrDWOSection; const MCSection *DwarfLineDWOSection; @@ -261,7 +262,9 @@ public: return DwarfInfoDWOSection; } const MCSection *getDwarfTypesSection(uint64_t Hash) const; - const MCSection *getDwarfTypesDWOSection(uint64_t Hash) const; + const MCSection *getDwarfTypesDWOSection() const { + return DwarfTypesDWOSection; + } const MCSection *getDwarfAbbrevDWOSection() const { return DwarfAbbrevDWOSection; } diff --git a/contrib/llvm/include/llvm/MC/MCObjectStreamer.h b/contrib/llvm/include/llvm/MC/MCObjectStreamer.h index 8d37c85b0585..0866ff5a9fc0 100644 --- a/contrib/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCObjectStreamer.h @@ -10,6 +10,7 @@ #ifndef LLVM_MC_MCOBJECTSTREAMER_H #define LLVM_MC_MCOBJECTSTREAMER_H +#include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCStreamer.h" @@ -37,11 +38,16 @@ class MCObjectStreamer : public MCStreamer { MCSectionData::iterator CurInsertionPoint; bool EmitEHFrame; bool EmitDebugFrame; + SmallVector PendingLabels; virtual void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0; void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override; void EmitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override; + // If any labels have been emitted but not assigned fragments, ensure that + // they get assigned, either to F if possible or to a new data fragment. + void flushPendingLabels(MCFragment *F); + protected: MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, raw_ostream &_OS, MCCodeEmitter *_Emitter); @@ -69,14 +75,15 @@ protected: MCFragment *getCurrentFragment() const; - void insert(MCFragment *F) const { + void insert(MCFragment *F) { + flushPendingLabels(F); CurSectionData->getFragmentList().insert(CurInsertionPoint, F); F->setParent(CurSectionData); } /// Get a data fragment to write into, creating a new one if the current /// fragment is not a data fragment. - MCDataFragment *getOrCreateDataFragment() const; + MCDataFragment *getOrCreateDataFragment(); public: void visitUsedSymbol(const MCSymbol &Sym) override; @@ -126,7 +133,7 @@ public: void EmitZeros(uint64_t NumBytes) override; void FinishImpl() override; - virtual bool mayHaveInstructions() const { + bool mayHaveInstructions() const override { return getCurrentSectionData()->hasInstructions(); } }; diff --git a/contrib/llvm/include/llvm/MC/MCObjectSymbolizer.h b/contrib/llvm/include/llvm/MC/MCObjectSymbolizer.h deleted file mode 100644 index f75b7f58a141..000000000000 --- a/contrib/llvm/include/llvm/MC/MCObjectSymbolizer.h +++ /dev/null @@ -1,83 +0,0 @@ -//===-- llvm/MC/MCObjectSymbolizer.h --------------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares the MCObjectSymbolizer class, an MCSymbolizer that is -// backed by an object::ObjectFile. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCOBJECTSYMBOLIZER_H -#define LLVM_MC_MCOBJECTSYMBOLIZER_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/MC/MCSymbolizer.h" -#include "llvm/Object/ObjectFile.h" -#include - -namespace llvm { - -class MCExpr; -class MCInst; -class MCRelocationInfo; -class raw_ostream; - -/// \brief An ObjectFile-backed symbolizer. -class MCObjectSymbolizer : public MCSymbolizer { -protected: - const object::ObjectFile *Obj; - - // Map a load address to the first relocation that applies there. As far as I - // know, if there are several relocations at the exact same address, they are - // related and the others can be determined from the first that was found in - // the relocation table. For instance, on x86-64 mach-o, a SUBTRACTOR - // relocation (referencing the minuend symbol) is followed by an UNSIGNED - // relocation (referencing the subtrahend symbol). - const object::RelocationRef *findRelocationAt(uint64_t Addr); - const object::SectionRef *findSectionContaining(uint64_t Addr); - - MCObjectSymbolizer(MCContext &Ctx, std::unique_ptr RelInfo, - const object::ObjectFile *Obj); - -public: - /// \name Overridden MCSymbolizer methods: - /// @{ - bool tryAddingSymbolicOperand(MCInst &MI, raw_ostream &cStream, - int64_t Value, uint64_t Address, - bool IsBranch, uint64_t Offset, - uint64_t InstSize) override; - - void tryAddingPcLoadReferenceComment(raw_ostream &cStream, - int64_t Value, - uint64_t Address) override; - /// @} - - /// \brief Look for an external function symbol at \p Addr. - /// (References through the ELF PLT, Mach-O stubs, and similar). - /// \returns An MCExpr representing the external symbol, or 0 if not found. - virtual StringRef findExternalFunctionAt(uint64_t Addr); - - /// \brief Create an object symbolizer for \p Obj. - static MCObjectSymbolizer * - createObjectSymbolizer(MCContext &Ctx, - std::unique_ptr RelInfo, - const object::ObjectFile *Obj); - -private: - typedef DenseMap AddrToRelocMap; - typedef std::vector SortedSectionList; - SortedSectionList SortedSections; - AddrToRelocMap AddrToReloc; - - void buildSectionList(); - void buildRelocationByAddrMap(); -}; - -} - -#endif diff --git a/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h b/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h index 0b550ba627e9..a9a30f172071 100644 --- a/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h +++ b/contrib/llvm/include/llvm/MC/MCParser/AsmLexer.h @@ -49,7 +49,7 @@ public: const AsmToken peekTok(bool ShouldSkipSpace = true) override; - bool isAtStartOfComment(char Char); + bool isAtStartOfComment(const char *Ptr); bool isAtStatementSeparator(const char *Ptr); const MCAsmInfo &getMAI() const { return MAI; } diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h b/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h index e3d4181e086f..b05891c13250 100644 --- a/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h +++ b/contrib/llvm/include/llvm/MC/MCParser/MCAsmLexer.h @@ -18,7 +18,7 @@ namespace llvm { -/// AsmToken - Target independent representation for an assembler token. +/// Target independent representation for an assembler token. class AsmToken { public: enum TokenKind { @@ -74,25 +74,26 @@ public: SMLoc getLoc() const; SMLoc getEndLoc() const; + SMRange getLocRange() const; - /// getStringContents - Get the contents of a string token (without quotes). + /// Get the contents of a string token (without quotes). StringRef getStringContents() const { assert(Kind == String && "This token isn't a string!"); return Str.slice(1, Str.size() - 1); } - /// getIdentifier - Get the identifier string for the current token, which - /// should be an identifier or a string. This gets the portion of the string - /// which should be used as the identifier, e.g., it does not include the - /// quotes on strings. + /// Get the identifier string for the current token, which should be an + /// identifier or a string. This gets the portion of the string which should + /// be used as the identifier, e.g., it does not include the quotes on + /// strings. StringRef getIdentifier() const { if (Kind == Identifier) return getString(); return getStringContents(); } - /// getString - Get the string for the current token, this includes all - /// characters (for example, the quotes on strings) in the token. + /// Get the string for the current token, this includes all characters (for + /// example, the quotes on strings) in the token. /// /// The returned StringRef points into the source manager's memory buffer, and /// is safe to store across calls to Lex(). @@ -113,8 +114,8 @@ public: } }; -/// MCAsmLexer - Generic assembler lexer interface, for use by target specific -/// assembly lexers. +/// Generic assembler lexer interface, for use by target specific assembly +/// lexers. class MCAsmLexer { /// The current token, stored in the base class for faster access. AsmToken CurTok; @@ -142,7 +143,7 @@ protected: // Can only create subclasses. public: virtual ~MCAsmLexer(); - /// Lex - Consume the next token from the input stream and return it. + /// Consume the next token from the input stream and return it. /// /// The lexer will continuosly return the end-of-file token once the end of /// the main input file has been reached. @@ -152,37 +153,37 @@ public: virtual StringRef LexUntilEndOfStatement() = 0; - /// getLoc - Get the current source location. + /// Get the current source location. SMLoc getLoc() const; - /// getTok - Get the current (last) lexed token. - const AsmToken &getTok() { + /// Get the current (last) lexed token. + const AsmToken &getTok() const { return CurTok; } - /// peekTok - Look ahead at the next token to be lexed. + /// Look ahead at the next token to be lexed. virtual const AsmToken peekTok(bool ShouldSkipSpace = true) = 0; - /// getErrLoc - Get the current error location + /// Get the current error location const SMLoc &getErrLoc() { return ErrLoc; } - /// getErr - Get the current error string + /// Get the current error string const std::string &getErr() { return Err; } - /// getKind - Get the kind of current token. + /// Get the kind of current token. AsmToken::TokenKind getKind() const { return CurTok.getKind(); } - /// is - Check if the current token has kind \p K. + /// Check if the current token has kind \p K. bool is(AsmToken::TokenKind K) const { return CurTok.is(K); } - /// isNot - Check if the current token has kind \p K. + /// Check if the current token has kind \p K. bool isNot(AsmToken::TokenKind K) const { return CurTok.isNot(K); } - /// setSkipSpace - Set whether spaces should be ignored by the lexer + /// Set whether spaces should be ignored by the lexer void setSkipSpace(bool val) { SkipSpace = val; } bool getAllowAtInIdentifier() { return AllowAtInIdentifier; } diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h index 9836795450ff..34188e66e62d 100644 --- a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h +++ b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h @@ -45,20 +45,22 @@ public: } }; -/// MCAsmParserSemaCallback - Generic Sema callback for assembly parser. +/// Generic Sema callback for assembly parser. class MCAsmParserSemaCallback { public: virtual ~MCAsmParserSemaCallback(); virtual void *LookupInlineAsmIdentifier(StringRef &LineBuf, InlineAsmIdentifierInfo &Info, bool IsUnevaluatedContext) = 0; + virtual StringRef LookupInlineAsmLabel(StringRef Identifier, SourceMgr &SM, + SMLoc Location, bool Create) = 0; virtual bool LookupInlineAsmField(StringRef Base, StringRef Member, unsigned &Offset) = 0; }; -/// MCAsmParser - Generic assembler parser interface, for use by target specific -/// assembly parsers. +/// Generic assembler parser interface, for use by target specific assembly +/// parsers. class MCAsmParser { public: typedef bool (*DirectiveHandler)(MCAsmParserExtension*, StringRef, SMLoc); @@ -85,10 +87,13 @@ public: virtual SourceMgr &getSourceManager() = 0; virtual MCAsmLexer &getLexer() = 0; + const MCAsmLexer &getLexer() const { + return const_cast(this)->getLexer(); + } virtual MCContext &getContext() = 0; - /// getStreamer - Return the output streamer for the assembler. + /// Return the output streamer for the assembler. virtual MCStreamer &getStreamer() = 0; MCTargetAsmParser &getTargetParser() const { return *TargetParser; } @@ -100,51 +105,49 @@ public: bool getShowParsedOperands() const { return ShowParsedOperands; } void setShowParsedOperands(bool Value) { ShowParsedOperands = Value; } - /// Run - Run the parser on the input source buffer. + /// Run the parser on the input source buffer. virtual bool Run(bool NoInitialTextSection, bool NoFinalize = false) = 0; virtual void setParsingInlineAsm(bool V) = 0; virtual bool isParsingInlineAsm() = 0; - /// parseMSInlineAsm - Parse ms-style inline assembly. - virtual bool parseMSInlineAsm(void *AsmLoc, std::string &AsmString, - unsigned &NumOutputs, unsigned &NumInputs, - SmallVectorImpl > &OpDecls, - SmallVectorImpl &Constraints, - SmallVectorImpl &Clobbers, - const MCInstrInfo *MII, - const MCInstPrinter *IP, - MCAsmParserSemaCallback &SI) = 0; + /// Parse ms-style inline assembly. + virtual bool parseMSInlineAsm( + void *AsmLoc, std::string &AsmString, unsigned &NumOutputs, + unsigned &NumInputs, SmallVectorImpl> &OpDecls, + SmallVectorImpl &Constraints, + SmallVectorImpl &Clobbers, const MCInstrInfo *MII, + const MCInstPrinter *IP, MCAsmParserSemaCallback &SI) = 0; - /// Note - Emit a note at the location \p L, with the message \p Msg. + /// Emit a note at the location \p L, with the message \p Msg. virtual void Note(SMLoc L, const Twine &Msg, ArrayRef Ranges = None) = 0; - /// Warning - Emit a warning at the location \p L, with the message \p Msg. + /// Emit a warning at the location \p L, with the message \p Msg. /// /// \return The return value is true, if warnings are fatal. virtual bool Warning(SMLoc L, const Twine &Msg, ArrayRef Ranges = None) = 0; - /// Error - Emit an error at the location \p L, with the message \p Msg. + /// Emit an error at the location \p L, with the message \p Msg. /// /// \return The return value is always true, as an idiomatic convenience to /// clients. virtual bool Error(SMLoc L, const Twine &Msg, ArrayRef Ranges = None) = 0; - /// Lex - Get the next AsmToken in the stream, possibly handling file - /// inclusion first. + /// Get the next AsmToken in the stream, possibly handling file inclusion + /// first. virtual const AsmToken &Lex() = 0; - /// getTok - Get the current AsmToken from the stream. - const AsmToken &getTok(); + /// Get the current AsmToken from the stream. + const AsmToken &getTok() const; /// \brief Report an error at the current lexer location. bool TokError(const Twine &Msg, ArrayRef Ranges = None); - /// parseIdentifier - Parse an identifier or string (as a quoted identifier) - /// and set \p Res to the identifier contents. + /// Parse an identifier or string (as a quoted identifier) and set \p Res to + /// the identifier contents. virtual bool parseIdentifier(StringRef &Res) = 0; /// \brief Parse up to the end of statement and return the contents from the @@ -152,15 +155,14 @@ public: /// will be either the EndOfStatement or EOF. virtual StringRef parseStringToEndOfStatement() = 0; - /// parseEscapedString - Parse the current token as a string which may include - /// escaped characters and return the string contents. + /// Parse the current token as a string which may include escaped characters + /// and return the string contents. virtual bool parseEscapedString(std::string &Data) = 0; - /// eatToEndOfStatement - Skip to the end of the current statement, for error - /// recovery. + /// Skip to the end of the current statement, for error recovery. virtual void eatToEndOfStatement() = 0; - /// parseExpression - Parse an arbitrary expression. + /// Parse an arbitrary expression. /// /// @param Res - The value of the expression. The result is undefined /// on error. @@ -168,31 +170,30 @@ public: virtual bool parseExpression(const MCExpr *&Res, SMLoc &EndLoc) = 0; bool parseExpression(const MCExpr *&Res); - /// parsePrimaryExpr - Parse a primary expression. + /// Parse a primary expression. /// /// @param Res - The value of the expression. The result is undefined /// on error. /// @result - False on success. virtual bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) = 0; - /// parseParenExpression - Parse an arbitrary expression, assuming that an - /// initial '(' has already been consumed. + /// Parse an arbitrary expression, assuming that an initial '(' has already + /// been consumed. /// /// @param Res - The value of the expression. The result is undefined /// on error. /// @result - False on success. virtual bool parseParenExpression(const MCExpr *&Res, SMLoc &EndLoc) = 0; - /// parseAbsoluteExpression - Parse an expression which must evaluate to an - /// absolute value. + /// Parse an expression which must evaluate to an absolute value. /// /// @param Res - The value of the absolute expression. The result is undefined /// on error. /// @result - False on success. virtual bool parseAbsoluteExpression(int64_t &Res) = 0; - /// checkForValidSection - Ensure that we have a valid section set in the - /// streamer. Otherwise, report an error and switch to .text. + /// Ensure that we have a valid section set in the streamer. Otherwise, report + /// an error and switch to .text. virtual void checkForValidSection() = 0; }; diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h index 2eda3a9a2143..bfc0afa132b7 100644 --- a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h +++ b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h @@ -52,8 +52,17 @@ public: /// @{ MCContext &getContext() { return getParser().getContext(); } + MCAsmLexer &getLexer() { return getParser().getLexer(); } + const MCAsmLexer &getLexer() const { + return const_cast(this)->getLexer(); + } + MCAsmParser &getParser() { return *Parser; } + const MCAsmParser &getParser() const { + return const_cast(this)->getParser(); + } + SourceMgr &getSourceManager() { return getParser().getSourceManager(); } MCStreamer &getStreamer() { return getParser().getStreamer(); } bool Warning(SMLoc L, const Twine &Msg) { diff --git a/contrib/llvm/include/llvm/MC/MCRegisterInfo.h b/contrib/llvm/include/llvm/MC/MCRegisterInfo.h index 766f63182925..8e25ee18e08d 100644 --- a/contrib/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/contrib/llvm/include/llvm/MC/MCRegisterInfo.h @@ -32,9 +32,9 @@ public: typedef const MCPhysReg* iterator; typedef const MCPhysReg* const_iterator; - const char *Name; const iterator RegsBegin; const uint8_t *const RegSet; + const uint32_t NameIdx; const uint16_t RegsSize; const uint16_t RegSetSize; const uint16_t ID; @@ -46,10 +46,6 @@ public: /// unsigned getID() const { return ID; } - /// getName() - Return the register class name for debugging. - /// - const char *getName() const { return Name; } - /// begin/end - Return all of the registers in this class. /// iterator begin() const { return RegsBegin; } @@ -118,6 +114,10 @@ struct MCRegisterDesc { // RegUnits - Points to the list of register units. The low 4 bits holds the // Scale, the high bits hold an offset into DiffLists. See MCRegUnitIterator. uint32_t RegUnits; + + /// Index into list with lane mask sequences. The sequence contains a lanemask + /// for every register unit. + uint16_t RegUnitLaneMasks; }; /// MCRegisterInfo base class - We assume that the target defines a static @@ -161,7 +161,10 @@ private: unsigned NumRegUnits; // Number of regunits. const MCPhysReg (*RegUnitRoots)[2]; // Pointer to regunit root table. const MCPhysReg *DiffLists; // Pointer to the difflists array + const unsigned *RegUnitMaskSequences; // Pointer to lane mask sequences + // for register units. const char *RegStrings; // Pointer to the string table. + const char *RegClassStrings; // Pointer to the class strings. const uint16_t *SubRegIndices; // Pointer to the subreg lookup // array. const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered @@ -230,8 +233,10 @@ public: // These iterators are allowed to sub-class DiffListIterator and access // internal list pointers. friend class MCSubRegIterator; + friend class MCSubRegIndexIterator; friend class MCSuperRegIterator; friend class MCRegUnitIterator; + friend class MCRegUnitMaskIterator; friend class MCRegUnitRootIterator; /// \brief Initialize MCRegisterInfo, called by TableGen @@ -242,7 +247,9 @@ public: const MCPhysReg (*RURoots)[2], unsigned NRU, const MCPhysReg *DL, + const unsigned *RUMS, const char *Strings, + const char *ClassStrings, const uint16_t *SubIndices, unsigned NumIndices, const SubRegCoveredBits *SubIdxRanges, @@ -253,7 +260,9 @@ public: PCReg = PC; Classes = C; DiffLists = DL; + RegUnitMaskSequences = RUMS; RegStrings = Strings; + RegClassStrings = ClassStrings; NumClasses = NC; RegUnitRoots = RURoots; NumRegUnits = NRU; @@ -401,6 +410,10 @@ public: return Classes[i]; } + const char *getRegClassName(const MCRegisterClass *Class) const { + return RegClassStrings + Class->NameIdx; + } + /// \brief Returns the encoding for RegNo uint16_t getEncodingValue(unsigned RegNo) const { assert(RegNo < NumRegs && @@ -449,6 +462,38 @@ public: } }; +/// Iterator that enumerates the sub-registers of a Reg and the associated +/// sub-register indices. +class MCSubRegIndexIterator { + MCSubRegIterator SRIter; + const uint16_t *SRIndex; +public: + /// Constructs an iterator that traverses subregisters and their + /// associated subregister indices. + MCSubRegIndexIterator(unsigned Reg, const MCRegisterInfo *MCRI) + : SRIter(Reg, MCRI) { + SRIndex = MCRI->SubRegIndices + MCRI->get(Reg).SubRegIndices; + } + + /// Returns current sub-register. + unsigned getSubReg() const { + return *SRIter; + } + /// Returns sub-register index of the current sub-register. + unsigned getSubRegIndex() const { + return *SRIndex; + } + + /// Returns true if this iterator is not yet at the end. + bool isValid() const { return SRIter.isValid(); } + + /// Moves to the next position. + void operator++() { + ++SRIter; + ++SRIndex; + } +}; + /// MCSuperRegIterator enumerates all super-registers of Reg. /// If IncludeSelf is set, Reg itself is included in the list. class MCSuperRegIterator : public MCRegisterInfo::DiffListIterator { @@ -510,6 +555,36 @@ public: } }; +/// MCRegUnitIterator enumerates a list of register units and their associated +/// lane masks for Reg. The register units are in ascending numerical order. +class MCRegUnitMaskIterator { + MCRegUnitIterator RUIter; + const unsigned *MaskListIter; +public: + MCRegUnitMaskIterator() {} + /// Constructs an iterator that traverses the register units and their + /// associated LaneMasks in Reg. + MCRegUnitMaskIterator(unsigned Reg, const MCRegisterInfo *MCRI) + : RUIter(Reg, MCRI) { + uint16_t Idx = MCRI->get(Reg).RegUnitLaneMasks; + MaskListIter = &MCRI->RegUnitMaskSequences[Idx]; + } + + /// Returns a (RegUnit, LaneMask) pair. + std::pair operator*() const { + return std::make_pair(*RUIter, *MaskListIter); + } + + /// Returns true if this iterator is not yet at the end. + bool isValid() const { return RUIter.isValid(); } + + /// Moves to the next position. + void operator++() { + ++MaskListIter; + ++RUIter; + } +}; + // Each register unit has one or two root registers. The complete set of // registers containing a register unit is the union of the roots and their // super-registers. All registers aliasing Unit can be visited like this: diff --git a/contrib/llvm/include/llvm/MC/MCSchedule.h b/contrib/llvm/include/llvm/MC/MCSchedule.h index 43b8672265c6..1adfedd2638a 100644 --- a/contrib/llvm/include/llvm/MC/MCSchedule.h +++ b/contrib/llvm/include/llvm/MC/MCSchedule.h @@ -133,10 +133,7 @@ struct MCSchedClassDesc { /// provides a detailed reservation table describing each cycle of instruction /// execution. Subtargets may define any or all of the above categories of data /// depending on the type of CPU and selected scheduler. -class MCSchedModel { -public: - static MCSchedModel DefaultSchedModel; // For unknown processors. - +struct MCSchedModel { // IssueWidth is the maximum number of instructions that may be scheduled in // the same per-cycle group. unsigned IssueWidth; @@ -191,7 +188,6 @@ public: bool CompleteModel; -private: unsigned ProcID; const MCProcResourceDesc *ProcResourceTable; const MCSchedClassDesc *SchedClassTable; @@ -201,37 +197,6 @@ private: friend class InstrItineraryData; const InstrItinerary *InstrItineraries; -public: - // Default's must be specified as static const literals so that tablegenerated - // target code can use it in static initializers. The defaults need to be - // initialized in this default ctor because some clients directly instantiate - // MCSchedModel instead of using a generated itinerary. - MCSchedModel(): IssueWidth(DefaultIssueWidth), - MicroOpBufferSize(DefaultMicroOpBufferSize), - LoopMicroOpBufferSize(DefaultLoopMicroOpBufferSize), - LoadLatency(DefaultLoadLatency), - HighLatency(DefaultHighLatency), - MispredictPenalty(DefaultMispredictPenalty), - PostRAScheduler(false), CompleteModel(true), - ProcID(0), ProcResourceTable(nullptr), - SchedClassTable(nullptr), NumProcResourceKinds(0), - NumSchedClasses(0), InstrItineraries(nullptr) { - (void)NumProcResourceKinds; - (void)NumSchedClasses; - } - - // Table-gen driven ctor. - MCSchedModel(unsigned iw, int mbs, int lmbs, unsigned ll, unsigned hl, - unsigned mp, bool postRASched, bool cm, unsigned pi, - const MCProcResourceDesc *pr, const MCSchedClassDesc *sc, - unsigned npr, unsigned nsc, const InstrItinerary *ii): - IssueWidth(iw), MicroOpBufferSize(mbs), LoopMicroOpBufferSize(lmbs), - LoadLatency(ll), HighLatency(hl), - MispredictPenalty(mp), PostRAScheduler(postRASched), - CompleteModel(cm), ProcID(pi), - ProcResourceTable(pr), SchedClassTable(sc), NumProcResourceKinds(npr), - NumSchedClasses(nsc), InstrItineraries(ii) {} - unsigned getProcessorID() const { return ProcID; } /// Does this machine model include instruction-level scheduling. @@ -258,6 +223,26 @@ public: assert(SchedClassIdx < NumSchedClasses && "bad scheduling class idx"); return &SchedClassTable[SchedClassIdx]; } + + // /\brief Returns a default initialized model. Used for unknown processors. + static MCSchedModel GetDefaultSchedModel() { + MCSchedModel Ret = { DefaultIssueWidth, + DefaultMicroOpBufferSize, + DefaultLoopMicroOpBufferSize, + DefaultLoadLatency, + DefaultHighLatency, + DefaultMispredictPenalty, + false, + true, + 0, + nullptr, + nullptr, + 0, + 0, + nullptr + }; + return Ret; + } }; } // End llvm namespace diff --git a/contrib/llvm/include/llvm/MC/MCStreamer.h b/contrib/llvm/include/llvm/MC/MCStreamer.h index 63a43d08c3f2..18855f9cf03d 100644 --- a/contrib/llvm/include/llvm/MC/MCStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCStreamer.h @@ -20,7 +20,7 @@ #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCLinkerOptimizationHint.h" -#include "llvm/MC/MCWin64EH.h" +#include "llvm/MC/MCWinEH.h" #include "llvm/Support/DataTypes.h" #include @@ -49,14 +49,14 @@ typedef std::pair MCSectionSubPair; /// /// If target foo wants to use this, it should implement 3 classes: /// * FooTargetStreamer : public MCTargetStreamer -/// * FooTargetAsmSreamer : public FooTargetStreamer +/// * FooTargetAsmStreamer : public FooTargetStreamer /// * FooTargetELFStreamer : public FooTargetStreamer /// /// FooTargetStreamer should have a pure virtual method for each directive. For /// example, for a ".bar symbol_name" directive, it should have /// virtual emitBar(const MCSymbol &Symbol) = 0; /// -/// The FooTargetAsmSreamer and FooTargetELFStreamer classes implement the +/// The FooTargetAsmStreamer and FooTargetELFStreamer classes implement the /// method. The assembly streamer just prints ".bar symbol_name". The object /// streamer does whatever is needed to implement .bar in the object file. /// @@ -66,8 +66,9 @@ typedef std::pair MCSectionSubPair; /// MCTargetStreamer &TS = OutStreamer.getTargetStreamer(); /// FooTargetStreamer &ATS = static_cast(TS); /// -/// The base classes FooTargetAsmSreamer and FooTargetELFStreamer should *never* -/// be treated differently. Callers should always talk to a FooTargetStreamer. +/// The base classes FooTargetAsmStreamer and FooTargetELFStreamer should +/// *never* be treated differently. Callers should always talk to a +/// FooTargetStreamer. class MCTargetStreamer { protected: MCStreamer &Streamer; @@ -91,7 +92,6 @@ public: AArch64TargetStreamer(MCStreamer &S); ~AArch64TargetStreamer(); - void finish() override; /// Callback used to implement the ldr= pseudo. @@ -103,6 +103,9 @@ public: /// Emit contents of constant pool for the current section. void emitCurrentConstantPool(); + /// Callback used to implement the .inst directive. + virtual void emitInst(uint32_t Inst); + private: std::unique_ptr ConstantPools; }; @@ -181,8 +184,8 @@ class MCStreamer { MCSymbol *EmitCFICommon(); - std::vector WinFrameInfos; - MCWinFrameInfo *CurrentWinFrameInfo; + std::vector WinFrameInfos; + WinEH::FrameInfo *CurrentWinFrameInfo; void EnsureValidWinFrameInfo(); // SymbolOrdering - Tracks an index to represent the order @@ -196,19 +199,14 @@ class MCStreamer { protected: MCStreamer(MCContext &Ctx); - const MCExpr *BuildSymbolDiff(MCContext &Context, const MCSymbol *A, - const MCSymbol *B); - - const MCExpr *ForceExpAbs(const MCExpr *Expr); - virtual void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame); virtual void EmitCFIEndProcImpl(MCDwarfFrameInfo &CurFrame); - MCWinFrameInfo *getCurrentWinFrameInfo() { + WinEH::FrameInfo *getCurrentWinFrameInfo() { return CurrentWinFrameInfo; } - void EmitWindowsUnwindTables(); + virtual void EmitWindowsUnwindTables(); virtual void EmitRawTextImpl(StringRef String); @@ -238,7 +236,7 @@ public: } unsigned getNumWinFrameInfos() { return WinFrameInfos.size(); } - ArrayRef getWinFrameInfos() const { + ArrayRef getWinFrameInfos() const { return WinFrameInfos; } @@ -349,8 +347,8 @@ public: /// @p Section. This is required to update CurSection. /// /// This corresponds to assembler directives like .section, .text, etc. - void SwitchSection(const MCSection *Section, - const MCExpr *Subsection = nullptr) { + virtual void SwitchSection(const MCSection *Section, + const MCExpr *Subsection = nullptr) { assert(Section && "Cannot switch to a null section!"); MCSectionSubPair curSection = SectionStack.back().first; SectionStack.back().second = curSection; @@ -373,7 +371,7 @@ public: } /// Create the default sections and set the initial one. - virtual void InitSections(); + virtual void InitSections(bool NoExecStack); /// AssignSection - Sets the symbol's section. /// @@ -552,12 +550,6 @@ public: /// to pass in a MCExpr for constant integers. virtual void EmitIntValue(uint64_t Value, unsigned Size); - /// EmitAbsValue - Emit the Value, but try to avoid relocations. On MachO - /// this is done by producing - /// foo = value - /// .long foo - void EmitAbsValue(const MCExpr *Value, unsigned Size); - virtual void EmitULEB128Value(const MCExpr *Value); virtual void EmitSLEB128Value(const MCExpr *Value); @@ -669,11 +661,6 @@ public: StringRef FileName); virtual MCSymbol *getDwarfLineTableSymbol(unsigned CUID); - - void EmitDwarfSetLineAddr(int64_t LineDelta, const MCSymbol *Label, - int PointerSize); - - virtual void EmitCompactUnwindEncoding(uint32_t CompactUnwindEncoding); virtual void EmitCFISections(bool EH, bool Debug); void EmitCFIStartProc(bool IsSimple); void EmitCFIEndProc(); @@ -782,8 +769,8 @@ MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB, /// createELFStreamer - Create a machine code streamer which will generate /// ELF format object files. MCStreamer *createELFStreamer(MCContext &Ctx, MCAsmBackend &TAB, - raw_ostream &OS, MCCodeEmitter *CE, bool RelaxAll, - bool NoExecStack); + raw_ostream &OS, MCCodeEmitter *CE, + bool RelaxAll); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h b/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h index ebcfee904c2f..3f38bd59a576 100644 --- a/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h +++ b/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_MC_MCSUBTARGET_H -#define LLVM_MC_MCSUBTARGET_H +#ifndef LLVM_MC_MCSUBTARGETINFO_H +#define LLVM_MC_MCSUBTARGETINFO_H #include "llvm/MC/MCInstrItineraries.h" #include "llvm/MC/SubtargetFeature.h" @@ -36,7 +36,7 @@ class MCSubtargetInfo { const MCWriteProcResEntry *WriteProcResTable; const MCWriteLatencyEntry *WriteLatencyTable; const MCReadAdvanceEntry *ReadAdvanceTable; - const MCSchedModel *CPUSchedModel; + MCSchedModel CPUSchedModel; const InstrStage *Stages; // Instruction itinerary stages const unsigned *OperandCycles; // Itinerary operand cycles @@ -65,6 +65,10 @@ public: return FeatureBits; } + /// setFeatureBits - Set the feature bits. + /// + void setFeatureBits(uint64_t FeatureBits_) { FeatureBits = FeatureBits_; } + /// InitMCProcessorInfo - Set or change the CPU (optionally supplemented with /// feature string). Recompute feature bits and scheduling model. void InitMCProcessorInfo(StringRef CPU, StringRef FS); @@ -82,11 +86,11 @@ public: /// getSchedModelForCPU - Get the machine model of a CPU. /// - const MCSchedModel *getSchedModelForCPU(StringRef CPU) const; + MCSchedModel getSchedModelForCPU(StringRef CPU) const; /// getSchedModel - Get the machine model for this subtarget's CPU. /// - const MCSchedModel *getSchedModel() const { return CPUSchedModel; } + const MCSchedModel &getSchedModel() const { return CPUSchedModel; } /// Return an iterator at the first process resource consumed by the given /// scheduling class. diff --git a/contrib/llvm/include/llvm/MC/MCSymbol.h b/contrib/llvm/include/llvm/MC/MCSymbol.h index 0b3c3ceb210f..47a8789d463b 100644 --- a/contrib/llvm/include/llvm/MC/MCSymbol.h +++ b/contrib/llvm/include/llvm/MC/MCSymbol.h @@ -53,6 +53,9 @@ namespace llvm { /// "Lfoo" or ".foo". unsigned IsTemporary : 1; + /// \brief True if this symbol can be redefined. + unsigned IsRedefinable : 1; + /// IsUsed - True if this symbol has been used. mutable unsigned IsUsed : 1; @@ -61,7 +64,7 @@ namespace llvm { friend class MCContext; MCSymbol(StringRef name, bool isTemporary) : Name(name), Section(nullptr), Value(nullptr), - IsTemporary(isTemporary), IsUsed(false) {} + IsTemporary(isTemporary), IsRedefinable(false), IsUsed(false) {} MCSymbol(const MCSymbol&) LLVM_DELETED_FUNCTION; void operator=(const MCSymbol&) LLVM_DELETED_FUNCTION; @@ -79,6 +82,19 @@ namespace llvm { bool isUsed() const { return IsUsed; } void setUsed(bool Value) const { IsUsed = Value; } + /// \brief Check if this symbol is redefinable. + bool isRedefinable() const { return IsRedefinable; } + /// \brief Mark this symbol as redefinable. + void setRedefinable(bool Value) { IsRedefinable = Value; } + /// \brief Prepare this symbol to be redefined. + void redefineIfPossible() { + if (IsRedefinable) { + Value = nullptr; + Section = nullptr; + IsRedefinable = false; + } + } + /// @} /// @name Associated Sections /// @{ diff --git a/contrib/llvm/include/llvm/MC/MCTargetAsmParser.h b/contrib/llvm/include/llvm/MC/MCTargetAsmParser.h index 9a5881b14b44..ea71d1f433a4 100644 --- a/contrib/llvm/include/llvm/MC/MCTargetAsmParser.h +++ b/contrib/llvm/include/llvm/MC/MCTargetAsmParser.h @@ -7,13 +7,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_MC_TARGETPARSER_H -#define LLVM_MC_TARGETPARSER_H +#ifndef LLVM_MC_MCTARGETASMPARSER_H +#define LLVM_MC_MCTARGETASMPARSER_H #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCParser/MCAsmParserExtension.h" #include "llvm/MC/MCTargetOptions.h" - #include namespace llvm { @@ -38,20 +37,22 @@ enum AsmRewriteKind { AOK_Input, // Rewrite in terms of $N. AOK_Output, // Rewrite in terms of $N. AOK_SizeDirective, // Add a sizing directive (e.g., dword ptr). + AOK_Label, // Rewrite local labels. AOK_Skip // Skip emission (e.g., offset/type operators). }; const char AsmRewritePrecedence [] = { 0, // AOK_Delete - 1, // AOK_Align - 1, // AOK_DotOperator - 1, // AOK_Emit - 3, // AOK_Imm - 3, // AOK_ImmPrefix - 2, // AOK_Input - 2, // AOK_Output - 4, // AOK_SizeDirective - 1 // AOK_Skip + 2, // AOK_Align + 2, // AOK_DotOperator + 2, // AOK_Emit + 4, // AOK_Imm + 4, // AOK_ImmPrefix + 3, // AOK_Input + 3, // AOK_Output + 5, // AOK_SizeDirective + 1, // AOK_Label + 2 // AOK_Skip }; struct AsmRewrite { @@ -59,9 +60,12 @@ struct AsmRewrite { SMLoc Loc; unsigned Len; unsigned Val; + StringRef Label; public: AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len = 0, unsigned val = 0) : Kind(kind), Loc(loc), Len(len), Val(val) {} + AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len, StringRef label) + : Kind(kind), Loc(loc), Len(len), Val(0), Label(label) {} }; struct ParseInstructionInfo { @@ -93,7 +97,7 @@ protected: // Can only create subclasses. MCTargetAsmParser(); /// AvailableFeatures - The current set of available features. - unsigned AvailableFeatures; + uint64_t AvailableFeatures; /// ParsingInlineAsm - Are we parsing ms-style inline assembly? bool ParsingInlineAsm; @@ -108,12 +112,14 @@ protected: // Can only create subclasses. public: virtual ~MCTargetAsmParser(); - unsigned getAvailableFeatures() const { return AvailableFeatures; } - void setAvailableFeatures(unsigned Value) { AvailableFeatures = Value; } + uint64_t getAvailableFeatures() const { return AvailableFeatures; } + void setAvailableFeatures(uint64_t Value) { AvailableFeatures = Value; } bool isParsingInlineAsm () { return ParsingInlineAsm; } void setParsingInlineAsm (bool Value) { ParsingInlineAsm = Value; } + MCTargetOptions getTargetOptions() const { return MCOptions; } + void setSemaCallback(MCAsmParserSemaCallback *Callback) { SemaCallback = Callback; } @@ -121,6 +127,9 @@ public: virtual bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) = 0; + /// Sets frame register corresponding to the current MachineFunction. + virtual void SetFrameRegister(unsigned RegNo) {} + /// ParseInstruction - Parse one assembly instruction. /// /// The parser is positioned following the instruction name. The target @@ -161,7 +170,7 @@ public: /// explaining the match failure. virtual bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, OperandVector &Operands, MCStreamer &Out, - unsigned &ErrorInfo, + uint64_t &ErrorInfo, bool MatchingInlineAsm) = 0; /// Allows targets to let registers opt out of clobber lists. diff --git a/contrib/llvm/include/llvm/MC/MCTargetOptions.h b/contrib/llvm/include/llvm/MC/MCTargetOptions.h index eb4348ed3ec2..ce28a196e974 100644 --- a/contrib/llvm/include/llvm/MC/MCTargetOptions.h +++ b/contrib/llvm/include/llvm/MC/MCTargetOptions.h @@ -10,8 +10,12 @@ #ifndef LLVM_MC_MCTARGETOPTIONS_H #define LLVM_MC_MCTARGETOPTIONS_H +#include + namespace llvm { +class StringRef; + class MCTargetOptions { public: enum AsmInstrumentation { @@ -24,12 +28,18 @@ public: bool MCRelaxAll : 1; bool MCNoExecStack : 1; + bool MCFatalWarnings : 1; bool MCSaveTempLabels : 1; bool MCUseDwarfDirectory : 1; bool ShowMCEncoding : 1; bool ShowMCInst : 1; bool AsmVerbose : 1; int DwarfVersion; + /// getABIName - If this returns a non-empty string this represents the + /// textual name of the ABI that we want the backend to use, e.g. o32, or + /// aapcs-linux. + StringRef getABIName() const; + std::string ABIName; MCTargetOptions(); }; @@ -38,12 +48,14 @@ inline bool operator==(const MCTargetOptions &LHS, const MCTargetOptions &RHS) { return (ARE_EQUAL(SanitizeAddress) && ARE_EQUAL(MCRelaxAll) && ARE_EQUAL(MCNoExecStack) && + ARE_EQUAL(MCFatalWarnings) && ARE_EQUAL(MCSaveTempLabels) && ARE_EQUAL(MCUseDwarfDirectory) && ARE_EQUAL(ShowMCEncoding) && ARE_EQUAL(ShowMCInst) && ARE_EQUAL(AsmVerbose) && - ARE_EQUAL(DwarfVersion)); + ARE_EQUAL(DwarfVersion) && + ARE_EQUAL(ABIName)); #undef ARE_EQUAL } diff --git a/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h b/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h index 6d4eb0ef5911..af23a92e6e99 100644 --- a/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h +++ b/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h @@ -15,8 +15,8 @@ #ifndef LLVM_MC_MCTARGETOPTIONSCOMMANDFLAGS_H #define LLVM_MC_MCTARGETOPTIONSCOMMANDFLAGS_H -#include "llvm/Support/CommandLine.h" #include "llvm/MC/MCTargetOptions.h" +#include "llvm/Support/CommandLine.h" using namespace llvm; cl::opt AsmInstrumentation( @@ -40,6 +40,11 @@ cl::opt ShowMCInst("asm-show-inst", cl::desc("Emit internal instruction representation to " "assembly file")); +cl::opt +ABIName("target-abi", cl::Hidden, + cl::desc("The name of the ABI to be targeted from the backend."), + cl::init("")); + static inline MCTargetOptions InitMCTargetOptionsFromFlags() { MCTargetOptions Options; Options.SanitizeAddress = @@ -47,6 +52,7 @@ static inline MCTargetOptions InitMCTargetOptionsFromFlags() { Options.MCRelaxAll = RelaxAll; Options.DwarfVersion = DwarfVersion; Options.ShowMCInst = ShowMCInst; + Options.ABIName = ABIName; return Options; } diff --git a/contrib/llvm/include/llvm/MC/MCWin64EH.h b/contrib/llvm/include/llvm/MC/MCWin64EH.h index 3df0d0ab8c46..0e81a191cd2c 100644 --- a/contrib/llvm/include/llvm/MC/MCWin64EH.h +++ b/contrib/llvm/include/llvm/MC/MCWin64EH.h @@ -20,9 +20,8 @@ #include namespace llvm { - class StringRef; - class MCStreamer; - class MCSymbol; +class MCStreamer; +class MCSymbol; namespace Win64EH { struct Instruction { @@ -52,36 +51,13 @@ struct Instruction { return WinEH::Instruction(UOP_SetFPReg, L, Reg, Off); } }; + +class UnwindEmitter : public WinEH::UnwindEmitter { +public: + void Emit(MCStreamer &Streamer) const override; + void EmitUnwindInfo(MCStreamer &Streamer, WinEH::FrameInfo *FI) const override; +}; } - - struct MCWinFrameInfo { - MCWinFrameInfo() - : Begin(nullptr), End(nullptr),ExceptionHandler(nullptr), - Function(nullptr), PrologEnd(nullptr), Symbol(nullptr), - HandlesUnwind(false), HandlesExceptions(false), LastFrameInst(-1), - ChainedParent(nullptr), Instructions() {} - MCSymbol *Begin; - MCSymbol *End; - const MCSymbol *ExceptionHandler; - const MCSymbol *Function; - MCSymbol *PrologEnd; - MCSymbol *Symbol; - bool HandlesUnwind; - bool HandlesExceptions; - int LastFrameInst; - MCWinFrameInfo *ChainedParent; - std::vector Instructions; - }; - - class MCWin64EHUnwindEmitter { - public: - static StringRef GetSectionSuffix(const MCSymbol *func); - // - // This emits the unwind info sections (.pdata and .xdata in PE/COFF). - // - static void Emit(MCStreamer &streamer); - static void EmitUnwindInfo(MCStreamer &streamer, MCWinFrameInfo *info); - }; } // end namespace llvm #endif diff --git a/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h index 7d2d0e4f5560..57a75cec2204 100644 --- a/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h @@ -30,10 +30,16 @@ public: MCWinCOFFStreamer(MCContext &Context, MCAsmBackend &MAB, MCCodeEmitter &CE, raw_ostream &OS); + /// state management + void reset() override { + CurSymbol = nullptr; + MCObjectStreamer::reset(); + } + /// \name MCStreamer interface /// \{ - void InitSections() override; + void InitSections(bool NoExecStack) override; void EmitLabel(MCSymbol *Symbol) override; void EmitAssemblerFlag(MCAssemblerFlag Flag) override; void EmitThumbFunc(MCSymbol *Func) override; diff --git a/contrib/llvm/include/llvm/MC/MCWinEH.h b/contrib/llvm/include/llvm/MC/MCWinEH.h index 1cd1b0f1b8ca..05b58c753b66 100644 --- a/contrib/llvm/include/llvm/MC/MCWinEH.h +++ b/contrib/llvm/include/llvm/MC/MCWinEH.h @@ -10,8 +10,14 @@ #ifndef LLVM_MC_MCWINEH_H #define LLVM_MC_MCWINEH_H +#include + namespace llvm { +class MCContext; +class MCSection; +class MCStreamer; class MCSymbol; +class StringRef; namespace WinEH { struct Instruction { @@ -23,6 +29,55 @@ struct Instruction { Instruction(unsigned Op, MCSymbol *L, unsigned Reg, unsigned Off) : Label(L), Offset(Off), Register(Reg), Operation(Op) {} }; + +struct FrameInfo { + const MCSymbol *Begin; + const MCSymbol *End; + const MCSymbol *ExceptionHandler; + const MCSymbol *Function; + const MCSymbol *PrologEnd; + const MCSymbol *Symbol; + + bool HandlesUnwind; + bool HandlesExceptions; + + int LastFrameInst; + const FrameInfo *ChainedParent; + std::vector Instructions; + + FrameInfo() + : Begin(nullptr), End(nullptr), ExceptionHandler(nullptr), + Function(nullptr), PrologEnd(nullptr), Symbol(nullptr), + HandlesUnwind(false), HandlesExceptions(false), LastFrameInst(-1), + ChainedParent(nullptr), Instructions() {} + FrameInfo(const MCSymbol *Function, const MCSymbol *BeginFuncEHLabel) + : Begin(BeginFuncEHLabel), End(nullptr), ExceptionHandler(nullptr), + Function(Function), PrologEnd(nullptr), Symbol(nullptr), + HandlesUnwind(false), HandlesExceptions(false), LastFrameInst(-1), + ChainedParent(nullptr), Instructions() {} + FrameInfo(const MCSymbol *Function, const MCSymbol *BeginFuncEHLabel, + const FrameInfo *ChainedParent) + : Begin(BeginFuncEHLabel), End(nullptr), ExceptionHandler(nullptr), + Function(Function), PrologEnd(nullptr), Symbol(nullptr), + HandlesUnwind(false), HandlesExceptions(false), LastFrameInst(-1), + ChainedParent(ChainedParent), Instructions() {} +}; + +class UnwindEmitter { +public: + static const MCSection *getPDataSection(const MCSymbol *Function, + MCContext &Context); + static const MCSection *getXDataSection(const MCSymbol *Function, + MCContext &Context); + + virtual ~UnwindEmitter() { } + + // + // This emits the unwind info sections (.pdata and .xdata in PE/COFF). + // + virtual void Emit(MCStreamer &Streamer) const = 0; + virtual void EmitUnwindInfo(MCStreamer &Streamer, FrameInfo *FI) const = 0; +}; } } diff --git a/contrib/llvm/include/llvm/MC/StringTableBuilder.h b/contrib/llvm/include/llvm/MC/StringTableBuilder.h index 065e9e06e2c9..897d449254ea 100644 --- a/contrib/llvm/include/llvm/MC/StringTableBuilder.h +++ b/contrib/llvm/include/llvm/MC/StringTableBuilder.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_MC_STRINGTABLE_BUILDER_H -#define LLVM_MC_STRINGTABLE_BUILDER_H +#ifndef LLVM_MC_STRINGTABLEBUILDER_H +#define LLVM_MC_STRINGTABLEBUILDER_H #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" @@ -26,12 +26,18 @@ public: /// copy of s. Can only be used before the table is finalized. StringRef add(StringRef s) { assert(!isFinalized()); - return StringIndexMap.GetOrCreateValue(s, 0).getKey(); + return StringIndexMap.insert(std::make_pair(s, 0)).first->first(); } + enum Kind { + ELF, + WinCOFF, + MachO + }; + /// \brief Analyze the strings and build the final table. No more strings can /// be added after this point. - void finalize(); + void finalize(Kind kind); /// \brief Retrieve the string table data. Can only be used after the table /// is finalized. @@ -48,6 +54,8 @@ public: return StringIndexMap[s]; } + void clear(); + private: bool isFinalized() { return !StringTable.empty(); diff --git a/contrib/llvm/include/llvm/MC/SubtargetFeature.h b/contrib/llvm/include/llvm/MC/SubtargetFeature.h index c5d62a6cbae8..bfecb8ba6ab0 100644 --- a/contrib/llvm/include/llvm/MC/SubtargetFeature.h +++ b/contrib/llvm/include/llvm/MC/SubtargetFeature.h @@ -72,21 +72,21 @@ struct SubtargetInfoKV { class SubtargetFeatures { std::vector Features; // Subtarget features as a vector public: - explicit SubtargetFeatures(const StringRef Initial = ""); + explicit SubtargetFeatures(StringRef Initial = ""); /// Features string accessors. std::string getString() const; /// Adding Features. - void AddFeature(const StringRef String); + void AddFeature(StringRef String); /// ToggleFeature - Toggle a feature and returns the newly updated feature /// bits. - uint64_t ToggleFeature(uint64_t Bits, const StringRef String, + uint64_t ToggleFeature(uint64_t Bits, StringRef String, ArrayRef FeatureTable); /// Get feature bits of a CPU. - uint64_t getFeatureBits(const StringRef CPU, + uint64_t getFeatureBits(StringRef CPU, ArrayRef CPUTable, ArrayRef FeatureTable); diff --git a/contrib/llvm/include/llvm/Object/Archive.h b/contrib/llvm/include/llvm/Object/Archive.h index af6c995c1d05..4e96205a93ba 100644 --- a/contrib/llvm/include/llvm/Object/Archive.h +++ b/contrib/llvm/include/llvm/Object/Archive.h @@ -15,6 +15,7 @@ #define LLVM_OBJECT_ARCHIVE_H #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/Object/Binary.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ErrorOr.h" @@ -83,14 +84,13 @@ public: return getHeader()->getAccessMode(); } /// \return the size of the archive member without the header or padding. - uint64_t getSize() const { return Data.size() - StartOfFile; } + uint64_t getSize() const; StringRef getBuffer() const { return StringRef(Data.data() + StartOfFile, getSize()); } - ErrorOr> - getMemoryBuffer(bool FullPath = false) const; + ErrorOr getMemoryBufferRef() const; ErrorOr> getAsBinary(LLVMContext *Context = nullptr) const; @@ -98,12 +98,12 @@ public: class child_iterator { Child child; + public: child_iterator() : child(Child(nullptr, nullptr)) {} child_iterator(const Child &c) : child(c) {} - const Child* operator->() const { - return &child; - } + const Child *operator->() const { return &child; } + const Child &operator*() const { return child; } bool operator==(const child_iterator &other) const { return child == other.child; @@ -113,11 +113,11 @@ public: return !(*this == other); } - bool operator <(const child_iterator &other) const { + bool operator<(const child_iterator &other) const { return child < other.child; } - child_iterator& operator++() { // Preincrement + child_iterator &operator++() { // Preincrement child = child.getNext(); return *this; } @@ -164,8 +164,8 @@ public: } }; - Archive(std::unique_ptr Source, std::error_code &EC); - static ErrorOr create(std::unique_ptr Source); + Archive(MemoryBufferRef Source, std::error_code &EC); + static ErrorOr> create(MemoryBufferRef Source); enum Kind { K_GNU, @@ -173,12 +173,14 @@ public: K_COFF }; - Kind kind() const { - return Format; - } + Kind kind() const { return (Kind)Format; } child_iterator child_begin(bool SkipInternal = true) const; child_iterator child_end() const; + iterator_range children(bool SkipInternal = true) const { + return iterator_range(child_begin(SkipInternal), + child_end()); + } symbol_iterator symbol_begin() const; symbol_iterator symbol_end() const; @@ -197,7 +199,8 @@ private: child_iterator SymbolTable; child_iterator StringTable; child_iterator FirstRegular; - Kind Format; + unsigned Format : 2; + unsigned IsThin : 1; }; } diff --git a/contrib/llvm/include/llvm/Object/Binary.h b/contrib/llvm/include/llvm/Object/Binary.h index 91984cb52715..4b2b7e6835cc 100644 --- a/contrib/llvm/include/llvm/Object/Binary.h +++ b/contrib/llvm/include/llvm/Object/Binary.h @@ -17,11 +17,11 @@ #include "llvm/Object/Error.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" namespace llvm { class LLVMContext; -class MemoryBuffer; class StringRef; namespace object { @@ -34,9 +34,9 @@ private: unsigned int TypeID; protected: - std::unique_ptr Data; + MemoryBufferRef Data; - Binary(unsigned int Type, std::unique_ptr Source); + Binary(unsigned int Type, MemoryBufferRef Source); enum { ID_Archive, @@ -78,8 +78,8 @@ public: virtual ~Binary(); StringRef getData() const; - MemoryBuffer *releaseBuffer() { return Data.release(); } StringRef getFileName() const; + MemoryBufferRef getMemoryBufferRef() const; // Cast methods. unsigned int getType() const { return TypeID; } @@ -126,10 +126,58 @@ public: /// @brief Create a Binary from Source, autodetecting the file type. /// /// @param Source The data to create the Binary from. -ErrorOr createBinary(std::unique_ptr Source, - LLVMContext *Context = nullptr); +ErrorOr> createBinary(MemoryBufferRef Source, + LLVMContext *Context = nullptr); -ErrorOr createBinary(StringRef Path); +template class OwningBinary { + std::unique_ptr Bin; + std::unique_ptr Buf; + +public: + OwningBinary(); + OwningBinary(std::unique_ptr Bin, std::unique_ptr Buf); + OwningBinary(OwningBinary&& Other); + OwningBinary &operator=(OwningBinary &&Other); + + std::pair, std::unique_ptr> takeBinary(); + + T* getBinary(); + const T* getBinary() const; +}; + +template +OwningBinary::OwningBinary(std::unique_ptr Bin, + std::unique_ptr Buf) + : Bin(std::move(Bin)), Buf(std::move(Buf)) {} + +template OwningBinary::OwningBinary() {} + +template +OwningBinary::OwningBinary(OwningBinary &&Other) + : Bin(std::move(Other.Bin)), Buf(std::move(Other.Buf)) {} + +template +OwningBinary &OwningBinary::operator=(OwningBinary &&Other) { + Bin = std::move(Other.Bin); + Buf = std::move(Other.Buf); + return *this; +} + +template +std::pair, std::unique_ptr> +OwningBinary::takeBinary() { + return std::make_pair(std::move(Bin), std::move(Buf)); +} + +template T* OwningBinary::getBinary() { + return Bin.get(); +} + +template const T* OwningBinary::getBinary() const { + return Bin.get(); +} + +ErrorOr> createBinary(StringRef Path); } } diff --git a/contrib/llvm/include/llvm/Object/COFF.h b/contrib/llvm/include/llvm/Object/COFF.h index e2da070d47bd..522bf68c6cbd 100644 --- a/contrib/llvm/include/llvm/Object/COFF.h +++ b/contrib/llvm/include/llvm/Object/COFF.h @@ -14,22 +14,31 @@ #ifndef LLVM_OBJECT_COFF_H #define LLVM_OBJECT_COFF_H +#include "llvm/ADT/PointerUnion.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/COFF.h" #include "llvm/Support/Endian.h" +#include "llvm/Support/ErrorOr.h" namespace llvm { template class ArrayRef; namespace object { class ImportDirectoryEntryRef; +class DelayImportDirectoryEntryRef; class ExportDirectoryEntryRef; +class ImportedSymbolRef; +class BaseRelocRef; typedef content_iterator import_directory_iterator; +typedef content_iterator + delay_import_directory_iterator; typedef content_iterator export_directory_iterator; +typedef content_iterator imported_symbol_iterator; +typedef content_iterator base_reloc_iterator; /// The DOS compatible header at the front of all PE/COFF executables. struct dos_header { - support::ulittle16_t Magic; + char Magic[2]; support::ulittle16_t UsedBytesInTheLastPage; support::ulittle16_t FileSizeInPages; support::ulittle16_t NumberOfRelocationItems; @@ -62,6 +71,22 @@ struct coff_file_header { bool isImportLibrary() const { return NumberOfSections == 0xffff; } }; +struct coff_bigobj_file_header { + support::ulittle16_t Sig1; + support::ulittle16_t Sig2; + support::ulittle16_t Version; + support::ulittle16_t Machine; + support::ulittle32_t TimeDateStamp; + uint8_t UUID[16]; + support::ulittle32_t unused1; + support::ulittle32_t unused2; + support::ulittle32_t unused3; + support::ulittle32_t unused4; + support::ulittle32_t NumberOfSections; + support::ulittle32_t PointerToSymbolTable; + support::ulittle32_t NumberOfSymbols; +}; + /// The 32-bit PE header that follows the COFF header. struct pe32_header { support::ulittle16_t Magic; @@ -87,12 +112,14 @@ struct pe32_header { support::ulittle32_t SizeOfHeaders; support::ulittle32_t CheckSum; support::ulittle16_t Subsystem; + // FIXME: This should be DllCharacteristics. support::ulittle16_t DLLCharacteristics; support::ulittle32_t SizeOfStackReserve; support::ulittle32_t SizeOfStackCommit; support::ulittle32_t SizeOfHeapReserve; support::ulittle32_t SizeOfHeapCommit; support::ulittle32_t LoaderFlags; + // FIXME: This should be NumberOfRvaAndSizes. support::ulittle32_t NumberOfRvaAndSize; }; @@ -142,22 +169,40 @@ struct import_directory_table_entry { support::ulittle32_t ImportAddressTableRVA; }; -struct import_lookup_table_entry32 { - support::ulittle32_t data; +template +struct import_lookup_table_entry { + IntTy Data; - bool isOrdinal() const { return data & 0x80000000; } + bool isOrdinal() const { return Data < 0; } uint16_t getOrdinal() const { assert(isOrdinal() && "ILT entry is not an ordinal!"); - return data & 0xFFFF; + return Data & 0xFFFF; } uint32_t getHintNameRVA() const { assert(!isOrdinal() && "ILT entry is not a Hint/Name RVA!"); - return data; + return Data & 0xFFFFFFFF; } }; +typedef import_lookup_table_entry + import_lookup_table_entry32; +typedef import_lookup_table_entry + import_lookup_table_entry64; + +struct delay_import_directory_table_entry { + // dumpbin reports this field as "Characteristics" instead of "Attributes". + support::ulittle32_t Attributes; + support::ulittle32_t Name; + support::ulittle32_t ModuleHandle; + support::ulittle32_t DelayImportAddressTable; + support::ulittle32_t DelayImportNameTable; + support::ulittle32_t BoundDelayImportTable; + support::ulittle32_t UnloadDelayImportTable; + support::ulittle32_t TimeStamp; +}; + struct export_directory_table_entry { support::ulittle32_t ExportFlags; support::ulittle32_t TimeDateStamp; @@ -180,67 +225,156 @@ union export_address_table_entry { typedef support::ulittle32_t export_name_pointer_table_entry; typedef support::ulittle16_t export_ordinal_table_entry; -struct coff_symbol { - struct StringTableOffset { - support::ulittle32_t Zeroes; - support::ulittle32_t Offset; - }; +struct StringTableOffset { + support::ulittle32_t Zeroes; + support::ulittle32_t Offset; +}; +template +struct coff_symbol { union { - char ShortName[8]; + char ShortName[COFF::NameSize]; StringTableOffset Offset; } Name; support::ulittle32_t Value; - support::ulittle16_t SectionNumber; + SectionNumberType SectionNumber; support::ulittle16_t Type; - support::ulittle8_t StorageClass; - support::ulittle8_t NumberOfAuxSymbols; + uint8_t StorageClass; + uint8_t NumberOfAuxSymbols; +}; - uint8_t getBaseType() const { return Type & 0x0F; } +typedef coff_symbol coff_symbol16; +typedef coff_symbol coff_symbol32; - uint8_t getComplexType() const { return (Type & 0xF0) >> 4; } +class COFFSymbolRef { +public: + COFFSymbolRef(const coff_symbol16 *CS) : CS16(CS), CS32(nullptr) {} + COFFSymbolRef(const coff_symbol32 *CS) : CS16(nullptr), CS32(CS) {} + COFFSymbolRef() : CS16(nullptr), CS32(nullptr) {} - bool isFunctionDefinition() const { - return StorageClass == COFF::IMAGE_SYM_CLASS_EXTERNAL && - getBaseType() == COFF::IMAGE_SYM_TYPE_NULL && - getComplexType() == COFF::IMAGE_SYM_DTYPE_FUNCTION && - !COFF::isReservedSectionNumber(SectionNumber); + const void *getRawPtr() const { + return CS16 ? static_cast(CS16) : CS32; } - bool isFunctionLineInfo() const { - return StorageClass == COFF::IMAGE_SYM_CLASS_FUNCTION; + friend bool operator<(COFFSymbolRef A, COFFSymbolRef B) { + return A.getRawPtr() < B.getRawPtr(); + } + + bool isBigObj() const { + if (CS16) + return false; + if (CS32) + return true; + llvm_unreachable("COFFSymbolRef points to nothing!"); + } + + const char *getShortName() const { + return CS16 ? CS16->Name.ShortName : CS32->Name.ShortName; + } + + const StringTableOffset &getStringTableOffset() const { + assert(isSet() && "COFFSymbolRef points to nothing!"); + return CS16 ? CS16->Name.Offset : CS32->Name.Offset; + } + + uint32_t getValue() const { return CS16 ? CS16->Value : CS32->Value; } + + int32_t getSectionNumber() const { + assert(isSet() && "COFFSymbolRef points to nothing!"); + if (CS16) { + // Reserved sections are returned as negative numbers. + if (CS16->SectionNumber <= COFF::MaxNumberOfSections16) + return CS16->SectionNumber; + return static_cast(CS16->SectionNumber); + } + return static_cast(CS32->SectionNumber); + } + + uint16_t getType() const { + assert(isSet() && "COFFSymbolRef points to nothing!"); + return CS16 ? CS16->Type : CS32->Type; + } + + uint8_t getStorageClass() const { + assert(isSet() && "COFFSymbolRef points to nothing!"); + return CS16 ? CS16->StorageClass : CS32->StorageClass; + } + + uint8_t getNumberOfAuxSymbols() const { + assert(isSet() && "COFFSymbolRef points to nothing!"); + return CS16 ? CS16->NumberOfAuxSymbols : CS32->NumberOfAuxSymbols; + } + + uint8_t getBaseType() const { return getType() & 0x0F; } + + uint8_t getComplexType() const { + return (getType() & 0xF0) >> COFF::SCT_COMPLEX_TYPE_SHIFT; + } + + bool isExternal() const { + return getStorageClass() == COFF::IMAGE_SYM_CLASS_EXTERNAL; + } + + bool isCommon() const { + return isExternal() && getSectionNumber() == COFF::IMAGE_SYM_UNDEFINED && + getValue() != 0; + } + + bool isUndefined() const { + return isExternal() && getSectionNumber() == COFF::IMAGE_SYM_UNDEFINED && + getValue() == 0; } bool isWeakExternal() const { - return StorageClass == COFF::IMAGE_SYM_CLASS_WEAK_EXTERNAL || - (StorageClass == COFF::IMAGE_SYM_CLASS_EXTERNAL && - SectionNumber == COFF::IMAGE_SYM_UNDEFINED && Value == 0); + return getStorageClass() == COFF::IMAGE_SYM_CLASS_WEAK_EXTERNAL; + } + + bool isFunctionDefinition() const { + return isExternal() && getBaseType() == COFF::IMAGE_SYM_TYPE_NULL && + getComplexType() == COFF::IMAGE_SYM_DTYPE_FUNCTION && + !COFF::isReservedSectionNumber(getSectionNumber()); + } + + bool isFunctionLineInfo() const { + return getStorageClass() == COFF::IMAGE_SYM_CLASS_FUNCTION; + } + + bool isAnyUndefined() const { + return isUndefined() || isWeakExternal(); } bool isFileRecord() const { - return StorageClass == COFF::IMAGE_SYM_CLASS_FILE; + return getStorageClass() == COFF::IMAGE_SYM_CLASS_FILE; } bool isSectionDefinition() const { // C++/CLI creates external ABS symbols for non-const appdomain globals. // These are also followed by an auxiliary section definition. - bool isAppdomainGlobal = StorageClass == COFF::IMAGE_SYM_CLASS_EXTERNAL && - SectionNumber == COFF::IMAGE_SYM_ABSOLUTE; - bool isOrdinarySection = - StorageClass == COFF::IMAGE_SYM_CLASS_STATIC && Value == 0; + bool isAppdomainGlobal = + getStorageClass() == COFF::IMAGE_SYM_CLASS_EXTERNAL && + getSectionNumber() == COFF::IMAGE_SYM_ABSOLUTE; + bool isOrdinarySection = getStorageClass() == COFF::IMAGE_SYM_CLASS_STATIC; + if (!getNumberOfAuxSymbols()) + return false; return isAppdomainGlobal || isOrdinarySection; } bool isCLRToken() const { - return StorageClass == COFF::IMAGE_SYM_CLASS_CLR_TOKEN; + return getStorageClass() == COFF::IMAGE_SYM_CLASS_CLR_TOKEN; } + +private: + bool isSet() const { return CS16 || CS32; } + + const coff_symbol16 *CS16; + const coff_symbol32 *CS32; }; struct coff_section { - char Name[8]; + char Name[COFF::NameSize]; support::ulittle32_t VirtualSize; support::ulittle32_t VirtualAddress; support::ulittle32_t SizeOfRawData; @@ -254,9 +388,9 @@ struct coff_section { // Returns true if the actual number of relocations is stored in // VirtualAddress field of the first relocation table entry. bool hasExtendedRelocations() const { - return Characteristics & COFF::IMAGE_SCN_LNK_NRELOC_OVFL && - NumberOfRelocations == UINT16_MAX; - }; + return (Characteristics & COFF::IMAGE_SCN_LNK_NRELOC_OVFL) && + NumberOfRelocations == UINT16_MAX; + } }; struct coff_relocation { @@ -270,7 +404,6 @@ struct coff_aux_function_definition { support::ulittle32_t TotalSize; support::ulittle32_t PointerToLinenumber; support::ulittle32_t PointerToNextFunction; - char Unused[2]; }; struct coff_aux_bf_and_ef_symbol { @@ -278,17 +411,11 @@ struct coff_aux_bf_and_ef_symbol { support::ulittle16_t Linenumber; char Unused2[6]; support::ulittle32_t PointerToNextFunction; - char Unused3[2]; }; struct coff_aux_weak_external { support::ulittle32_t TagIndex; support::ulittle32_t Characteristics; - char Unused[10]; -}; - -struct coff_aux_file { - char FileName[18]; }; struct coff_aux_section_definition { @@ -296,16 +423,22 @@ struct coff_aux_section_definition { support::ulittle16_t NumberOfRelocations; support::ulittle16_t NumberOfLinenumbers; support::ulittle32_t CheckSum; - support::ulittle16_t Number; - support::ulittle8_t Selection; - char Unused[3]; + support::ulittle16_t NumberLowPart; + uint8_t Selection; + uint8_t Unused; + support::ulittle16_t NumberHighPart; + int32_t getNumber(bool IsBigObj) const { + uint32_t Number = static_cast(NumberLowPart); + if (IsBigObj) + Number |= static_cast(NumberHighPart) << 16; + return static_cast(Number); + } }; struct coff_aux_clr_token { - support::ulittle8_t AuxType; - support::ulittle8_t Reserved; + uint8_t AuxType; + uint8_t Reserved; support::ulittle32_t SymbolTableIndex; - char Unused[12]; }; struct coff_load_configuration32 { @@ -324,7 +457,7 @@ struct coff_load_configuration32 { support::ulittle32_t ProcessAffinityMask; support::ulittle32_t ProcessHeapFlags; support::ulittle16_t CSDVersion; - uint16_t Reserved; + support::ulittle16_t Reserved; support::ulittle32_t EditList; support::ulittle32_t SecurityCookie; support::ulittle32_t SEHandlerTable; @@ -337,32 +470,114 @@ struct coff_runtime_function_x64 { support::ulittle32_t UnwindInformation; }; +struct coff_base_reloc_block_header { + support::ulittle32_t PageRVA; + support::ulittle32_t BlockSize; +}; + +struct coff_base_reloc_block_entry { + support::ulittle16_t Data; + int getType() const { return Data >> 12; } + int getOffset() const { return Data & ((1 << 12) - 1); } +}; + class COFFObjectFile : public ObjectFile { private: friend class ImportDirectoryEntryRef; friend class ExportDirectoryEntryRef; const coff_file_header *COFFHeader; + const coff_bigobj_file_header *COFFBigObjHeader; const pe32_header *PE32Header; const pe32plus_header *PE32PlusHeader; const data_directory *DataDirectory; const coff_section *SectionTable; - const coff_symbol *SymbolTable; + const coff_symbol16 *SymbolTable16; + const coff_symbol32 *SymbolTable32; const char *StringTable; uint32_t StringTableSize; const import_directory_table_entry *ImportDirectory; uint32_t NumberOfImportDirectory; + const delay_import_directory_table_entry *DelayImportDirectory; + uint32_t NumberOfDelayImportDirectory; const export_directory_table_entry *ExportDirectory; + const coff_base_reloc_block_header *BaseRelocHeader; + const coff_base_reloc_block_header *BaseRelocEnd; std::error_code getString(uint32_t offset, StringRef &Res) const; - const coff_symbol *toSymb(DataRefImpl Symb) const; + template + const coff_symbol_type *toSymb(DataRefImpl Symb) const; const coff_section *toSec(DataRefImpl Sec) const; const coff_relocation *toRel(DataRefImpl Rel) const; std::error_code initSymbolTablePtr(); std::error_code initImportTablePtr(); + std::error_code initDelayImportTablePtr(); std::error_code initExportTablePtr(); + std::error_code initBaseRelocPtr(); +public: + uintptr_t getSymbolTable() const { + if (SymbolTable16) + return reinterpret_cast(SymbolTable16); + if (SymbolTable32) + return reinterpret_cast(SymbolTable32); + return uintptr_t(0); + } + uint16_t getMachine() const { + if (COFFHeader) + return COFFHeader->Machine; + if (COFFBigObjHeader) + return COFFBigObjHeader->Machine; + llvm_unreachable("no COFF header!"); + } + uint16_t getSizeOfOptionalHeader() const { + if (COFFHeader) + return COFFHeader->isImportLibrary() ? 0 + : COFFHeader->SizeOfOptionalHeader; + // bigobj doesn't have this field. + if (COFFBigObjHeader) + return 0; + llvm_unreachable("no COFF header!"); + } + uint16_t getCharacteristics() const { + if (COFFHeader) + return COFFHeader->isImportLibrary() ? 0 : COFFHeader->Characteristics; + // bigobj doesn't have characteristics to speak of, + // editbin will silently lie to you if you attempt to set any. + if (COFFBigObjHeader) + return 0; + llvm_unreachable("no COFF header!"); + } + uint32_t getTimeDateStamp() const { + if (COFFHeader) + return COFFHeader->TimeDateStamp; + if (COFFBigObjHeader) + return COFFBigObjHeader->TimeDateStamp; + llvm_unreachable("no COFF header!"); + } + uint32_t getNumberOfSections() const { + if (COFFHeader) + return COFFHeader->isImportLibrary() ? 0 : COFFHeader->NumberOfSections; + if (COFFBigObjHeader) + return COFFBigObjHeader->NumberOfSections; + llvm_unreachable("no COFF header!"); + } + uint32_t getPointerToSymbolTable() const { + if (COFFHeader) + return COFFHeader->isImportLibrary() ? 0 + : COFFHeader->PointerToSymbolTable; + if (COFFBigObjHeader) + return COFFBigObjHeader->PointerToSymbolTable; + llvm_unreachable("no COFF header!"); + } + uint32_t getNumberOfSymbols() const { + if (COFFHeader) + return COFFHeader->isImportLibrary() ? 0 : COFFHeader->NumberOfSymbols; + if (COFFBigObjHeader) + return COFFBigObjHeader->NumberOfSymbols; + llvm_unreachable("no COFF header!"); + } protected: void moveSymbolNext(DataRefImpl &Symb) const override; std::error_code getSymbolName(DataRefImpl Symb, @@ -378,24 +593,16 @@ protected: void moveSectionNext(DataRefImpl &Sec) const override; std::error_code getSectionName(DataRefImpl Sec, StringRef &Res) const override; - std::error_code getSectionAddress(DataRefImpl Sec, - uint64_t &Res) const override; - std::error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const override; + uint64_t getSectionAddress(DataRefImpl Sec) const override; + uint64_t getSectionSize(DataRefImpl Sec) const override; std::error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const override; - std::error_code getSectionAlignment(DataRefImpl Sec, - uint64_t &Res) const override; - std::error_code isSectionText(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionData(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionBSS(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionVirtual(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionZeroInit(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionReadOnlyData(DataRefImpl Sec, - bool &Res) const override; - std::error_code isSectionRequiredForExecution(DataRefImpl Sec, - bool &Res) const override; - std::error_code sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb, - bool &Result) const override; + uint64_t getSectionAlignment(DataRefImpl Sec) const override; + bool isSectionText(DataRefImpl Sec) const override; + bool isSectionData(DataRefImpl Sec) const override; + bool isSectionBSS(DataRefImpl Sec) const override; + bool isSectionVirtual(DataRefImpl Sec) const override; + bool sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb) const override; relocation_iterator section_rel_begin(DataRefImpl Sec) const override; relocation_iterator section_rel_end(DataRefImpl Sec) const override; @@ -414,54 +621,93 @@ protected: getRelocationValueString(DataRefImpl Rel, SmallVectorImpl &Result) const override; - std::error_code getLibraryNext(DataRefImpl LibData, - LibraryRef &Result) const override; - std::error_code getLibraryPath(DataRefImpl LibData, - StringRef &Result) const override; - public: - COFFObjectFile(std::unique_ptr Object, std::error_code &EC); + COFFObjectFile(MemoryBufferRef Object, std::error_code &EC); basic_symbol_iterator symbol_begin_impl() const override; basic_symbol_iterator symbol_end_impl() const override; - library_iterator needed_library_begin() const override; - library_iterator needed_library_end() const override; section_iterator section_begin() const override; section_iterator section_end() const override; const coff_section *getCOFFSection(const SectionRef &Section) const; - const coff_symbol *getCOFFSymbol(const SymbolRef &Symbol) const; + COFFSymbolRef getCOFFSymbol(const DataRefImpl &Ref) const; + COFFSymbolRef getCOFFSymbol(const SymbolRef &Symbol) const; const coff_relocation *getCOFFRelocation(const RelocationRef &Reloc) const; uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; unsigned getArch() const override; - StringRef getLoadName() const override; import_directory_iterator import_directory_begin() const; import_directory_iterator import_directory_end() const; + delay_import_directory_iterator delay_import_directory_begin() const; + delay_import_directory_iterator delay_import_directory_end() const; export_directory_iterator export_directory_begin() const; export_directory_iterator export_directory_end() const; + base_reloc_iterator base_reloc_begin() const; + base_reloc_iterator base_reloc_end() const; - std::error_code getHeader(const coff_file_header *&Res) const; - std::error_code getCOFFHeader(const coff_file_header *&Res) const; + iterator_range import_directories() const; + iterator_range + delay_import_directories() const; + iterator_range export_directories() const; + iterator_range base_relocs() const; + + const dos_header *getDOSHeader() const { + if (!PE32Header && !PE32PlusHeader) + return nullptr; + return reinterpret_cast(base()); + } std::error_code getPE32Header(const pe32_header *&Res) const; std::error_code getPE32PlusHeader(const pe32plus_header *&Res) const; std::error_code getDataDirectory(uint32_t index, const data_directory *&Res) const; std::error_code getSection(int32_t index, const coff_section *&Res) const; - std::error_code getSymbol(uint32_t index, const coff_symbol *&Res) const; + template + std::error_code getSymbol(uint32_t Index, + const coff_symbol_type *&Res) const { + if (Index >= getNumberOfSymbols()) + return object_error::parse_failed; + + Res = reinterpret_cast(getSymbolTable()) + Index; + return object_error::success; + } + ErrorOr getSymbol(uint32_t index) const { + if (SymbolTable16) { + const coff_symbol16 *Symb = nullptr; + if (std::error_code EC = getSymbol(index, Symb)) + return EC; + return COFFSymbolRef(Symb); + } + if (SymbolTable32) { + const coff_symbol32 *Symb = nullptr; + if (std::error_code EC = getSymbol(index, Symb)) + return EC; + return COFFSymbolRef(Symb); + } + return object_error::parse_failed; + } template std::error_code getAuxSymbol(uint32_t index, const T *&Res) const { - const coff_symbol *s; - std::error_code ec = getSymbol(index, s); - Res = reinterpret_cast(s); - return ec; + ErrorOr s = getSymbol(index); + if (std::error_code EC = s.getError()) + return EC; + Res = reinterpret_cast(s->getRawPtr()); + return object_error::success; + } + std::error_code getSymbolName(COFFSymbolRef Symbol, StringRef &Res) const; + + ArrayRef getSymbolAuxData(COFFSymbolRef Symbol) const; + + size_t getSymbolTableEntrySize() const { + if (COFFHeader) + return sizeof(coff_symbol16); + if (COFFBigObjHeader) + return sizeof(coff_symbol32); + llvm_unreachable("null symbol table pointer!"); } - std::error_code getSymbolName(const coff_symbol *symbol, - StringRef &Res) const; - ArrayRef getSymbolAuxData(const coff_symbol *symbol) const; std::error_code getSectionName(const coff_section *Sec, StringRef &Res) const; + uint64_t getSectionSize(const coff_section *Sec) const; std::error_code getSectionContents(const coff_section *Sec, ArrayRef &Res) const; @@ -470,6 +716,9 @@ public: std::error_code getHintName(uint32_t Rva, uint16_t &Hint, StringRef &Name) const; + bool isRelocatableObject() const override; + bool is64() const { return PE32PlusHeader; } + static inline bool classof(const Binary *v) { return v->isCOFF(); } }; @@ -483,7 +732,14 @@ public: bool operator==(const ImportDirectoryEntryRef &Other) const; void moveNext(); + + imported_symbol_iterator imported_symbol_begin() const; + imported_symbol_iterator imported_symbol_end() const; + iterator_range imported_symbols() const; + std::error_code getName(StringRef &Result) const; + std::error_code getImportLookupTableRVA(uint32_t &Result) const; + std::error_code getImportAddressTableRVA(uint32_t &Result) const; std::error_code getImportTableEntry(const import_directory_table_entry *&Result) const; @@ -497,6 +753,31 @@ private: const COFFObjectFile *OwningObject; }; +class DelayImportDirectoryEntryRef { +public: + DelayImportDirectoryEntryRef() : OwningObject(nullptr) {} + DelayImportDirectoryEntryRef(const delay_import_directory_table_entry *T, + uint32_t I, const COFFObjectFile *Owner) + : Table(T), Index(I), OwningObject(Owner) {} + + bool operator==(const DelayImportDirectoryEntryRef &Other) const; + void moveNext(); + + imported_symbol_iterator imported_symbol_begin() const; + imported_symbol_iterator imported_symbol_end() const; + iterator_range imported_symbols() const; + + std::error_code getName(StringRef &Result) const; + std::error_code getDelayImportTable( + const delay_import_directory_table_entry *&Result) const; + std::error_code getImportAddress(int AddrIndex, uint64_t &Result) const; + +private: + const delay_import_directory_table_entry *Table; + uint32_t Index; + const COFFObjectFile *OwningObject; +}; + // The iterator for the export directory table entry. class ExportDirectoryEntryRef { public: @@ -519,6 +800,49 @@ private: uint32_t Index; const COFFObjectFile *OwningObject; }; + +class ImportedSymbolRef { +public: + ImportedSymbolRef() : OwningObject(nullptr) {} + ImportedSymbolRef(const import_lookup_table_entry32 *Entry, uint32_t I, + const COFFObjectFile *Owner) + : Entry32(Entry), Entry64(nullptr), Index(I), OwningObject(Owner) {} + ImportedSymbolRef(const import_lookup_table_entry64 *Entry, uint32_t I, + const COFFObjectFile *Owner) + : Entry32(nullptr), Entry64(Entry), Index(I), OwningObject(Owner) {} + + bool operator==(const ImportedSymbolRef &Other) const; + void moveNext(); + + std::error_code getSymbolName(StringRef &Result) const; + std::error_code getOrdinal(uint16_t &Result) const; + +private: + const import_lookup_table_entry32 *Entry32; + const import_lookup_table_entry64 *Entry64; + uint32_t Index; + const COFFObjectFile *OwningObject; +}; + +class BaseRelocRef { +public: + BaseRelocRef() : OwningObject(nullptr) {} + BaseRelocRef(const coff_base_reloc_block_header *Header, + const COFFObjectFile *Owner) + : Header(Header), Index(0), OwningObject(Owner) {} + + bool operator==(const BaseRelocRef &Other) const; + void moveNext(); + + std::error_code getType(uint8_t &Type) const; + std::error_code getRVA(uint32_t &Result) const; + +private: + const coff_base_reloc_block_header *Header; + uint32_t Index; + const COFFObjectFile *OwningObject; +}; + } // end namespace object } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Object/COFFYAML.h b/contrib/llvm/include/llvm/Object/COFFYAML.h index 4aba08f75ddc..12a25223bd37 100644 --- a/contrib/llvm/include/llvm/Object/COFFYAML.h +++ b/contrib/llvm/include/llvm/Object/COFFYAML.h @@ -31,6 +31,12 @@ inline SectionCharacteristics operator|(SectionCharacteristics a, uint32_t Ret = static_cast(a) | static_cast(b); return static_cast(Ret); } + +inline DLLCharacteristics operator|(DLLCharacteristics a, + DLLCharacteristics b) { + uint16_t Ret = static_cast(a) | static_cast(b); + return static_cast(Ret); +} } // The structure of the yaml files is not an exact 1:1 match to COFF. In order @@ -69,7 +75,13 @@ namespace COFFYAML { Symbol(); }; + struct PEHeader { + COFF::PE32Header Header; + Optional DataDirectories[COFF::NUM_DATA_DIRECTORIES]; + }; + struct Object { + Optional OptionalHeader; COFF::header Header; std::vector
Sections; std::vector Symbols; @@ -130,6 +142,11 @@ struct ScalarEnumerationTraits { static void enumeration(IO &IO, COFF::RelocationTypeAMD64 &Value); }; +template <> +struct ScalarEnumerationTraits { + static void enumeration(IO &IO, COFF::WindowsSubsystem &Value); +}; + template <> struct ScalarBitSetTraits { static void bitset(IO &IO, COFF::Characteristics &Value); @@ -140,11 +157,26 @@ struct ScalarBitSetTraits { static void bitset(IO &IO, COFF::SectionCharacteristics &Value); }; +template <> +struct ScalarBitSetTraits { + static void bitset(IO &IO, COFF::DLLCharacteristics &Value); +}; + template <> struct MappingTraits { static void mapping(IO &IO, COFFYAML::Relocation &Rel); }; +template <> +struct MappingTraits { + static void mapping(IO &IO, COFFYAML::PEHeader &PH); +}; + +template <> +struct MappingTraits { + static void mapping(IO &IO, COFF::DataDirectory &DD); +}; + template <> struct MappingTraits { static void mapping(IO &IO, COFF::header &H); diff --git a/contrib/llvm/include/llvm/Object/ELF.h b/contrib/llvm/include/llvm/Object/ELF.h index fbc48e6d7218..7c10bbf6e5ab 100644 --- a/contrib/llvm/include/llvm/Object/ELF.h +++ b/contrib/llvm/include/llvm/Object/ELF.h @@ -540,7 +540,7 @@ ELFFile::getSectionContents(const Elf_Shdr *Sec) const { if (Sec->sh_offset + Sec->sh_size > Buf.size()) return object_error::parse_failed; const uint8_t *Start = base() + Sec->sh_offset; - return ArrayRef(Start, Sec->sh_size); + return makeArrayRef(Start, Sec->sh_size); } template diff --git a/contrib/llvm/include/llvm/Object/ELFObjectFile.h b/contrib/llvm/include/llvm/Object/ELFObjectFile.h index 069f38112def..c2d6438f45ba 100644 --- a/contrib/llvm/include/llvm/Object/ELFObjectFile.h +++ b/contrib/llvm/include/llvm/Object/ELFObjectFile.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_OBJECT_ELF_OBJECT_FILE_H -#define LLVM_OBJECT_ELF_OBJECT_FILE_H +#ifndef LLVM_OBJECT_ELFOBJECTFILE_H +#define LLVM_OBJECT_ELFOBJECTFILE_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerIntPair.h" @@ -35,8 +35,26 @@ namespace llvm { namespace object { -template -class ELFObjectFile : public ObjectFile { +class ELFObjectFileBase : public ObjectFile { +protected: + ELFObjectFileBase(unsigned int Type, MemoryBufferRef Source); + +public: + virtual std::error_code getRelocationAddend(DataRefImpl Rel, + int64_t &Res) const = 0; + virtual std::pair + getELFDynamicSymbolIterators() const = 0; + + virtual std::error_code getSymbolVersion(SymbolRef Symb, StringRef &Version, + bool &IsDefault) const = 0; + + virtual uint64_t getSectionFlags(SectionRef Sec) const = 0; + virtual uint32_t getSectionType(SectionRef Sec) const = 0; + + static inline bool classof(const Binary *v) { return v->isELF(); } +}; + +template class ELFObjectFile : public ELFObjectFileBase { public: LLVM_ELF_IMPORT_TYPES_ELFT(ELFT) @@ -71,32 +89,19 @@ protected: std::error_code getSymbolSection(DataRefImpl Symb, section_iterator &Res) const override; - std::error_code getLibraryNext(DataRefImpl Data, - LibraryRef &Result) const override; - std::error_code getLibraryPath(DataRefImpl Data, - StringRef &Res) const override; - void moveSectionNext(DataRefImpl &Sec) const override; std::error_code getSectionName(DataRefImpl Sec, StringRef &Res) const override; - std::error_code getSectionAddress(DataRefImpl Sec, - uint64_t &Res) const override; - std::error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const override; + uint64_t getSectionAddress(DataRefImpl Sec) const override; + uint64_t getSectionSize(DataRefImpl Sec) const override; std::error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const override; - std::error_code getSectionAlignment(DataRefImpl Sec, - uint64_t &Res) const override; - std::error_code isSectionText(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionData(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionBSS(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionRequiredForExecution(DataRefImpl Sec, - bool &Res) const override; - std::error_code isSectionVirtual(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionZeroInit(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionReadOnlyData(DataRefImpl Sec, - bool &Res) const override; - std::error_code sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb, - bool &Result) const override; + uint64_t getSectionAlignment(DataRefImpl Sec) const override; + bool isSectionText(DataRefImpl Sec) const override; + bool isSectionData(DataRefImpl Sec) const override; + bool isSectionBSS(DataRefImpl Sec) const override; + bool isSectionVirtual(DataRefImpl Sec) const override; + bool sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb) const override; relocation_iterator section_rel_begin(DataRefImpl Sec) const override; relocation_iterator section_rel_end(DataRefImpl Sec) const override; section_iterator getRelocatedSection(DataRefImpl Sec) const override; @@ -178,7 +183,7 @@ protected: bool isDyldELFObject; public: - ELFObjectFile(std::unique_ptr Object, std::error_code &EC); + ELFObjectFile(MemoryBufferRef Object, std::error_code &EC); const Elf_Sym *getSymbol(DataRefImpl Symb) const; @@ -191,17 +196,18 @@ public: section_iterator section_begin() const override; section_iterator section_end() const override; - library_iterator needed_library_begin() const override; - library_iterator needed_library_end() const override; - - std::error_code getRelocationAddend(DataRefImpl Rel, int64_t &Res) const; + std::error_code getRelocationAddend(DataRefImpl Rel, + int64_t &Res) const override; std::error_code getSymbolVersion(SymbolRef Symb, StringRef &Version, - bool &IsDefault) const; + bool &IsDefault) const override; + + uint64_t getSectionFlags(SectionRef Sec) const override; + uint32_t getSectionType(SectionRef Sec) const override; uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; unsigned getArch() const override; - StringRef getLoadName() const override; + StringRef getLoadName() const; std::error_code getPlatformFlags(unsigned &Result) const override { Result = EF.getHeader()->e_flags; @@ -215,6 +221,11 @@ public: return v->getType() == getELFType(ELFT::TargetEndianness == support::little, ELFT::Is64Bits); } + + std::pair + getELFDynamicSymbolIterators() const override; + + bool isRelocatableObject() const override; }; // Use an alignment of 2 for the typedefs since that is the worst case for @@ -253,6 +264,18 @@ std::error_code ELFObjectFile::getSymbolVersion(SymbolRef SymRef, return object_error::success; } +template +uint64_t ELFObjectFile::getSectionFlags(SectionRef Sec) const { + DataRefImpl DRI = Sec.getRawDataRefImpl(); + return toELFShdrIter(DRI)->sh_flags; +} + +template +uint32_t ELFObjectFile::getSectionType(SectionRef Sec) const { + DataRefImpl DRI = Sec.getRawDataRefImpl(); + return toELFShdrIter(DRI)->sh_type; +} + template std::error_code ELFObjectFile::getSymbolAddress(DataRefImpl Symb, uint64_t &Result) const { @@ -272,8 +295,9 @@ std::error_code ELFObjectFile::getSymbolAddress(DataRefImpl Symb, const Elf_Ehdr *Header = EF.getHeader(); Result = ESym->st_value; - // Clear the ARM/Thumb indicator flag. - if (Header->e_machine == ELF::EM_ARM && ESym->getType() == ELF::STT_FUNC) + // Clear the ARM/Thumb or microMIPS indicator flag. + if ((Header->e_machine == ELF::EM_ARM || Header->e_machine == ELF::EM_MIPS) && + ESym->getType() == ELF::STT_FUNC) Result &= ~1; if (Header->e_type == ELF::ET_REL) @@ -400,17 +424,13 @@ std::error_code ELFObjectFile::getSectionName(DataRefImpl Sec, } template -std::error_code ELFObjectFile::getSectionAddress(DataRefImpl Sec, - uint64_t &Result) const { - Result = toELFShdrIter(Sec)->sh_addr; - return object_error::success; +uint64_t ELFObjectFile::getSectionAddress(DataRefImpl Sec) const { + return toELFShdrIter(Sec)->sh_addr; } template -std::error_code ELFObjectFile::getSectionSize(DataRefImpl Sec, - uint64_t &Result) const { - Result = toELFShdrIter(Sec)->sh_size; - return object_error::success; +uint64_t ELFObjectFile::getSectionSize(DataRefImpl Sec) const { + return toELFShdrIter(Sec)->sh_size; } template @@ -423,79 +443,43 @@ ELFObjectFile::getSectionContents(DataRefImpl Sec, } template -std::error_code -ELFObjectFile::getSectionAlignment(DataRefImpl Sec, - uint64_t &Result) const { - Result = toELFShdrIter(Sec)->sh_addralign; - return object_error::success; +uint64_t ELFObjectFile::getSectionAlignment(DataRefImpl Sec) const { + return toELFShdrIter(Sec)->sh_addralign; } template -std::error_code ELFObjectFile::isSectionText(DataRefImpl Sec, - bool &Result) const { - Result = toELFShdrIter(Sec)->sh_flags & ELF::SHF_EXECINSTR; - return object_error::success; +bool ELFObjectFile::isSectionText(DataRefImpl Sec) const { + return toELFShdrIter(Sec)->sh_flags & ELF::SHF_EXECINSTR; } template -std::error_code ELFObjectFile::isSectionData(DataRefImpl Sec, - bool &Result) const { +bool ELFObjectFile::isSectionData(DataRefImpl Sec) const { Elf_Shdr_Iter EShdr = toELFShdrIter(Sec); - Result = EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) && - EShdr->sh_type == ELF::SHT_PROGBITS; - return object_error::success; + return EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) && + EShdr->sh_type == ELF::SHT_PROGBITS; } template -std::error_code ELFObjectFile::isSectionBSS(DataRefImpl Sec, - bool &Result) const { +bool ELFObjectFile::isSectionBSS(DataRefImpl Sec) const { Elf_Shdr_Iter EShdr = toELFShdrIter(Sec); - Result = EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) && - EShdr->sh_type == ELF::SHT_NOBITS; - return object_error::success; + return EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) && + EShdr->sh_type == ELF::SHT_NOBITS; } template -std::error_code -ELFObjectFile::isSectionRequiredForExecution(DataRefImpl Sec, - bool &Result) const { - Result = toELFShdrIter(Sec)->sh_flags & ELF::SHF_ALLOC; - return object_error::success; +bool ELFObjectFile::isSectionVirtual(DataRefImpl Sec) const { + return toELFShdrIter(Sec)->sh_type == ELF::SHT_NOBITS; } template -std::error_code ELFObjectFile::isSectionVirtual(DataRefImpl Sec, - bool &Result) const { - Result = toELFShdrIter(Sec)->sh_type == ELF::SHT_NOBITS; - return object_error::success; -} - -template -std::error_code ELFObjectFile::isSectionZeroInit(DataRefImpl Sec, - bool &Result) const { - Result = toELFShdrIter(Sec)->sh_type == ELF::SHT_NOBITS; - return object_error::success; -} - -template -std::error_code ELFObjectFile::isSectionReadOnlyData(DataRefImpl Sec, - bool &Result) const { - Elf_Shdr_Iter EShdr = toELFShdrIter(Sec); - Result = !(EShdr->sh_flags & (ELF::SHF_WRITE | ELF::SHF_EXECINSTR)); - return object_error::success; -} - -template -std::error_code ELFObjectFile::sectionContainsSymbol(DataRefImpl Sec, - DataRefImpl Symb, - bool &Result) const { +bool ELFObjectFile::sectionContainsSymbol(DataRefImpl Sec, + DataRefImpl Symb) const { Elf_Sym_Iter ESym = toELFSymIter(Symb); uintX_t Index = ESym->st_shndx; bool Reserved = Index >= ELF::SHN_LORESERVE && Index <= ELF::SHN_HIRESERVE; - Result = !Reserved && (&*toELFShdrIter(Sec) == EF.getSection(ESym->st_shndx)); - return object_error::success; + return !Reserved && (&*toELFShdrIter(Sec) == EF.getSection(ESym->st_shndx)); } template @@ -754,6 +738,7 @@ std::error_code ELFObjectFile::getRelocationValueString( Result.append(fmtbuf.begin(), fmtbuf.end()); break; } + case ELF::EM_386: case ELF::EM_ARM: case ELF::EM_HEXAGON: case ELF::EM_MIPS: @@ -786,13 +771,13 @@ ELFObjectFile::getRela(DataRefImpl Rela) const { } template -ELFObjectFile::ELFObjectFile(std::unique_ptr Object, - std::error_code &EC) - : ObjectFile(getELFType(static_cast(ELFT::TargetEndianness) == - support::little, - ELFT::Is64Bits), - std::move(Object)), - EF(Data->getBuffer(), EC) {} +ELFObjectFile::ELFObjectFile(MemoryBufferRef Object, std::error_code &EC) + : ELFObjectFileBase( + getELFType(static_cast(ELFT::TargetEndianness) == + support::little, + ELFT::Is64Bits), + Object), + EF(Data.getBuffer(), EC) {} template basic_symbol_iterator ELFObjectFile::symbol_begin_impl() const { @@ -837,44 +822,6 @@ StringRef ELFObjectFile::getLoadName() const { return ""; } -template -library_iterator ELFObjectFile::needed_library_begin() const { - Elf_Dyn_Iter DI = EF.begin_dynamic_table(); - Elf_Dyn_Iter DE = EF.end_dynamic_table(); - - while (DI != DE && DI->getTag() != ELF::DT_SONAME) - ++DI; - - return library_iterator(LibraryRef(toDRI(DI), this)); -} - -template -std::error_code ELFObjectFile::getLibraryNext(DataRefImpl Data, - LibraryRef &Result) const { - Elf_Dyn_Iter DI = toELFDynIter(Data); - Elf_Dyn_Iter DE = EF.end_dynamic_table(); - - // Skip to the next DT_NEEDED entry. - do - ++DI; - while (DI != DE && DI->getTag() != ELF::DT_NEEDED); - - Result = LibraryRef(toDRI(DI), this); - return object_error::success; -} - -template -std::error_code ELFObjectFile::getLibraryPath(DataRefImpl Data, - StringRef &Res) const { - Res = EF.getDynamicString(toELFDynIter(Data)->getVal()); - return object_error::success; -} - -template -library_iterator ELFObjectFile::needed_library_end() const { - return library_iterator(LibraryRef(toDRI(EF.end_dynamic_table()), this)); -} - template uint8_t ELFObjectFile::getBytesInAddress() const { return ELFT::Is64Bits ? 8 : 4; @@ -882,6 +829,7 @@ uint8_t ELFObjectFile::getBytesInAddress() const { template StringRef ELFObjectFile::getFileFormatName() const { + bool IsLittleEndian = ELFT::TargetEndianness == support::little; switch (EF.getHeader()->e_ident[ELF::EI_CLASS]) { case ELF::ELFCLASS32: switch (EF.getHeader()->e_machine) { @@ -890,7 +838,7 @@ StringRef ELFObjectFile::getFileFormatName() const { case ELF::EM_X86_64: return "ELF32-x86-64"; case ELF::EM_ARM: - return "ELF32-arm"; + return (IsLittleEndian ? "ELF32-arm-little" : "ELF32-arm-big"); case ELF::EM_HEXAGON: return "ELF32-hexagon"; case ELF::EM_MIPS: @@ -910,7 +858,7 @@ StringRef ELFObjectFile::getFileFormatName() const { case ELF::EM_X86_64: return "ELF64-x86-64"; case ELF::EM_AARCH64: - return "ELF64-aarch64"; + return (IsLittleEndian ? "ELF64-aarch64-little" : "ELF64-aarch64-big"); case ELF::EM_PPC64: return "ELF64-ppc64"; case ELF::EM_S390: @@ -951,6 +899,8 @@ unsigned ELFObjectFile::getArch() const { default: report_fatal_error("Invalid ELFCLASS!"); } + case ELF::EM_PPC: + return Triple::ppc; case ELF::EM_PPC64: return IsLittleEndian ? Triple::ppc64le : Triple::ppc64; case ELF::EM_S390: @@ -967,73 +917,34 @@ unsigned ELFObjectFile::getArch() const { } } -/// FIXME: Maybe we should have a base ElfObjectFile that is not a template -/// and make these member functions? +template +std::pair +ELFObjectFile::getELFDynamicSymbolIterators() const { + return std::make_pair(dynamic_symbol_begin(), dynamic_symbol_end()); +} + +template bool ELFObjectFile::isRelocatableObject() const { + return EF.getHeader()->e_type == ELF::ET_REL; +} + inline std::error_code getELFRelocationAddend(const RelocationRef R, int64_t &Addend) { const ObjectFile *Obj = R.getObjectFile(); DataRefImpl DRI = R.getRawDataRefImpl(); - // Little-endian 32-bit - if (const ELF32LEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getRelocationAddend(DRI, Addend); - - // Big-endian 32-bit - if (const ELF32BEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getRelocationAddend(DRI, Addend); - - // Little-endian 64-bit - if (const ELF64LEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getRelocationAddend(DRI, Addend); - - // Big-endian 64-bit - if (const ELF64BEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getRelocationAddend(DRI, Addend); - - llvm_unreachable("Object passed to getELFRelocationAddend() is not ELF"); + return cast(Obj)->getRelocationAddend(DRI, Addend); } inline std::pair -getELFDynamicSymbolIterators(SymbolicFile *Obj) { - if (const ELF32LEObjectFile *ELF = dyn_cast(Obj)) - return std::make_pair(ELF->dynamic_symbol_begin(), - ELF->dynamic_symbol_end()); - if (const ELF64LEObjectFile *ELF = dyn_cast(Obj)) - return std::make_pair(ELF->dynamic_symbol_begin(), - ELF->dynamic_symbol_end()); - if (const ELF32BEObjectFile *ELF = dyn_cast(Obj)) - return std::make_pair(ELF->dynamic_symbol_begin(), - ELF->dynamic_symbol_end()); - if (const ELF64BEObjectFile *ELF = cast(Obj)) - return std::make_pair(ELF->dynamic_symbol_begin(), - ELF->dynamic_symbol_end()); - - llvm_unreachable( - "Object passed to getELFDynamicSymbolIterators() is not ELF"); +getELFDynamicSymbolIterators(const SymbolicFile *Obj) { + return cast(Obj)->getELFDynamicSymbolIterators(); } -/// This is a generic interface for retrieving GNU symbol version -/// information from an ELFObjectFile. inline std::error_code GetELFSymbolVersion(const ObjectFile *Obj, const SymbolRef &Sym, StringRef &Version, bool &IsDefault) { - // Little-endian 32-bit - if (const ELF32LEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getSymbolVersion(Sym, Version, IsDefault); - - // Big-endian 32-bit - if (const ELF32BEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getSymbolVersion(Sym, Version, IsDefault); - - // Little-endian 64-bit - if (const ELF64LEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getSymbolVersion(Sym, Version, IsDefault); - - // Big-endian 64-bit - if (const ELF64BEObjectFile *ELFObj = dyn_cast(Obj)) - return ELFObj->getSymbolVersion(Sym, Version, IsDefault); - - llvm_unreachable("Object passed to GetELFSymbolVersion() is not ELF"); + return cast(Obj) + ->getSymbolVersion(Sym, Version, IsDefault); } } } diff --git a/contrib/llvm/include/llvm/Object/ELFTypes.h b/contrib/llvm/include/llvm/Object/ELFTypes.h index 84b603125a69..4bc0c7c2016e 100644 --- a/contrib/llvm/include/llvm/Object/ELFTypes.h +++ b/contrib/llvm/include/llvm/Object/ELFTypes.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_OBJECT_ELF_TYPES_H -#define LLVM_OBJECT_ELF_TYPES_H +#ifndef LLVM_OBJECT_ELFTYPES_H +#define LLVM_OBJECT_ELFTYPES_H #include "llvm/Support/AlignOf.h" #include "llvm/Support/DataTypes.h" @@ -176,6 +176,7 @@ struct Elf_Sym_Base > { template struct Elf_Sym_Impl : Elf_Sym_Base { using Elf_Sym_Base::st_info; + using Elf_Sym_Base::st_other; // These accessors and mutators correspond to the ELF32_ST_BIND, // ELF32_ST_TYPE, and ELF32_ST_INFO macros defined in the ELF specification: @@ -186,6 +187,9 @@ struct Elf_Sym_Impl : Elf_Sym_Base { void setBindingAndType(unsigned char b, unsigned char t) { st_info = (b << 4) + (t & 0x0f); } + + /// Access to the STV_xxx flag stored in the first two bits of st_other. + unsigned char getVisibility() const { return st_other & 0x3; } }; /// Elf_Versym: This is the structure of entries in the SHT_GNU_versym section diff --git a/contrib/llvm/include/llvm/Object/ELFYAML.h b/contrib/llvm/include/llvm/Object/ELFYAML.h index fc8cc9581655..b71946d98b5d 100644 --- a/contrib/llvm/include/llvm/Object/ELFYAML.h +++ b/contrib/llvm/include/llvm/Object/ELFYAML.h @@ -40,11 +40,12 @@ LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFOSABI) // Just use 64, since it can hold 32-bit values too. LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_EF) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_SHT) -LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_REL) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_REL) // Just use 64, since it can hold 32-bit values too. LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_SHF) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STT) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STV) +LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STO) // For now, hardcode 64 bits everywhere that 32 or 64 would be needed // since 64-bit can hold 32-bit values too. @@ -63,7 +64,7 @@ struct Symbol { StringRef Section; llvm::yaml::Hex64 Value; llvm::yaml::Hex64 Size; - ELF_STV Visibility; + uint8_t Other; }; struct LocalGlobalWeakSymbols { std::vector Local; @@ -174,6 +175,11 @@ struct ScalarEnumerationTraits { static void enumeration(IO &IO, ELFYAML::ELF_STV &Value); }; +template <> +struct ScalarBitSetTraits { + static void bitset(IO &IO, ELFYAML::ELF_STO &Value); +}; + template <> struct ScalarEnumerationTraits { static void enumeration(IO &IO, ELFYAML::ELF_REL &Value); diff --git a/contrib/llvm/include/llvm/Object/Error.h b/contrib/llvm/include/llvm/Object/Error.h index 701da1272cd5..90c2bd74b43c 100644 --- a/contrib/llvm/include/llvm/Object/Error.h +++ b/contrib/llvm/include/llvm/Object/Error.h @@ -26,7 +26,8 @@ enum class object_error { arch_not_found, invalid_file_type, parse_failed, - unexpected_eof + unexpected_eof, + bitcode_section_not_found, }; inline std::error_code make_error_code(object_error e) { diff --git a/contrib/llvm/include/llvm/Object/IRObjectFile.h b/contrib/llvm/include/llvm/Object/IRObjectFile.h index b33cc263b9bd..74f46667ce64 100644 --- a/contrib/llvm/include/llvm/Object/IRObjectFile.h +++ b/contrib/llvm/include/llvm/Object/IRObjectFile.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_OBJECT_IR_OBJECT_FILE_H -#define LLVM_OBJECT_IR_OBJECT_FILE_H +#ifndef LLVM_OBJECT_IROBJECTFILE_H +#define LLVM_OBJECT_IROBJECTFILE_H #include "llvm/Object/SymbolicFile.h" @@ -22,19 +22,24 @@ class Module; class GlobalValue; namespace object { +class ObjectFile; + class IRObjectFile : public SymbolicFile { std::unique_ptr M; std::unique_ptr Mang; std::vector> AsmSymbols; public: - IRObjectFile(std::unique_ptr Object, std::unique_ptr M); + IRObjectFile(MemoryBufferRef Object, std::unique_ptr M); ~IRObjectFile(); void moveSymbolNext(DataRefImpl &Symb) const override; std::error_code printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override; uint32_t getSymbolFlags(DataRefImpl Symb) const override; - const GlobalValue *getSymbolGV(DataRefImpl Symb) const; + GlobalValue *getSymbolGV(DataRefImpl Symb); + const GlobalValue *getSymbolGV(DataRefImpl Symb) const { + return const_cast(this)->getSymbolGV(Symb); + } basic_symbol_iterator symbol_begin_impl() const override; basic_symbol_iterator symbol_end_impl() const override; @@ -44,14 +49,24 @@ public: Module &getModule() { return *M; } + std::unique_ptr takeModule(); static inline bool classof(const Binary *v) { return v->isIR(); } - static ErrorOr - createIRObjectFile(std::unique_ptr Object, - LLVMContext &Context); + /// \brief Finds and returns bitcode embedded in the given object file, or an + /// error code if not found. + static ErrorOr findBitcodeInObject(const ObjectFile &Obj); + + /// \brief Finds and returns bitcode in the given memory buffer (which may + /// be either a bitcode file or a native object file with embedded bitcode), + /// or an error code if not found. + static ErrorOr + findBitcodeInMemBuffer(MemoryBufferRef Object); + + static ErrorOr> create(MemoryBufferRef Object, + LLVMContext &Context); }; } } diff --git a/contrib/llvm/include/llvm/Object/MachO.h b/contrib/llvm/include/llvm/Object/MachO.h index 4835eb80bd65..bee1f6ce0b01 100644 --- a/contrib/llvm/include/llvm/Object/MachO.h +++ b/contrib/llvm/include/llvm/Object/MachO.h @@ -49,6 +49,141 @@ public: }; typedef content_iterator dice_iterator; +/// ExportEntry encapsulates the current-state-of-the-walk used when doing a +/// non-recursive walk of the trie data structure. This allows you to iterate +/// across all exported symbols using: +/// for (const llvm::object::ExportEntry &AnExport : Obj->exports()) { +/// } +class ExportEntry { +public: + ExportEntry(ArrayRef Trie); + + StringRef name() const; + uint64_t flags() const; + uint64_t address() const; + uint64_t other() const; + StringRef otherName() const; + uint32_t nodeOffset() const; + + bool operator==(const ExportEntry &) const; + + void moveNext(); + +private: + friend class MachOObjectFile; + void moveToFirst(); + void moveToEnd(); + uint64_t readULEB128(const uint8_t *&p); + void pushDownUntilBottom(); + void pushNode(uint64_t Offset); + + // Represents a node in the mach-o exports trie. + struct NodeState { + NodeState(const uint8_t *Ptr); + const uint8_t *Start; + const uint8_t *Current; + uint64_t Flags; + uint64_t Address; + uint64_t Other; + const char *ImportName; + unsigned ChildCount; + unsigned NextChildIndex; + unsigned ParentStringLength; + bool IsExportNode; + }; + + ArrayRef Trie; + SmallString<256> CumulativeString; + SmallVector Stack; + bool Malformed; + bool Done; +}; +typedef content_iterator export_iterator; + +/// MachORebaseEntry encapsulates the current state in the decompression of +/// rebasing opcodes. This allows you to iterate through the compressed table of +/// rebasing using: +/// for (const llvm::object::MachORebaseEntry &Entry : Obj->rebaseTable()) { +/// } +class MachORebaseEntry { +public: + MachORebaseEntry(ArrayRef opcodes, bool is64Bit); + + uint32_t segmentIndex() const; + uint64_t segmentOffset() const; + StringRef typeName() const; + + bool operator==(const MachORebaseEntry &) const; + + void moveNext(); + +private: + friend class MachOObjectFile; + void moveToFirst(); + void moveToEnd(); + uint64_t readULEB128(); + + ArrayRef Opcodes; + const uint8_t *Ptr; + uint64_t SegmentOffset; + uint32_t SegmentIndex; + uint64_t RemainingLoopCount; + uint64_t AdvanceAmount; + uint8_t RebaseType; + uint8_t PointerSize; + bool Malformed; + bool Done; +}; +typedef content_iterator rebase_iterator; + +/// MachOBindEntry encapsulates the current state in the decompression of +/// binding opcodes. This allows you to iterate through the compressed table of +/// bindings using: +/// for (const llvm::object::MachOBindEntry &Entry : Obj->bindTable()) { +/// } +class MachOBindEntry { +public: + enum class Kind { Regular, Lazy, Weak }; + + MachOBindEntry(ArrayRef Opcodes, bool is64Bit, MachOBindEntry::Kind); + + uint32_t segmentIndex() const; + uint64_t segmentOffset() const; + StringRef typeName() const; + StringRef symbolName() const; + uint32_t flags() const; + int64_t addend() const; + int ordinal() const; + + bool operator==(const MachOBindEntry &) const; + + void moveNext(); + +private: + friend class MachOObjectFile; + void moveToFirst(); + void moveToEnd(); + uint64_t readULEB128(); + int64_t readSLEB128(); + + ArrayRef Opcodes; + const uint8_t *Ptr; + uint64_t SegmentOffset; + uint32_t SegmentIndex; + StringRef SymbolName; + int Ordinal; + uint32_t Flags; + int64_t Addend; + uint64_t RemainingLoopCount; + uint64_t AdvanceAmount; + uint8_t BindType; + uint8_t PointerSize; + Kind TableKind; + bool Malformed; + bool Done; +}; +typedef content_iterator bind_iterator; + class MachOObjectFile : public ObjectFile { public: struct LoadCommandInfo { @@ -56,8 +191,8 @@ public: MachO::load_command C; // The command itself. }; - MachOObjectFile(std::unique_ptr Object, bool IsLittleEndian, - bool Is64Bits, std::error_code &EC); + MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian, bool Is64Bits, + std::error_code &EC); void moveSymbolNext(DataRefImpl &Symb) const override; std::error_code getSymbolName(DataRefImpl Symb, @@ -65,6 +200,7 @@ public: // MachO specific. std::error_code getIndirectName(DataRefImpl Symb, StringRef &Res) const; + unsigned getSectionType(SectionRef Sec) const; std::error_code getSymbolAddress(DataRefImpl Symb, uint64_t &Res) const override; @@ -80,24 +216,16 @@ public: void moveSectionNext(DataRefImpl &Sec) const override; std::error_code getSectionName(DataRefImpl Sec, StringRef &Res) const override; - std::error_code getSectionAddress(DataRefImpl Sec, - uint64_t &Res) const override; - std::error_code getSectionSize(DataRefImpl Sec, uint64_t &Res) const override; + uint64_t getSectionAddress(DataRefImpl Sec) const override; + uint64_t getSectionSize(DataRefImpl Sec) const override; std::error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const override; - std::error_code getSectionAlignment(DataRefImpl Sec, - uint64_t &Res) const override; - std::error_code isSectionText(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionData(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionBSS(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionRequiredForExecution(DataRefImpl Sec, - bool &Res) const override; - std::error_code isSectionVirtual(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionZeroInit(DataRefImpl Sec, bool &Res) const override; - std::error_code isSectionReadOnlyData(DataRefImpl Sec, - bool &Res) const override; - std::error_code sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb, - bool &Result) const override; + uint64_t getSectionAlignment(DataRefImpl Sec) const override; + bool isSectionText(DataRefImpl Sec) const override; + bool isSectionData(DataRefImpl Sec) const override; + bool isSectionBSS(DataRefImpl Sec) const override; + bool isSectionVirtual(DataRefImpl Sec) const override; + bool sectionContainsSymbol(DataRefImpl Sec, DataRefImpl Symb) const override; relocation_iterator section_rel_begin(DataRefImpl Sec) const override; relocation_iterator section_rel_end(DataRefImpl Sec) const override; @@ -118,13 +246,8 @@ public: std::error_code getRelocationHidden(DataRefImpl Rel, bool &Result) const override; - std::error_code getLibraryNext(DataRefImpl LibData, - LibraryRef &Res) const override; - std::error_code getLibraryPath(DataRefImpl LibData, - StringRef &Res) const override; - // MachO specific. - std::error_code getLibraryShortNameByIndex(unsigned Index, StringRef &Res); + std::error_code getLibraryShortNameByIndex(unsigned Index, StringRef &) const; // TODO: Would be useful to have an iterator based version // of the load command interface too. @@ -138,21 +261,45 @@ public: section_iterator section_begin() const override; section_iterator section_end() const override; - library_iterator needed_library_begin() const override; - library_iterator needed_library_end() const override; - uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; unsigned getArch() const override; - - StringRef getLoadName() const override; + Triple getArch(const char **McpuDefault, Triple *ThumbTriple) const; relocation_iterator section_rel_begin(unsigned Index) const; relocation_iterator section_rel_end(unsigned Index) const; dice_iterator begin_dices() const; dice_iterator end_dices() const; + + /// For use iterating over all exported symbols. + iterator_range exports() const; + + /// For use examining a trie not in a MachOObjectFile. + static iterator_range exports(ArrayRef Trie); + + /// For use iterating over all rebase table entries. + iterator_range rebaseTable() const; + + /// For use examining rebase opcodes not in a MachOObjectFile. + static iterator_range rebaseTable(ArrayRef Opcodes, + bool is64); + + /// For use iterating over all bind table entries. + iterator_range bindTable() const; + + /// For use iterating over all lazy bind table entries. + iterator_range lazyBindTable() const; + + /// For use iterating over all lazy bind table entries. + iterator_range weakBindTable() const; + + /// For use examining bind opcodes not in a MachOObjectFile. + static iterator_range bindTable(ArrayRef Opcodes, + bool is64, + MachOBindEntry::Kind); + // In a MachO file, sections have a segment name. This is used in the .o // files. They have a single segment, but this field specifies which segment @@ -173,6 +320,8 @@ public: const MachO::any_relocation_info &RE) const; uint32_t getScatteredRelocationValue( const MachO::any_relocation_info &RE) const; + uint32_t getScatteredRelocationType( + const MachO::any_relocation_info &RE) const; unsigned getAnyRelocationAddress(const MachO::any_relocation_info &RE) const; unsigned getAnyRelocationPCRel(const MachO::any_relocation_info &RE) const; unsigned getAnyRelocationLength(const MachO::any_relocation_info &RE) const; @@ -197,12 +346,42 @@ public: getSegmentLoadCommand(const LoadCommandInfo &L) const; MachO::segment_command_64 getSegment64LoadCommand(const LoadCommandInfo &L) const; - MachO::linker_options_command - getLinkerOptionsLoadCommand(const LoadCommandInfo &L) const; + MachO::linker_option_command + getLinkerOptionLoadCommand(const LoadCommandInfo &L) const; MachO::version_min_command getVersionMinLoadCommand(const LoadCommandInfo &L) const; MachO::dylib_command getDylibIDLoadCommand(const LoadCommandInfo &L) const; + MachO::dyld_info_command + getDyldInfoLoadCommand(const LoadCommandInfo &L) const; + MachO::dylinker_command + getDylinkerCommand(const LoadCommandInfo &L) const; + MachO::uuid_command + getUuidCommand(const LoadCommandInfo &L) const; + MachO::rpath_command + getRpathCommand(const LoadCommandInfo &L) const; + MachO::source_version_command + getSourceVersionCommand(const LoadCommandInfo &L) const; + MachO::entry_point_command + getEntryPointCommand(const LoadCommandInfo &L) const; + MachO::encryption_info_command + getEncryptionInfoCommand(const LoadCommandInfo &L) const; + MachO::encryption_info_command_64 + getEncryptionInfoCommand64(const LoadCommandInfo &L) const; + MachO::sub_framework_command + getSubFrameworkCommand(const LoadCommandInfo &L) const; + MachO::sub_umbrella_command + getSubUmbrellaCommand(const LoadCommandInfo &L) const; + MachO::sub_library_command + getSubLibraryCommand(const LoadCommandInfo &L) const; + MachO::sub_client_command + getSubClientCommand(const LoadCommandInfo &L) const; + MachO::routines_command + getRoutinesCommand(const LoadCommandInfo &L) const; + MachO::routines_command_64 + getRoutinesCommand64(const LoadCommandInfo &L) const; + MachO::thread_command + getThreadCommand(const LoadCommandInfo &L) const; MachO::any_relocation_info getRelocation(DataRefImpl Rel) const; MachO::data_in_code_entry getDice(DataRefImpl Rel) const; @@ -216,6 +395,12 @@ public: MachO::symtab_command getSymtabLoadCommand() const; MachO::dysymtab_command getDysymtabLoadCommand() const; MachO::linkedit_data_command getDataInCodeLoadCommand() const; + ArrayRef getDyldInfoRebaseOpcodes() const; + ArrayRef getDyldInfoBindOpcodes() const; + ArrayRef getDyldInfoWeakBindOpcodes() const; + ArrayRef getDyldInfoLazyBindOpcodes() const; + ArrayRef getDyldInfoExportsTrie() const; + ArrayRef getUuid() const; StringRef getStringTableData() const; bool is64Bit() const; @@ -225,10 +410,19 @@ public: StringRef &Suffix); static Triple::ArchType getArch(uint32_t CPUType); - static Triple getArch(uint32_t CPUType, uint32_t CPUSubType); - static Triple getArch(StringRef ArchFlag); + static Triple getArch(uint32_t CPUType, uint32_t CPUSubType, + const char **McpuDefault = nullptr); + static Triple getThumbArch(uint32_t CPUType, uint32_t CPUSubType, + const char **McpuDefault = nullptr); + static Triple getArch(uint32_t CPUType, uint32_t CPUSubType, + const char **McpuDefault, Triple *ThumbTriple); + static bool isValidArch(StringRef ArchFlag); static Triple getHostArch(); + bool isRelocatableObject() const override; + + bool hasPageZeroSegment() const { return HasPageZeroSegment; } + static bool classof(const Binary *v) { return v->isMachO(); } @@ -239,10 +433,13 @@ private: typedef SmallVector LibraryList; LibraryList Libraries; typedef SmallVector LibraryShortName; - LibraryShortName LibrariesShortNames; + mutable LibraryShortName LibrariesShortNames; const char *SymtabLoadCmd; const char *DysymtabLoadCmd; const char *DataInCodeLoadCmd; + const char *DyldInfoLoadCmd; + const char *UuidLoadCmd; + bool HasPageZeroSegment; }; /// DiceRef diff --git a/contrib/llvm/include/llvm/Object/MachOUniversal.h b/contrib/llvm/include/llvm/Object/MachOUniversal.h index e6677f5bf28b..93f665438821 100644 --- a/contrib/llvm/include/llvm/Object/MachOUniversal.h +++ b/contrib/llvm/include/llvm/Object/MachOUniversal.h @@ -16,8 +16,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" -#include "llvm/Object/Binary.h" #include "llvm/Object/Archive.h" +#include "llvm/Object/Binary.h" #include "llvm/Object/MachO.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MachO.h" @@ -25,8 +25,6 @@ namespace llvm { namespace object { -class ObjectFile; - class MachOUniversalBinary : public Binary { virtual void anchor(); @@ -53,14 +51,18 @@ public: ObjectForArch getNext() const { return ObjectForArch(Parent, Index + 1); } uint32_t getCPUType() const { return Header.cputype; } + uint32_t getCPUSubType() const { return Header.cpusubtype; } + uint32_t getOffset() const { return Header.offset; } + uint32_t getSize() const { return Header.size; } + uint32_t getAlign() const { return Header.align; } std::string getArchTypeName() const { Triple T = MachOObjectFile::getArch(Header.cputype, Header.cpusubtype); return T.getArchName(); } - ErrorOr> getAsObjectFile() const; + ErrorOr> getAsObjectFile() const; - std::error_code getAsArchive(std::unique_ptr &Result) const; + ErrorOr> getAsArchive() const; }; class object_iterator { @@ -84,10 +86,9 @@ public: } }; - MachOUniversalBinary(std::unique_ptr Source, - std::error_code &ec); - static ErrorOr - create(std::unique_ptr Source); + MachOUniversalBinary(MemoryBufferRef Souce, std::error_code &EC); + static ErrorOr> + create(MemoryBufferRef Source); object_iterator begin_objects() const { return ObjectForArch(this, 0); @@ -103,7 +104,7 @@ public: return V->isMachOUniversalBinary(); } - ErrorOr> + ErrorOr> getObjectForArch(Triple::ArchType Arch) const; }; diff --git a/contrib/llvm/include/llvm/Object/ObjectFile.h b/contrib/llvm/include/llvm/Object/ObjectFile.h index a4370a3f30d0..6aa985d7e67b 100644 --- a/contrib/llvm/include/llvm/Object/ObjectFile.h +++ b/contrib/llvm/include/llvm/Object/ObjectFile.h @@ -27,6 +27,8 @@ namespace llvm { namespace object { class ObjectFile; +class COFFObjectFile; +class MachOObjectFile; class SymbolRef; class symbol_iterator; @@ -93,23 +95,19 @@ public: void moveNext(); std::error_code getName(StringRef &Result) const; - std::error_code getAddress(uint64_t &Result) const; - std::error_code getSize(uint64_t &Result) const; + uint64_t getAddress() const; + uint64_t getSize() const; std::error_code getContents(StringRef &Result) const; /// @brief Get the alignment of this section as the actual value (not log 2). - std::error_code getAlignment(uint64_t &Result) const; + uint64_t getAlignment() const; - // FIXME: Move to the normalization layer when it's created. - std::error_code isText(bool &Result) const; - std::error_code isData(bool &Result) const; - std::error_code isBSS(bool &Result) const; - std::error_code isRequiredForExecution(bool &Result) const; - std::error_code isVirtual(bool &Result) const; - std::error_code isZeroInit(bool &Result) const; - std::error_code isReadOnlyData(bool &Result) const; + bool isText() const; + bool isData() const; + bool isBSS() const; + bool isVirtual() const; - std::error_code containsSymbol(SymbolRef S, bool &Result) const; + bool containsSymbol(SymbolRef S) const; relocation_iterator relocation_begin() const; relocation_iterator relocation_end() const; @@ -120,6 +118,7 @@ public: section_iterator getRelocatedSection() const; DataRefImpl getRawDataRefImpl() const; + const ObjectFile *getObject() const; }; /// SymbolRef - This is a value type class that represents a single symbol in @@ -176,30 +175,6 @@ public: } }; -/// LibraryRef - This is a value type class that represents a single library in -/// the list of libraries needed by a shared or dynamic object. -class LibraryRef { - friend class SectionRef; - DataRefImpl LibraryPimpl; - const ObjectFile *OwningObject; - -public: - LibraryRef() : OwningObject(nullptr) { } - - LibraryRef(DataRefImpl LibraryP, const ObjectFile *Owner); - - bool operator==(const LibraryRef &Other) const; - bool operator<(const LibraryRef &Other) const; - - std::error_code getNext(LibraryRef &Result) const; - - // Get the path to this library, as stored in the object file. - std::error_code getPath(StringRef &Result) const; - - DataRefImpl getRawDataRefImpl() const; -}; -typedef content_iterator library_iterator; - /// ObjectFile - This class is the base class for all object file types. /// Concrete instances of this object are created by createObjectFile, which /// figures out which type to create. @@ -209,10 +184,10 @@ class ObjectFile : public SymbolicFile { ObjectFile(const ObjectFile &other) LLVM_DELETED_FUNCTION; protected: - ObjectFile(unsigned int Type, std::unique_ptr Source); + ObjectFile(unsigned int Type, MemoryBufferRef Source); const uint8_t *base() const { - return reinterpret_cast(Data->getBufferStart()); + return reinterpret_cast(Data.getBufferStart()); } // These functions are for SymbolRef to call internally. The main goal of @@ -248,29 +223,18 @@ protected: virtual void moveSectionNext(DataRefImpl &Sec) const = 0; virtual std::error_code getSectionName(DataRefImpl Sec, StringRef &Res) const = 0; - virtual std::error_code getSectionAddress(DataRefImpl Sec, - uint64_t &Res) const = 0; - virtual std::error_code getSectionSize(DataRefImpl Sec, - uint64_t &Res) const = 0; + virtual uint64_t getSectionAddress(DataRefImpl Sec) const = 0; + virtual uint64_t getSectionSize(DataRefImpl Sec) const = 0; virtual std::error_code getSectionContents(DataRefImpl Sec, StringRef &Res) const = 0; - virtual std::error_code getSectionAlignment(DataRefImpl Sec, - uint64_t &Res) const = 0; - virtual std::error_code isSectionText(DataRefImpl Sec, bool &Res) const = 0; - virtual std::error_code isSectionData(DataRefImpl Sec, bool &Res) const = 0; - virtual std::error_code isSectionBSS(DataRefImpl Sec, bool &Res) const = 0; - virtual std::error_code isSectionRequiredForExecution(DataRefImpl Sec, - bool &Res) const = 0; + virtual uint64_t getSectionAlignment(DataRefImpl Sec) const = 0; + virtual bool isSectionText(DataRefImpl Sec) const = 0; + virtual bool isSectionData(DataRefImpl Sec) const = 0; + virtual bool isSectionBSS(DataRefImpl Sec) const = 0; // A section is 'virtual' if its contents aren't present in the object image. - virtual std::error_code isSectionVirtual(DataRefImpl Sec, - bool &Res) const = 0; - virtual std::error_code isSectionZeroInit(DataRefImpl Sec, - bool &Res) const = 0; - virtual std::error_code isSectionReadOnlyData(DataRefImpl Sec, - bool &Res) const = 0; - virtual std::error_code sectionContainsSymbol(DataRefImpl Sec, - DataRefImpl Symb, - bool &Result) const = 0; + virtual bool isSectionVirtual(DataRefImpl Sec) const = 0; + virtual bool sectionContainsSymbol(DataRefImpl Sec, + DataRefImpl Symb) const = 0; virtual relocation_iterator section_rel_begin(DataRefImpl Sec) const = 0; virtual relocation_iterator section_rel_end(DataRefImpl Sec) const = 0; virtual section_iterator getRelocatedSection(DataRefImpl Sec) const; @@ -297,13 +261,6 @@ protected: return object_error::success; } - // Same for LibraryRef - friend class LibraryRef; - virtual std::error_code getLibraryNext(DataRefImpl Lib, - LibraryRef &Res) const = 0; - virtual std::error_code getLibraryPath(DataRefImpl Lib, - StringRef &Res) const = 0; - public: typedef iterator_range symbol_iterator_range; symbol_iterator_range symbols() const { @@ -318,9 +275,6 @@ public: return section_iterator_range(section_begin(), section_end()); } - virtual library_iterator needed_library_begin() const = 0; - virtual library_iterator needed_library_end() const = 0; - /// @brief The number of bytes used to represent an address in this object /// file format. virtual uint8_t getBytesInAddress() const = 0; @@ -328,27 +282,26 @@ public: virtual StringRef getFileFormatName() const = 0; virtual /* Triple::ArchType */ unsigned getArch() const = 0; - /// For shared objects, returns the name which this object should be - /// loaded from at runtime. This corresponds to DT_SONAME on ELF and - /// LC_ID_DYLIB (install name) on MachO. - virtual StringRef getLoadName() const = 0; - /// Returns platform-specific object flags, if any. virtual std::error_code getPlatformFlags(unsigned &Result) const { Result = 0; return object_error::invalid_file_type; } + /// True if this is a relocatable object (.o/.obj). + virtual bool isRelocatableObject() const = 0; + /// @returns Pointer to ObjectFile subclass to handle this type of object. /// @param ObjectPath The path to the object file. ObjectPath.isObject must /// return true. /// @brief Create ObjectFile from path. - static ErrorOr createObjectFile(StringRef ObjectPath); - static ErrorOr - createObjectFile(std::unique_ptr &Object, - sys::fs::file_magic Type); - static ErrorOr - createObjectFile(std::unique_ptr &Object) { + static ErrorOr> + createObjectFile(StringRef ObjectPath); + + static ErrorOr> + createObjectFile(MemoryBufferRef Object, sys::fs::file_magic Type); + static ErrorOr> + createObjectFile(MemoryBufferRef Object) { return createObjectFile(Object, sys::fs::file_magic::unknown); } @@ -357,13 +310,14 @@ public: return v->isObject(); } -public: - static ErrorOr - createCOFFObjectFile(std::unique_ptr Object); - static ErrorOr - createELFObjectFile(std::unique_ptr &Object); - static ErrorOr - createMachOObjectFile(std::unique_ptr &Object); + static ErrorOr> + createCOFFObjectFile(MemoryBufferRef Object); + + static ErrorOr> + createELFObjectFile(MemoryBufferRef Object); + + static ErrorOr> + createMachOObjectFile(MemoryBufferRef Object); }; // Inline function definitions. @@ -430,54 +384,41 @@ inline std::error_code SectionRef::getName(StringRef &Result) const { return OwningObject->getSectionName(SectionPimpl, Result); } -inline std::error_code SectionRef::getAddress(uint64_t &Result) const { - return OwningObject->getSectionAddress(SectionPimpl, Result); +inline uint64_t SectionRef::getAddress() const { + return OwningObject->getSectionAddress(SectionPimpl); } -inline std::error_code SectionRef::getSize(uint64_t &Result) const { - return OwningObject->getSectionSize(SectionPimpl, Result); +inline uint64_t SectionRef::getSize() const { + return OwningObject->getSectionSize(SectionPimpl); } inline std::error_code SectionRef::getContents(StringRef &Result) const { return OwningObject->getSectionContents(SectionPimpl, Result); } -inline std::error_code SectionRef::getAlignment(uint64_t &Result) const { - return OwningObject->getSectionAlignment(SectionPimpl, Result); +inline uint64_t SectionRef::getAlignment() const { + return OwningObject->getSectionAlignment(SectionPimpl); } -inline std::error_code SectionRef::isText(bool &Result) const { - return OwningObject->isSectionText(SectionPimpl, Result); +inline bool SectionRef::isText() const { + return OwningObject->isSectionText(SectionPimpl); } -inline std::error_code SectionRef::isData(bool &Result) const { - return OwningObject->isSectionData(SectionPimpl, Result); +inline bool SectionRef::isData() const { + return OwningObject->isSectionData(SectionPimpl); } -inline std::error_code SectionRef::isBSS(bool &Result) const { - return OwningObject->isSectionBSS(SectionPimpl, Result); +inline bool SectionRef::isBSS() const { + return OwningObject->isSectionBSS(SectionPimpl); } -inline std::error_code SectionRef::isRequiredForExecution(bool &Result) const { - return OwningObject->isSectionRequiredForExecution(SectionPimpl, Result); +inline bool SectionRef::isVirtual() const { + return OwningObject->isSectionVirtual(SectionPimpl); } -inline std::error_code SectionRef::isVirtual(bool &Result) const { - return OwningObject->isSectionVirtual(SectionPimpl, Result); -} - -inline std::error_code SectionRef::isZeroInit(bool &Result) const { - return OwningObject->isSectionZeroInit(SectionPimpl, Result); -} - -inline std::error_code SectionRef::isReadOnlyData(bool &Result) const { - return OwningObject->isSectionReadOnlyData(SectionPimpl, Result); -} - -inline std::error_code SectionRef::containsSymbol(SymbolRef S, - bool &Result) const { +inline bool SectionRef::containsSymbol(SymbolRef S) const { return OwningObject->sectionContainsSymbol(SectionPimpl, - S.getRawDataRefImpl(), Result); + S.getRawDataRefImpl()); } inline relocation_iterator SectionRef::relocation_begin() const { @@ -496,6 +437,10 @@ inline DataRefImpl SectionRef::getRawDataRefImpl() const { return SectionPimpl; } +inline const ObjectFile *SectionRef::getObject() const { + return OwningObject; +} + /// RelocationRef inline RelocationRef::RelocationRef(DataRefImpl RelocationP, const ObjectFile *Owner) @@ -548,26 +493,6 @@ inline const ObjectFile *RelocationRef::getObjectFile() const { return OwningObject; } -// Inline function definitions. -inline LibraryRef::LibraryRef(DataRefImpl LibraryP, const ObjectFile *Owner) - : LibraryPimpl(LibraryP) - , OwningObject(Owner) {} - -inline bool LibraryRef::operator==(const LibraryRef &Other) const { - return LibraryPimpl == Other.LibraryPimpl; -} - -inline bool LibraryRef::operator<(const LibraryRef &Other) const { - return LibraryPimpl < Other.LibraryPimpl; -} - -inline std::error_code LibraryRef::getNext(LibraryRef &Result) const { - return OwningObject->getLibraryNext(LibraryPimpl, Result); -} - -inline std::error_code LibraryRef::getPath(StringRef &Result) const { - return OwningObject->getLibraryPath(LibraryPimpl, Result); -} } // end namespace object } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Object/RelocVisitor.h b/contrib/llvm/include/llvm/Object/RelocVisitor.h index 5ca245057a55..91eafd55ad76 100644 --- a/contrib/llvm/include/llvm/Object/RelocVisitor.h +++ b/contrib/llvm/include/llvm/Object/RelocVisitor.h @@ -17,6 +17,7 @@ #define LLVM_OBJECT_RELOCVISITOR_H #include "llvm/ADT/StringRef.h" +#include "llvm/Object/COFF.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" @@ -40,128 +41,18 @@ struct RelocToApply { /// @brief Base class for object file relocation visitors. class RelocVisitor { public: - explicit RelocVisitor(StringRef FileFormat) - : FileFormat(FileFormat), HasError(false) {} + explicit RelocVisitor(const ObjectFile &Obj) + : ObjToVisit(Obj), HasError(false) {} // TODO: Should handle multiple applied relocations via either passing in the // previously computed value or just count paired relocations as a single // visit. - RelocToApply visit(uint32_t RelocType, RelocationRef R, uint64_t SecAddr = 0, - uint64_t Value = 0) { - if (FileFormat == "ELF64-x86-64") { - switch (RelocType) { - case llvm::ELF::R_X86_64_NONE: - return visitELF_X86_64_NONE(R); - case llvm::ELF::R_X86_64_64: - return visitELF_X86_64_64(R, Value); - case llvm::ELF::R_X86_64_PC32: - return visitELF_X86_64_PC32(R, Value, SecAddr); - case llvm::ELF::R_X86_64_32: - return visitELF_X86_64_32(R, Value); - case llvm::ELF::R_X86_64_32S: - return visitELF_X86_64_32S(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF32-i386") { - switch (RelocType) { - case llvm::ELF::R_386_NONE: - return visitELF_386_NONE(R); - case llvm::ELF::R_386_32: - return visitELF_386_32(R, Value); - case llvm::ELF::R_386_PC32: - return visitELF_386_PC32(R, Value, SecAddr); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF64-ppc64") { - switch (RelocType) { - case llvm::ELF::R_PPC64_ADDR32: - return visitELF_PPC64_ADDR32(R, Value); - case llvm::ELF::R_PPC64_ADDR64: - return visitELF_PPC64_ADDR64(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF32-ppc") { - switch (RelocType) { - case llvm::ELF::R_PPC_ADDR32: - return visitELF_PPC_ADDR32(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF32-mips") { - switch (RelocType) { - case llvm::ELF::R_MIPS_32: - return visitELF_MIPS_32(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF64-mips") { - switch (RelocType) { - case llvm::ELF::R_MIPS_32: - return visitELF_MIPS_32(R, Value); - case llvm::ELF::R_MIPS_64: - return visitELF_MIPS_64(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF64-aarch64") { - switch (RelocType) { - case llvm::ELF::R_AARCH64_ABS32: - return visitELF_AARCH64_ABS32(R, Value); - case llvm::ELF::R_AARCH64_ABS64: - return visitELF_AARCH64_ABS64(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF64-s390") { - switch (RelocType) { - case llvm::ELF::R_390_32: - return visitELF_390_32(R, Value); - case llvm::ELF::R_390_64: - return visitELF_390_64(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF32-sparc") { - switch (RelocType) { - case llvm::ELF::R_SPARC_32: - case llvm::ELF::R_SPARC_UA32: - return visitELF_SPARC_32(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF64-sparc") { - switch (RelocType) { - case llvm::ELF::R_SPARC_32: - case llvm::ELF::R_SPARC_UA32: - return visitELF_SPARCV9_32(R, Value); - case llvm::ELF::R_SPARC_64: - case llvm::ELF::R_SPARC_UA64: - return visitELF_SPARCV9_64(R, Value); - default: - HasError = true; - return RelocToApply(); - } - } else if (FileFormat == "ELF32-arm") { - switch (RelocType) { - default: - HasError = true; - return RelocToApply(); - case llvm::ELF::R_ARM_ABS32: - return visitELF_ARM_ABS32(R, Value); - } - } + RelocToApply visit(uint32_t RelocType, RelocationRef R, uint64_t Value = 0) { + if (isa(ObjToVisit)) + return visitELF(RelocType, R, Value); + if (isa(ObjToVisit)) + return visitCOFF(RelocType, R, Value); + HasError = true; return RelocToApply(); } @@ -169,10 +60,168 @@ public: bool error() { return HasError; } private: - StringRef FileFormat; + const ObjectFile &ObjToVisit; bool HasError; - int64_t getAddend32LE(RelocationRef R) { + RelocToApply visitELF(uint32_t RelocType, RelocationRef R, uint64_t Value) { + if (ObjToVisit.getBytesInAddress() == 8) { // 64-bit object file + switch (ObjToVisit.getArch()) { + case Triple::x86_64: + switch (RelocType) { + case llvm::ELF::R_X86_64_NONE: + return visitELF_X86_64_NONE(R); + case llvm::ELF::R_X86_64_64: + return visitELF_X86_64_64(R, Value); + case llvm::ELF::R_X86_64_PC32: + return visitELF_X86_64_PC32(R, Value); + case llvm::ELF::R_X86_64_32: + return visitELF_X86_64_32(R, Value); + case llvm::ELF::R_X86_64_32S: + return visitELF_X86_64_32S(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::aarch64: + switch (RelocType) { + case llvm::ELF::R_AARCH64_ABS32: + return visitELF_AARCH64_ABS32(R, Value); + case llvm::ELF::R_AARCH64_ABS64: + return visitELF_AARCH64_ABS64(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::mips64el: + case Triple::mips64: + switch (RelocType) { + case llvm::ELF::R_MIPS_32: + return visitELF_MIPS_32(R, Value); + case llvm::ELF::R_MIPS_64: + return visitELF_MIPS_64(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::ppc64le: + case Triple::ppc64: + switch (RelocType) { + case llvm::ELF::R_PPC64_ADDR32: + return visitELF_PPC64_ADDR32(R, Value); + case llvm::ELF::R_PPC64_ADDR64: + return visitELF_PPC64_ADDR64(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::systemz: + switch (RelocType) { + case llvm::ELF::R_390_32: + return visitELF_390_32(R, Value); + case llvm::ELF::R_390_64: + return visitELF_390_64(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::sparcv9: + switch (RelocType) { + case llvm::ELF::R_SPARC_32: + case llvm::ELF::R_SPARC_UA32: + return visitELF_SPARCV9_32(R, Value); + case llvm::ELF::R_SPARC_64: + case llvm::ELF::R_SPARC_UA64: + return visitELF_SPARCV9_64(R, Value); + default: + HasError = true; + return RelocToApply(); + } + default: + HasError = true; + return RelocToApply(); + } + } else if (ObjToVisit.getBytesInAddress() == 4) { // 32-bit object file + switch (ObjToVisit.getArch()) { + case Triple::x86: + switch (RelocType) { + case llvm::ELF::R_386_NONE: + return visitELF_386_NONE(R); + case llvm::ELF::R_386_32: + return visitELF_386_32(R, Value); + case llvm::ELF::R_386_PC32: + return visitELF_386_PC32(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::ppc: + switch (RelocType) { + case llvm::ELF::R_PPC_ADDR32: + return visitELF_PPC_ADDR32(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::arm: + case Triple::armeb: + switch (RelocType) { + default: + HasError = true; + return RelocToApply(); + case llvm::ELF::R_ARM_ABS32: + return visitELF_ARM_ABS32(R, Value); + } + case Triple::mipsel: + case Triple::mips: + switch (RelocType) { + case llvm::ELF::R_MIPS_32: + return visitELF_MIPS_32(R, Value); + default: + HasError = true; + return RelocToApply(); + } + case Triple::sparc: + switch (RelocType) { + case llvm::ELF::R_SPARC_32: + case llvm::ELF::R_SPARC_UA32: + return visitELF_SPARC_32(R, Value); + default: + HasError = true; + return RelocToApply(); + } + default: + HasError = true; + return RelocToApply(); + } + } else { + report_fatal_error("Invalid word size in object file"); + } + } + + RelocToApply visitCOFF(uint32_t RelocType, RelocationRef R, uint64_t Value) { + switch (ObjToVisit.getArch()) { + case Triple::x86: + switch (RelocType) { + case COFF::IMAGE_REL_I386_SECREL: + return visitCOFF_I386_SECREL(R, Value); + case COFF::IMAGE_REL_I386_DIR32: + return visitCOFF_I386_DIR32(R, Value); + } + break; + case Triple::x86_64: + switch (RelocType) { + case COFF::IMAGE_REL_AMD64_SECREL: + return visitCOFF_AMD64_SECREL(R, Value); + case COFF::IMAGE_REL_AMD64_ADDR64: + return visitCOFF_AMD64_ADDR64(R, Value); + } + break; + } + HasError = true; + return RelocToApply(); + } + + int64_t getELFAddend32LE(RelocationRef R) { const ELF32LEObjectFile *Obj = cast(R.getObjectFile()); DataRefImpl DRI = R.getRawDataRefImpl(); int64_t Addend; @@ -180,7 +229,7 @@ private: return Addend; } - int64_t getAddend64LE(RelocationRef R) { + int64_t getELFAddend64LE(RelocationRef R) { const ELF64LEObjectFile *Obj = cast(R.getObjectFile()); DataRefImpl DRI = R.getRawDataRefImpl(); int64_t Addend; @@ -188,7 +237,7 @@ private: return Addend; } - int64_t getAddend32BE(RelocationRef R) { + int64_t getELFAddend32BE(RelocationRef R) { const ELF32BEObjectFile *Obj = cast(R.getObjectFile()); DataRefImpl DRI = R.getRawDataRefImpl(); int64_t Addend; @@ -196,7 +245,7 @@ private: return Addend; } - int64_t getAddend64BE(RelocationRef R) { + int64_t getELFAddend64BE(RelocationRef R) { const ELF64BEObjectFile *Obj = cast(R.getObjectFile()); DataRefImpl DRI = R.getRawDataRefImpl(); int64_t Addend; @@ -213,13 +262,12 @@ private: // Ideally the Addend here will be the addend in the data for // the relocation. It's not actually the case for Rel relocations. RelocToApply visitELF_386_32(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend32LE(R); + int64_t Addend = getELFAddend32LE(R); return RelocToApply(Value + Addend, 4); } - RelocToApply visitELF_386_PC32(RelocationRef R, uint64_t Value, - uint64_t SecAddr) { - int64_t Addend = getAddend32LE(R); + RelocToApply visitELF_386_PC32(RelocationRef R, uint64_t Value) { + int64_t Addend = getELFAddend32LE(R); uint64_t Address; R.getOffset(Address); return RelocToApply(Value + Addend - Address, 4); @@ -230,23 +278,22 @@ private: return RelocToApply(0, 0); } RelocToApply visitELF_X86_64_64(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64LE(R); + int64_t Addend = getELFAddend64LE(R); return RelocToApply(Value + Addend, 8); } - RelocToApply visitELF_X86_64_PC32(RelocationRef R, uint64_t Value, - uint64_t SecAddr) { - int64_t Addend = getAddend64LE(R); + RelocToApply visitELF_X86_64_PC32(RelocationRef R, uint64_t Value) { + int64_t Addend = getELFAddend64LE(R); uint64_t Address; R.getOffset(Address); return RelocToApply(Value + Addend - Address, 4); } RelocToApply visitELF_X86_64_32(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64LE(R); + int64_t Addend = getELFAddend64LE(R); uint32_t Res = (Value + Addend) & 0xFFFFFFFF; return RelocToApply(Res, 4); } RelocToApply visitELF_X86_64_32S(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64LE(R); + int64_t Addend = getELFAddend64LE(R); int32_t Res = (Value + Addend) & 0xFFFFFFFF; return RelocToApply(Res, 4); } @@ -266,7 +313,7 @@ private: /// PPC32 ELF RelocToApply visitELF_PPC_ADDR32(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend32BE(R); + int64_t Addend = getELFAddend32BE(R); uint32_t Res = (Value + Addend) & 0xFFFFFFFF; return RelocToApply(Res, 4); } @@ -288,7 +335,8 @@ private: // AArch64 ELF RelocToApply visitELF_AARCH64_ABS32(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64LE(R); + int64_t Addend; + getELFRelocationAddend(R, Addend); int64_t Res = Value + Addend; // Overflow check allows for both signed and unsigned interpretation. @@ -299,13 +347,14 @@ private: } RelocToApply visitELF_AARCH64_ABS64(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64LE(R); + int64_t Addend; + getELFRelocationAddend(R, Addend); return RelocToApply(Value + Addend, 8); } // SystemZ ELF RelocToApply visitELF_390_32(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64BE(R); + int64_t Addend = getELFAddend64BE(R); int64_t Res = Value + Addend; // Overflow check allows for both signed and unsigned interpretation. @@ -316,30 +365,54 @@ private: } RelocToApply visitELF_390_64(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64BE(R); + int64_t Addend = getELFAddend64BE(R); return RelocToApply(Value + Addend, 8); } RelocToApply visitELF_SPARC_32(RelocationRef R, uint32_t Value) { - int32_t Addend = getAddend32BE(R); + int32_t Addend = getELFAddend32BE(R); return RelocToApply(Value + Addend, 4); } RelocToApply visitELF_SPARCV9_32(RelocationRef R, uint64_t Value) { - int32_t Addend = getAddend64BE(R); + int32_t Addend = getELFAddend64BE(R); return RelocToApply(Value + Addend, 4); } RelocToApply visitELF_SPARCV9_64(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend64BE(R); + int64_t Addend = getELFAddend64BE(R); return RelocToApply(Value + Addend, 8); } RelocToApply visitELF_ARM_ABS32(RelocationRef R, uint64_t Value) { - int64_t Addend = getAddend32LE(R); - return RelocToApply(Value + Addend, 4); + int64_t Addend; + getELFRelocationAddend(R, Addend); + int64_t Res = Value + Addend; + + // Overflow check allows for both signed and unsigned interpretation. + if (Res < INT32_MIN || Res > UINT32_MAX) + HasError = true; + + return RelocToApply(static_cast(Res), 4); } + /// I386 COFF + RelocToApply visitCOFF_I386_SECREL(RelocationRef R, uint64_t Value) { + return RelocToApply(static_cast(Value), /*Width=*/4); + } + + RelocToApply visitCOFF_I386_DIR32(RelocationRef R, uint64_t Value) { + return RelocToApply(static_cast(Value), /*Width=*/4); + } + + /// AMD64 COFF + RelocToApply visitCOFF_AMD64_SECREL(RelocationRef R, uint64_t Value) { + return RelocToApply(static_cast(Value), /*Width=*/4); + } + + RelocToApply visitCOFF_AMD64_ADDR64(RelocationRef R, uint64_t Value) { + return RelocToApply(Value, /*Width=*/8); + } }; } diff --git a/contrib/llvm/include/llvm/Object/SymbolicFile.h b/contrib/llvm/include/llvm/Object/SymbolicFile.h index 77eef4a546aa..435799a34ebc 100644 --- a/contrib/llvm/include/llvm/Object/SymbolicFile.h +++ b/contrib/llvm/include/llvm/Object/SymbolicFile.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_OBJECT_SYMBOLIC_FILE_H -#define LLVM_OBJECT_SYMBOLIC_FILE_H +#ifndef LLVM_OBJECT_SYMBOLICFILE_H +#define LLVM_OBJECT_SYMBOLICFILE_H #include "llvm/Object/Binary.h" @@ -87,8 +87,9 @@ public: SF_Absolute = 1U << 3, // Absolute symbol SF_Common = 1U << 4, // Symbol has common linkage SF_Indirect = 1U << 5, // Symbol is an alias to another symbol - SF_FormatSpecific = 1U << 6 // Specific to the object file format + SF_FormatSpecific = 1U << 6, // Specific to the object file format // (e.g. section symbols) + SF_Thumb = 1U << 7 // Thumb symbol in a 32-bit ARM binary }; BasicSymbolRef() : OwningObject(nullptr) { } @@ -115,7 +116,7 @@ const uint64_t UnknownAddressOrSize = ~0ULL; class SymbolicFile : public Binary { public: virtual ~SymbolicFile(); - SymbolicFile(unsigned int Type, std::unique_ptr Source); + SymbolicFile(unsigned int Type, MemoryBufferRef Source); // virtual interface. virtual void moveSymbolNext(DataRefImpl &Symb) const = 0; @@ -142,15 +143,16 @@ public: } // construction aux. - static ErrorOr - createSymbolicFile(std::unique_ptr &Object, - sys::fs::file_magic Type, LLVMContext *Context); + static ErrorOr> + createSymbolicFile(MemoryBufferRef Object, sys::fs::file_magic Type, + LLVMContext *Context); - static ErrorOr - createSymbolicFile(std::unique_ptr &Object) { + static ErrorOr> + createSymbolicFile(MemoryBufferRef Object) { return createSymbolicFile(Object, sys::fs::file_magic::unknown, nullptr); } - static ErrorOr createSymbolicFile(StringRef ObjectPath); + static ErrorOr> + createSymbolicFile(StringRef ObjectPath); static inline bool classof(const Binary *v) { return v->isSymbolic(); diff --git a/contrib/llvm/include/llvm/Option/ArgList.h b/contrib/llvm/include/llvm/Option/ArgList.h index d46b0e892faf..3f8547e7fe41 100644 --- a/contrib/llvm/include/llvm/Option/ArgList.h +++ b/contrib/llvm/include/llvm/Option/ArgList.h @@ -187,6 +187,7 @@ public: /// /// \p Claim Whether the argument should be claimed, if it exists. Arg *getLastArgNoClaim(OptSpecifier Id) const; + Arg *getLastArgNoClaim(OptSpecifier Id0, OptSpecifier Id1) const; Arg *getLastArg(OptSpecifier Id) const; Arg *getLastArg(OptSpecifier Id0, OptSpecifier Id1) const; Arg *getLastArg(OptSpecifier Id0, OptSpecifier Id1, OptSpecifier Id2) const; diff --git a/contrib/llvm/include/llvm/PassRegistry.h b/contrib/llvm/include/llvm/PassRegistry.h index 1558c51bde48..8c28ef5e7e61 100644 --- a/contrib/llvm/include/llvm/PassRegistry.h +++ b/contrib/llvm/include/llvm/PassRegistry.h @@ -42,61 +42,51 @@ class PassRegistry { mutable sys::SmartRWMutex Lock; /// PassInfoMap - Keep track of the PassInfo object for each registered pass. - typedef DenseMap MapType; + typedef DenseMap MapType; MapType PassInfoMap; - - typedef StringMap StringMapType; + + typedef StringMap StringMapType; StringMapType PassInfoStringMap; - - /// AnalysisGroupInfo - Keep track of information for each analysis group. - struct AnalysisGroupInfo { - SmallPtrSet Implementations; - }; - DenseMap AnalysisGroupInfoMap; - + std::vector> ToFree; - std::vector Listeners; - + std::vector Listeners; + public: - PassRegistry() { } + PassRegistry() {} ~PassRegistry(); - - /// getPassRegistry - Access the global registry object, which is + + /// getPassRegistry - Access the global registry object, which is /// automatically initialized at application launch and destroyed by /// llvm_shutdown. static PassRegistry *getPassRegistry(); - + /// getPassInfo - Look up a pass' corresponding PassInfo, indexed by the pass' /// type identifier (&MyPass::ID). const PassInfo *getPassInfo(const void *TI) const; - + /// getPassInfo - Look up a pass' corresponding PassInfo, indexed by the pass' /// argument string. const PassInfo *getPassInfo(StringRef Arg) const; - - /// registerPass - Register a pass (by means of its PassInfo) with the + + /// registerPass - Register a pass (by means of its PassInfo) with the /// registry. Required in order to use the pass with a PassManager. void registerPass(const PassInfo &PI, bool ShouldFree = false); - - /// registerPass - Unregister a pass (by means of its PassInfo) with the - /// registry. - void unregisterPass(const PassInfo &PI); - + /// registerAnalysisGroup - Register an analysis group (or a pass implementing - // an analysis group) with the registry. Like registerPass, this is required + // an analysis group) with the registry. Like registerPass, this is required // in order for a PassManager to be able to use this group/pass. void registerAnalysisGroup(const void *InterfaceID, const void *PassID, - PassInfo& Registeree, bool isDefault, + PassInfo &Registeree, bool isDefault, bool ShouldFree = false); - + /// enumerateWith - Enumerate the registered passes, calling the provided /// PassRegistrationListener's passEnumerate() callback on each of them. void enumerateWith(PassRegistrationListener *L); - + /// addRegistrationListener - Register the given PassRegistrationListener /// to receive passRegistered() callbacks whenever a new pass is registered. void addRegistrationListener(PassRegistrationListener *L); - + /// removeRegistrationListener - Unregister a PassRegistrationListener so that /// it no longer receives passRegistered() callbacks. void removeRegistrationListener(PassRegistrationListener *L); diff --git a/contrib/llvm/include/llvm/PassSupport.h b/contrib/llvm/include/llvm/PassSupport.h index 449bc9281084..6cb6516412e8 100644 --- a/contrib/llvm/include/llvm/PassSupport.h +++ b/contrib/llvm/include/llvm/PassSupport.h @@ -82,6 +82,15 @@ class TargetMachine; CALL_ONCE_INITIALIZATION(initialize##passName##PassOnce) \ } +#define INITIALIZE_PASS_WITH_OPTIONS(PassName, Arg, Name, Cfg, Analysis) \ + INITIALIZE_PASS_BEGIN(PassName, Arg, Name, Cfg, Analysis) \ + PassName::registerOptions(); \ + INITIALIZE_PASS_END(PassName, Arg, Name, Cfg, Analysis) + +#define INITIALIZE_PASS_WITH_OPTIONS_BEGIN(PassName, Arg, Name, Cfg, Analysis) \ + INITIALIZE_PASS_BEGIN(PassName, Arg, Name, Cfg, Analysis) \ + PassName::registerOptions(); \ + template Pass *callDefaultCtor() { return new PassName(); } diff --git a/contrib/llvm/include/llvm/ProfileData/CoverageMapping.h b/contrib/llvm/include/llvm/ProfileData/CoverageMapping.h new file mode 100644 index 000000000000..38fc8ca7c77c --- /dev/null +++ b/contrib/llvm/include/llvm/ProfileData/CoverageMapping.h @@ -0,0 +1,448 @@ +//=-- CoverageMapping.h - Code coverage mapping support ---------*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Code coverage mapping data is generated by clang and read by +// llvm-cov to show code coverage statistics for a file. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_COVERAGEMAPPING_H_ +#define LLVM_PROFILEDATA_COVERAGEMAPPING_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/iterator.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/raw_ostream.h" +#include + +namespace llvm { +class IndexedInstrProfReader; +namespace coverage { + +class ObjectFileCoverageMappingReader; + +class CoverageMapping; +struct CounterExpressions; + +enum CoverageMappingVersion { CoverageMappingVersion1 }; + +/// \brief A Counter is an abstract value that describes how to compute the +/// execution count for a region of code using the collected profile count data. +struct Counter { + enum CounterKind { Zero, CounterValueReference, Expression }; + static const unsigned EncodingTagBits = 2; + static const unsigned EncodingTagMask = 0x3; + static const unsigned EncodingCounterTagAndExpansionRegionTagBits = + EncodingTagBits + 1; + +private: + CounterKind Kind; + unsigned ID; + + Counter(CounterKind Kind, unsigned ID) : Kind(Kind), ID(ID) {} + +public: + Counter() : Kind(Zero), ID(0) {} + + CounterKind getKind() const { return Kind; } + + bool isZero() const { return Kind == Zero; } + + bool isExpression() const { return Kind == Expression; } + + unsigned getCounterID() const { return ID; } + + unsigned getExpressionID() const { return ID; } + + bool operator==(const Counter &Other) const { + return Kind == Other.Kind && ID == Other.ID; + } + + friend bool operator<(const Counter &LHS, const Counter &RHS) { + return std::tie(LHS.Kind, LHS.ID) < std::tie(RHS.Kind, RHS.ID); + } + + /// \brief Return the counter that represents the number zero. + static Counter getZero() { return Counter(); } + + /// \brief Return the counter that corresponds to a specific profile counter. + static Counter getCounter(unsigned CounterId) { + return Counter(CounterValueReference, CounterId); + } + + /// \brief Return the counter that corresponds to a specific + /// addition counter expression. + static Counter getExpression(unsigned ExpressionId) { + return Counter(Expression, ExpressionId); + } +}; + +/// \brief A Counter expression is a value that represents an arithmetic +/// operation with two counters. +struct CounterExpression { + enum ExprKind { Subtract, Add }; + ExprKind Kind; + Counter LHS, RHS; + + CounterExpression(ExprKind Kind, Counter LHS, Counter RHS) + : Kind(Kind), LHS(LHS), RHS(RHS) {} +}; + +/// \brief A Counter expression builder is used to construct the +/// counter expressions. It avoids unecessary duplication +/// and simplifies algebraic expressions. +class CounterExpressionBuilder { + /// \brief A list of all the counter expressions + std::vector Expressions; + /// \brief A lookup table for the index of a given expression. + llvm::DenseMap ExpressionIndices; + + /// \brief Return the counter which corresponds to the given expression. + /// + /// If the given expression is already stored in the builder, a counter + /// that references that expression is returned. Otherwise, the given + /// expression is added to the builder's collection of expressions. + Counter get(const CounterExpression &E); + + /// \brief Gather the terms of the expression tree for processing. + /// + /// This collects each addition and subtraction referenced by the counter into + /// a sequence that can be sorted and combined to build a simplified counter + /// expression. + void extractTerms(Counter C, int Sign, + SmallVectorImpl> &Terms); + + /// \brief Simplifies the given expression tree + /// by getting rid of algebraically redundant operations. + Counter simplify(Counter ExpressionTree); + +public: + ArrayRef getExpressions() const { return Expressions; } + + /// \brief Return a counter that represents the expression + /// that adds LHS and RHS. + Counter add(Counter LHS, Counter RHS); + + /// \brief Return a counter that represents the expression + /// that subtracts RHS from LHS. + Counter subtract(Counter LHS, Counter RHS); +}; + +/// \brief A Counter mapping region associates a source range with +/// a specific counter. +struct CounterMappingRegion { + enum RegionKind { + /// \brief A CodeRegion associates some code with a counter + CodeRegion, + + /// \brief An ExpansionRegion represents a file expansion region that + /// associates a source range with the expansion of a virtual source file, + /// such as for a macro instantiation or #include file. + ExpansionRegion, + + /// \brief A SkippedRegion represents a source range with code that + /// was skipped by a preprocessor or similar means. + SkippedRegion + }; + + static const unsigned EncodingHasCodeBeforeBits = 1; + + Counter Count; + unsigned FileID, ExpandedFileID; + unsigned LineStart, ColumnStart, LineEnd, ColumnEnd; + RegionKind Kind; + /// \brief A flag that is set to true when there is already code before + /// this region on the same line. + /// This is useful to accurately compute the execution counts for a line. + bool HasCodeBefore; + + CounterMappingRegion(Counter Count, unsigned FileID, unsigned LineStart, + unsigned ColumnStart, unsigned LineEnd, + unsigned ColumnEnd, bool HasCodeBefore = false, + RegionKind Kind = CodeRegion) + : Count(Count), FileID(FileID), ExpandedFileID(0), LineStart(LineStart), + ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd), + Kind(Kind), HasCodeBefore(HasCodeBefore) {} + + inline std::pair startLoc() const { + return std::pair(LineStart, ColumnStart); + } + + inline std::pair endLoc() const { + return std::pair(LineEnd, ColumnEnd); + } + + bool operator<(const CounterMappingRegion &Other) const { + if (FileID != Other.FileID) + return FileID < Other.FileID; + return startLoc() < Other.startLoc(); + } + + bool contains(const CounterMappingRegion &Other) const { + if (FileID != Other.FileID) + return false; + if (startLoc() > Other.startLoc()) + return false; + if (endLoc() < Other.endLoc()) + return false; + return true; + } +}; + +/// \brief Associates a source range with an execution count. +struct CountedRegion : public CounterMappingRegion { + uint64_t ExecutionCount; + + CountedRegion(const CounterMappingRegion &R, uint64_t ExecutionCount) + : CounterMappingRegion(R), ExecutionCount(ExecutionCount) {} +}; + +/// \brief A Counter mapping context is used to connect the counters, +/// expressions and the obtained counter values. +class CounterMappingContext { + ArrayRef Expressions; + ArrayRef CounterValues; + +public: + CounterMappingContext(ArrayRef Expressions, + ArrayRef CounterValues = ArrayRef()) + : Expressions(Expressions), CounterValues(CounterValues) {} + + void dump(const Counter &C, llvm::raw_ostream &OS) const; + void dump(const Counter &C) const { dump(C, llvm::outs()); } + + /// \brief Return the number of times that a region of code associated with + /// this counter was executed. + ErrorOr evaluate(const Counter &C) const; +}; + +/// \brief Code coverage information for a single function. +struct FunctionRecord { + /// \brief Raw function name. + std::string Name; + /// \brief Associated files. + std::vector Filenames; + /// \brief Regions in the function along with their counts. + std::vector CountedRegions; + /// \brief The number of times this function was executed. + uint64_t ExecutionCount; + + FunctionRecord(StringRef Name, ArrayRef Filenames, + uint64_t ExecutionCount) + : Name(Name), Filenames(Filenames.begin(), Filenames.end()), + ExecutionCount(ExecutionCount) {} +}; + +/// \brief Iterator over Functions, optionally filtered to a single file. +class FunctionRecordIterator + : public iterator_facade_base { + ArrayRef Records; + ArrayRef::iterator Current; + StringRef Filename; + + /// \brief Skip records whose primary file is not \c Filename. + void skipOtherFiles(); + +public: + FunctionRecordIterator(ArrayRef Records_, + StringRef Filename = "") + : Records(Records_), Current(Records.begin()), Filename(Filename) { + skipOtherFiles(); + } + + FunctionRecordIterator() : Current(Records.begin()) {} + + bool operator==(const FunctionRecordIterator &RHS) const { + return Current == RHS.Current && Filename == RHS.Filename; + } + + const FunctionRecord &operator*() const { return *Current; } + + FunctionRecordIterator &operator++() { + assert(Current != Records.end() && "incremented past end"); + ++Current; + skipOtherFiles(); + return *this; + } +}; + +/// \brief Coverage information for a macro expansion or #included file. +/// +/// When covered code has pieces that can be expanded for more detail, such as a +/// preprocessor macro use and its definition, these are represented as +/// expansions whose coverage can be looked up independently. +struct ExpansionRecord { + /// \brief The abstract file this expansion covers. + unsigned FileID; + /// \brief The region that expands to this record. + const CountedRegion &Region; + /// \brief Coverage for the expansion. + const FunctionRecord &Function; + + ExpansionRecord(const CountedRegion &Region, + const FunctionRecord &Function) + : FileID(Region.ExpandedFileID), Region(Region), Function(Function) {} +}; + +/// \brief The execution count information starting at a point in a file. +/// +/// A sequence of CoverageSegments gives execution counts for a file in format +/// that's simple to iterate through for processing. +struct CoverageSegment { + /// \brief The line where this segment begins. + unsigned Line; + /// \brief The column where this segment begins. + unsigned Col; + /// \brief The execution count, or zero if no count was recorded. + uint64_t Count; + /// \brief When false, the segment was uninstrumented or skipped. + bool HasCount; + /// \brief Whether this enters a new region or returns to a previous count. + bool IsRegionEntry; + + CoverageSegment(unsigned Line, unsigned Col, bool IsRegionEntry) + : Line(Line), Col(Col), Count(0), HasCount(false), + IsRegionEntry(IsRegionEntry) {} + void setCount(uint64_t NewCount) { + Count = NewCount; + HasCount = true; + } + void addCount(uint64_t NewCount) { setCount(Count + NewCount); } +}; + +/// \brief Coverage information to be processed or displayed. +/// +/// This represents the coverage of an entire file, expansion, or function. It +/// provides a sequence of CoverageSegments to iterate through, as well as the +/// list of expansions that can be further processed. +class CoverageData { + std::string Filename; + std::vector Segments; + std::vector Expansions; + friend class CoverageMapping; + +public: + CoverageData() {} + + CoverageData(StringRef Filename) : Filename(Filename) {} + + CoverageData(CoverageData &&RHS) + : Filename(std::move(RHS.Filename)), Segments(std::move(RHS.Segments)), + Expansions(std::move(RHS.Expansions)) {} + + /// \brief Get the name of the file this data covers. + StringRef getFilename() { return Filename; } + + std::vector::iterator begin() { return Segments.begin(); } + std::vector::iterator end() { return Segments.end(); } + bool empty() { return Segments.empty(); } + + /// \brief Expansions that can be further processed. + std::vector getExpansions() { return Expansions; } +}; + +/// \brief The mapping of profile information to coverage data. +/// +/// This is the main interface to get coverage information, using a profile to +/// fill out execution counts. +class CoverageMapping { + std::vector Functions; + unsigned MismatchedFunctionCount; + + CoverageMapping() : MismatchedFunctionCount(0) {} + +public: + /// \brief Load the coverage mapping using the given readers. + static ErrorOr> + load(ObjectFileCoverageMappingReader &CoverageReader, + IndexedInstrProfReader &ProfileReader); + + /// \brief Load the coverage mapping from the given files. + static ErrorOr> + load(StringRef ObjectFilename, StringRef ProfileFilename); + + /// \brief The number of functions that couldn't have their profiles mapped. + /// + /// This is a count of functions whose profile is out of date or otherwise + /// can't be associated with any coverage information. + unsigned getMismatchedCount() { return MismatchedFunctionCount; } + + /// \brief Returns the list of files that are covered. + std::vector getUniqueSourceFiles() const; + + /// \brief Get the coverage for a particular file. + /// + /// The given filename must be the name as recorded in the coverage + /// information. That is, only names returned from getUniqueSourceFiles will + /// yield a result. + CoverageData getCoverageForFile(StringRef Filename); + + /// \brief Gets all of the functions covered by this profile. + iterator_range getCoveredFunctions() const { + return make_range(FunctionRecordIterator(Functions), + FunctionRecordIterator()); + } + + /// \brief Gets all of the functions in a particular file. + iterator_range + getCoveredFunctions(StringRef Filename) const { + return make_range(FunctionRecordIterator(Functions, Filename), + FunctionRecordIterator()); + } + + /// \brief Get the list of function instantiations in the file. + /// + /// Fucntions that are instantiated more than once, such as C++ template + /// specializations, have distinct coverage records for each instantiation. + std::vector getInstantiations(StringRef Filename); + + /// \brief Get the coverage for a particular function. + CoverageData getCoverageForFunction(const FunctionRecord &Function); + + /// \brief Get the coverage for an expansion within a coverage set. + CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion); +}; + +} // end namespace coverage + +/// \brief Provide DenseMapInfo for CounterExpression +template<> struct DenseMapInfo { + static inline coverage::CounterExpression getEmptyKey() { + using namespace coverage; + return CounterExpression(CounterExpression::ExprKind::Subtract, + Counter::getCounter(~0U), + Counter::getCounter(~0U)); + } + + static inline coverage::CounterExpression getTombstoneKey() { + using namespace coverage; + return CounterExpression(CounterExpression::ExprKind::Add, + Counter::getCounter(~0U), + Counter::getCounter(~0U)); + } + + static unsigned getHashValue(const coverage::CounterExpression &V) { + return static_cast( + hash_combine(V.Kind, V.LHS.getKind(), V.LHS.getCounterID(), + V.RHS.getKind(), V.RHS.getCounterID())); + } + + static bool isEqual(const coverage::CounterExpression &LHS, + const coverage::CounterExpression &RHS) { + return LHS.Kind == RHS.Kind && LHS.LHS == RHS.LHS && LHS.RHS == RHS.RHS; + } +}; + + +} // end namespace llvm + +#endif // LLVM_PROFILEDATA_COVERAGEMAPPING_H_ diff --git a/contrib/llvm/include/llvm/ProfileData/CoverageMappingReader.h b/contrib/llvm/include/llvm/ProfileData/CoverageMappingReader.h new file mode 100644 index 000000000000..5a6b44b8d9ef --- /dev/null +++ b/contrib/llvm/include/llvm/ProfileData/CoverageMappingReader.h @@ -0,0 +1,208 @@ +//=-- CoverageMappingReader.h - Code coverage mapping reader ------*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for reading coverage mapping data for +// instrumentation based coverage. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_COVERAGEMAPPINGREADER_H +#define LLVM_PROFILEDATA_COVERAGEMAPPINGREADER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/ProfileData/CoverageMapping.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include + +namespace llvm { +namespace coverage { + +class ObjectFileCoverageMappingReader; + +/// \brief Coverage mapping information for a single function. +struct CoverageMappingRecord { + StringRef FunctionName; + uint64_t FunctionHash; + ArrayRef Filenames; + ArrayRef Expressions; + ArrayRef MappingRegions; +}; + +/// \brief A file format agnostic iterator over coverage mapping data. +class CoverageMappingIterator + : public std::iterator { + ObjectFileCoverageMappingReader *Reader; + CoverageMappingRecord Record; + + void increment(); + +public: + CoverageMappingIterator() : Reader(nullptr) {} + CoverageMappingIterator(ObjectFileCoverageMappingReader *Reader) + : Reader(Reader) { + increment(); + } + + CoverageMappingIterator &operator++() { + increment(); + return *this; + } + bool operator==(const CoverageMappingIterator &RHS) { + return Reader == RHS.Reader; + } + bool operator!=(const CoverageMappingIterator &RHS) { + return Reader != RHS.Reader; + } + CoverageMappingRecord &operator*() { return Record; } + CoverageMappingRecord *operator->() { return &Record; } +}; + +/// \brief Base class for the raw coverage mapping and filenames data readers. +class RawCoverageReader { +protected: + StringRef Data; + + /// \brief Return the error code. + std::error_code error(std::error_code EC) { return EC; } + + /// \brief Clear the current error code and return a successful one. + std::error_code success() { return error(instrprof_error::success); } + + RawCoverageReader(StringRef Data) : Data(Data) {} + + std::error_code readULEB128(uint64_t &Result); + std::error_code readIntMax(uint64_t &Result, uint64_t MaxPlus1); + std::error_code readSize(uint64_t &Result); + std::error_code readString(StringRef &Result); +}; + +/// \brief Reader for the raw coverage filenames. +class RawCoverageFilenamesReader : public RawCoverageReader { + std::vector &Filenames; + + RawCoverageFilenamesReader(const RawCoverageFilenamesReader &) + LLVM_DELETED_FUNCTION; + RawCoverageFilenamesReader & + operator=(const RawCoverageFilenamesReader &) LLVM_DELETED_FUNCTION; + +public: + RawCoverageFilenamesReader(StringRef Data, std::vector &Filenames) + : RawCoverageReader(Data), Filenames(Filenames) {} + + std::error_code read(); +}; + +/// \brief Reader for the raw coverage mapping data. +class RawCoverageMappingReader : public RawCoverageReader { + StringRef FunctionName; + ArrayRef TranslationUnitFilenames; + std::vector &Filenames; + std::vector &Expressions; + std::vector &MappingRegions; + + RawCoverageMappingReader(const RawCoverageMappingReader &) + LLVM_DELETED_FUNCTION; + RawCoverageMappingReader & + operator=(const RawCoverageMappingReader &) LLVM_DELETED_FUNCTION; + +public: + RawCoverageMappingReader(StringRef FunctionName, StringRef MappingData, + ArrayRef TranslationUnitFilenames, + std::vector &Filenames, + std::vector &Expressions, + std::vector &MappingRegions) + : RawCoverageReader(MappingData), FunctionName(FunctionName), + TranslationUnitFilenames(TranslationUnitFilenames), + Filenames(Filenames), Expressions(Expressions), + MappingRegions(MappingRegions) {} + + std::error_code read(CoverageMappingRecord &Record); + +private: + std::error_code decodeCounter(unsigned Value, Counter &C); + std::error_code readCounter(Counter &C); + std::error_code + readMappingRegionsSubArray(std::vector &MappingRegions, + unsigned InferredFileID, size_t NumFileIDs); +}; + +/// \brief Reader for the coverage mapping data that is emitted by the +/// frontend and stored in an object file. +class ObjectFileCoverageMappingReader { +public: + struct ProfileMappingRecord { + CoverageMappingVersion Version; + StringRef FunctionName; + uint64_t FunctionHash; + StringRef CoverageMapping; + size_t FilenamesBegin; + size_t FilenamesSize; + + ProfileMappingRecord(CoverageMappingVersion Version, StringRef FunctionName, + uint64_t FunctionHash, StringRef CoverageMapping, + size_t FilenamesBegin, size_t FilenamesSize) + : Version(Version), FunctionName(FunctionName), + FunctionHash(FunctionHash), CoverageMapping(CoverageMapping), + FilenamesBegin(FilenamesBegin), FilenamesSize(FilenamesSize) {} + }; + +private: + std::error_code LastError; + object::OwningBinary Object; + std::vector Filenames; + std::vector MappingRecords; + size_t CurrentRecord; + std::vector FunctionsFilenames; + std::vector Expressions; + std::vector MappingRegions; + + ObjectFileCoverageMappingReader(const ObjectFileCoverageMappingReader &) + LLVM_DELETED_FUNCTION; + ObjectFileCoverageMappingReader & + operator=(const ObjectFileCoverageMappingReader &) LLVM_DELETED_FUNCTION; + + /// \brief Set the current error_code and return same. + std::error_code error(std::error_code EC) { + LastError = EC; + return EC; + } + + /// \brief Clear the current error code and return a successful one. + std::error_code success() { return error(instrprof_error::success); } + +public: + ObjectFileCoverageMappingReader(StringRef FileName); + ObjectFileCoverageMappingReader( + std::unique_ptr &ObjectBuffer, + sys::fs::file_magic Type = sys::fs::file_magic::unknown); + + std::error_code readHeader(); + std::error_code readNextRecord(CoverageMappingRecord &Record); + + /// Iterator over profile data. + CoverageMappingIterator begin() { return CoverageMappingIterator(this); } + CoverageMappingIterator end() { return CoverageMappingIterator(); } + + /// \brief Return true if the reader has finished reading the profile data. + bool isEOF() { return LastError == instrprof_error::eof; } + /// \brief Return true if the reader encountered an error reading profiling + /// data. + bool hasError() { return LastError && !isEOF(); } + /// \brief Get the current error code. + std::error_code getError() { return LastError; } +}; + +} // end namespace coverage +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/ProfileData/CoverageMappingWriter.h b/contrib/llvm/include/llvm/ProfileData/CoverageMappingWriter.h new file mode 100644 index 000000000000..2e3b0378d032 --- /dev/null +++ b/contrib/llvm/include/llvm/ProfileData/CoverageMappingWriter.h @@ -0,0 +1,63 @@ +//=-- CoverageMappingWriter.h - Code coverage mapping writer ------*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for writing coverage mapping data for +// instrumentation based coverage. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_COVERAGEMAPPINGWRITER_H +#define LLVM_PROFILEDATA_COVERAGEMAPPINGWRITER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ProfileData/CoverageMapping.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace coverage { + +/// \brief Writer of the filenames section for the instrumentation +/// based code coverage. +class CoverageFilenamesSectionWriter { + ArrayRef Filenames; + +public: + CoverageFilenamesSectionWriter(ArrayRef Filenames) + : Filenames(Filenames) {} + + /// \brief Write encoded filenames to the given output stream. + void write(raw_ostream &OS); +}; + +/// \brief Writer for instrumentation based coverage mapping data. +class CoverageMappingWriter { + ArrayRef VirtualFileMapping; + ArrayRef Expressions; + MutableArrayRef MappingRegions; + +public: + CoverageMappingWriter(ArrayRef VirtualFileMapping, + ArrayRef Expressions, + MutableArrayRef MappingRegions) + : VirtualFileMapping(VirtualFileMapping), Expressions(Expressions), + MappingRegions(MappingRegions) {} + + CoverageMappingWriter(ArrayRef Expressions, + MutableArrayRef MappingRegions) + : Expressions(Expressions), MappingRegions(MappingRegions) {} + + /// \brief Write encoded coverage mapping data to the given output stream. + void write(raw_ostream &OS); +}; + +} // end namespace coverage +} // end namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h b/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h index 7a5a71dc6a31..9655d66edc92 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -12,17 +12,17 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_PROFILEDATA_INSTRPROF_READER_H_ -#define LLVM_PROFILEDATA_INSTRPROF_READER_H_ +#ifndef LLVM_PROFILEDATA_INSTRPROFREADER_H +#define LLVM_PROFILEDATA_INSTRPROFREADER_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/ErrorOr.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/EndianStream.h" #include "llvm/Support/OnDiskHashTable.h" - #include namespace llvm { @@ -94,8 +94,7 @@ public: /// Factory method to create an appropriately typed reader for the given /// instrprof file. - static std::error_code create(std::string Path, - std::unique_ptr &Result); + static ErrorOr> create(std::string Path); }; /// Reader for the simple text based instrprof format. @@ -120,7 +119,7 @@ private: LLVM_DELETED_FUNCTION; public: TextInstrProfReader(std::unique_ptr DataBuffer_) - : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, '#') {} + : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} /// Read the header. std::error_code readHeader() override { return success(); } @@ -206,12 +205,17 @@ enum class HashT : uint32_t; /// Trait for lookups into the on-disk hash table for the binary instrprof /// format. class InstrProfLookupTrait { - std::vector CountBuffer; + std::vector DataBuffer; IndexedInstrProf::HashT HashType; public: InstrProfLookupTrait(IndexedInstrProf::HashT HashType) : HashType(HashType) {} - typedef InstrProfRecord data_type; + struct data_type { + data_type(StringRef Name, ArrayRef Data) + : Name(Name), Data(Data) {} + StringRef Name; + ArrayRef Data; + }; typedef StringRef internal_key_type; typedef StringRef external_key_type; typedef uint64_t hash_value_type; @@ -234,25 +238,20 @@ public: return StringRef((const char *)D, N); } - InstrProfRecord ReadData(StringRef K, const unsigned char *D, offset_type N) { - if (N < 2 * sizeof(uint64_t) || N % sizeof(uint64_t)) { + data_type ReadData(StringRef K, const unsigned char *D, offset_type N) { + DataBuffer.clear(); + if (N % sizeof(uint64_t)) // The data is corrupt, don't try to read it. - CountBuffer.clear(); - return InstrProfRecord("", 0, CountBuffer); - } + return data_type("", DataBuffer); using namespace support; - - // The first stored value is the hash. - uint64_t Hash = endian::readNext(D); - // Each counter follows. - unsigned NumCounters = N / sizeof(uint64_t) - 1; - CountBuffer.clear(); - CountBuffer.reserve(NumCounters - 1); - for (unsigned I = 0; I < NumCounters; ++I) - CountBuffer.push_back(endian::readNext(D)); - - return InstrProfRecord(K, Hash, CountBuffer); + // We just treat the data as opaque here. It's simpler to handle in + // IndexedInstrProfReader. + unsigned NumEntries = N / sizeof(uint64_t); + DataBuffer.reserve(NumEntries); + for (unsigned I = 0; I < NumEntries; ++I) + DataBuffer.push_back(endian::readNext(D)); + return data_type(K, DataBuffer); } }; typedef OnDiskIterableChainedHashTable @@ -267,7 +266,11 @@ private: std::unique_ptr Index; /// Iterator over the profile data. InstrProfReaderIndex::data_iterator RecordIterator; - /// The maximal execution count among all fucntions. + /// Offset into our current data set. + size_t CurrentOffset; + /// The file format version of the profile data. + uint64_t FormatVersion; + /// The maximal execution count among all functions. uint64_t MaxFunctionCount; IndexedInstrProfReader(const IndexedInstrProfReader &) LLVM_DELETED_FUNCTION; @@ -275,8 +278,7 @@ private: LLVM_DELETED_FUNCTION; public: IndexedInstrProfReader(std::unique_ptr DataBuffer) - : DataBuffer(std::move(DataBuffer)), Index(nullptr), - RecordIterator(InstrProfReaderIndex::data_iterator()) {} + : DataBuffer(std::move(DataBuffer)), Index(nullptr), CurrentOffset(0) {} /// Return true if the given buffer is in an indexed instrprof format. static bool hasFormat(const MemoryBuffer &DataBuffer); @@ -287,7 +289,7 @@ public: std::error_code readNextRecord(InstrProfRecord &Record) override; /// Fill Counts with the profile data for the given function name. - std::error_code getFunctionCounts(StringRef FuncName, uint64_t &FuncHash, + std::error_code getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts); /// Return the maximum of all known function counts. uint64_t getMaximumFunctionCount() { return MaxFunctionCount; } @@ -299,4 +301,4 @@ public: } // end namespace llvm -#endif // LLVM_PROFILEDATA_INSTRPROF_READER_H_ +#endif diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h b/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h index 6e68bee30eb8..a23c56772a2f 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -12,15 +12,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_PROFILEDATA_INSTRPROF_WRITER_H_ -#define LLVM_PROFILEDATA_INSTRPROF_WRITER_H_ +#ifndef LLVM_PROFILEDATA_INSTRPROFWRITER_H +#define LLVM_PROFILEDATA_INSTRPROFWRITER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/raw_ostream.h" - #include namespace llvm { @@ -28,13 +28,13 @@ namespace llvm { /// Writer for instrumentation based profile data. class InstrProfWriter { public: - struct CounterData { - uint64_t Hash; - std::vector Counts; - }; + typedef SmallDenseMap, 1> CounterData; private: StringMap FunctionData; + uint64_t MaxFunctionCount; public: + InstrProfWriter() : MaxFunctionCount(0) {} + /// Add function counts for the given function. If there are already counts /// for this function and the hash and number of counts match, each counter is /// summed. @@ -47,4 +47,4 @@ public: } // end namespace llvm -#endif // LLVM_PROFILE_INSTRPROF_WRITER_H_ +#endif diff --git a/contrib/llvm/include/llvm/ProfileData/SampleProf.h b/contrib/llvm/include/llvm/ProfileData/SampleProf.h new file mode 100644 index 000000000000..df0a055c2a79 --- /dev/null +++ b/contrib/llvm/include/llvm/ProfileData/SampleProf.h @@ -0,0 +1,247 @@ +//=-- SampleProf.h - Sampling profiling format support --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains common definitions used in the reading and writing of +// sample profile data. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_PROFILEDATA_SAMPLEPROF_H_ +#define LLVM_PROFILEDATA_SAMPLEPROF_H_ + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include + +namespace llvm { + +const std::error_category &sampleprof_category(); + +enum class sampleprof_error { + success = 0, + bad_magic, + unsupported_version, + too_large, + truncated, + malformed, + unrecognized_format +}; + +inline std::error_code make_error_code(sampleprof_error E) { + return std::error_code(static_cast(E), sampleprof_category()); +} + +} // end namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} + +namespace llvm { + +namespace sampleprof { + +static inline uint64_t SPMagic() { + return uint64_t('S') << (64 - 8) | uint64_t('P') << (64 - 16) | + uint64_t('R') << (64 - 24) | uint64_t('O') << (64 - 32) | + uint64_t('F') << (64 - 40) | uint64_t('4') << (64 - 48) | + uint64_t('2') << (64 - 56) | uint64_t(0xff); +} + +static inline uint64_t SPVersion() { return 100; } + +/// \brief Represents the relative location of an instruction. +/// +/// Instruction locations are specified by the line offset from the +/// beginning of the function (marked by the line where the function +/// header is) and the discriminator value within that line. +/// +/// The discriminator value is useful to distinguish instructions +/// that are on the same line but belong to different basic blocks +/// (e.g., the two post-increment instructions in "if (p) x++; else y++;"). +struct LineLocation { + LineLocation(int L, unsigned D) : LineOffset(L), Discriminator(D) {} + int LineOffset; + unsigned Discriminator; +}; + +} // End namespace sampleprof + +template <> struct DenseMapInfo { + typedef DenseMapInfo OffsetInfo; + typedef DenseMapInfo DiscriminatorInfo; + static inline sampleprof::LineLocation getEmptyKey() { + return sampleprof::LineLocation(OffsetInfo::getEmptyKey(), + DiscriminatorInfo::getEmptyKey()); + } + static inline sampleprof::LineLocation getTombstoneKey() { + return sampleprof::LineLocation(OffsetInfo::getTombstoneKey(), + DiscriminatorInfo::getTombstoneKey()); + } + static inline unsigned getHashValue(sampleprof::LineLocation Val) { + return DenseMapInfo>::getHashValue( + std::pair(Val.LineOffset, Val.Discriminator)); + } + static inline bool isEqual(sampleprof::LineLocation LHS, + sampleprof::LineLocation RHS) { + return LHS.LineOffset == RHS.LineOffset && + LHS.Discriminator == RHS.Discriminator; + } +}; + +namespace sampleprof { + +/// \brief Representation of a single sample record. +/// +/// A sample record is represented by a positive integer value, which +/// indicates how frequently was the associated line location executed. +/// +/// Additionally, if the associated location contains a function call, +/// the record will hold a list of all the possible called targets. For +/// direct calls, this will be the exact function being invoked. For +/// indirect calls (function pointers, virtual table dispatch), this +/// will be a list of one or more functions. +class SampleRecord { +public: + typedef StringMap CallTargetMap; + + SampleRecord() : NumSamples(0), CallTargets() {} + + /// \brief Increment the number of samples for this record by \p S. + /// + /// Sample counts accumulate using saturating arithmetic, to avoid wrapping + /// around unsigned integers. + void addSamples(unsigned S) { + if (NumSamples <= std::numeric_limits::max() - S) + NumSamples += S; + else + NumSamples = std::numeric_limits::max(); + } + + /// \brief Add called function \p F with samples \p S. + /// + /// Sample counts accumulate using saturating arithmetic, to avoid wrapping + /// around unsigned integers. + void addCalledTarget(StringRef F, unsigned S) { + unsigned &TargetSamples = CallTargets[F]; + if (TargetSamples <= std::numeric_limits::max() - S) + TargetSamples += S; + else + TargetSamples = std::numeric_limits::max(); + } + + /// \brief Return true if this sample record contains function calls. + bool hasCalls() const { return CallTargets.size() > 0; } + + unsigned getSamples() const { return NumSamples; } + const CallTargetMap &getCallTargets() const { return CallTargets; } + + /// \brief Merge the samples in \p Other into this record. + void merge(const SampleRecord &Other) { + addSamples(Other.getSamples()); + for (const auto &I : Other.getCallTargets()) + addCalledTarget(I.first(), I.second); + } + +private: + unsigned NumSamples; + CallTargetMap CallTargets; +}; + +typedef DenseMap BodySampleMap; + +/// \brief Representation of the samples collected for a function. +/// +/// This data structure contains all the collected samples for the body +/// of a function. Each sample corresponds to a LineLocation instance +/// within the body of the function. +class FunctionSamples { +public: + FunctionSamples() : TotalSamples(0), TotalHeadSamples(0) {} + void print(raw_ostream &OS = dbgs()); + void addTotalSamples(unsigned Num) { TotalSamples += Num; } + void addHeadSamples(unsigned Num) { TotalHeadSamples += Num; } + void addBodySamples(int LineOffset, unsigned Discriminator, unsigned Num) { + assert(LineOffset >= 0); + // When dealing with instruction weights, we use the value + // zero to indicate the absence of a sample. If we read an + // actual zero from the profile file, use the value 1 to + // avoid the confusion later on. + if (Num == 0) + Num = 1; + BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(Num); + } + void addCalledTargetSamples(int LineOffset, unsigned Discriminator, + std::string FName, unsigned Num) { + assert(LineOffset >= 0); + BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(FName, + Num); + } + + /// \brief Return the sample record at the given location. + /// Each location is specified by \p LineOffset and \p Discriminator. + SampleRecord &sampleRecordAt(const LineLocation &Loc) { + return BodySamples[Loc]; + } + + /// \brief Return the number of samples collected at the given location. + /// Each location is specified by \p LineOffset and \p Discriminator. + unsigned samplesAt(int LineOffset, unsigned Discriminator) { + return sampleRecordAt(LineLocation(LineOffset, Discriminator)).getSamples(); + } + + bool empty() const { return BodySamples.empty(); } + + /// \brief Return the total number of samples collected inside the function. + unsigned getTotalSamples() const { return TotalSamples; } + + /// \brief Return the total number of samples collected at the head of the + /// function. + unsigned getHeadSamples() const { return TotalHeadSamples; } + + /// \brief Return all the samples collected in the body of the function. + const BodySampleMap &getBodySamples() const { return BodySamples; } + + /// \brief Merge the samples in \p Other into this one. + void merge(const FunctionSamples &Other) { + addTotalSamples(Other.getTotalSamples()); + addHeadSamples(Other.getHeadSamples()); + for (const auto &I : Other.getBodySamples()) { + const LineLocation &Loc = I.first; + const SampleRecord &Rec = I.second; + sampleRecordAt(Loc).merge(Rec); + } + } + +private: + /// \brief Total number of samples collected inside this function. + /// + /// Samples are cumulative, they include all the samples collected + /// inside this function and all its inlined callees. + unsigned TotalSamples; + + /// \brief Total number of samples collected at the head of the function. + unsigned TotalHeadSamples; + + /// \brief Map instruction locations to collected samples. + /// + /// Each entry in this map contains the number of samples + /// collected at the corresponding line offset. All line locations + /// are an offset from the start of the function. + BodySampleMap BodySamples; +}; + +} // End namespace sampleprof + +} // End namespace llvm + +#endif // LLVM_PROFILEDATA_SAMPLEPROF_H_ diff --git a/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h b/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h new file mode 100644 index 000000000000..c082a1abe951 --- /dev/null +++ b/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -0,0 +1,170 @@ +//===- SampleProfReader.h - Read LLVM sample profile data -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains definitions needed for reading sample profiles. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_PROFILEDATA_SAMPLEPROFREADER_H +#define LLVM_PROFILEDATA_SAMPLEPROFREADER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +namespace sampleprof { + +/// \brief Sample-based profile reader. +/// +/// Each profile contains sample counts for all the functions +/// executed. Inside each function, statements are annotated with the +/// collected samples on all the instructions associated with that +/// statement. +/// +/// For this to produce meaningful data, the program needs to be +/// compiled with some debug information (at minimum, line numbers: +/// -gline-tables-only). Otherwise, it will be impossible to match IR +/// instructions to the line numbers collected by the profiler. +/// +/// From the profile file, we are interested in collecting the +/// following information: +/// +/// * A list of functions included in the profile (mangled names). +/// +/// * For each function F: +/// 1. The total number of samples collected in F. +/// +/// 2. The samples collected at each line in F. To provide some +/// protection against source code shuffling, line numbers should +/// be relative to the start of the function. +/// +/// The reader supports two file formats: text and binary. The text format +/// is useful for debugging and testing, while the binary format is more +/// compact. They can both be used interchangeably. +class SampleProfileReader { +public: + SampleProfileReader(std::unique_ptr B, LLVMContext &C) + : Profiles(0), Ctx(C), Buffer(std::move(B)) {} + + virtual ~SampleProfileReader() {} + + /// \brief Read and validate the file header. + virtual std::error_code readHeader() = 0; + + /// \brief Read sample profiles from the associated file. + virtual std::error_code read() = 0; + + /// \brief Print the profile for \p FName on stream \p OS. + void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs()); + + /// \brief Print all the profiles on stream \p OS. + void dump(raw_ostream &OS = dbgs()); + + /// \brief Return the samples collected for function \p F. + FunctionSamples *getSamplesFor(const Function &F) { + return &Profiles[F.getName()]; + } + + /// \brief Return all the profiles. + StringMap &getProfiles() { return Profiles; } + + /// \brief Report a parse error message. + void reportParseError(int64_t LineNumber, Twine Msg) const { + Ctx.diagnose(DiagnosticInfoSampleProfile(Buffer->getBufferIdentifier(), + LineNumber, Msg)); + } + + /// \brief Create a sample profile reader appropriate to the file format. + static ErrorOr> + create(StringRef Filename, LLVMContext &C); + +protected: + /// \brief Map every function to its associated profile. + /// + /// The profile of every function executed at runtime is collected + /// in the structure FunctionSamples. This maps function objects + /// to their corresponding profiles. + StringMap Profiles; + + /// \brief LLVM context used to emit diagnostics. + LLVMContext &Ctx; + + /// \brief Memory buffer holding the profile file. + std::unique_ptr Buffer; +}; + +class SampleProfileReaderText : public SampleProfileReader { +public: + SampleProfileReaderText(std::unique_ptr B, LLVMContext &C) + : SampleProfileReader(std::move(B), C) {} + + /// \brief Read and validate the file header. + std::error_code readHeader() override { return sampleprof_error::success; } + + /// \brief Read sample profiles from the associated file. + std::error_code read() override; +}; + +class SampleProfileReaderBinary : public SampleProfileReader { +public: + SampleProfileReaderBinary(std::unique_ptr B, LLVMContext &C) + : SampleProfileReader(std::move(B), C), Data(nullptr), End(nullptr) {} + + /// \brief Read and validate the file header. + std::error_code readHeader() override; + + /// \brief Read sample profiles from the associated file. + std::error_code read() override; + + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); + +protected: + /// \brief Read a numeric value of type T from the profile. + /// + /// If an error occurs during decoding, a diagnostic message is emitted and + /// EC is set. + /// + /// \returns the read value. + template ErrorOr readNumber(); + + /// \brief Read a string from the profile. + /// + /// If an error occurs during decoding, a diagnostic message is emitted and + /// EC is set. + /// + /// \returns the read value. + ErrorOr readString(); + + /// \brief Return true if we've reached the end of file. + bool at_eof() const { return Data >= End; } + + /// \brief Points to the current location in the buffer. + const uint8_t *Data; + + /// \brief Points to the end of the buffer. + const uint8_t *End; +}; + +} // End namespace sampleprof + +} // End namespace llvm + +#endif // LLVM_PROFILEDATA_SAMPLEPROFREADER_H diff --git a/contrib/llvm/include/llvm/ProfileData/SampleProfWriter.h b/contrib/llvm/include/llvm/ProfileData/SampleProfWriter.h new file mode 100644 index 000000000000..302a82d32861 --- /dev/null +++ b/contrib/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -0,0 +1,110 @@ +//===- SampleProfWriter.h - Write LLVM sample profile data ----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains definitions needed for writing sample profiles. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_PROFILEDATA_SAMPLEPROFWRITER_H +#define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +namespace sampleprof { + +enum SampleProfileFormat { SPF_None = 0, SPF_Text, SPF_Binary, SPF_GCC }; + +/// \brief Sample-based profile writer. Base class. +class SampleProfileWriter { +public: + SampleProfileWriter(StringRef Filename, std::error_code &EC, + sys::fs::OpenFlags Flags) + : OS(Filename, EC, Flags) {} + virtual ~SampleProfileWriter() {} + + /// \brief Write sample profiles in \p S for function \p FName. + /// + /// \returns true if the file was updated successfully. False, otherwise. + virtual bool write(StringRef FName, const FunctionSamples &S) = 0; + + /// \brief Write sample profiles in \p S for function \p F. + bool write(const Function &F, const FunctionSamples &S) { + return write(F.getName(), S); + } + + /// \brief Write all the sample profiles for all the functions in \p M. + /// + /// \returns true if the file was updated successfully. False, otherwise. + bool write(const Module &M, StringMap &P) { + for (const auto &F : M) { + StringRef Name = F.getName(); + if (!write(Name, P[Name])) + return false; + } + return true; + } + + /// \brief Write all the sample profiles in the given map of samples. + /// + /// \returns true if the file was updated successfully. False, otherwise. + bool write(StringMap &ProfileMap) { + for (auto &I : ProfileMap) { + StringRef FName = I.first(); + FunctionSamples &Profile = I.second; + if (!write(FName, Profile)) + return false; + } + return true; + } + + /// \brief Profile writer factory. Create a new writer based on the value of + /// \p Format. + static ErrorOr> + create(StringRef Filename, SampleProfileFormat Format); + +protected: + /// \brief Output stream where to emit the profile to. + raw_fd_ostream OS; +}; + +/// \brief Sample-based profile writer (text format). +class SampleProfileWriterText : public SampleProfileWriter { +public: + SampleProfileWriterText(StringRef F, std::error_code &EC) + : SampleProfileWriter(F, EC, sys::fs::F_Text) {} + + bool write(StringRef FName, const FunctionSamples &S) override; + bool write(const Module &M, StringMap &P) { + return SampleProfileWriter::write(M, P); + } +}; + +/// \brief Sample-based profile writer (binary format). +class SampleProfileWriterBinary : public SampleProfileWriter { +public: + SampleProfileWriterBinary(StringRef F, std::error_code &EC); + + bool write(StringRef F, const FunctionSamples &S) override; + bool write(const Module &M, StringMap &P) { + return SampleProfileWriter::write(M, P); + } +}; + +} // End namespace sampleprof + +} // End namespace llvm + +#endif // LLVM_PROFILEDATA_SAMPLEPROFWRITER_H diff --git a/contrib/llvm/include/llvm/Support/ARMBuildAttributes.h b/contrib/llvm/include/llvm/Support/ARMBuildAttributes.h index f63e0a61f639..96a8219bfb85 100644 --- a/contrib/llvm/include/llvm/Support/ARMBuildAttributes.h +++ b/contrib/llvm/include/llvm/Support/ARMBuildAttributes.h @@ -16,8 +16,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_ARM_BUILD_ATTRIBUTES_H -#define LLVM_SUPPORT_ARM_BUILD_ATTRIBUTES_H +#ifndef LLVM_SUPPORT_ARMBUILDATTRIBUTES_H +#define LLVM_SUPPORT_ARMBUILDATTRIBUTES_H namespace llvm { class StringRef; @@ -146,6 +146,12 @@ enum { AllowNeon2 = 2, // SIMDv2 was permitted (Half-precision FP, MAC operations) AllowNeonARMv8 = 3, // ARM v8-A SIMD was permitted + // Tag_ABI_PCS_R9_use, (=14), uleb128 + R9IsGPR = 0, // R9 used as v6 (just another callee-saved register) + R9IsSB = 1, // R9 used as a global static base rgister + R9IsTLSPointer = 2, // R9 used as a thread local storage pointer + R9Reserved = 3, // R9 not used by code associated with attributed entity + // Tag_ABI_PCS_RW_data, (=15), uleb128 AddressRWPCRel = 1, // Address RW static data PC-relative AddressRWSBRel = 2, // Address RW static data SB-relative @@ -165,6 +171,8 @@ enum { WCharWidth4Bytes = 4, // sizeof(wchar_t) == 4 // Tag_ABI_FP_denormal, (=20), uleb128 + PositiveZero = 0, + IEEEDenormals = 1, PreserveFPSign = 2, // sign when flushed-to-zero is preserved // Tag_ABI_FP_number_model, (=23), uleb128 @@ -192,6 +200,9 @@ enum { // Tag_FP_HP_extension, (=36), uleb128 AllowHPFP = 1, // Allow use of Half Precision FP + // Tag_FP_16bit_format, (=38), uleb128 + FP16FormatIEEE = 1, + // Tag_MPextension_use, (=42), uleb128 AllowMP = 1, // Allow use of MP extensions @@ -214,4 +225,4 @@ enum { } // namespace ARMBuildAttrs } // namespace llvm -#endif // LLVM_SUPPORT_ARM_BUILD_ATTRIBUTES_H +#endif diff --git a/contrib/llvm/include/llvm/Support/ARMEHABI.h b/contrib/llvm/include/llvm/Support/ARMEHABI.h index c7ac54aeb6de..9b052df0a908 100644 --- a/contrib/llvm/include/llvm/Support/ARMEHABI.h +++ b/contrib/llvm/include/llvm/Support/ARMEHABI.h @@ -19,8 +19,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_ARM_EHABI_H -#define LLVM_SUPPORT_ARM_EHABI_H +#ifndef LLVM_SUPPORT_ARMEHABI_H +#define LLVM_SUPPORT_ARMEHABI_H namespace llvm { namespace ARM { @@ -131,4 +131,4 @@ namespace EHABI { } } -#endif // ARM_UNWIND_OP_H +#endif diff --git a/contrib/llvm/include/llvm/Support/ARMWinEH.h b/contrib/llvm/include/llvm/Support/ARMWinEH.h index 78deb8d36a98..1463629f45dc 100644 --- a/contrib/llvm/include/llvm/Support/ARMWinEH.h +++ b/contrib/llvm/include/llvm/Support/ARMWinEH.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_WINARMEH_H -#define LLVM_SUPPORT_WINARMEH_H +#ifndef LLVM_SUPPORT_ARMWINEH_H +#define LLVM_SUPPORT_ARMWINEH_H #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/Endian.h" @@ -350,16 +350,15 @@ struct ExceptionDataRecord { ArrayRef EpilogueScopes() const { assert(E() == 0 && "epilogue scopes are only present when the E bit is 0"); size_t Offset = HeaderWords(*this); - return ArrayRef(&Data[Offset], EpilogueCount()); + return makeArrayRef(&Data[Offset], EpilogueCount()); } - ArrayRef UnwindByteCode() const { + ArrayRef UnwindByteCode() const { const size_t Offset = HeaderWords(*this) + (E() ? 0 : EpilogueCount()); - const support::ulittle8_t *ByteCode = - reinterpret_cast(&Data[Offset]); - return ArrayRef(ByteCode, - CodeWords() * sizeof(uint32_t)); + const uint8_t *ByteCode = + reinterpret_cast(&Data[Offset]); + return makeArrayRef(ByteCode, CodeWords() * sizeof(uint32_t)); } uint32_t ExceptionHandlerRVA() const { @@ -381,4 +380,3 @@ inline size_t HeaderWords(const ExceptionDataRecord &XR) { } #endif - diff --git a/contrib/llvm/include/llvm/Support/Allocator.h b/contrib/llvm/include/llvm/Support/Allocator.h index 7a7e4c0a13e2..de317719714d 100644 --- a/contrib/llvm/include/llvm/Support/Allocator.h +++ b/contrib/llvm/include/llvm/Support/Allocator.h @@ -90,7 +90,10 @@ class MallocAllocator : public AllocatorBase { public: void Reset() {} - void *Allocate(size_t Size, size_t /*Alignment*/) { return malloc(Size); } + LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t Size, + size_t /*Alignment*/) { + return malloc(Size); + } // Pull in base class overloads. using AllocatorBase::Allocate; @@ -116,8 +119,8 @@ void printBumpPtrAllocatorStats(unsigned NumSlabs, size_t BytesAllocated, /// \brief Allocate memory in an ever growing pool, as if by bump-pointer. /// /// This isn't strictly a bump-pointer allocator as it uses backing slabs of -/// memory rather than relying on boundless contiguous heap. However, it has -/// bump-pointer semantics in that is a monotonically growing pool of memory +/// memory rather than relying on a boundless contiguous heap. However, it has +/// bump-pointer semantics in that it is a monotonically growing pool of memory /// where every allocation is found by merely allocating the next N bytes in /// the slab, or the next N bytes in the next slab. /// @@ -200,28 +203,24 @@ public: } /// \brief Allocate space at the specified alignment. - void *Allocate(size_t Size, size_t Alignment) { - if (!CurPtr) // Start a new slab if we haven't allocated one already. - StartNewSlab(); + LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t Size, size_t Alignment) { + assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead."); // Keep track of how many bytes we've allocated. BytesAllocated += Size; - // 0-byte alignment means 1-byte alignment. - if (Alignment == 0) - Alignment = 1; + size_t Adjustment = alignmentAdjustment(CurPtr, Alignment); + assert(Adjustment + Size >= Size && "Adjustment + Size must not overflow"); - // Allocate the aligned space, going forwards from CurPtr. - char *Ptr = alignPtr(CurPtr, Alignment); - - // Check if we can hold it. - if (Ptr + Size <= End) { - CurPtr = Ptr + Size; + // Check if we have enough space. + if (Adjustment + Size <= size_t(End - CurPtr)) { + char *AlignedPtr = CurPtr + Adjustment; + CurPtr = AlignedPtr + Size; // Update the allocation point of this memory block in MemorySanitizer. // Without this, MemorySanitizer messages for values originated from here // will point to the allocation of the entire slab. - __msan_allocated_memory(Ptr, Size); - return Ptr; + __msan_allocated_memory(AlignedPtr, Size); + return AlignedPtr; } // If Size is really big, allocate a separate slab for it. @@ -230,19 +229,22 @@ public: void *NewSlab = Allocator.Allocate(PaddedSize, 0); CustomSizedSlabs.push_back(std::make_pair(NewSlab, PaddedSize)); - Ptr = alignPtr((char *)NewSlab, Alignment); - assert((uintptr_t)Ptr + Size <= (uintptr_t)NewSlab + PaddedSize); - __msan_allocated_memory(Ptr, Size); - return Ptr; + uintptr_t AlignedAddr = alignAddr(NewSlab, Alignment); + assert(AlignedAddr + Size <= (uintptr_t)NewSlab + PaddedSize); + char *AlignedPtr = (char*)AlignedAddr; + __msan_allocated_memory(AlignedPtr, Size); + return AlignedPtr; } // Otherwise, start a new slab and try again. StartNewSlab(); - Ptr = alignPtr(CurPtr, Alignment); - CurPtr = Ptr + Size; - assert(CurPtr <= End && "Unable to allocate memory!"); - __msan_allocated_memory(Ptr, Size); - return Ptr; + uintptr_t AlignedAddr = alignAddr(CurPtr, Alignment); + assert(AlignedAddr + Size <= (uintptr_t)End && + "Unable to allocate memory!"); + char *AlignedPtr = (char*)AlignedAddr; + CurPtr = AlignedPtr + Size; + __msan_allocated_memory(AlignedPtr, Size); + return AlignedPtr; } // Pull in base class overloads. @@ -320,8 +322,10 @@ private: #ifndef NDEBUG // Poison the memory so stale pointers crash sooner. Note we must // preserve the Size and NextPtr fields at the beginning. - sys::Memory::setRangeWritable(*I, AllocatedSlabSize); - memset(*I, 0xCD, AllocatedSlabSize); + if (AllocatedSlabSize != 0) { + sys::Memory::setRangeWritable(*I, AllocatedSlabSize); + memset(*I, 0xCD, AllocatedSlabSize); + } #endif Allocator.Deallocate(*I, AllocatedSlabSize); } @@ -373,7 +377,7 @@ public: /// all memory allocated so far. void DestroyAll() { auto DestroyElements = [](char *Begin, char *End) { - assert(Begin == alignPtr(Begin, alignOf())); + assert(Begin == (char*)alignAddr(Begin, alignOf())); for (char *Ptr = Begin; Ptr + sizeof(T) <= End; Ptr += sizeof(T)) reinterpret_cast(Ptr)->~T(); }; @@ -382,7 +386,7 @@ public: ++I) { size_t AllocatedSlabSize = BumpPtrAllocator::computeSlabSize( std::distance(Allocator.Slabs.begin(), I)); - char *Begin = alignPtr((char *)*I, alignOf()); + char *Begin = (char*)alignAddr(*I, alignOf()); char *End = *I == Allocator.Slabs.back() ? Allocator.CurPtr : (char *)*I + AllocatedSlabSize; @@ -392,7 +396,7 @@ public: for (auto &PtrAndSize : Allocator.CustomSizedSlabs) { void *Ptr = PtrAndSize.first; size_t Size = PtrAndSize.second; - DestroyElements(alignPtr((char *)Ptr, alignOf()), (char *)Ptr + Size); + DestroyElements((char*)alignAddr(Ptr, alignOf()), (char *)Ptr + Size); } Allocator.Reset(); diff --git a/contrib/llvm/include/llvm/Support/CBindingWrapping.h b/contrib/llvm/include/llvm/Support/CBindingWrapping.h index 51097b820220..786ba183b3b0 100644 --- a/contrib/llvm/include/llvm/Support/CBindingWrapping.h +++ b/contrib/llvm/include/llvm/Support/CBindingWrapping.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_C_BINDING_WRAPPING_H -#define LLVM_C_BINDING_WRAPPING_H +#ifndef LLVM_SUPPORT_CBINDINGWRAPPING_H +#define LLVM_SUPPORT_CBINDINGWRAPPING_H #include "llvm/Support/Casting.h" diff --git a/contrib/llvm/include/llvm/Support/COFF.h b/contrib/llvm/include/llvm/Support/COFF.h index e09ef07d81db..150bce50d9ae 100644 --- a/contrib/llvm/include/llvm/Support/COFF.h +++ b/contrib/llvm/include/llvm/Support/COFF.h @@ -31,23 +31,30 @@ namespace llvm { namespace COFF { // The maximum number of sections that a COFF object can have (inclusive). - const int MaxNumberOfSections = 65299; + const int32_t MaxNumberOfSections16 = 65279; // The PE signature bytes that follows the DOS stub header. static const char PEMagic[] = { 'P', 'E', '\0', '\0' }; + static const char BigObjMagic[] = { + '\xc7', '\xa1', '\xba', '\xd1', '\xee', '\xba', '\xa9', '\x4b', + '\xaf', '\x20', '\xfa', '\xf6', '\x6a', '\xa4', '\xdc', '\xb8', + }; + // Sizes in bytes of various things in the COFF format. enum { - HeaderSize = 20, + Header16Size = 20, + Header32Size = 56, NameSize = 8, - SymbolSize = 18, + Symbol16Size = 18, + Symbol32Size = 20, SectionSize = 40, RelocationSize = 10 }; struct header { uint16_t Machine; - uint16_t NumberOfSections; + int32_t NumberOfSections; uint32_t TimeDateStamp; uint32_t PointerToSymbolTable; uint32_t NumberOfSymbols; @@ -55,6 +62,24 @@ namespace COFF { uint16_t Characteristics; }; + struct BigObjHeader { + enum : uint16_t { MinBigObjectVersion = 2 }; + + uint16_t Sig1; ///< Must be IMAGE_FILE_MACHINE_UNKNOWN (0). + uint16_t Sig2; ///< Must be 0xFFFF. + uint16_t Version; + uint16_t Machine; + uint32_t TimeDateStamp; + uint8_t UUID[16]; + uint32_t unused1; + uint32_t unused2; + uint32_t unused3; + uint32_t unused4; + uint32_t NumberOfSections; + uint32_t PointerToSymbolTable; + uint32_t NumberOfSymbols; + }; + enum MachineTypes { MT_Invalid = 0xffff, @@ -124,7 +149,7 @@ namespace COFF { struct symbol { char Name[NameSize]; uint32_t Value; - uint16_t SectionNumber; + int32_t SectionNumber; uint16_t Type; uint8_t StorageClass; uint8_t NumberOfAuxSymbols; @@ -140,9 +165,9 @@ namespace COFF { SF_WeakExternal = 0x01000000 }; - enum SymbolSectionNumber { - IMAGE_SYM_DEBUG = 0xFFFE, - IMAGE_SYM_ABSOLUTE = 0xFFFF, + enum SymbolSectionNumber : int32_t { + IMAGE_SYM_DEBUG = -2, + IMAGE_SYM_ABSOLUTE = -1, IMAGE_SYM_UNDEFINED = 0 }; @@ -367,18 +392,14 @@ namespace COFF { IMAGE_WEAK_EXTERN_SEARCH_ALIAS = 3 }; - struct AuxiliaryFile { - uint8_t FileName[18]; - }; - struct AuxiliarySectionDefinition { uint32_t Length; uint16_t NumberOfRelocations; uint16_t NumberOfLinenumbers; uint32_t CheckSum; - uint16_t Number; + uint32_t Number; uint8_t Selection; - char unused[3]; + char unused; }; struct AuxiliaryCLRToken { @@ -392,7 +413,6 @@ namespace COFF { AuxiliaryFunctionDefinition FunctionDefinition; AuxiliarybfAndefSymbol bfAndefSymbol; AuxiliaryWeakExternal WeakExternal; - AuxiliaryFile File; AuxiliarySectionDefinition SectionDefinition; }; @@ -495,12 +515,14 @@ namespace COFF { uint32_t SizeOfHeaders; uint32_t CheckSum; uint16_t Subsystem; + // FIXME: This should be DllCharacteristics to match the COFF spec. uint16_t DLLCharacteristics; uint32_t SizeOfStackReserve; uint32_t SizeOfStackCommit; uint32_t SizeOfHeapReserve; uint32_t SizeOfHeapCommit; uint32_t LoaderFlags; + // FIXME: This should be NumberOfRvaAndSizes to match the COFF spec. uint32_t NumberOfRvaAndSize; }; @@ -524,7 +546,9 @@ namespace COFF { BOUND_IMPORT, IAT, DELAY_IMPORT_DESCRIPTOR, - CLR_RUNTIME_HEADER + CLR_RUNTIME_HEADER, + + NUM_DATA_DIRECTORIES }; enum WindowsSubsystem { @@ -642,13 +666,18 @@ namespace COFF { enum CodeViewLineTableIdentifiers { DEBUG_SECTION_MAGIC = 0x4, + DEBUG_SYMBOL_SUBSECTION = 0xF1, DEBUG_LINE_TABLE_SUBSECTION = 0xF2, DEBUG_STRING_TABLE_SUBSECTION = 0xF3, - DEBUG_INDEX_SUBSECTION = 0xF4 + DEBUG_INDEX_SUBSECTION = 0xF4, + + // Symbol subsections are split into records of different types. + DEBUG_SYMBOL_TYPE_PROC_START = 0x1147, + DEBUG_SYMBOL_TYPE_PROC_END = 0x114F }; - inline bool isReservedSectionNumber(int N) { - return N == IMAGE_SYM_UNDEFINED || N > MaxNumberOfSections; + inline bool isReservedSectionNumber(int32_t SectionNumber) { + return SectionNumber <= 0; } } // End namespace COFF. diff --git a/contrib/llvm/include/llvm/Support/Casting.h b/contrib/llvm/include/llvm/Support/Casting.h index beed31a4084f..6ba5efa47554 100644 --- a/contrib/llvm/include/llvm/Support/Casting.h +++ b/contrib/llvm/include/llvm/Support/Casting.h @@ -242,6 +242,26 @@ inline typename cast_retty::ret_type cast(Y *Val) { // cast_or_null - Functionally identical to cast, except that a null value is // accepted. // +template +LLVM_ATTRIBUTE_UNUSED_RESULT inline typename std::enable_if< + !is_simple_type::value, typename cast_retty::ret_type>::type +cast_or_null(const Y &Val) { + if (!Val) + return nullptr; + assert(isa(Val) && "cast_or_null() argument of incompatible type!"); + return cast(Val); +} + +template +LLVM_ATTRIBUTE_UNUSED_RESULT inline typename std::enable_if< + !is_simple_type::value, typename cast_retty::ret_type>::type +cast_or_null(Y &Val) { + if (!Val) + return nullptr; + assert(isa(Val) && "cast_or_null() argument of incompatible type!"); + return cast(Val); +} + template LLVM_ATTRIBUTE_UNUSED_RESULT inline typename cast_retty::ret_type cast_or_null(Y *Val) { @@ -281,6 +301,20 @@ dyn_cast(Y *Val) { // dyn_cast_or_null - Functionally identical to dyn_cast, except that a null // value is accepted. // +template +LLVM_ATTRIBUTE_UNUSED_RESULT inline typename std::enable_if< + !is_simple_type::value, typename cast_retty::ret_type>::type +dyn_cast_or_null(const Y &Val) { + return (Val && isa(Val)) ? cast(Val) : nullptr; +} + +template +LLVM_ATTRIBUTE_UNUSED_RESULT inline typename std::enable_if< + !is_simple_type::value, typename cast_retty::ret_type>::type +dyn_cast_or_null(Y &Val) { + return (Val && isa(Val)) ? cast(Val) : nullptr; +} + template LLVM_ATTRIBUTE_UNUSED_RESULT inline typename cast_retty::ret_type dyn_cast_or_null(Y *Val) { diff --git a/contrib/llvm/include/llvm/Support/CommandLine.h b/contrib/llvm/include/llvm/Support/CommandLine.h index fdd901200fe9..1c06bf5f8c07 100644 --- a/contrib/llvm/include/llvm/Support/CommandLine.h +++ b/contrib/llvm/include/llvm/Support/CommandLine.h @@ -40,7 +40,7 @@ namespace cl { //===----------------------------------------------------------------------===// // ParseCommandLineOptions - Command line option processing entry point. // -void ParseCommandLineOptions(int argc, const char * const *argv, +void ParseCommandLineOptions(int argc, const char *const *argv, const char *Overview = nullptr); //===----------------------------------------------------------------------===// @@ -66,7 +66,6 @@ void SetVersionPrinter(void (*func)()); /// information specific to the tool. void AddExtraVersionPrinter(void (*func)()); - // PrintOptionValues - Print option values. // With -print-options print the difference between option values and defaults. // With -print-all-options print all option values. @@ -80,11 +79,11 @@ void MarkOptionsChanged(); // Flags permitted to be passed to command line arguments // -enum NumOccurrencesFlag { // Flags for the number of occurrences allowed - Optional = 0x00, // Zero or One occurrence - ZeroOrMore = 0x01, // Zero or more occurrences allowed - Required = 0x02, // One occurrence required - OneOrMore = 0x03, // One or more occurrences required +enum NumOccurrencesFlag { // Flags for the number of occurrences allowed + Optional = 0x00, // Zero or One occurrence + ZeroOrMore = 0x01, // Zero or more occurrences allowed + Required = 0x02, // One occurrence required + OneOrMore = 0x03, // One or more occurrences required // ConsumeAfter - Indicates that this option is fed anything that follows the // last positional argument required by the application (it is an error if @@ -93,20 +92,20 @@ enum NumOccurrencesFlag { // Flags for the number of occurrences allowed // found. Once a filename is found, all of the succeeding arguments are // passed, unprocessed, to the ConsumeAfter option. // - ConsumeAfter = 0x04 + ConsumeAfter = 0x04 }; -enum ValueExpected { // Is a value required for the option? +enum ValueExpected { // Is a value required for the option? // zero reserved for the unspecified value - ValueOptional = 0x01, // The value can appear... or not - ValueRequired = 0x02, // The value is required to appear! - ValueDisallowed = 0x03 // A value may not be specified (for flags) + ValueOptional = 0x01, // The value can appear... or not + ValueRequired = 0x02, // The value is required to appear! + ValueDisallowed = 0x03 // A value may not be specified (for flags) }; -enum OptionHidden { // Control whether -help shows this option - NotHidden = 0x00, // Option included in -help & -help-hidden - Hidden = 0x01, // -help doesn't, but -help-hidden does - ReallyHidden = 0x02 // Neither -help nor -help-hidden show this arg +enum OptionHidden { // Control whether -help shows this option + NotHidden = 0x00, // Option included in -help & -help-hidden + Hidden = 0x01, // -help doesn't, but -help-hidden does + ReallyHidden = 0x02 // Neither -help nor -help-hidden show this arg }; // Formatting flags - This controls special features that the option might have @@ -125,16 +124,16 @@ enum OptionHidden { // Control whether -help shows this option // enum FormattingFlags { - NormalFormatting = 0x00, // Nothing special - Positional = 0x01, // Is a positional argument, no '-' required - Prefix = 0x02, // Can this option directly prefix its value? - Grouping = 0x03 // Can this option group with other options? + NormalFormatting = 0x00, // Nothing special + Positional = 0x01, // Is a positional argument, no '-' required + Prefix = 0x02, // Can this option directly prefix its value? + Grouping = 0x03 // Can this option group with other options? }; -enum MiscFlags { // Miscellaneous flags to adjust argument - CommaSeparated = 0x01, // Should this cl::list split between commas? - PositionalEatsArgs = 0x02, // Should this positional cl::list eat -args? - Sink = 0x04 // Should this cl::list eat all unknown options? +enum MiscFlags { // Miscellaneous flags to adjust argument + CommaSeparated = 0x01, // Should this cl::list split between commas? + PositionalEatsArgs = 0x02, // Should this positional cl::list eat -args? + Sink = 0x04 // Should this cl::list eat all unknown options? }; //===----------------------------------------------------------------------===// @@ -145,9 +144,13 @@ private: const char *const Name; const char *const Description; void registerCategory(); + public: - OptionCategory(const char *const Name, const char *const Description = nullptr) - : Name(Name), Description(Description) { registerCategory(); } + OptionCategory(const char *const Name, + const char *const Description = nullptr) + : Name(Name), Description(Description) { + registerCategory(); + } const char *getName() const { return Name; } const char *getDescription() const { return Description; } }; @@ -176,7 +179,7 @@ class Option { // Out of line virtual function to provide home for the class. virtual void anchor(); - int NumOccurrences; // The number of times specified + int NumOccurrences; // The number of times specified // Occurrences, HiddenFlag, and Formatting are all enum types but to avoid // problems with signed enums in bitfields. unsigned Occurrences : 3; // enum NumOccurrencesFlag @@ -186,9 +189,9 @@ class Option { unsigned HiddenFlag : 2; // enum OptionHidden unsigned Formatting : 2; // enum FormattingFlags unsigned Misc : 3; - unsigned Position; // Position of last occurrence of the option - unsigned AdditionalVals;// Greater than 0 for multi-valued option. - Option *NextRegistered; // Singly linked list of registered options. + unsigned Position; // Position of last occurrence of the option + unsigned AdditionalVals; // Greater than 0 for multi-valued option. + Option *NextRegistered; // Singly linked list of registered options. public: const char *ArgStr; // The argument string itself (ex: "help", "o") @@ -200,8 +203,7 @@ public: return (enum NumOccurrencesFlag)Occurrences; } inline enum ValueExpected getValueExpectedFlag() const { - return Value ? ((enum ValueExpected)Value) - : getValueExpectedFlagDefault(); + return Value ? ((enum ValueExpected)Value) : getValueExpectedFlagDefault(); } inline enum OptionHidden getOptionHiddenFlag() const { return (enum OptionHidden)HiddenFlag; @@ -209,9 +211,7 @@ public: inline enum FormattingFlags getFormattingFlag() const { return (enum FormattingFlags)Formatting; } - inline unsigned getMiscFlags() const { - return Misc; - } + inline unsigned getMiscFlags() const { return Misc; } inline unsigned getPosition() const { return Position; } inline unsigned getNumAdditionalVals() const { return AdditionalVals; } @@ -224,25 +224,24 @@ public: void setArgStr(const char *S) { ArgStr = S; } void setDescription(const char *S) { HelpStr = S; } void setValueStr(const char *S) { ValueStr = S; } - void setNumOccurrencesFlag(enum NumOccurrencesFlag Val) { - Occurrences = Val; - } + void setNumOccurrencesFlag(enum NumOccurrencesFlag Val) { Occurrences = Val; } void setValueExpectedFlag(enum ValueExpected Val) { Value = Val; } void setHiddenFlag(enum OptionHidden Val) { HiddenFlag = Val; } void setFormattingFlag(enum FormattingFlags V) { Formatting = V; } void setMiscFlag(enum MiscFlags M) { Misc |= M; } void setPosition(unsigned pos) { Position = pos; } void setCategory(OptionCategory &C) { Category = &C; } + protected: explicit Option(enum NumOccurrencesFlag OccurrencesFlag, enum OptionHidden Hidden) - : NumOccurrences(0), Occurrences(OccurrencesFlag), Value(0), - HiddenFlag(Hidden), Formatting(NormalFormatting), Misc(0), - Position(0), AdditionalVals(0), NextRegistered(nullptr), - ArgStr(""), HelpStr(""), ValueStr(""), Category(&GeneralCategory) { - } + : NumOccurrences(0), Occurrences(OccurrencesFlag), Value(0), + HiddenFlag(Hidden), Formatting(NormalFormatting), Misc(0), Position(0), + AdditionalVals(0), NextRegistered(nullptr), ArgStr(""), HelpStr(""), + ValueStr(""), Category(&GeneralCategory) {} inline void setNumAdditionalVals(unsigned n) { AdditionalVals = n; } + public: // addArgument - Register this argument with the commandline system. // @@ -266,12 +265,12 @@ public: virtual void printOptionValue(size_t GlobalWidth, bool Force) const = 0; - virtual void getExtraOptionNames(SmallVectorImpl &) {} + virtual void getExtraOptionNames(SmallVectorImpl &) {} // addOccurrence - Wrapper around handleOccurrence that enforces Flags. // - virtual bool addOccurrence(unsigned pos, StringRef ArgName, - StringRef Value, bool MultiArg = false); + virtual bool addOccurrence(unsigned pos, StringRef ArgName, StringRef Value, + bool MultiArg = false); // Prints option name followed by message. Always returns true. bool error(const Twine &Message, StringRef ArgName = StringRef()); @@ -281,7 +280,6 @@ public: virtual ~Option() {} }; - //===----------------------------------------------------------------------===// // Command line option modifiers that can be used to modify the behavior of // command line option parsers... @@ -306,36 +304,31 @@ struct value_desc { // the default constructor for the argument type does not give you what you // want. This is only valid on "opt" arguments, not on "list" arguments. // -template -struct initializer { +template struct initializer { const Ty &Init; initializer(const Ty &Val) : Init(Val) {} - template - void apply(Opt &O) const { O.setInitialValue(Init); } + template void apply(Opt &O) const { O.setInitialValue(Init); } }; -template -initializer init(const Ty &Val) { +template initializer init(const Ty &Val) { return initializer(Val); } - // location - Allow the user to specify which external variable they want to // store the results of the command line argument processing into, if they don't // want to store it in the option itself. // -template -struct LocationClass { +template struct LocationClass { Ty &Loc; LocationClass(Ty &L) : Loc(L) {} - template - void apply(Opt &O) const { O.setLocation(O, Loc); } + template void apply(Opt &O) const { O.setLocation(O, Loc); } }; -template -LocationClass location(Ty &L) { return LocationClass(L); } +template LocationClass location(Ty &L) { + return LocationClass(L); +} // cat - Specifiy the Option category for the command line argument to belong // to. @@ -343,11 +336,9 @@ struct cat { OptionCategory &Category; cat(OptionCategory &c) : Category(c) {} - template - void apply(Opt &O) const { O.setCategory(Category); } + template void apply(Opt &O) const { O.setCategory(Category); } }; - //===----------------------------------------------------------------------===// // OptionValue class @@ -360,11 +351,11 @@ private: virtual void anchor(); }; -template struct OptionValue; +template struct OptionValue; // The default value safely does nothing. Option value printing is only // best-effort. -template +template struct OptionValueBase : public GenericOptionValue { // Temporary storage for argument passing. typedef OptionValue WrapperType; @@ -374,21 +365,20 @@ struct OptionValueBase : public GenericOptionValue { const DataType &getValue() const { llvm_unreachable("no default value"); } // Some options may take their value from a different data type. - template - void setValue(const DT& /*V*/) {} + template void setValue(const DT & /*V*/) {} - bool compare(const DataType &/*V*/) const { return false; } + bool compare(const DataType & /*V*/) const { return false; } - bool compare(const GenericOptionValue& /*V*/) const override { + bool compare(const GenericOptionValue & /*V*/) const override { return false; } }; // Simple copy of the option value. -template -class OptionValueCopy : public GenericOptionValue { +template class OptionValueCopy : public GenericOptionValue { DataType Value; bool Valid; + public: OptionValueCopy() : Valid(false) {} @@ -399,37 +389,36 @@ public: return Value; } - void setValue(const DataType &V) { Valid = true; Value = V; } - - bool compare(const DataType &V) const { - return Valid && (Value != V); + void setValue(const DataType &V) { + Valid = true; + Value = V; } + bool compare(const DataType &V) const { return Valid && (Value != V); } + bool compare(const GenericOptionValue &V) const override { const OptionValueCopy &VC = - static_cast< const OptionValueCopy& >(V); - if (!VC.hasValue()) return false; + static_cast &>(V); + if (!VC.hasValue()) + return false; return compare(VC.getValue()); } }; // Non-class option values. -template +template struct OptionValueBase : OptionValueCopy { typedef DataType WrapperType; }; // Top-level option class. -template +template struct OptionValue : OptionValueBase::value> { OptionValue() {} - OptionValue(const DataType& V) { - this->setValue(V); - } + OptionValue(const DataType &V) { this->setValue(V); } // Some options may take their value from a different data type. - template - OptionValue &operator=(const DT& V) { + template OptionValue &operator=(const DT &V) { this->setValue(V); return *this; } @@ -437,36 +426,33 @@ struct OptionValue : OptionValueBase::value> { // Other safe-to-copy-by-value common option types. enum boolOrDefault { BOU_UNSET, BOU_TRUE, BOU_FALSE }; -template<> +template <> struct OptionValue : OptionValueCopy { typedef cl::boolOrDefault WrapperType; OptionValue() {} - OptionValue(const cl::boolOrDefault& V) { - this->setValue(V); - } - OptionValue &operator=(const cl::boolOrDefault& V) { + OptionValue(const cl::boolOrDefault &V) { this->setValue(V); } + OptionValue &operator=(const cl::boolOrDefault &V) { setValue(V); return *this; } + private: void anchor() override; }; -template<> -struct OptionValue : OptionValueCopy { +template <> struct OptionValue : OptionValueCopy { typedef StringRef WrapperType; OptionValue() {} - OptionValue(const std::string& V) { - this->setValue(V); - } - OptionValue &operator=(const std::string& V) { + OptionValue(const std::string &V) { this->setValue(V); } + OptionValue &operator=(const std::string &V) { setValue(V); return *this; } + private: void anchor() override; }; @@ -476,20 +462,20 @@ private: // #define clEnumVal(ENUMVAL, DESC) #ENUMVAL, int(ENUMVAL), DESC #define clEnumValN(ENUMVAL, FLAGNAME, DESC) FLAGNAME, int(ENUMVAL), DESC -#define clEnumValEnd (reinterpret_cast(0)) +#define clEnumValEnd (reinterpret_cast(0)) // values - For custom data types, allow specifying a group of values together // as the values that go into the mapping that the option handler uses. Note // that the values list must always have a 0 at the end of the list to indicate // that the list has ended. // -template -class ValuesClass { +template class ValuesClass { // Use a vector instead of a map, because the lists should be short, // the overhead is less, and most importantly, it keeps them in the order // inserted so we can print our option out nicely. - SmallVector >,4> Values; + SmallVector>, 4> Values; void processValues(va_list Vals); + public: ValuesClass(const char *EnumName, DataType Val, const char *Desc, va_list ValueArgs) { @@ -500,27 +486,26 @@ public: while (const char *enumName = va_arg(ValueArgs, const char *)) { DataType EnumVal = static_cast(va_arg(ValueArgs, int)); const char *EnumDesc = va_arg(ValueArgs, const char *); - Values.push_back(std::make_pair(enumName, // Add value to value map + Values.push_back(std::make_pair(enumName, // Add value to value map std::make_pair(EnumVal, EnumDesc))); } } - template - void apply(Opt &O) const { + template void apply(Opt &O) const { for (size_t i = 0, e = Values.size(); i != e; ++i) O.getParser().addLiteralOption(Values[i].first, Values[i].second.first, Values[i].second.second); } }; -template -ValuesClass END_WITH_NULL values(const char *Arg, DataType Val, - const char *Desc, ...) { - va_list ValueArgs; - va_start(ValueArgs, Desc); - ValuesClass Vals(Arg, Val, Desc, ValueArgs); - va_end(ValueArgs); - return Vals; +template +ValuesClass LLVM_END_WITH_NULL +values(const char *Arg, DataType Val, const char *Desc, ...) { + va_list ValueArgs; + va_start(ValueArgs, Desc); + ValuesClass Vals(Arg, Val, Desc, ValueArgs); + va_end(ValueArgs); + return Vals; } //===----------------------------------------------------------------------===// @@ -539,13 +524,14 @@ class generic_parser_base { protected: class GenericOptionInfo { public: - GenericOptionInfo(const char *name, const char *helpStr) : - Name(name), HelpStr(helpStr) {} + GenericOptionInfo(const char *name, const char *helpStr) + : Name(name), HelpStr(helpStr) {} const char *Name; const char *HelpStr; }; + public: - virtual ~generic_parser_base() {} // Base class should have virtual-dtor + virtual ~generic_parser_base() {} // Base class should have virtual-dtor // getNumOptions - Virtual function implemented by generic subclass to // indicate how many entries are in Values. @@ -576,7 +562,7 @@ public: // // Template definition ensures that the option and default have the same // DataType (via the same AnyOptionValue). - template + template void printOptionDiff(const Option &O, const AnyOptionValue &V, const AnyOptionValue &Default, size_t GlobalWidth) const { @@ -590,7 +576,7 @@ public: hasArgStr = O.hasArgStr(); } - void getExtraOptionNames(SmallVectorImpl &OptionNames) { + void getExtraOptionNames(SmallVectorImpl &OptionNames) { // If there has been no argstr specified, that means that we need to add an // argument for every possible option. This ensures that our options are // vectored to us. @@ -599,7 +585,6 @@ public: OptionNames.push_back(getOption(i)); } - enum ValueExpected getValueExpectedFlagDefault() const { // If there is an ArgStr specified, then we are of the form: // @@ -633,16 +618,16 @@ protected: // command line option for -help. Because this is a simple mapping parser, the // data type can be any unsupported type. // -template -class parser : public generic_parser_base { +template class parser : public generic_parser_base { protected: class OptionInfo : public GenericOptionInfo { public: - OptionInfo(const char *name, DataType v, const char *helpStr) : - GenericOptionInfo(name, helpStr), V(v) {} + OptionInfo(const char *name, DataType v, const char *helpStr) + : GenericOptionInfo(name, helpStr), V(v) {} OptionValue V; }; SmallVector Values; + public: typedef DataType parser_data_type; @@ -690,14 +675,14 @@ public: void removeLiteralOption(const char *Name) { unsigned N = findOption(Name); assert(N != Values.size() && "Option not found!"); - Values.erase(Values.begin()+N); + Values.erase(Values.begin() + N); } }; //-------------------------------------------------- // basic_parser - Super class of parsers to provide boilerplate code // -class basic_parser_impl { // non-template implementation of basic_parser +class basic_parser_impl { // non-template implementation of basic_parser public: virtual ~basic_parser_impl() {} @@ -705,7 +690,7 @@ public: return ValueRequired; } - void getExtraOptionNames(SmallVectorImpl &) {} + void getExtraOptionNames(SmallVectorImpl &) {} void initialize(Option &) {} @@ -735,8 +720,7 @@ protected: // basic_parser - The real basic parser is just a template wrapper that provides // a typedef for the provided data type. // -template -class basic_parser : public basic_parser_impl { +template class basic_parser : public basic_parser_impl { public: typedef DataType parser_data_type; typedef OptionValue OptVal; @@ -745,18 +729,14 @@ public: //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { const char *ArgStr; -public: +public: // parse - Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, bool &Val); - template - void initialize(Opt &O) { - ArgStr = O.ArgStr; - } + template void initialize(Opt &O) { ArgStr = O.ArgStr; } enum ValueExpected getValueExpectedFlagDefault() const { return ValueOptional; @@ -776,8 +756,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, boolOrDefault &Val); @@ -801,8 +780,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, int &Val); @@ -819,12 +797,10 @@ public: EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); - //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, unsigned &Val); @@ -844,7 +820,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser // -template<> +template <> class parser : public basic_parser { public: // parse - Return true on error. @@ -866,8 +842,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, double &Val); @@ -887,8 +862,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &O, StringRef ArgName, StringRef Arg, float &Val); @@ -908,8 +882,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &, StringRef, StringRef Arg, std::string &Value) { @@ -932,8 +905,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); //-------------------------------------------------- // parser // -template<> -class parser : public basic_parser { +template <> class parser : public basic_parser { public: // parse - Return true on error. bool parse(Option &, StringRef, StringRef Arg, char &Value) { @@ -960,7 +932,7 @@ EXTERN_TEMPLATE_INSTANTIATION(class basic_parser); // parser to handle all the template nastiness. // This overloaded function is selected by the generic parser. -template +template void printOptionDiff(const Option &O, const generic_parser_base &P, const DT &V, const OptionValue
&Default, size_t GlobalWidth) { OptionValue
OV = V; @@ -969,18 +941,16 @@ void printOptionDiff(const Option &O, const generic_parser_base &P, const DT &V, // This is instantiated for basic parsers when the parsed value has a different // type than the option value. e.g. HelpPrinter. -template -struct OptionDiffPrinter { - void print(const Option &O, const parser P, const ValDT &/*V*/, - const OptionValue &/*Default*/, size_t GlobalWidth) { +template struct OptionDiffPrinter { + void print(const Option &O, const parser P, const ValDT & /*V*/, + const OptionValue & /*Default*/, size_t GlobalWidth) { P.printOptionNoValue(O, GlobalWidth); } }; // This is instantiated for basic parsers when the parsed value has the same // type as the option value. -template -struct OptionDiffPrinter { +template struct OptionDiffPrinter { void print(const Option &O, const parser
P, const DT &V, const OptionValue
&Default, size_t GlobalWidth) { P.printOptionDiff(O, V, Default, GlobalWidth); @@ -989,15 +959,14 @@ struct OptionDiffPrinter { // This overloaded function is selected by the basic parser, which may parse a // different type than the option type. -template +template void printOptionDiff( - const Option &O, - const basic_parser &P, - const ValDT &V, const OptionValue &Default, - size_t GlobalWidth) { + const Option &O, + const basic_parser &P, + const ValDT &V, const OptionValue &Default, size_t GlobalWidth) { OptionDiffPrinter printer; - printer.print(O, static_cast(P), V, Default, + printer.print(O, static_cast(P), V, Default, GlobalWidth); } @@ -1007,46 +976,47 @@ void printOptionDiff( // not correctly respond to the apply method). Because the syntax to use this // is a pain, we have the 'apply' method below to handle the nastiness... // -template struct applicator { - template - static void opt(const Mod &M, Opt &O) { M.apply(O); } +template struct applicator { + template static void opt(const Mod &M, Opt &O) { M.apply(O); } }; // Handle const char* as a special case... -template struct applicator { - template - static void opt(const char *Str, Opt &O) { O.setArgStr(Str); } +template struct applicator { + template static void opt(const char *Str, Opt &O) { + O.setArgStr(Str); + } }; -template struct applicator { - template - static void opt(const char *Str, Opt &O) { O.setArgStr(Str); } +template struct applicator { + template static void opt(const char *Str, Opt &O) { + O.setArgStr(Str); + } }; -template<> struct applicator { - template - static void opt(const char *Str, Opt &O) { O.setArgStr(Str); } +template <> struct applicator { + template static void opt(const char *Str, Opt &O) { + O.setArgStr(Str); + } }; -template<> struct applicator { +template <> struct applicator { static void opt(NumOccurrencesFlag N, Option &O) { O.setNumOccurrencesFlag(N); } }; -template<> struct applicator { +template <> struct applicator { static void opt(ValueExpected VE, Option &O) { O.setValueExpectedFlag(VE); } }; -template<> struct applicator { +template <> struct applicator { static void opt(OptionHidden OH, Option &O) { O.setHiddenFlag(OH); } }; -template<> struct applicator { +template <> struct applicator { static void opt(FormattingFlags FF, Option &O) { O.setFormattingFlag(FF); } }; -template<> struct applicator { +template <> struct applicator { static void opt(MiscFlags MF, Option &O) { O.setMiscFlag(MF); } }; // apply method - Apply a modifier to an option in a type safe way. -template -void apply(const Mod &M, Opt *O) { +template void apply(const Mod &M, Opt *O) { applicator::opt(M, *O); } @@ -1057,16 +1027,17 @@ void apply(const Mod &M, Opt *O) { // assumes the user will specify a variable to store the data into with the // cl::location(x) modifier. // -template +template class opt_storage { - DataType *Location; // Where to store the object... + DataType *Location; // Where to store the object... OptionValue Default; void check_location() const { assert(Location && "cl::location(...) not specified for a command " - "line option with external storage, " - "or cl::init specified before cl::location()!!"); + "line option with external storage, " + "or cl::init specified before cl::location()!!"); } + public: opt_storage() : Location(nullptr) {} @@ -1078,16 +1049,21 @@ public: return false; } - template - void setValue(const T &V, bool initial = false) { + template void setValue(const T &V, bool initial = false) { check_location(); *Location = V; if (initial) Default = V; } - DataType &getValue() { check_location(); return *Location; } - const DataType &getValue() const { check_location(); return *Location; } + DataType &getValue() { + check_location(); + return *Location; + } + const DataType &getValue() const { + check_location(); + return *Location; + } operator DataType() const { return this->getValue(); } @@ -1098,13 +1074,12 @@ public: // inherit from a class, we do so. This makes us exactly compatible with the // object in all cases that it is used. // -template -class opt_storage : public DataType { +template +class opt_storage : public DataType { public: OptionValue Default; - template - void setValue(const T &V, bool initial = false) { + template void setValue(const T &V, bool initial = false) { DataType::operator=(V); if (initial) Default = V; @@ -1120,8 +1095,7 @@ public: // this case, we store an instance through containment, and overload operators // to get at the value. // -template -class opt_storage { +template class opt_storage { public: DataType Value; OptionValue Default; @@ -1130,8 +1104,7 @@ public: // type. opt_storage() : Value(DataType()), Default(DataType()) {} - template - void setValue(const T &V, bool initial = false) { + template void setValue(const T &V, bool initial = false) { Value = V; if (initial) Default = V; @@ -1147,12 +1120,11 @@ public: DataType operator->() const { return Value; } }; - //===----------------------------------------------------------------------===// // opt - A scalar command line option. // template > + class ParserClass = parser> class opt : public Option, public opt_storage::value> { @@ -1161,9 +1133,9 @@ class opt : public Option, bool handleOccurrence(unsigned pos, StringRef ArgName, StringRef Arg) override { typename ParserClass::parser_data_type Val = - typename ParserClass::parser_data_type(); + typename ParserClass::parser_data_type(); if (Parser.parse(*this, ArgName, Arg, Val)) - return true; // Parse error! + return true; // Parse error! this->setValue(Val); this->setPosition(pos); return false; @@ -1172,20 +1144,23 @@ class opt : public Option, enum ValueExpected getValueExpectedFlagDefault() const override { return Parser.getValueExpectedFlagDefault(); } - void getExtraOptionNames(SmallVectorImpl &OptionNames) override { + void + getExtraOptionNames(SmallVectorImpl &OptionNames) override { return Parser.getExtraOptionNames(OptionNames); } // Forward printing stuff to the parser... - size_t getOptionWidth() const override {return Parser.getOptionWidth(*this);} + size_t getOptionWidth() const override { + return Parser.getOptionWidth(*this); + } void printOptionInfo(size_t GlobalWidth) const override { Parser.printOptionInfo(*this, GlobalWidth); } void printOptionValue(size_t GlobalWidth, bool Force) const override { if (Force || this->getDefault().compare(this->getValue())) { - cl::printOptionDiff( - *this, Parser, this->getValue(), this->getDefault(), GlobalWidth); + cl::printOptionDiff(*this, Parser, this->getValue(), + this->getDefault(), GlobalWidth); } } @@ -1193,81 +1168,107 @@ class opt : public Option, addArgument(); Parser.initialize(*this); } + public: // setInitialValue - Used by the cl::init modifier... void setInitialValue(const DataType &V) { this->setValue(V, true); } ParserClass &getParser() { return Parser; } - template - DataType &operator=(const T &Val) { + template DataType &operator=(const T &Val) { this->setValue(Val); return this->getValue(); } // One option... - template - explicit opt(const M0t &M0) : Option(Optional, NotHidden) { + template + explicit opt(const M0t &M0) + : Option(Optional, NotHidden) { apply(M0, this); done(); } // Two options... - template - opt(const M0t &M0, const M1t &M1) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); + template + opt(const M0t &M0, const M1t &M1) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); done(); } // Three options... - template - opt(const M0t &M0, const M1t &M1, - const M2t &M2) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); + template + opt(const M0t &M0, const M1t &M1, const M2t &M2) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); done(); } // Four options... - template - opt(const M0t &M0, const M1t &M1, const M2t &M2, - const M3t &M3) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + template + opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); done(); } // Five options... - template - opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + template + opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, const M4t &M4) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); apply(M4, this); done(); } // Six options... - template - opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); + template + opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, const M4t &M4, + const M5t &M5) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); done(); } // Seven options... - template - opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5, - const M6t &M6) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); apply(M6, this); + template + opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, const M4t &M4, + const M5t &M5, const M6t &M6) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); + apply(M6, this); done(); } // Eight options... - template - opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5, const M6t &M6, - const M7t &M7) : Option(Optional, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); apply(M6, this); apply(M7, this); + template + opt(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, const M4t &M4, + const M5t &M5, const M6t &M6, const M7t &M7) + : Option(Optional, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); + apply(M6, this); + apply(M7, this); done(); } }; @@ -1285,9 +1286,8 @@ EXTERN_TEMPLATE_INSTANTIATION(class opt); // assumes the user will specify a variable to store the data into with the // cl::location(x) modifier. // -template -class list_storage { - StorageClass *Location; // Where to store the object... +template class list_storage { + StorageClass *Location; // Where to store the object... public: list_storage() : Location(0) {} @@ -1299,32 +1299,30 @@ public: return false; } - template - void addValue(const T &V) { + template void addValue(const T &V) { assert(Location != 0 && "cl::location(...) not specified for a command " - "line option with external storage!"); + "line option with external storage!"); Location->push_back(V); } }; - // Define how to hold a class type object, such as a string. Since we can // inherit from a class, we do so. This makes us exactly compatible with the // object in all cases that it is used. // -template +template class list_storage : public std::vector { public: - template - void addValue(const T &V) { std::vector::push_back(V); } + template void addValue(const T &V) { + std::vector::push_back(V); + } }; - //===----------------------------------------------------------------------===// // list - A list of command line options. // template > + class ParserClass = parser> class list : public Option, public list_storage { std::vector Positions; ParserClass Parser; @@ -1332,16 +1330,17 @@ class list : public Option, public list_storage { enum ValueExpected getValueExpectedFlagDefault() const override { return Parser.getValueExpectedFlagDefault(); } - void getExtraOptionNames(SmallVectorImpl &OptionNames) override { + void + getExtraOptionNames(SmallVectorImpl &OptionNames) override { return Parser.getExtraOptionNames(OptionNames); } bool handleOccurrence(unsigned pos, StringRef ArgName, StringRef Arg) override { typename ParserClass::parser_data_type Val = - typename ParserClass::parser_data_type(); + typename ParserClass::parser_data_type(); if (Parser.parse(*this, ArgName, Arg, Val)) - return true; // Parse Error! + return true; // Parse Error! list_storage::addValue(Val); setPosition(pos); Positions.push_back(pos); @@ -1349,19 +1348,22 @@ class list : public Option, public list_storage { } // Forward printing stuff to the parser... - size_t getOptionWidth() const override {return Parser.getOptionWidth(*this);} + size_t getOptionWidth() const override { + return Parser.getOptionWidth(*this); + } void printOptionInfo(size_t GlobalWidth) const override { Parser.printOptionInfo(*this, GlobalWidth); } // Unimplemented: list options don't currently store their default value. - void printOptionValue(size_t /*GlobalWidth*/, - bool /*Force*/) const override {} + void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { + } void done() { addArgument(); Parser.initialize(*this); } + public: ParserClass &getParser() { return Parser; } @@ -1370,71 +1372,96 @@ public: return Positions[optnum]; } - void setNumAdditionalVals(unsigned n) { - Option::setNumAdditionalVals(n); - } + void setNumAdditionalVals(unsigned n) { Option::setNumAdditionalVals(n); } // One option... - template - explicit list(const M0t &M0) : Option(ZeroOrMore, NotHidden) { + template + explicit list(const M0t &M0) + : Option(ZeroOrMore, NotHidden) { apply(M0, this); done(); } // Two options... - template - list(const M0t &M0, const M1t &M1) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); + template + list(const M0t &M0, const M1t &M1) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); done(); } // Three options... - template + template list(const M0t &M0, const M1t &M1, const M2t &M2) - : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); done(); } // Four options... - template + template list(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3) - : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); done(); } // Five options... - template + template list(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + const M4t &M4) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); apply(M4, this); done(); } // Six options... - template + template list(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); + const M4t &M4, const M5t &M5) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); done(); } // Seven options... - template + template list(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, const M4t &M4, const M5t &M5, const M6t &M6) - : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); apply(M6, this); + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); + apply(M6, this); done(); } // Eight options... - template + template list(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5, const M6t &M6, - const M7t &M7) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); apply(M6, this); apply(M7, this); + const M4t &M4, const M5t &M5, const M6t &M6, const M7t &M7) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); + apply(M6, this); + apply(M7, this); done(); } }; @@ -1445,10 +1472,11 @@ struct multi_val { explicit multi_val(unsigned N) : AdditionalVals(N) {} template - void apply(list &L) const { L.setNumAdditionalVals(AdditionalVals); } + void apply(list &L) const { + L.setNumAdditionalVals(AdditionalVals); + } }; - //===----------------------------------------------------------------------===// // bits_storage class @@ -1456,15 +1484,13 @@ struct multi_val { // assumes the user will specify a variable to store the data into with the // cl::location(x) modifier. // -template -class bits_storage { - unsigned *Location; // Where to store the bits... +template class bits_storage { + unsigned *Location; // Where to store the bits... - template - static unsigned Bit(const T &V) { + template static unsigned Bit(const T &V) { unsigned BitPos = reinterpret_cast(V); assert(BitPos < sizeof(unsigned) * CHAR_BIT && - "enum exceeds width of bit vector!"); + "enum exceeds width of bit vector!"); return 1 << BitPos; } @@ -1478,57 +1504,45 @@ public: return false; } - template - void addValue(const T &V) { + template void addValue(const T &V) { assert(Location != 0 && "cl::location(...) not specified for a command " - "line option with external storage!"); + "line option with external storage!"); *Location |= Bit(V); } unsigned getBits() { return *Location; } - template - bool isSet(const T &V) { + template bool isSet(const T &V) { return (*Location & Bit(V)) != 0; } }; - // Define how to hold bits. Since we can inherit from a class, we do so. // This makes us exactly compatible with the bits in all cases that it is used. // -template -class bits_storage { - unsigned Bits; // Where to store the bits... +template class bits_storage { + unsigned Bits; // Where to store the bits... - template - static unsigned Bit(const T &V) { + template static unsigned Bit(const T &V) { unsigned BitPos = (unsigned)V; assert(BitPos < sizeof(unsigned) * CHAR_BIT && - "enum exceeds width of bit vector!"); + "enum exceeds width of bit vector!"); return 1 << BitPos; } public: - template - void addValue(const T &V) { - Bits |= Bit(V); - } + template void addValue(const T &V) { Bits |= Bit(V); } unsigned getBits() { return Bits; } - template - bool isSet(const T &V) { - return (Bits & Bit(V)) != 0; - } + template bool isSet(const T &V) { return (Bits & Bit(V)) != 0; } }; - //===----------------------------------------------------------------------===// // bits - A bit vector of command options. // template > + class ParserClass = parser> class bits : public Option, public bits_storage { std::vector Positions; ParserClass Parser; @@ -1536,16 +1550,17 @@ class bits : public Option, public bits_storage { enum ValueExpected getValueExpectedFlagDefault() const override { return Parser.getValueExpectedFlagDefault(); } - void getExtraOptionNames(SmallVectorImpl &OptionNames) override { + void + getExtraOptionNames(SmallVectorImpl &OptionNames) override { return Parser.getExtraOptionNames(OptionNames); } bool handleOccurrence(unsigned pos, StringRef ArgName, StringRef Arg) override { typename ParserClass::parser_data_type Val = - typename ParserClass::parser_data_type(); + typename ParserClass::parser_data_type(); if (Parser.parse(*this, ArgName, Arg, Val)) - return true; // Parse Error! + return true; // Parse Error! this->addValue(Val); setPosition(pos); Positions.push_back(pos); @@ -1553,19 +1568,22 @@ class bits : public Option, public bits_storage { } // Forward printing stuff to the parser... - size_t getOptionWidth() const override {return Parser.getOptionWidth(*this);} + size_t getOptionWidth() const override { + return Parser.getOptionWidth(*this); + } void printOptionInfo(size_t GlobalWidth) const override { Parser.printOptionInfo(*this, GlobalWidth); } // Unimplemented: bits options don't currently store their default values. - void printOptionValue(size_t /*GlobalWidth*/, - bool /*Force*/) const override {} + void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { + } void done() { addArgument(); Parser.initialize(*this); } + public: ParserClass &getParser() { return Parser; } @@ -1575,66 +1593,93 @@ public: } // One option... - template - explicit bits(const M0t &M0) : Option(ZeroOrMore, NotHidden) { + template + explicit bits(const M0t &M0) + : Option(ZeroOrMore, NotHidden) { apply(M0, this); done(); } // Two options... - template - bits(const M0t &M0, const M1t &M1) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); + template + bits(const M0t &M0, const M1t &M1) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); done(); } // Three options... - template + template bits(const M0t &M0, const M1t &M1, const M2t &M2) - : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); done(); } // Four options... - template + template bits(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3) - : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); done(); } // Five options... - template + template bits(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + const M4t &M4) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); apply(M4, this); done(); } // Six options... - template + template bits(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); + const M4t &M4, const M5t &M5) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); done(); } // Seven options... - template + template bits(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, const M4t &M4, const M5t &M5, const M6t &M6) - : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); apply(M6, this); + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); + apply(M6, this); done(); } // Eight options... - template + template bits(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3, - const M4t &M4, const M5t &M5, const M6t &M6, - const M7t &M7) : Option(ZeroOrMore, NotHidden) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); - apply(M4, this); apply(M5, this); apply(M6, this); apply(M7, this); + const M4t &M4, const M5t &M5, const M6t &M6, const M7t &M7) + : Option(ZeroOrMore, NotHidden) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); + apply(M4, this); + apply(M5, this); + apply(M6, this); + apply(M7, this); done(); } }; @@ -1646,11 +1691,11 @@ public: class alias : public Option { Option *AliasFor; bool handleOccurrence(unsigned pos, StringRef /*ArgName*/, - StringRef Arg) override { + StringRef Arg) override { return AliasFor->handleOccurrence(pos, AliasFor->ArgStr, Arg); } - bool addOccurrence(unsigned pos, StringRef /*ArgName*/, - StringRef Value, bool MultiArg = false) override { + bool addOccurrence(unsigned pos, StringRef /*ArgName*/, StringRef Value, + bool MultiArg = false) override { return AliasFor->addOccurrence(pos, AliasFor->ArgStr, Value, MultiArg); } // Handle printing stuff... @@ -1658,8 +1703,8 @@ class alias : public Option { void printOptionInfo(size_t GlobalWidth) const override; // Aliases do not need to print their values. - void printOptionValue(size_t /*GlobalWidth*/, - bool /*Force*/) const override {} + void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { + } ValueExpected getValueExpectedFlagDefault() const override { return AliasFor->getValueExpectedFlag(); @@ -1670,8 +1715,9 @@ class alias : public Option { error("cl::alias must have argument name specified!"); if (!AliasFor) error("cl::alias must have an cl::aliasopt(option) specified!"); - addArgument(); + addArgument(); } + public: void setAliasFor(Option &O) { if (AliasFor) @@ -1680,30 +1726,37 @@ public: } // One option... - template - explicit alias(const M0t &M0) : Option(Optional, Hidden), AliasFor(nullptr) { + template + explicit alias(const M0t &M0) + : Option(Optional, Hidden), AliasFor(nullptr) { apply(M0, this); done(); } // Two options... - template + template alias(const M0t &M0, const M1t &M1) - : Option(Optional, Hidden), AliasFor(nullptr) { - apply(M0, this); apply(M1, this); + : Option(Optional, Hidden), AliasFor(nullptr) { + apply(M0, this); + apply(M1, this); done(); } // Three options... - template + template alias(const M0t &M0, const M1t &M1, const M2t &M2) - : Option(Optional, Hidden), AliasFor(nullptr) { - apply(M0, this); apply(M1, this); apply(M2, this); + : Option(Optional, Hidden), AliasFor(nullptr) { + apply(M0, this); + apply(M1, this); + apply(M2, this); done(); } // Four options... - template + template alias(const M0t &M0, const M1t &M1, const M2t &M2, const M3t &M3) - : Option(Optional, Hidden), AliasFor(nullptr) { - apply(M0, this); apply(M1, this); apply(M2, this); apply(M3, this); + : Option(Optional, Hidden), AliasFor(nullptr) { + apply(M0, this); + apply(M1, this); + apply(M2, this); + apply(M3, this); done(); } }; @@ -1720,8 +1773,8 @@ struct aliasopt { // printed to stderr at the end of the regular help, just before // exit is called. struct extrahelp { - const char * morehelp; - explicit extrahelp(const char* help); + const char *morehelp; + explicit extrahelp(const char *help); }; void PrintVersionMessage(); @@ -1733,8 +1786,7 @@ void PrintVersionMessage(); /// /// \param Hidden if true will print hidden options /// \param Categorized if true print options in categories -void PrintHelpMessage(bool Hidden=false, bool Categorized=false); - +void PrintHelpMessage(bool Hidden = false, bool Categorized = false); //===----------------------------------------------------------------------===// // Public interface for accessing registered options. @@ -1766,7 +1818,7 @@ void PrintHelpMessage(bool Hidden=false, bool Categorized=false); /// This interface is useful for modifying options in libraries that are out of /// the control of the client. The options should be modified before calling /// llvm::cl::ParseCommandLineOptions(). -void getRegisteredOptions(StringMap &Map); +void getRegisteredOptions(StringMap