diff --git a/contrib/compiler-rt/lib/asan/asan_errors.cc b/contrib/compiler-rt/lib/asan/asan_errors.cc index 57490ad180b5..b7a38eb7cece 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.cc +++ b/contrib/compiler-rt/lib/asan/asan_errors.cc @@ -61,10 +61,9 @@ static void MaybeDumpRegisters(void *context) { static void MaybeReportNonExecRegion(uptr pc) { #if SANITIZER_FREEBSD || SANITIZER_LINUX MemoryMappingLayout proc_maps(/*cache_enabled*/ true); - uptr start, end, protection; - while (proc_maps.Next(&start, &end, nullptr, nullptr, 0, &protection)) { - if (pc >= start && pc < end && - !(protection & MemoryMappingLayout::kProtectionExecute)) + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (pc >= segment.start && pc < segment.end && !segment.IsExecutable()) Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n"); } #endif diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h index 3b70695249e4..f09bbd83af25 100644 --- a/contrib/compiler-rt/lib/asan/asan_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_internal.h @@ -75,6 +75,7 @@ void NORETURN ShowStatsAndAbort(); void ReplaceSystemMalloc(); // asan_linux.cc / asan_mac.cc / asan_win.cc +uptr FindDynamicShadowStart(); void *AsanDoesNotSupportStaticLinkage(); void AsanCheckDynamicRTPrereqs(); void AsanCheckIncompatibleRT(); diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc index 50ef84c39a66..6d47ba432a61 100644 --- a/contrib/compiler-rt/lib/asan/asan_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_linux.cc @@ -77,6 +77,11 @@ void *AsanDoesNotSupportStaticLinkage() { return &_DYNAMIC; // defined in link.h } +uptr FindDynamicShadowStart() { + UNREACHABLE("FindDynamicShadowStart is not available"); + return 0; +} + void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { UNIMPLEMENTED(); } @@ -140,9 +145,9 @@ void AsanCheckIncompatibleRT() { // system libraries, causing crashes later in ASan initialization. MemoryMappingLayout proc_maps(/*cache_enabled*/true); char filename[128]; - while (proc_maps.Next(nullptr, nullptr, nullptr, filename, - sizeof(filename), nullptr)) { - if (IsDynamicRTName(filename)) { + MemoryMappedSegment segment(filename, sizeof(filename)); + while (proc_maps.Next(&segment)) { + if (IsDynamicRTName(segment.filename)) { Report("Your application is linked against " "incompatible ASan runtimes.\n"); Die(); diff --git a/contrib/compiler-rt/lib/asan/asan_mac.cc b/contrib/compiler-rt/lib/asan/asan_mac.cc index 3c93b26d9bf6..b7af1a58664c 100644 --- a/contrib/compiler-rt/lib/asan/asan_mac.cc +++ b/contrib/compiler-rt/lib/asan/asan_mac.cc @@ -55,6 +55,29 @@ void *AsanDoesNotSupportStaticLinkage() { return 0; } +uptr FindDynamicShadowStart() { + uptr granularity = GetMmapGranularity(); + uptr alignment = 8 * granularity; + uptr left_padding = granularity; + uptr space_size = kHighShadowEnd + left_padding; + + uptr largest_gap_found = 0; + uptr shadow_start = FindAvailableMemoryRange(space_size, alignment, + granularity, &largest_gap_found); + // If the shadow doesn't fit, restrict the address space to make it fit. + if (shadow_start == 0) { + uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment); + RestrictMemoryToMaxAddress(new_max_vm); + kHighMemEnd = new_max_vm - 1; + space_size = kHighShadowEnd + left_padding; + shadow_start = + FindAvailableMemoryRange(space_size, alignment, granularity, nullptr); + } + CHECK_NE((uptr)0, shadow_start); + CHECK(IsAligned(shadow_start, alignment)); + return shadow_start; +} + // No-op. Mac does not support static linkage anyway. void AsanCheckDynamicRTPrereqs() {} diff --git a/contrib/compiler-rt/lib/asan/asan_new_delete.cc b/contrib/compiler-rt/lib/asan/asan_new_delete.cc index 942b169d920c..e68c7f3e2400 100644 --- a/contrib/compiler-rt/lib/asan/asan_new_delete.cc +++ b/contrib/compiler-rt/lib/asan/asan_new_delete.cc @@ -26,7 +26,7 @@ // VS2015 dynamic CRT (MD) work. #if SANITIZER_WINDOWS #define CXX_OPERATOR_ATTRIBUTE -#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:"##sym)) +#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:" sym)) #ifdef _WIN64 COMMENT_EXPORT("??2@YAPEAX_K@Z") // operator new COMMENT_EXPORT("??2@YAPEAX_KAEBUnothrow_t@std@@@Z") // operator new nothrow diff --git a/contrib/compiler-rt/lib/asan/asan_posix.cc b/contrib/compiler-rt/lib/asan/asan_posix.cc index 68fde9139232..added746ace8 100644 --- a/contrib/compiler-rt/lib/asan/asan_posix.cc +++ b/contrib/compiler-rt/lib/asan/asan_posix.cc @@ -59,7 +59,7 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { // lis r0,-10000 // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000 // If the store faults then sp will not have been updated, so test above - // will not work, becase the fault address will be more than just "slightly" + // will not work, because the fault address will be more than just "slightly" // below sp. if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) { u32 inst = *(unsigned *)sig.pc; diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc index d9d7d7e4f13d..5ae3568ae04a 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtl.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc @@ -438,15 +438,7 @@ static void InitializeShadowMemory() { if (shadow_start == kDefaultShadowSentinel) { __asan_shadow_memory_dynamic_address = 0; CHECK_EQ(0, kLowShadowBeg); - - uptr granularity = GetMmapGranularity(); - uptr alignment = 8 * granularity; - uptr left_padding = granularity; - uptr space_size = kHighShadowEnd + left_padding; - - shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity); - CHECK_NE((uptr)0, shadow_start); - CHECK(IsAligned(shadow_start, alignment)); + shadow_start = FindDynamicShadowStart(); } // Update the shadow memory address (potentially) used by instrumentation. __asan_shadow_memory_dynamic_address = shadow_start; diff --git a/contrib/compiler-rt/lib/asan/asan_thread.cc b/contrib/compiler-rt/lib/asan/asan_thread.cc index 714496d5c606..b1a0d9a3b37f 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.cc +++ b/contrib/compiler-rt/lib/asan/asan_thread.cc @@ -200,7 +200,6 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { uptr stack_size = this->stack_size(); if (stack_size == 0) // stack_size is not yet available, don't use FakeStack. return nullptr; - CHECK_LE(stack_size, 0x10000000); uptr old_val = 0; // fake_stack_ has 3 states: // 0 -- not initialized diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc index 26db32465da8..8a839d913f95 100644 --- a/contrib/compiler-rt/lib/asan/asan_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_win.cc @@ -217,6 +217,18 @@ void *AsanDoesNotSupportStaticLinkage() { return 0; } +uptr FindDynamicShadowStart() { + uptr granularity = GetMmapGranularity(); + uptr alignment = 8 * granularity; + uptr left_padding = granularity; + uptr space_size = kHighShadowEnd + left_padding; + uptr shadow_start = + FindAvailableMemoryRange(space_size, alignment, granularity, nullptr); + CHECK_NE((uptr)0, shadow_start); + CHECK(IsAligned(shadow_start, alignment)); + return shadow_start; +} + void AsanCheckDynamicRTPrereqs() {} void AsanCheckIncompatibleRT() {} diff --git a/contrib/compiler-rt/lib/builtins/cpu_model.c b/contrib/compiler-rt/lib/builtins/cpu_model.c index 5ff6baf43876..c6b30eda0a77 100644 --- a/contrib/compiler-rt/lib/builtins/cpu_model.c +++ b/contrib/compiler-rt/lib/builtins/cpu_model.c @@ -44,29 +44,16 @@ enum ProcessorVendors { }; enum ProcessorTypes { - INTEL_ATOM = 1, + INTEL_BONNELL = 1, INTEL_CORE2, INTEL_COREI7, AMDFAM10H, AMDFAM15H, - INTEL_i386, - INTEL_i486, - INTEL_PENTIUM, - INTEL_PENTIUM_PRO, - INTEL_PENTIUM_II, - INTEL_PENTIUM_III, - INTEL_PENTIUM_IV, - INTEL_PENTIUM_M, - INTEL_CORE_DUO, - INTEL_XEONPHI, - INTEL_X86_64, - INTEL_NOCONA, - INTEL_PRESCOTT, - AMD_i486, - AMDPENTIUM, - AMDATHLON, - AMDFAM14H, - AMDFAM16H, + INTEL_SILVERMONT, + INTEL_KNL, + AMD_BTVER1, + AMD_BTVER2, + AMDFAM17H, CPU_TYPE_MAX }; @@ -79,32 +66,14 @@ enum ProcessorSubtypes { AMDFAM10H_ISTANBUL, AMDFAM15H_BDVER1, AMDFAM15H_BDVER2, - INTEL_PENTIUM_MMX, - INTEL_CORE2_65, - INTEL_CORE2_45, + AMDFAM15H_BDVER3, + AMDFAM15H_BDVER4, + AMDFAM17H_ZNVER1, INTEL_COREI7_IVYBRIDGE, INTEL_COREI7_HASWELL, INTEL_COREI7_BROADWELL, INTEL_COREI7_SKYLAKE, INTEL_COREI7_SKYLAKE_AVX512, - INTEL_ATOM_BONNELL, - INTEL_ATOM_SILVERMONT, - INTEL_KNIGHTS_LANDING, - AMDPENTIUM_K6, - AMDPENTIUM_K62, - AMDPENTIUM_K63, - AMDPENTIUM_GEODE, - AMDATHLON_TBIRD, - AMDATHLON_MP, - AMDATHLON_XP, - AMDATHLON_K8SSE3, - AMDATHLON_OPTERON, - AMDATHLON_FX, - AMDATHLON_64, - AMD_BTVER1, - AMD_BTVER2, - AMDFAM15H_BDVER3, - AMDFAM15H_BDVER4, CPU_SUBTYPE_MAX }; @@ -120,11 +89,26 @@ enum ProcessorFeatures { FEATURE_SSE4_2, FEATURE_AVX, FEATURE_AVX2, - FEATURE_AVX512, - FEATURE_AVX512SAVE, - FEATURE_MOVBE, - FEATURE_ADX, - FEATURE_EM64T + FEATURE_SSE4_A, + FEATURE_FMA4, + FEATURE_XOP, + FEATURE_FMA, + FEATURE_AVX512F, + FEATURE_BMI, + FEATURE_BMI2, + FEATURE_AES, + FEATURE_PCLMUL, + FEATURE_AVX512VL, + FEATURE_AVX512BW, + FEATURE_AVX512DQ, + FEATURE_AVX512CD, + FEATURE_AVX512ER, + FEATURE_AVX512PF, + FEATURE_AVX512VBMI, + FEATURE_AVX512IFMA, + FEATURE_AVX5124VNNIW, + FEATURE_AVX5124FMAPS, + FEATURE_AVX512VPOPCNTDQ }; // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max). @@ -164,26 +148,27 @@ static bool isCpuIdSupported() { /// getX86CpuIDAndInfo - Execute the specified cpuid and return the 4 values in /// the specified arguments. If we can't run cpuid on the host, return true. -static void getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, +static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, unsigned *rECX, unsigned *rEDX) { #if defined(__GNUC__) || defined(__clang__) #if defined(__x86_64__) - // gcc doesn't know cpuid would clobber ebx/rbx. Preseve it manually. + // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually. + // FIXME: should we save this for Clang? __asm__("movq\t%%rbx, %%rsi\n\t" "cpuid\n\t" "xchgq\t%%rbx, %%rsi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value)); + return false; #elif defined(__i386__) __asm__("movl\t%%ebx, %%esi\n\t" "cpuid\n\t" "xchgl\t%%ebx, %%esi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value)); -// pedantic #else returns to appease -Wunreachable-code (so we don't generate -// postprocessed code that looks like "return true; return false;") + return false; #else - assert(0 && "This method is defined only for x86."); + return true; #endif #elif defined(_MSC_VER) // The MSVC intrinsic is portable across x86 and x64. @@ -193,15 +178,16 @@ static void getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, *rEBX = registers[1]; *rECX = registers[2]; *rEDX = registers[3]; + return false; #else - assert(0 && "This method is defined only for GNUC, Clang or MSVC."); + return true; #endif } /// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return /// the 4 values in the specified arguments. If we can't run cpuid on the host, /// return true. -static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, +static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, unsigned *rEAX, unsigned *rEBX, unsigned *rECX, unsigned *rEDX) { #if defined(__x86_64__) || defined(_M_X64) @@ -213,6 +199,7 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, "xchgq\t%%rbx, %%rsi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value), "c"(subleaf)); + return false; #elif defined(_MSC_VER) int registers[4]; __cpuidex(registers, value, subleaf); @@ -220,8 +207,9 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, *rEBX = registers[1]; *rECX = registers[2]; *rEDX = registers[3]; + return false; #else - assert(0 && "This method is defined only for GNUC, Clang or MSVC."); + return true; #endif #elif defined(__i386__) || defined(_M_IX86) #if defined(__GNUC__) || defined(__clang__) @@ -230,6 +218,7 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, "xchgl\t%%ebx, %%esi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value), "c"(subleaf)); + return false; #elif defined(_MSC_VER) __asm { mov eax,value @@ -244,11 +233,12 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, mov esi,rEDX mov dword ptr [esi],edx } + return false; #else - assert(0 && "This method is defined only for GNUC, Clang or MSVC."); + return true; #endif #else - assert(0 && "This method is defined only for x86."); + return true; #endif } @@ -283,84 +273,15 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family, } } -static void getIntelProcessorTypeAndSubtype(unsigned int Family, - unsigned int Model, - unsigned int Brand_id, - unsigned int Features, - unsigned *Type, unsigned *Subtype) { +static void +getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, + unsigned Brand_id, unsigned Features, + unsigned *Type, unsigned *Subtype) { if (Brand_id != 0) return; switch (Family) { - case 3: - *Type = INTEL_i386; - break; - case 4: - switch (Model) { - case 0: // Intel486 DX processors - case 1: // Intel486 DX processors - case 2: // Intel486 SX processors - case 3: // Intel487 processors, IntelDX2 OverDrive processors, - // IntelDX2 processors - case 4: // Intel486 SL processor - case 5: // IntelSX2 processors - case 7: // Write-Back Enhanced IntelDX2 processors - case 8: // IntelDX4 OverDrive processors, IntelDX4 processors - default: - *Type = INTEL_i486; - break; - } - case 5: - switch (Model) { - case 1: // Pentium OverDrive processor for Pentium processor (60, 66), - // Pentium processors (60, 66) - case 2: // Pentium OverDrive processor for Pentium processor (75, 90, - // 100, 120, 133), Pentium processors (75, 90, 100, 120, 133, - // 150, 166, 200) - case 3: // Pentium OverDrive processors for Intel486 processor-based - // systems - *Type = INTEL_PENTIUM; - break; - case 4: // Pentium OverDrive processor with MMX technology for Pentium - // processor (75, 90, 100, 120, 133), Pentium processor with - // MMX technology (166, 200) - *Type = INTEL_PENTIUM; - *Subtype = INTEL_PENTIUM_MMX; - break; - default: - *Type = INTEL_PENTIUM; - break; - } case 6: switch (Model) { - case 0x01: // Pentium Pro processor - *Type = INTEL_PENTIUM_PRO; - break; - case 0x03: // Intel Pentium II OverDrive processor, Pentium II processor, - // model 03 - case 0x05: // Pentium II processor, model 05, Pentium II Xeon processor, - // model 05, and Intel Celeron processor, model 05 - case 0x06: // Celeron processor, model 06 - *Type = INTEL_PENTIUM_II; - break; - case 0x07: // Pentium III processor, model 07, and Pentium III Xeon - // processor, model 07 - case 0x08: // Pentium III processor, model 08, Pentium III Xeon processor, - // model 08, and Celeron processor, model 08 - case 0x0a: // Pentium III Xeon processor, model 0Ah - case 0x0b: // Pentium III processor, model 0Bh - *Type = INTEL_PENTIUM_III; - break; - case 0x09: // Intel Pentium M processor, Intel Celeron M processor model 09. - case 0x0d: // Intel Pentium M processor, Intel Celeron M processor, model - // 0Dh. All processors are manufactured using the 90 nm process. - case 0x15: // Intel EP80579 Integrated Processor and Intel EP80579 - // Integrated Processor with Intel QuickAssist Technology - *Type = INTEL_PENTIUM_M; - break; - case 0x0e: // Intel Core Duo processor, Intel Core Solo processor, model - // 0Eh. All processors are manufactured using the 65 nm process. - *Type = INTEL_CORE_DUO; - break; // yonah case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile // processor, Intel Core 2 Quad processor, Intel Core 2 Quad // mobile processor, Intel Core 2 Extreme processor, Intel @@ -368,9 +289,6 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family, // 0Fh. All processors are manufactured using the 65 nm process. case 0x16: // Intel Celeron processor model 16h. All processors are // manufactured using the 65 nm process - *Type = INTEL_CORE2; // "core2" - *Subtype = INTEL_CORE2_65; - break; case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model // 17h. All processors are manufactured using the 45 nm process. // @@ -378,14 +296,13 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family, case 0x1d: // Intel Xeon processor MP. All processors are manufactured using // the 45 nm process. *Type = INTEL_CORE2; // "penryn" - *Subtype = INTEL_CORE2_45; break; case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All // processors are manufactured using the 45 nm process. case 0x1e: // Intel(R) Core(TM) i7 CPU 870 @ 2.93GHz. // As found in a Summer 2010 model iMac. case 0x1f: - case 0x2e: // Nehalem EX + case 0x2e: // Nehalem EX *Type = INTEL_COREI7; // "nehalem" *Subtype = INTEL_COREI7_NEHALEM; break; @@ -403,7 +320,7 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family, *Subtype = INTEL_COREI7_SANDYBRIDGE; break; case 0x3a: - case 0x3e: // Ivy Bridge EP + case 0x3e: // Ivy Bridge EP *Type = INTEL_COREI7; // "ivybridge" *Subtype = INTEL_COREI7_IVYBRIDGE; break; @@ -427,22 +344,26 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family, break; // Skylake: - case 0x4e: - *Type = INTEL_COREI7; // "skylake-avx512" - *Subtype = INTEL_COREI7_SKYLAKE_AVX512; - break; - case 0x5e: + case 0x4e: // Skylake mobile + case 0x5e: // Skylake desktop + case 0x8e: // Kaby Lake mobile + case 0x9e: // Kaby Lake desktop *Type = INTEL_COREI7; // "skylake" *Subtype = INTEL_COREI7_SKYLAKE; break; + // Skylake Xeon: + case 0x55: + *Type = INTEL_COREI7; + *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" + break; + case 0x1c: // Most 45 nm Intel Atom processors case 0x26: // 45 nm Atom Lincroft case 0x27: // 32 nm Atom Medfield case 0x35: // 32 nm Atom Midview case 0x36: // 32 nm Atom Midview - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_BONNELL; + *Type = INTEL_BONNELL; break; // "bonnell" // Atom Silvermont codes from the Intel software optimization guide. @@ -452,185 +373,29 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family, case 0x5a: case 0x5d: case 0x4c: // really airmont - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_SILVERMONT; + *Type = INTEL_SILVERMONT; break; // "silvermont" case 0x57: - *Type = INTEL_XEONPHI; // knl - *Subtype = INTEL_KNIGHTS_LANDING; + *Type = INTEL_KNL; // knl break; - default: // Unknown family 6 CPU, try to guess. - if (Features & (1 << FEATURE_AVX512)) { - *Type = INTEL_XEONPHI; // knl - *Subtype = INTEL_KNIGHTS_LANDING; - break; - } - if (Features & (1 << FEATURE_ADX)) { - *Type = INTEL_COREI7; - *Subtype = INTEL_COREI7_BROADWELL; - break; - } - if (Features & (1 << FEATURE_AVX2)) { - *Type = INTEL_COREI7; - *Subtype = INTEL_COREI7_HASWELL; - break; - } - if (Features & (1 << FEATURE_AVX)) { - *Type = INTEL_COREI7; - *Subtype = INTEL_COREI7_SANDYBRIDGE; - break; - } - if (Features & (1 << FEATURE_SSE4_2)) { - if (Features & (1 << FEATURE_MOVBE)) { - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_SILVERMONT; - } else { - *Type = INTEL_COREI7; - *Subtype = INTEL_COREI7_NEHALEM; - } - break; - } - if (Features & (1 << FEATURE_SSE4_1)) { - *Type = INTEL_CORE2; // "penryn" - *Subtype = INTEL_CORE2_45; - break; - } - if (Features & (1 << FEATURE_SSSE3)) { - if (Features & (1 << FEATURE_MOVBE)) { - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_BONNELL; // "bonnell" - } else { - *Type = INTEL_CORE2; // "core2" - *Subtype = INTEL_CORE2_65; - } - break; - } - if (Features & (1 << FEATURE_EM64T)) { - *Type = INTEL_X86_64; - break; // x86-64 - } - if (Features & (1 << FEATURE_SSE2)) { - *Type = INTEL_PENTIUM_M; - break; - } - if (Features & (1 << FEATURE_SSE)) { - *Type = INTEL_PENTIUM_III; - break; - } - if (Features & (1 << FEATURE_MMX)) { - *Type = INTEL_PENTIUM_II; - break; - } - *Type = INTEL_PENTIUM_PRO; + default: // Unknown family 6 CPU. break; + break; } - case 15: { - switch (Model) { - case 0: // Pentium 4 processor, Intel Xeon processor. All processors are - // model 00h and manufactured using the 0.18 micron process. - case 1: // Pentium 4 processor, Intel Xeon processor, Intel Xeon - // processor MP, and Intel Celeron processor. All processors are - // model 01h and manufactured using the 0.18 micron process. - case 2: // Pentium 4 processor, Mobile Intel Pentium 4 processor - M, - // Intel Xeon processor, Intel Xeon processor MP, Intel Celeron - // processor, and Mobile Intel Celeron processor. All processors - // are model 02h and manufactured using the 0.13 micron process. - *Type = - ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV); - break; - - case 3: // Pentium 4 processor, Intel Xeon processor, Intel Celeron D - // processor. All processors are model 03h and manufactured using - // the 90 nm process. - case 4: // Pentium 4 processor, Pentium 4 processor Extreme Edition, - // Pentium D processor, Intel Xeon processor, Intel Xeon - // processor MP, Intel Celeron D processor. All processors are - // model 04h and manufactured using the 90 nm process. - case 6: // Pentium 4 processor, Pentium D processor, Pentium processor - // Extreme Edition, Intel Xeon processor, Intel Xeon processor - // MP, Intel Celeron D processor. All processors are model 06h - // and manufactured using the 65 nm process. - *Type = - ((Features & (1 << FEATURE_EM64T)) ? INTEL_NOCONA : INTEL_PRESCOTT); - break; - - default: - *Type = - ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV); - break; - } - } default: - break; /*"generic"*/ + break; // Unknown. } } -static void getAMDProcessorTypeAndSubtype(unsigned int Family, - unsigned int Model, - unsigned int Features, unsigned *Type, +static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, + unsigned Features, unsigned *Type, unsigned *Subtype) { // FIXME: this poorly matches the generated SubtargetFeatureKV table. There // appears to be no way to generate the wide variety of AMD-specific targets // from the information returned from CPUID. switch (Family) { - case 4: - *Type = AMD_i486; - case 5: - *Type = AMDPENTIUM; - switch (Model) { - case 6: - case 7: - *Subtype = AMDPENTIUM_K6; - break; // "k6" - case 8: - *Subtype = AMDPENTIUM_K62; - break; // "k6-2" - case 9: - case 13: - *Subtype = AMDPENTIUM_K63; - break; // "k6-3" - case 10: - *Subtype = AMDPENTIUM_GEODE; - break; // "geode" - default: - break; - } - case 6: - *Type = AMDATHLON; - switch (Model) { - case 4: - *Subtype = AMDATHLON_TBIRD; - break; // "athlon-tbird" - case 6: - case 7: - case 8: - *Subtype = AMDATHLON_MP; - break; // "athlon-mp" - case 10: - *Subtype = AMDATHLON_XP; - break; // "athlon-xp" - default: - break; - } - case 15: - *Type = AMDATHLON; - if (Features & (1 << FEATURE_SSE3)) { - *Subtype = AMDATHLON_K8SSE3; - break; // "k8-sse3" - } - switch (Model) { - case 1: - *Subtype = AMDATHLON_OPTERON; - break; // "opteron" - case 5: - *Subtype = AMDATHLON_FX; - break; // "athlon-fx"; also opteron - default: - *Subtype = AMDATHLON_64; - break; // "athlon64" - } case 16: *Type = AMDFAM10H; // "amdfam10" switch (Model) { @@ -643,23 +408,16 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family, case 8: *Subtype = AMDFAM10H_ISTANBUL; break; - default: - break; } + break; case 20: - *Type = AMDFAM14H; - *Subtype = AMD_BTVER1; + *Type = AMD_BTVER1; break; // "btver1"; case 21: *Type = AMDFAM15H; - if (!(Features & - (1 << FEATURE_AVX))) { // If no AVX support, provide a sane fallback. - *Subtype = AMD_BTVER1; - break; // "btver1" - } - if (Model >= 0x50 && Model <= 0x6f) { + if (Model >= 0x60 && Model <= 0x7f) { *Subtype = AMDFAM15H_BDVER4; - break; // "bdver4"; 50h-6Fh: Excavator + break; // "bdver4"; 60h-7Fh: Excavator } if (Model >= 0x30 && Model <= 0x3f) { *Subtype = AMDFAM15H_BDVER3; @@ -675,31 +433,47 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family, } break; case 22: - *Type = AMDFAM16H; - if (!(Features & - (1 << FEATURE_AVX))) { // If no AVX support provide a sane fallback. - *Subtype = AMD_BTVER1; - break; // "btver1"; - } - *Subtype = AMD_BTVER2; + *Type = AMD_BTVER2; break; // "btver2" + case 23: + *Type = AMDFAM17H; + *Subtype = AMDFAM17H_ZNVER1; + break; default: break; // "generic" } } -static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX, - unsigned MaxLeaf) { +static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, + unsigned *FeaturesOut) { unsigned Features = 0; - unsigned int EAX, EBX; - Features |= (((EDX >> 23) & 1) << FEATURE_MMX); - Features |= (((EDX >> 25) & 1) << FEATURE_SSE); - Features |= (((EDX >> 26) & 1) << FEATURE_SSE2); - Features |= (((ECX >> 0) & 1) << FEATURE_SSE3); - Features |= (((ECX >> 9) & 1) << FEATURE_SSSE3); - Features |= (((ECX >> 19) & 1) << FEATURE_SSE4_1); - Features |= (((ECX >> 20) & 1) << FEATURE_SSE4_2); - Features |= (((ECX >> 22) & 1) << FEATURE_MOVBE); + unsigned EAX, EBX; + + if ((EDX >> 15) & 1) + Features |= 1 << FEATURE_CMOV; + if ((EDX >> 23) & 1) + Features |= 1 << FEATURE_MMX; + if ((EDX >> 25) & 1) + Features |= 1 << FEATURE_SSE; + if ((EDX >> 26) & 1) + Features |= 1 << FEATURE_SSE2; + + if ((ECX >> 0) & 1) + Features |= 1 << FEATURE_SSE3; + if ((ECX >> 1) & 1) + Features |= 1 << FEATURE_PCLMUL; + if ((ECX >> 9) & 1) + Features |= 1 << FEATURE_SSSE3; + if ((ECX >> 12) & 1) + Features |= 1 << FEATURE_FMA; + if ((ECX >> 19) & 1) + Features |= 1 << FEATURE_SSE4_1; + if ((ECX >> 20) & 1) + Features |= 1 << FEATURE_SSE4_2; + if ((ECX >> 23) & 1) + Features |= 1 << FEATURE_POPCNT; + if ((ECX >> 25) & 1) + Features |= 1 << FEATURE_AES; // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV // indicates that the AVX registers will be saved and restored on context @@ -708,20 +482,59 @@ static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX, bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) && ((EAX & 0x6) == 0x6); bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0); - bool HasLeaf7 = MaxLeaf >= 0x7; - getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); - bool HasADX = HasLeaf7 && ((EBX >> 19) & 1); - bool HasAVX2 = HasAVX && HasLeaf7 && (EBX & 0x20); - bool HasAVX512 = HasLeaf7 && HasAVX512Save && ((EBX >> 16) & 1); - Features |= (HasAVX << FEATURE_AVX); - Features |= (HasAVX2 << FEATURE_AVX2); - Features |= (HasAVX512 << FEATURE_AVX512); - Features |= (HasAVX512Save << FEATURE_AVX512SAVE); - Features |= (HasADX << FEATURE_ADX); - getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); - Features |= (((EDX >> 29) & 0x1) << FEATURE_EM64T); - return Features; + if (HasAVX) + Features |= 1 << FEATURE_AVX; + + bool HasLeaf7 = + MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); + + if (HasLeaf7 && ((EBX >> 3) & 1)) + Features |= 1 << FEATURE_BMI; + if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX) + Features |= 1 << FEATURE_AVX2; + if (HasLeaf7 && ((EBX >> 9) & 1)) + Features |= 1 << FEATURE_BMI2; + if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512F; + if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512DQ; + if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512IFMA; + if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512PF; + if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512ER; + if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512CD; + if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512BW; + if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512VL; + + if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512VBMI; + if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512VPOPCNTDQ; + + if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX5124VNNIW; + if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX5124FMAPS; + + unsigned MaxExtLevel; + getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX); + + bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 && + !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); + if (HasExtLeaf1 && ((ECX >> 6) & 1)) + Features |= 1 << FEATURE_SSE4_A; + if (HasExtLeaf1 && ((ECX >> 11) & 1)) + Features |= 1 << FEATURE_XOP; + if (HasExtLeaf1 && ((ECX >> 16) & 1)) + Features |= 1 << FEATURE_FMA4; + + *FeaturesOut = Features; } #if defined(HAVE_INIT_PRIORITY) @@ -751,11 +564,11 @@ struct __processor_model { int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { - unsigned int EAX, EBX, ECX, EDX; - unsigned int MaxLeaf = 5; - unsigned int Vendor; - unsigned int Model, Family, Brand_id; - unsigned int Features = 0; + unsigned EAX, EBX, ECX, EDX; + unsigned MaxLeaf = 5; + unsigned Vendor; + unsigned Model, Family, Brand_id; + unsigned Features = 0; /* This function needs to run just once. */ if (__cpu_model.__cpu_vendor) @@ -765,9 +578,7 @@ __cpu_indicator_init(void) { return -1; /* Assume cpuid insn present. Run in level 0 to get vendor id. */ - getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX); - - if (MaxLeaf < 1) { + if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) { __cpu_model.__cpu_vendor = VENDOR_OTHER; return -1; } @@ -776,7 +587,7 @@ __cpu_indicator_init(void) { Brand_id = EBX & 0xff; /* Find available features. */ - Features = getAvailableFeatures(ECX, EDX, MaxLeaf); + getAvailableFeatures(ECX, EDX, MaxLeaf, &Features); __cpu_model.__cpu_features[0] = Features; if (Vendor == SIG_INTEL) { diff --git a/contrib/compiler-rt/lib/builtins/int_util.c b/contrib/compiler-rt/lib/builtins/int_util.c index 420d1e237aae..de87410dbca2 100644 --- a/contrib/compiler-rt/lib/builtins/int_util.c +++ b/contrib/compiler-rt/lib/builtins/int_util.c @@ -45,6 +45,16 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) { __assert_rtn(function, file, line, "libcompiler_rt abort"); } +#elif __Fuchsia__ + +#ifndef _WIN32 +__attribute__((weak)) +__attribute__((visibility("hidden"))) +#endif +void compilerrt_abort_impl(const char *file, int line, const char *function) { + __builtin_trap(); +} + #else /* Get the system definition of abort() */ diff --git a/contrib/compiler-rt/lib/esan/working_set.cpp b/contrib/compiler-rt/lib/esan/working_set.cpp index f39111993c33..e56902c8f32a 100644 --- a/contrib/compiler-rt/lib/esan/working_set.cpp +++ b/contrib/compiler-rt/lib/esan/working_set.cpp @@ -160,15 +160,16 @@ static u32 countAndClearShadowValues(u32 BitIdx, uptr ShadowStart, static u32 computeWorkingSizeAndReset(u32 BitIdx) { u32 WorkingSetSize = 0; MemoryMappingLayout MemIter(true/*cache*/); - uptr Start, End, Prot; - while (MemIter.Next(&Start, &End, nullptr/*offs*/, nullptr/*file*/, - 0/*file size*/, &Prot)) { - VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n", - __FUNCTION__, Start, End, Prot, isAppMem(Start), - isShadowMem(Start)); - if (isShadowMem(Start) && (Prot & MemoryMappingLayout::kProtectionWrite)) { - VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Start, End); - WorkingSetSize += countAndClearShadowValues(BitIdx, Start, End); + MemoryMappedSegment Segment; + while (MemIter.Next(&Segment)) { + VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n", __FUNCTION__, + Segment.start, Segment.end, Segment.protection, + isAppMem(Segment.start), isShadowMem(Segment.start)); + if (isShadowMem(Segment.start) && Segment.IsWritable()) { + VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Segment.start, + Segment.end); + WorkingSetSize += + countAndClearShadowValues(BitIdx, Segment.start, Segment.end); } } return WorkingSetSize; diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.cc b/contrib/compiler-rt/lib/lsan/lsan_common.cc index a5ffc6835f5f..4ffa91568cc8 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common.cc @@ -74,6 +74,10 @@ static const char kStdSuppressions[] = // definition. "leak:*pthread_exit*\n" #endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT +#if SANITIZER_MAC + // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173 + "leak:*_os_trace*\n" +#endif // TLS leak in some glibc versions, described in // https://sourceware.org/bugzilla/show_bug.cgi?id=12650. "leak:*tls_get_addr*\n"; @@ -301,11 +305,10 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, } void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, - uptr region_begin, uptr region_end, uptr prot) { + uptr region_begin, uptr region_end, bool is_readable) { uptr intersection_begin = Max(root_region.begin, region_begin); uptr intersection_end = Min(region_end, root_region.begin + root_region.size); if (intersection_begin >= intersection_end) return; - bool is_readable = prot & MemoryMappingLayout::kProtectionRead; LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n", root_region.begin, root_region.begin + root_region.size, region_begin, region_end, @@ -318,11 +321,10 @@ void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, static void ProcessRootRegion(Frontier *frontier, const RootRegion &root_region) { MemoryMappingLayout proc_maps(/*cache_enabled*/ true); - uptr begin, end, prot; - while (proc_maps.Next(&begin, &end, - /*offset*/ nullptr, /*filename*/ nullptr, - /*filename_size*/ 0, &prot)) { - ScanRootRegion(frontier, root_region, begin, end, prot); + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + ScanRootRegion(frontier, root_region, segment.start, segment.end, + segment.IsReadable()); } } diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.h b/contrib/compiler-rt/lib/lsan/lsan_common.h index beb31d6f40e4..d93ac1b10919 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common.h +++ b/contrib/compiler-rt/lib/lsan/lsan_common.h @@ -127,7 +127,7 @@ struct RootRegion { InternalMmapVector const *GetRootRegions(); void ScanRootRegion(Frontier *frontier, RootRegion const ®ion, - uptr region_begin, uptr region_end, uptr prot); + uptr region_begin, uptr region_end, bool is_readable); // Run stoptheworld while holding any platform-specific locks. void DoStopTheWorld(StopTheWorldCallback callback, void* argument); diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc index adde3a1b4035..f87c6b7e0425 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc @@ -156,7 +156,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { if (flags()->use_root_regions) { for (uptr i = 0; i < root_regions->size(); i++) { ScanRootRegion(frontier, (*root_regions)[i], address, end_address, - info.protection); + info.protection & kProtectionRead); } } diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc index ce8444a3bb2f..069777c7f5e7 100644 --- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc +++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc @@ -27,6 +27,7 @@ #include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" @@ -48,15 +49,9 @@ DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen) DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n) DECLARE_REAL(void *, memset, void *dest, int c, uptr n) -#if SANITIZER_FREEBSD -#define __errno_location __error -#endif - // True if this is a nested interceptor. static THREADLOCAL int in_interceptor_scope; -extern "C" int *__errno_location(void); - struct InterceptorScope { InterceptorScope() { ++in_interceptor_scope; } ~InterceptorScope() { --in_interceptor_scope; } @@ -915,7 +910,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, ENSURE_MSAN_INITED(); if (addr && !MEM_IS_APP(addr)) { if (flags & map_fixed) { - *__errno_location() = errno_EINVAL; + errno = errno_EINVAL; return (void *)-1; } else { addr = nullptr; @@ -933,7 +928,7 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags, ENSURE_MSAN_INITED(); if (addr && !MEM_IS_APP(addr)) { if (flags & map_fixed) { - *__errno_location() = errno_EINVAL; + errno = errno_EINVAL; return (void *)-1; } else { addr = nullptr; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index d44c71513896..89aae579856a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -107,7 +107,8 @@ bool MprotectNoAccess(uptr addr, uptr size); bool MprotectReadOnly(uptr addr, uptr size); // Find an available address space. -uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding); +uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding, + uptr *largest_gap_found); // Used to check if we can map shadow memory to a fixed location. bool MemoryRangeIsAvailable(uptr range_start, uptr range_end); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 459530aa95ba..8607bf44902d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -40,6 +40,7 @@ #include "interception/interception.h" #include "sanitizer_addrhashmap.h" +#include "sanitizer_errno.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_interceptors.h" #include "sanitizer_tls_get_addr.h" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.cc new file mode 100644 index 000000000000..a6f9fc6125eb --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.cc @@ -0,0 +1,35 @@ +//===-- sanitizer_errno.cc --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizers run-time libraries. +// +// Defines errno to avoid including errno.h and its dependencies into other +// files (e.g. interceptors are not supposed to include any system headers). +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_errno_codes.h" +#include "sanitizer_internal_defs.h" + +#include + +namespace __sanitizer { + +COMPILER_CHECK(errno_ENOMEM == ENOMEM); +COMPILER_CHECK(errno_EBUSY == EBUSY); +COMPILER_CHECK(errno_EINVAL == EINVAL); + +// EOWNERDEAD is not present in some older platforms. +#if defined(EOWNERDEAD) +extern const int errno_EOWNERDEAD = EOWNERDEAD; +#else +extern const int errno_EOWNERDEAD = -1; +#endif + +} // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h new file mode 100644 index 000000000000..c405307ba8ec --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h @@ -0,0 +1,35 @@ +//===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizers run-time libraries. +// +// Defines errno to avoid including errno.h and its dependencies into sensitive +// files (e.g. interceptors are not supposed to include any system headers). +// It's ok to use errno.h directly when your file already depend on other system +// includes though. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ERRNO_H +#define SANITIZER_ERRNO_H + +#include "sanitizer_errno_codes.h" +#include "sanitizer_platform.h" + +#if SANITIZER_FREEBSD || SANITIZER_MAC +# define __errno_location __error +#elif SANITIZER_ANDROID +# define __errno_location __errno +#endif + +extern "C" int *__errno_location(); + +#define errno (*__errno_location()) + +#endif // SANITIZER_ERRNO_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h new file mode 100644 index 000000000000..dba774c5b6c5 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno_codes.h @@ -0,0 +1,34 @@ +//===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizers run-time libraries. +// +// Defines errno codes to avoid including errno.h and its dependencies into +// sensitive files (e.g. interceptors are not supposed to include any system +// headers). +// It's ok to use errno.h directly when your file already depend on other system +// includes though. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ERRNO_CODES_H +#define SANITIZER_ERRNO_CODES_H + +namespace __sanitizer { + +#define errno_ENOMEM 12 +#define errno_EBUSY 16 +#define errno_EINVAL 22 + +// Those might not present or their value differ on different platforms. +extern const int errno_EOWNERDEAD; + +} // namespace __sanitizer + +#endif // SANITIZER_ERRNO_CODES_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index a94a63c7f16d..a79a2a155db9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -59,6 +59,14 @@ #include #include +#if SANITIZER_LINUX +#include +#endif + +#if SANITIZER_LINUX && !SANITIZER_ANDROID +#include +#endif + #if SANITIZER_FREEBSD #include #include @@ -209,7 +217,6 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { out->st_atime = in->st_atime; out->st_mtime = in->st_mtime; out->st_ctime = in->st_ctime; - out->st_ino = in->st_ino; } #endif @@ -229,7 +236,6 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { out->st_atime = in->st_atime_nsec; out->st_mtime = in->st_mtime_nsec; out->st_ctime = in->st_ctime_nsec; - out->st_ino = in->st_ino; } #endif @@ -815,6 +821,72 @@ bool ThreadLister::GetDirectoryEntries() { return true; } +#if SANITIZER_WORDSIZE == 32 +// Take care of unusable kernel area in top gigabyte. +static uptr GetKernelAreaSize() { +#if SANITIZER_LINUX && !SANITIZER_X32 + const uptr gbyte = 1UL << 30; + + // Firstly check if there are writable segments + // mapped to top gigabyte (e.g. stack). + MemoryMappingLayout proc_maps(/*cache_enabled*/true); + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0; + } + +#if !SANITIZER_ANDROID + // Even if nothing is mapped, top Gb may still be accessible + // if we are running on 64-bit kernel. + // Uname may report misleading results if personality type + // is modified (e.g. under schroot) so check this as well. + struct utsname uname_info; + int pers = personality(0xffffffffUL); + if (!(pers & PER_MASK) + && uname(&uname_info) == 0 + && internal_strstr(uname_info.machine, "64")) + return 0; +#endif // SANITIZER_ANDROID + + // Top gigabyte is reserved for kernel. + return gbyte; +#else + return 0; +#endif // SANITIZER_LINUX && !SANITIZER_X32 +} +#endif // SANITIZER_WORDSIZE == 32 + +uptr GetMaxVirtualAddress() { +#if SANITIZER_WORDSIZE == 64 +# if defined(__powerpc64__) || defined(__aarch64__) + // On PowerPC64 we have two different address space layouts: 44- and 46-bit. + // We somehow need to figure out which one we are using now and choose + // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL. + // Note that with 'ulimit -s unlimited' the stack is moved away from the top + // of the address space, so simply checking the stack address is not enough. + // This should (does) work for both PowerPC64 Endian modes. + // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit. + return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1; +# elif defined(__mips64) + return (1ULL << 40) - 1; // 0x000000ffffffffffUL; +# elif defined(__s390x__) + return (1ULL << 53) - 1; // 0x001fffffffffffffUL; +# else + return (1ULL << 47) - 1; // 0x00007fffffffffffUL; +# endif +#else // SANITIZER_WORDSIZE == 32 +# if defined(__s390__) + return (1ULL << 31) - 1; // 0x7fffffff; +# else + uptr res = (1ULL << 32) - 1; // 0xffffffff; + if (!common_flags()->full_address_space) + res -= GetKernelAreaSize(); + CHECK_LT(reinterpret_cast(&res), res); + return res; +# endif +#endif // SANITIZER_WORDSIZE +} + uptr GetPageSize() { // Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. #if SANITIZER_ANDROID @@ -1599,7 +1671,8 @@ void CheckNoDeepBind(const char *filename, int flag) { #endif } -uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) { +uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding, + uptr *largest_gap_found) { UNREACHABLE("FindAvailableMemoryRange is not available"); return 0; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc index b9a48a1e496b..52196db12731 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -81,28 +81,25 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, // Find the mapping that contains a stack variable. MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr start, end, offset; + MemoryMappedSegment segment; uptr prev_end = 0; - while (proc_maps.Next(&start, &end, &offset, nullptr, 0, - /* protection */nullptr)) { - if ((uptr)&rl < end) - break; - prev_end = end; + while (proc_maps.Next(&segment)) { + if ((uptr)&rl < segment.end) break; + prev_end = segment.end; } - CHECK((uptr)&rl >= start && (uptr)&rl < end); + CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end); // Get stacksize from rlimit, but clip it so that it does not overlap // with other mappings. uptr stacksize = rl.rlim_cur; - if (stacksize > end - prev_end) - stacksize = end - prev_end; + if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end; // When running with unlimited stack size, we still want to set some limit. // The unlimited stack size is caused by 'ulimit -s unlimited'. // Also, for some reason, GNU make spawns subprocesses with unlimited stack. if (stacksize > kMaxThreadStackSize) stacksize = kMaxThreadStackSize; - *stack_top = end; - *stack_bottom = end - stacksize; + *stack_top = segment.end; + *stack_bottom = segment.end - stacksize; return; } pthread_attr_t attr; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc index b48238106dd9..8df01815f9f7 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -191,7 +191,8 @@ void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); } uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { - return sigprocmask(how, set, oldset); + // Don't use sigprocmask here, because it affects all threads. + return pthread_sigmask(how, set, oldset); } // Doesn't call pthread_atfork() handlers (but not available on 10.6). @@ -799,9 +800,48 @@ char **GetArgv() { return *_NSGetArgv(); } +#if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM +// The task_vm_info struct is normally provided by the macOS SDK, but we need +// fields only available in 10.12+. Declare the struct manually to be able to +// build against older SDKs. +struct __sanitizer_task_vm_info { + uptr _unused[(SANITIZER_WORDSIZE == 32) ? 20 : 19]; + uptr min_address; + uptr max_address; +}; + +uptr GetTaskInfoMaxAddress() { + __sanitizer_task_vm_info vm_info = {{0}, 0, 0}; + mach_msg_type_number_t count = sizeof(vm_info) / sizeof(int); + int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count); + if (err == 0) { + return vm_info.max_address - 1; + } else { + // xnu cannot provide vm address limit + return 0x200000000 - 1; + } +} +#endif + +uptr GetMaxVirtualAddress() { +#if SANITIZER_WORDSIZE == 64 +# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM + // Get the maximum VM address + static uptr max_vm = GetTaskInfoMaxAddress(); + CHECK(max_vm); + return max_vm; +# else + return (1ULL << 47) - 1; // 0x00007fffffffffffUL; +# endif +#else // SANITIZER_WORDSIZE == 32 + return (1ULL << 32) - 1; // 0xffffffff; +#endif // SANITIZER_WORDSIZE +} + uptr FindAvailableMemoryRange(uptr shadow_size, uptr alignment, - uptr left_padding) { + uptr left_padding, + uptr *largest_gap_found) { typedef vm_region_submap_short_info_data_64_t RegionInfo; enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 }; // Start searching for available memory region past PAGEZERO, which is @@ -812,6 +852,7 @@ uptr FindAvailableMemoryRange(uptr shadow_size, mach_vm_address_t address = start_address; mach_vm_address_t free_begin = start_address; kern_return_t kr = KERN_SUCCESS; + if (largest_gap_found) *largest_gap_found = 0; while (kr == KERN_SUCCESS) { mach_vm_size_t vmsize = 0; natural_t depth = 0; @@ -821,10 +862,15 @@ uptr FindAvailableMemoryRange(uptr shadow_size, (vm_region_info_t)&vminfo, &count); if (free_begin != address) { // We found a free region [free_begin..address-1]. - uptr shadow_address = RoundUpTo((uptr)free_begin + left_padding, - alignment); - if (shadow_address + shadow_size < (uptr)address) { - return shadow_address; + uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment); + uptr gap_end = RoundDownTo((uptr)address, alignment); + uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0; + if (shadow_size < gap_size) { + return gap_start; + } + + if (largest_gap_found && *largest_gap_found < gap_size) { + *largest_gap_found = gap_size; } } // Move to the next region. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index 636d9bfeac8c..3f1c68c8610a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -36,6 +36,8 @@ MacosVersion GetMacosVersion(); char **GetEnviron(); +void RestrictMemoryToMaxAddress(uptr max_address); + } // namespace __sanitizer extern "C" { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc new file mode 100644 index 000000000000..c95daa9372ad --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc @@ -0,0 +1,30 @@ +//===-- sanitizer_mac_libcdep.cc ------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between various sanitizers' runtime libraries and +// implements OSX-specific functions. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_MAC +#include "sanitizer_mac.h" + +#include + +namespace __sanitizer { + +void RestrictMemoryToMaxAddress(uptr max_address) { + uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address; + void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap"); + CHECK(res != MAP_FAILED); +} + +} // namespace __sanitizer + +#endif // SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index 683f019d70c3..83f4fd22f623 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -25,7 +25,6 @@ #endif #include #include -#include #include #include #include @@ -931,14 +930,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE; #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID - const int errno_EINVAL = EINVAL; -// EOWNERDEAD is not present in some older platforms. -#if defined(EOWNERDEAD) - const int errno_EOWNERDEAD = EOWNERDEAD; -#else - const int errno_EOWNERDEAD = -1; -#endif - const int si_SEGV_MAPERR = SEGV_MAPERR; const int si_SEGV_ACCERR = SEGV_ACCERR; } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index def2ff56e693..02c5ef609a37 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -1468,9 +1468,6 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_PIO_SCRNMAP; #endif - extern const int errno_EINVAL; - extern const int errno_EOWNERDEAD; - extern const int si_SEGV_MAPERR; extern const int si_SEGV_ACCERR; } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc index 63f1bf713b24..8d3128ae199d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc @@ -27,14 +27,6 @@ #include #include -#if SANITIZER_LINUX -#include -#endif - -#if SANITIZER_LINUX && !SANITIZER_ANDROID -#include -#endif - #if SANITIZER_FREEBSD // The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before // that, it was never implemented. So just define it to zero. @@ -49,80 +41,6 @@ uptr GetMmapGranularity() { return GetPageSize(); } -#if SANITIZER_WORDSIZE == 32 -// Take care of unusable kernel area in top gigabyte. -static uptr GetKernelAreaSize() { -#if SANITIZER_LINUX && !SANITIZER_X32 - const uptr gbyte = 1UL << 30; - - // Firstly check if there are writable segments - // mapped to top gigabyte (e.g. stack). - MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr end, prot; - while (proc_maps.Next(/*start*/nullptr, &end, - /*offset*/nullptr, /*filename*/nullptr, - /*filename_size*/0, &prot)) { - if ((end >= 3 * gbyte) - && (prot & MemoryMappingLayout::kProtectionWrite) != 0) - return 0; - } - -#if !SANITIZER_ANDROID - // Even if nothing is mapped, top Gb may still be accessible - // if we are running on 64-bit kernel. - // Uname may report misleading results if personality type - // is modified (e.g. under schroot) so check this as well. - struct utsname uname_info; - int pers = personality(0xffffffffUL); - if (!(pers & PER_MASK) - && uname(&uname_info) == 0 - && internal_strstr(uname_info.machine, "64")) - return 0; -#endif // SANITIZER_ANDROID - - // Top gigabyte is reserved for kernel. - return gbyte; -#else - return 0; -#endif // SANITIZER_LINUX && !SANITIZER_X32 -} -#endif // SANITIZER_WORDSIZE == 32 - -uptr GetMaxVirtualAddress() { -#if SANITIZER_WORDSIZE == 64 -# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM - // Ideally, we would derive the upper bound from MACH_VM_MAX_ADDRESS. The - // upper bound can change depending on the device. - return 0x200000000 - 1; -# elif defined(__powerpc64__) || defined(__aarch64__) - // On PowerPC64 we have two different address space layouts: 44- and 46-bit. - // We somehow need to figure out which one we are using now and choose - // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL. - // Note that with 'ulimit -s unlimited' the stack is moved away from the top - // of the address space, so simply checking the stack address is not enough. - // This should (does) work for both PowerPC64 Endian modes. - // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit. - return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1; -# elif defined(__mips64) - return (1ULL << 40) - 1; // 0x000000ffffffffffUL; -# elif defined(__s390x__) - return (1ULL << 53) - 1; // 0x001fffffffffffffUL; -# else - return (1ULL << 47) - 1; // 0x00007fffffffffffUL; -# endif -#else // SANITIZER_WORDSIZE == 32 -# if defined(__s390__) - return (1ULL << 31) - 1; // 0x7fffffff; -# else - uptr res = (1ULL << 32) - 1; // 0xffffffff; - if (!common_flags()->full_address_space) - res -= GetKernelAreaSize(); - CHECK_LT(reinterpret_cast(&res), res); - return res; -# endif -#endif // SANITIZER_WORDSIZE -} - void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { size = RoundUpTo(size, GetPageSizeCached()); uptr res = internal_mmap(nullptr, size, @@ -162,7 +80,7 @@ void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { } // We want to map a chunk of address space aligned to 'alignment'. -// We do it by maping a bit more and then unmaping redundant pieces. +// We do it by mapping a bit more and then unmapping redundant pieces. // We probably can do it with fewer syscalls in some OS-dependent way. void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, const char *mem_type) { @@ -313,13 +231,12 @@ static inline bool IntervalsAreSeparate(uptr start1, uptr end1, // memory). bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr start, end; - while (proc_maps.Next(&start, &end, - /*offset*/nullptr, /*filename*/nullptr, - /*filename_size*/0, /*protection*/nullptr)) { - if (start == end) continue; // Empty range. - CHECK_NE(0, end); - if (!IntervalsAreSeparate(start, end - 1, range_start, range_end)) + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (segment.start == segment.end) continue; // Empty range. + CHECK_NE(0, segment.end); + if (!IntervalsAreSeparate(segment.start, segment.end - 1, range_start, + range_end)) return false; } return true; @@ -327,13 +244,13 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { void DumpProcessMap() { MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr start, end; const sptr kBufSize = 4095; char *filename = (char*)MmapOrDie(kBufSize, __func__); + MemoryMappedSegment segment(filename, kBufSize); Report("Process memory map follows:\n"); - while (proc_maps.Next(&start, &end, /* file_offset */nullptr, - filename, kBufSize, /* protection */nullptr)) { - Printf("\t%p-%p\t%s\n", (void*)start, (void*)end, filename); + while (proc_maps.Next(&segment)) { + Printf("\t%p-%p\t%s\n", (void *)segment.start, (void *)segment.end, + segment.filename); } Report("End of process memory map.\n"); UnmapOrDie(filename, kBufSize); @@ -363,14 +280,14 @@ void ReportFile::Write(const char *buffer, uptr length) { } bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) { - uptr s, e, off, prot; - InternalScopedString buff(kMaxPathLength); MemoryMappingLayout proc_maps(/*cache_enabled*/false); - while (proc_maps.Next(&s, &e, &off, buff.data(), buff.size(), &prot)) { - if ((prot & MemoryMappingLayout::kProtectionExecute) != 0 - && internal_strcmp(module, buff.data()) == 0) { - *start = s; - *end = e; + InternalScopedString buff(kMaxPathLength); + MemoryMappedSegment segment(buff.data(), kMaxPathLength); + while (proc_maps.Next(&segment)) { + if (segment.IsExecutable() && + internal_strcmp(module, segment.filename) == 0) { + *start = segment.start; + *end = segment.end; return true; } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h index 5aad6b959ad4..06d072b4dc77 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h @@ -31,13 +31,37 @@ struct ProcSelfMapsBuff { void ReadProcMaps(ProcSelfMapsBuff *proc_maps); #endif // SANITIZER_FREEBSD || SANITIZER_LINUX +// Memory protection masks. +static const uptr kProtectionRead = 1; +static const uptr kProtectionWrite = 2; +static const uptr kProtectionExecute = 4; +static const uptr kProtectionShared = 8; + +struct MemoryMappedSegment { + MemoryMappedSegment(char *buff = nullptr, uptr size = 0) + : filename(buff), filename_size(size) {} + ~MemoryMappedSegment() {} + + bool IsReadable() { return protection & kProtectionRead; } + bool IsWritable() { return protection & kProtectionWrite; } + bool IsExecutable() { return protection & kProtectionExecute; } + bool IsShared() { return protection & kProtectionShared; } + + uptr start; + uptr end; + uptr offset; + char *filename; // owned by caller + uptr filename_size; + uptr protection; + ModuleArch arch; + u8 uuid[kModuleUUIDSize]; +}; + class MemoryMappingLayout { public: explicit MemoryMappingLayout(bool cache_enabled); ~MemoryMappingLayout(); - bool Next(uptr *start, uptr *end, uptr *offset, char filename[], - uptr filename_size, uptr *protection, ModuleArch *arch = nullptr, - u8 *uuid = nullptr); + bool Next(MemoryMappedSegment *segment); void Reset(); // In some cases, e.g. when running under a sandbox on Linux, ASan is unable // to obtain the memory mappings. It should fall back to pre-cached data @@ -47,12 +71,6 @@ class MemoryMappingLayout { // Adds all mapped objects into a vector. void DumpListOfModules(InternalMmapVector *modules); - // Memory protection masks. - static const uptr kProtectionRead = 1; - static const uptr kProtectionWrite = 2; - static const uptr kProtectionExecute = 4; - static const uptr kProtectionShared = 8; - private: void LoadFromCache(); @@ -67,10 +85,7 @@ class MemoryMappingLayout { static StaticSpinMutex cache_lock_; // protects cached_proc_self_maps_. # elif SANITIZER_MAC template - bool NextSegmentLoad(uptr *start, uptr *end, uptr *offset, char filename[], - uptr filename_size, ModuleArch *arch, u8 *uuid, - uptr *protection); - void GetSegmentAddrRange(uptr *start, uptr *end, uptr vmaddr, uptr vmsize); + bool NextSegmentLoad(MemoryMappedSegment *segment); int current_image_; u32 current_magic_; u32 current_filetype_; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc index c583f42f25d8..b95f301a437d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -119,12 +119,10 @@ void MemoryMappingLayout::LoadFromCache() { void MemoryMappingLayout::DumpListOfModules( InternalMmapVector *modules) { Reset(); - uptr cur_beg, cur_end, cur_offset, prot; InternalScopedString module_name(kMaxPathLength); - for (uptr i = 0; Next(&cur_beg, &cur_end, &cur_offset, module_name.data(), - module_name.size(), &prot); - i++) { - const char *cur_name = module_name.data(); + MemoryMappedSegment segment(module_name.data(), module_name.size()); + for (uptr i = 0; Next(&segment); i++) { + const char *cur_name = segment.filename; if (cur_name[0] == '\0') continue; // Don't subtract 'cur_beg' from the first entry: @@ -138,11 +136,11 @@ void MemoryMappingLayout::DumpListOfModules( // mapped high at address space (in particular, higher than // shadow memory of the tool), so the module can't be the // first entry. - uptr base_address = (i ? cur_beg : 0) - cur_offset; + uptr base_address = (i ? segment.start : 0) - segment.offset; LoadedModule cur_module; cur_module.set(cur_name, base_address); - cur_module.addAddressRange(cur_beg, cur_end, prot & kProtectionExecute, - prot & kProtectionWrite); + cur_module.addAddressRange(segment.start, segment.end, + segment.IsExecutable(), segment.IsWritable()); modules->push_back(cur_module); } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc index 30216456330e..f0cdbeb4483a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc @@ -48,36 +48,27 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { proc_maps->len = Size; } -bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, - char filename[], uptr filename_size, - uptr *protection, ModuleArch *arch, u8 *uuid) { - CHECK(!arch && "not implemented"); - CHECK(!uuid && "not implemented"); +bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { char *last = proc_self_maps_.data + proc_self_maps_.len; if (current_ >= last) return false; - uptr dummy; - if (!start) start = &dummy; - if (!end) end = &dummy; - if (!offset) offset = &dummy; - if (!protection) protection = &dummy; struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_; - *start = (uptr)VmEntry->kve_start; - *end = (uptr)VmEntry->kve_end; - *offset = (uptr)VmEntry->kve_offset; + segment->start = (uptr)VmEntry->kve_start; + segment->end = (uptr)VmEntry->kve_end; + segment->offset = (uptr)VmEntry->kve_offset; - *protection = 0; + segment->protection = 0; if ((VmEntry->kve_protection & KVME_PROT_READ) != 0) - *protection |= kProtectionRead; + segment->protection |= kProtectionRead; if ((VmEntry->kve_protection & KVME_PROT_WRITE) != 0) - *protection |= kProtectionWrite; + segment->protection |= kProtectionWrite; if ((VmEntry->kve_protection & KVME_PROT_EXEC) != 0) - *protection |= kProtectionExecute; + segment->protection |= kProtectionExecute; - if (filename != NULL && filename_size > 0) { - internal_snprintf(filename, - Min(filename_size, (uptr)PATH_MAX), - "%s", VmEntry->kve_path); + if (segment->filename != NULL && segment->filename_size > 0) { + internal_snprintf(segment->filename, + Min(segment->filename_size, (uptr)PATH_MAX), "%s", + VmEntry->kve_path); } current_ += VmEntry->kve_structsize; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc index 7e4a44be95b6..1bcad2bf70e6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -26,41 +26,28 @@ static bool IsOneOf(char c, char c1, char c2) { return c == c1 || c == c2; } -bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, - char filename[], uptr filename_size, - uptr *protection, ModuleArch *arch, u8 *uuid) { - CHECK(!arch && "not implemented"); - CHECK(!uuid && "not implemented"); +bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { char *last = proc_self_maps_.data + proc_self_maps_.len; if (current_ >= last) return false; - uptr dummy; - if (!start) start = &dummy; - if (!end) end = &dummy; - if (!offset) offset = &dummy; - if (!protection) protection = &dummy; char *next_line = (char*)internal_memchr(current_, '\n', last - current_); if (next_line == 0) next_line = last; // Example: 08048000-08056000 r-xp 00000000 03:0c 64593 /foo/bar - *start = ParseHex(¤t_); + segment->start = ParseHex(¤t_); CHECK_EQ(*current_++, '-'); - *end = ParseHex(¤t_); + segment->end = ParseHex(¤t_); CHECK_EQ(*current_++, ' '); CHECK(IsOneOf(*current_, '-', 'r')); - *protection = 0; - if (*current_++ == 'r') - *protection |= kProtectionRead; + segment->protection = 0; + if (*current_++ == 'r') segment->protection |= kProtectionRead; CHECK(IsOneOf(*current_, '-', 'w')); - if (*current_++ == 'w') - *protection |= kProtectionWrite; + if (*current_++ == 'w') segment->protection |= kProtectionWrite; CHECK(IsOneOf(*current_, '-', 'x')); - if (*current_++ == 'x') - *protection |= kProtectionExecute; + if (*current_++ == 'x') segment->protection |= kProtectionExecute; CHECK(IsOneOf(*current_, 's', 'p')); - if (*current_++ == 's') - *protection |= kProtectionShared; + if (*current_++ == 's') segment->protection |= kProtectionShared; CHECK_EQ(*current_++, ' '); - *offset = ParseHex(¤t_); + segment->offset = ParseHex(¤t_); CHECK_EQ(*current_++, ' '); ParseHex(¤t_); CHECK_EQ(*current_++, ':'); @@ -75,14 +62,12 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, while (current_ < next_line && *current_ == ' ') current_++; // Fill in the filename. - uptr i = 0; - while (current_ < next_line) { - if (filename && i < filename_size - 1) - filename[i++] = *current_; - current_++; + if (segment->filename) { + uptr len = Min((uptr)(next_line - current_), segment->filename_size - 1); + internal_strncpy(segment->filename, current_, len); + segment->filename[len] = 0; } - if (filename && i < filename_size) - filename[i] = 0; + current_ = next_line + 1; return true; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 131017458d4c..560451a16d90 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -88,6 +88,48 @@ void MemoryMappingLayout::LoadFromCache() { // No-op on Mac for now. } +// _dyld_get_image_header() and related APIs don't report dyld itself. +// We work around this by manually recursing through the memory map +// until we hit a Mach header matching dyld instead. These recurse +// calls are expensive, but the first memory map generation occurs +// early in the process, when dyld is one of the only images loaded, +// so it will be hit after only a few iterations. +static mach_header *get_dyld_image_header() { + mach_port_name_t port; + if (task_for_pid(mach_task_self(), internal_getpid(), &port) != + KERN_SUCCESS) { + return nullptr; + } + + unsigned depth = 1; + vm_size_t size = 0; + vm_address_t address = 0; + kern_return_t err = KERN_SUCCESS; + mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64; + + while (true) { + struct vm_region_submap_info_64 info; + err = vm_region_recurse_64(port, &address, &size, &depth, + (vm_region_info_t)&info, &count); + if (err != KERN_SUCCESS) return nullptr; + + if (size >= sizeof(mach_header) && info.protection & kProtectionRead) { + mach_header *hdr = (mach_header *)address; + if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) && + hdr->filetype == MH_DYLINKER) { + return hdr; + } + } + address += size; + } +} + +const mach_header *get_dyld_hdr() { + if (!dyld_hdr) dyld_hdr = get_dyld_image_header(); + + return dyld_hdr; +} + // Next and NextSegmentLoad were inspired by base/sysinfo.cc in // Google Perftools, https://github.com/gperftools/gperftools. @@ -96,40 +138,39 @@ void MemoryMappingLayout::LoadFromCache() { // segment. // Note that the segment addresses are not necessarily sorted. template -bool MemoryMappingLayout::NextSegmentLoad(uptr *start, uptr *end, uptr *offset, - char filename[], uptr filename_size, - ModuleArch *arch, u8 *uuid, - uptr *protection) { +bool MemoryMappingLayout::NextSegmentLoad(MemoryMappedSegment *segment) { const char *lc = current_load_cmd_addr_; current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize; if (((const load_command *)lc)->cmd == kLCSegment) { const SegmentCommand* sc = (const SegmentCommand *)lc; - GetSegmentAddrRange(start, end, sc->vmaddr, sc->vmsize); - if (protection) { - // Return the initial protection. - *protection = sc->initprot; + + if (current_image_ == kDyldImageIdx) { + // vmaddr is masked with 0xfffff because on macOS versions < 10.12, + // it contains an absolute address rather than an offset for dyld. + // To make matters even more complicated, this absolute address + // isn't actually the absolute segment address, but the offset portion + // of the address is accurate when combined with the dyld base address, + // and the mask will give just this offset. + segment->start = (sc->vmaddr & 0xfffff) + (uptr)get_dyld_hdr(); + segment->end = (sc->vmaddr & 0xfffff) + sc->vmsize + (uptr)get_dyld_hdr(); + } else { + const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_); + segment->start = sc->vmaddr + dlloff; + segment->end = sc->vmaddr + sc->vmsize + dlloff; } - if (offset) { - if (current_filetype_ == /*MH_EXECUTE*/ 0x2) { - *offset = sc->vmaddr; - } else { - *offset = sc->fileoff; - } - } - if (filename) { - if (current_image_ == kDyldImageIdx) { - internal_strncpy(filename, kDyldPath, filename_size); - } else { - internal_strncpy(filename, _dyld_get_image_name(current_image_), - filename_size); - } - } - if (arch) { - *arch = current_arch_; - } - if (uuid) { - internal_memcpy(uuid, current_uuid_, kModuleUUIDSize); + + // Return the initial protection. + segment->protection = sc->initprot; + segment->offset = + (current_filetype_ == /*MH_EXECUTE*/ 0x2) ? sc->vmaddr : sc->fileoff; + if (segment->filename) { + const char *src = (current_image_ == kDyldImageIdx) + ? kDyldPath + : _dyld_get_image_name(current_image_); + internal_strncpy(segment->filename, src, segment->filename_size); } + segment->arch = current_arch_; + internal_memcpy(segment->uuid, current_uuid_, kModuleUUIDSize); return true; } return false; @@ -190,70 +231,7 @@ static bool IsModuleInstrumented(const load_command *first_lc) { return false; } -// _dyld_get_image_header() and related APIs don't report dyld itself. -// We work around this by manually recursing through the memory map -// until we hit a Mach header matching dyld instead. These recurse -// calls are expensive, but the first memory map generation occurs -// early in the process, when dyld is one of the only images loaded, -// so it will be hit after only a few iterations. -static mach_header *get_dyld_image_header() { - mach_port_name_t port; - if (task_for_pid(mach_task_self(), internal_getpid(), &port) != - KERN_SUCCESS) { - return nullptr; - } - - unsigned depth = 1; - vm_size_t size = 0; - vm_address_t address = 0; - kern_return_t err = KERN_SUCCESS; - mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64; - - while (true) { - struct vm_region_submap_info_64 info; - err = vm_region_recurse_64(port, &address, &size, &depth, - (vm_region_info_t)&info, &count); - if (err != KERN_SUCCESS) return nullptr; - - if (size >= sizeof(mach_header) && - info.protection & MemoryMappingLayout::kProtectionRead) { - mach_header *hdr = (mach_header *)address; - if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) && - hdr->filetype == MH_DYLINKER) { - return hdr; - } - } - address += size; - } -} - -const mach_header *get_dyld_hdr() { - if (!dyld_hdr) dyld_hdr = get_dyld_image_header(); - - return dyld_hdr; -} - -void MemoryMappingLayout::GetSegmentAddrRange(uptr *start, uptr *end, - uptr vmaddr, uptr vmsize) { - if (current_image_ == kDyldImageIdx) { - // vmaddr is masked with 0xfffff because on macOS versions < 10.12, - // it contains an absolute address rather than an offset for dyld. - // To make matters even more complicated, this absolute address - // isn't actually the absolute segment address, but the offset portion - // of the address is accurate when combined with the dyld base address, - // and the mask will give just this offset. - if (start) *start = (vmaddr & 0xfffff) + (uptr)get_dyld_hdr(); - if (end) *end = (vmaddr & 0xfffff) + vmsize + (uptr)get_dyld_hdr(); - } else { - const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_); - if (start) *start = vmaddr + dlloff; - if (end) *end = vmaddr + vmsize + dlloff; - } -} - -bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, - char filename[], uptr filename_size, - uptr *protection, ModuleArch *arch, u8 *uuid) { +bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { for (; current_image_ >= kDyldImageIdx; current_image_--) { const mach_header *hdr = (current_image_ == kDyldImageIdx) ? get_dyld_hdr() @@ -291,16 +269,13 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, #ifdef MH_MAGIC_64 case MH_MAGIC_64: { if (NextSegmentLoad( - start, end, offset, filename, filename_size, arch, uuid, - protection)) + segment)) return true; break; } #endif case MH_MAGIC: { - if (NextSegmentLoad( - start, end, offset, filename, filename_size, arch, uuid, - protection)) + if (NextSegmentLoad(segment)) return true; break; } @@ -315,28 +290,22 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, void MemoryMappingLayout::DumpListOfModules( InternalMmapVector *modules) { Reset(); - uptr cur_beg, cur_end, prot; - ModuleArch cur_arch; - u8 cur_uuid[kModuleUUIDSize]; InternalScopedString module_name(kMaxPathLength); - for (uptr i = 0; Next(&cur_beg, &cur_end, 0, module_name.data(), - module_name.size(), &prot, &cur_arch, &cur_uuid[0]); - i++) { - const char *cur_name = module_name.data(); - if (cur_name[0] == '\0') - continue; + MemoryMappedSegment segment(module_name.data(), kMaxPathLength); + for (uptr i = 0; Next(&segment); i++) { + if (segment.filename[0] == '\0') continue; LoadedModule *cur_module = nullptr; if (!modules->empty() && - 0 == internal_strcmp(cur_name, modules->back().full_name())) { + 0 == internal_strcmp(segment.filename, modules->back().full_name())) { cur_module = &modules->back(); } else { modules->push_back(LoadedModule()); cur_module = &modules->back(); - cur_module->set(cur_name, cur_beg, cur_arch, cur_uuid, - current_instrumented_); + cur_module->set(segment.filename, segment.start, segment.arch, + segment.uuid, current_instrumented_); } - cur_module->addAddressRange(cur_beg, cur_end, prot & kProtectionExecute, - prot & kProtectionWrite); + cur_module->addAddressRange(segment.start, segment.end, + segment.IsExecutable(), segment.IsWritable()); } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc index 36c98d057bd3..747a4a701728 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -43,7 +43,8 @@ void StackTrace::Print() const { if (dedup_frames-- > 0) { if (dedup_token.length()) dedup_token.append("--"); - dedup_token.append(cur->info.function); + if (cur->info.function != nullptr) + dedup_token.append(cur->info.function); } } frames->ClearAll(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc index 89d9cf61c3e4..de01e8d119a1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -291,7 +291,8 @@ void DontDumpShadowMemory(uptr addr, uptr length) { // FIXME: add madvise-analog when we move to 64-bits. } -uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) { +uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding, + uptr *largest_gap_found) { uptr address = 0; while (true) { MEMORY_BASIC_INFORMATION info; diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp index 00fa192181ad..ec9132f90a4f 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -264,7 +264,7 @@ ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) { ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder); } -Xorshift128Plus *getPrng(ScudoThreadContext *ThreadContext) { +ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) { return &ThreadContext->Prng; } @@ -283,7 +283,7 @@ struct ScudoAllocator { StaticSpinMutex FallbackMutex; AllocatorCache FallbackAllocatorCache; ScudoQuarantineCache FallbackQuarantineCache; - Xorshift128Plus FallbackPrng; + ScudoPrng FallbackPrng; bool DeallocationTypeMismatch; bool ZeroContents; @@ -333,8 +333,8 @@ struct ScudoAllocator { static_cast(Options.QuarantineSizeMb) << 20, static_cast(Options.ThreadLocalQuarantineSizeKb) << 10); BackendAllocator.InitCache(&FallbackAllocatorCache); - FallbackPrng.initFromURandom(); - Cookie = FallbackPrng.getNext(); + FallbackPrng.init(); + Cookie = FallbackPrng.getU64(); } // Helper function that checks for a valid Scudo chunk. nullptr isn't. @@ -373,19 +373,19 @@ struct ScudoAllocator { bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment); void *Ptr; - uptr Salt; + u8 Salt; uptr AllocationSize = FromPrimary ? AlignedSize : NeededSize; uptr AllocationAlignment = FromPrimary ? MinAlignment : Alignment; ScudoThreadContext *ThreadContext = getThreadContextAndLock(); if (LIKELY(ThreadContext)) { - Salt = getPrng(ThreadContext)->getNext(); + Salt = getPrng(ThreadContext)->getU8(); Ptr = BackendAllocator.Allocate(getAllocatorCache(ThreadContext), AllocationSize, AllocationAlignment, FromPrimary); ThreadContext->unlock(); } else { SpinMutexLock l(&FallbackMutex); - Salt = FallbackPrng.getNext(); + Salt = FallbackPrng.getU8(); Ptr = BackendAllocator.Allocate(&FallbackAllocatorCache, AllocationSize, AllocationAlignment, FromPrimary); } @@ -612,7 +612,7 @@ static void initScudoInternal(const AllocatorOptions &Options) { void ScudoThreadContext::init() { getBackendAllocator().InitCache(&Cache); - Prng.initFromURandom(); + Prng.init(); memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder)); } diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls.h b/contrib/compiler-rt/lib/scudo/scudo_tls.h index f6039bebec44..20c49204cf13 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_tls.h +++ b/contrib/compiler-rt/lib/scudo/scudo_tls.h @@ -30,7 +30,7 @@ namespace __scudo { struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform { AllocatorCache Cache; - Xorshift128Plus Prng; + ScudoPrng Prng; uptr QuarantineCachePlaceHolder[4]; void init(); void commitBack(); diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp index 31c391946c15..f7903ff34c73 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp @@ -123,40 +123,4 @@ bool testCPUFeature(CPUFeature Feature) { } #endif // defined(__x86_64__) || defined(__i386__) -// readRetry will attempt to read Count bytes from the Fd specified, and if -// interrupted will retry to read additional bytes to reach Count. -static ssize_t readRetry(int Fd, u8 *Buffer, size_t Count) { - ssize_t AmountRead = 0; - while (static_cast(AmountRead) < Count) { - ssize_t Result = read(Fd, Buffer + AmountRead, Count - AmountRead); - if (Result > 0) - AmountRead += Result; - else if (!Result) - break; - else if (errno != EINTR) { - AmountRead = -1; - break; - } - } - return AmountRead; -} - -static void fillRandom(u8 *Data, ssize_t Size) { - int Fd = open("/dev/urandom", O_RDONLY); - if (Fd < 0) { - dieWithMessage("ERROR: failed to open /dev/urandom.\n"); - } - bool Success = readRetry(Fd, Data, Size) == Size; - close(Fd); - if (!Success) { - dieWithMessage("ERROR: failed to read enough data from /dev/urandom.\n"); - } -} - -// Seeds the xorshift state with /dev/urandom. -// TODO(kostyak): investigate using getrandom() if available. -void Xorshift128Plus::initFromURandom() { - fillRandom(reinterpret_cast(State), sizeof(State)); -} - } // namespace __scudo diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.h b/contrib/compiler-rt/lib/scudo/scudo_utils.h index 7198476f42cf..6c6c9d893404 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_utils.h +++ b/contrib/compiler-rt/lib/scudo/scudo_utils.h @@ -36,23 +36,58 @@ enum CPUFeature { }; bool testCPUFeature(CPUFeature feature); -// Tiny PRNG based on https://en.wikipedia.org/wiki/Xorshift#xorshift.2B -// The state (128 bits) will be stored in thread local storage. -struct Xorshift128Plus { +INLINE u64 rotl(const u64 X, int K) { + return (X << K) | (X >> (64 - K)); +} + +// XoRoShiRo128+ PRNG (http://xoroshiro.di.unimi.it/). +struct XoRoShiRo128Plus { public: - void initFromURandom(); - u64 getNext() { - u64 x = State[0]; - const u64 y = State[1]; - State[0] = y; - x ^= x << 23; - State[1] = x ^ y ^ (x >> 17) ^ (y >> 26); - return State[1] + y; + void init() { + if (UNLIKELY(!GetRandom(reinterpret_cast(State), sizeof(State)))) { + // Early processes (eg: init) do not have /dev/urandom yet, but we still + // have to provide them with some degree of entropy. Not having a secure + // seed is not as problematic for them, as they are less likely to be + // the target of heap based vulnerabilities exploitation attempts. + State[0] = NanoTime(); + State[1] = 0; + } + fillCache(); } + u8 getU8() { + if (UNLIKELY(isCacheEmpty())) + fillCache(); + const u8 Result = static_cast(CachedBytes & 0xff); + CachedBytes >>= 8; + CachedBytesAvailable--; + return Result; + } + u64 getU64() { return next(); } + private: + u8 CachedBytesAvailable; + u64 CachedBytes; u64 State[2]; + u64 next() { + const u64 S0 = State[0]; + u64 S1 = State[1]; + const u64 Result = S0 + S1; + S1 ^= S0; + State[0] = rotl(S0, 55) ^ S1 ^ (S1 << 14); + State[1] = rotl(S1, 36); + return Result; + } + bool isCacheEmpty() { + return CachedBytesAvailable == 0; + } + void fillCache() { + CachedBytes = next(); + CachedBytesAvailable = sizeof(CachedBytes); + } }; +typedef XoRoShiRo128Plus ScudoPrng; + } // namespace __scudo #endif // SCUDO_UTILS_H_ diff --git a/contrib/compiler-rt/lib/tsan/dd/dd_interceptors.cc b/contrib/compiler-rt/lib/tsan/dd/dd_interceptors.cc index 97c72dd2b7fd..a39218f0454b 100644 --- a/contrib/compiler-rt/lib/tsan/dd/dd_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/dd/dd_interceptors.cc @@ -270,20 +270,19 @@ namespace __dsan { static void InitDataSeg() { MemoryMappingLayout proc_maps(true); - uptr start, end, offset; char name[128]; + MemoryMappedSegment segment(name, ARRAY_SIZE(name)); bool prev_is_data = false; - while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), - /*protection*/ 0)) { - bool is_data = offset != 0 && name[0] != 0; + while (proc_maps.Next(&segment)) { + bool is_data = segment.offset != 0 && segment.filename[0] != 0; // BSS may get merged with [heap] in /proc/self/maps. This is not very // reliable. - bool is_bss = offset == 0 && - (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data; - if (g_data_start == 0 && is_data) - g_data_start = start; - if (is_bss) - g_data_end = end; + bool is_bss = segment.offset == 0 && + (segment.filename[0] == 0 || + internal_strcmp(segment.filename, "[heap]") == 0) && + prev_is_data; + if (g_data_start == 0 && is_data) g_data_start = segment.start; + if (is_bss) g_data_end = segment.end; prev_is_data = is_data; } VPrintf(1, "guessed data_start=%p data_end=%p\n", g_data_start, g_data_end); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc index 32435adfdf33..9ee9104283f8 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.cc @@ -101,6 +101,9 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused) clk_[tid_].reused = reused_; } +void ThreadClock::ResetCached(ClockCache *c) { +} + void ThreadClock::acquire(ClockCache *c, const SyncClock *src) { DCHECK_LE(nclk_, kMaxTid); DCHECK_LE(src->size_, kMaxTid); @@ -116,9 +119,7 @@ void ThreadClock::acquire(ClockCache *c, const SyncClock *src) { // Check if we've already acquired src after the last release operation on src bool acquired = false; if (nclk > tid_) { - CPP_STAT_INC(StatClockAcquireLarge); if (src->elem(tid_).reused == reused_) { - CPP_STAT_INC(StatClockAcquireRepeat); for (unsigned i = 0; i < kDirtyTids; i++) { unsigned tid = src->dirty_tids_[i]; if (tid != kInvalidTid) { @@ -266,11 +267,11 @@ void ThreadClock::UpdateCurrentThread(SyncClock *dst) const { for (unsigned i = 0; i < kDirtyTids; i++) { if (dst->dirty_tids_[i] == tid_) { - CPP_STAT_INC(StatClockReleaseFast1); + CPP_STAT_INC(StatClockReleaseFast); return; } if (dst->dirty_tids_[i] == kInvalidTid) { - CPP_STAT_INC(StatClockReleaseFast2); + CPP_STAT_INC(StatClockReleaseFast); dst->dirty_tids_[i] = tid_; return; } @@ -297,6 +298,64 @@ bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const { return true; } +// Sets a single element in the vector clock. +// This function is called only from weird places like AcquireGlobal. +void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) { + DCHECK_LT(tid, kMaxTid); + DCHECK_GE(v, clk_[tid].epoch); + clk_[tid].epoch = v; + if (nclk_ <= tid) + nclk_ = tid + 1; + last_acquire_ = clk_[tid_].epoch; +} + +void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) { + printf("clock=["); + for (uptr i = 0; i < nclk_; i++) + printf("%s%llu", i == 0 ? "" : ",", clk_[i].epoch); + printf("] reused=["); + for (uptr i = 0; i < nclk_; i++) + printf("%s%llu", i == 0 ? "" : ",", clk_[i].reused); + printf("] tid=%u/%u last_acq=%llu", + tid_, reused_, last_acquire_); +} + +SyncClock::SyncClock() { + ResetImpl(); +} + +SyncClock::~SyncClock() { + // Reset must be called before dtor. + CHECK_EQ(size_, 0); + CHECK_EQ(tab_, 0); + CHECK_EQ(tab_idx_, 0); +} + +void SyncClock::Reset(ClockCache *c) { + if (size_ == 0) { + // nothing + } else if (size_ <= ClockBlock::kClockCount) { + // One-level table. + ctx->clock_alloc.Free(c, tab_idx_); + } else { + // Two-level table. + for (uptr i = 0; i < size_; i += ClockBlock::kClockCount) + ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]); + ctx->clock_alloc.Free(c, tab_idx_); + } + ResetImpl(); +} + +void SyncClock::ResetImpl() { + tab_ = 0; + tab_idx_ = 0; + size_ = 0; + release_store_tid_ = kInvalidTid; + release_store_reused_ = 0; + for (uptr i = 0; i < kDirtyTids; i++) + dirty_tids_[i] = kInvalidTid; +} + void SyncClock::Resize(ClockCache *c, uptr nclk) { CPP_STAT_INC(StatClockReleaseResize); if (RoundUpTo(nclk, ClockBlock::kClockCount) <= @@ -344,66 +403,6 @@ void SyncClock::Resize(ClockCache *c, uptr nclk) { size_ = nclk; } -// Sets a single element in the vector clock. -// This function is called only from weird places like AcquireGlobal. -void ThreadClock::set(unsigned tid, u64 v) { - DCHECK_LT(tid, kMaxTid); - DCHECK_GE(v, clk_[tid].epoch); - clk_[tid].epoch = v; - if (nclk_ <= tid) - nclk_ = tid + 1; - last_acquire_ = clk_[tid_].epoch; -} - -void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) { - printf("clock=["); - for (uptr i = 0; i < nclk_; i++) - printf("%s%llu", i == 0 ? "" : ",", clk_[i].epoch); - printf("] reused=["); - for (uptr i = 0; i < nclk_; i++) - printf("%s%llu", i == 0 ? "" : ",", clk_[i].reused); - printf("] tid=%u/%u last_acq=%llu", - tid_, reused_, last_acquire_); -} - -SyncClock::SyncClock() - : release_store_tid_(kInvalidTid) - , release_store_reused_() - , tab_() - , tab_idx_() - , size_() { - for (uptr i = 0; i < kDirtyTids; i++) - dirty_tids_[i] = kInvalidTid; -} - -SyncClock::~SyncClock() { - // Reset must be called before dtor. - CHECK_EQ(size_, 0); - CHECK_EQ(tab_, 0); - CHECK_EQ(tab_idx_, 0); -} - -void SyncClock::Reset(ClockCache *c) { - if (size_ == 0) { - // nothing - } else if (size_ <= ClockBlock::kClockCount) { - // One-level table. - ctx->clock_alloc.Free(c, tab_idx_); - } else { - // Two-level table. - for (uptr i = 0; i < size_; i += ClockBlock::kClockCount) - ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]); - ctx->clock_alloc.Free(c, tab_idx_); - } - tab_ = 0; - tab_idx_ = 0; - size_ = 0; - release_store_tid_ = kInvalidTid; - release_store_reused_ = 0; - for (uptr i = 0; i < kDirtyTids; i++) - dirty_tids_[i] = kInvalidTid; -} - ClockElem &SyncClock::elem(unsigned tid) const { DCHECK_LT(tid, size_); if (size_ <= ClockBlock::kClockCount) diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.h index 4e352cb81d11..378b550fd11b 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_clock.h @@ -74,6 +74,7 @@ class SyncClock { u32 tab_idx_; u32 size_; + void ResetImpl(); ClockElem &elem(unsigned tid) const; }; @@ -89,7 +90,7 @@ struct ThreadClock { return clk_[tid].epoch; } - void set(unsigned tid, u64 v); + void set(ClockCache *c, unsigned tid, u64 v); void set(u64 v) { DCHECK_GE(v, clk_[tid_].epoch); @@ -108,6 +109,7 @@ struct ThreadClock { void release(ClockCache *c, SyncClock *dst) const; void acq_rel(ClockCache *c, SyncClock *dst); void ReleaseStore(ClockCache *c, SyncClock *dst) const; + void ResetCached(ClockCache *c); void DebugReset(); void DebugDump(int(*printf)(const char *s, ...)); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h index e9815c90a953..16dbdf391085 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h @@ -39,7 +39,7 @@ class DenseSlabAlloc { typedef DenseSlabAllocCache Cache; typedef typename Cache::IndexT IndexT; - DenseSlabAlloc() { + explicit DenseSlabAlloc(const char *name) { // Check that kL1Size and kL2Size are sane. CHECK_EQ(kL1Size & (kL1Size - 1), 0); CHECK_EQ(kL2Size & (kL2Size - 1), 0); @@ -49,6 +49,7 @@ class DenseSlabAlloc { internal_memset(map_, 0, sizeof(map_)); freelist_ = 0; fillpos_ = 0; + name_ = name; } ~DenseSlabAlloc() { @@ -96,15 +97,19 @@ class DenseSlabAlloc { SpinMutex mtx_; IndexT freelist_; uptr fillpos_; + const char *name_; void Refill(Cache *c) { SpinMutexLock lock(&mtx_); if (freelist_ == 0) { if (fillpos_ == kL1Size) { - Printf("ThreadSanitizer: DenseSlabAllocator overflow. Dying.\n"); + Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", + name_, kL1Size, kL2Size); Die(); } - T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), "DenseSlabAllocator"); + VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", + name_, fillpos_, kL1Size, kL2Size); + T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_); // Reserve 0 as invalid index. IndexT start = fillpos_ == 0 ? 1 : 0; for (IndexT i = start; i < kL2Size; i++) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 334cc326daf6..001123f4941e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" @@ -34,13 +35,11 @@ using namespace __tsan; // NOLINT #if SANITIZER_FREEBSD || SANITIZER_MAC -#define __errno_location __error #define stdout __stdoutp #define stderr __stderrp #endif #if SANITIZER_ANDROID -#define __errno_location __errno #define mallopt(a, b) #endif @@ -84,7 +83,6 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); -extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID @@ -98,9 +96,6 @@ const int PTHREAD_MUTEX_RECURSIVE_NP = 1; const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif -const int EINVAL = 22; -const int EBUSY = 16; -const int EOWNERDEAD = 130; #if !SANITIZER_FREEBSD && !SANITIZER_MAC const int EPOLL_CTL_ADD = 1; #endif @@ -130,8 +125,6 @@ typedef long long_t; // NOLINT # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ -#define errno (*__errno_location()) - typedef void (*sighandler_t)(int sig); typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); @@ -268,7 +261,7 @@ ScopedInterceptor::~ScopedInterceptor() { void ScopedInterceptor::EnableIgnores() { if (ignoring_) { - ThreadIgnoreBegin(thr_, pc_, false); + ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++; if (in_ignored_lib_) { DCHECK(!thr_->in_ignored_lib); @@ -466,8 +459,14 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { static void LongJmp(ThreadState *thr, uptr *env) { #ifdef __powerpc__ uptr mangled_sp = env[0]; -#elif SANITIZER_FREEBSD || SANITIZER_MAC +#elif SANITIZER_FREEBSD uptr mangled_sp = env[2]; +#elif SANITIZER_MAC +# ifdef __aarch64__ + uptr mangled_sp = env[13]; +# else + uptr mangled_sp = env[2]; +# endif #elif defined(SANITIZER_LINUX) # ifdef __aarch64__ uptr mangled_sp = env[13]; @@ -665,7 +664,7 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) { if (*addr) { if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { if (flags & MAP_FIXED) { - errno = EINVAL; + errno = errno_EINVAL; return false; } else { *addr = 0; @@ -1122,7 +1121,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); int res = REAL(pthread_mutex_destroy)(m); - if (res == 0 || res == EBUSY) { + if (res == 0 || res == errno_EBUSY) { MutexDestroy(thr, pc, (uptr)m); } return res; @@ -1131,9 +1130,9 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); int res = REAL(pthread_mutex_trylock)(m); - if (res == EOWNERDEAD) + if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); - if (res == 0 || res == EOWNERDEAD) + if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); return res; } @@ -1311,7 +1310,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); if (o == 0 || f == 0) - return EINVAL; + return errno_EINVAL; atomic_uint32_t *a; if (!SANITIZER_MAC) a = static_cast(o); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc index f6bf8a0e586b..4f1079467331 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -21,7 +21,10 @@ #include "tsan_interface_ann.h" #include + +#if defined(__has_include) && __has_include() #include +#endif // #if defined(__has_include) && __has_include() typedef long long_t; // NOLINT @@ -235,6 +238,8 @@ TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { REAL(os_lock_unlock)(lock); } +#if defined(__has_include) && __has_include() + TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, xpc_connection_t connection, xpc_handler_t handler) { SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, @@ -287,6 +292,8 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { REAL(xpc_connection_cancel)(connection); } +#endif // #if defined(__has_include) && __has_include() + // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc index 45ec45bbdbbe..f68a0468de53 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc @@ -483,8 +483,8 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) { else MutexPreLock(thr, pc, (uptr)m); } - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); } INTERFACE_ATTRIBUTE @@ -510,8 +510,8 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) { } else { ret = MutexUnlock(thr, pc, (uptr)m, flagz); } - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); return ret; } @@ -525,8 +525,8 @@ void __tsan_mutex_post_unlock(void *m, unsigned flagz) { INTERFACE_ATTRIBUTE void __tsan_mutex_pre_signal(void *addr, unsigned flagz) { SCOPED_ANNOTATION(__tsan_mutex_pre_signal); - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); } INTERFACE_ATTRIBUTE @@ -547,7 +547,7 @@ void __tsan_mutex_pre_divert(void *addr, unsigned flagz) { INTERFACE_ATTRIBUTE void __tsan_mutex_post_divert(void *addr, unsigned flagz) { SCOPED_ANNOTATION(__tsan_mutex_post_divert); - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); } } // extern "C" diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc index b22d5c1ecef8..d334394f5303 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc @@ -220,8 +220,7 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { #endif template -static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, - morder mo) { +static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { CHECK(IsLoadOrder(mo)); // This fast-path is critical for performance. // Assume the access is atomic. @@ -229,10 +228,17 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, MemoryReadAtomic(thr, pc, (uptr)a, SizeLog()); return NoTsanAtomicLoad(a, mo); } - SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false); - AcquireImpl(thr, pc, &s->clock); + // Don't create sync object if it does not exist yet. For example, an atomic + // pointer is initialized to nullptr and then periodically acquire-loaded. T v = NoTsanAtomicLoad(a, mo); - s->mtx.ReadUnlock(); + SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false); + if (s) { + AcquireImpl(thr, pc, &s->clock); + // Re-read under sync mutex because we need a consistent snapshot + // of the value and the clock we acquire. + v = NoTsanAtomicLoad(a, mo); + s->mtx.ReadUnlock(); + } MemoryReadAtomic(thr, pc, (uptr)a, SizeLog()); return v; } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc index 7169d5b02c04..1434cf688ce9 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc @@ -294,6 +294,8 @@ uptr __sanitizer_get_allocated_size(const void *p) { void __tsan_on_thread_idle() { ThreadState *thr = cur_thread(); + thr->clock.ResetCached(&thr->proc()->clock_cache); + thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); allocator()->SwallowCache(&thr->proc()->alloc_cache); internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache); ctx->metamap.OnProcIdle(thr->proc()); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h index 60d9b9d8c452..bea1daba3952 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -100,6 +100,37 @@ struct Mapping { }; #define TSAN_MID_APP_RANGE 1 +#elif defined(__aarch64__) && defined(__APPLE__) +/* +C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) +0000 0000 00 - 0100 0000 00: - (4 GB) +0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) +0200 0000 00 - 0300 0000 00: heap (4 GB) +0300 0000 00 - 0400 0000 00: - (4 GB) +0400 0000 00 - 0c00 0000 00: shadow memory (32 GB) +0c00 0000 00 - 0d00 0000 00: - (4 GB) +0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) +0e00 0000 00 - 0f00 0000 00: - (4 GB) +0f00 0000 00 - 1000 0000 00: traces (4 GB) +*/ +struct Mapping { + static const uptr kLoAppMemBeg = 0x0100000000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kHeapMemBeg = 0x0200000000ull; + static const uptr kHeapMemEnd = 0x0300000000ull; + static const uptr kShadowBeg = 0x0400000000ull; + static const uptr kShadowEnd = 0x0c00000000ull; + static const uptr kMetaShadowBeg = 0x0d00000000ull; + static const uptr kMetaShadowEnd = 0x0e00000000ull; + static const uptr kTraceMemBeg = 0x0f00000000ull; + static const uptr kTraceMemEnd = 0x1000000000ull; + static const uptr kHiAppMemBeg = 0x1000000000ull; + static const uptr kHiAppMemEnd = 0x1000000000ull; + static const uptr kAppMemMsk = 0x0ull; + static const uptr kAppMemXor = 0x0ull; + static const uptr kVdsoBeg = 0x7000000000000000ull; +}; + #elif defined(__aarch64__) // AArch64 supports multiple VMA which leads to multiple address transformation // functions. To support these multiple VMAS transformations and mappings TSAN @@ -389,7 +420,7 @@ uptr MappingImpl(void) { template uptr MappingArchImpl(void) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return MappingImpl(); case 42: return MappingImpl(); @@ -542,7 +573,7 @@ bool IsAppMemImpl(uptr mem) { ALWAYS_INLINE bool IsAppMem(uptr mem) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return IsAppMemImpl(mem); case 42: return IsAppMemImpl(mem); @@ -569,7 +600,7 @@ bool IsShadowMemImpl(uptr mem) { ALWAYS_INLINE bool IsShadowMem(uptr mem) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return IsShadowMemImpl(mem); case 42: return IsShadowMemImpl(mem); @@ -596,7 +627,7 @@ bool IsMetaMemImpl(uptr mem) { ALWAYS_INLINE bool IsMetaMem(uptr mem) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return IsMetaMemImpl(mem); case 42: return IsMetaMemImpl(mem); @@ -633,7 +664,7 @@ uptr MemToShadowImpl(uptr x) { ALWAYS_INLINE uptr MemToShadow(uptr x) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return MemToShadowImpl(x); case 42: return MemToShadowImpl(x); @@ -672,7 +703,7 @@ u32 *MemToMetaImpl(uptr x) { ALWAYS_INLINE u32 *MemToMeta(uptr x) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return MemToMetaImpl(x); case 42: return MemToMetaImpl(x); @@ -724,7 +755,7 @@ uptr ShadowToMemImpl(uptr s) { ALWAYS_INLINE uptr ShadowToMem(uptr s) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return ShadowToMemImpl(s); case 42: return ShadowToMemImpl(s); @@ -759,7 +790,7 @@ uptr GetThreadTraceImpl(int tid) { ALWAYS_INLINE uptr GetThreadTrace(int tid) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return GetThreadTraceImpl(tid); case 42: return GetThreadTraceImpl(tid); @@ -789,7 +820,7 @@ uptr GetThreadTraceHeaderImpl(int tid) { ALWAYS_INLINE uptr GetThreadTraceHeader(int tid) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return GetThreadTraceHeaderImpl(tid); case 42: return GetThreadTraceHeaderImpl(tid); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc index d05c0e701e72..0ba01babe69a 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #if SANITIZER_LINUX @@ -182,17 +181,15 @@ static void MapRodata() { } // Map the file into shadow of .rodata sections. MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr start, end, offset, prot; // Reusing the buffer 'name'. - while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) { - if (name[0] != 0 && name[0] != '[' - && (prot & MemoryMappingLayout::kProtectionRead) - && (prot & MemoryMappingLayout::kProtectionExecute) - && !(prot & MemoryMappingLayout::kProtectionWrite) - && IsAppMem(start)) { + MemoryMappedSegment segment(name, ARRAY_SIZE(name)); + while (proc_maps.Next(&segment)) { + if (segment.filename[0] != 0 && segment.filename[0] != '[' && + segment.IsReadable() && segment.IsExecutable() && + !segment.IsWritable() && IsAppMem(segment.start)) { // Assume it's .rodata - char *shadow_start = (char*)MemToShadow(start); - char *shadow_end = (char*)MemToShadow(end); + char *shadow_start = (char *)MemToShadow(segment.start); + char *shadow_end = (char *)MemToShadow(segment.end); for (char *p = shadow_start; p < shadow_end; p += marker.size()) { internal_mmap(p, Min(marker.size(), shadow_end - p), PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc index a82bcd01bbf4..73a656ffca5e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc @@ -230,6 +230,14 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, #endif void InitializePlatformEarly() { +#if defined(__aarch64__) + uptr max_vm = GetMaxVirtualAddress() + 1; + if (max_vm != Mapping::kHiAppMemEnd) { + Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n", + max_vm, Mapping::kHiAppMemEnd); + Die(); + } +#endif } void InitializePlatform() { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cc index 0732c83d689d..e4f90a811c35 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cc @@ -46,6 +46,9 @@ void InitializeShadowMemory() { #elif defined(__mips64) const uptr kMadviseRangeBeg = 0xff00000000ull; const uptr kMadviseRangeSize = 0x0100000000ull; +#elif defined(__aarch64__) && defined(__APPLE__) + uptr kMadviseRangeBeg = LoAppMemBeg(); + uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg(); #elif defined(__aarch64__) uptr kMadviseRangeBeg = 0; uptr kMadviseRangeSize = 0; @@ -115,21 +118,24 @@ static void ProtectRange(uptr beg, uptr end) { void CheckAndProtect() { // Ensure that the binary is indeed compiled with -pie. MemoryMappingLayout proc_maps(true); - uptr p, end, prot; - while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) { - if (IsAppMem(p)) + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (IsAppMem(segment.start)) continue; + if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue; + if (segment.protection == 0) // Zero page or mprotected. continue; - if (p >= HeapMemEnd() && - p < HeapEnd()) - continue; - if (prot == 0) // Zero page or mprotected. - continue; - if (p >= VdsoBeg()) // vdso + if (segment.start >= VdsoBeg()) // vdso break; - Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end); + Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", + segment.start, segment.end); Die(); } +#if defined(__aarch64__) && defined(__APPLE__) + ProtectRange(HeapMemEnd(), ShadowBeg()); + ProtectRange(ShadowEnd(), MetaShadowBeg()); + ProtectRange(MetaShadowEnd(), TraceMemBeg()); +#else ProtectRange(LoAppMemEnd(), ShadowBeg()); ProtectRange(ShadowEnd(), MetaShadowBeg()); #ifdef TSAN_MID_APP_RANGE @@ -143,6 +149,7 @@ void CheckAndProtect() { ProtectRange(TraceMemBeg(), TraceMemEnd()); ProtectRange(TraceMemEnd(), HeapMemBeg()); ProtectRange(HeapEnd(), HiAppMemBeg()); +#endif } #endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index fa60f3247c38..a01525302b02 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -104,7 +104,8 @@ Context::Context() , racy_stacks(MBlockRacyStacks) , racy_addresses(MBlockRacyAddresses) , fired_suppressions_mtx(MutexTypeFired, StatMtxFired) - , fired_suppressions(8) { + , fired_suppressions(8) + , clock_alloc("clock allocator") { } // The objects are allocated in TLS, so one may rely on zero-initialization. diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S index c2a69816be2f..f8c6453a5da4 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -1,13 +1,46 @@ +// The content of this file is AArch64-only: +#if defined(__aarch64__) + #include "sanitizer_common/sanitizer_asm.h" +#if !defined(__APPLE__) .section .bss .type __tsan_pointer_chk_guard, %object -.size __tsan_pointer_chk_guard, 8 +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) __tsan_pointer_chk_guard: .zero 8 +#endif +#if defined(__APPLE__) +.align 2 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long _setjmp$non_lazy_ptr +_setjmp$non_lazy_ptr: +.indirect_symbol _setjmp +.long 0 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long __setjmp$non_lazy_ptr +__setjmp$non_lazy_ptr: +.indirect_symbol __setjmp +.long 0 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long _sigsetjmp$non_lazy_ptr +_sigsetjmp$non_lazy_ptr: +.indirect_symbol _sigsetjmp +.long 0 +#endif + +#if !defined(__APPLE__) .section .text +#else +.section __TEXT,__text +.align 3 +#endif +#if !defined(__APPLE__) // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp // functions) by XORing them with a random guard pointer. For AArch64 it is a // global variable rather than a TCB one (as for x86_64/powerpc) and althought @@ -16,9 +49,9 @@ __tsan_pointer_chk_guard: // not stable). So InitializeGuardPtr obtains the pointer guard value by // issuing a setjmp and checking the resulting pointers values against the // original ones. -.hidden _Z18InitializeGuardPtrv +ASM_HIDDEN(_Z18InitializeGuardPtrv) .global _Z18InitializeGuardPtrv -.type _Z18InitializeGuardPtrv, @function +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) _Z18InitializeGuardPtrv: CFI_STARTPROC // Allocates a jmp_buf for the setjmp call. @@ -55,12 +88,14 @@ _Z18InitializeGuardPtrv: CFI_DEF_CFA (31, 0) ret CFI_ENDPROC -.size _Z18InitializeGuardPtrv, .-_Z18InitializeGuardPtrv +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) +#endif -.hidden __tsan_setjmp +ASM_HIDDEN(__tsan_setjmp) .comm _ZN14__interception11real_setjmpE,8,8 -.type setjmp, @function -setjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC // save env parameters for function call @@ -78,14 +113,19 @@ setjmp: CFI_OFFSET (19, -16) mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#else + add x0, x29, 32 + mov x1, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter mov x0, x19 @@ -96,18 +136,24 @@ setjmp: CFI_DEF_CFA (31, 0) // tail jump to libc setjmp +#if !defined(__APPLE__) adrp x1, :got:_ZN14__interception11real_setjmpE ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE] ldr x1, [x1] +#else + adrp x1, _setjmp$non_lazy_ptr@page + add x1, x1, _setjmp$non_lazy_ptr@pageoff + ldr x1, [x1] +#endif br x1 CFI_ENDPROC -.size setjmp, .-setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) .comm _ZN14__interception12real__setjmpE,8,8 -.globl _setjmp -.type _setjmp, @function -_setjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameters for function call @@ -125,14 +171,19 @@ _setjmp: CFI_OFFSET (19, -16) mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#else + add x0, x29, 32 + mov x1, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) // Restore jmp_buf parameter mov x0, x19 @@ -143,18 +194,24 @@ _setjmp: CFI_DEF_CFA (31, 0) // tail jump to libc setjmp +#if !defined(__APPLE__) adrp x1, :got:_ZN14__interception12real__setjmpE ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE] ldr x1, [x1] +#else + adrp x1, __setjmp$non_lazy_ptr@page + add x1, x1, __setjmp$non_lazy_ptr@pageoff + ldr x1, [x1] +#endif br x1 CFI_ENDPROC -.size _setjmp, .-_setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl sigsetjmp -.type sigsetjmp, @function -sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -174,14 +231,19 @@ sigsetjmp: mov w20, w1 mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#else + add x0, x29, 32 + mov x1, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter mov w1, w20 @@ -195,17 +257,24 @@ sigsetjmp: CFI_DEF_CFA (31, 0) // tail jump to libc sigsetjmp +#if !defined(__APPLE__) adrp x2, :got:_ZN14__interception14real_sigsetjmpE ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE] ldr x2, [x2] +#else + adrp x2, _sigsetjmp$non_lazy_ptr@page + add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff + ldr x2, [x2] +#endif br x2 CFI_ENDPROC -.size sigsetjmp, .-sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +#if !defined(__APPLE__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl __sigsetjmp -.type __sigsetjmp, @function -__sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -225,14 +294,16 @@ __sigsetjmp: mov w20, w1 mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) mov w1, w20 mov x0, x19 @@ -245,14 +316,22 @@ __sigsetjmp: CFI_DEF_CFA (31, 0) // tail jump to libc __sigsetjmp +#if !defined(__APPLE__) adrp x2, :got:_ZN14__interception16real___sigsetjmpE ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE] ldr x2, [x2] +#else + adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page + add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff +#endif br x2 CFI_ENDPROC -.size __sigsetjmp, .-__sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif #if defined(__FreeBSD__) || defined(__linux__) /* We do not need executable stack. */ .section .note.GNU-stack,"",@progbits #endif + +#endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S index caa832375e52..98947fd2a1ba 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S @@ -1,4 +1,8 @@ +// The content of this file is x86_64-only: +#if defined(__x86_64__) + #include "sanitizer_common/sanitizer_asm.h" + #if !defined(__APPLE__) .section .text #else @@ -357,3 +361,5 @@ ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) /* We do not need executable stack. */ .section .note.GNU-stack,"",@progbits #endif + +#endif diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc index 54938f37e243..2f85811620f1 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -413,10 +413,10 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) { static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) { ThreadState *thr = reinterpret_cast(arg); ThreadContext *tctx = static_cast(tctx_base); + u64 epoch = tctx->epoch1; if (tctx->status == ThreadStatusRunning) - thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch()); - else - thr->clock.set(tctx->tid, tctx->epoch1); + epoch = tctx->thr->fast_state.epoch(); + thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); } void AcquireGlobal(ThreadState *thr, uptr pc) { @@ -456,10 +456,10 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { ThreadState *thr = reinterpret_cast(arg); ThreadContext *tctx = static_cast(tctx_base); + u64 epoch = tctx->epoch1; if (tctx->status == ThreadStatusRunning) - thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch()); - else - thr->last_sleep_clock.set(tctx->tid, tctx->epoch1); + epoch = tctx->thr->fast_state.epoch(); + thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); } void AfterSleep(ThreadState *thr, uptr pc) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc index 68b9f50308ea..85a982941ed7 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc @@ -314,7 +314,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) { return; #if !SANITIZER_GO int fd = -1; - int creat_tid = -1; + int creat_tid = kInvalidTid; u32 creat_stack = 0; if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) { ReportLocation *loc = ReportLocation::New(ReportLocationFD); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc index 67eebf5d0c38..83fab082afe3 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc @@ -142,6 +142,10 @@ void ThreadContext::OnFinished() { if (common_flags()->detect_deadlocks) ctx->dd->DestroyLogicalThread(thr->dd_lt); + thr->clock.ResetCached(&thr->proc()->clock_cache); +#if !SANITIZER_GO + thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); +#endif thr->~ThreadState(); #if TSAN_COLLECT_STATS StatAggregate(ctx->stat, thr->stat); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc index 2ee688bf5771..18c83d5c6dac 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.cc @@ -75,14 +75,11 @@ void StatOutput(u64 *stat) { name[StatClockAcquire] = "Clock acquire "; name[StatClockAcquireEmpty] = " empty clock "; name[StatClockAcquireFastRelease] = " fast from release-store "; - name[StatClockAcquireLarge] = " contains my tid "; - name[StatClockAcquireRepeat] = " repeated (fast) "; name[StatClockAcquireFull] = " full (slow) "; name[StatClockAcquiredSomething] = " acquired something "; name[StatClockRelease] = "Clock release "; name[StatClockReleaseResize] = " resize "; - name[StatClockReleaseFast1] = " fast1 "; - name[StatClockReleaseFast2] = " fast2 "; + name[StatClockReleaseFast] = " fast "; name[StatClockReleaseSlow] = " dirty overflow (slow) "; name[StatClockReleaseFull] = " full (slow) "; name[StatClockReleaseAcquired] = " was acquired "; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.h index 7d2791ebbfcc..42d6a2b63532 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_stat.h @@ -74,15 +74,12 @@ enum StatType { StatClockAcquire, StatClockAcquireEmpty, StatClockAcquireFastRelease, - StatClockAcquireLarge, - StatClockAcquireRepeat, StatClockAcquireFull, StatClockAcquiredSomething, // Clocks - release. StatClockRelease, StatClockReleaseResize, - StatClockReleaseFast1, - StatClockReleaseFast2, + StatClockReleaseFast, StatClockReleaseSlow, StatClockReleaseFull, StatClockReleaseAcquired, diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.cc index 4cc3cb89c34f..44ae558fa1b2 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.cc @@ -53,7 +53,9 @@ void SyncVar::Reset(Processor *proc) { } } -MetaMap::MetaMap() { +MetaMap::MetaMap() + : block_alloc_("heap block allocator") + , sync_alloc_("sync allocator") { atomic_store(&uid_gen_, 0, memory_order_relaxed); } diff --git a/contrib/libc++/include/cmath b/contrib/libc++/include/cmath index b41f247af59c..917928a1f42f 100644 --- a/contrib/libc++/include/cmath +++ b/contrib/libc++/include/cmath @@ -549,7 +549,7 @@ hypot(_A1 __lcpp_x, _A2 __lcpp_y, _A3 __lcpp_z) _NOEXCEPT template _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR typename enable_if::value, bool>::type -__libcpp_isnan(_A1 __lcpp_x) _NOEXCEPT +__libcpp_isnan_or_builtin(_A1 __lcpp_x) _NOEXCEPT { #if __has_builtin(__builtin_isnan) return __builtin_isnan(__lcpp_x); @@ -561,7 +561,7 @@ __libcpp_isnan(_A1 __lcpp_x) _NOEXCEPT template _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR typename enable_if::value, bool>::type -__libcpp_isnan(_A1 __lcpp_x) _NOEXCEPT +__libcpp_isnan_or_builtin(_A1 __lcpp_x) _NOEXCEPT { return isnan(__lcpp_x); } @@ -569,7 +569,7 @@ __libcpp_isnan(_A1 __lcpp_x) _NOEXCEPT template _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR typename enable_if::value, bool>::type -__libcpp_isinf(_A1 __lcpp_x) _NOEXCEPT +__libcpp_isinf_or_builtin(_A1 __lcpp_x) _NOEXCEPT { #if __has_builtin(__builtin_isinf) return __builtin_isinf(__lcpp_x); @@ -581,7 +581,7 @@ __libcpp_isinf(_A1 __lcpp_x) _NOEXCEPT template _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR typename enable_if::value, bool>::type -__libcpp_isinf(_A1 __lcpp_x) _NOEXCEPT +__libcpp_isinf_or_builtin(_A1 __lcpp_x) _NOEXCEPT { return isinf(__lcpp_x); } @@ -589,7 +589,7 @@ __libcpp_isinf(_A1 __lcpp_x) _NOEXCEPT template _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR typename enable_if::value, bool>::type -__libcpp_isfinite(_A1 __lcpp_x) _NOEXCEPT +__libcpp_isfinite_or_builtin(_A1 __lcpp_x) _NOEXCEPT { #if __has_builtin(__builtin_isfinite) return __builtin_isfinite(__lcpp_x); @@ -601,7 +601,7 @@ __libcpp_isfinite(_A1 __lcpp_x) _NOEXCEPT template _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR typename enable_if::value, bool>::type -__libcpp_isfinite(_A1 __lcpp_x) _NOEXCEPT +__libcpp_isfinite_or_builtin(_A1 __lcpp_x) _NOEXCEPT { return isfinite(__lcpp_x); } diff --git a/contrib/libc++/include/complex b/contrib/libc++/include/complex index d41971b3b4db..41a47cfba47c 100644 --- a/contrib/libc++/include/complex +++ b/contrib/libc++/include/complex @@ -599,39 +599,39 @@ operator*(const complex<_Tp>& __z, const complex<_Tp>& __w) _Tp __bc = __b * __c; _Tp __x = __ac - __bd; _Tp __y = __ad + __bc; - if (__libcpp_isnan(__x) && __libcpp_isnan(__y)) + if (__libcpp_isnan_or_builtin(__x) && __libcpp_isnan_or_builtin(__y)) { bool __recalc = false; - if (__libcpp_isinf(__a) || __libcpp_isinf(__b)) + if (__libcpp_isinf_or_builtin(__a) || __libcpp_isinf_or_builtin(__b)) { - __a = copysign(__libcpp_isinf(__a) ? _Tp(1) : _Tp(0), __a); - __b = copysign(__libcpp_isinf(__b) ? _Tp(1) : _Tp(0), __b); - if (__libcpp_isnan(__c)) + __a = copysign(__libcpp_isinf_or_builtin(__a) ? _Tp(1) : _Tp(0), __a); + __b = copysign(__libcpp_isinf_or_builtin(__b) ? _Tp(1) : _Tp(0), __b); + if (__libcpp_isnan_or_builtin(__c)) __c = copysign(_Tp(0), __c); - if (__libcpp_isnan(__d)) + if (__libcpp_isnan_or_builtin(__d)) __d = copysign(_Tp(0), __d); __recalc = true; } - if (__libcpp_isinf(__c) || __libcpp_isinf(__d)) + if (__libcpp_isinf_or_builtin(__c) || __libcpp_isinf_or_builtin(__d)) { - __c = copysign(__libcpp_isinf(__c) ? _Tp(1) : _Tp(0), __c); - __d = copysign(__libcpp_isinf(__d) ? _Tp(1) : _Tp(0), __d); - if (__libcpp_isnan(__a)) + __c = copysign(__libcpp_isinf_or_builtin(__c) ? _Tp(1) : _Tp(0), __c); + __d = copysign(__libcpp_isinf_or_builtin(__d) ? _Tp(1) : _Tp(0), __d); + if (__libcpp_isnan_or_builtin(__a)) __a = copysign(_Tp(0), __a); - if (__libcpp_isnan(__b)) + if (__libcpp_isnan_or_builtin(__b)) __b = copysign(_Tp(0), __b); __recalc = true; } - if (!__recalc && (__libcpp_isinf(__ac) || __libcpp_isinf(__bd) || - __libcpp_isinf(__ad) || __libcpp_isinf(__bc))) + if (!__recalc && (__libcpp_isinf_or_builtin(__ac) || __libcpp_isinf_or_builtin(__bd) || + __libcpp_isinf_or_builtin(__ad) || __libcpp_isinf_or_builtin(__bc))) { - if (__libcpp_isnan(__a)) + if (__libcpp_isnan_or_builtin(__a)) __a = copysign(_Tp(0), __a); - if (__libcpp_isnan(__b)) + if (__libcpp_isnan_or_builtin(__b)) __b = copysign(_Tp(0), __b); - if (__libcpp_isnan(__c)) + if (__libcpp_isnan_or_builtin(__c)) __c = copysign(_Tp(0), __c); - if (__libcpp_isnan(__d)) + if (__libcpp_isnan_or_builtin(__d)) __d = copysign(_Tp(0), __d); __recalc = true; } @@ -674,7 +674,7 @@ operator/(const complex<_Tp>& __z, const complex<_Tp>& __w) _Tp __c = __w.real(); _Tp __d = __w.imag(); _Tp __logbw = logb(fmax(fabs(__c), fabs(__d))); - if (__libcpp_isfinite(__logbw)) + if (__libcpp_isfinite_or_builtin(__logbw)) { __ilogbw = static_cast(__logbw); __c = scalbn(__c, -__ilogbw); @@ -683,24 +683,24 @@ operator/(const complex<_Tp>& __z, const complex<_Tp>& __w) _Tp __denom = __c * __c + __d * __d; _Tp __x = scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); _Tp __y = scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); - if (__libcpp_isnan(__x) && __libcpp_isnan(__y)) + if (__libcpp_isnan_or_builtin(__x) && __libcpp_isnan_or_builtin(__y)) { - if ((__denom == _Tp(0)) && (!__libcpp_isnan(__a) || !__libcpp_isnan(__b))) + if ((__denom == _Tp(0)) && (!__libcpp_isnan_or_builtin(__a) || !__libcpp_isnan_or_builtin(__b))) { __x = copysign(_Tp(INFINITY), __c) * __a; __y = copysign(_Tp(INFINITY), __c) * __b; } - else if ((__libcpp_isinf(__a) || __libcpp_isinf(__b)) && __libcpp_isfinite(__c) && __libcpp_isfinite(__d)) + else if ((__libcpp_isinf_or_builtin(__a) || __libcpp_isinf_or_builtin(__b)) && __libcpp_isfinite_or_builtin(__c) && __libcpp_isfinite_or_builtin(__d)) { - __a = copysign(__libcpp_isinf(__a) ? _Tp(1) : _Tp(0), __a); - __b = copysign(__libcpp_isinf(__b) ? _Tp(1) : _Tp(0), __b); + __a = copysign(__libcpp_isinf_or_builtin(__a) ? _Tp(1) : _Tp(0), __a); + __b = copysign(__libcpp_isinf_or_builtin(__b) ? _Tp(1) : _Tp(0), __b); __x = _Tp(INFINITY) * (__a * __c + __b * __d); __y = _Tp(INFINITY) * (__b * __c - __a * __d); } - else if (__libcpp_isinf(__logbw) && __logbw > _Tp(0) && __libcpp_isfinite(__a) && __libcpp_isfinite(__b)) + else if (__libcpp_isinf_or_builtin(__logbw) && __logbw > _Tp(0) && __libcpp_isfinite_or_builtin(__a) && __libcpp_isfinite_or_builtin(__b)) { - __c = copysign(__libcpp_isinf(__c) ? _Tp(1) : _Tp(0), __c); - __d = copysign(__libcpp_isinf(__d) ? _Tp(1) : _Tp(0), __d); + __c = copysign(__libcpp_isinf_or_builtin(__c) ? _Tp(1) : _Tp(0), __c); + __d = copysign(__libcpp_isinf_or_builtin(__d) ? _Tp(1) : _Tp(0), __d); __x = _Tp(0) * (__a * __c + __b * __d); __y = _Tp(0) * (__b * __c - __a * __d); } @@ -910,9 +910,9 @@ inline _LIBCPP_INLINE_VISIBILITY _Tp norm(const complex<_Tp>& __c) { - if (__libcpp_isinf(__c.real())) + if (__libcpp_isinf_or_builtin(__c.real())) return abs(__c.real()); - if (__libcpp_isinf(__c.imag())) + if (__libcpp_isinf_or_builtin(__c.imag())) return abs(__c.imag()); return __c.real() * __c.real() + __c.imag() * __c.imag(); } @@ -955,7 +955,7 @@ complex<_Tp> proj(const complex<_Tp>& __c) { std::complex<_Tp> __r = __c; - if (__libcpp_isinf(__c.real()) || __libcpp_isinf(__c.imag())) + if (__libcpp_isinf_or_builtin(__c.real()) || __libcpp_isinf_or_builtin(__c.imag())) __r = complex<_Tp>(INFINITY, copysign(_Tp(0), __c.imag())); return __r; } @@ -969,7 +969,7 @@ typename enable_if >::type proj(_Tp __re) { - if (__libcpp_isinf(__re)) + if (__libcpp_isinf_or_builtin(__re)) __re = abs(__re); return complex<_Tp>(__re); } @@ -993,25 +993,25 @@ template complex<_Tp> polar(const _Tp& __rho, const _Tp& __theta = _Tp(0)) { - if (__libcpp_isnan(__rho) || signbit(__rho)) + if (__libcpp_isnan_or_builtin(__rho) || signbit(__rho)) return complex<_Tp>(_Tp(NAN), _Tp(NAN)); - if (__libcpp_isnan(__theta)) + if (__libcpp_isnan_or_builtin(__theta)) { - if (__libcpp_isinf(__rho)) + if (__libcpp_isinf_or_builtin(__rho)) return complex<_Tp>(__rho, __theta); return complex<_Tp>(__theta, __theta); } - if (__libcpp_isinf(__theta)) + if (__libcpp_isinf_or_builtin(__theta)) { - if (__libcpp_isinf(__rho)) + if (__libcpp_isinf_or_builtin(__rho)) return complex<_Tp>(__rho, _Tp(NAN)); return complex<_Tp>(_Tp(NAN), _Tp(NAN)); } _Tp __x = __rho * cos(__theta); - if (__libcpp_isnan(__x)) + if (__libcpp_isnan_or_builtin(__x)) __x = 0; _Tp __y = __rho * sin(__theta); - if (__libcpp_isnan(__y)) + if (__libcpp_isnan_or_builtin(__y)) __y = 0; return complex<_Tp>(__x, __y); } @@ -1042,13 +1042,13 @@ template complex<_Tp> sqrt(const complex<_Tp>& __x) { - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(_Tp(INFINITY), __x.imag()); - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { if (__x.real() > _Tp(0)) - return complex<_Tp>(__x.real(), __libcpp_isnan(__x.imag()) ? __x.imag() : copysign(_Tp(0), __x.imag())); - return complex<_Tp>(__libcpp_isnan(__x.imag()) ? __x.imag() : _Tp(0), copysign(__x.real(), __x.imag())); + return complex<_Tp>(__x.real(), __libcpp_isnan_or_builtin(__x.imag()) ? __x.imag() : copysign(_Tp(0), __x.imag())); + return complex<_Tp>(__libcpp_isnan_or_builtin(__x.imag()) ? __x.imag() : _Tp(0), copysign(__x.real(), __x.imag())); } return polar(sqrt(abs(__x)), arg(__x) / _Tp(2)); } @@ -1060,21 +1060,21 @@ complex<_Tp> exp(const complex<_Tp>& __x) { _Tp __i = __x.imag(); - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { if (__x.real() < _Tp(0)) { - if (!__libcpp_isfinite(__i)) + if (!__libcpp_isfinite_or_builtin(__i)) __i = _Tp(1); } - else if (__i == 0 || !__libcpp_isfinite(__i)) + else if (__i == 0 || !__libcpp_isfinite_or_builtin(__i)) { - if (__libcpp_isinf(__i)) + if (__libcpp_isinf_or_builtin(__i)) __i = _Tp(NAN); return complex<_Tp>(__x.real(), __i); } } - else if (__libcpp_isnan(__x.real()) && __x.imag() == 0) + else if (__libcpp_isnan_or_builtin(__x.real()) && __x.imag() == 0) return __x; _Tp __e = exp(__x.real()); return complex<_Tp>(__e * cos(__i), __e * sin(__i)); @@ -1132,23 +1132,23 @@ complex<_Tp> asinh(const complex<_Tp>& __x) { const _Tp __pi(atan2(+0., -0.)); - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { - if (__libcpp_isnan(__x.imag())) + if (__libcpp_isnan_or_builtin(__x.imag())) return __x; - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(__x.real(), copysign(__pi * _Tp(0.25), __x.imag())); return complex<_Tp>(__x.real(), copysign(_Tp(0), __x.imag())); } - if (__libcpp_isnan(__x.real())) + if (__libcpp_isnan_or_builtin(__x.real())) { - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(__x.imag(), __x.real()); if (__x.imag() == 0) return __x; return complex<_Tp>(__x.real(), __x.real()); } - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(copysign(__x.imag(), __x.real()), copysign(__pi/_Tp(2), __x.imag())); complex<_Tp> __z = log(__x + sqrt(pow(__x, _Tp(2)) + _Tp(1))); return complex<_Tp>(copysign(__z.real(), __x.real()), copysign(__z.imag(), __x.imag())); @@ -1161,11 +1161,11 @@ complex<_Tp> acosh(const complex<_Tp>& __x) { const _Tp __pi(atan2(+0., -0.)); - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { - if (__libcpp_isnan(__x.imag())) + if (__libcpp_isnan_or_builtin(__x.imag())) return complex<_Tp>(abs(__x.real()), __x.imag()); - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) { if (__x.real() > 0) return complex<_Tp>(__x.real(), copysign(__pi * _Tp(0.25), __x.imag())); @@ -1176,13 +1176,13 @@ acosh(const complex<_Tp>& __x) return complex<_Tp>(-__x.real(), copysign(__pi, __x.imag())); return complex<_Tp>(__x.real(), copysign(_Tp(0), __x.imag())); } - if (__libcpp_isnan(__x.real())) + if (__libcpp_isnan_or_builtin(__x.real())) { - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(abs(__x.imag()), __x.real()); return complex<_Tp>(__x.real(), __x.real()); } - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(abs(__x.imag()), copysign(__pi/_Tp(2), __x.imag())); complex<_Tp> __z = log(__x + sqrt(pow(__x, _Tp(2)) - _Tp(1))); return complex<_Tp>(copysign(__z.real(), _Tp(0)), copysign(__z.imag(), __x.imag())); @@ -1195,21 +1195,21 @@ complex<_Tp> atanh(const complex<_Tp>& __x) { const _Tp __pi(atan2(+0., -0.)); - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) { return complex<_Tp>(copysign(_Tp(0), __x.real()), copysign(__pi/_Tp(2), __x.imag())); } - if (__libcpp_isnan(__x.imag())) + if (__libcpp_isnan_or_builtin(__x.imag())) { - if (__libcpp_isinf(__x.real()) || __x.real() == 0) + if (__libcpp_isinf_or_builtin(__x.real()) || __x.real() == 0) return complex<_Tp>(copysign(_Tp(0), __x.real()), __x.imag()); return complex<_Tp>(__x.imag(), __x.imag()); } - if (__libcpp_isnan(__x.real())) + if (__libcpp_isnan_or_builtin(__x.real())) { return complex<_Tp>(__x.real(), __x.real()); } - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { return complex<_Tp>(copysign(_Tp(0), __x.real()), copysign(__pi/_Tp(2), __x.imag())); } @@ -1227,11 +1227,11 @@ template complex<_Tp> sinh(const complex<_Tp>& __x) { - if (__libcpp_isinf(__x.real()) && !__libcpp_isfinite(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.real()) && !__libcpp_isfinite_or_builtin(__x.imag())) return complex<_Tp>(__x.real(), _Tp(NAN)); - if (__x.real() == 0 && !__libcpp_isfinite(__x.imag())) + if (__x.real() == 0 && !__libcpp_isfinite_or_builtin(__x.imag())) return complex<_Tp>(__x.real(), _Tp(NAN)); - if (__x.imag() == 0 && !__libcpp_isfinite(__x.real())) + if (__x.imag() == 0 && !__libcpp_isfinite_or_builtin(__x.real())) return __x; return complex<_Tp>(sinh(__x.real()) * cos(__x.imag()), cosh(__x.real()) * sin(__x.imag())); } @@ -1242,13 +1242,13 @@ template complex<_Tp> cosh(const complex<_Tp>& __x) { - if (__libcpp_isinf(__x.real()) && !__libcpp_isfinite(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.real()) && !__libcpp_isfinite_or_builtin(__x.imag())) return complex<_Tp>(abs(__x.real()), _Tp(NAN)); - if (__x.real() == 0 && !__libcpp_isfinite(__x.imag())) + if (__x.real() == 0 && !__libcpp_isfinite_or_builtin(__x.imag())) return complex<_Tp>(_Tp(NAN), __x.real()); if (__x.real() == 0 && __x.imag() == 0) return complex<_Tp>(_Tp(1), __x.imag()); - if (__x.imag() == 0 && !__libcpp_isfinite(__x.real())) + if (__x.imag() == 0 && !__libcpp_isfinite_or_builtin(__x.real())) return complex<_Tp>(abs(__x.real()), __x.imag()); return complex<_Tp>(cosh(__x.real()) * cos(__x.imag()), sinh(__x.real()) * sin(__x.imag())); } @@ -1259,19 +1259,19 @@ template complex<_Tp> tanh(const complex<_Tp>& __x) { - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { - if (!__libcpp_isfinite(__x.imag())) + if (!__libcpp_isfinite_or_builtin(__x.imag())) return complex<_Tp>(_Tp(1), _Tp(0)); return complex<_Tp>(_Tp(1), copysign(_Tp(0), sin(_Tp(2) * __x.imag()))); } - if (__libcpp_isnan(__x.real()) && __x.imag() == 0) + if (__libcpp_isnan_or_builtin(__x.real()) && __x.imag() == 0) return __x; _Tp __2r(_Tp(2) * __x.real()); _Tp __2i(_Tp(2) * __x.imag()); _Tp __d(cosh(__2r) + cos(__2i)); _Tp __2rsh(sinh(__2r)); - if (__libcpp_isinf(__2rsh) && __libcpp_isinf(__d)) + if (__libcpp_isinf_or_builtin(__2rsh) && __libcpp_isinf_or_builtin(__d)) return complex<_Tp>(__2rsh > _Tp(0) ? _Tp(1) : _Tp(-1), __2i > _Tp(0) ? _Tp(0) : _Tp(-0.)); return complex<_Tp>(__2rsh/__d, sin(__2i)/__d); @@ -1294,11 +1294,11 @@ complex<_Tp> acos(const complex<_Tp>& __x) { const _Tp __pi(atan2(+0., -0.)); - if (__libcpp_isinf(__x.real())) + if (__libcpp_isinf_or_builtin(__x.real())) { - if (__libcpp_isnan(__x.imag())) + if (__libcpp_isnan_or_builtin(__x.imag())) return complex<_Tp>(__x.imag(), __x.real()); - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) { if (__x.real() < _Tp(0)) return complex<_Tp>(_Tp(0.75) * __pi, -__x.imag()); @@ -1308,13 +1308,13 @@ acos(const complex<_Tp>& __x) return complex<_Tp>(__pi, signbit(__x.imag()) ? -__x.real() : __x.real()); return complex<_Tp>(_Tp(0), signbit(__x.imag()) ? __x.real() : -__x.real()); } - if (__libcpp_isnan(__x.real())) + if (__libcpp_isnan_or_builtin(__x.real())) { - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(__x.real(), -__x.imag()); return complex<_Tp>(__x.real(), __x.real()); } - if (__libcpp_isinf(__x.imag())) + if (__libcpp_isinf_or_builtin(__x.imag())) return complex<_Tp>(__pi/_Tp(2), -__x.imag()); if (__x.real() == 0 && (__x.imag() == 0 || isnan(__x.imag()))) return complex<_Tp>(__pi/_Tp(2), -__x.imag()); diff --git a/contrib/libc++/include/optional b/contrib/libc++/include/optional index c0fd0e7bc49f..1fb953bab743 100644 --- a/contrib/libc++/include/optional +++ b/contrib/libc++/include/optional @@ -439,46 +439,122 @@ struct __optional_storage_base<_Tp, true> } }; -template ::value> -struct __optional_storage; - -template -struct __optional_storage<_Tp, true> : __optional_storage_base<_Tp> +template ::value> +struct __optional_copy_base : __optional_storage_base<_Tp> { using __optional_storage_base<_Tp>::__optional_storage_base; }; template -struct __optional_storage<_Tp, false> : __optional_storage_base<_Tp> +struct __optional_copy_base<_Tp, false> : __optional_storage_base<_Tp> { - using value_type = _Tp; using __optional_storage_base<_Tp>::__optional_storage_base; _LIBCPP_INLINE_VISIBILITY - __optional_storage() = default; + __optional_copy_base() = default; _LIBCPP_INLINE_VISIBILITY - __optional_storage(const __optional_storage& __opt) + __optional_copy_base(const __optional_copy_base& __opt) { this->__construct_from(__opt); } _LIBCPP_INLINE_VISIBILITY - __optional_storage(__optional_storage&& __opt) + __optional_copy_base(__optional_copy_base&&) = default; + _LIBCPP_INLINE_VISIBILITY + __optional_copy_base& operator=(const __optional_copy_base&) = default; + _LIBCPP_INLINE_VISIBILITY + __optional_copy_base& operator=(__optional_copy_base&&) = default; +}; + +template ::value> +struct __optional_move_base : __optional_copy_base<_Tp> +{ + using __optional_copy_base<_Tp>::__optional_copy_base; +}; + +template +struct __optional_move_base<_Tp, false> : __optional_copy_base<_Tp> +{ + using value_type = _Tp; + using __optional_copy_base<_Tp>::__optional_copy_base; + + _LIBCPP_INLINE_VISIBILITY + __optional_move_base() = default; + _LIBCPP_INLINE_VISIBILITY + __optional_move_base(const __optional_move_base&) = default; + + _LIBCPP_INLINE_VISIBILITY + __optional_move_base(__optional_move_base&& __opt) noexcept(is_nothrow_move_constructible_v) { this->__construct_from(_VSTD::move(__opt)); } _LIBCPP_INLINE_VISIBILITY - __optional_storage& operator=(const __optional_storage& __opt) + __optional_move_base& operator=(const __optional_move_base&) = default; + _LIBCPP_INLINE_VISIBILITY + __optional_move_base& operator=(__optional_move_base&&) = default; +}; + +template ::value && + is_trivially_copy_constructible<_Tp>::value && + is_trivially_copy_assignable<_Tp>::value> +struct __optional_copy_assign_base : __optional_move_base<_Tp> +{ + using __optional_move_base<_Tp>::__optional_move_base; +}; + +template +struct __optional_copy_assign_base<_Tp, false> : __optional_move_base<_Tp> +{ + using __optional_move_base<_Tp>::__optional_move_base; + + _LIBCPP_INLINE_VISIBILITY + __optional_copy_assign_base() = default; + _LIBCPP_INLINE_VISIBILITY + __optional_copy_assign_base(const __optional_copy_assign_base&) = default; + _LIBCPP_INLINE_VISIBILITY + __optional_copy_assign_base(__optional_copy_assign_base&&) = default; + + _LIBCPP_INLINE_VISIBILITY + __optional_copy_assign_base& operator=(const __optional_copy_assign_base& __opt) { this->__assign_from(__opt); return *this; } _LIBCPP_INLINE_VISIBILITY - __optional_storage& operator=(__optional_storage&& __opt) + __optional_copy_assign_base& operator=(__optional_copy_assign_base&&) = default; +}; + +template ::value && + is_trivially_move_constructible<_Tp>::value && + is_trivially_move_assignable<_Tp>::value> +struct __optional_move_assign_base : __optional_copy_assign_base<_Tp> +{ + using __optional_copy_assign_base<_Tp>::__optional_copy_assign_base; +}; + +template +struct __optional_move_assign_base<_Tp, false> : __optional_copy_assign_base<_Tp> +{ + using value_type = _Tp; + using __optional_copy_assign_base<_Tp>::__optional_copy_assign_base; + + _LIBCPP_INLINE_VISIBILITY + __optional_move_assign_base() = default; + _LIBCPP_INLINE_VISIBILITY + __optional_move_assign_base(const __optional_move_assign_base& __opt) = default; + _LIBCPP_INLINE_VISIBILITY + __optional_move_assign_base(__optional_move_assign_base&&) = default; + _LIBCPP_INLINE_VISIBILITY + __optional_move_assign_base& operator=(const __optional_move_assign_base&) = default; + + _LIBCPP_INLINE_VISIBILITY + __optional_move_assign_base& operator=(__optional_move_assign_base&& __opt) noexcept(is_nothrow_move_assignable_v && is_nothrow_move_constructible_v) { @@ -501,11 +577,11 @@ using __optional_sfinae_assign_base_t = __sfinae_assign_base< template class optional - : private __optional_storage<_Tp> + : private __optional_move_assign_base<_Tp> , private __optional_sfinae_ctor_base_t<_Tp> , private __optional_sfinae_assign_base_t<_Tp> { - using __base = __optional_storage<_Tp>; + using __base = __optional_move_assign_base<_Tp>; public: using value_type = _Tp; diff --git a/contrib/libc++/include/regex b/contrib/libc++/include/regex index 443c2e303465..77ca648109b2 100644 --- a/contrib/libc++/include/regex +++ b/contrib/libc++/include/regex @@ -6142,7 +6142,7 @@ regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() { __flags_ |= regex_constants::__no_update_pos; _BidirectionalIterator __start = __match_[0].second; - if (__match_.empty()) + if (__match_[0].first == __match_[0].second) { if (__start == __end_) { diff --git a/contrib/libc++/include/string b/contrib/libc++/include/string index d1a3a1f8e6cc..010a4c7816ea 100644 --- a/contrib/libc++/include/string +++ b/contrib/libc++/include/string @@ -676,11 +676,11 @@ private: }; #if _LIBCPP_BIG_ENDIAN - enum {__short_mask = 0x01}; - enum {__long_mask = 0x1ul}; + static const size_type __short_mask = 0x01; + static const size_type __long_mask = 0x1ul; #else // _LIBCPP_BIG_ENDIAN - enum {__short_mask = 0x80}; - enum {__long_mask = ~(size_type(~0) >> 1)}; + static const size_type __short_mask = 0x80; + static const size_type __long_mask = ~(size_type(~0) >> 1); #endif // _LIBCPP_BIG_ENDIAN enum {__min_cap = (sizeof(__long) - 1)/sizeof(value_type) > 2 ? @@ -706,11 +706,11 @@ private: }; #if _LIBCPP_BIG_ENDIAN - enum {__short_mask = 0x80}; - enum {__long_mask = ~(size_type(~0) >> 1)}; + static const size_type __short_mask = 0x80; + static const size_type __long_mask = ~(size_type(~0) >> 1); #else // _LIBCPP_BIG_ENDIAN - enum {__short_mask = 0x01}; - enum {__long_mask = 0x1ul}; + static const size_type __short_mask = 0x01; + static const size_type __long_mask = 0x1ul; #endif // _LIBCPP_BIG_ENDIAN enum {__min_cap = (sizeof(__long) - 1)/sizeof(value_type) > 2 ? diff --git a/contrib/libc++/src/experimental/filesystem/filesystem_time_helper.h b/contrib/libc++/src/experimental/filesystem/filesystem_time_helper.h new file mode 100644 index 000000000000..a60fdef5f0d9 --- /dev/null +++ b/contrib/libc++/src/experimental/filesystem/filesystem_time_helper.h @@ -0,0 +1,173 @@ +//===----------------------------------------------------------------------===//// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===//// + +#ifndef FILESYSTEM_TIME_HELPER_H +#define FILESYSTEM_TIME_HELPER_H + +#include "experimental/__config" +#include "chrono" +#include "cstdlib" +#include "climits" + +#include +#include +#if !defined(UTIME_OMIT) +#include // for ::utimes as used in __last_write_time +#endif + +_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL_FILESYSTEM + +namespace time_detail { namespace { + +using namespace chrono; + +template ::value> +struct fs_time_util_base { + static constexpr auto max_seconds = + duration_cast(FileTimeT::duration::max()).count(); + + static constexpr auto max_nsec = + duration_cast(FileTimeT::duration::max() - + seconds(max_seconds)) + .count(); + + static constexpr auto min_seconds = + duration_cast(FileTimeT::duration::min()).count(); + + static constexpr auto min_nsec_timespec = + duration_cast( + (FileTimeT::duration::min() - seconds(min_seconds)) + seconds(1)) + .count(); + + // Static assert that these values properly round trip. + static_assert((seconds(min_seconds) + + duration_cast(nanoseconds(min_nsec_timespec))) - + duration_cast(seconds(1)) == + FileTimeT::duration::min(), + ""); +}; + +template +struct fs_time_util_base { + static const long long max_seconds; + static const long long max_nsec; + static const long long min_seconds; + static const long long min_nsec_timespec; +}; + +template +const long long fs_time_util_base::max_seconds = + duration_cast(FileTimeT::duration::max()).count(); + +template +const long long fs_time_util_base::max_nsec = + duration_cast(FileTimeT::duration::max() - + seconds(max_seconds)) + .count(); + +template +const long long fs_time_util_base::min_seconds = + duration_cast(FileTimeT::duration::min()).count(); + +template +const long long fs_time_util_base::min_nsec_timespec = + duration_cast((FileTimeT::duration::min() - + seconds(min_seconds)) + + seconds(1)) + .count(); + +template +struct fs_time_util : fs_time_util_base { + using Base = fs_time_util_base; + using Base::max_nsec; + using Base::max_seconds; + using Base::min_nsec_timespec; + using Base::min_seconds; + +public: + template + static bool checked_set(CType* out, ChronoType time) { + using Lim = numeric_limits; + if (time > Lim::max() || time < Lim::min()) + return false; + *out = static_cast(time); + return true; + } + + static _LIBCPP_CONSTEXPR_AFTER_CXX11 bool is_representable(TimeSpecT tm) { + if (tm.tv_sec >= 0) { + return (tm.tv_sec < max_seconds) || + (tm.tv_sec == max_seconds && tm.tv_nsec <= max_nsec); + } else if (tm.tv_sec == (min_seconds - 1)) { + return tm.tv_nsec >= min_nsec_timespec; + } else { + return (tm.tv_sec >= min_seconds); + } + } + + static _LIBCPP_CONSTEXPR_AFTER_CXX11 bool is_representable(FileTimeT tm) { + auto secs = duration_cast(tm.time_since_epoch()); + auto nsecs = duration_cast(tm.time_since_epoch() - secs); + if (nsecs.count() < 0) { + secs = secs + seconds(1); + nsecs = nsecs + seconds(1); + } + using TLim = numeric_limits; + if (secs.count() >= 0) + return secs.count() <= TLim::max(); + return secs.count() >= TLim::min(); + } + + static _LIBCPP_CONSTEXPR_AFTER_CXX11 FileTimeT + convert_timespec(TimeSpecT tm) { + auto adj_msec = duration_cast(nanoseconds(tm.tv_nsec)); + if (tm.tv_sec >= 0) { + auto Dur = seconds(tm.tv_sec) + microseconds(adj_msec); + return FileTimeT(Dur); + } else if (duration_cast(nanoseconds(tm.tv_nsec)).count() == + 0) { + return FileTimeT(seconds(tm.tv_sec)); + } else { // tm.tv_sec < 0 + auto adj_subsec = + duration_cast(seconds(1) - nanoseconds(tm.tv_nsec)); + auto Dur = seconds(tm.tv_sec + 1) - adj_subsec; + return FileTimeT(Dur); + } + } + + template + static bool set_times_checked(TimeT* sec_out, SubSecT* subsec_out, + FileTimeT tp) { + using namespace chrono; + auto dur = tp.time_since_epoch(); + auto sec_dur = duration_cast(dur); + auto subsec_dur = duration_cast(dur - sec_dur); + // The tv_nsec and tv_usec fields must not be negative so adjust accordingly + if (subsec_dur.count() < 0) { + if (sec_dur.count() > min_seconds) { + sec_dur -= seconds(1); + subsec_dur += seconds(1); + } else { + subsec_dur = SubSecDurT::zero(); + } + } + return checked_set(sec_out, sec_dur.count()) && + checked_set(subsec_out, subsec_dur.count()); + } +}; + +} // end namespace +} // end namespace time_detail + +using time_detail::fs_time_util; + +_LIBCPP_END_NAMESPACE_EXPERIMENTAL_FILESYSTEM + +#endif // FILESYSTEM_TIME_HELPER_H diff --git a/contrib/libc++/src/experimental/filesystem/operations.cpp b/contrib/libc++/src/experimental/filesystem/operations.cpp index e9bc0eb6e30e..641a3c53636d 100644 --- a/contrib/libc++/src/experimental/filesystem/operations.cpp +++ b/contrib/libc++/src/experimental/filesystem/operations.cpp @@ -15,6 +15,8 @@ #include "cstdlib" #include "climits" +#include "filesystem_time_helper.h" + #include #include #include @@ -426,17 +428,20 @@ void __current_path(const path& p, std::error_code *ec) { bool __equivalent(const path& p1, const path& p2, std::error_code *ec) { + auto make_unsupported_error = [&]() { + set_or_throw(make_error_code(errc::not_supported), ec, + "equivalent", p1, p2); + return false; + }; std::error_code ec1, ec2; struct ::stat st1 = {}; struct ::stat st2 = {}; auto s1 = detail::posix_stat(p1.native(), st1, &ec1); + if (!exists(s1)) + return make_unsupported_error(); auto s2 = detail::posix_stat(p2.native(), st2, &ec2); - - if ((!exists(s1) && !exists(s2)) || (is_other(s1) && is_other(s2))) { - set_or_throw(make_error_code(errc::not_supported), ec, - "equivalent", p1, p2); - return false; - } + if (!exists(s2)) + return make_unsupported_error(); if (ec) ec->clear(); return (st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino); } @@ -502,17 +507,6 @@ bool __fs_is_empty(const path& p, std::error_code *ec) namespace detail { namespace { -using namespace std::chrono; - -template -bool checked_set(CType* out, ChronoType time) { - using Lim = numeric_limits; - if (time > Lim::max() || time < Lim::min()) - return false; - *out = static_cast(time); - return true; -} - using TimeSpec = struct timespec; using StatT = struct stat; @@ -525,137 +519,10 @@ __attribute__((unused)) // Suppress warning TimeSpec extract_atime(StatT const& st) { return st.st_atim; } #endif -constexpr auto max_seconds = duration_cast( - file_time_type::duration::max()).count(); - -constexpr auto max_nsec = duration_cast( - file_time_type::duration::max() - seconds(max_seconds)).count(); - -constexpr auto min_seconds = duration_cast( - file_time_type::duration::min()).count(); - -constexpr auto min_nsec_timespec = duration_cast( - (file_time_type::duration::min() - seconds(min_seconds)) + seconds(1)).count(); - -// Static assert that these values properly round trip. -static_assert((seconds(min_seconds) + duration_cast(nanoseconds(min_nsec_timespec))) - - duration_cast(seconds(1)) - == file_time_type::duration::min(), ""); - -constexpr auto max_time_t = numeric_limits::max(); -constexpr auto min_time_t = numeric_limits::min(); - -#if !defined(__LP64__) && defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wtautological-constant-out-of-range-compare" -#endif - -_LIBCPP_CONSTEXPR_AFTER_CXX11 -bool is_representable(TimeSpec const& tm) { - if (tm.tv_sec >= 0) { - return (tm.tv_sec < max_seconds) || - (tm.tv_sec == max_seconds && tm.tv_nsec <= max_nsec); - } else if (tm.tv_sec == (min_seconds - 1)) { - return tm.tv_nsec >= min_nsec_timespec; - } else { - return (tm.tv_sec >= min_seconds); - } -} -#ifndef _LIBCPP_HAS_NO_CXX14_CONSTEXPR -#if defined(__LP64__) -static_assert(is_representable({max_seconds, max_nsec}), ""); -static_assert(!is_representable({max_seconds + 1, 0}), ""); -static_assert(!is_representable({max_seconds, max_nsec + 1}), ""); -static_assert(!is_representable({max_time_t, 0}), ""); -static_assert(is_representable({min_seconds, 0}), ""); -static_assert(is_representable({min_seconds - 1, min_nsec_timespec}), ""); -static_assert(is_representable({min_seconds - 1, min_nsec_timespec + 1}), ""); -static_assert(!is_representable({min_seconds - 1, min_nsec_timespec - 1}), ""); -static_assert(!is_representable({min_time_t, 999999999}), ""); -#else -static_assert(is_representable({max_time_t, 999999999}), ""); -static_assert(is_representable({max_time_t, 1000000000}), ""); -static_assert(is_representable({min_time_t, 0}), ""); -#endif -#endif - -_LIBCPP_CONSTEXPR_AFTER_CXX11 -bool is_representable(file_time_type const& tm) { - auto secs = duration_cast(tm.time_since_epoch()); - auto nsecs = duration_cast(tm.time_since_epoch() - secs); - if (nsecs.count() < 0) { - secs = secs + seconds(1); - nsecs = nsecs + seconds(1); - } - using TLim = numeric_limits; - if (secs.count() >= 0) - return secs.count() <= TLim::max(); - return secs.count() >= TLim::min(); -} -#ifndef _LIBCPP_HAS_NO_CXX14_CONSTEXPR -#if defined(__LP64__) -static_assert(is_representable(file_time_type::max()), ""); -static_assert(is_representable(file_time_type::min()), ""); -#else -static_assert(!is_representable(file_time_type::max()), ""); -static_assert(!is_representable(file_time_type::min()), ""); -static_assert(is_representable(file_time_type(seconds(max_time_t))), ""); -static_assert(is_representable(file_time_type(seconds(min_time_t))), ""); -#endif -#endif - -_LIBCPP_CONSTEXPR_AFTER_CXX11 -file_time_type convert_timespec(TimeSpec const& tm) { - auto adj_msec = duration_cast(nanoseconds(tm.tv_nsec)); - if (tm.tv_sec >= 0) { - auto Dur = seconds(tm.tv_sec) + microseconds(adj_msec); - return file_time_type(Dur); - } else if (duration_cast(nanoseconds(tm.tv_nsec)).count() == 0) { - return file_time_type(seconds(tm.tv_sec)); - } else { // tm.tv_sec < 0 - auto adj_subsec = duration_cast(seconds(1) - nanoseconds(tm.tv_nsec)); - auto Dur = seconds(tm.tv_sec + 1) - adj_subsec; - return file_time_type(Dur); - } -} -#ifndef _LIBCPP_HAS_NO_CXX14_CONSTEXPR -#if defined(__LP64__) -static_assert(convert_timespec({max_seconds, max_nsec}) == file_time_type::max(), ""); -static_assert(convert_timespec({max_seconds, max_nsec - 1}) < file_time_type::max(), ""); -static_assert(convert_timespec({max_seconds - 1, 999999999}) < file_time_type::max(), ""); -static_assert(convert_timespec({min_seconds - 1, min_nsec_timespec}) == file_time_type::min(), ""); -static_assert(convert_timespec({min_seconds - 1, min_nsec_timespec + 1}) > file_time_type::min(), ""); -static_assert(convert_timespec({min_seconds , 0}) > file_time_type::min(), ""); -#else -// FIXME add tests for 32 bit builds -#endif -#endif - -#if !defined(__LP64__) && defined(__clang__) -#pragma clang diagnostic pop -#endif - -template -bool set_times_checked(time_t* sec_out, SubSecT* subsec_out, file_time_type tp) { - using namespace chrono; - auto dur = tp.time_since_epoch(); - auto sec_dur = duration_cast(dur); - auto subsec_dur = duration_cast(dur - sec_dur); - // The tv_nsec and tv_usec fields must not be negative so adjust accordingly - if (subsec_dur.count() < 0) { - if (sec_dur.count() > min_seconds) { - sec_dur -= seconds(1); - subsec_dur += seconds(1); - } else { - subsec_dur = SubSecDurT::zero(); - } - } - return checked_set(sec_out, sec_dur.count()) - && checked_set(subsec_out, subsec_dur.count()); -} - }} // end namespace detail +using FSTime = fs_time_util; + file_time_type __last_write_time(const path& p, std::error_code *ec) { using namespace ::std::chrono; @@ -668,12 +535,12 @@ file_time_type __last_write_time(const path& p, std::error_code *ec) } if (ec) ec->clear(); auto ts = detail::extract_mtime(st); - if (!detail::is_representable(ts)) { + if (!FSTime::is_representable(ts)) { set_or_throw(error_code(EOVERFLOW, generic_category()), ec, "last_write_time", p); return file_time_type::min(); } - return detail::convert_timespec(ts); + return FSTime::convert_timespec(ts); } void __last_write_time(const path& p, file_time_type new_time, @@ -698,7 +565,7 @@ void __last_write_time(const path& p, file_time_type new_time, struct ::timeval tbuf[2]; tbuf[0].tv_sec = atime.tv_sec; tbuf[0].tv_usec = duration_cast(nanoseconds(atime.tv_nsec)).count(); - const bool overflowed = !detail::set_times_checked( + const bool overflowed = !FSTime::set_times_checked( &tbuf[1].tv_sec, &tbuf[1].tv_usec, new_time); if (overflowed) { @@ -714,7 +581,7 @@ void __last_write_time(const path& p, file_time_type new_time, tbuf[0].tv_sec = 0; tbuf[0].tv_nsec = UTIME_OMIT; - const bool overflowed = !detail::set_times_checked( + const bool overflowed = !FSTime::set_times_checked( &tbuf[1].tv_sec, &tbuf[1].tv_nsec, new_time); if (overflowed) { set_or_throw(make_error_code(errc::invalid_argument), diff --git a/contrib/libc++/include/__refstring b/contrib/libc++/src/include/refstring.h similarity index 97% rename from contrib/libc++/include/__refstring rename to contrib/libc++/src/include/refstring.h index 7f417a03c491..f0d5b4456da5 100644 --- a/contrib/libc++/include/__refstring +++ b/contrib/libc++/src/include/refstring.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___REFSTRING -#define _LIBCPP___REFSTRING +#ifndef _LIBCPP_REFSTRING_H +#define _LIBCPP_REFSTRING_H #include <__config> #include @@ -124,4 +124,4 @@ bool __libcpp_refstring::__uses_refcount() const { _LIBCPP_END_NAMESPACE_STD -#endif //_LIBCPP___REFSTRING +#endif //_LIBCPP_REFSTRING_H diff --git a/contrib/libc++/src/stdexcept.cpp b/contrib/libc++/src/stdexcept.cpp index 3f333309dd8f..5e06e521e404 100644 --- a/contrib/libc++/src/stdexcept.cpp +++ b/contrib/libc++/src/stdexcept.cpp @@ -11,7 +11,7 @@ #include "new" #include "string" #include "system_error" -#include "__refstring" +#include "include/refstring.h" /* For _LIBCPPABI_VERSION */ #if !defined(_LIBCPP_BUILDING_HAS_NO_ABI_LIBRARY) && \ diff --git a/contrib/llvm/include/llvm-c/OrcBindings.h b/contrib/llvm/include/llvm-c/OrcBindings.h index d86ea8808889..7ee395431358 100644 --- a/contrib/llvm/include/llvm-c/OrcBindings.h +++ b/contrib/llvm/include/llvm-c/OrcBindings.h @@ -113,8 +113,9 @@ void LLVMOrcDisposeMangledSymbol(char *MangledSymbol); /** * Create a lazy compile callback. */ -LLVMOrcTargetAddress +LLVMOrcErrorCode LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx); @@ -135,8 +136,9 @@ LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, /** * Add module to be eagerly compiled. */ -LLVMOrcModuleHandle +LLVMOrcErrorCode LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, LLVMSharedModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx); @@ -144,8 +146,9 @@ LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, /** * Add module to be lazily compiled one function at a time. */ -LLVMOrcModuleHandle +LLVMOrcErrorCode LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, LLVMSharedModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx); @@ -153,10 +156,11 @@ LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, /** * Add an object file. */ -LLVMOrcModuleHandle LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, - LLVMSharedObjectBufferRef Obj, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx); +LLVMOrcErrorCode LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMSharedObjectBufferRef Obj, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); /** * Remove a module set from the JIT. @@ -164,18 +168,20 @@ LLVMOrcModuleHandle LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, * This works for all modules that can be added via OrcAdd*, including object * files. */ -void LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, LLVMOrcModuleHandle H); +LLVMOrcErrorCode LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle H); /** * Get symbol address from JIT instance. */ -LLVMOrcTargetAddress LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, - const char *SymbolName); +LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + const char *SymbolName); /** * Dispose of an ORC JIT stack. */ -void LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack); +LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack); #ifdef __cplusplus } diff --git a/contrib/llvm/include/llvm/ADT/APInt.h b/contrib/llvm/include/llvm/ADT/APInt.h index e5f0c35534ac..a1cce6e5fe17 100644 --- a/contrib/llvm/include/llvm/ADT/APInt.h +++ b/contrib/llvm/include/llvm/ADT/APInt.h @@ -401,7 +401,11 @@ class LLVM_NODISCARD APInt { /// \brief Determine if this is a value of 1. /// /// This checks to see if the value of this APInt is one. - bool isOneValue() const { return getActiveBits() == 1; } + bool isOneValue() const { + if (isSingleWord()) + return U.VAL == 1; + return countLeadingZerosSlowCase() == BitWidth - 1; + } /// \brief Determine if this is the largest unsigned value. /// diff --git a/contrib/llvm/include/llvm/ADT/STLExtras.h b/contrib/llvm/include/llvm/ADT/STLExtras.h index 8c28412bb607..83f289c42a23 100644 --- a/contrib/llvm/include/llvm/ADT/STLExtras.h +++ b/contrib/llvm/include/llvm/ADT/STLExtras.h @@ -100,6 +100,8 @@ class function_ref { } public: + function_ref() : callback(nullptr) {} + template function_ref(Callable &&callable, typename std::enable_if< @@ -110,6 +112,8 @@ class function_ref { Ret operator()(Params ...params) const { return callback(callable, std::forward(params)...); } + + operator bool() const { return callback; } }; // deleter - Very very very simple method that is used to invoke operator diff --git a/contrib/llvm/include/llvm/ADT/SmallPtrSet.h b/contrib/llvm/include/llvm/ADT/SmallPtrSet.h index a2ad74b1e04a..4e8a2490ee3c 100644 --- a/contrib/llvm/include/llvm/ADT/SmallPtrSet.h +++ b/contrib/llvm/include/llvm/ADT/SmallPtrSet.h @@ -15,9 +15,9 @@ #ifndef LLVM_ADT_SMALLPTRSET_H #define LLVM_ADT_SMALLPTRSET_H -#include "llvm/Config/abi-breaking.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/ReverseIteration.h" #include "llvm/Support/type_traits.h" #include #include @@ -29,15 +29,6 @@ namespace llvm { -#if LLVM_ENABLE_ABI_BREAKING_CHECKS -template struct ReverseIterate { static bool value; }; -#if LLVM_ENABLE_REVERSE_ITERATION -template bool ReverseIterate::value = true; -#else -template bool ReverseIterate::value = false; -#endif -#endif - /// SmallPtrSetImplBase - This is the common code shared among all the /// SmallPtrSet<>'s, which is almost everything. SmallPtrSet has two modes, one /// for small and one for large sets. diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 3e05e09900a5..5de3821242e0 100644 --- a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -1353,4 +1353,4 @@ struct BFIDOTGraphTraitsBase : public DefaultDOTGraphTraits { #undef DEBUG_TYPE -#endif +#endif // LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H diff --git a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h index a15a9e18c815..32868cbecdcf 100644 --- a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -577,12 +577,17 @@ class CGSCCToFunctionPassAdaptor // analyses will eventually occur when the module pass completes. PA.intersect(std::move(PassPA)); - // Update the call graph based on this function pass. This may also - // update the current SCC to point to a smaller, more refined SCC. - CurrentC = &updateCGAndAnalysisManagerForFunctionPass( - CG, *CurrentC, *N, AM, UR, DebugLogging); - assert(CG.lookupSCC(*N) == CurrentC && - "Current SCC not updated to the SCC containing the current node!"); + // If the call graph hasn't been preserved, update it based on this + // function pass. This may also update the current SCC to point to + // a smaller, more refined SCC. + auto PAC = PA.getChecker(); + if (!PAC.preserved() && !PAC.preservedSet>()) { + CurrentC = &updateCGAndAnalysisManagerForFunctionPass( + CG, *CurrentC, *N, AM, UR, DebugLogging); + assert( + CG.lookupSCC(*N) == CurrentC && + "Current SCC not updated to the SCC containing the current node!"); + } } // By definition we preserve the proxy. And we preserve all analyses on diff --git a/contrib/llvm/include/llvm/Analysis/InlineCost.h b/contrib/llvm/include/llvm/Analysis/InlineCost.h index ce0b7895f253..f33a2de5a5f4 100644 --- a/contrib/llvm/include/llvm/Analysis/InlineCost.h +++ b/contrib/llvm/include/llvm/Analysis/InlineCost.h @@ -160,7 +160,7 @@ InlineParams getInlineParams(int Threshold); /// the -Oz flag. InlineParams getInlineParams(unsigned OptLevel, unsigned SizeOptLevel); -/// Return the cost associated with a callsite, including paramater passing +/// Return the cost associated with a callsite, including parameter passing /// and the call/return instruction. int getCallsiteCost(CallSite CS, const DataLayout &DL); diff --git a/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h b/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h index ad7f5c80549f..3a052761ad7d 100644 --- a/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h +++ b/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h @@ -652,17 +652,23 @@ class LazyCallGraph { /// Make an existing internal ref edge into a call edge. /// /// This may form a larger cycle and thus collapse SCCs into TargetN's SCC. - /// If that happens, the deleted SCC pointers are returned. These SCCs are - /// not in a valid state any longer but the pointers will remain valid - /// until destruction of the parent graph instance for the purpose of - /// clearing cached information. + /// If that happens, the optional callback \p MergedCB will be invoked (if + /// provided) on the SCCs being merged away prior to actually performing + /// the merge. Note that this will never include the target SCC as that + /// will be the SCC functions are merged into to resolve the cycle. Once + /// this function returns, these merged SCCs are not in a valid state but + /// the pointers will remain valid until destruction of the parent graph + /// instance for the purpose of clearing cached information. This function + /// also returns 'true' if a cycle was formed and some SCCs merged away as + /// a convenience. /// /// After this operation, both SourceN's SCC and TargetN's SCC may move /// position within this RefSCC's postorder list. Any SCCs merged are /// merged into the TargetN's SCC in order to preserve reachability analyses /// which took place on that SCC. - SmallVector switchInternalEdgeToCall(Node &SourceN, - Node &TargetN); + bool switchInternalEdgeToCall( + Node &SourceN, Node &TargetN, + function_ref MergedSCCs)> MergeCB = {}); /// Make an existing internal call edge between separate SCCs into a ref /// edge. diff --git a/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h b/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h index 60dafccd84bd..23ab372703ee 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -224,6 +224,9 @@ class ObjectSizeOffsetVisitor SizeOffsetType visitSelectInst(SelectInst &I); SizeOffsetType visitUndefValue(UndefValue&); SizeOffsetType visitInstruction(Instruction &I); + +private: + bool CheckedZextOrTrunc(APInt &I); }; typedef std::pair SizeOffsetEvalType; diff --git a/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h b/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h index c0337b6daf37..cd4ec0a03a9e 100644 --- a/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h @@ -34,10 +34,10 @@ #include #include -namespace llvm { - #define DEBUG_TYPE "region" +namespace llvm { + //===----------------------------------------------------------------------===// /// RegionBase Implementation template @@ -901,8 +901,8 @@ void RegionInfoBase::calculate(FuncT &F) { buildRegionsTree(DT->getNode(BB), TopLevelRegion); } -#undef DEBUG_TYPE - } // end namespace llvm +#undef DEBUG_TYPE + #endif // LLVM_ANALYSIS_REGIONINFOIMPL_H diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h index 68fbf640994c..dfb525e3de7a 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -753,6 +753,28 @@ class TargetTransformInfo { Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType) const; + /// \returns The type to use in a loop expansion of a memcpy call. + Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAlign, unsigned DestAlign) const; + + /// \param[out] OpsOut The operand types to copy RemainingBytes of memory. + /// \param RemainingBytes The number of bytes to copy. + /// + /// Calculates the operand types to use when copying \p RemainingBytes of + /// memory, where source and destination alignments are \p SrcAlign and + /// \p DestAlign respectively. + void getMemcpyLoopResidualLoweringType(SmallVectorImpl &OpsOut, + LLVMContext &Context, + unsigned RemainingBytes, + unsigned SrcAlign, + unsigned DestAlign) const; + + /// \returns True if we want to test the new memcpy lowering functionality in + /// Transform/Utils. + /// Temporary. Will be removed once we move to the new functionality and + /// remove the old. + bool useWideIRMemcpyLoopLowering() const; + /// \returns True if the two functions have compatible attributes for inlining /// purposes. bool areInlineCompatible(const Function *Caller, @@ -953,6 +975,12 @@ class TargetTransformInfo::Concept { virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0; virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType) = 0; + virtual Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAlign, + unsigned DestAlign) const = 0; + virtual void getMemcpyLoopResidualLoweringType( + SmallVectorImpl &OpsOut, LLVMContext &Context, + unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const = 0; virtual bool areInlineCompatible(const Function *Caller, const Function *Callee) const = 0; virtual unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const = 0; @@ -1266,6 +1294,19 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept { Type *ExpectedType) override { return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); } + Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAlign, + unsigned DestAlign) const override { + return Impl.getMemcpyLoopLoweringType(Context, Length, SrcAlign, DestAlign); + } + void getMemcpyLoopResidualLoweringType(SmallVectorImpl &OpsOut, + LLVMContext &Context, + unsigned RemainingBytes, + unsigned SrcAlign, + unsigned DestAlign) const override { + Impl.getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, + SrcAlign, DestAlign); + } bool areInlineCompatible(const Function *Caller, const Function *Callee) const override { return Impl.areInlineCompatible(Caller, Callee); diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 0246fc1c02cc..8740ee92eed5 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -444,6 +444,20 @@ class TargetTransformInfoImplBase { return nullptr; } + Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAlign, unsigned DestAlign) const { + return Type::getInt8Ty(Context); + } + + void getMemcpyLoopResidualLoweringType(SmallVectorImpl &OpsOut, + LLVMContext &Context, + unsigned RemainingBytes, + unsigned SrcAlign, + unsigned DestAlign) const { + for (unsigned i = 0; i != RemainingBytes; ++i) + OpsOut.push_back(Type::getInt8Ty(Context)); + } + bool areInlineCompatible(const Function *Caller, const Function *Callee) const { return (Caller->getFnAttribute("target-cpu") == diff --git a/contrib/llvm/include/llvm/Analysis/ValueTracking.h b/contrib/llvm/include/llvm/Analysis/ValueTracking.h index e953ec8ab6ab..f4c57d4289fc 100644 --- a/contrib/llvm/include/llvm/Analysis/ValueTracking.h +++ b/contrib/llvm/include/llvm/Analysis/ValueTracking.h @@ -523,8 +523,7 @@ template class ArrayRef; /// (A) Optional isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, - bool InvertAPred = false, - unsigned Depth = 0, + bool LHSIsFalse = false, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr); diff --git a/contrib/llvm/include/llvm/BinaryFormat/Wasm.h b/contrib/llvm/include/llvm/BinaryFormat/Wasm.h index eef473b20dde..23e30b7a868d 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/contrib/llvm/include/llvm/BinaryFormat/Wasm.h @@ -94,7 +94,7 @@ struct WasmFunction { }; struct WasmDataSegment { - uint32_t Index; + uint32_t MemoryIndex; WasmInitExpr Offset; ArrayRef Content; }; @@ -107,7 +107,7 @@ struct WasmElemSegment { struct WasmRelocation { uint32_t Type; // The type of the relocation. - int32_t Index; // Index into function to global index space. + uint32_t Index; // Index into function to global index space. uint64_t Offset; // Offset from the start of the section. int64_t Addend; // A value to add to the symbol. }; diff --git a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 5435e48ff424..3777f956cf27 100644 --- a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -59,6 +59,8 @@ enum BlockIDs { FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID, SYMTAB_BLOCK_ID, + + SYNC_SCOPE_NAMES_BLOCK_ID, }; /// Identification block contains a string that describes the producer details, @@ -172,6 +174,10 @@ enum OperandBundleTagCode { OPERAND_BUNDLE_TAG = 1, // TAG: [strchr x N] }; +enum SyncScopeNameCode { + SYNC_SCOPE_NAME = 1, +}; + // Value symbol table codes. enum ValueSymtabCodes { VST_CODE_ENTRY = 1, // VST_ENTRY: [valueid, namechar x N] @@ -404,12 +410,6 @@ enum AtomicOrderingCodes { ORDERING_SEQCST = 6 }; -/// Encoded SynchronizationScope values. -enum AtomicSynchScopeCodes { - SYNCHSCOPE_SINGLETHREAD = 0, - SYNCHSCOPE_CROSSTHREAD = 1 -}; - /// Markers and flags for call instruction. enum CallMarkersFlags { CALL_TAIL = 0, diff --git a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h index c898667f1474..60bbc9aaa5bd 100644 --- a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -608,8 +608,8 @@ class AsmPrinter : public MachineFunctionPass { // Internal Implementation Details //===------------------------------------------------------------------===// - /// This emits visibility information about symbol, if this is suported by the - /// target. + /// This emits visibility information about symbol, if this is supported by + /// the target. void EmitVisibility(MCSymbol *Sym, unsigned Visibility, bool IsDefinition = true) const; diff --git a/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h index a740df96899d..b59fd60e8aed 100644 --- a/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -428,7 +428,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { std::pair LT = TLI->getTypeLegalizationCost(DL, Ty); - bool IsFloat = Ty->getScalarType()->isFloatingPointTy(); + bool IsFloat = Ty->isFPOrFPVectorTy(); // Assume that floating point arithmetic operations cost twice as much as // integer operations. unsigned OpCost = (IsFloat ? 2 : 1); diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index ec60123e54b1..59a4073646eb 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -16,14 +16,17 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H +#include "llvm/ADT/SmallVector.h" #include #include #include #include #include +#include namespace llvm { +class LLT; class MachineInstr; class MachineInstrBuilder; class MachineOperand; @@ -58,6 +61,131 @@ class PredicateBitsetImpl : public std::bitset { } }; +enum { + /// Record the specified instruction + /// - NewInsnID - Instruction ID to define + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + GIM_RecordInsn, + + /// Check the feature bits + /// - Expected features + GIM_CheckFeatures, + + /// Check the opcode on the specified instruction + /// - InsnID - Instruction ID + /// - Expected opcode + GIM_CheckOpcode, + /// Check the instruction has the right number of operands + /// - InsnID - Instruction ID + /// - Expected number of operands + GIM_CheckNumOperands, + + /// Check the type for the specified operand + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - Expected type + GIM_CheckType, + /// Check the register bank for the specified operand + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - Expected register bank (specified as a register class) + GIM_CheckRegBankForClass, + /// Check the operand matches a complex predicate + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - RendererID - The renderer to hold the result + /// - Complex predicate ID + GIM_CheckComplexPattern, + /// Check the operand is a specific integer + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - Expected integer + GIM_CheckConstantInt, + /// Check the operand is a specific literal integer (i.e. MO.isImm() or MO.isCImm() is true). + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - Expected integer + GIM_CheckLiteralInt, + /// Check the operand is a specific intrinsic ID + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - Expected Intrinsic ID + GIM_CheckIntrinsicID, + /// Check the specified operand is an MBB + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + GIM_CheckIsMBB, + + /// Check if the specified operand is safe to fold into the current + /// instruction. + /// - InsnID - Instruction ID + GIM_CheckIsSafeToFold, + + //=== Renderers === + + /// Mutate an instruction + /// - NewInsnID - Instruction ID to define + /// - OldInsnID - Instruction ID to mutate + /// - NewOpcode - The new opcode to use + GIR_MutateOpcode, + /// Build a new instruction + /// - InsnID - Instruction ID to define + /// - Opcode - The new opcode to use + GIR_BuildMI, + + /// Copy an operand to the specified instruction + /// - NewInsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to copy from + /// - OpIdx - The operand to copy + GIR_Copy, + /// Copy an operand to the specified instruction + /// - NewInsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to copy from + /// - OpIdx - The operand to copy + /// - SubRegIdx - The subregister to copy + GIR_CopySubReg, + /// Add an implicit register def to the specified instruction + /// - InsnID - Instruction ID to modify + /// - RegNum - The register to add + GIR_AddImplicitDef, + /// Add an implicit register use to the specified instruction + /// - InsnID - Instruction ID to modify + /// - RegNum - The register to add + GIR_AddImplicitUse, + /// Add an register to the specified instruction + /// - InsnID - Instruction ID to modify + /// - RegNum - The register to add + GIR_AddRegister, + /// Add an immediate to the specified instruction + /// - InsnID - Instruction ID to modify + /// - Imm - The immediate to add + GIR_AddImm, + /// Render complex operands to the specified instruction + /// - InsnID - Instruction ID to modify + /// - RendererID - The renderer to call + GIR_ComplexRenderer, + + /// Constrain an instruction operand to a register class. + /// - InsnID - Instruction ID to modify + /// - OpIdx - Operand index + /// - RCEnum - Register class enumeration value + GIR_ConstrainOperandRC, + /// Constrain an instructions operands according to the instruction + /// description. + /// - InsnID - Instruction ID to modify + GIR_ConstrainSelectedInstOperands, + /// Merge all memory operands into instruction. + /// - InsnID - Instruction ID to modify + GIR_MergeMemOperands, + /// Erase from parent. + /// - InsnID - Instruction ID to erase + GIR_EraseFromParent, + + /// A successful emission + GIR_Done, +}; + /// Provides the logic to select generic machine instructions. class InstructionSelector { public: @@ -78,9 +206,39 @@ class InstructionSelector { protected: using ComplexRendererFn = std::function; + using RecordedMIVector = SmallVector; + using NewMIVector = SmallVector; + struct MatcherState { + std::vector Renderers; + RecordedMIVector MIs; + + MatcherState(unsigned MaxRenderers); + }; + +public: + template + struct MatcherInfoTy { + const LLT *TypeObjects; + const PredicateBitset *FeatureBitsets; + const std::vector ComplexPredicates; + }; + +protected: InstructionSelector(); + /// Execute a given matcher table and return true if the match was successful + /// and false otherwise. + template + bool executeMatchTable( + TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, + const MatcherInfoTy &MatcherInfo, + const int64_t *MatchTable, const TargetInstrInfo &TII, + MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, + const RegisterBankInfo &RBI, + const PredicateBitset &AvailableFeatures) const; + /// Constrain a register operand of an instruction \p I to a specified /// register class. This could involve inserting COPYs before (for uses) or /// after (for defs) and may replace the operand of \p I. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h new file mode 100644 index 000000000000..98b6b859b9e2 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -0,0 +1,337 @@ +//==-- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h ---------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file This file declares the API for the instruction selector. +/// This class is responsible for selecting machine instructions. +/// It's implemented by the target. It's used by the InstructionSelect pass. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H +#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H + +namespace llvm { +template +bool InstructionSelector::executeMatchTable( + TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, + const MatcherInfoTy &MatcherInfo, + const int64_t *MatchTable, const TargetInstrInfo &TII, + MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, + const RegisterBankInfo &RBI, + const PredicateBitset &AvailableFeatures) const { + const int64_t *Command = MatchTable; + while (true) { + switch (*Command++) { + case GIM_RecordInsn: { + int64_t NewInsnID = *Command++; + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + + // As an optimisation we require that MIs[0] is always the root. Refuse + // any attempt to modify it. + assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); + (void)NewInsnID; + + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isReg()) { + DEBUG(dbgs() << "Rejected (not a register)\n"); + return false; + } + if (TRI.isPhysicalRegister(MO.getReg())) { + DEBUG(dbgs() << "Rejected (is a physical register)\n"); + return false; + } + + assert((size_t)NewInsnID == State.MIs.size() && + "Expected to store MIs in order"); + State.MIs.push_back(MRI.getVRegDef(MO.getReg())); + DEBUG(dbgs() << "MIs[" << NewInsnID << "] = GIM_RecordInsn(" << InsnID + << ", " << OpIdx << ")\n"); + break; + } + + case GIM_CheckFeatures: { + int64_t ExpectedBitsetID = *Command++; + DEBUG(dbgs() << "GIM_CheckFeatures(ExpectedBitsetID=" << ExpectedBitsetID + << ")\n"); + if ((AvailableFeatures & MatcherInfo.FeatureBitsets[ExpectedBitsetID]) != + MatcherInfo.FeatureBitsets[ExpectedBitsetID]) { + DEBUG(dbgs() << "Rejected\n"); + return false; + } + break; + } + + case GIM_CheckOpcode: { + int64_t InsnID = *Command++; + int64_t Expected = *Command++; + + unsigned Opcode = State.MIs[InsnID]->getOpcode(); + DEBUG(dbgs() << "GIM_CheckOpcode(MIs[" << InsnID << "], ExpectedOpcode=" + << Expected << ") // Got=" << Opcode << "\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (Opcode != Expected) + return false; + break; + } + case GIM_CheckNumOperands: { + int64_t InsnID = *Command++; + int64_t Expected = *Command++; + DEBUG(dbgs() << "GIM_CheckNumOperands(MIs[" << InsnID + << "], Expected=" << Expected << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (State.MIs[InsnID]->getNumOperands() != Expected) + return false; + break; + } + + case GIM_CheckType: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t TypeID = *Command++; + DEBUG(dbgs() << "GIM_CheckType(MIs[" << InsnID << "]->getOperand(" + << OpIdx << "), TypeID=" << TypeID << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) != + MatcherInfo.TypeObjects[TypeID]) + return false; + break; + } + case GIM_CheckRegBankForClass: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t RCEnum = *Command++; + DEBUG(dbgs() << "GIM_CheckRegBankForClass(MIs[" << InsnID + << "]->getOperand(" << OpIdx << "), RCEnum=" << RCEnum + << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (&RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != + RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, TRI)) + return false; + break; + } + case GIM_CheckComplexPattern: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t RendererID = *Command++; + int64_t ComplexPredicateID = *Command++; + DEBUG(dbgs() << "State.Renderers[" << RendererID + << "] = GIM_CheckComplexPattern(MIs[" << InsnID + << "]->getOperand(" << OpIdx + << "), ComplexPredicateID=" << ComplexPredicateID << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + // FIXME: Use std::invoke() when it's available. + if (!(State.Renderers[RendererID] = + (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( + State.MIs[InsnID]->getOperand(OpIdx)))) + return false; + break; + } + case GIM_CheckConstantInt: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t Value = *Command++; + DEBUG(dbgs() << "GIM_CheckConstantInt(MIs[" << InsnID << "]->getOperand(" + << OpIdx << "), Value=" << Value << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, MRI)) + return false; + break; + } + case GIM_CheckLiteralInt: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t Value = *Command++; + DEBUG(dbgs() << "GIM_CheckLiteralInt(MIs[" << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx); + if (!OM.isCImm() || !OM.getCImm()->equalsInt(Value)) + return false; + break; + } + case GIM_CheckIntrinsicID: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t Value = *Command++; + DEBUG(dbgs() << "GIM_CheckIntrinsicID(MIs[" << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx); + if (!OM.isIntrinsicID() || OM.getIntrinsicID() != Value) + return false; + break; + } + case GIM_CheckIsMBB: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + DEBUG(dbgs() << "GIM_CheckIsMBB(MIs[" << InsnID << "]->getOperand(" + << OpIdx << "))\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) + return false; + break; + } + + case GIM_CheckIsSafeToFold: { + int64_t InsnID = *Command++; + DEBUG(dbgs() << "GIM_CheckIsSafeToFold(MIs[" << InsnID << "])\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (!isObviouslySafeToFold(*State.MIs[InsnID])) + return false; + break; + } + + case GIR_MutateOpcode: { + int64_t OldInsnID = *Command++; + int64_t NewInsnID = *Command++; + int64_t NewOpcode = *Command++; + assert((size_t)NewInsnID == OutMIs.size() && + "Expected to store MIs in order"); + OutMIs.push_back( + MachineInstrBuilder(*State.MIs[OldInsnID]->getParent()->getParent(), + State.MIs[OldInsnID])); + OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); + DEBUG(dbgs() << "GIR_MutateOpcode(OutMIs[" << NewInsnID << "], MIs[" + << OldInsnID << "], " << NewOpcode << ")\n"); + break; + } + case GIR_BuildMI: { + int64_t InsnID = *Command++; + int64_t Opcode = *Command++; + assert((size_t)InsnID == OutMIs.size() && + "Expected to store MIs in order"); + (void)InsnID; + OutMIs.push_back(BuildMI(*State.MIs[0]->getParent(), State.MIs[0], + State.MIs[0]->getDebugLoc(), TII.get(Opcode))); + DEBUG(dbgs() << "GIR_BuildMI(OutMIs[" << InsnID << "], " << Opcode + << ")\n"); + break; + } + + case GIR_Copy: { + int64_t NewInsnID = *Command++; + int64_t OldInsnID = *Command++; + int64_t OpIdx = *Command++; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); + DEBUG(dbgs() << "GIR_Copy(OutMIs[" << NewInsnID << "], MIs[" << OldInsnID + << "], " << OpIdx << ")\n"); + break; + } + case GIR_CopySubReg: { + int64_t NewInsnID = *Command++; + int64_t OldInsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t SubRegIdx = *Command++; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), + 0, SubRegIdx); + DEBUG(dbgs() << "GIR_CopySubReg(OutMIs[" << NewInsnID << "], MIs[" + << OldInsnID << "], " << OpIdx << ", " << SubRegIdx + << ")\n"); + break; + } + case GIR_AddImplicitDef: { + int64_t InsnID = *Command++; + int64_t RegNum = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addDef(RegNum, RegState::Implicit); + DEBUG(dbgs() << "GIR_AddImplicitDef(OutMIs[" << InsnID << "], " << RegNum + << ")\n"); + break; + } + case GIR_AddImplicitUse: { + int64_t InsnID = *Command++; + int64_t RegNum = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addUse(RegNum, RegState::Implicit); + DEBUG(dbgs() << "GIR_AddImplicitUse(OutMIs[" << InsnID << "], " << RegNum + << ")\n"); + break; + } + case GIR_AddRegister: { + int64_t InsnID = *Command++; + int64_t RegNum = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addReg(RegNum); + DEBUG(dbgs() << "GIR_AddRegister(OutMIs[" << InsnID << "], " << RegNum + << ")\n"); + break; + } + case GIR_AddImm: { + int64_t InsnID = *Command++; + int64_t Imm = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addImm(Imm); + DEBUG(dbgs() << "GIR_AddImm(OutMIs[" << InsnID << "], " << Imm << ")\n"); + break; + } + case GIR_ComplexRenderer: { + int64_t InsnID = *Command++; + int64_t RendererID = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + State.Renderers[RendererID](OutMIs[InsnID]); + DEBUG(dbgs() << "GIR_ComplexRenderer(OutMIs[" << InsnID << "], " + << RendererID << ")\n"); + break; + } + + case GIR_ConstrainOperandRC: { + int64_t InsnID = *Command++; + int64_t OpIdx = *Command++; + int64_t RCEnum = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, + *TRI.getRegClass(RCEnum), TII, TRI, RBI); + DEBUG(dbgs() << "GIR_ConstrainOperandRC(OutMIs[" << InsnID << "], " + << OpIdx << ", " << RCEnum << ")\n"); + break; + } + case GIR_ConstrainSelectedInstOperands: { + int64_t InsnID = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, + RBI); + DEBUG(dbgs() << "GIR_ConstrainSelectedInstOperands(OutMIs[" << InsnID + << "])\n"); + break; + } + case GIR_MergeMemOperands: { + int64_t InsnID = *Command++; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + for (const auto *FromMI : State.MIs) + for (const auto &MMO : FromMI->memoperands()) + OutMIs[InsnID].addMemOperand(MMO); + DEBUG(dbgs() << "GIR_MergeMemOperands(OutMIs[" << InsnID << "])\n"); + break; + } + case GIR_EraseFromParent: { + int64_t InsnID = *Command++; + assert(State.MIs[InsnID] && + "Attempted to erase an undefined instruction"); + State.MIs[InsnID]->eraseFromParent(); + DEBUG(dbgs() << "GIR_EraseFromParent(MIs[" << InsnID << "])\n"); + break; + } + + case GIR_Done: + DEBUG(dbgs() << "GIR_Done"); + return true; + + default: + llvm_unreachable("Unexpected command"); + } + } +} + +} // end namespace llvm + +#endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 5197ba869c0a..1fd45b52e3ac 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -101,11 +101,11 @@ class LegalizerHelper { const LegalizerInfo &LI; }; -/// Helper function that replaces \p MI with a libcall. +/// Helper function that creates the given libcall. LegalizerHelper::LegalizeResult -replaceWithLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, - RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result, - ArrayRef Args); +createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, + const CallLowering::ArgInfo &Result, + ArrayRef Args); } // End namespace llvm. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index c9327d50432e..85e6fef1f3c2 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugLoc.h" @@ -59,6 +60,21 @@ class MachineIRBuilder { } void validateTruncExt(unsigned Dst, unsigned Src, bool IsExtend); + MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Res, unsigned Op0, unsigned Op1); + + unsigned getDestFromArg(unsigned Reg) { return Reg; } + unsigned getDestFromArg(LLT Ty) { + return getMF().getRegInfo().createGenericVirtualRegister(Ty); + } + unsigned getDestFromArg(const TargetRegisterClass *RC) { + return getMF().getRegInfo().createVirtualRegister(RC); + } + + unsigned getRegFromArg(unsigned Reg) { return Reg; } + + unsigned getRegFromArg(const MachineInstrBuilder &MIB) { + return MIB->getOperand(0).getReg(); + } public: /// Getter for the function we currently build. @@ -120,6 +136,22 @@ class MachineIRBuilder { /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildInstr(unsigned Opcode); + /// DAG like Generic method for building arbitrary instructions as above. + /// \Opc opcode for the instruction. + /// \Ty Either LLT/TargetRegisterClass/unsigned types for Dst + /// \Args Variadic list of uses of types(unsigned/MachineInstrBuilder) + /// Uses of type MachineInstrBuilder will perform + /// getOperand(0).getReg() to convert to register. + template + MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, + UseArgsTy &&... Args) { + auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); + unsigned It[] = {(getRegFromArg(Args))...}; + for (const auto &i : It) + MIB.addUse(i); + return MIB; + } + /// Build but don't insert = \p Opcode . /// /// \pre setMF, setBasicBlock or setMI must have been called. @@ -188,6 +220,11 @@ class MachineIRBuilder { /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildAdd(unsigned Res, unsigned Op0, unsigned Op1); + template + MachineInstrBuilder buildAdd(DstTy &&Ty, UseArgsTy &&... UseArgs) { + unsigned Res = getDestFromArg(Ty); + return buildAdd(Res, (getRegFromArg(UseArgs))...); + } /// Build and insert \p Res = G_SUB \p Op0, \p Op1 /// @@ -295,6 +332,18 @@ class MachineIRBuilder { MachineInstrBuilder buildAnd(unsigned Res, unsigned Op0, unsigned Op1); + /// Build and insert \p Res = G_OR \p Op0, \p Op1 + /// + /// G_OR sets \p Res to the bitwise or of integer parameters \p Op0 and \p + /// Op1. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers + /// with the same (scalar or vector) type). + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildOr(unsigned Res, unsigned Op0, unsigned Op1); + /// Build and insert \p Res = G_ANYEXT \p Op0 /// /// G_ANYEXT produces a register of the specified width, with bits 0 to @@ -416,6 +465,10 @@ class MachineIRBuilder { /// \return The newly created instruction. MachineInstrBuilder buildConstant(unsigned Res, int64_t Val); + template + MachineInstrBuilder buildConstant(DstType &&Res, int64_t Val) { + return buildConstant(getDestFromArg(Res), Val); + } /// Build and insert \p Res = G_FCONSTANT \p Val /// /// G_FCONSTANT is a floating-point constant with the specified size and diff --git a/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h b/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h index fa1ec867ea3d..c28b1a06854f 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h @@ -93,12 +93,14 @@ class LiveRegUnits { } /// Updates liveness when stepping backwards over the instruction \p MI. + /// This removes all register units defined or clobbered in \p MI and then + /// adds the units used (as in use operands) in \p MI. void stepBackward(const MachineInstr &MI); - /// Mark all register units live during instruction \p MI. - /// This can be used to accumulate live/unoccupied registers over a range of - /// instructions. - void accumulateBackward(const MachineInstr &MI); + /// Adds all register units used, defined or clobbered in \p MI. + /// This is useful when walking over a range of instruction to find registers + /// unused over the whole range. + void accumulate(const MachineInstr &MI); /// Adds registers living out of block \p MBB. /// Live out registers are the union of the live-in registers of the successor diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h index f67da7b01c54..19173fa39bdc 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h @@ -650,7 +650,7 @@ class MachineFunction { MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr, - SynchronizationScope SynchScope = CrossThread, + SyncScope::ID SSID = SyncScope::System, AtomicOrdering Ordering = AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); diff --git a/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h b/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h index 78adce507b8c..a9de0db05d72 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -114,6 +114,9 @@ class MachineMemOperand { MOInvariant = 1u << 5, // Reserved for use by target-specific passes. + // Targets may override getSerializableMachineMemOperandTargetFlags() to + // enable MIR serialization/parsing of these flags. If more of these flags + // are added, the MIR printing/parsing code will need to be updated as well. MOTargetFlag1 = 1u << 6, MOTargetFlag2 = 1u << 7, MOTargetFlag3 = 1u << 8, @@ -124,8 +127,8 @@ class MachineMemOperand { private: /// Atomic information for this memory operation. struct MachineAtomicInfo { - /// Synchronization scope for this memory operation. - unsigned SynchScope : 1; // enum SynchronizationScope + /// Synchronization scope ID for this memory operation. + unsigned SSID : 8; // SyncScope::ID /// Atomic ordering requirements for this memory operation. For cmpxchg /// atomic operations, atomic ordering requirements when store occurs. unsigned Ordering : 4; // enum AtomicOrdering @@ -152,7 +155,7 @@ class MachineMemOperand { unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr, - SynchronizationScope SynchScope = CrossThread, + SyncScope::ID SSID = SyncScope::System, AtomicOrdering Ordering = AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); @@ -202,9 +205,9 @@ class MachineMemOperand { /// Return the range tag for the memory reference. const MDNode *getRanges() const { return Ranges; } - /// Return the synchronization scope for this memory operation. - SynchronizationScope getSynchScope() const { - return static_cast(AtomicInfo.SynchScope); + /// Returns the synchronization scope ID for this memory operation. + SyncScope::ID getSyncScopeID() const { + return static_cast(AtomicInfo.SSID); } /// Return the atomic ordering requirements for this memory operation. For diff --git a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h index 8c3aacaa8efc..08151be11083 100644 --- a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h @@ -340,6 +340,18 @@ namespace RTLIB { MEMCPY_ELEMENT_UNORDERED_ATOMIC_8, MEMCPY_ELEMENT_UNORDERED_ATOMIC_16, + MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1, + MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2, + MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4, + MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8, + MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16, + + MEMSET_ELEMENT_UNORDERED_ATOMIC_1, + MEMSET_ELEMENT_UNORDERED_ATOMIC_2, + MEMSET_ELEMENT_UNORDERED_ATOMIC_4, + MEMSET_ELEMENT_UNORDERED_ATOMIC_8, + MEMSET_ELEMENT_UNORDERED_ATOMIC_16, + // EXCEPTION HANDLING UNWIND_RESUME, @@ -515,6 +527,17 @@ namespace RTLIB { /// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or /// UNKNOW_LIBCALL if there is none. Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize); + + /// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return + /// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or + /// UNKNOW_LIBCALL if there is none. + Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize); + + /// getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return + /// MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or + /// UNKNOW_LIBCALL if there is none. + Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize); + } } diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h index 4d72eda5c71a..25afc5b506df 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h @@ -235,6 +235,9 @@ class TargetRegisterInfo; "SDep::Output edge cannot use the zero register!"); Contents.Reg = Reg; } + + raw_ostream &print(raw_ostream &O, + const TargetRegisterInfo *TRI = nullptr) const; }; template <> @@ -458,7 +461,10 @@ class TargetRegisterInfo; void dump(const ScheduleDAG *G) const; void dumpAll(const ScheduleDAG *G) const; - void print(raw_ostream &O, const ScheduleDAG *G) const; + raw_ostream &print(raw_ostream &O, + const SUnit *N = nullptr, + const SUnit *X = nullptr) const; + raw_ostream &print(raw_ostream &O, const ScheduleDAG *G) const; private: void ComputeDepth(); diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h index f3f3003b7e20..55a23c3cca9b 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -927,7 +927,7 @@ class SelectionDAG { SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); + SyncScope::ID SSID); SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); @@ -937,7 +937,7 @@ class SelectionDAG { SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, - SynchronizationScope SynchScope); + SyncScope::ID SSID); SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO); diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index d9f8af0e21d1..db42fb6c170c 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1213,8 +1213,8 @@ class MemSDNode : public SDNode { /// Returns the Ranges that describes the dereference. const MDNode *getRanges() const { return MMO->getRanges(); } - /// Return the synchronization scope for this memory operation. - SynchronizationScope getSynchScope() const { return MMO->getSynchScope(); } + /// Returns the synchronization scope ID for this memory operation. + SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } /// Return the atomic ordering requirements for this memory operation. For /// cmpxchg atomic operations, return the atomic ordering requirements when @@ -1432,8 +1432,8 @@ class ConstantSDNode : public SDNode { int64_t getSExtValue() const { return Value->getSExtValue(); } bool isOne() const { return Value->isOne(); } - bool isNullValue() const { return Value->isNullValue(); } - bool isAllOnesValue() const { return Value->isAllOnesValue(); } + bool isNullValue() const { return Value->isZero(); } + bool isAllOnesValue() const { return Value->isMinusOne(); } bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h index 7941af8be8af..cdfc1745cea5 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -735,6 +735,10 @@ class Compile3Sym : public SymbolRecord { uint16_t VersionBackendQFE; StringRef Version; + void setLanguage(SourceLanguage Lang) { + Flags = CompileSym3Flags((uint32_t(Flags) & 0xFFFFFF00) | uint32_t(Lang)); + } + uint8_t getLanguage() const { return static_cast(Flags) & 0xFF; } uint32_t getFlags() const { return static_cast(Flags) & ~0xFF; } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h index 10d51c2d6244..e0c2226bdbd7 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -10,9 +10,11 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H #define LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/Support/Endian.h" #include #include +#include namespace llvm { @@ -265,6 +267,23 @@ struct TypeIndexOffset { void printTypeIndex(ScopedPrinter &Printer, StringRef FieldName, TypeIndex TI, TypeCollection &Types); } -} + +template <> struct DenseMapInfo { + static inline codeview::TypeIndex getEmptyKey() { + return codeview::TypeIndex{DenseMapInfo::getEmptyKey()}; + } + static inline codeview::TypeIndex getTombstoneKey() { + return codeview::TypeIndex{DenseMapInfo::getTombstoneKey()}; + } + static unsigned getHashValue(const codeview::TypeIndex &TI) { + return DenseMapInfo::getHashValue(TI.getIndex()); + } + static bool isEqual(const codeview::TypeIndex &LHS, + const codeview::TypeIndex &RHS) { + return LHS == RHS; + } +}; + +} // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/DebugInfo/DIContext.h b/contrib/llvm/include/llvm/DebugInfo/DIContext.h index 4126e245ff13..936813dc6abc 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DIContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DIContext.h @@ -204,7 +204,9 @@ class LoadedObjectInfo { /// need to be consistent with the addresses used to query the DIContext and /// the output of this function should be deterministic, i.e. repeated calls with /// the same Sec should give the same address. - virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const = 0; + virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const { + return 0; + } /// If conveniently available, return the content of the given Section. /// @@ -221,12 +223,28 @@ class LoadedObjectInfo { return false; } + // FIXME: This is untested and unused anywhere in the LLVM project, it's + // used/needed by Julia (an external project). It should have some coverage + // (at least tests, but ideally example functionality). /// Obtain a copy of this LoadedObjectInfo. - /// - /// The caller is responsible for deallocation once the copy is no longer required. virtual std::unique_ptr clone() const = 0; }; +template +struct LoadedObjectInfoHelper : Base { +protected: + LoadedObjectInfoHelper(const LoadedObjectInfoHelper &) = default; + LoadedObjectInfoHelper() = default; + +public: + template + LoadedObjectInfoHelper(Ts &&... Args) : Base(std::forward(Args)...) {} + + std::unique_ptr clone() const override { + return llvm::make_unique(static_cast(*this)); + } +}; + } // end namespace llvm #endif // LLVM_DEBUGINFO_DICONTEXT_H diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h index 739aa1f9ee74..ee2e805050c0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -226,11 +226,7 @@ class DWARFContext : public DIContext { virtual bool isLittleEndian() const = 0; virtual uint8_t getAddressSize() const = 0; virtual const DWARFSection &getInfoSection() = 0; - - using TypeSectionMap = MapVector>; - - virtual const TypeSectionMap &getTypesSections() = 0; + virtual void forEachTypesSections(function_ref F) = 0; virtual StringRef getAbbrevSection() = 0; virtual const DWARFSection &getLocSection() = 0; virtual StringRef getARangeSection() = 0; @@ -252,7 +248,8 @@ class DWARFContext : public DIContext { // Sections for DWARF5 split dwarf proposal. virtual const DWARFSection &getInfoDWOSection() = 0; - virtual const TypeSectionMap &getTypesDWOSections() = 0; + virtual void + forEachTypesDWOSections(function_ref F) = 0; virtual StringRef getAbbrevDWOSection() = 0; virtual const DWARFSection &getLineDWOSection() = 0; virtual const DWARFSection &getLocDWOSection() = 0; @@ -294,6 +291,9 @@ enum class ErrorPolicy { Halt, Continue }; class DWARFContextInMemory : public DWARFContext { virtual void anchor(); + using TypeSectionMap = MapVector>; + StringRef FileName; bool IsLittleEndian; uint8_t AddressSize; @@ -338,7 +338,8 @@ class DWARFContextInMemory : public DWARFContext { SmallVector, 4> UncompressedSections; - StringRef *MapSectionToMember(StringRef Name); + DWARFSection *mapNameToDWARFSection(StringRef Name); + StringRef *mapSectionToMember(StringRef Name); /// If Sec is compressed section, decompresses and updates its contents /// provided by Data. Otherwise leaves it unchanged. @@ -362,7 +363,10 @@ class DWARFContextInMemory : public DWARFContext { bool isLittleEndian() const override { return IsLittleEndian; } uint8_t getAddressSize() const override { return AddressSize; } const DWARFSection &getInfoSection() override { return InfoSection; } - const TypeSectionMap &getTypesSections() override { return TypesSections; } + void forEachTypesSections(function_ref F) override { + for (auto &P : TypesSections) + F(P.second); + } StringRef getAbbrevSection() override { return AbbrevSection; } const DWARFSection &getLocSection() override { return LocSection; } StringRef getARangeSection() override { return ARangeSection; } @@ -389,8 +393,9 @@ class DWARFContextInMemory : public DWARFContext { // Sections for DWARF5 split dwarf proposal. const DWARFSection &getInfoDWOSection() override { return InfoDWOSection; } - const TypeSectionMap &getTypesDWOSections() override { - return TypesDWOSections; + void forEachTypesDWOSections(function_ref F) override { + for (auto &P : TypesDWOSections) + F(P.second); } StringRef getAbbrevDWOSection() override { return AbbrevDWOSection; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h index e4cb1b24e30d..c918a5d5e976 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h @@ -47,6 +47,7 @@ class DbiModuleDescriptorBuilder { DbiModuleDescriptorBuilder & operator=(const DbiModuleDescriptorBuilder &) = delete; + void setPdbFilePathNI(uint32_t NI); void setObjFileName(StringRef Name); void addSymbol(codeview::CVSymbol Symbol); @@ -68,6 +69,10 @@ class DbiModuleDescriptorBuilder { uint32_t calculateSerializedLength() const; + /// Return the offset within the module symbol stream of the next symbol + /// record passed to addSymbol. Add four to account for the signature. + uint32_t getNextSymbolOffset() const { return SymbolByteSize + 4; } + void finalize(); Error finalizeMsfLayout(); @@ -81,6 +86,7 @@ class DbiModuleDescriptorBuilder { msf::MSFBuilder &MSF; uint32_t SymbolByteSize = 0; + uint32_t PdbFilePathNI = 0; std::string ModuleName; std::string ObjFileName; std::vector SourceFiles; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h index 3bf790726656..4be113f28d6f 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h @@ -83,6 +83,8 @@ class DbiStream { FixedStreamArray getSectionMap() const; void visitSectionContributions(ISectionContribVisitor &Visitor) const; + Expected getECName(uint32_t NI) const; + private: Error initializeSectionContributionData(); Error initializeSectionHeadersData(); diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h index 744411854181..63eb34f0326a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h @@ -15,6 +15,7 @@ #include "llvm/Support/Error.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryByteStream.h" @@ -54,8 +55,13 @@ class DbiStreamBuilder { // Add given bytes as a new stream. Error addDbgStream(pdb::DbgHeaderType Type, ArrayRef Data); + uint32_t addECName(StringRef Name); + uint32_t calculateSerializedLength() const; + void setPublicsStreamIndex(uint32_t Index); + void setSymbolRecordStreamIndex(uint32_t Index); + Expected addModuleInfo(StringRef ModuleName); Error addModuleSourceFile(StringRef Module, StringRef File); Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module, StringRef File); @@ -75,7 +81,7 @@ class DbiStreamBuilder { private: struct DebugStream { ArrayRef Data; - uint16_t StreamNumber = 0; + uint16_t StreamNumber = kInvalidStreamIndex; }; Error finalize(); @@ -87,7 +93,6 @@ class DbiStreamBuilder { uint32_t calculateNamesBufferSize() const; uint32_t calculateDbgStreamsSize() const; - Error generateModiSubstream(); Error generateFileInfoSubstream(); msf::MSFBuilder &Msf; @@ -100,6 +105,8 @@ class DbiStreamBuilder { uint16_t PdbDllRbld; uint16_t Flags; PDB_Machine MachineType; + uint32_t PublicsStreamIndex = kInvalidStreamIndex; + uint32_t SymRecordStreamIndex = kInvalidStreamIndex; const DbiStreamHeader *Header; @@ -108,6 +115,7 @@ class DbiStreamBuilder { StringMap SourceFileNames; + PDBStringTableBuilder ECNamesBuilder; WritableBinaryStreamRef NamesBuffer; MutableBinaryByteStream FileInfoBuffer; std::vector SectionContribs; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h index 25f66240a6a2..17a82b7ce12d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h @@ -44,7 +44,7 @@ class NamedStreamMap { bool get(StringRef Stream, uint32_t &StreamNo) const; void set(StringRef Stream, uint32_t StreamNo); void remove(StringRef Stream); - + const StringMap &getStringMap() const { return Mapping; } iterator_range> entries() const; private: diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h new file mode 100644 index 000000000000..4f532c6e3829 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h @@ -0,0 +1,49 @@ +//===- NativeBuiltinSymbol.h -------------------------------------- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEBUILTINSYMBOL_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEBUILTINSYMBOL_H + +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" + +#include "llvm/DebugInfo/PDB/PDBTypes.h" + +namespace llvm { +namespace pdb { + +class NativeSession; + +class NativeBuiltinSymbol : public NativeRawSymbol { +public: + NativeBuiltinSymbol(NativeSession &PDBSession, SymIndexId Id, + PDB_BuiltinType T, uint64_t L); + ~NativeBuiltinSymbol() override; + + virtual std::unique_ptr clone() const override; + + void dump(raw_ostream &OS, int Indent) const override; + + PDB_SymType getSymTag() const override; + + PDB_BuiltinType getBuiltinType() const override; + bool isConstType() const override; + uint64_t getLength() const override; + bool isUnalignedType() const override; + bool isVolatileType() const override; + +protected: + NativeSession &Session; + PDB_BuiltinType Type; + uint64_t Length; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h index 1687737f0e7f..bd5c09e5ff76 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h @@ -18,7 +18,7 @@ namespace pdb { class NativeCompilandSymbol : public NativeRawSymbol { public: - NativeCompilandSymbol(NativeSession &Session, uint32_t SymbolId, + NativeCompilandSymbol(NativeSession &Session, SymIndexId SymbolId, DbiModuleDescriptor MI); std::unique_ptr clone() const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h index 15bac78df191..ddb7f811da38 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h @@ -18,7 +18,7 @@ namespace pdb { class NativeExeSymbol : public NativeRawSymbol { public: - NativeExeSymbol(NativeSession &Session, uint32_t SymbolId); + NativeExeSymbol(NativeSession &Session, SymIndexId SymbolId); std::unique_ptr clone() const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h index a24a972879d2..66a9eae28e23 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h @@ -19,9 +19,11 @@ namespace pdb { class NativeSession; +typedef uint32_t SymIndexId; + class NativeRawSymbol : public IPDBRawSymbol { public: - NativeRawSymbol(NativeSession &PDBSession, uint32_t SymbolId); + NativeRawSymbol(NativeSession &PDBSession, SymIndexId SymbolId); virtual std::unique_ptr clone() const = 0; @@ -205,7 +207,7 @@ class NativeRawSymbol : public IPDBRawSymbol { protected: NativeSession &Session; - uint32_t SymbolId; + SymIndexId SymbolId; }; } // end namespace pdb diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h index dd40874dc5f2..b16ce231c349 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h @@ -10,9 +10,13 @@ #ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVESESSION_H #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVESESSION_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" #include "llvm/DebugInfo/PDB/IPDBSession.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" +#include "llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" @@ -35,6 +39,8 @@ class NativeSession : public IPDBSession { std::unique_ptr createCompilandSymbol(DbiModuleDescriptor MI); + SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI); + uint64_t getLoadAddress() const override; void setLoadAddress(uint64_t Address) override; std::unique_ptr getGlobalScope() override; @@ -77,6 +83,7 @@ class NativeSession : public IPDBSession { std::unique_ptr Pdb; std::unique_ptr Allocator; std::vector> SymbolCache; + DenseMap TypeIndexToSymbolId; }; } } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h index cd7d3b063793..2dc23f819d3b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -31,11 +31,13 @@ class MSFBuilder; namespace pdb { class DbiStreamBuilder; class InfoStreamBuilder; +class PublicsStreamBuilder; class TpiStreamBuilder; class PDBFileBuilder { public: explicit PDBFileBuilder(BumpPtrAllocator &Allocator); + ~PDBFileBuilder(); PDBFileBuilder(const PDBFileBuilder &) = delete; PDBFileBuilder &operator=(const PDBFileBuilder &) = delete; @@ -47,6 +49,7 @@ class PDBFileBuilder { TpiStreamBuilder &getTpiBuilder(); TpiStreamBuilder &getIpiBuilder(); PDBStringTableBuilder &getStringTableBuilder(); + PublicsStreamBuilder &getPublicsBuilder(); Error commit(StringRef Filename); @@ -61,6 +64,7 @@ class PDBFileBuilder { std::unique_ptr Msf; std::unique_ptr Info; std::unique_ptr Dbi; + std::unique_ptr Publics; std::unique_ptr Tpi; std::unique_ptr Ipi; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h index 86ef1136b41d..29167c966d42 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h @@ -56,7 +56,6 @@ class PDBStringTable { const PDBStringTableHeader *Header = nullptr; codeview::DebugStringTableSubsectionRef Strings; FixedStreamArray IDs; - uint32_t ByteSize = 0; uint32_t NameCount = 0; }; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h index 4570c80c76d7..9ace826bd8f7 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h @@ -25,8 +25,6 @@ struct GSIHashHeader; class PDBFile; class PublicsStream { - struct HeaderInfo; - public: PublicsStream(PDBFile &File, std::unique_ptr Stream); ~PublicsStream(); @@ -65,7 +63,7 @@ class PublicsStream { FixedStreamArray ThunkMap; FixedStreamArray SectionOffsets; - const HeaderInfo *Header; + const PublicsStreamHeader *Header; const GSIHashHeader *HashHdr; }; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h new file mode 100644 index 000000000000..5ab57ebef53d --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h @@ -0,0 +1,54 @@ +//===- PublicsStreamBuilder.h - PDB Publics Stream Creation -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBPUBLICSTREAMBUILDER_H +#define LLVM_DEBUGINFO_PDB_RAW_PDBPUBLICSTREAMBUILDER_H + +#include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/BinaryStreamRef.h" +#include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace msf { +class MSFBuilder; +} +namespace pdb { +class PublicsStream; +struct PublicsStreamHeader; + +class PublicsStreamBuilder { +public: + explicit PublicsStreamBuilder(msf::MSFBuilder &Msf); + ~PublicsStreamBuilder(); + + PublicsStreamBuilder(const PublicsStreamBuilder &) = delete; + PublicsStreamBuilder &operator=(const PublicsStreamBuilder &) = delete; + + Error finalizeMsfLayout(); + uint32_t calculateSerializedLength() const; + + Error commit(BinaryStreamWriter &PublicsWriter); + + uint32_t getStreamIndex() const { return StreamIdx; } + uint32_t getRecordStreamIdx() const { return RecordStreamIdx; } + +private: + uint32_t StreamIdx = kInvalidStreamIndex; + uint32_t RecordStreamIdx = kInvalidStreamIndex; + std::vector HashRecords; + msf::MSFBuilder &Msf; +}; +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h index 771272d6a47d..a3cdd3f09a44 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h @@ -255,6 +255,19 @@ struct ModuleInfoHeader { /// char ObjFileName[]; }; +// This is PSGSIHDR struct defined in +// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h +struct PublicsStreamHeader { + support::ulittle32_t SymHash; + support::ulittle32_t AddrMap; + support::ulittle32_t NumThunks; + support::ulittle32_t SizeOfThunk; + support::ulittle16_t ISectThunkTable; + char Padding[2]; + support::ulittle32_t OffThunkTable; + support::ulittle32_t NumSections; +}; + /// Defines a 128-bit unique identifier. This maps to a GUID on Windows, but /// is abstracted here for the purposes of non-Windows platforms that don't have /// the GUID structure defined. diff --git a/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h index f09e95fddb97..4172f240ba39 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h @@ -21,6 +21,8 @@ #include #include +#include "llvm/Support/Error.h" + namespace llvm { class GlobalValue; @@ -41,10 +43,11 @@ class JITSymbolFlags { enum FlagNames : UnderlyingType { None = 0, - Weak = 1U << 0, - Common = 1U << 1, - Absolute = 1U << 2, - Exported = 1U << 3 + HasError = 1U << 0, + Weak = 1U << 1, + Common = 1U << 2, + Absolute = 1U << 3, + Exported = 1U << 4 }; /// @brief Default-construct a JITSymbolFlags instance. @@ -53,6 +56,11 @@ class JITSymbolFlags { /// @brief Construct a JITSymbolFlags instance from the given flags. JITSymbolFlags(FlagNames Flags) : Flags(Flags) {} + /// @brief Return true if there was an error retrieving this symbol. + bool hasError() const { + return (Flags & HasError) == HasError; + } + /// @brief Returns true is the Weak flag is set. bool isWeak() const { return (Flags & Weak) == Weak; @@ -113,11 +121,17 @@ class JITEvaluatedSymbol { /// @brief Represents a symbol in the JIT. class JITSymbol { public: - using GetAddressFtor = std::function; + using GetAddressFtor = std::function()>; - /// @brief Create a 'null' symbol that represents failure to find a symbol - /// definition. - JITSymbol(std::nullptr_t) {} + /// @brief Create a 'null' symbol, used to represent a "symbol not found" + /// result from a successful (non-erroneous) lookup. + JITSymbol(std::nullptr_t) + : CachedAddr(0) {} + + /// @brief Create a JITSymbol representing an error in the symbol lookup + /// process (e.g. a network failure during a remote lookup). + JITSymbol(Error Err) + : Err(std::move(Err)), Flags(JITSymbolFlags::HasError) {} /// @brief Create a symbol for a definition with a known address. JITSymbol(JITTargetAddress Addr, JITSymbolFlags Flags) @@ -137,18 +151,59 @@ class JITSymbol { /// user can materialize the definition at any time by calling the getAddress /// method. JITSymbol(GetAddressFtor GetAddress, JITSymbolFlags Flags) - : GetAddress(std::move(GetAddress)), Flags(Flags) {} + : GetAddress(std::move(GetAddress)), CachedAddr(0), Flags(Flags) {} + + JITSymbol(const JITSymbol&) = delete; + JITSymbol& operator=(const JITSymbol&) = delete; + + JITSymbol(JITSymbol &&Other) + : GetAddress(std::move(Other.GetAddress)), Flags(std::move(Other.Flags)) { + if (Flags.hasError()) + Err = std::move(Other.Err); + else + CachedAddr = std::move(Other.CachedAddr); + } + + JITSymbol& operator=(JITSymbol &&Other) { + GetAddress = std::move(Other.GetAddress); + Flags = std::move(Other.Flags); + if (Flags.hasError()) + Err = std::move(Other.Err); + else + CachedAddr = std::move(Other.CachedAddr); + return *this; + } + + ~JITSymbol() { + if (Flags.hasError()) + Err.~Error(); + else + CachedAddr.~JITTargetAddress(); + } /// @brief Returns true if the symbol exists, false otherwise. - explicit operator bool() const { return CachedAddr || GetAddress; } + explicit operator bool() const { + return !Flags.hasError() && (CachedAddr || GetAddress); + } + + /// @brief Move the error field value out of this JITSymbol. + Error takeError() { + if (Flags.hasError()) + return std::move(Err); + return Error::success(); + } /// @brief Get the address of the symbol in the target address space. Returns /// '0' if the symbol does not exist. - JITTargetAddress getAddress() { + Expected getAddress() { + assert(!Flags.hasError() && "getAddress called on error value"); if (GetAddress) { - CachedAddr = GetAddress(); - assert(CachedAddr && "Symbol could not be materialized."); - GetAddress = nullptr; + if (auto CachedAddrOrErr = GetAddress()) { + GetAddress = nullptr; + CachedAddr = *CachedAddrOrErr; + assert(CachedAddr && "Symbol could not be materialized."); + } else + return CachedAddrOrErr.takeError(); } return CachedAddr; } @@ -157,7 +212,10 @@ class JITSymbol { private: GetAddressFtor GetAddress; - JITTargetAddress CachedAddr = 0; + union { + JITTargetAddress CachedAddr; + Error Err; + }; JITSymbolFlags Flags; }; diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 8ac1b6bca0a7..c1acca386820 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -146,7 +146,7 @@ class CompileOnDemandLayer { std::unique_ptr)>; struct SourceModuleEntry { - std::unique_ptr> SourceMod; + std::shared_ptr SourceMod; std::set StubsToClone; }; @@ -154,7 +154,7 @@ class CompileOnDemandLayer { using SourceModuleHandle = typename SourceModulesList::size_type; SourceModuleHandle - addSourceModule(std::unique_ptr> M) { + addSourceModule(std::shared_ptr M) { SourceModuleHandle H = SourceModules.size(); SourceModules.push_back(SourceModuleEntry()); SourceModules.back().SourceMod = std::move(M); @@ -162,7 +162,7 @@ class CompileOnDemandLayer { } Module& getSourceModule(SourceModuleHandle H) { - return SourceModules[H].SourceMod->getResource(); + return *SourceModules[H].SourceMod; } std::set& getStubsToClone(SourceModuleHandle H) { @@ -176,19 +176,21 @@ class CompileOnDemandLayer { for (auto BLH : BaseLayerHandles) if (auto Sym = BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly)) return Sym; + else if (auto Err = Sym.takeError()) + return std::move(Err); return nullptr; } - void removeModulesFromBaseLayer(BaseLayerT &BaseLayer) { + Error removeModulesFromBaseLayer(BaseLayerT &BaseLayer) { for (auto &BLH : BaseLayerHandles) - BaseLayer.removeModule(BLH); + if (auto Err = BaseLayer.removeModule(BLH)) + return Err; + return Error::success(); } - std::unique_ptr ExternalSymbolResolver; - std::unique_ptr> MemMgr; + std::shared_ptr ExternalSymbolResolver; std::unique_ptr StubsMgr; StaticGlobalRenamer StaticRenamer; - ModuleAdderFtor ModuleAdder; SourceModulesList SourceModules; std::vector BaseLayerHandles; }; @@ -196,6 +198,7 @@ class CompileOnDemandLayer { using LogicalDylibList = std::list; public: + /// @brief Handle to loaded module. using ModuleHandleT = typename LogicalDylibList::iterator; @@ -217,48 +220,41 @@ class CompileOnDemandLayer { CloneStubsIntoPartitions(CloneStubsIntoPartitions) {} ~CompileOnDemandLayer() { + // FIXME: Report error on log. while (!LogicalDylibs.empty()) - removeModule(LogicalDylibs.begin()); + consumeError(removeModule(LogicalDylibs.begin())); } /// @brief Add a module to the compile-on-demand layer. - template - ModuleHandleT addModule(std::shared_ptr M, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { + Expected + addModule(std::shared_ptr M, + std::shared_ptr Resolver) { LogicalDylibs.push_back(LogicalDylib()); auto &LD = LogicalDylibs.back(); LD.ExternalSymbolResolver = std::move(Resolver); LD.StubsMgr = CreateIndirectStubsManager(); - auto &MemMgrRef = *MemMgr; - LD.MemMgr = wrapOwnership(std::move(MemMgr)); - - LD.ModuleAdder = - [&MemMgrRef](BaseLayerT &B, std::unique_ptr M, - std::unique_ptr R) { - return B.addModule(std::move(M), &MemMgrRef, std::move(R)); - }; - // Process each of the modules in this module set. - addLogicalModule(LogicalDylibs.back(), std::move(M)); + if (auto Err = addLogicalModule(LD, std::move(M))) + return std::move(Err); return std::prev(LogicalDylibs.end()); } /// @brief Add extra modules to an existing logical module. - void addExtraModule(ModuleHandleT H, std::shared_ptr M) { - addLogicalModule(*H, std::move(M)); + Error addExtraModule(ModuleHandleT H, std::shared_ptr M) { + return addLogicalModule(*H, std::move(M)); } /// @brief Remove the module represented by the given handle. /// /// This will remove all modules in the layers below that were derived from /// the module represented by H. - void removeModule(ModuleHandleT H) { - H->removeModulesFromBaseLayer(BaseLayer); + Error removeModule(ModuleHandleT H) { + auto Err = H->removeModulesFromBaseLayer(BaseLayer); LogicalDylibs.erase(H); + return Err; } /// @brief Search for the given named symbol. @@ -272,6 +268,8 @@ class CompileOnDemandLayer { return Sym; if (auto Sym = findSymbolIn(LDI, Name, ExportedSymbolsOnly)) return Sym; + else if (auto Err = Sym.takeError()) + return std::move(Err); } return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } @@ -309,8 +307,9 @@ class CompileOnDemandLayer { } private: - template - void addLogicalModule(LogicalDylib &LD, ModulePtrT SrcMPtr) { + + Error addLogicalModule(LogicalDylib &LD, std::shared_ptr SrcMPtr) { + // Rename all static functions / globals to $static.X : // This will unique the names across all modules in the logical dylib, // simplifying symbol lookup. @@ -322,7 +321,7 @@ class CompileOnDemandLayer { // Create a logical module handle for SrcM within the logical dylib. Module &SrcM = *SrcMPtr; - auto LMId = LD.addSourceModule(wrapOwnership(std::move(SrcMPtr))); + auto LMId = LD.addSourceModule(std::move(SrcMPtr)); // Create stub functions. const DataLayout &DL = SrcM.getDataLayout(); @@ -335,9 +334,12 @@ class CompileOnDemandLayer { // Skip weak functions for which we already have definitions. auto MangledName = mangle(F.getName(), DL); - if (F.hasWeakLinkage() || F.hasLinkOnceLinkage()) + if (F.hasWeakLinkage() || F.hasLinkOnceLinkage()) { if (auto Sym = LD.findSymbol(BaseLayer, MangledName, false)) continue; + else if (auto Err = Sym.takeError()) + return std::move(Err); + } // Record all functions defined by this module. if (CloneStubsIntoPartitions) @@ -350,9 +352,15 @@ class CompileOnDemandLayer { StubInits[MangledName] = std::make_pair(CCInfo.getAddress(), JITSymbolFlags::fromGlobalValue(F)); - CCInfo.setCompileAction([this, &LD, LMId, &F]() { - return this->extractAndCompile(LD, LMId, F); - }); + CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress { + if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) + return *FnImplAddrOrErr; + else { + // FIXME: Report error, return to 'abort' or something similar. + consumeError(FnImplAddrOrErr.takeError()); + return 0; + } + }); } auto EC = LD.StubsMgr->createStubs(StubInits); @@ -367,7 +375,7 @@ class CompileOnDemandLayer { // empty globals module. if (SrcM.global_empty() && SrcM.alias_empty() && !SrcM.getModuleFlagsMetadata()) - return; + return Error::success(); // Create the GlobalValues module. auto GVsM = llvm::make_unique((SrcM.getName() + ".globals").str(), @@ -393,8 +401,9 @@ class CompileOnDemandLayer { // Initializers may refer to functions declared (but not defined) in this // module. Build a materializer to clone decls on demand. + Error MaterializerErrors = Error::success(); auto Materializer = createLambdaMaterializer( - [&LD, &GVsM](Value *V) -> Value* { + [&LD, &GVsM, &MaterializerErrors](Value *V) -> Value* { if (auto *F = dyn_cast(V)) { // Decls in the original module just get cloned. if (F->isDeclaration()) @@ -405,13 +414,24 @@ class CompileOnDemandLayer { // instead. const DataLayout &DL = GVsM->getDataLayout(); std::string FName = mangle(F->getName(), DL); - auto StubSym = LD.StubsMgr->findStub(FName, false); unsigned PtrBitWidth = DL.getPointerTypeSizeInBits(F->getType()); - ConstantInt *StubAddr = - ConstantInt::get(GVsM->getContext(), - APInt(PtrBitWidth, StubSym.getAddress())); + JITTargetAddress StubAddr = 0; + + // Get the address for the stub. If we encounter an error while + // doing so, stash it in the MaterializerErrors variable and use a + // null address as a placeholder. + if (auto StubSym = LD.StubsMgr->findStub(FName, false)) { + if (auto StubAddrOrErr = StubSym.getAddress()) + StubAddr = *StubAddrOrErr; + else + MaterializerErrors = joinErrors(std::move(MaterializerErrors), + StubAddrOrErr.takeError()); + } + + ConstantInt *StubAddrCI = + ConstantInt::get(GVsM->getContext(), APInt(PtrBitWidth, StubAddr)); Constant *Init = ConstantExpr::getCast(Instruction::IntToPtr, - StubAddr, F->getType()); + StubAddrCI, F->getType()); return GlobalAlias::create(F->getFunctionType(), F->getType()->getAddressSpace(), F->getLinkage(), F->getName(), @@ -435,22 +455,31 @@ class CompileOnDemandLayer { NewA->setAliasee(cast(Init)); } + if (MaterializerErrors) + return MaterializerErrors; + // Build a resolver for the globals module and add it to the base layer. auto GVsResolver = createLambdaResolver( - [this, &LD](const std::string &Name) { + [this, &LD](const std::string &Name) -> JITSymbol { if (auto Sym = LD.StubsMgr->findStub(Name, false)) return Sym; if (auto Sym = LD.findSymbol(BaseLayer, Name, false)) return Sym; + else if (auto Err = Sym.takeError()) + return std::move(Err); return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name); }, [&LD](const std::string &Name) { return LD.ExternalSymbolResolver->findSymbol(Name); }); - auto GVsH = LD.ModuleAdder(BaseLayer, std::move(GVsM), - std::move(GVsResolver)); - LD.BaseLayerHandles.push_back(GVsH); + if (auto GVsHOrErr = + BaseLayer.addModule(std::move(GVsM), std::move(GVsResolver))) + LD.BaseLayerHandles.push_back(*GVsHOrErr); + else + return GVsHOrErr.takeError(); + + return Error::success(); } static std::string mangle(StringRef Name, const DataLayout &DL) { @@ -462,7 +491,7 @@ class CompileOnDemandLayer { return MangledName; } - JITTargetAddress + Expected extractAndCompile(LogicalDylib &LD, typename LogicalDylib::SourceModuleHandle LMId, Function &F) { @@ -475,34 +504,42 @@ class CompileOnDemandLayer { // Grab the name of the function being called here. std::string CalledFnName = mangle(F.getName(), SrcM.getDataLayout()); - auto Part = Partition(F); - auto PartH = emitPartition(LD, LMId, Part); - JITTargetAddress CalledAddr = 0; - for (auto *SubF : Part) { - std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout()); - auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false); - assert(FnBodySym && "Couldn't find function body."); + auto Part = Partition(F); + if (auto PartHOrErr = emitPartition(LD, LMId, Part)) { + auto &PartH = *PartHOrErr; + for (auto *SubF : Part) { + std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout()); + if (auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false)) { + if (auto FnBodyAddrOrErr = FnBodySym.getAddress()) { + JITTargetAddress FnBodyAddr = *FnBodyAddrOrErr; - JITTargetAddress FnBodyAddr = FnBodySym.getAddress(); + // If this is the function we're calling record the address so we can + // return it from this function. + if (SubF == &F) + CalledAddr = FnBodyAddr; - // If this is the function we're calling record the address so we can - // return it from this function. - if (SubF == &F) - CalledAddr = FnBodyAddr; + // Update the function body pointer for the stub. + if (auto EC = LD.StubsMgr->updatePointer(FnName, FnBodyAddr)) + return 0; - // Update the function body pointer for the stub. - if (auto EC = LD.StubsMgr->updatePointer(FnName, FnBodyAddr)) - return 0; - } + } else + return FnBodyAddrOrErr.takeError(); + } else if (auto Err = FnBodySym.takeError()) + return std::move(Err); + else + llvm_unreachable("Function not emitted for partition"); + } - LD.BaseLayerHandles.push_back(PartH); + LD.BaseLayerHandles.push_back(PartH); + } else + return PartHOrErr.takeError(); return CalledAddr; } template - BaseLayerModuleHandleT + Expected emitPartition(LogicalDylib &LD, typename LogicalDylib::SourceModuleHandle LMId, const PartitionT &Part) { @@ -566,16 +603,18 @@ class CompileOnDemandLayer { // Create memory manager and symbol resolver. auto Resolver = createLambdaResolver( - [this, &LD](const std::string &Name) { + [this, &LD](const std::string &Name) -> JITSymbol { if (auto Sym = LD.findSymbol(BaseLayer, Name, false)) return Sym; + else if (auto Err = Sym.takeError()) + return std::move(Err); return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name); }, [&LD](const std::string &Name) { return LD.ExternalSymbolResolver->findSymbol(Name); }); - return LD.ModuleAdder(BaseLayer, std::move(M), std::move(Resolver)); + return BaseLayer.addModule(std::move(M), std::move(Resolver)); } BaseLayerT &BaseLayer; diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index bf8cca406844..d9b45c6a1e29 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -17,6 +17,8 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/iterator_range.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" #include #include #include @@ -99,19 +101,24 @@ class CtorDtorRunner { /// @brief Run the recorded constructors/destructors through the given JIT /// layer. - bool runViaLayer(JITLayerT &JITLayer) const { + Error runViaLayer(JITLayerT &JITLayer) const { using CtorDtorTy = void (*)(); - bool Error = false; for (const auto &CtorDtorName : CtorDtorNames) if (auto CtorDtorSym = JITLayer.findSymbolIn(H, CtorDtorName, false)) { - CtorDtorTy CtorDtor = - reinterpret_cast( - static_cast(CtorDtorSym.getAddress())); - CtorDtor(); - } else - Error = true; - return !Error; + if (auto AddrOrErr = CtorDtorSym.getAddress()) { + CtorDtorTy CtorDtor = + reinterpret_cast(static_cast(*AddrOrErr)); + CtorDtor(); + } else + return AddrOrErr.takeError(); + } else { + if (auto Err = CtorDtorSym.takeError()) + return Err; + else + return make_error(CtorDtorName); + } + return Error::success(); } private: diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h index d582e9a33241..ff54ef625ebb 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h @@ -17,9 +17,14 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include +#include #include namespace llvm { + +class Module; +class JITSymbolResolver; + namespace orc { /// @brief Global mapping layer. @@ -32,25 +37,22 @@ namespace orc { template class GlobalMappingLayer { public: - /// @brief Handle to a set of added modules. - using ModuleSetHandleT = typename BaseLayerT::ModuleSetHandleT; + + /// @brief Handle to an added module. + using ModuleHandleT = typename BaseLayerT::ModuleHandleT; /// @brief Construct an GlobalMappingLayer with the given BaseLayer GlobalMappingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} - /// @brief Add the given module set to the JIT. + /// @brief Add the given module to the JIT. /// @return A handle for the added modules. - template - ModuleSetHandleT addModuleSet(ModuleSetT Ms, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { - return BaseLayer.addModuleSet(std::move(Ms), std::move(MemMgr), - std::move(Resolver)); + ModuleHandleT addModule(std::shared_ptr M, + std::shared_ptr Resolver) { + return BaseLayer.addModule(std::move(M), std::move(Resolver)); } /// @brief Remove the module set associated with the handle H. - void removeModuleSet(ModuleSetHandleT H) { BaseLayer.removeModuleSet(H); } + void removeModule(ModuleHandleT H) { BaseLayer.removeModule(H); } /// @brief Manually set the address to return for the given symbol. void setGlobalMapping(const std::string &Name, JITTargetAddress Addr) { @@ -78,15 +80,15 @@ class GlobalMappingLayer { return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } - /// @brief Get the address of the given symbol in the context of the set of - /// modules represented by the handle H. This call is forwarded to the + /// @brief Get the address of the given symbol in the context of the of the + /// module represented by the handle H. This call is forwarded to the /// base layer's implementation. - /// @param H The handle for the module set to search in. + /// @param H The handle for the module to search in. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it is found in the - /// given module set. - JITSymbol findSymbolIn(ModuleSetHandleT H, const std::string &Name, + /// given module. + JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, bool ExportedSymbolsOnly) { return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly); } @@ -94,7 +96,7 @@ class GlobalMappingLayer { /// @brief Immediately emit and finalize the module set represented by the /// given handle. /// @param H Handle for module set to emit/finalize. - void emitAndFinalize(ModuleSetHandleT H) { + void emitAndFinalize(ModuleHandleT H) { BaseLayer.emitAndFinalize(H); } diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h index 99ccd4d221a5..fadd334bed0f 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -50,18 +50,18 @@ class IRCompileLayer { /// along with the given memory manager and symbol resolver. /// /// @return A handle for the added module. - template - ModuleHandleT addModule(std::shared_ptr M, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { + Expected + addModule(std::shared_ptr M, + std::shared_ptr Resolver) { using CompileResult = decltype(Compile(*M)); auto Obj = std::make_shared(Compile(*M)); - return BaseLayer.addObject(std::move(Obj), std::move(MemMgr), - std::move(Resolver)); + return BaseLayer.addObject(std::move(Obj), std::move(Resolver)); } /// @brief Remove the module associated with the handle H. - void removeModule(ModuleHandleT H) { BaseLayer.removeObject(H); } + Error removeModule(ModuleHandleT H) { + return BaseLayer.removeObject(H); + } /// @brief Search for the given named symbol. /// @param Name The name of the symbol to search for. @@ -87,8 +87,8 @@ class IRCompileLayer { /// @brief Immediately emit and finalize the module represented by the given /// handle. /// @param H Handle for module to emit/finalize. - void emitAndFinalize(ModuleHandleT H) { - BaseLayer.emitAndFinalize(H); + Error emitAndFinalize(ModuleHandleT H) { + return BaseLayer.emitAndFinalize(H); } private: diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index cf6556a33bbd..476061afda59 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -42,16 +42,14 @@ class IRTransformLayer { /// the layer below, along with the memory manager and symbol resolver. /// /// @return A handle for the added modules. - template - ModuleHandleT addModule(std::shared_ptr M, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { - return BaseLayer.addModule(Transform(std::move(M)), std::move(MemMgr), - std::move(Resolver)); + Expected + addModule(std::shared_ptr M, + std::shared_ptr Resolver) { + return BaseLayer.addModule(Transform(std::move(M)), std::move(Resolver)); } /// @brief Remove the module associated with the handle H. - void removeModule(ModuleHandleT H) { BaseLayer.removeModule(H); } + Error removeModule(ModuleHandleT H) { return BaseLayer.removeModule(H); } /// @brief Search for the given named symbol. /// @param Name The name of the symbol to search for. @@ -77,8 +75,8 @@ class IRTransformLayer { /// @brief Immediately emit and finalize the module represented by the given /// handle. /// @param H Handle for module to emit/finalize. - void emitAndFinalize(ModuleHandleT H) { - BaseLayer.emitAndFinalize(H); + Error emitAndFinalize(ModuleHandleT H) { + return BaseLayer.emitAndFinalize(H); } /// @brief Access the transform functor directly. diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/LambdaResolver.h index 6868640d38e8..228392ae0d4a 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/LambdaResolver.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/LambdaResolver.h @@ -45,7 +45,7 @@ class LambdaResolver : public JITSymbolResolver { template -std::unique_ptr> +std::shared_ptr> createLambdaResolver(DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver; diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index 38769aac12af..6c951fab6185 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -46,8 +46,9 @@ template class LazyEmittingLayer { private: class EmissionDeferredModule { public: - EmissionDeferredModule() = default; - virtual ~EmissionDeferredModule() = default; + EmissionDeferredModule(std::shared_ptr M, + std::shared_ptr Resolver) + : M(std::move(M)), Resolver(std::move(Resolver)) {} JITSymbol find(StringRef Name, bool ExportedSymbolsOnly, BaseLayerT &B) { switch (EmitState) { @@ -59,16 +60,24 @@ template class LazyEmittingLayer { std::string PName = Name; JITSymbolFlags Flags = JITSymbolFlags::fromGlobalValue(*GV); auto GetAddress = - [this, ExportedSymbolsOnly, PName, &B]() -> JITTargetAddress { + [this, ExportedSymbolsOnly, PName, &B]() -> Expected { if (this->EmitState == Emitting) return 0; else if (this->EmitState == NotEmitted) { this->EmitState = Emitting; - Handle = this->emitToBaseLayer(B); + if (auto HandleOrErr = this->emitToBaseLayer(B)) + Handle = std::move(*HandleOrErr); + else + return HandleOrErr.takeError(); this->EmitState = Emitted; } - auto Sym = B.findSymbolIn(Handle, PName, ExportedSymbolsOnly); - return Sym.getAddress(); + if (auto Sym = B.findSymbolIn(Handle, PName, ExportedSymbolsOnly)) + return Sym.getAddress(); + else if (auto Err = Sym.takeError()) + return std::move(Err); + else + llvm_unreachable("Successful symbol lookup should return " + "definition address here"); }; return JITSymbol(std::move(GetAddress), Flags); } else @@ -101,33 +110,10 @@ template class LazyEmittingLayer { BaseLayer.emitAndFinalize(Handle); } - template - static std::unique_ptr - create(BaseLayerT &B, std::shared_ptr M, MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver); - - protected: - virtual const GlobalValue* searchGVs(StringRef Name, - bool ExportedSymbolsOnly) const = 0; - virtual BaseLayerHandleT emitToBaseLayer(BaseLayerT &BaseLayer) = 0; - private: - enum { NotEmitted, Emitting, Emitted } EmitState = NotEmitted; - BaseLayerHandleT Handle; - }; - template - class EmissionDeferredModuleImpl : public EmissionDeferredModule { - public: - EmissionDeferredModuleImpl(std::shared_ptr M, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) - : M(std::move(M)), MemMgr(std::move(MemMgr)), - Resolver(std::move(Resolver)) {} - - protected: const GlobalValue* searchGVs(StringRef Name, - bool ExportedSymbolsOnly) const override { + bool ExportedSymbolsOnly) const { // FIXME: We could clean all this up if we had a way to reliably demangle // names: We could just demangle name and search, rather than // mangling everything else. @@ -149,15 +135,13 @@ template class LazyEmittingLayer { return buildMangledSymbols(Name, ExportedSymbolsOnly); } - BaseLayerHandleT emitToBaseLayer(BaseLayerT &BaseLayer) override { + Expected emitToBaseLayer(BaseLayerT &BaseLayer) { // We don't need the mangled names set any more: Once we've emitted this // to the base layer we'll just look for symbols there. MangledSymbols.reset(); - return BaseLayer.addModule(std::move(M), std::move(MemMgr), - std::move(Resolver)); + return BaseLayer.addModule(std::move(M), std::move(Resolver)); } - private: // If the mangled name of the given GlobalValue matches the given search // name (and its visibility conforms to the ExportedSymbolsOnly flag) then // return the symbol. Otherwise, add the mangled name to the Names map and @@ -207,9 +191,10 @@ template class LazyEmittingLayer { return nullptr; } + enum { NotEmitted, Emitting, Emitted } EmitState = NotEmitted; + BaseLayerHandleT Handle; std::shared_ptr M; - MemoryManagerPtrT MemMgr; - SymbolResolverPtrT Resolver; + std::shared_ptr Resolver; mutable std::unique_ptr> MangledSymbols; }; @@ -219,6 +204,7 @@ template class LazyEmittingLayer { ModuleListT ModuleList; public: + /// @brief Handle to a loaded module. using ModuleHandleT = typename ModuleListT::iterator; @@ -226,24 +212,23 @@ template class LazyEmittingLayer { LazyEmittingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} /// @brief Add the given module to the lazy emitting layer. - template - ModuleHandleT addModule(std::shared_ptr M, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { + Expected + addModule(std::shared_ptr M, + std::shared_ptr Resolver) { return ModuleList.insert( ModuleList.end(), - EmissionDeferredModule::create(BaseLayer, std::move(M), - std::move(MemMgr), - std::move(Resolver))); + llvm::make_unique(std::move(M), + std::move(Resolver))); } /// @brief Remove the module represented by the given handle. /// /// This method will free the memory associated with the given module, both /// in this layer, and the base layer. - void removeModule(ModuleHandleT H) { + Error removeModule(ModuleHandleT H) { (*H)->removeModuleFromBaseLayer(BaseLayer); ModuleList.erase(H); + return Error::success(); } /// @brief Search for the given named symbol. @@ -276,22 +261,11 @@ template class LazyEmittingLayer { /// @brief Immediately emit and finalize the module represented by the given /// handle. /// @param H Handle for module to emit/finalize. - void emitAndFinalize(ModuleHandleT H) { - (*H)->emitAndFinalize(BaseLayer); + Error emitAndFinalize(ModuleHandleT H) { + return (*H)->emitAndFinalize(BaseLayer); } }; -template -template -std::unique_ptr::EmissionDeferredModule> -LazyEmittingLayer::EmissionDeferredModule::create( - BaseLayerT &B, std::shared_ptr M, MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { - using EDS = EmissionDeferredModuleImpl; - return llvm::make_unique(std::move(M), std::move(MemMgr), - std::move(Resolver)); -} - } // end namespace orc } // end namespace llvm diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h index c41c1233c0d9..cb47e7520b1a 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h @@ -16,6 +16,7 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include +#include #include namespace llvm { @@ -42,16 +43,14 @@ class ObjectTransformLayer { /// memory manager and symbol resolver. /// /// @return A handle for the added objects. - template - ObjHandleT addObject(ObjPtrT Obj, MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { - return BaseLayer.addObject(Transform(std::move(Obj)), std::move(MemMgr), - std::move(Resolver)); + template + Expected addObject(ObjectPtr Obj, + std::shared_ptr Resolver) { + return BaseLayer.addObject(Transform(std::move(Obj)), std::move(Resolver)); } /// @brief Remove the object set associated with the handle H. - void removeObject(ObjHandleT H) { BaseLayer.removeObject(H); } + Error removeObject(ObjHandleT H) { return BaseLayer.removeObject(H); } /// @brief Search for the given named symbol. /// @param Name The name of the symbol to search for. @@ -77,7 +76,9 @@ class ObjectTransformLayer { /// @brief Immediately emit and finalize the object set represented by the /// given handle. /// @param H Handle for object set to emit/finalize. - void emitAndFinalize(ObjHandleT H) { BaseLayer.emitAndFinalize(H); } + Error emitAndFinalize(ObjHandleT H) { + return BaseLayer.emitAndFinalize(H); + } /// @brief Map section addresses for the objects associated with the handle H. void mapSectionAddress(ObjHandleT H, const void *LocalAddress, diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h index cbb40fad0223..e6374b70967a 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -22,7 +22,8 @@ namespace orc { enum class OrcErrorCode : int { // RPC Errors - RemoteAllocatorDoesNotExist = 1, + JITSymbolNotFound = 1, + RemoteAllocatorDoesNotExist, RemoteAllocatorIdAlreadyInUse, RemoteMProtectAddrUnrecognized, RemoteIndirectStubsOwnerDoesNotExist, @@ -37,6 +38,18 @@ enum class OrcErrorCode : int { std::error_code orcError(OrcErrorCode ErrCode); +class JITSymbolNotFound : public ErrorInfo { +public: + static char ID; + + JITSymbolNotFound(std::string SymbolName); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const std::string &getSymbolName() const; +private: + std::string SymbolName; +}; + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index 66ad36be01c8..e1016ef95f0c 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -228,13 +228,20 @@ class RTDyldObjectLinkingLayer : public RTDyldObjectLinkingLayerBase { public: + /// @brief Functor for creating memory managers. + using MemoryManagerGetter = + std::function()>; + /// @brief Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyFinalized functors. RTDyldObjectLinkingLayer( + MemoryManagerGetter GetMemMgr, NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor()) - : NotifyLoaded(std::move(NotifyLoaded)), - NotifyFinalized(std::move(NotifyFinalized)) {} + : GetMemMgr(GetMemMgr), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), + ProcessAllSections(false) {} /// @brief Set the 'ProcessAllSections' flag. /// @@ -251,12 +258,8 @@ class RTDyldObjectLinkingLayer : public RTDyldObjectLinkingLayerBase { /// /// @return A handle that can be used to refer to the loaded objects (for /// symbol searching, finalization, freeing memory, etc.). - template - ObjHandleT addObject(ObjectPtr Obj, - MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) { - + Expected addObject(ObjectPtr Obj, + std::shared_ptr Resolver) { auto Finalizer = [&](ObjHandleT H, RuntimeDyld &RTDyld, const ObjectPtr &ObjToLoad, std::function LOSHandleLoad) { @@ -275,8 +278,9 @@ class RTDyldObjectLinkingLayer : public RTDyldObjectLinkingLayerBase { }; auto LO = - createLinkedObject(std::move(Obj), std::move(MemMgr), std::move(Resolver), - std::move(Finalizer), ProcessAllSections); + createLinkedObject(std::move(Obj), GetMemMgr(), + std::move(Resolver), std::move(Finalizer), + ProcessAllSections); // LOS is an owning-ptr. Keep a non-owning one so that we can set the handle // below. auto *LOPtr = LO.get(); @@ -295,9 +299,10 @@ class RTDyldObjectLinkingLayer : public RTDyldObjectLinkingLayerBase { /// indirectly) will result in undefined behavior. If dependence tracking is /// required to detect or resolve such issues it should be added at a higher /// layer. - void removeObject(ObjHandleT H) { + Error removeObject(ObjHandleT H) { // How do we invalidate the symbols in H? LinkedObjList.erase(H); + return Error::success(); } /// @brief Search for the given named symbol. @@ -334,13 +339,15 @@ class RTDyldObjectLinkingLayer : public RTDyldObjectLinkingLayerBase { /// @brief Immediately emit and finalize the object set represented by the /// given handle. /// @param H Handle for object set to emit/finalize. - void emitAndFinalize(ObjHandleT H) { + Error emitAndFinalize(ObjHandleT H) { (*H)->finalize(); + return Error::success(); } private: LinkedObjectListT LinkedObjList; + MemoryManagerGetter GetMemMgr; NotifyLoadedFtor NotifyLoaded; NotifyFinalizedFtor NotifyFinalized; bool ProcessAllSections = false; diff --git a/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h b/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h index 1925489f7952..56aa04ce694a 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -88,21 +88,6 @@ class RuntimeDyld { ObjSectionToIDMap ObjSecToIDMap; }; - template struct LoadedObjectInfoHelper : LoadedObjectInfo { - protected: - LoadedObjectInfoHelper(const LoadedObjectInfoHelper &) = default; - LoadedObjectInfoHelper() = default; - - public: - LoadedObjectInfoHelper(RuntimeDyldImpl &RTDyld, - LoadedObjectInfo::ObjSectionToIDMap ObjSecToIDMap) - : LoadedObjectInfo(RTDyld, std::move(ObjSecToIDMap)) {} - - std::unique_ptr clone() const override { - return llvm::make_unique(static_cast(*this)); - } - }; - /// \brief Memory Management. class MemoryManager { friend class RuntimeDyld; diff --git a/contrib/llvm/include/llvm/IR/Constants.h b/contrib/llvm/include/llvm/IR/Constants.h index 8b3a90fa065b..2e72c41ccee3 100644 --- a/contrib/llvm/include/llvm/IR/Constants.h +++ b/contrib/llvm/include/llvm/IR/Constants.h @@ -680,11 +680,6 @@ class ConstantDataArray final : public ConstantDataSequential { explicit ConstantDataArray(Type *ty, const char *Data) : ConstantDataSequential(ty, ConstantDataArrayVal, Data) {} - /// Allocate space for exactly zero operands. - void *operator new(size_t s) { - return User::operator new(s, 0); - } - public: ConstantDataArray(const ConstantDataArray &) = delete; @@ -739,11 +734,6 @@ class ConstantDataVector final : public ConstantDataSequential { explicit ConstantDataVector(Type *ty, const char *Data) : ConstantDataSequential(ty, ConstantDataVectorVal, Data) {} - // allocate space for exactly zero operands. - void *operator new(size_t s) { - return User::operator new(s, 0); - } - public: ConstantDataVector(const ConstantDataVector &) = delete; diff --git a/contrib/llvm/include/llvm/IR/IRBuilder.h b/contrib/llvm/include/llvm/IR/IRBuilder.h index ec33f82f7022..5344a93efb33 100644 --- a/contrib/llvm/include/llvm/IR/IRBuilder.h +++ b/contrib/llvm/include/llvm/IR/IRBuilder.h @@ -1062,7 +1062,7 @@ class IRBuilder : public IRBuilderBase, public Inserter { Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") { if (Constant *RC = dyn_cast(RHS)) { - if (isa(RC) && cast(RC)->isAllOnesValue()) + if (isa(RC) && cast(RC)->isMinusOne()) return LHS; // LHS & -1 -> LHS if (Constant *LC = dyn_cast(LHS)) return Insert(Folder.CreateAnd(LC, RC), Name); @@ -1203,22 +1203,22 @@ class IRBuilder : public IRBuilderBase, public Inserter { return SI; } FenceInst *CreateFence(AtomicOrdering Ordering, - SynchronizationScope SynchScope = CrossThread, + SyncScope::ID SSID = SyncScope::System, const Twine &Name = "") { - return Insert(new FenceInst(Context, Ordering, SynchScope), Name); + return Insert(new FenceInst(Context, Ordering, SSID), Name); } AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope = CrossThread) { + SyncScope::ID SSID = SyncScope::System) { return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, - FailureOrdering, SynchScope)); + FailureOrdering, SSID)); } AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, AtomicOrdering Ordering, - SynchronizationScope SynchScope = CrossThread) { - return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SynchScope)); + SyncScope::ID SSID = SyncScope::System) { + return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SSID)); } Value *CreateGEP(Value *Ptr, ArrayRef IdxList, const Twine &Name = "") { @@ -1517,11 +1517,9 @@ class IRBuilder : public IRBuilderBase, public Inserter { const Twine &Name = "") { if (V->getType() == DestTy) return V; - if (V->getType()->getScalarType()->isPointerTy() && - DestTy->getScalarType()->isIntegerTy()) + if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy()) return CreatePtrToInt(V, DestTy, Name); - if (V->getType()->getScalarType()->isIntegerTy() && - DestTy->getScalarType()->isPointerTy()) + if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy()) return CreateIntToPtr(V, DestTy, Name); return CreateBitCast(V, DestTy, Name); diff --git a/contrib/llvm/include/llvm/IR/Instructions.h b/contrib/llvm/include/llvm/IR/Instructions.h index dc5f37450b48..60ae98869e55 100644 --- a/contrib/llvm/include/llvm/IR/Instructions.h +++ b/contrib/llvm/include/llvm/IR/Instructions.h @@ -52,11 +52,6 @@ class ConstantInt; class DataLayout; class LLVMContext; -enum SynchronizationScope { - SingleThread = 0, - CrossThread = 1 -}; - //===----------------------------------------------------------------------===// // AllocaInst Class //===----------------------------------------------------------------------===// @@ -195,17 +190,16 @@ class LoadInst : public UnaryInstruction { LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, BasicBlock *InsertAtEnd); LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, - AtomicOrdering Order, SynchronizationScope SynchScope = CrossThread, + AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System, Instruction *InsertBefore = nullptr) : LoadInst(cast(Ptr->getType())->getElementType(), Ptr, - NameStr, isVolatile, Align, Order, SynchScope, InsertBefore) {} + NameStr, isVolatile, Align, Order, SSID, InsertBefore) {} LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope = CrossThread, + SyncScope::ID SSID = SyncScope::System, Instruction *InsertBefore = nullptr); LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, - unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, + unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); LoadInst(Value *Ptr, const char *NameStr, Instruction *InsertBefore); LoadInst(Value *Ptr, const char *NameStr, BasicBlock *InsertAtEnd); @@ -235,34 +229,34 @@ class LoadInst : public UnaryInstruction { void setAlignment(unsigned Align); - /// Returns the ordering effect of this fence. + /// Returns the ordering constraint of this load instruction. AtomicOrdering getOrdering() const { return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7); } - /// Set the ordering constraint on this load. May not be Release or - /// AcquireRelease. + /// Sets the ordering constraint of this load instruction. May not be Release + /// or AcquireRelease. void setOrdering(AtomicOrdering Ordering) { setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) | ((unsigned)Ordering << 7)); } - SynchronizationScope getSynchScope() const { - return SynchronizationScope((getSubclassDataFromInstruction() >> 6) & 1); + /// Returns the synchronization scope ID of this load instruction. + SyncScope::ID getSyncScopeID() const { + return SSID; } - /// Specify whether this load is ordered with respect to all - /// concurrently executing threads, or only with respect to signal handlers - /// executing in the same thread. - void setSynchScope(SynchronizationScope xthread) { - setInstructionSubclassData((getSubclassDataFromInstruction() & ~(1 << 6)) | - (xthread << 6)); + /// Sets the synchronization scope ID of this load instruction. + void setSyncScopeID(SyncScope::ID SSID) { + this->SSID = SSID; } + /// Sets the ordering constraint and the synchronization scope ID of this load + /// instruction. void setAtomic(AtomicOrdering Ordering, - SynchronizationScope SynchScope = CrossThread) { + SyncScope::ID SSID = SyncScope::System) { setOrdering(Ordering); - setSynchScope(SynchScope); + setSyncScopeID(SSID); } bool isSimple() const { return !isAtomic() && !isVolatile(); } @@ -297,6 +291,11 @@ class LoadInst : public UnaryInstruction { void setInstructionSubclassData(unsigned short D) { Instruction::setInstructionSubclassData(D); } + + /// The synchronization scope ID of this load instruction. Not quite enough + /// room in SubClassData for everything, so synchronization scope ID gets its + /// own field. + SyncScope::ID SSID; }; //===----------------------------------------------------------------------===// @@ -325,11 +324,10 @@ class StoreInst : public Instruction { unsigned Align, BasicBlock *InsertAtEnd); StoreInst(Value *Val, Value *Ptr, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope = CrossThread, + SyncScope::ID SSID = SyncScope::System, Instruction *InsertBefore = nullptr); StoreInst(Value *Val, Value *Ptr, bool isVolatile, - unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, + unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); // allocate space for exactly two operands @@ -356,34 +354,34 @@ class StoreInst : public Instruction { void setAlignment(unsigned Align); - /// Returns the ordering effect of this store. + /// Returns the ordering constraint of this store instruction. AtomicOrdering getOrdering() const { return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7); } - /// Set the ordering constraint on this store. May not be Acquire or - /// AcquireRelease. + /// Sets the ordering constraint of this store instruction. May not be + /// Acquire or AcquireRelease. void setOrdering(AtomicOrdering Ordering) { setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) | ((unsigned)Ordering << 7)); } - SynchronizationScope getSynchScope() const { - return SynchronizationScope((getSubclassDataFromInstruction() >> 6) & 1); + /// Returns the synchronization scope ID of this store instruction. + SyncScope::ID getSyncScopeID() const { + return SSID; } - /// Specify whether this store instruction is ordered with respect to all - /// concurrently executing threads, or only with respect to signal handlers - /// executing in the same thread. - void setSynchScope(SynchronizationScope xthread) { - setInstructionSubclassData((getSubclassDataFromInstruction() & ~(1 << 6)) | - (xthread << 6)); + /// Sets the synchronization scope ID of this store instruction. + void setSyncScopeID(SyncScope::ID SSID) { + this->SSID = SSID; } + /// Sets the ordering constraint and the synchronization scope ID of this + /// store instruction. void setAtomic(AtomicOrdering Ordering, - SynchronizationScope SynchScope = CrossThread) { + SyncScope::ID SSID = SyncScope::System) { setOrdering(Ordering); - setSynchScope(SynchScope); + setSyncScopeID(SSID); } bool isSimple() const { return !isAtomic() && !isVolatile(); } @@ -421,6 +419,11 @@ class StoreInst : public Instruction { void setInstructionSubclassData(unsigned short D) { Instruction::setInstructionSubclassData(D); } + + /// The synchronization scope ID of this store instruction. Not quite enough + /// room in SubClassData for everything, so synchronization scope ID gets its + /// own field. + SyncScope::ID SSID; }; template <> @@ -435,7 +438,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value) /// An instruction for ordering other memory operations. class FenceInst : public Instruction { - void Init(AtomicOrdering Ordering, SynchronizationScope SynchScope); + void Init(AtomicOrdering Ordering, SyncScope::ID SSID); protected: // Note: Instruction needs to be a friend here to call cloneImpl. @@ -447,10 +450,9 @@ class FenceInst : public Instruction { // Ordering may only be Acquire, Release, AcquireRelease, or // SequentiallyConsistent. FenceInst(LLVMContext &C, AtomicOrdering Ordering, - SynchronizationScope SynchScope = CrossThread, + SyncScope::ID SSID = SyncScope::System, Instruction *InsertBefore = nullptr); - FenceInst(LLVMContext &C, AtomicOrdering Ordering, - SynchronizationScope SynchScope, + FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID, BasicBlock *InsertAtEnd); // allocate space for exactly zero operands @@ -458,28 +460,26 @@ class FenceInst : public Instruction { return User::operator new(s, 0); } - /// Returns the ordering effect of this fence. + /// Returns the ordering constraint of this fence instruction. AtomicOrdering getOrdering() const { return AtomicOrdering(getSubclassDataFromInstruction() >> 1); } - /// Set the ordering constraint on this fence. May only be Acquire, Release, - /// AcquireRelease, or SequentiallyConsistent. + /// Sets the ordering constraint of this fence instruction. May only be + /// Acquire, Release, AcquireRelease, or SequentiallyConsistent. void setOrdering(AtomicOrdering Ordering) { setInstructionSubclassData((getSubclassDataFromInstruction() & 1) | ((unsigned)Ordering << 1)); } - SynchronizationScope getSynchScope() const { - return SynchronizationScope(getSubclassDataFromInstruction() & 1); + /// Returns the synchronization scope ID of this fence instruction. + SyncScope::ID getSyncScopeID() const { + return SSID; } - /// Specify whether this fence orders other operations with respect to all - /// concurrently executing threads, or only with respect to signal handlers - /// executing in the same thread. - void setSynchScope(SynchronizationScope xthread) { - setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) | - xthread); + /// Sets the synchronization scope ID of this fence instruction. + void setSyncScopeID(SyncScope::ID SSID) { + this->SSID = SSID; } // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -496,6 +496,11 @@ class FenceInst : public Instruction { void setInstructionSubclassData(unsigned short D) { Instruction::setInstructionSubclassData(D); } + + /// The synchronization scope ID of this fence instruction. Not quite enough + /// room in SubClassData for everything, so synchronization scope ID gets its + /// own field. + SyncScope::ID SSID; }; //===----------------------------------------------------------------------===// @@ -509,7 +514,7 @@ class FenceInst : public Instruction { class AtomicCmpXchgInst : public Instruction { void Init(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); + SyncScope::ID SSID); protected: // Note: Instruction needs to be a friend here to call cloneImpl. @@ -521,13 +526,11 @@ class AtomicCmpXchgInst : public Instruction { AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope, - Instruction *InsertBefore = nullptr); + SyncScope::ID SSID, Instruction *InsertBefore = nullptr); AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope, - BasicBlock *InsertAtEnd); + SyncScope::ID SSID, BasicBlock *InsertAtEnd); // allocate space for exactly three operands void *operator new(size_t s) { @@ -561,7 +564,12 @@ class AtomicCmpXchgInst : public Instruction { /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - /// Set the ordering constraint on this cmpxchg. + /// Returns the success ordering constraint of this cmpxchg instruction. + AtomicOrdering getSuccessOrdering() const { + return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7); + } + + /// Sets the success ordering constraint of this cmpxchg instruction. void setSuccessOrdering(AtomicOrdering Ordering) { assert(Ordering != AtomicOrdering::NotAtomic && "CmpXchg instructions can only be atomic."); @@ -569,6 +577,12 @@ class AtomicCmpXchgInst : public Instruction { ((unsigned)Ordering << 2)); } + /// Returns the failure ordering constraint of this cmpxchg instruction. + AtomicOrdering getFailureOrdering() const { + return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7); + } + + /// Sets the failure ordering constraint of this cmpxchg instruction. void setFailureOrdering(AtomicOrdering Ordering) { assert(Ordering != AtomicOrdering::NotAtomic && "CmpXchg instructions can only be atomic."); @@ -576,28 +590,14 @@ class AtomicCmpXchgInst : public Instruction { ((unsigned)Ordering << 5)); } - /// Specify whether this cmpxchg is atomic and orders other operations with - /// respect to all concurrently executing threads, or only with respect to - /// signal handlers executing in the same thread. - void setSynchScope(SynchronizationScope SynchScope) { - setInstructionSubclassData((getSubclassDataFromInstruction() & ~2) | - (SynchScope << 1)); + /// Returns the synchronization scope ID of this cmpxchg instruction. + SyncScope::ID getSyncScopeID() const { + return SSID; } - /// Returns the ordering constraint on this cmpxchg. - AtomicOrdering getSuccessOrdering() const { - return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7); - } - - /// Returns the ordering constraint on this cmpxchg. - AtomicOrdering getFailureOrdering() const { - return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7); - } - - /// Returns whether this cmpxchg is atomic between threads or only within a - /// single thread. - SynchronizationScope getSynchScope() const { - return SynchronizationScope((getSubclassDataFromInstruction() & 2) >> 1); + /// Sets the synchronization scope ID of this cmpxchg instruction. + void setSyncScopeID(SyncScope::ID SSID) { + this->SSID = SSID; } Value *getPointerOperand() { return getOperand(0); } @@ -652,6 +652,11 @@ class AtomicCmpXchgInst : public Instruction { void setInstructionSubclassData(unsigned short D) { Instruction::setInstructionSubclassData(D); } + + /// The synchronization scope ID of this cmpxchg instruction. Not quite + /// enough room in SubClassData for everything, so synchronization scope ID + /// gets its own field. + SyncScope::ID SSID; }; template <> @@ -711,10 +716,10 @@ class AtomicRMWInst : public Instruction { }; AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, - AtomicOrdering Ordering, SynchronizationScope SynchScope, + AtomicOrdering Ordering, SyncScope::ID SSID, Instruction *InsertBefore = nullptr); AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, - AtomicOrdering Ordering, SynchronizationScope SynchScope, + AtomicOrdering Ordering, SyncScope::ID SSID, BasicBlock *InsertAtEnd); // allocate space for exactly two operands @@ -748,7 +753,12 @@ class AtomicRMWInst : public Instruction { /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - /// Set the ordering constraint on this RMW. + /// Returns the ordering constraint of this rmw instruction. + AtomicOrdering getOrdering() const { + return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7); + } + + /// Sets the ordering constraint of this rmw instruction. void setOrdering(AtomicOrdering Ordering) { assert(Ordering != AtomicOrdering::NotAtomic && "atomicrmw instructions can only be atomic."); @@ -756,23 +766,14 @@ class AtomicRMWInst : public Instruction { ((unsigned)Ordering << 2)); } - /// Specify whether this RMW orders other operations with respect to all - /// concurrently executing threads, or only with respect to signal handlers - /// executing in the same thread. - void setSynchScope(SynchronizationScope SynchScope) { - setInstructionSubclassData((getSubclassDataFromInstruction() & ~2) | - (SynchScope << 1)); + /// Returns the synchronization scope ID of this rmw instruction. + SyncScope::ID getSyncScopeID() const { + return SSID; } - /// Returns the ordering constraint on this RMW. - AtomicOrdering getOrdering() const { - return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7); - } - - /// Returns whether this RMW is atomic between threads or only within a - /// single thread. - SynchronizationScope getSynchScope() const { - return SynchronizationScope((getSubclassDataFromInstruction() & 2) >> 1); + /// Sets the synchronization scope ID of this rmw instruction. + void setSyncScopeID(SyncScope::ID SSID) { + this->SSID = SSID; } Value *getPointerOperand() { return getOperand(0); } @@ -797,13 +798,18 @@ class AtomicRMWInst : public Instruction { private: void Init(BinOp Operation, Value *Ptr, Value *Val, - AtomicOrdering Ordering, SynchronizationScope SynchScope); + AtomicOrdering Ordering, SyncScope::ID SSID); // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. void setInstructionSubclassData(unsigned short D) { Instruction::setInstructionSubclassData(D); } + + /// The synchronization scope ID of this rmw instruction. Not quite enough + /// room in SubClassData for everything, so synchronization scope ID gets its + /// own field. + SyncScope::ID SSID; }; template <> @@ -1101,8 +1107,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value) /// Represent an integer comparison operator. class ICmpInst: public CmpInst { void AssertOK() { - assert(getPredicate() >= CmpInst::FIRST_ICMP_PREDICATE && - getPredicate() <= CmpInst::LAST_ICMP_PREDICATE && + assert(isIntPredicate() && "Invalid ICmp predicate value"); assert(getOperand(0)->getType() == getOperand(1)->getType() && "Both operands to ICmp instruction are not of the same type!"); @@ -1244,8 +1249,7 @@ class ICmpInst: public CmpInst { /// Represents a floating point comparison operator. class FCmpInst: public CmpInst { void AssertOK() { - assert(getPredicate() <= FCmpInst::LAST_FCMP_PREDICATE && - "Invalid FCmp predicate value"); + assert(isFPPredicate() && "Invalid FCmp predicate value"); assert(getOperand(0)->getType() == getOperand(1)->getType() && "Both operands to FCmp instruction are not of the same type!"); // Check that the operands are the right type diff --git a/contrib/llvm/include/llvm/IR/IntrinsicInst.h b/contrib/llvm/include/llvm/IR/IntrinsicInst.h index 944af57a7800..f55d17ec72c8 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicInst.h +++ b/contrib/llvm/include/llvm/IR/IntrinsicInst.h @@ -296,6 +296,175 @@ namespace llvm { } }; + class ElementUnorderedAtomicMemMoveInst : public IntrinsicInst { + private: + enum { ARG_DEST = 0, ARG_SOURCE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 }; + + public: + Value *getRawDest() const { + return const_cast(getArgOperand(ARG_DEST)); + } + const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); } + Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); } + + /// Return the arguments to the instruction. + Value *getRawSource() const { + return const_cast(getArgOperand(ARG_SOURCE)); + } + const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); } + Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); } + + Value *getLength() const { + return const_cast(getArgOperand(ARG_LENGTH)); + } + const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); } + Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); } + + bool isVolatile() const { return false; } + + Value *getRawElementSizeInBytes() const { + return const_cast(getArgOperand(ARG_ELEMENTSIZE)); + } + + ConstantInt *getElementSizeInBytesCst() const { + return cast(getRawElementSizeInBytes()); + } + + uint32_t getElementSizeInBytes() const { + return getElementSizeInBytesCst()->getZExtValue(); + } + + /// This is just like getRawDest, but it strips off any cast + /// instructions that feed it, giving the original input. The returned + /// value is guaranteed to be a pointer. + Value *getDest() const { return getRawDest()->stripPointerCasts(); } + + /// This is just like getRawSource, but it strips off any cast + /// instructions that feed it, giving the original input. The returned + /// value is guaranteed to be a pointer. + Value *getSource() const { return getRawSource()->stripPointerCasts(); } + + unsigned getDestAddressSpace() const { + return cast(getRawDest()->getType())->getAddressSpace(); + } + + unsigned getSourceAddressSpace() const { + return cast(getRawSource()->getType())->getAddressSpace(); + } + + /// Set the specified arguments of the instruction. + void setDest(Value *Ptr) { + assert(getRawDest()->getType() == Ptr->getType() && + "setDest called with pointer of wrong type!"); + setArgOperand(ARG_DEST, Ptr); + } + + void setSource(Value *Ptr) { + assert(getRawSource()->getType() == Ptr->getType() && + "setSource called with pointer of wrong type!"); + setArgOperand(ARG_SOURCE, Ptr); + } + + void setLength(Value *L) { + assert(getLength()->getType() == L->getType() && + "setLength called with value of wrong type!"); + setArgOperand(ARG_LENGTH, L); + } + + void setElementSizeInBytes(Constant *V) { + assert(V->getType() == Type::getInt8Ty(getContext()) && + "setElementSizeInBytes called with value of wrong type!"); + setArgOperand(ARG_ELEMENTSIZE, V); + } + + static inline bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memmove_element_unordered_atomic; + } + static inline bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + }; + + /// This class represents atomic memset intrinsic + /// TODO: Integrate this class into MemIntrinsic hierarchy; for now this is + /// C&P of all methods from that hierarchy + class ElementUnorderedAtomicMemSetInst : public IntrinsicInst { + private: + enum { ARG_DEST = 0, ARG_VALUE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 }; + + public: + Value *getRawDest() const { + return const_cast(getArgOperand(ARG_DEST)); + } + const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); } + Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); } + + Value *getValue() const { return const_cast(getArgOperand(ARG_VALUE)); } + const Use &getValueUse() const { return getArgOperandUse(ARG_VALUE); } + Use &getValueUse() { return getArgOperandUse(ARG_VALUE); } + + Value *getLength() const { + return const_cast(getArgOperand(ARG_LENGTH)); + } + const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); } + Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); } + + bool isVolatile() const { return false; } + + Value *getRawElementSizeInBytes() const { + return const_cast(getArgOperand(ARG_ELEMENTSIZE)); + } + + ConstantInt *getElementSizeInBytesCst() const { + return cast(getRawElementSizeInBytes()); + } + + uint32_t getElementSizeInBytes() const { + return getElementSizeInBytesCst()->getZExtValue(); + } + + /// This is just like getRawDest, but it strips off any cast + /// instructions that feed it, giving the original input. The returned + /// value is guaranteed to be a pointer. + Value *getDest() const { return getRawDest()->stripPointerCasts(); } + + unsigned getDestAddressSpace() const { + return cast(getRawDest()->getType())->getAddressSpace(); + } + + /// Set the specified arguments of the instruction. + void setDest(Value *Ptr) { + assert(getRawDest()->getType() == Ptr->getType() && + "setDest called with pointer of wrong type!"); + setArgOperand(ARG_DEST, Ptr); + } + + void setValue(Value *Val) { + assert(getValue()->getType() == Val->getType() && + "setValue called with value of wrong type!"); + setArgOperand(ARG_VALUE, Val); + } + + void setLength(Value *L) { + assert(getLength()->getType() == L->getType() && + "setLength called with value of wrong type!"); + setArgOperand(ARG_LENGTH, L); + } + + void setElementSizeInBytes(Constant *V) { + assert(V->getType() == Type::getInt8Ty(getContext()) && + "setElementSizeInBytes called with value of wrong type!"); + setArgOperand(ARG_ELEMENTSIZE, V); + } + + static inline bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memset_element_unordered_atomic; + } + static inline bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + }; + /// This is the common base class for memset/memcpy/memmove. class MemIntrinsic : public IntrinsicInst { public: diff --git a/contrib/llvm/include/llvm/IR/Intrinsics.td b/contrib/llvm/include/llvm/IR/Intrinsics.td index 45936a6e9b66..14c88e519435 100644 --- a/contrib/llvm/include/llvm/IR/Intrinsics.td +++ b/contrib/llvm/include/llvm/IR/Intrinsics.td @@ -873,6 +873,22 @@ def int_memcpy_element_unordered_atomic ReadOnly<1> ]>; +// @llvm.memmove.element.unordered.atomic.*(dest, src, length, elementsize) +def int_memmove_element_unordered_atomic + : Intrinsic<[], + [ + llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty + ], + [ + IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>, + ReadOnly<1> + ]>; + +// @llvm.memset.element.unordered.atomic.*(dest, value, length, elementsize) +def int_memset_element_unordered_atomic + : Intrinsic<[], [ llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i32_ty ], + [ IntrArgMemOnly, NoCapture<0>, WriteOnly<0> ]>; + //===------------------------ Reduction Intrinsics ------------------------===// // def int_experimental_vector_reduce_fadd : Intrinsic<[llvm_anyfloat_ty], diff --git a/contrib/llvm/include/llvm/IR/LLVMContext.h b/contrib/llvm/include/llvm/IR/LLVMContext.h index b27abad618c9..4cb77701f762 100644 --- a/contrib/llvm/include/llvm/IR/LLVMContext.h +++ b/contrib/llvm/include/llvm/IR/LLVMContext.h @@ -42,6 +42,24 @@ class Output; } // end namespace yaml +namespace SyncScope { + +typedef uint8_t ID; + +/// Known synchronization scope IDs, which always have the same value. All +/// synchronization scope IDs that LLVM has special knowledge of are listed +/// here. Additionally, this scheme allows LLVM to efficiently check for +/// specific synchronization scope ID without comparing strings. +enum { + /// Synchronized with respect to signal handlers executing in the same thread. + SingleThread = 0, + + /// Synchronized with respect to all concurrently executing threads. + System = 1 +}; + +} // end namespace SyncScope + /// This is an important class for using LLVM in a threaded context. It /// (opaquely) owns and manages the core "global" data of LLVM's core /// infrastructure, including the type and constant uniquing tables. @@ -111,6 +129,16 @@ class LLVMContext { /// tag registered with an LLVMContext has an unique ID. uint32_t getOperandBundleTagID(StringRef Tag) const; + /// getOrInsertSyncScopeID - Maps synchronization scope name to + /// synchronization scope ID. Every synchronization scope registered with + /// LLVMContext has unique ID except pre-defined ones. + SyncScope::ID getOrInsertSyncScopeID(StringRef SSN); + + /// getSyncScopeNames - Populates client supplied SmallVector with + /// synchronization scope names registered with LLVMContext. Synchronization + /// scope names are ordered by increasing synchronization scope IDs. + void getSyncScopeNames(SmallVectorImpl &SSNs) const; + /// Define the GC for a function void setGC(const Function &Fn, std::string GCName); diff --git a/contrib/llvm/include/llvm/IR/Module.h b/contrib/llvm/include/llvm/IR/Module.h index d47d82a57bff..196e32e3615c 100644 --- a/contrib/llvm/include/llvm/IR/Module.h +++ b/contrib/llvm/include/llvm/IR/Module.h @@ -249,7 +249,7 @@ class Module { /// when other randomness consuming passes are added or removed. In /// addition, the random stream will be reproducible across LLVM /// versions when the pass does not change. - RandomNumberGenerator *createRNG(const Pass* P) const; + std::unique_ptr createRNG(const Pass* P) const; /// @} /// @name Module Level Mutators diff --git a/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h b/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h index b43d58865862..4aa8a0199ab1 100644 --- a/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -47,7 +47,13 @@ template struct MappingTraits; /// \brief Class to accumulate and hold information about a callee. struct CalleeInfo { - enum class HotnessType : uint8_t { Unknown = 0, Cold = 1, None = 2, Hot = 3 }; + enum class HotnessType : uint8_t { + Unknown = 0, + Cold = 1, + None = 2, + Hot = 3, + Critical = 4 + }; HotnessType Hotness = HotnessType::Unknown; CalleeInfo() = default; @@ -516,7 +522,7 @@ using ModulePathStringTableTy = StringMap>; /// Map of global value GUID to its summary, used to identify values defined in /// a particular module, and provide efficient access to their summary. -using GVSummaryMapTy = std::map; +using GVSummaryMapTy = DenseMap; /// Class to hold module path string table and global value map, /// and encapsulate methods for operating on them. diff --git a/contrib/llvm/include/llvm/IR/PassManager.h b/contrib/llvm/include/llvm/IR/PassManager.h index d03b7b65f81e..393175675034 100644 --- a/contrib/llvm/include/llvm/IR/PassManager.h +++ b/contrib/llvm/include/llvm/IR/PassManager.h @@ -162,6 +162,14 @@ class PreservedAnalyses { return PA; } + /// \brief Construct a preserved analyses object with a single preserved set. + template + static PreservedAnalyses allInSet() { + PreservedAnalyses PA; + PA.preserveSet(); + return PA; + } + /// Mark an analysis as preserved. template void preserve() { preserve(AnalysisT::ID()); } @@ -1062,10 +1070,27 @@ class OuterAnalysisManagerProxy const AnalysisManagerT &getManager() const { return *AM; } - /// \brief Handle invalidation by ignoring it; this pass is immutable. + /// When invalidation occurs, remove any registered invalidation events. bool invalidate( - IRUnitT &, const PreservedAnalyses &, - typename AnalysisManager::Invalidator &) { + IRUnitT &IRUnit, const PreservedAnalyses &PA, + typename AnalysisManager::Invalidator &Inv) { + // Loop over the set of registered outer invalidation mappings and if any + // of them map to an analysis that is now invalid, clear it out. + SmallVector DeadKeys; + for (auto &KeyValuePair : OuterAnalysisInvalidationMap) { + AnalysisKey *OuterID = KeyValuePair.first; + auto &InnerIDs = KeyValuePair.second; + InnerIDs.erase(llvm::remove_if(InnerIDs, [&](AnalysisKey *InnerID) { + return Inv.invalidate(InnerID, IRUnit, PA); }), + InnerIDs.end()); + if (InnerIDs.empty()) + DeadKeys.push_back(OuterID); + } + + for (auto OuterID : DeadKeys) + OuterAnalysisInvalidationMap.erase(OuterID); + + // The proxy itself remains valid regardless of anything else. return false; } diff --git a/contrib/llvm/include/llvm/IR/PatternMatch.h b/contrib/llvm/include/llvm/IR/PatternMatch.h index 5b69e7855cc7..acb895211644 100644 --- a/contrib/llvm/include/llvm/IR/PatternMatch.h +++ b/contrib/llvm/include/llvm/IR/PatternMatch.h @@ -158,12 +158,18 @@ struct match_neg_zero { /// zero inline match_neg_zero m_NegZero() { return match_neg_zero(); } +struct match_any_zero { + template bool match(ITy *V) { + if (const auto *C = dyn_cast(V)) + return C->isZeroValue(); + return false; + } +}; + /// \brief - Match an arbitrary zero/null constant. This includes /// zero_initializer for vectors and ConstantPointerNull for pointers. For /// floating point constants, this will match negative zero and positive zero -inline match_combine_or m_AnyZero() { - return m_CombineOr(m_Zero(), m_NegZero()); -} +inline match_any_zero m_AnyZero() { return match_any_zero(); } struct match_nan { template bool match(ITy *V) { @@ -176,6 +182,39 @@ struct match_nan { /// Match an arbitrary NaN constant. This includes quiet and signalling nans. inline match_nan m_NaN() { return match_nan(); } +struct match_one { + template bool match(ITy *V) { + if (const auto *C = dyn_cast(V)) + return C->isOneValue(); + return false; + } +}; + +/// \brief Match an integer 1 or a vector with all elements equal to 1. +inline match_one m_One() { return match_one(); } + +struct match_all_ones { + template bool match(ITy *V) { + if (const auto *C = dyn_cast(V)) + return C->isAllOnesValue(); + return false; + } +}; + +/// \brief Match an integer or vector with all bits set to true. +inline match_all_ones m_AllOnes() { return match_all_ones(); } + +struct match_sign_mask { + template bool match(ITy *V) { + if (const auto *C = dyn_cast(V)) + return C->isMinSignedValue(); + return false; + } +}; + +/// \brief Match an integer or vector with only the sign bit(s) set. +inline match_sign_mask m_SignMask() { return match_sign_mask(); } + struct apint_match { const APInt *&Res; @@ -259,34 +298,6 @@ template struct api_pred_ty : public Predicate { } }; -struct is_one { - bool isValue(const APInt &C) { return C.isOneValue(); } -}; - -/// \brief Match an integer 1 or a vector with all elements equal to 1. -inline cst_pred_ty m_One() { return cst_pred_ty(); } -inline api_pred_ty m_One(const APInt *&V) { return V; } - -struct is_all_ones { - bool isValue(const APInt &C) { return C.isAllOnesValue(); } -}; - -/// \brief Match an integer or vector with all bits set to true. -inline cst_pred_ty m_AllOnes() { - return cst_pred_ty(); -} -inline api_pred_ty m_AllOnes(const APInt *&V) { return V; } - -struct is_sign_mask { - bool isValue(const APInt &C) { return C.isSignMask(); } -}; - -/// \brief Match an integer or vector with only the sign bit(s) set. -inline cst_pred_ty m_SignMask() { - return cst_pred_ty(); -} -inline api_pred_ty m_SignMask(const APInt *&V) { return V; } - struct is_power2 { bool isValue(const APInt &C) { return C.isPowerOf2(); } }; diff --git a/contrib/llvm/include/llvm/IR/SafepointIRVerifier.h b/contrib/llvm/include/llvm/IR/SafepointIRVerifier.h new file mode 100644 index 000000000000..092050d1d207 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/SafepointIRVerifier.h @@ -0,0 +1,35 @@ +//===- SafepointIRVerifier.h - Checks for GC relocation problems *- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines a verifier which is useful for enforcing the relocation +// properties required by a relocating GC. Specifically, it looks for uses of +// the unrelocated value of pointer SSA values after a possible safepoint. It +// attempts to report no false negatives, but may end up reporting false +// positives in rare cases (see the note at the top of the corresponding cpp +// file.) +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_SAFEPOINT_IR_VERIFIER +#define LLVM_IR_SAFEPOINT_IR_VERIFIER + +namespace llvm { + +class Function; +class FunctionPass; + +/// Run the safepoint verifier over a single function. Crashes on failure. +void verifySafepointIR(Function &F); + +/// Create an instance of the safepoint verifier pass which can be added to +/// a pass pipeline to check for relocation bugs. +FunctionPass *createSafepointIRVerifierPass(); +} + +#endif // LLVM_IR_SAFEPOINT_IR_VERIFIER diff --git a/contrib/llvm/include/llvm/IR/Type.h b/contrib/llvm/include/llvm/IR/Type.h index b37b59288e3f..ef7801266777 100644 --- a/contrib/llvm/include/llvm/IR/Type.h +++ b/contrib/llvm/include/llvm/IR/Type.h @@ -202,6 +202,12 @@ class Type { /// Return true if this is an integer type or a vector of integer types. bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); } + /// Return true if this is an integer type or a vector of integer types of + /// the given width. + bool isIntOrIntVectorTy(unsigned BitWidth) const { + return getScalarType()->isIntegerTy(BitWidth); + } + /// True if this is an instance of FunctionType. bool isFunctionTy() const { return getTypeID() == FunctionTyID; } diff --git a/contrib/llvm/include/llvm/InitializePasses.h b/contrib/llvm/include/llvm/InitializePasses.h index aab14070dbda..39ac4649b70d 100644 --- a/contrib/llvm/include/llvm/InitializePasses.h +++ b/contrib/llvm/include/llvm/InitializePasses.h @@ -303,6 +303,7 @@ void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&); void initializePromoteLegacyPassPass(PassRegistry&); void initializePruneEHPass(PassRegistry&); void initializeRABasicPass(PassRegistry&); +void initializeRAFastPass(PassRegistry&); void initializeRAGreedyPass(PassRegistry&); void initializeReassociateLegacyPassPass(PassRegistry&); void initializeRegBankSelectPass(PassRegistry&); @@ -318,6 +319,7 @@ void initializeResetMachineFunctionPass(PassRegistry&); void initializeReversePostOrderFunctionAttrsLegacyPassPass(PassRegistry&); void initializeRewriteStatepointsForGCPass(PassRegistry&); void initializeRewriteSymbolsLegacyPassPass(PassRegistry&); +void initializeSafepointIRVerifierPass(PassRegistry&); void initializeSCCPLegacyPassPass(PassRegistry&); void initializeSCEVAAWrapperPassPass(PassRegistry&); void initializeSLPVectorizerPass(PassRegistry&); diff --git a/contrib/llvm/include/llvm/MC/MCAsmBackend.h b/contrib/llvm/include/llvm/MC/MCAsmBackend.h index c9c43a22da5d..5a8e29d08ad2 100644 --- a/contrib/llvm/include/llvm/MC/MCAsmBackend.h +++ b/contrib/llvm/include/llvm/MC/MCAsmBackend.h @@ -73,7 +73,7 @@ class MCAsmBackend { /// reported via \p Ctx. virtual void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const = 0; + uint64_t Value, bool IsResolved) const = 0; /// @} diff --git a/contrib/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h b/contrib/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h index 5e626f186986..7f09c05ccf2a 100644 --- a/contrib/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h +++ b/contrib/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h @@ -68,6 +68,7 @@ class MCDisassembler { /// an invalid instruction. /// \param Address - The address, in the memory space of region, of the first /// byte of the instruction. + /// \param Bytes - A reference to the actual bytes of the instruction. /// \param VStream - The stream to print warnings and diagnostic messages on. /// \param CStream - The stream to print comments and annotations on. /// \return - MCDisassembler::Success if the instruction is valid, diff --git a/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h b/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h index 2d2480a27223..42dc90da3049 100644 --- a/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h @@ -233,8 +233,7 @@ class MachObjectWriter : public MCObjectWriter { void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, - MCValue Target, bool &IsPCRel, - uint64_t &FixedValue) override; + MCValue Target, uint64_t &FixedValue) override; void bindIndirectSymbols(MCAssembler &Asm); diff --git a/contrib/llvm/include/llvm/MC/MCObjectWriter.h b/contrib/llvm/include/llvm/MC/MCObjectWriter.h index 86bcbb6861d7..cd90690fb186 100644 --- a/contrib/llvm/include/llvm/MC/MCObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCObjectWriter.h @@ -86,7 +86,7 @@ class MCObjectWriter { virtual void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, - bool &IsPCRel, uint64_t &FixedValue) = 0; + uint64_t &FixedValue) = 0; /// Check whether the difference (A - B) between two symbol references is /// fully resolved. diff --git a/contrib/llvm/include/llvm/MC/MCSymbolWasm.h b/contrib/llvm/include/llvm/MC/MCSymbolWasm.h index 7ea89629efda..9bae6c582faa 100644 --- a/contrib/llvm/include/llvm/MC/MCSymbolWasm.h +++ b/contrib/llvm/include/llvm/MC/MCSymbolWasm.h @@ -21,6 +21,8 @@ class MCSymbolWasm : public MCSymbol { std::string ModuleName; SmallVector Returns; SmallVector Params; + bool ParamsSet = false; + bool ReturnsSet = false; /// An expression describing how to calculate the size of a symbol. If a /// symbol has no size this field will be NULL. @@ -45,15 +47,23 @@ class MCSymbolWasm : public MCSymbol { const StringRef getModuleName() const { return ModuleName; } - const SmallVector &getReturns() const { return Returns; } + const SmallVector &getReturns() const { + assert(ReturnsSet); + return Returns; + } void setReturns(SmallVectorImpl &&Rets) { + ReturnsSet = true; Returns = std::move(Rets); } - const SmallVector &getParams() const { return Params; } + const SmallVector &getParams() const { + assert(ParamsSet); + return Params; + } void setParams(SmallVectorImpl &&Pars) { + ParamsSet = true; Params = std::move(Pars); } }; diff --git a/contrib/llvm/include/llvm/Object/COFF.h b/contrib/llvm/include/llvm/Object/COFF.h index 78e0b5f6ed30..89c1ba6be35f 100644 --- a/contrib/llvm/include/llvm/Object/COFF.h +++ b/contrib/llvm/include/llvm/Object/COFF.h @@ -698,6 +698,9 @@ struct coff_resource_dir_entry { uint32_t getNameOffset() const { return maskTrailingOnes(31) & NameOffset; } + // Even though the PE/COFF spec doesn't mention this, the high bit of a name + // offset is set. + void setNameOffset(uint32_t Offset) { NameOffset = Offset | (1 << 31); } } Identifier; union { support::ulittle32_t DataEntryOffset; diff --git a/contrib/llvm/include/llvm/Object/Wasm.h b/contrib/llvm/include/llvm/Object/Wasm.h index 5c8445f10f44..07ee4a4d6c4d 100644 --- a/contrib/llvm/include/llvm/Object/Wasm.h +++ b/contrib/llvm/include/llvm/Object/Wasm.h @@ -61,7 +61,7 @@ class WasmSymbol { void print(raw_ostream &Out) const { Out << "Name=" << Name << ", Type=" << static_cast(Type) - << ", Flags=" << Flags; + << ", Flags=" << Flags << " ElemIndex=" << ElementIndex; } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -69,8 +69,7 @@ class WasmSymbol { #endif }; -class WasmSection { -public: +struct WasmSection { WasmSection() = default; uint32_t Type = 0; // Section type (See below) @@ -80,6 +79,11 @@ class WasmSection { std::vector Relocations; // Relocations for this section }; +struct WasmSegment { + uint32_t SectionOffset; + wasm::WasmDataSegment Data; +}; + class WasmObjectFile : public ObjectFile { public: @@ -110,7 +114,7 @@ class WasmObjectFile : public ObjectFile { return ElemSegments; } - const std::vector& dataSegments() const { + const std::vector& dataSegments() const { return DataSegments; } @@ -210,7 +214,7 @@ class WasmObjectFile : public ObjectFile { std::vector Imports; std::vector Exports; std::vector ElemSegments; - std::vector DataSegments; + std::vector DataSegments; std::vector Functions; std::vector Symbols; ArrayRef CodeSection; diff --git a/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h b/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h index 6bf08d340eeb..709ad8ec3b77 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h +++ b/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h @@ -98,7 +98,8 @@ struct Relocation { }; struct DataSegment { - uint32_t Index; + uint32_t MemoryIndex; + uint32_t SectionOffset; wasm::WasmInitExpr Offset; yaml::BinaryRef Content; }; diff --git a/contrib/llvm/include/llvm/Option/OptTable.h b/contrib/llvm/include/llvm/Option/OptTable.h index 3e7b019a0d4e..a35e182f00e5 100644 --- a/contrib/llvm/include/llvm/Option/OptTable.h +++ b/contrib/llvm/include/llvm/Option/OptTable.h @@ -140,7 +140,8 @@ class OptTable { // to start with. /// /// \return The vector of flags which start with Cur. - std::vector findByPrefix(StringRef Cur) const; + std::vector findByPrefix(StringRef Cur, + unsigned short DisableFlags) const; /// \brief Parse a single argument; returning the new argument and /// updating Index. diff --git a/contrib/llvm/include/llvm/Passes/PassBuilder.h b/contrib/llvm/include/llvm/Passes/PassBuilder.h index ff1958397331..33433f6b4a10 100644 --- a/contrib/llvm/include/llvm/Passes/PassBuilder.h +++ b/contrib/llvm/include/llvm/Passes/PassBuilder.h @@ -46,6 +46,19 @@ class PassBuilder { Optional PGOOpt; public: + /// \brief A struct to capture parsed pass pipeline names. + /// + /// A pipeline is defined as a series of names, each of which may in itself + /// recursively contain a nested pipeline. A name is either the name of a pass + /// (e.g. "instcombine") or the name of a pipeline type (e.g. "cgscc"). If the + /// name is the name of a pass, the InnerPipeline is empty, since passes + /// cannot contain inner pipelines. See parsePassPipeline() for a more + /// detailed description of the textual pipeline format. + struct PipelineElement { + StringRef Name; + std::vector InnerPipeline; + }; + /// \brief LLVM-provided high-level optimization levels. /// /// This enumerates the LLVM-provided high-level optimization levels. Each @@ -188,9 +201,14 @@ class PassBuilder { /// only intended for use when attempting to optimize code. If frontends /// require some transformations for semantic reasons, they should explicitly /// build them. + /// + /// \p PrepareForThinLTO indicates whether this is invoked in + /// PrepareForThinLTO phase. Special handling is needed for sample PGO to + /// ensure profile accurate in the backend profile annotation phase. FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging = false); + bool DebugLogging = false, + bool PrepareForThinLTO = false); /// Construct the core LLVM module canonicalization and simplification /// pipeline. @@ -205,9 +223,14 @@ class PassBuilder { /// only intended for use when attempting to optimize code. If frontends /// require some transformations for semantic reasons, they should explicitly /// build them. + /// + /// \p PrepareForThinLTO indicates whether this is invoked in + /// PrepareForThinLTO phase. Special handling is needed for sample PGO to + /// ensure profile accurate in the backend profile annotation phase. ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging = false); + bool DebugLogging = false, + bool PrepareForThinLTO = false); /// Construct the core LLVM module optimization pipeline. /// @@ -302,7 +325,8 @@ class PassBuilder { /// registered. AAManager buildDefaultAAPipeline(); - /// \brief Parse a textual pass pipeline description into a \c ModulePassManager. + /// \brief Parse a textual pass pipeline description into a \c + /// ModulePassManager. /// /// The format of the textual pass pipeline description looks something like: /// @@ -312,8 +336,8 @@ class PassBuilder { /// are comma separated. As a special shortcut, if the very first pass is not /// a module pass (as a module pass manager is), this will automatically form /// the shortest stack of pass managers that allow inserting that first pass. - /// So, assuming function passes 'fpassN', CGSCC passes 'cgpassN', and loop passes - /// 'lpassN', all of these are valid: + /// So, assuming function passes 'fpassN', CGSCC passes 'cgpassN', and loop + /// passes 'lpassN', all of these are valid: /// /// fpass1,fpass2,fpass3 /// cgpass1,cgpass2,cgpass3 @@ -326,13 +350,28 @@ class PassBuilder { /// module(function(loop(lpass1,lpass2,lpass3))) /// /// This shortcut is especially useful for debugging and testing small pass - /// combinations. Note that these shortcuts don't introduce any other magic. If - /// the sequence of passes aren't all the exact same kind of pass, it will be - /// an error. You cannot mix different levels implicitly, you must explicitly - /// form a pass manager in which to nest passes. + /// combinations. Note that these shortcuts don't introduce any other magic. + /// If the sequence of passes aren't all the exact same kind of pass, it will + /// be an error. You cannot mix different levels implicitly, you must + /// explicitly form a pass manager in which to nest passes. bool parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText, bool VerifyEachPass = true, bool DebugLogging = false); + /// {{@ Parse a textual pass pipeline description into a specific PassManager + /// + /// Automatic deduction of an appropriate pass manager stack is not supported. + /// For example, to insert a loop pass 'lpass' into a FunctinoPassManager, + /// this is the valid pipeline text: + /// + /// function(lpass) + bool parsePassPipeline(CGSCCPassManager &CGPM, StringRef PipelineText, + bool VerifyEachPass = true, bool DebugLogging = false); + bool parsePassPipeline(FunctionPassManager &FPM, StringRef PipelineText, + bool VerifyEachPass = true, bool DebugLogging = false); + bool parsePassPipeline(LoopPassManager &LPM, StringRef PipelineText, + bool VerifyEachPass = true, bool DebugLogging = false); + /// @}} + /// Parse a textual alias analysis pipeline into the provided AA manager. /// /// The format of the textual AA pipeline is a comma separated list of AA @@ -350,13 +389,139 @@ class PassBuilder { /// returns false. bool parseAAPipeline(AAManager &AA, StringRef PipelineText); -private: - /// A struct to capture parsed pass pipeline names. - struct PipelineElement { - StringRef Name; - std::vector InnerPipeline; - }; + /// \brief Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding passes that perform peephole + /// optimizations similar to the instruction combiner. These passes will be + /// inserted after each instance of the instruction combiner pass. + void registerPeepholeEPCallback( + const std::function &C) { + PeepholeEPCallbacks.push_back(C); + } + /// \brief Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding late loop canonicalization and + /// simplification passes. This is the last point in the loop optimization + /// pipeline before loop deletion. Each pass added + /// here must be an instance of LoopPass. + /// This is the place to add passes that can remove loops, such as target- + /// specific loop idiom recognition. + void registerLateLoopOptimizationsEPCallback( + const std::function &C) { + LateLoopOptimizationsEPCallbacks.push_back(C); + } + + /// \brief Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding loop passes to the end of the loop + /// optimizer. + void registerLoopOptimizerEndEPCallback( + const std::function &C) { + LoopOptimizerEndEPCallbacks.push_back(C); + } + + /// \brief Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding optimization passes after most of the + /// main optimizations, but before the last cleanup-ish optimizations. + void registerScalarOptimizerLateEPCallback( + const std::function &C) { + ScalarOptimizerLateEPCallbacks.push_back(C); + } + + /// \brief Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding CallGraphSCC passes at the end of the + /// main CallGraphSCC passes and before any function simplification passes run + /// by CGPassManager. + void registerCGSCCOptimizerLateEPCallback( + const std::function &C) { + CGSCCOptimizerLateEPCallbacks.push_back(C); + } + + /// \brief Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding optimization passes before the + /// vectorizer and other highly target specific optimization passes are + /// executed. + void registerVectorizerStartEPCallback( + const std::function &C) { + VectorizerStartEPCallbacks.push_back(C); + } + + /// \brief Register a callback for parsing an AliasAnalysis Name to populate + /// the given AAManager \p AA + void registerParseAACallback( + const std::function &C) { + AAParsingCallbacks.push_back(C); + } + + /// {{@ Register callbacks for analysis registration with this PassBuilder + /// instance. + /// Callees register their analyses with the given AnalysisManager objects. + void registerAnalysisRegistrationCallback( + const std::function &C) { + CGSCCAnalysisRegistrationCallbacks.push_back(C); + } + void registerAnalysisRegistrationCallback( + const std::function &C) { + FunctionAnalysisRegistrationCallbacks.push_back(C); + } + void registerAnalysisRegistrationCallback( + const std::function &C) { + LoopAnalysisRegistrationCallbacks.push_back(C); + } + void registerAnalysisRegistrationCallback( + const std::function &C) { + ModuleAnalysisRegistrationCallbacks.push_back(C); + } + /// @}} + + /// {{@ Register pipeline parsing callbacks with this pass builder instance. + /// Using these callbacks, callers can parse both a single pass name, as well + /// as entire sub-pipelines, and populate the PassManager instance + /// accordingly. + void registerPipelineParsingCallback( + const std::function)> &C) { + CGSCCPipelineParsingCallbacks.push_back(C); + } + void registerPipelineParsingCallback( + const std::function)> &C) { + FunctionPipelineParsingCallbacks.push_back(C); + } + void registerPipelineParsingCallback( + const std::function)> &C) { + LoopPipelineParsingCallbacks.push_back(C); + } + void registerPipelineParsingCallback( + const std::function)> &C) { + ModulePipelineParsingCallbacks.push_back(C); + } + /// @}} + + /// \brief Register a callback for a top-level pipeline entry. + /// + /// If the PassManager type is not given at the top level of the pipeline + /// text, this Callback should be used to determine the appropriate stack of + /// PassManagers and populate the passed ModulePassManager. + void registerParseTopLevelPipelineCallback( + const std::function, + bool VerifyEachPass, bool DebugLogging)> &C) { + TopLevelPipelineParsingCallbacks.push_back(C); + } + +private: static Optional> parsePipelineText(StringRef Text); @@ -382,7 +547,106 @@ class PassBuilder { bool parseModulePassPipeline(ModulePassManager &MPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging); + + void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, + OptimizationLevel Level, bool RunProfileGen, + std::string ProfileGenFile, + std::string ProfileUseFile); + + void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); + + // Extension Point callbacks + SmallVector, 2> + PeepholeEPCallbacks; + SmallVector, 2> + LateLoopOptimizationsEPCallbacks; + SmallVector, 2> + LoopOptimizerEndEPCallbacks; + SmallVector, 2> + ScalarOptimizerLateEPCallbacks; + SmallVector, 2> + CGSCCOptimizerLateEPCallbacks; + SmallVector, 2> + VectorizerStartEPCallbacks; + // Module callbacks + SmallVector, 2> + ModuleAnalysisRegistrationCallbacks; + SmallVector)>, + 2> + ModulePipelineParsingCallbacks; + SmallVector, + bool VerifyEachPass, bool DebugLogging)>, + 2> + TopLevelPipelineParsingCallbacks; + // CGSCC callbacks + SmallVector, 2> + CGSCCAnalysisRegistrationCallbacks; + SmallVector)>, + 2> + CGSCCPipelineParsingCallbacks; + // Function callbacks + SmallVector, 2> + FunctionAnalysisRegistrationCallbacks; + SmallVector)>, + 2> + FunctionPipelineParsingCallbacks; + // Loop callbacks + SmallVector, 2> + LoopAnalysisRegistrationCallbacks; + SmallVector)>, + 2> + LoopPipelineParsingCallbacks; + // AA callbacks + SmallVector, 2> + AAParsingCallbacks; }; + +/// This utility template takes care of adding require<> and invalidate<> +/// passes for an analysis to a given \c PassManager. It is intended to be used +/// during parsing of a pass pipeline when parsing a single PipelineName. +/// When registering a new function analysis FancyAnalysis with the pass +/// pipeline name "fancy-analysis", a matching ParsePipelineCallback could look +/// like this: +/// +/// static bool parseFunctionPipeline(StringRef Name, FunctionPassManager &FPM, +/// ArrayRef P) { +/// if (parseAnalysisUtilityPasses("fancy-analysis", Name, +/// FPM)) +/// return true; +/// return false; +/// } +template +bool parseAnalysisUtilityPasses( + StringRef AnalysisName, StringRef PipelineName, + PassManager &PM) { + if (!PipelineName.endswith(">")) + return false; + // See if this is an invalidate<> pass name + if (PipelineName.startswith("invalidate<")) { + PipelineName = PipelineName.substr(11, PipelineName.size() - 12); + if (PipelineName != AnalysisName) + return false; + PM.addPass(InvalidateAnalysisPass()); + return true; + } + + // See if this is a require<> pass name + if (PipelineName.startswith("require<")) { + PipelineName = PipelineName.substr(8, PipelineName.size() - 9); + if (PipelineName != AnalysisName) + return false; + PM.addPass(RequireAnalysisPass()); + return true; + } + + return false; +} } #endif diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProf.h b/contrib/llvm/include/llvm/ProfileData/InstrProf.h index a6b2850ccd22..772187f70153 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProf.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProf.h @@ -249,9 +249,8 @@ void annotateValueSite(Module &M, Instruction &Inst, /// Same as the above interface but using an ArrayRef, as well as \p Sum. void annotateValueSite(Module &M, Instruction &Inst, - ArrayRef VDs, - uint64_t Sum, InstrProfValueKind ValueKind, - uint32_t MaxMDCount); + ArrayRef VDs, uint64_t Sum, + InstrProfValueKind ValueKind, uint32_t MaxMDCount); /// Extract the value profile data from \p Inst which is annotated with /// value profile meta data. Return false if there is no value data annotated, @@ -582,34 +581,27 @@ struct InstrProfValueSiteRecord { /// Merge data from another InstrProfValueSiteRecord /// Optionally scale merged counts by \p Weight. - void merge(SoftInstrProfErrors &SIPE, InstrProfValueSiteRecord &Input, - uint64_t Weight = 1); + void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, + function_ref Warn); /// Scale up value profile data counts. - void scale(SoftInstrProfErrors &SIPE, uint64_t Weight); + void scale(uint64_t Weight, function_ref Warn); }; /// Profiling information for a single function. struct InstrProfRecord { - StringRef Name; - uint64_t Hash; std::vector Counts; - SoftInstrProfErrors SIPE; InstrProfRecord() = default; - InstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) - : Name(Name), Hash(Hash), Counts(std::move(Counts)) {} + InstrProfRecord(std::vector Counts) : Counts(std::move(Counts)) {} InstrProfRecord(InstrProfRecord &&) = default; InstrProfRecord(const InstrProfRecord &RHS) - : Name(RHS.Name), Hash(RHS.Hash), Counts(RHS.Counts), SIPE(RHS.SIPE), + : Counts(RHS.Counts), ValueData(RHS.ValueData ? llvm::make_unique(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { - Name = RHS.Name; - Hash = RHS.Hash; Counts = RHS.Counts; - SIPE = RHS.SIPE; if (!RHS.ValueData) { ValueData = nullptr; return *this; @@ -626,7 +618,6 @@ struct InstrProfRecord { /// Return the number of value profile kinds with non-zero number /// of profile sites. inline uint32_t getNumValueKinds() const; - /// Return the number of instrumented sites for ValueKind. inline uint32_t getNumValueSites(uint32_t ValueKind) const; @@ -661,11 +652,12 @@ struct InstrProfRecord { /// Merge the counts in \p Other into this one. /// Optionally scale merged counts by \p Weight. - void merge(InstrProfRecord &Other, uint64_t Weight = 1); + void merge(InstrProfRecord &Other, uint64_t Weight, + function_ref Warn); /// Scale up profile counts (including value profile data) by /// \p Weight. - void scale(uint64_t Weight); + void scale(uint64_t Weight, function_ref Warn); /// Sort value profile data (per site) by count. void sortValueData() { @@ -683,9 +675,6 @@ struct InstrProfRecord { /// Clear value data entries void clearValueData() { ValueData = nullptr; } - /// Get the error contained within the record's soft error counter. - Error takeError() { return SIPE.takeError(); } - private: struct ValueProfData { std::vector IndirectCallSites; @@ -737,11 +726,23 @@ struct InstrProfRecord { // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. - void mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, - uint64_t Weight); + void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src, + uint64_t Weight, + function_ref Warn); // Scale up value profile data count. - void scaleValueProfData(uint32_t ValueKind, uint64_t Weight); + void scaleValueProfData(uint32_t ValueKind, uint64_t Weight, + function_ref Warn); +}; + +struct NamedInstrProfRecord : InstrProfRecord { + StringRef Name; + uint64_t Hash; + + NamedInstrProfRecord() = default; + NamedInstrProfRecord(StringRef Name, uint64_t Hash, + std::vector Counts) + : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} }; uint32_t InstrProfRecord::getNumValueKinds() const { @@ -753,11 +754,8 @@ uint32_t InstrProfRecord::getNumValueKinds() const { uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { uint32_t N = 0; - const std::vector &SiteRecords = - getValueSitesForKind(ValueKind); - for (auto &SR : SiteRecords) { + for (auto &SR : getValueSitesForKind(ValueKind)) N += SR.ValueData.size(); - } return N; } diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h b/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h index 8163ca159209..424360e0f765 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -40,9 +40,9 @@ class InstrProfReader; /// A file format agnostic iterator over profiling data. class InstrProfIterator : public std::iterator { + NamedInstrProfRecord> { InstrProfReader *Reader = nullptr; - InstrProfRecord Record; + value_type Record; void Increment(); @@ -53,12 +53,12 @@ class InstrProfIterator : public std::iterator() { return &Record; } + value_type &operator*() { return Record; } + value_type *operator->() { return &Record; } }; /// Base class and interface for reading profiling data of any known instrprof -/// format. Provides an iterator over InstrProfRecords. +/// format. Provides an iterator over NamedInstrProfRecords. class InstrProfReader { instrprof_error LastError = instrprof_error::success; @@ -70,7 +70,7 @@ class InstrProfReader { virtual Error readHeader() = 0; /// Read a single record. - virtual Error readNextRecord(InstrProfRecord &Record) = 0; + virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0; /// Iterator over profile data. InstrProfIterator begin() { return InstrProfIterator(this); } @@ -161,7 +161,7 @@ class TextInstrProfReader : public InstrProfReader { Error readHeader() override; /// Read a single record. - Error readNextRecord(InstrProfRecord &Record) override; + Error readNextRecord(NamedInstrProfRecord &Record) override; InstrProfSymtab &getSymtab() override { assert(Symtab.get()); @@ -209,7 +209,7 @@ class RawInstrProfReader : public InstrProfReader { static bool hasFormat(const MemoryBuffer &DataBuffer); Error readHeader() override; - Error readNextRecord(InstrProfRecord &Record) override; + Error readNextRecord(NamedInstrProfRecord &Record) override; bool isIRLevelProfile() const override { return (Version & VARIANT_MASK_IR_PROF) != 0; @@ -243,8 +243,8 @@ class RawInstrProfReader : public InstrProfReader { return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); } - Error readName(InstrProfRecord &Record); - Error readFuncHash(InstrProfRecord &Record); + Error readName(NamedInstrProfRecord &Record); + Error readFuncHash(NamedInstrProfRecord &Record); Error readRawCounts(InstrProfRecord &Record); Error readValueProfilingData(InstrProfRecord &Record); bool atEnd() const { return Data == DataEnd; } @@ -281,7 +281,7 @@ enum class HashT : uint32_t; /// Trait for lookups into the on-disk hash table for the binary instrprof /// format. class InstrProfLookupTrait { - std::vector DataBuffer; + std::vector DataBuffer; IndexedInstrProf::HashT HashType; unsigned FormatVersion; // Endianness of the input value profile data. @@ -293,7 +293,7 @@ class InstrProfLookupTrait { InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion) : HashType(HashType), FormatVersion(FormatVersion) {} - using data_type = ArrayRef; + using data_type = ArrayRef; using internal_key_type = StringRef; using external_key_type = StringRef; @@ -334,11 +334,11 @@ struct InstrProfReaderIndexBase { // Read all the profile records with the same key pointed to the current // iterator. - virtual Error getRecords(ArrayRef &Data) = 0; + virtual Error getRecords(ArrayRef &Data) = 0; // Read all the profile records with the key equal to FuncName virtual Error getRecords(StringRef FuncName, - ArrayRef &Data) = 0; + ArrayRef &Data) = 0; virtual void advanceToNextKey() = 0; virtual bool atEnd() const = 0; virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; @@ -364,9 +364,9 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase { IndexedInstrProf::HashT HashType, uint64_t Version); ~InstrProfReaderIndex() override = default; - Error getRecords(ArrayRef &Data) override; + Error getRecords(ArrayRef &Data) override; Error getRecords(StringRef FuncName, - ArrayRef &Data) override; + ArrayRef &Data) override; void advanceToNextKey() override { RecordIterator++; } bool atEnd() const override { @@ -419,10 +419,9 @@ class IndexedInstrProfReader : public InstrProfReader { /// Read the file header. Error readHeader() override; /// Read a single record. - Error readNextRecord(InstrProfRecord &Record) override; + Error readNextRecord(NamedInstrProfRecord &Record) override; - /// Return the pointer to InstrProfRecord associated with FuncName - /// and FuncHash + /// Return the NamedInstrProfRecord associated with FuncName and FuncHash Expected getInstrProfRecord(StringRef FuncName, uint64_t FuncHash); diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h b/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h index fff10af30295..8107ab386fe2 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -33,7 +33,7 @@ class raw_fd_ostream; class InstrProfWriter { public: - using ProfilingData = SmallDenseMap; + using ProfilingData = SmallDenseMap; enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; private: @@ -50,10 +50,15 @@ class InstrProfWriter { /// Add function counts for the given function. If there are already counts /// for this function and the hash and number of counts match, each counter is /// summed. Optionally scale counts by \p Weight. - Error addRecord(InstrProfRecord &&I, uint64_t Weight = 1); + void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, + function_ref Warn); + void addRecord(NamedInstrProfRecord &&I, function_ref Warn) { + addRecord(std::move(I), 1, Warn); + } /// Merge existing function counts from the given writer. - Error mergeRecordsFromWriter(InstrProfWriter &&IPW); + void mergeRecordsFromWriter(InstrProfWriter &&IPW, + function_ref Warn); /// Write the profile to \c OS void write(raw_fd_ostream &OS); @@ -62,7 +67,8 @@ class InstrProfWriter { Error writeText(raw_fd_ostream &OS); /// Write \c Record in text format to \c OS - static void writeRecordInText(const InstrProfRecord &Record, + static void writeRecordInText(StringRef Name, uint64_t Hash, + const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS); /// Write the profile, returning the raw data. For testing. @@ -85,6 +91,8 @@ class InstrProfWriter { void setOutputSparse(bool Sparse); private: + void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I, + uint64_t Weight, function_ref Warn); bool shouldEncodeData(const ProfilingData &PD); void writeImpl(ProfOStream &OS); }; diff --git a/contrib/llvm/include/llvm/ProfileData/ProfileCommon.h b/contrib/llvm/include/llvm/ProfileData/ProfileCommon.h index 987e3160ccae..51b065bcdb70 100644 --- a/contrib/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/contrib/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/ProfileSummary.h" +#include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Error.h" #include #include @@ -27,8 +28,6 @@ namespace llvm { -struct InstrProfRecord; - namespace sampleprof { class FunctionSamples; diff --git a/contrib/llvm/include/llvm/Support/BlockFrequency.h b/contrib/llvm/include/llvm/Support/BlockFrequency.h index 1b45cc52973f..2e75cbdd29c1 100644 --- a/contrib/llvm/include/llvm/Support/BlockFrequency.h +++ b/contrib/llvm/include/llvm/Support/BlockFrequency.h @@ -71,6 +71,10 @@ class BlockFrequency { bool operator>=(BlockFrequency RHS) const { return Frequency >= RHS.Frequency; } + + bool operator==(BlockFrequency RHS) const { + return Frequency == RHS.Frequency; + } }; } diff --git a/contrib/llvm/include/llvm/Support/Compiler.h b/contrib/llvm/include/llvm/Support/Compiler.h index be9e46540016..b19e37235df5 100644 --- a/contrib/llvm/include/llvm/Support/Compiler.h +++ b/contrib/llvm/include/llvm/Support/Compiler.h @@ -493,4 +493,14 @@ void AnnotateIgnoreWritesEnd(const char *file, int line); #define LLVM_THREAD_LOCAL #endif +/// \macro LLVM_ENABLE_EXCEPTIONS +/// \brief Whether LLVM is built with exception support. +#if __has_feature(cxx_exceptions) +#define LLVM_ENABLE_EXCEPTIONS 1 +#elif defined(__GNUC__) && defined(__EXCEPTIONS) +#define LLVM_ENABLE_EXCEPTIONS 1 +#elif defined(_MSC_VER) && defined(_CPPUNWIND) +#define LLVM_ENABLE_EXCEPTIONS 1 +#endif + #endif diff --git a/contrib/llvm/include/llvm/Support/DynamicLibrary.h b/contrib/llvm/include/llvm/Support/DynamicLibrary.h index a8874a10d461..469d5dfad062 100644 --- a/contrib/llvm/include/llvm/Support/DynamicLibrary.h +++ b/contrib/llvm/include/llvm/Support/DynamicLibrary.h @@ -88,6 +88,22 @@ namespace sys { return !getPermanentLibrary(Filename, ErrMsg).isValid(); } + enum SearchOrdering { + /// SO_Linker - Search as a call to dlsym(dlopen(NULL)) would when + /// DynamicLibrary::getPermanentLibrary(NULL) has been called or + /// search the list of explcitly loaded symbols if not. + SO_Linker, + /// SO_LoadedFirst - Search all loaded libraries, then as SO_Linker would. + SO_LoadedFirst, + /// SO_LoadedLast - Search as SO_Linker would, then loaded libraries. + /// Only useful to search if libraries with RTLD_LOCAL have been added. + SO_LoadedLast, + /// SO_LoadOrder - Or this in to search libraries in the ordered loaded. + /// The default bahaviour is to search loaded libraries in reverse. + SO_LoadOrder = 4 + }; + static SearchOrdering SearchOrder; // = SO_Linker + /// This function will search through all previously loaded dynamic /// libraries for the symbol \p symbolName. If it is found, the address of /// that symbol is returned. If not, null is returned. Note that this will diff --git a/contrib/llvm/include/llvm/Support/ErrorHandling.h b/contrib/llvm/include/llvm/Support/ErrorHandling.h index 7c1edd801571..b45f6348390e 100644 --- a/contrib/llvm/include/llvm/Support/ErrorHandling.h +++ b/contrib/llvm/include/llvm/Support/ErrorHandling.h @@ -78,12 +78,48 @@ LLVM_ATTRIBUTE_NORETURN void report_fatal_error(StringRef reason, LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const Twine &reason, bool gen_crash_diag = true); - /// This function calls abort(), and prints the optional message to stderr. - /// Use the llvm_unreachable macro (that adds location info), instead of - /// calling this function directly. - LLVM_ATTRIBUTE_NORETURN void - llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, - unsigned line=0); +/// Installs a new bad alloc error handler that should be used whenever a +/// bad alloc error, e.g. failing malloc/calloc, is encountered by LLVM. +/// +/// The user can install a bad alloc handler, in order to define the behavior +/// in case of failing allocations, e.g. throwing an exception. Note that this +/// handler must not trigger any additional allocations itself. +/// +/// If no error handler is installed the default is to print the error message +/// to stderr, and call exit(1). If an error handler is installed then it is +/// the handler's responsibility to log the message, it will no longer be +/// printed to stderr. If the error handler returns, then exit(1) will be +/// called. +/// +/// +/// \param user_data - An argument which will be passed to the installed error +/// handler. +void install_bad_alloc_error_handler(fatal_error_handler_t handler, + void *user_data = nullptr); + +/// Restores default bad alloc error handling behavior. +void remove_bad_alloc_error_handler(); + +/// Reports a bad alloc error, calling any user defined bad alloc +/// error handler. In contrast to the generic 'report_fatal_error' +/// functions, this function is expected to return, e.g. the user +/// defined error handler throws an exception. +/// +/// Note: When throwing an exception in the bad alloc handler, make sure that +/// the following unwind succeeds, e.g. do not trigger additional allocations +/// in the unwind chain. +/// +/// If no error handler is installed (default), then a bad_alloc exception +/// is thrown if LLVM is compiled with exception support, otherwise an assertion +/// is called. +void report_bad_alloc_error(const char *Reason, bool GenCrashDiag = true); + +/// This function calls abort(), and prints the optional message to stderr. +/// Use the llvm_unreachable macro (that adds location info), instead of +/// calling this function directly. +LLVM_ATTRIBUTE_NORETURN void +llvm_unreachable_internal(const char *msg = nullptr, const char *file = nullptr, + unsigned line = 0); } /// Marks that the current location is not supposed to be reachable. diff --git a/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h index 9edf03aa3621..a0fec668e05c 100644 --- a/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -32,6 +32,20 @@ namespace llvm { namespace DomTreeBuilder { +template +struct ChildrenGetter { + static auto Get(NodePtr N) -> decltype(reverse(children(N))) { + return reverse(children(N)); + } +}; + +template +struct ChildrenGetter { + static auto Get(NodePtr N) -> decltype(inverse_children(N)) { + return inverse_children(N); + } +}; + // Information record used by Semi-NCA during tree construction. template struct SemiNCAInfo { @@ -45,6 +59,7 @@ struct SemiNCAInfo { unsigned Semi = 0; NodePtr Label = nullptr; NodePtr IDom = nullptr; + SmallVector ReverseChildren; }; std::vector NumToNode; @@ -79,66 +94,49 @@ struct SemiNCAInfo { .get(); } - // External storage for depth first iterator that reuses the info lookup map - // SemiNCAInfo already has. We don't have a set, but a map instead, so we are - // converting the one argument insert calls. - struct df_iterator_dom_storage { - public: - using BaseSet = decltype(NodeToInfo); - df_iterator_dom_storage(BaseSet &Storage) : Storage(Storage) {} + static bool AlwaysDescend(NodePtr, NodePtr) { return true; } - using iterator = typename BaseSet::iterator; - std::pair insert(NodePtr N) { - return Storage.insert({N, InfoRec()}); - } - void completed(NodePtr) {} + // Custom DFS implementation which can skip nodes based on a provided + // predicate. It also collects ReverseChildren so that we don't have to spend + // time getting predecessors in SemiNCA. + template + unsigned runDFS(NodePtr V, unsigned LastNum, DescendCondition Condition, + unsigned AttachToNum) { + assert(V); + SmallVector WorkList = {V}; + if (NodeToInfo.count(V) != 0) NodeToInfo[V].Parent = AttachToNum; - private: - BaseSet &Storage; - }; - - df_iterator_dom_storage getStorage() { return {NodeToInfo}; } - - unsigned runReverseDFS(NodePtr V, unsigned N) { - auto DFStorage = getStorage(); - - bool IsChildOfArtificialExit = (N != 0); - for (auto I = idf_ext_begin(V, DFStorage), E = idf_ext_end(V, DFStorage); - I != E; ++I) { - NodePtr BB = *I; + while (!WorkList.empty()) { + const NodePtr BB = WorkList.pop_back_val(); auto &BBInfo = NodeToInfo[BB]; - BBInfo.DFSNum = BBInfo.Semi = ++N; + + // Visited nodes always have positive DFS numbers. + if (BBInfo.DFSNum != 0) continue; + BBInfo.DFSNum = BBInfo.Semi = ++LastNum; BBInfo.Label = BB; - // Set the parent to the top of the visited stack. The stack includes us, - // and is 1 based, so we subtract to account for both of these. - if (I.getPathLength() > 1) - BBInfo.Parent = NodeToInfo[I.getPath(I.getPathLength() - 2)].DFSNum; - NumToNode.push_back(BB); // NumToNode[n] = V; + NumToNode.push_back(BB); - if (IsChildOfArtificialExit) - BBInfo.Parent = 1; + for (const NodePtr Succ : ChildrenGetter::Get(BB)) { + const auto SIT = NodeToInfo.find(Succ); + // Don't visit nodes more than once but remember to collect + // RerverseChildren. + if (SIT != NodeToInfo.end() && SIT->second.DFSNum != 0) { + if (Succ != BB) SIT->second.ReverseChildren.push_back(BB); + continue; + } - IsChildOfArtificialExit = false; + if (!Condition(BB, Succ)) continue; + + // It's fine to add Succ to the map, because we know that it will be + // visited later. + auto &SuccInfo = NodeToInfo[Succ]; + WorkList.push_back(Succ); + SuccInfo.Parent = LastNum; + SuccInfo.ReverseChildren.push_back(BB); + } } - return N; - } - unsigned runForwardDFS(NodePtr V, unsigned N) { - auto DFStorage = getStorage(); - - for (auto I = df_ext_begin(V, DFStorage), E = df_ext_end(V, DFStorage); - I != E; ++I) { - NodePtr BB = *I; - auto &BBInfo = NodeToInfo[BB]; - BBInfo.DFSNum = BBInfo.Semi = ++N; - BBInfo.Label = BB; - // Set the parent to the top of the visited stack. The stack includes us, - // and is 1 based, so we subtract to account for both of these. - if (I.getPathLength() > 1) - BBInfo.Parent = NodeToInfo[I.getPath(I.getPathLength() - 2)].DFSNum; - NumToNode.push_back(BB); // NumToNode[n] = V; - } - return N; + return LastNum; } NodePtr eval(NodePtr VIn, unsigned LastLinked) { @@ -181,31 +179,14 @@ struct SemiNCAInfo { template void runSemiNCA(DomTreeT &DT, unsigned NumBlocks) { - unsigned N = 0; - NumToNode.push_back(nullptr); - - bool MultipleRoots = (DT.Roots.size() > 1); - if (MultipleRoots) { - auto &BBInfo = NodeToInfo[nullptr]; - BBInfo.DFSNum = BBInfo.Semi = ++N; - BBInfo.Label = nullptr; - - NumToNode.push_back(nullptr); // NumToNode[n] = V; - } - // Step #1: Number blocks in depth-first order and initialize variables used // in later stages of the algorithm. - if (DT.isPostDominator()){ - for (unsigned i = 0, e = static_cast(DT.Roots.size()); - i != e; ++i) - N = runReverseDFS(DT.Roots[i], N); - } else { - N = runForwardDFS(DT.Roots[0], N); - } + const unsigned N = doFullDFSWalk(DT, AlwaysDescend); // It might be that some blocks did not get a DFS number (e.g., blocks of // infinite loops). In these cases an artificial exit node is required. - MultipleRoots |= (DT.isPostDominator() && N != NumBlocks); + const bool MultipleRoots = + DT.Roots.size() > 1 || (DT.isPostDominator() && N != NumBlocks); // Initialize IDoms to spanning tree parents. for (unsigned i = 1; i <= N; ++i) { @@ -221,7 +202,7 @@ struct SemiNCAInfo { // Initialize the semi dominator to point to the parent node. WInfo.Semi = WInfo.Parent; - for (const auto &N : inverse_children(W)) + for (const auto &N : WInfo.ReverseChildren) if (NodeToInfo.count(N)) { // Only if this predecessor is reachable! unsigned SemiU = NodeToInfo[eval(N, i + 1)].Semi; if (SemiU < WInfo.Semi) @@ -279,14 +260,27 @@ struct SemiNCAInfo { } } - void doFullDFSWalk(const DomTreeT &DT) { - NumToNode.push_back(nullptr); + template + unsigned doFullDFSWalk(const DomTreeT &DT, DescendCondition DC) { unsigned Num = 0; - for (auto *Root : DT.Roots) - if (!DT.isPostDominator()) - Num = runForwardDFS(Root, Num); - else - Num = runReverseDFS(Root, Num); + NumToNode.push_back(nullptr); + + if (DT.Roots.size() > 1) { + auto &BBInfo = NodeToInfo[nullptr]; + BBInfo.DFSNum = BBInfo.Semi = ++Num; + BBInfo.Label = nullptr; + + NumToNode.push_back(nullptr); // NumToNode[n] = V; + } + + if (DT.isPostDominator()) { + for (auto *Root : DT.Roots) Num = runDFS(Root, Num, DC, 1); + } else { + assert(DT.Roots.size() == 1); + Num = runDFS(DT.Roots[0], Num, DC, Num); + } + + return Num; } static void PrintBlockOrNullptr(raw_ostream &O, NodePtr Obj) { @@ -299,7 +293,7 @@ struct SemiNCAInfo { // Checks if the tree contains all reachable nodes in the input graph. bool verifyReachability(const DomTreeT &DT) { clear(); - doFullDFSWalk(DT); + doFullDFSWalk(DT, AlwaysDescend); for (auto &NodeToTN : DT.DomTreeNodes) { const TreeNodePtr TN = NodeToTN.second.get(); @@ -356,7 +350,7 @@ struct SemiNCAInfo { // NCD(From, To) == IDom(To) or To. bool verifyNCD(const DomTreeT &DT) { clear(); - doFullDFSWalk(DT); + doFullDFSWalk(DT, AlwaysDescend); for (auto &BlockToInfo : NodeToInfo) { auto &Info = BlockToInfo.second; @@ -440,8 +434,9 @@ struct SemiNCAInfo { if (!BB || TN->getChildren().empty()) continue; clear(); - NodeToInfo.insert({BB, {}}); - doFullDFSWalk(DT); + doFullDFSWalk(DT, [BB](NodePtr From, NodePtr To) { + return From != BB && To != BB; + }); for (TreeNodePtr Child : TN->getChildren()) if (NodeToInfo.count(Child->getBlock()) != 0) { @@ -473,8 +468,10 @@ struct SemiNCAInfo { const auto &Siblings = TN->getChildren(); for (const TreeNodePtr N : Siblings) { clear(); - NodeToInfo.insert({N->getBlock(), {}}); - doFullDFSWalk(DT); + NodePtr BBN = N->getBlock(); + doFullDFSWalk(DT, [BBN](NodePtr From, NodePtr To) { + return From != BBN && To != BBN; + }); for (const TreeNodePtr S : Siblings) { if (S == N) continue; diff --git a/contrib/llvm/include/llvm/Support/ReverseIteration.h b/contrib/llvm/include/llvm/Support/ReverseIteration.h new file mode 100644 index 000000000000..cb97b60f06dd --- /dev/null +++ b/contrib/llvm/include/llvm/Support/ReverseIteration.h @@ -0,0 +1,17 @@ +#ifndef LLVM_SUPPORT_REVERSEITERATION_H +#define LLVM_SUPPORT_REVERSEITERATION_H + +#include "llvm/Config/abi-breaking.h" + +namespace llvm { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS +template struct ReverseIterate { static bool value; }; +#if LLVM_ENABLE_REVERSE_ITERATION +template bool ReverseIterate::value = true; +#else +template bool ReverseIterate::value = false; +#endif +#endif +} + +#endif diff --git a/contrib/llvm/include/llvm/Support/UnicodeCharRanges.h b/contrib/llvm/include/llvm/Support/UnicodeCharRanges.h index d4d4d8eb84a4..4c655833b396 100644 --- a/contrib/llvm/include/llvm/Support/UnicodeCharRanges.h +++ b/contrib/llvm/include/llvm/Support/UnicodeCharRanges.h @@ -18,11 +18,11 @@ #include "llvm/Support/raw_ostream.h" #include +#define DEBUG_TYPE "unicode" + namespace llvm { namespace sys { -#define DEBUG_TYPE "unicode" - /// \brief Represents a closed range of Unicode code points [Lower, Upper]. struct UnicodeCharRange { uint32_t Lower; @@ -99,10 +99,9 @@ class UnicodeCharSet { const CharRanges Ranges; }; -#undef DEBUG_TYPE // "unicode" - } // namespace sys } // namespace llvm +#undef DEBUG_TYPE // "unicode" #endif // LLVM_SUPPORT_UNICODECHARRANGES_H diff --git a/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 3a3118139bcb..178b08d7b8b7 100644 --- a/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -64,6 +64,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. diff --git a/contrib/llvm/include/llvm/Target/TargetInstrInfo.h b/contrib/llvm/include/llvm/Target/TargetInstrInfo.h index 2fc3ec996e7f..1843a2eed9bf 100644 --- a/contrib/llvm/include/llvm/Target/TargetInstrInfo.h +++ b/contrib/llvm/include/llvm/Target/TargetInstrInfo.h @@ -1545,6 +1545,16 @@ class TargetInstrInfo : public MCInstrInfo { return None; } + /// Return an array that contains the MMO target flag values and their + /// names. + /// + /// MIR Serialization is able to serialize only the MMO target flags that are + /// defined by this method. + virtual ArrayRef> + getSerializableMachineMemOperandTargetFlags() const { + return None; + } + /// Determines whether \p Inst is a tail call instruction. Override this /// method on targets that do not properly set MCID::Return and MCID::Call on /// tail call instructions." diff --git a/contrib/llvm/include/llvm/Target/TargetLowering.h b/contrib/llvm/include/llvm/Target/TargetLowering.h index 964d6314b127..60a03bdc182d 100644 --- a/contrib/llvm/include/llvm/Target/TargetLowering.h +++ b/contrib/llvm/include/llvm/Target/TargetLowering.h @@ -415,7 +415,8 @@ class TargetLoweringBase { virtual bool mergeStoresAfterLegalization() const { return false; } /// Returns if it's reasonable to merge stores to MemVT size. - virtual bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT) const { + virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, + const SelectionDAG &DAG) const { return true; } @@ -2726,6 +2727,18 @@ class TargetLowering : public TargetLoweringBase { return true; } + // Return true if it is profitable to combine a BUILD_VECTOR to a TRUNCATE. + // Example of such a combine: + // v4i32 build_vector((extract_elt V, 0), + // (extract_elt V, 2), + // (extract_elt V, 4), + // (extract_elt V, 6)) + // --> + // v4i32 truncate (bitcast V to v4i64) + virtual bool isDesirableToCombineBuildVectorToTruncate() const { + return false; + } + /// Return true if the target has native support for the specified value type /// and it is 'desirable' to use the type for the given node type. e.g. On x86 /// i16 is legal, but undesirable since i16 instruction encodings are longer @@ -2815,6 +2828,9 @@ class TargetLowering : public TargetLoweringBase { // TargetLowering::LowerCall that perform tail call conversions. bool IsTailCall = false; + // Is Call lowering done post SelectionDAG type legalization. + bool IsPostTypeLegalization = false; + unsigned NumFixedArgs = -1; CallingConv::ID CallConv = CallingConv::C; SDValue Callee; @@ -2937,6 +2953,11 @@ class TargetLowering : public TargetLoweringBase { return *this; } + CallLoweringInfo &setIsPostTypeLegalization(bool Value=true) { + IsPostTypeLegalization = Value; + return *this; + } + ArgListTy &getArgs() { return Args; } @@ -3055,6 +3076,13 @@ class TargetLowering : public TargetLoweringBase { return Chain; } + /// This callback is used to inspect load/store instructions and add + /// target-specific MachineMemOperand flags to them. The default + /// implementation does nothing. + virtual MachineMemOperand::Flags getMMOFlags(const Instruction &I) const { + return MachineMemOperand::MONone; + } + /// This callback is invoked by the type legalizer to legalize nodes with an /// illegal operand type but legal result types. It replaces the /// LowerOperation callback in the type Legalizer. The reason we can not do diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h b/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h index 4c585a20021c..f25ab40640df 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -68,21 +68,6 @@ class GVN : public PassInfoMixin { class ValueTable { DenseMap valueNumbering; DenseMap expressionNumbering; - - // Expressions is the vector of Expression. ExprIdx is the mapping from - // value number to the index of Expression in Expressions. We use it - // instead of a DenseMap because filling such mapping is faster than - // filling a DenseMap and the compile time is a little better. - uint32_t nextExprNumber; - std::vector Expressions; - std::vector ExprIdx; - // Value number to PHINode mapping. Used for phi-translate in scalarpre. - DenseMap NumberingPhi; - // Cache for phi-translate in scalarpre. - typedef DenseMap, uint32_t> - PhiTranslateMap; - PhiTranslateMap PhiTranslateTable; - AliasAnalysis *AA; MemoryDependenceResults *MD; DominatorTree *DT; @@ -94,10 +79,6 @@ class GVN : public PassInfoMixin { Value *LHS, Value *RHS); Expression createExtractvalueExpr(ExtractValueInst *EI); uint32_t lookupOrAddCall(CallInst *C); - uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock, - uint32_t Num, GVN &Gvn); - std::pair assignExpNewValueNum(Expression &exp); - bool areAllValsInBB(uint32_t num, const BasicBlock *BB, GVN &Gvn); public: ValueTable(); @@ -106,11 +87,9 @@ class GVN : public PassInfoMixin { ~ValueTable(); uint32_t lookupOrAdd(Value *V); - uint32_t lookup(Value *V, bool Verify = true) const; + uint32_t lookup(Value *V) const; uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred, Value *LHS, Value *RHS); - uint32_t phiTranslate(const BasicBlock *BB, const BasicBlock *PhiBlock, - uint32_t Num, GVN &Gvn); bool exists(Value *V) const; void add(Value *V, uint32_t num); void clear(); @@ -152,10 +131,6 @@ class GVN : public PassInfoMixin { SmallMapVector ReplaceWithConstMap; SmallVector InstrsToErase; - // Map the block to reversed postorder traversal number. It is used to - // find back edge easily. - DenseMap BlockRPONumber; - typedef SmallVector LoadDepVect; typedef SmallVector AvailValInBlkVect; typedef SmallVector UnavailBlkVect; @@ -239,7 +214,7 @@ class GVN : public PassInfoMixin { bool performPRE(Function &F); bool performScalarPRE(Instruction *I); bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, - BasicBlock *Curr, unsigned int ValNo); + unsigned int ValNo); Value *findLeader(const BasicBlock *BB, uint32_t num); void cleanupGlobalSets(); void verifyRemoved(const Instruction *I) const; @@ -251,7 +226,6 @@ class GVN : public PassInfoMixin { bool processFoldableCondBr(BranchInst *BI); void addDeadBlock(BasicBlock *BB); void assignValNumForDeadCode(); - void assignBlockRPONumber(Function &F); }; /// Create a legacy GVN pass. This also allows parameterizing whether or not diff --git a/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h index e4906b709e4b..4554b5cbc644 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h @@ -17,21 +17,39 @@ namespace llvm { +class ConstantInt; class Instruction; class MemCpyInst; class MemMoveInst; class MemSetInst; +class TargetTransformInfo; class Value; /// Emit a loop implementing the semantics of llvm.memcpy with the equivalent /// arguments at \p InsertBefore. -void createMemCpyLoop(Instruction *InsertBefore, - Value *SrcAddr, Value *DstAddr, Value *CopyLen, - unsigned SrcAlign, unsigned DestAlign, +void createMemCpyLoop(Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, + Value *CopyLen, unsigned SrcAlign, unsigned DestAlign, bool SrcIsVolatile, bool DstIsVolatile); +/// Emit a loop implementing the semantics of llvm.memcpy where the size is not +/// a compile-time constant. Loop will be insterted at \p InsertBefore. +void createMemCpyLoopUnknownSize(Instruction *InsertBefore, Value *SrcAddr, + Value *DstAddr, Value *CopyLen, + unsigned SrcAlign, unsigned DestAlign, + bool SrcIsVolatile, bool DstIsVolatile, + const TargetTransformInfo &TTI); + +/// Emit a loop implementing the semantics of an llvm.memcpy whose size is a +/// compile time constant. Loop is inserted at \p InsertBefore. +void createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr, + Value *DstAddr, ConstantInt *CopyLen, + unsigned SrcAlign, unsigned DestAlign, + bool SrcIsVolatile, bool DstIsVolatile, + const TargetTransformInfo &TTI); + + /// Expand \p MemCpy as a loop. \p MemCpy is not deleted. -void expandMemCpyAsLoop(MemCpyInst *MemCpy); +void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI); /// Expand \p MemMove as a loop. \p MemMove is not deleted. void expandMemMoveAsLoop(MemMoveInst *MemMove); diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h index b0448fed9f4d..2dd205d8b2af 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h @@ -22,10 +22,10 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" -namespace llvm { - #define DEBUG_TYPE "ssaupdater" +namespace llvm { + class CastInst; class PHINode; template class SSAUpdaterTraits; @@ -453,8 +453,8 @@ class SSAUpdaterImpl { } }; +} // end llvm namespace + #undef DEBUG_TYPE // "ssaupdater" -} // End llvm namespace - -#endif +#endif // LLVM_TRANSFORMS_UTILS_SSAUPDATERIMPL_H diff --git a/contrib/llvm/include/llvm/module.modulemap b/contrib/llvm/include/llvm/module.modulemap index dd419e861316..766198bbc5de 100644 --- a/contrib/llvm/include/llvm/module.modulemap +++ b/contrib/llvm/include/llvm/module.modulemap @@ -23,6 +23,7 @@ module LLVM_Backend { exclude header "CodeGen/CommandFlags.h" exclude header "CodeGen/LinkAllAsmWriterComponents.h" exclude header "CodeGen/LinkAllCodegenComponents.h" + exclude header "CodeGen/GlobalISel/InstructionSelectorImpl.h" // These are intended for (repeated) textual inclusion. textual header "CodeGen/DIEValue.def" diff --git a/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp index b52a1d7b24d6..e682a644ef2c 100644 --- a/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/contrib/llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -1006,7 +1006,7 @@ static AliasResult aliasSameBasePointerGEPs(const GEPOperator *GEP1, // Because they cannot partially overlap and because fields in an array // cannot overlap, if we can prove the final indices are different between // GEP1 and GEP2, we can conclude GEP1 and GEP2 don't alias. - + // If the last indices are constants, we've already checked they don't // equal each other so we can exit early. if (C1 && C2) diff --git a/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp index 23d5a887c34a..a329e5ad48c9 100644 --- a/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/contrib/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -538,7 +538,7 @@ bool BranchProbabilityInfo::calcZeroHeuristics(const BasicBlock *BB, // InstCombine canonicalizes X <= 0 into X < 1. // X <= 0 -> Unlikely isProb = false; - } else if (CV->isAllOnesValue()) { + } else if (CV->isMinusOne()) { switch (CI->getPredicate()) { case CmpInst::ICMP_EQ: // X == -1 -> Unlikely diff --git a/contrib/llvm/lib/Analysis/CGSCCPassManager.cpp b/contrib/llvm/lib/Analysis/CGSCCPassManager.cpp index 9d4521221f47..3ddefc6520a7 100644 --- a/contrib/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/contrib/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -196,18 +196,117 @@ FunctionAnalysisManagerCGSCCProxy::run(LazyCallGraph::SCC &C, bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate( LazyCallGraph::SCC &C, const PreservedAnalyses &PA, CGSCCAnalysisManager::Invalidator &Inv) { - for (LazyCallGraph::Node &N : C) - FAM->invalidate(N.getFunction(), PA); + // If literally everything is preserved, we're done. + if (PA.areAllPreserved()) + return false; // This is still a valid proxy. - // This proxy doesn't need to handle invalidation itself. Instead, the - // module-level CGSCC proxy handles it above by ensuring that if the - // module-level FAM proxy becomes invalid the entire SCC layer, which - // includes this proxy, is cleared. + // If this proxy isn't marked as preserved, then even if the result remains + // valid, the key itself may no longer be valid, so we clear everything. + // + // Note that in order to preserve this proxy, a module pass must ensure that + // the FAM has been completely updated to handle the deletion of functions. + // Specifically, any FAM-cached results for those functions need to have been + // forcibly cleared. When preserved, this proxy will only invalidate results + // cached on functions *still in the module* at the end of the module pass. + auto PAC = PA.getChecker(); + if (!PAC.preserved() && !PAC.preservedSet>()) { + for (LazyCallGraph::Node &N : C) + FAM->clear(N.getFunction()); + + return true; + } + + // Directly check if the relevant set is preserved. + bool AreFunctionAnalysesPreserved = + PA.allAnalysesInSetPreserved>(); + + // Now walk all the functions to see if any inner analysis invalidation is + // necessary. + for (LazyCallGraph::Node &N : C) { + Function &F = N.getFunction(); + Optional FunctionPA; + + // Check to see whether the preserved set needs to be pruned based on + // SCC-level analysis invalidation that triggers deferred invalidation + // registered with the outer analysis manager proxy for this function. + if (auto *OuterProxy = + FAM->getCachedResult(F)) + for (const auto &OuterInvalidationPair : + OuterProxy->getOuterInvalidations()) { + AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; + const auto &InnerAnalysisIDs = OuterInvalidationPair.second; + if (Inv.invalidate(OuterAnalysisID, C, PA)) { + if (!FunctionPA) + FunctionPA = PA; + for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) + FunctionPA->abandon(InnerAnalysisID); + } + } + + // Check if we needed a custom PA set, and if so we'll need to run the + // inner invalidation. + if (FunctionPA) { + FAM->invalidate(F, *FunctionPA); + continue; + } + + // Otherwise we only need to do invalidation if the original PA set didn't + // preserve all function analyses. + if (!AreFunctionAnalysesPreserved) + FAM->invalidate(F, PA); + } + + // Return false to indicate that this result is still a valid proxy. return false; } } // End llvm namespace +/// When a new SCC is created for the graph and there might be function +/// analysis results cached for the functions now in that SCC two forms of +/// updates are required. +/// +/// First, a proxy from the SCC to the FunctionAnalysisManager needs to be +/// created so that any subsequent invalidation events to the SCC are +/// propagated to the function analysis results cached for functions within it. +/// +/// Second, if any of the functions within the SCC have analysis results with +/// outer analysis dependencies, then those dependencies would point to the +/// *wrong* SCC's analysis result. We forcibly invalidate the necessary +/// function analyses so that they don't retain stale handles. +static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C, + LazyCallGraph &G, + CGSCCAnalysisManager &AM) { + // Get the relevant function analysis manager. + auto &FAM = + AM.getResult(C, G).getManager(); + + // Now walk the functions in this SCC and invalidate any function analysis + // results that might have outer dependencies on an SCC analysis. + for (LazyCallGraph::Node &N : C) { + Function &F = N.getFunction(); + + auto *OuterProxy = + FAM.getCachedResult(F); + if (!OuterProxy) + // No outer analyses were queried, nothing to do. + continue; + + // Forcibly abandon all the inner analyses with dependencies, but + // invalidate nothing else. + auto PA = PreservedAnalyses::all(); + for (const auto &OuterInvalidationPair : + OuterProxy->getOuterInvalidations()) { + const auto &InnerAnalysisIDs = OuterInvalidationPair.second; + for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) + PA.abandon(InnerAnalysisID); + } + + // Now invalidate anything we found. + FAM.invalidate(F, PA); + } +} + namespace { /// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c /// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly @@ -236,7 +335,6 @@ incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G, dbgs() << "Enqueuing the existing SCC in the worklist:" << *C << "\n"; SCC *OldC = C; - (void)OldC; // Update the current SCC. Note that if we have new SCCs, this must actually // change the SCC. @@ -245,6 +343,26 @@ incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G, C = &*NewSCCRange.begin(); assert(G.lookupSCC(N) == C && "Failed to update current SCC!"); + // If we had a cached FAM proxy originally, we will want to create more of + // them for each SCC that was split off. + bool NeedFAMProxy = + AM.getCachedResult(*OldC) != nullptr; + + // We need to propagate an invalidation call to all but the newly current SCC + // because the outer pass manager won't do that for us after splitting them. + // FIXME: We should accept a PreservedAnalysis from the CG updater so that if + // there are preserved ananalyses we can avoid invalidating them here for + // split-off SCCs. + // We know however that this will preserve any FAM proxy so go ahead and mark + // that. + PreservedAnalyses PA; + PA.preserve(); + AM.invalidate(*OldC, PA); + + // Ensure the now-current SCC's function analyses are updated. + if (NeedFAMProxy) + updateNewSCCFunctionAnalyses(*C, G, AM); + for (SCC &NewC : reverse(make_range(std::next(NewSCCRange.begin()), NewSCCRange.end()))) { assert(C != &NewC && "No need to re-visit the current SCC!"); @@ -252,6 +370,14 @@ incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G, UR.CWorklist.insert(&NewC); if (DebugLogging) dbgs() << "Enqueuing a newly formed SCC:" << NewC << "\n"; + + // Ensure new SCCs' function analyses are updated. + if (NeedFAMProxy) + updateNewSCCFunctionAnalyses(NewC, G, AM); + + // Also propagate a normal invalidation to the new SCC as only the current + // will get one from the pass manager infrastructure. + AM.invalidate(NewC, PA); } return C; } @@ -349,14 +475,6 @@ LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass( // For separate SCCs this is trivial. RC->switchTrivialInternalEdgeToRef(N, TargetN); } else { - // Otherwise we may end up re-structuring the call graph. First, - // invalidate any SCC analyses. We have to do this before we split - // functions into new SCCs and lose track of where their analyses are - // cached. - // FIXME: We should accept a more precise preserved set here. For - // example, it might be possible to preserve some function analyses - // even as the SCC structure is changed. - AM.invalidate(*C, PreservedAnalyses::none()); // Now update the call graph. C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, TargetN), G, N, C, AM, UR, DebugLogging); @@ -424,13 +542,6 @@ LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass( continue; } - // Otherwise we may end up re-structuring the call graph. First, invalidate - // any SCC analyses. We have to do this before we split functions into new - // SCCs and lose track of where their analyses are cached. - // FIXME: We should accept a more precise preserved set here. For example, - // it might be possible to preserve some function analyses even as the SCC - // structure is changed. - AM.invalidate(*C, PreservedAnalyses::none()); // Now update the call graph. C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, *RefTarget), G, N, C, AM, UR, DebugLogging); @@ -459,25 +570,48 @@ LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass( // Otherwise we are switching an internal ref edge to a call edge. This // may merge away some SCCs, and we add those to the UpdateResult. We also // need to make sure to update the worklist in the event SCCs have moved - // before the current one in the post-order sequence. + // before the current one in the post-order sequence + bool HasFunctionAnalysisProxy = false; auto InitialSCCIndex = RC->find(*C) - RC->begin(); - auto InvalidatedSCCs = RC->switchInternalEdgeToCall(N, *CallTarget); - if (!InvalidatedSCCs.empty()) { + bool FormedCycle = RC->switchInternalEdgeToCall( + N, *CallTarget, [&](ArrayRef MergedSCCs) { + for (SCC *MergedC : MergedSCCs) { + assert(MergedC != &TargetC && "Cannot merge away the target SCC!"); + + HasFunctionAnalysisProxy |= + AM.getCachedResult( + *MergedC) != nullptr; + + // Mark that this SCC will no longer be valid. + UR.InvalidatedSCCs.insert(MergedC); + + // FIXME: We should really do a 'clear' here to forcibly release + // memory, but we don't have a good way of doing that and + // preserving the function analyses. + auto PA = PreservedAnalyses::allInSet>(); + PA.preserve(); + AM.invalidate(*MergedC, PA); + } + }); + + // If we formed a cycle by creating this call, we need to update more data + // structures. + if (FormedCycle) { C = &TargetC; assert(G.lookupSCC(N) == C && "Failed to update current SCC!"); + // If one of the invalidated SCCs had a cached proxy to a function + // analysis manager, we need to create a proxy in the new current SCC as + // the invaliadted SCCs had their functions moved. + if (HasFunctionAnalysisProxy) + AM.getResult(*C, G); + // Any analyses cached for this SCC are no longer precise as the shape - // has changed by introducing this cycle. - AM.invalidate(*C, PreservedAnalyses::none()); - - for (SCC *InvalidatedC : InvalidatedSCCs) { - assert(InvalidatedC != C && "Cannot invalidate the current SCC!"); - UR.InvalidatedSCCs.insert(InvalidatedC); - - // Also clear any cached analyses for the SCCs that are dead. This - // isn't really necessary for correctness but can release memory. - AM.clear(*InvalidatedC); - } + // has changed by introducing this cycle. However, we have taken care to + // update the proxies so it remains valide. + auto PA = PreservedAnalyses::allInSet>(); + PA.preserve(); + AM.invalidate(*C, PA); } auto NewSCCIndex = RC->find(*C) - RC->begin(); if (InitialSCCIndex < NewSCCIndex) { diff --git a/contrib/llvm/lib/Analysis/CaptureTracking.cpp b/contrib/llvm/lib/Analysis/CaptureTracking.cpp index 2093f0fdec12..3b0026ba10e9 100644 --- a/contrib/llvm/lib/Analysis/CaptureTracking.cpp +++ b/contrib/llvm/lib/Analysis/CaptureTracking.cpp @@ -94,8 +94,8 @@ namespace { // guarantee that 'I' never reaches 'BeforeHere' through a back-edge or // by its successors, i.e, prune if: // - // (1) BB is an entry block or have no sucessors. - // (2) There's no path coming back through BB sucessors. + // (1) BB is an entry block or have no successors. + // (2) There's no path coming back through BB successors. if (BB == &BB->getParent()->getEntryBlock() || !BB->getTerminator()->getNumSuccessors()) return true; diff --git a/contrib/llvm/lib/Analysis/DemandedBits.cpp b/contrib/llvm/lib/Analysis/DemandedBits.cpp index 926b28d6094a..9c53f9140ca3 100644 --- a/contrib/llvm/lib/Analysis/DemandedBits.cpp +++ b/contrib/llvm/lib/Analysis/DemandedBits.cpp @@ -143,9 +143,8 @@ void DemandedBits::determineLiveOperandBits( break; case Instruction::Shl: if (OperandNo == 0) - if (ConstantInt *CI = - dyn_cast(UserI->getOperand(1))) { - uint64_t ShiftAmt = CI->getLimitedValue(BitWidth-1); + if (auto *ShiftAmtC = dyn_cast(UserI->getOperand(1))) { + uint64_t ShiftAmt = ShiftAmtC->getLimitedValue(BitWidth - 1); AB = AOut.lshr(ShiftAmt); // If the shift is nuw/nsw, then the high bits are not dead @@ -159,9 +158,8 @@ void DemandedBits::determineLiveOperandBits( break; case Instruction::LShr: if (OperandNo == 0) - if (ConstantInt *CI = - dyn_cast(UserI->getOperand(1))) { - uint64_t ShiftAmt = CI->getLimitedValue(BitWidth-1); + if (auto *ShiftAmtC = dyn_cast(UserI->getOperand(1))) { + uint64_t ShiftAmt = ShiftAmtC->getLimitedValue(BitWidth - 1); AB = AOut.shl(ShiftAmt); // If the shift is exact, then the low bits are not dead @@ -172,9 +170,8 @@ void DemandedBits::determineLiveOperandBits( break; case Instruction::AShr: if (OperandNo == 0) - if (ConstantInt *CI = - dyn_cast(UserI->getOperand(1))) { - uint64_t ShiftAmt = CI->getLimitedValue(BitWidth-1); + if (auto *ShiftAmtC = dyn_cast(UserI->getOperand(1))) { + uint64_t ShiftAmt = ShiftAmtC->getLimitedValue(BitWidth - 1); AB = AOut.shl(ShiftAmt); // Because the high input bit is replicated into the // high-order bits of the result, if we need any of those diff --git a/contrib/llvm/lib/Analysis/DependenceAnalysis.cpp b/contrib/llvm/lib/Analysis/DependenceAnalysis.cpp index e4d58bf1b4eb..34eccc07f265 100644 --- a/contrib/llvm/lib/Analysis/DependenceAnalysis.cpp +++ b/contrib/llvm/lib/Analysis/DependenceAnalysis.cpp @@ -3342,7 +3342,8 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, UsefulGEP = isLoopInvariant(SrcPtrSCEV, LI->getLoopFor(Src->getParent())) && isLoopInvariant(DstPtrSCEV, LI->getLoopFor(Dst->getParent())) && - (SrcGEP->getNumOperands() == DstGEP->getNumOperands()); + (SrcGEP->getNumOperands() == DstGEP->getNumOperands()) && + isKnownPredicate(CmpInst::ICMP_EQ, SrcPtrSCEV, DstPtrSCEV); } unsigned Pairs = UsefulGEP ? SrcGEP->idx_end() - SrcGEP->idx_begin() : 1; SmallVector Pair(Pairs); @@ -3371,7 +3372,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, if (Delinearize && CommonLevels > 1) { if (tryDelinearize(Src, Dst, Pair)) { - DEBUG(dbgs() << " delinerized GEP\n"); + DEBUG(dbgs() << " delinearized GEP\n"); Pairs = Pair.size(); } } @@ -3796,7 +3797,7 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, if (Delinearize && CommonLevels > 1) { if (tryDelinearize(Src, Dst, Pair)) { - DEBUG(dbgs() << " delinerized GEP\n"); + DEBUG(dbgs() << " delinearized GEP\n"); Pairs = Pair.size(); } } diff --git a/contrib/llvm/lib/Analysis/InstructionSimplify.cpp b/contrib/llvm/lib/Analysis/InstructionSimplify.cpp index d9e32a3c417e..f6632020b8fc 100644 --- a/contrib/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/contrib/llvm/lib/Analysis/InstructionSimplify.cpp @@ -560,7 +560,7 @@ static Value *SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, return Y; /// i1 add -> xor. - if (MaxRecurse && Op0->getType()->getScalarType()->isIntegerTy(1)) + if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) if (Value *V = SimplifyXorInst(Op0, Op1, Q, MaxRecurse-1)) return V; @@ -598,7 +598,7 @@ Value *llvm::SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, /// folding. static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, bool AllowNonInbounds = false) { - assert(V->getType()->getScalarType()->isPointerTy()); + assert(V->getType()->isPtrOrPtrVectorTy()); Type *IntPtrTy = DL.getIntPtrType(V->getType())->getScalarType(); APInt Offset = APInt::getNullValue(IntPtrTy->getIntegerBitWidth()); @@ -627,8 +627,7 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, } break; } - assert(V->getType()->getScalarType()->isPointerTy() && - "Unexpected operand type!"); + assert(V->getType()->isPtrOrPtrVectorTy() && "Unexpected operand type!"); } while (Visited.insert(V).second); Constant *OffsetIntPtr = ConstantInt::get(IntPtrTy, Offset); @@ -771,7 +770,7 @@ static Value *SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, return ConstantExpr::getIntegerCast(Result, Op0->getType(), true); // i1 sub -> xor. - if (MaxRecurse && Op0->getType()->getScalarType()->isIntegerTy(1)) + if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) if (Value *V = SimplifyXorInst(Op0, Op1, Q, MaxRecurse-1)) return V; @@ -902,7 +901,7 @@ static Value *SimplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, return X; // i1 mul -> and. - if (MaxRecurse && Op0->getType()->getScalarType()->isIntegerTy(1)) + if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) if (Value *V = SimplifyAndInst(Op0, Op1, Q, MaxRecurse-1)) return V; @@ -998,7 +997,7 @@ static Value *simplifyDivRem(Value *Op0, Value *Op1, bool IsDiv) { // X % 1 -> 0 // If this is a boolean op (single-bit element type), we can't have // division-by-zero or remainder-by-zero, so assume the divisor is 1. - if (match(Op1, m_One()) || Ty->getScalarType()->isIntegerTy(1)) + if (match(Op1, m_One()) || Ty->isIntOrIntVectorTy(1)) return IsDiv ? Op0 : Constant::getNullValue(Ty); return nullptr; @@ -2251,7 +2250,7 @@ static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q) { Type *ITy = GetCompareTy(LHS); // The return type. Type *OpTy = LHS->getType(); // The operand type. - if (!OpTy->getScalarType()->isIntegerTy(1)) + if (!OpTy->isIntOrIntVectorTy(1)) return nullptr; // A boolean compared to true/false can be simplified in 14 out of the 20 diff --git a/contrib/llvm/lib/Analysis/LazyCallGraph.cpp b/contrib/llvm/lib/Analysis/LazyCallGraph.cpp index b6a9436cc1ec..a4c3e43b4b0c 100644 --- a/contrib/llvm/lib/Analysis/LazyCallGraph.cpp +++ b/contrib/llvm/lib/Analysis/LazyCallGraph.cpp @@ -456,8 +456,10 @@ updatePostorderSequenceForEdgeInsertion( return make_range(SCCs.begin() + SourceIdx, SCCs.begin() + TargetIdx); } -SmallVector -LazyCallGraph::RefSCC::switchInternalEdgeToCall(Node &SourceN, Node &TargetN) { +bool +LazyCallGraph::RefSCC::switchInternalEdgeToCall( + Node &SourceN, Node &TargetN, + function_ref MergeSCCs)> MergeCB) { assert(!(*SourceN)[TargetN].isCall() && "Must start with a ref edge!"); SmallVector DeletedSCCs; @@ -475,7 +477,7 @@ LazyCallGraph::RefSCC::switchInternalEdgeToCall(Node &SourceN, Node &TargetN) { // we've just added more connectivity. if (&SourceSCC == &TargetSCC) { SourceN->setEdgeKind(TargetN, Edge::Call); - return DeletedSCCs; + return false; // No new cycle. } // At this point we leverage the postorder list of SCCs to detect when the @@ -488,7 +490,7 @@ LazyCallGraph::RefSCC::switchInternalEdgeToCall(Node &SourceN, Node &TargetN) { int TargetIdx = SCCIndices[&TargetSCC]; if (TargetIdx < SourceIdx) { SourceN->setEdgeKind(TargetN, Edge::Call); - return DeletedSCCs; + return false; // No new cycle. } // Compute the SCCs which (transitively) reach the source. @@ -555,12 +557,16 @@ LazyCallGraph::RefSCC::switchInternalEdgeToCall(Node &SourceN, Node &TargetN) { SourceSCC, TargetSCC, SCCs, SCCIndices, ComputeSourceConnectedSet, ComputeTargetConnectedSet); + // Run the user's callback on the merged SCCs before we actually merge them. + if (MergeCB) + MergeCB(makeArrayRef(MergeRange.begin(), MergeRange.end())); + // If the merge range is empty, then adding the edge didn't actually form any // new cycles. We're done. if (MergeRange.begin() == MergeRange.end()) { // Now that the SCC structure is finalized, flip the kind to call. SourceN->setEdgeKind(TargetN, Edge::Call); - return DeletedSCCs; + return false; // No new cycle. } #ifndef NDEBUG @@ -596,8 +602,8 @@ LazyCallGraph::RefSCC::switchInternalEdgeToCall(Node &SourceN, Node &TargetN) { // Now that the SCC structure is finalized, flip the kind to call. SourceN->setEdgeKind(TargetN, Edge::Call); - // And we're done! - return DeletedSCCs; + // And we're done, but we did form a new cycle. + return true; } void LazyCallGraph::RefSCC::switchTrivialInternalEdgeToRef(Node &SourceN, diff --git a/contrib/llvm/lib/Analysis/Lint.cpp b/contrib/llvm/lib/Analysis/Lint.cpp index 9713588537b3..ada600a69b87 100644 --- a/contrib/llvm/lib/Analysis/Lint.cpp +++ b/contrib/llvm/lib/Analysis/Lint.cpp @@ -405,7 +405,7 @@ void Lint::visitMemoryReference(Instruction &I, Assert(!isa(UnderlyingObject), "Undefined behavior: Undef pointer dereference", &I); Assert(!isa(UnderlyingObject) || - !cast(UnderlyingObject)->isAllOnesValue(), + !cast(UnderlyingObject)->isMinusOne(), "Unusual: All-ones pointer dereference", &I); Assert(!isa(UnderlyingObject) || !cast(UnderlyingObject)->isOne(), diff --git a/contrib/llvm/lib/Analysis/LoopInfo.cpp b/contrib/llvm/lib/Analysis/LoopInfo.cpp index ff68810abb82..baf932432a0a 100644 --- a/contrib/llvm/lib/Analysis/LoopInfo.cpp +++ b/contrib/llvm/lib/Analysis/LoopInfo.cpp @@ -131,13 +131,13 @@ PHINode *Loop::getCanonicalInductionVariable() const { PHINode *PN = cast(I); if (ConstantInt *CI = dyn_cast(PN->getIncomingValueForBlock(Incoming))) - if (CI->isNullValue()) + if (CI->isZero()) if (Instruction *Inc = dyn_cast(PN->getIncomingValueForBlock(Backedge))) if (Inc->getOpcode() == Instruction::Add && Inc->getOperand(0) == PN) if (ConstantInt *CI = dyn_cast(Inc->getOperand(1))) - if (CI->equalsInt(1)) + if (CI->isOne()) return PN; } return nullptr; @@ -460,7 +460,7 @@ class UnloopUpdater { void UnloopUpdater::updateBlockParents() { if (Unloop.getNumBlocks()) { // Perform a post order CFG traversal of all blocks within this loop, - // propagating the nearest loop from sucessors to predecessors. + // propagating the nearest loop from successors to predecessors. LoopBlocksTraversal Traversal(DFS, LI); for (BasicBlock *POI : Traversal) { diff --git a/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp b/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp index f88d54b21e1e..7327c07499be 100644 --- a/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp +++ b/contrib/llvm/lib/Analysis/MemoryBuiltins.cpp @@ -505,6 +505,22 @@ SizeOffsetType ObjectSizeOffsetVisitor::compute(Value *V) { return unknown(); } +/// When we're compiling N-bit code, and the user uses parameters that are +/// greater than N bits (e.g. uint64_t on a 32-bit build), we can run into +/// trouble with APInt size issues. This function handles resizing + overflow +/// checks for us. Check and zext or trunc \p I depending on IntTyBits and +/// I's value. +bool ObjectSizeOffsetVisitor::CheckedZextOrTrunc(APInt &I) { + // More bits than we can handle. Checking the bit width isn't necessary, but + // it's faster than checking active bits, and should give `false` in the + // vast majority of cases. + if (I.getBitWidth() > IntTyBits && I.getActiveBits() > IntTyBits) + return false; + if (I.getBitWidth() != IntTyBits) + I = I.zextOrTrunc(IntTyBits); + return true; +} + SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) { if (!I.getAllocatedType()->isSized()) return unknown(); @@ -515,8 +531,14 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) { Value *ArraySize = I.getArraySize(); if (const ConstantInt *C = dyn_cast(ArraySize)) { - Size *= C->getValue().zextOrSelf(IntTyBits); - return std::make_pair(align(Size, I.getAlignment()), Zero); + APInt NumElems = C->getValue(); + if (!CheckedZextOrTrunc(NumElems)) + return unknown(); + + bool Overflow; + Size = Size.umul_ov(NumElems, Overflow); + return Overflow ? unknown() : std::make_pair(align(Size, I.getAlignment()), + Zero); } return unknown(); } @@ -561,21 +583,6 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitCallSite(CallSite CS) { if (!Arg) return unknown(); - // When we're compiling N-bit code, and the user uses parameters that are - // greater than N bits (e.g. uint64_t on a 32-bit build), we can run into - // trouble with APInt size issues. This function handles resizing + overflow - // checks for us. - auto CheckedZextOrTrunc = [&](APInt &I) { - // More bits than we can handle. Checking the bit width isn't necessary, but - // it's faster than checking active bits, and should give `false` in the - // vast majority of cases. - if (I.getBitWidth() > IntTyBits && I.getActiveBits() > IntTyBits) - return false; - if (I.getBitWidth() != IntTyBits) - I = I.zextOrTrunc(IntTyBits); - return true; - }; - APInt Size = Arg->getValue(); if (!CheckedZextOrTrunc(Size)) return unknown(); diff --git a/contrib/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/contrib/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 095647e1bd20..e9e354ebb88f 100644 --- a/contrib/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/contrib/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -266,7 +266,7 @@ computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, // sample PGO, to enable the same inlines as the profiled optimized binary. for (auto &I : F.getImportGUIDs()) CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness( - CalleeInfo::HotnessType::Hot); + CalleeInfo::HotnessType::Critical); bool NonRenamableLocal = isNonRenamableLocal(F); bool NotEligibleForImport = diff --git a/contrib/llvm/lib/Analysis/ScalarEvolution.cpp b/contrib/llvm/lib/Analysis/ScalarEvolution.cpp index 678ad3af5e85..3fb1ab980add 100644 --- a/contrib/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/contrib/llvm/lib/Analysis/ScalarEvolution.cpp @@ -326,7 +326,7 @@ bool SCEV::isOne() const { bool SCEV::isAllOnesValue() const { if (const SCEVConstant *SC = dyn_cast(this)) - return SC->getValue()->isAllOnesValue(); + return SC->getValue()->isMinusOne(); return false; } @@ -2743,7 +2743,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, } // If we are left with a constant one being multiplied, strip it off. - if (cast(Ops[0])->getValue()->equalsInt(1)) { + if (cast(Ops[0])->getValue()->isOne()) { Ops.erase(Ops.begin()); --Idx; } else if (cast(Ops[0])->getValue()->isZero()) { @@ -2939,7 +2939,7 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, "SCEVUDivExpr operand types don't match!"); if (const SCEVConstant *RHSC = dyn_cast(RHS)) { - if (RHSC->getValue()->equalsInt(1)) + if (RHSC->getValue()->isOne()) return LHS; // X udiv 1 --> x // If the denominator is zero, the result of the udiv is undefined. Don't // try to analyze it, because the resolution chosen here may differ from @@ -5421,9 +5421,9 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { // For an expression like x&255 that merely masks off the high bits, // use zext(trunc(x)) as the SCEV expression. if (ConstantInt *CI = dyn_cast(BO->RHS)) { - if (CI->isNullValue()) + if (CI->isZero()) return getSCEV(BO->RHS); - if (CI->isAllOnesValue()) + if (CI->isMinusOne()) return getSCEV(BO->LHS); const APInt &A = CI->getValue(); @@ -5498,7 +5498,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { case Instruction::Xor: if (ConstantInt *CI = dyn_cast(BO->RHS)) { // If the RHS of xor is -1, then this is a not operation. - if (CI->isAllOnesValue()) + if (CI->isMinusOne()) return getNotSCEV(getSCEV(BO->LHS)); // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask. @@ -5577,7 +5577,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { if (CI->getValue().uge(BitWidth)) break; - if (CI->isNullValue()) + if (CI->isZero()) return getSCEV(BO->LHS); // shift by zero --> noop uint64_t AShrAmt = CI->getZExtValue(); @@ -7626,7 +7626,7 @@ ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. // We have not yet seen any such cases. const SCEVConstant *StepC = dyn_cast(Step); - if (!StepC || StepC->getValue()->equalsInt(0)) + if (!StepC || StepC->getValue()->isZero()) return getCouldNotCompute(); // For positive steps (counting up until unsigned overflow): @@ -7640,7 +7640,7 @@ ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, // Handle unitary steps, which cannot wraparound. // 1*N = -Start; -1*N = Start (mod 2^BW), so: // N = Distance (as unsigned) - if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) { + if (StepC->getValue()->isOne() || StepC->getValue()->isMinusOne()) { APInt MaxBECount = getUnsignedRangeMax(Distance); // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated, @@ -7696,7 +7696,7 @@ ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) { // If the value is a constant, check to see if it is known to be non-zero // already. If so, the backedge will execute zero times. if (const SCEVConstant *C = dyn_cast(V)) { - if (!C->getValue()->isNullValue()) + if (!C->getValue()->isZero()) return getZero(C->getType()); return getCouldNotCompute(); // Otherwise it will loop infinitely. } diff --git a/contrib/llvm/lib/Analysis/TargetTransformInfo.cpp b/contrib/llvm/lib/Analysis/TargetTransformInfo.cpp index f938a9a52065..94bbc58541a7 100644 --- a/contrib/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/contrib/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -16,6 +16,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include @@ -23,6 +24,11 @@ using namespace llvm; #define DEBUG_TYPE "tti" +static cl::opt UseWideMemcpyLoopLowering( + "use-wide-memcpy-loop-lowering", cl::init(false), + cl::desc("Enables the new wide memcpy loop lowering in Transforms/Utils."), + cl::Hidden); + namespace { /// \brief No-op implementation of the TTI interface using the utility base /// classes. @@ -482,6 +488,25 @@ Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); } +Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context, + Value *Length, + unsigned SrcAlign, + unsigned DestAlign) const { + return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign, + DestAlign); +} + +void TargetTransformInfo::getMemcpyLoopResidualLoweringType( + SmallVectorImpl &OpsOut, LLVMContext &Context, + unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const { + TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, + SrcAlign, DestAlign); +} + +bool TargetTransformInfo::useWideIRMemcpyLoopLowering() const { + return UseWideMemcpyLoopLowering; +} + bool TargetTransformInfo::areInlineCompatible(const Function *Caller, const Function *Callee) const { return TTIImpl->areInlineCompatible(Caller, Callee); diff --git a/contrib/llvm/lib/Analysis/ValueTracking.cpp b/contrib/llvm/lib/Analysis/ValueTracking.cpp index fd6e3a643bf0..9e042da8801d 100644 --- a/contrib/llvm/lib/Analysis/ValueTracking.cpp +++ b/contrib/llvm/lib/Analysis/ValueTracking.cpp @@ -1500,12 +1500,10 @@ void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, assert(Depth <= MaxDepth && "Limit Search Depth"); unsigned BitWidth = Known.getBitWidth(); - assert((V->getType()->isIntOrIntVectorTy() || - V->getType()->getScalarType()->isPointerTy()) && + assert((V->getType()->isIntOrIntVectorTy(BitWidth) || + V->getType()->isPtrOrPtrVectorTy()) && "Not integer or pointer type!"); - assert((Q.DL.getTypeSizeInBits(V->getType()->getScalarType()) == BitWidth) && - (!V->getType()->isIntOrIntVectorTy() || - V->getType()->getScalarSizeInBits() == BitWidth) && + assert(Q.DL.getTypeSizeInBits(V->getType()->getScalarType()) == BitWidth && "V and Known should have same BitWidth"); (void)BitWidth; @@ -1952,7 +1950,7 @@ bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q) { } // Check if all incoming values are non-zero constant. bool AllNonZeroConstants = all_of(PN->operands(), [](Value *V) { - return isa(V) && !cast(V)->isZeroValue(); + return isa(V) && !cast(V)->isZero(); }); if (AllNonZeroConstants) return true; @@ -4393,7 +4391,7 @@ isImpliedCondMatchingImmOperands(CmpInst::Predicate APred, const Value *ALHS, } Optional llvm::isImpliedCondition(const Value *LHS, const Value *RHS, - const DataLayout &DL, bool InvertAPred, + const DataLayout &DL, bool LHSIsFalse, unsigned Depth, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { @@ -4402,26 +4400,51 @@ Optional llvm::isImpliedCondition(const Value *LHS, const Value *RHS, return None; Type *OpTy = LHS->getType(); - assert(OpTy->getScalarType()->isIntegerTy(1)); + assert(OpTy->isIntOrIntVectorTy(1)); // LHS ==> RHS by definition - if (!InvertAPred && LHS == RHS) - return true; + if (LHS == RHS) + return !LHSIsFalse; if (OpTy->isVectorTy()) // TODO: extending the code below to handle vectors return None; assert(OpTy->isIntegerTy(1) && "implied by above"); - ICmpInst::Predicate APred, BPred; - Value *ALHS, *ARHS; Value *BLHS, *BRHS; - - if (!match(LHS, m_ICmp(APred, m_Value(ALHS), m_Value(ARHS))) || - !match(RHS, m_ICmp(BPred, m_Value(BLHS), m_Value(BRHS)))) + ICmpInst::Predicate BPred; + // We expect the RHS to be an icmp. + if (!match(RHS, m_ICmp(BPred, m_Value(BLHS), m_Value(BRHS)))) return None; - if (InvertAPred) + Value *ALHS, *ARHS; + ICmpInst::Predicate APred; + // The LHS can be an 'or', 'and', or 'icmp'. + if (!match(LHS, m_ICmp(APred, m_Value(ALHS), m_Value(ARHS)))) { + // The remaining tests are all recursive, so bail out if we hit the limit. + if (Depth == MaxDepth) + return None; + // If the result of an 'or' is false, then we know both legs of the 'or' are + // false. Similarly, if the result of an 'and' is true, then we know both + // legs of the 'and' are true. + if ((LHSIsFalse && match(LHS, m_Or(m_Value(ALHS), m_Value(ARHS)))) || + (!LHSIsFalse && match(LHS, m_And(m_Value(ALHS), m_Value(ARHS))))) { + if (Optional Implication = isImpliedCondition( + ALHS, RHS, DL, LHSIsFalse, Depth + 1, AC, CxtI, DT)) + return Implication; + if (Optional Implication = isImpliedCondition( + ARHS, RHS, DL, LHSIsFalse, Depth + 1, AC, CxtI, DT)) + return Implication; + return None; + } + return None; + } + // All of the below logic assumes both LHS and RHS are icmps. + assert(isa(LHS) && isa(RHS) && "Expected icmps."); + + // The rest of the logic assumes the LHS condition is true. If that's not the + // case, invert the predicate to make it so. + if (LHSIsFalse) APred = CmpInst::getInversePredicate(APred); // Can we infer anything when the two compares have matching operands? diff --git a/contrib/llvm/lib/Analysis/VectorUtils.cpp b/contrib/llvm/lib/Analysis/VectorUtils.cpp index 0ace8fa382bc..554d132c2ab7 100644 --- a/contrib/llvm/lib/Analysis/VectorUtils.cpp +++ b/contrib/llvm/lib/Analysis/VectorUtils.cpp @@ -301,7 +301,7 @@ const llvm::Value *llvm::getSplatValue(const Value *V) { auto *InsertEltInst = dyn_cast(ShuffleInst->getOperand(0)); if (!InsertEltInst || !isa(InsertEltInst->getOperand(2)) || - !cast(InsertEltInst->getOperand(2))->isNullValue()) + !cast(InsertEltInst->getOperand(2))->isZero()) return nullptr; return InsertEltInst->getOperand(1); diff --git a/contrib/llvm/lib/AsmParser/LLLexer.cpp b/contrib/llvm/lib/AsmParser/LLLexer.cpp index a49276099f19..428bb21fbf51 100644 --- a/contrib/llvm/lib/AsmParser/LLLexer.cpp +++ b/contrib/llvm/lib/AsmParser/LLLexer.cpp @@ -542,7 +542,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(release); KEYWORD(acq_rel); KEYWORD(seq_cst); - KEYWORD(singlethread); + KEYWORD(syncscope); KEYWORD(nnan); KEYWORD(ninf); diff --git a/contrib/llvm/lib/AsmParser/LLParser.cpp b/contrib/llvm/lib/AsmParser/LLParser.cpp index 9ad31125f4b8..717eb0e00f4f 100644 --- a/contrib/llvm/lib/AsmParser/LLParser.cpp +++ b/contrib/llvm/lib/AsmParser/LLParser.cpp @@ -1919,20 +1919,42 @@ bool LLParser::parseAllocSizeArguments(unsigned &BaseSizeArg, } /// ParseScopeAndOrdering -/// if isAtomic: ::= 'singlethread'? AtomicOrdering +/// if isAtomic: ::= SyncScope? AtomicOrdering /// else: ::= /// /// This sets Scope and Ordering to the parsed values. -bool LLParser::ParseScopeAndOrdering(bool isAtomic, SynchronizationScope &Scope, +bool LLParser::ParseScopeAndOrdering(bool isAtomic, SyncScope::ID &SSID, AtomicOrdering &Ordering) { if (!isAtomic) return false; - Scope = CrossThread; - if (EatIfPresent(lltok::kw_singlethread)) - Scope = SingleThread; + return ParseScope(SSID) || ParseOrdering(Ordering); +} - return ParseOrdering(Ordering); +/// ParseScope +/// ::= syncscope("singlethread" | "")? +/// +/// This sets synchronization scope ID to the ID of the parsed value. +bool LLParser::ParseScope(SyncScope::ID &SSID) { + SSID = SyncScope::System; + if (EatIfPresent(lltok::kw_syncscope)) { + auto StartParenAt = Lex.getLoc(); + if (!EatIfPresent(lltok::lparen)) + return Error(StartParenAt, "Expected '(' in syncscope"); + + std::string SSN; + auto SSNAt = Lex.getLoc(); + if (ParseStringConstant(SSN)) + return Error(SSNAt, "Expected synchronization scope name"); + + auto EndParenAt = Lex.getLoc(); + if (!EatIfPresent(lltok::rparen)) + return Error(EndParenAt, "Expected ')' in syncscope"); + + SSID = Context.getOrInsertSyncScopeID(SSN); + } + + return false; } /// ParseOrdering @@ -3061,7 +3083,7 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { } else { assert(Opc == Instruction::ICmp && "Unexpected opcode for CmpInst!"); if (!Val0->getType()->isIntOrIntVectorTy() && - !Val0->getType()->getScalarType()->isPointerTy()) + !Val0->getType()->isPtrOrPtrVectorTy()) return Error(ID.Loc, "icmp requires pointer or integer operands"); ID.ConstantVal = ConstantExpr::getICmp(Pred, Val0, Val1); } @@ -3210,7 +3232,7 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { if (Opc == Instruction::GetElementPtr) { if (Elts.size() == 0 || - !Elts[0]->getType()->getScalarType()->isPointerTy()) + !Elts[0]->getType()->isPtrOrPtrVectorTy()) return Error(ID.Loc, "base of getelementptr must be a pointer"); Type *BaseType = Elts[0]->getType(); @@ -3226,7 +3248,7 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { ArrayRef Indices(Elts.begin() + 1, Elts.end()); for (Constant *Val : Indices) { Type *ValTy = Val->getType(); - if (!ValTy->getScalarType()->isIntegerTy()) + if (!ValTy->isIntOrIntVectorTy()) return Error(ID.Loc, "getelementptr index must be an integer"); if (ValTy->isVectorTy()) { unsigned ValNumEl = ValTy->getVectorNumElements(); @@ -5697,7 +5719,7 @@ bool LLParser::ParseCompare(Instruction *&Inst, PerFunctionState &PFS, } else { assert(Opc == Instruction::ICmp && "Unknown opcode for CmpInst!"); if (!LHS->getType()->isIntOrIntVectorTy() && - !LHS->getType()->getScalarType()->isPointerTy()) + !LHS->getType()->isPtrOrPtrVectorTy()) return Error(Loc, "icmp requires integer operands"); Inst = new ICmpInst(CmpInst::Predicate(Pred), LHS, RHS); } @@ -6100,7 +6122,7 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { bool AteExtraComma = false; bool isAtomic = false; AtomicOrdering Ordering = AtomicOrdering::NotAtomic; - SynchronizationScope Scope = CrossThread; + SyncScope::ID SSID = SyncScope::System; if (Lex.getKind() == lltok::kw_atomic) { isAtomic = true; @@ -6118,7 +6140,7 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { if (ParseType(Ty) || ParseToken(lltok::comma, "expected comma after load's type") || ParseTypeAndValue(Val, Loc, PFS) || - ParseScopeAndOrdering(isAtomic, Scope, Ordering) || + ParseScopeAndOrdering(isAtomic, SSID, Ordering) || ParseOptionalCommaAlign(Alignment, AteExtraComma)) return true; @@ -6134,7 +6156,7 @@ int LLParser::ParseLoad(Instruction *&Inst, PerFunctionState &PFS) { return Error(ExplicitTypeLoc, "explicit pointee type doesn't match operand's pointee type"); - Inst = new LoadInst(Ty, Val, "", isVolatile, Alignment, Ordering, Scope); + Inst = new LoadInst(Ty, Val, "", isVolatile, Alignment, Ordering, SSID); return AteExtraComma ? InstExtraComma : InstNormal; } @@ -6149,7 +6171,7 @@ int LLParser::ParseStore(Instruction *&Inst, PerFunctionState &PFS) { bool AteExtraComma = false; bool isAtomic = false; AtomicOrdering Ordering = AtomicOrdering::NotAtomic; - SynchronizationScope Scope = CrossThread; + SyncScope::ID SSID = SyncScope::System; if (Lex.getKind() == lltok::kw_atomic) { isAtomic = true; @@ -6165,7 +6187,7 @@ int LLParser::ParseStore(Instruction *&Inst, PerFunctionState &PFS) { if (ParseTypeAndValue(Val, Loc, PFS) || ParseToken(lltok::comma, "expected ',' after store operand") || ParseTypeAndValue(Ptr, PtrLoc, PFS) || - ParseScopeAndOrdering(isAtomic, Scope, Ordering) || + ParseScopeAndOrdering(isAtomic, SSID, Ordering) || ParseOptionalCommaAlign(Alignment, AteExtraComma)) return true; @@ -6181,7 +6203,7 @@ int LLParser::ParseStore(Instruction *&Inst, PerFunctionState &PFS) { Ordering == AtomicOrdering::AcquireRelease) return Error(Loc, "atomic store cannot use Acquire ordering"); - Inst = new StoreInst(Val, Ptr, isVolatile, Alignment, Ordering, Scope); + Inst = new StoreInst(Val, Ptr, isVolatile, Alignment, Ordering, SSID); return AteExtraComma ? InstExtraComma : InstNormal; } @@ -6193,7 +6215,7 @@ int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { bool AteExtraComma = false; AtomicOrdering SuccessOrdering = AtomicOrdering::NotAtomic; AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic; - SynchronizationScope Scope = CrossThread; + SyncScope::ID SSID = SyncScope::System; bool isVolatile = false; bool isWeak = false; @@ -6208,7 +6230,7 @@ int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { ParseTypeAndValue(Cmp, CmpLoc, PFS) || ParseToken(lltok::comma, "expected ',' after cmpxchg cmp operand") || ParseTypeAndValue(New, NewLoc, PFS) || - ParseScopeAndOrdering(true /*Always atomic*/, Scope, SuccessOrdering) || + ParseScopeAndOrdering(true /*Always atomic*/, SSID, SuccessOrdering) || ParseOrdering(FailureOrdering)) return true; @@ -6231,7 +6253,7 @@ int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) { if (!New->getType()->isFirstClassType()) return Error(NewLoc, "cmpxchg operand must be a first class value"); AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst( - Ptr, Cmp, New, SuccessOrdering, FailureOrdering, Scope); + Ptr, Cmp, New, SuccessOrdering, FailureOrdering, SSID); CXI->setVolatile(isVolatile); CXI->setWeak(isWeak); Inst = CXI; @@ -6245,7 +6267,7 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { Value *Ptr, *Val; LocTy PtrLoc, ValLoc; bool AteExtraComma = false; AtomicOrdering Ordering = AtomicOrdering::NotAtomic; - SynchronizationScope Scope = CrossThread; + SyncScope::ID SSID = SyncScope::System; bool isVolatile = false; AtomicRMWInst::BinOp Operation; @@ -6271,7 +6293,7 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { if (ParseTypeAndValue(Ptr, PtrLoc, PFS) || ParseToken(lltok::comma, "expected ',' after atomicrmw address") || ParseTypeAndValue(Val, ValLoc, PFS) || - ParseScopeAndOrdering(true /*Always atomic*/, Scope, Ordering)) + ParseScopeAndOrdering(true /*Always atomic*/, SSID, Ordering)) return true; if (Ordering == AtomicOrdering::Unordered) @@ -6288,7 +6310,7 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { " integer"); AtomicRMWInst *RMWI = - new AtomicRMWInst(Operation, Ptr, Val, Ordering, Scope); + new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID); RMWI->setVolatile(isVolatile); Inst = RMWI; return AteExtraComma ? InstExtraComma : InstNormal; @@ -6298,8 +6320,8 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { /// ::= 'fence' 'singlethread'? AtomicOrdering int LLParser::ParseFence(Instruction *&Inst, PerFunctionState &PFS) { AtomicOrdering Ordering = AtomicOrdering::NotAtomic; - SynchronizationScope Scope = CrossThread; - if (ParseScopeAndOrdering(true /*Always atomic*/, Scope, Ordering)) + SyncScope::ID SSID = SyncScope::System; + if (ParseScopeAndOrdering(true /*Always atomic*/, SSID, Ordering)) return true; if (Ordering == AtomicOrdering::Unordered) @@ -6307,7 +6329,7 @@ int LLParser::ParseFence(Instruction *&Inst, PerFunctionState &PFS) { if (Ordering == AtomicOrdering::Monotonic) return TokError("fence cannot be monotonic"); - Inst = new FenceInst(Context, Ordering, Scope); + Inst = new FenceInst(Context, Ordering, SSID); return InstNormal; } @@ -6349,7 +6371,7 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) { break; } if (ParseTypeAndValue(Val, EltLoc, PFS)) return true; - if (!Val->getType()->getScalarType()->isIntegerTy()) + if (!Val->getType()->isIntOrIntVectorTy()) return Error(EltLoc, "getelementptr index must be an integer"); if (Val->getType()->isVectorTy()) { diff --git a/contrib/llvm/lib/AsmParser/LLParser.h b/contrib/llvm/lib/AsmParser/LLParser.h index 4616c2e86947..d5b059355c42 100644 --- a/contrib/llvm/lib/AsmParser/LLParser.h +++ b/contrib/llvm/lib/AsmParser/LLParser.h @@ -241,8 +241,9 @@ namespace llvm { bool ParseOptionalCallingConv(unsigned &CC); bool ParseOptionalAlignment(unsigned &Alignment); bool ParseOptionalDerefAttrBytes(lltok::Kind AttrKind, uint64_t &Bytes); - bool ParseScopeAndOrdering(bool isAtomic, SynchronizationScope &Scope, + bool ParseScopeAndOrdering(bool isAtomic, SyncScope::ID &SSID, AtomicOrdering &Ordering); + bool ParseScope(SyncScope::ID &SSID); bool ParseOrdering(AtomicOrdering &Ordering); bool ParseOptionalStackAlignment(unsigned &Alignment); bool ParseOptionalCommaAlign(unsigned &Alignment, bool &AteExtraComma); diff --git a/contrib/llvm/lib/AsmParser/LLToken.h b/contrib/llvm/lib/AsmParser/LLToken.h index 6c8ed7da495d..9c7a06de81b4 100644 --- a/contrib/llvm/lib/AsmParser/LLToken.h +++ b/contrib/llvm/lib/AsmParser/LLToken.h @@ -93,7 +93,7 @@ enum Kind { kw_release, kw_acq_rel, kw_seq_cst, - kw_singlethread, + kw_syncscope, kw_nnan, kw_ninf, kw_nsz, diff --git a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 1ebef3173135..2b4970a80cdd 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -513,6 +513,7 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { TBAAVerifier TBAAVerifyHelper; std::vector BundleTags; + SmallVector SSIDs; public: BitcodeReader(BitstreamCursor Stream, StringRef Strtab, @@ -648,6 +649,7 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { Error parseTypeTable(); Error parseTypeTableBody(); Error parseOperandBundleTags(); + Error parseSyncScopeNames(); Expected recordValue(SmallVectorImpl &Record, unsigned NameIndex, Triple &TT); @@ -668,6 +670,8 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { Error findFunctionInStream( Function *F, DenseMap::iterator DeferredFunctionInfoIterator); + + SyncScope::ID getDecodedSyncScopeID(unsigned Val); }; /// Class to manage reading and parsing function summary index bitcode @@ -998,14 +1002,6 @@ static AtomicOrdering getDecodedOrdering(unsigned Val) { } } -static SynchronizationScope getDecodedSynchScope(unsigned Val) { - switch (Val) { - case bitc::SYNCHSCOPE_SINGLETHREAD: return SingleThread; - default: // Map unknown scopes to cross-thread. - case bitc::SYNCHSCOPE_CROSSTHREAD: return CrossThread; - } -} - static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) { switch (Val) { default: // Map unknown selection kinds to any. @@ -1745,6 +1741,44 @@ Error BitcodeReader::parseOperandBundleTags() { } } +Error BitcodeReader::parseSyncScopeNames() { + if (Stream.EnterSubBlock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID)) + return error("Invalid record"); + + if (!SSIDs.empty()) + return error("Invalid multiple synchronization scope names blocks"); + + SmallVector Record; + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + if (SSIDs.empty()) + return error("Invalid empty synchronization scope names block"); + return Error::success(); + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Synchronization scope names are implicitly mapped to synchronization + // scope IDs by their order. + + if (Stream.readRecord(Entry.ID, Record) != bitc::SYNC_SCOPE_NAME) + return error("Invalid record"); + + SmallString<16> SSN; + if (convertToString(Record, 0, SSN)) + return error("Invalid record"); + + SSIDs.push_back(Context.getOrInsertSyncScopeID(SSN)); + Record.clear(); + } +} + /// Associate a value with its name from the given index in the provided record. Expected BitcodeReader::recordValue(SmallVectorImpl &Record, unsigned NameIndex, Triple &TT) { @@ -3132,6 +3166,10 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, if (Error Err = parseOperandBundleTags()) return Err; break; + case bitc::SYNC_SCOPE_NAMES_BLOCK_ID: + if (Error Err = parseSyncScopeNames()) + return Err; + break; } continue; @@ -4204,7 +4242,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_INST_LOADATOMIC: { - // LOADATOMIC: [opty, op, align, vol, ordering, synchscope] + // LOADATOMIC: [opty, op, align, vol, ordering, ssid] unsigned OpNum = 0; Value *Op; if (getValueTypePair(Record, OpNum, NextValueNo, Op) || @@ -4226,12 +4264,12 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); unsigned Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SynchScope); + I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SSID); InstructionList.push_back(I); break; @@ -4260,7 +4298,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } case bitc::FUNC_CODE_INST_STOREATOMIC: case bitc::FUNC_CODE_INST_STOREATOMIC_OLD: { - // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, synchscope] + // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, ssid] unsigned OpNum = 0; Value *Val, *Ptr; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || @@ -4280,20 +4318,20 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::AcquireRelease) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0) return error("Invalid record"); unsigned Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new StoreInst(Val, Ptr, Record[OpNum+1], Align, Ordering, SynchScope); + I = new StoreInst(Val, Ptr, Record[OpNum+1], Align, Ordering, SSID); InstructionList.push_back(I); break; } case bitc::FUNC_CODE_INST_CMPXCHG_OLD: case bitc::FUNC_CODE_INST_CMPXCHG: { - // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, synchscope, + // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, ssid, // failureordering?, isweak?] unsigned OpNum = 0; Value *Ptr, *Cmp, *New; @@ -4310,7 +4348,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (SuccessOrdering == AtomicOrdering::NotAtomic || SuccessOrdering == AtomicOrdering::Unordered) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 2]); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 2]); if (Error Err = typeCheckLoadStoreInst(Cmp->getType(), Ptr->getType())) return Err; @@ -4322,7 +4360,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { FailureOrdering = getDecodedOrdering(Record[OpNum + 3]); I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering, - SynchScope); + SSID); cast(I)->setVolatile(Record[OpNum]); if (Record.size() < 8) { @@ -4339,7 +4377,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_INST_ATOMICRMW: { - // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, synchscope] + // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, ssid] unsigned OpNum = 0; Value *Ptr, *Val; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || @@ -4356,13 +4394,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (Ordering == AtomicOrdering::NotAtomic || Ordering == AtomicOrdering::Unordered) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); - I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SynchScope); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); + I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID); cast(I)->setVolatile(Record[OpNum+1]); InstructionList.push_back(I); break; } - case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, synchscope] + case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, ssid] if (2 != Record.size()) return error("Invalid record"); AtomicOrdering Ordering = getDecodedOrdering(Record[0]); @@ -4370,8 +4408,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Ordering == AtomicOrdering::Unordered || Ordering == AtomicOrdering::Monotonic) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[1]); - I = new FenceInst(Context, Ordering, SynchScope); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[1]); + I = new FenceInst(Context, Ordering, SSID); InstructionList.push_back(I); break; } @@ -4567,6 +4605,14 @@ Error BitcodeReader::findFunctionInStream( return Error::success(); } +SyncScope::ID BitcodeReader::getDecodedSyncScopeID(unsigned Val) { + if (Val == SyncScope::SingleThread || Val == SyncScope::System) + return SyncScope::ID(Val); + if (Val >= SSIDs.size()) + return SyncScope::System; // Map unknown synchronization scopes to system. + return SSIDs[Val]; +} + //===----------------------------------------------------------------------===// // GVMaterializer implementation //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index b2b1ea6de374..0e518d2bbc8f 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -114,6 +114,8 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { /// True if a module hash record should be written. bool GenerateHash; + SHA1 Hasher; + /// If non-null, when GenerateHash is true, the resulting hash is written /// into ModHash. When GenerateHash is false, that specified value /// is used as the hash instead of computing from the generated bitcode. @@ -176,6 +178,8 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { private: uint64_t bitcodeStartBit() { return BitcodeStartBit; } + size_t addToStrtab(StringRef Str); + void writeAttributeGroupTable(); void writeAttributeTable(); void writeTypeTable(); @@ -262,6 +266,7 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { const GlobalObject &GO); void writeModuleMetadataKinds(); void writeOperandBundleTags(); + void writeSyncScopeNames(); void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal); void writeModuleConstants(); bool pushValueAndType(const Value *V, unsigned InstID, @@ -312,6 +317,10 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { return VE.getValueID(VI.getValue()); } std::map &valueIds() { return GUIDToValueIdMap; } + + unsigned getEncodedSyncScopeID(SyncScope::ID SSID) { + return unsigned(SSID); + } }; /// Class to manage the bitcode writing for a combined index. @@ -481,14 +490,6 @@ static unsigned getEncodedOrdering(AtomicOrdering Ordering) { llvm_unreachable("Invalid ordering"); } -static unsigned getEncodedSynchScope(SynchronizationScope SynchScope) { - switch (SynchScope) { - case SingleThread: return bitc::SYNCHSCOPE_SINGLETHREAD; - case CrossThread: return bitc::SYNCHSCOPE_CROSSTHREAD; - } - llvm_unreachable("Invalid synch scope"); -} - static void writeStringRecord(BitstreamWriter &Stream, unsigned Code, StringRef Str, unsigned AbbrevToUse) { SmallVector Vals; @@ -947,11 +948,17 @@ static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) { llvm_unreachable("Invalid unnamed_addr"); } +size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) { + if (GenerateHash) + Hasher.update(Str); + return StrtabBuilder.add(Str); +} + void ModuleBitcodeWriter::writeComdats() { SmallVector Vals; for (const Comdat *C : VE.getComdats()) { // COMDAT: [strtab offset, strtab size, selection_kind] - Vals.push_back(StrtabBuilder.add(C->getName())); + Vals.push_back(addToStrtab(C->getName())); Vals.push_back(C->getName().size()); Vals.push_back(getEncodedComdatSelectionKind(*C)); Stream.EmitRecord(bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/0); @@ -1122,7 +1129,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // linkage, alignment, section, visibility, threadlocal, // unnamed_addr, externally_initialized, dllstorageclass, // comdat, attributes] - Vals.push_back(StrtabBuilder.add(GV.getName())); + Vals.push_back(addToStrtab(GV.getName())); Vals.push_back(GV.getName().size()); Vals.push_back(VE.getTypeID(GV.getValueType())); Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant()); @@ -1161,7 +1168,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // linkage, paramattrs, alignment, section, visibility, gc, // unnamed_addr, prologuedata, dllstorageclass, comdat, // prefixdata, personalityfn] - Vals.push_back(StrtabBuilder.add(F.getName())); + Vals.push_back(addToStrtab(F.getName())); Vals.push_back(F.getName().size()); Vals.push_back(VE.getTypeID(F.getFunctionType())); Vals.push_back(F.getCallingConv()); @@ -1191,7 +1198,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { for (const GlobalAlias &A : M.aliases()) { // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage, // visibility, dllstorageclass, threadlocal, unnamed_addr] - Vals.push_back(StrtabBuilder.add(A.getName())); + Vals.push_back(addToStrtab(A.getName())); Vals.push_back(A.getName().size()); Vals.push_back(VE.getTypeID(A.getValueType())); Vals.push_back(A.getType()->getAddressSpace()); @@ -1210,7 +1217,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { for (const GlobalIFunc &I : M.ifuncs()) { // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver // val#, linkage, visibility] - Vals.push_back(StrtabBuilder.add(I.getName())); + Vals.push_back(addToStrtab(I.getName())); Vals.push_back(I.getName().size()); Vals.push_back(VE.getTypeID(I.getValueType())); Vals.push_back(I.getType()->getAddressSpace()); @@ -2032,6 +2039,24 @@ void ModuleBitcodeWriter::writeOperandBundleTags() { Stream.ExitBlock(); } +void ModuleBitcodeWriter::writeSyncScopeNames() { + SmallVector SSNs; + M.getContext().getSyncScopeNames(SSNs); + if (SSNs.empty()) + return; + + Stream.EnterSubblock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID, 2); + + SmallVector Record; + for (auto SSN : SSNs) { + Record.append(SSN.begin(), SSN.end()); + Stream.EmitRecord(bitc::SYNC_SCOPE_NAME, Record, 0); + Record.clear(); + } + + Stream.ExitBlock(); +} + static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) { if ((int64_t)V >= 0) Vals.push_back(V << 1); @@ -2648,7 +2673,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(cast(I).isVolatile()); if (cast(I).isAtomic()) { Vals.push_back(getEncodedOrdering(cast(I).getOrdering())); - Vals.push_back(getEncodedSynchScope(cast(I).getSynchScope())); + Vals.push_back(getEncodedSyncScopeID(cast(I).getSyncScopeID())); } break; case Instruction::Store: @@ -2662,7 +2687,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(cast(I).isVolatile()); if (cast(I).isAtomic()) { Vals.push_back(getEncodedOrdering(cast(I).getOrdering())); - Vals.push_back(getEncodedSynchScope(cast(I).getSynchScope())); + Vals.push_back( + getEncodedSyncScopeID(cast(I).getSyncScopeID())); } break; case Instruction::AtomicCmpXchg: @@ -2674,7 +2700,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back( getEncodedOrdering(cast(I).getSuccessOrdering())); Vals.push_back( - getEncodedSynchScope(cast(I).getSynchScope())); + getEncodedSyncScopeID(cast(I).getSyncScopeID())); Vals.push_back( getEncodedOrdering(cast(I).getFailureOrdering())); Vals.push_back(cast(I).isWeak()); @@ -2688,12 +2714,12 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(cast(I).isVolatile()); Vals.push_back(getEncodedOrdering(cast(I).getOrdering())); Vals.push_back( - getEncodedSynchScope(cast(I).getSynchScope())); + getEncodedSyncScopeID(cast(I).getSyncScopeID())); break; case Instruction::Fence: Code = bitc::FUNC_CODE_INST_FENCE; Vals.push_back(getEncodedOrdering(cast(I).getOrdering())); - Vals.push_back(getEncodedSynchScope(cast(I).getSynchScope())); + Vals.push_back(getEncodedSyncScopeID(cast(I).getSyncScopeID())); break; case Instruction::Call: { const CallInst &CI = cast(I); @@ -3648,7 +3674,6 @@ void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { // Emit the module's hash. // MODULE_CODE_HASH: [5*i32] if (GenerateHash) { - SHA1 Hasher; uint32_t Vals[5]; Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos], Buffer.size() - BlockStartPos)); @@ -3707,6 +3732,7 @@ void ModuleBitcodeWriter::write() { writeUseListBlock(nullptr); writeOperandBundleTags(); + writeSyncScopeNames(); // Emit function bodies. DenseMap FunctionToBitcodeIndex; diff --git a/contrib/llvm/lib/CodeGen/AtomicExpandPass.cpp b/contrib/llvm/lib/CodeGen/AtomicExpandPass.cpp index 344136b1f195..aa9c8e94d08a 100644 --- a/contrib/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/contrib/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -361,7 +361,7 @@ LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) { auto *NewLI = Builder.CreateLoad(NewAddr); NewLI->setAlignment(LI->getAlignment()); NewLI->setVolatile(LI->isVolatile()); - NewLI->setAtomic(LI->getOrdering(), LI->getSynchScope()); + NewLI->setAtomic(LI->getOrdering(), LI->getSyncScopeID()); DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n"); Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType()); @@ -444,7 +444,7 @@ StoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) { StoreInst *NewSI = Builder.CreateStore(NewVal, NewAddr); NewSI->setAlignment(SI->getAlignment()); NewSI->setVolatile(SI->isVolatile()); - NewSI->setAtomic(SI->getOrdering(), SI->getSynchScope()); + NewSI->setAtomic(SI->getOrdering(), SI->getSyncScopeID()); DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n"); SI->eraseFromParent(); return NewSI; @@ -801,7 +801,7 @@ void AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) { Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted); AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg( PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(), - CI->getFailureOrdering(), CI->getSynchScope()); + CI->getFailureOrdering(), CI->getSyncScopeID()); NewCI->setVolatile(CI->isVolatile()); // When we're building a strong cmpxchg, we need a loop, so you // might think we could use a weak cmpxchg inside. But, using strong @@ -924,7 +924,7 @@ AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst * auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal, CI->getSuccessOrdering(), CI->getFailureOrdering(), - CI->getSynchScope()); + CI->getSyncScopeID()); NewCI->setVolatile(CI->isVolatile()); NewCI->setWeak(CI->isWeak()); DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n"); diff --git a/contrib/llvm/lib/CodeGen/CodeGen.cpp b/contrib/llvm/lib/CodeGen/CodeGen.cpp index faa5f139cf7b..b7fd45a3f6a6 100644 --- a/contrib/llvm/lib/CodeGen/CodeGen.cpp +++ b/contrib/llvm/lib/CodeGen/CodeGen.cpp @@ -78,6 +78,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializePreISelIntrinsicLoweringLegacyPassPass(Registry); initializeProcessImplicitDefsPass(Registry); initializeRABasicPass(Registry); + initializeRAFastPass(Registry); initializeRAGreedyPass(Registry); initializeRegisterCoalescerPass(Registry); initializeRenameIndependentSubregsPass(Registry); diff --git a/contrib/llvm/lib/CodeGen/CodeGenPrepare.cpp b/contrib/llvm/lib/CodeGen/CodeGenPrepare.cpp index b50e76f2e3ba..b7155ac2480a 100644 --- a/contrib/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/contrib/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -4270,6 +4270,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, Value *Consensus = nullptr; unsigned NumUsesConsensus = 0; bool IsNumUsesConsensusValid = false; + bool PhiSeen = false; SmallVector AddrModeInsts; ExtAddrMode AddrMode; TypePromotionTransaction TPT(RemovedInsts); @@ -4289,6 +4290,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, if (PHINode *P = dyn_cast(V)) { for (Value *IncValue : P->incoming_values()) worklist.push_back(IncValue); + PhiSeen = true; continue; } @@ -4342,9 +4344,10 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, TPT.commit(); // If all the instructions matched are already in this BB, don't do anything. - if (none_of(AddrModeInsts, [&](Value *V) { + // If we saw Phi node then it is not local definitely. + if (!PhiSeen && none_of(AddrModeInsts, [&](Value *V) { return IsNonLocalValue(V, MemoryInst->getParent()); - })) { + })) { DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode << "\n"); return false; } @@ -4390,6 +4393,20 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, AddrMode.Scale = 0; } + // It is only safe to sign extend the BaseReg if we know that the math + // required to create it did not overflow before we extend it. Since + // the original IR value was tossed in favor of a constant back when + // the AddrMode was created we need to bail out gracefully if widths + // do not match instead of extending it. + // + // (See below for code to add the scale.) + if (AddrMode.Scale) { + Type *ScaledRegTy = AddrMode.ScaledReg->getType(); + if (cast(IntPtrTy)->getBitWidth() > + cast(ScaledRegTy)->getBitWidth()) + return false; + } + if (AddrMode.BaseGV) { if (ResultPtr) return false; @@ -4440,19 +4457,11 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, Value *V = AddrMode.ScaledReg; if (V->getType() == IntPtrTy) { // done. - } else if (cast(IntPtrTy)->getBitWidth() < - cast(V->getType())->getBitWidth()) { - V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); } else { - // It is only safe to sign extend the BaseReg if we know that the math - // required to create it did not overflow before we extend it. Since - // the original IR value was tossed in favor of a constant back when - // the AddrMode was created we need to bail out gracefully if widths - // do not match instead of extending it. - Instruction *I = dyn_cast_or_null(ResultIndex); - if (I && (ResultIndex != AddrMode.BaseReg)) - I->eraseFromParent(); - return false; + assert(cast(IntPtrTy)->getBitWidth() < + cast(V->getType())->getBitWidth() && + "We can't transform if ScaledReg is too narrow"); + V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); } if (AddrMode.Scale != 1) diff --git a/contrib/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/contrib/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 521037f9d206..ed1bd995e60b 100644 --- a/contrib/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/contrib/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -345,7 +345,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { *MF->getMachineMemOperand(MachinePointerInfo(LI.getPointerOperand()), Flags, DL->getTypeStoreSize(LI.getType()), getMemOpAlignment(LI), AAMDNodes(), nullptr, - LI.getSynchScope(), LI.getOrdering())); + LI.getSyncScopeID(), LI.getOrdering())); return true; } @@ -363,7 +363,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) { *MF->getMachineMemOperand( MachinePointerInfo(SI.getPointerOperand()), Flags, DL->getTypeStoreSize(SI.getValueOperand()->getType()), - getMemOpAlignment(SI), AAMDNodes(), nullptr, SI.getSynchScope(), + getMemOpAlignment(SI), AAMDNodes(), nullptr, SI.getSyncScopeID(), SI.getOrdering())); return true; } diff --git a/contrib/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp b/contrib/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp index 860fc9a4f8b6..bf427225d6a9 100644 --- a/contrib/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp +++ b/contrib/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp @@ -16,7 +16,11 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCInstrDesc.h" +#include "llvm/IR/Constants.h" +#include "llvm/Target/TargetInstrInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -26,6 +30,9 @@ using namespace llvm; +InstructionSelector::MatcherState::MatcherState(unsigned MaxRenderers) + : Renderers(MaxRenderers, nullptr), MIs() {} + InstructionSelector::InstructionSelector() = default; bool InstructionSelector::constrainOperandRegToRegClass( diff --git a/contrib/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/contrib/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 84b0a0ac4157..49fb5e8f075b 100644 --- a/contrib/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/contrib/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -99,23 +99,19 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { llvm_unreachable("Unknown libcall function"); } -LegalizerHelper::LegalizeResult llvm::replaceWithLibcall( - MachineInstr &MI, MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, - const CallLowering::ArgInfo &Result, ArrayRef Args) { +LegalizerHelper::LegalizeResult +llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, + const CallLowering::ArgInfo &Result, + ArrayRef Args) { auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); const char *Name = TLI.getLibcallName(Libcall); + MIRBuilder.getMF().getFrameInfo().setHasCalls(true); - MIRBuilder.setInstr(MI); if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall), MachineOperand::CreateES(Name), Result, Args)) return LegalizerHelper::UnableToLegalize; - // We're about to remove MI, so move the insert point after it. - MIRBuilder.setInsertPt(MIRBuilder.getMBB(), - std::next(MIRBuilder.getInsertPt())); - - MI.eraseFromParent(); return LegalizerHelper::Legalized; } @@ -123,10 +119,9 @@ static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType) { auto Libcall = getRTLibDesc(MI.getOpcode(), Size); - return replaceWithLibcall(MI, MIRBuilder, Libcall, - {MI.getOperand(0).getReg(), OpType}, - {{MI.getOperand(1).getReg(), OpType}, - {MI.getOperand(2).getReg(), OpType}}); + return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType}, + {{MI.getOperand(1).getReg(), OpType}, + {MI.getOperand(2).getReg(), OpType}}); } LegalizerHelper::LegalizeResult @@ -135,6 +130,8 @@ LegalizerHelper::libcall(MachineInstr &MI) { unsigned Size = LLTy.getSizeInBits(); auto &Ctx = MIRBuilder.getMF().getFunction()->getContext(); + MIRBuilder.setInstr(MI); + switch (MI.getOpcode()) { default: return UnableToLegalize; @@ -143,15 +140,24 @@ LegalizerHelper::libcall(MachineInstr &MI) { case TargetOpcode::G_SREM: case TargetOpcode::G_UREM: { Type *HLTy = Type::getInt32Ty(Ctx); - return simpleLibcall(MI, MIRBuilder, Size, HLTy); + auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); + if (Status != Legalized) + return Status; + break; } case TargetOpcode::G_FADD: case TargetOpcode::G_FPOW: case TargetOpcode::G_FREM: { Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx); - return simpleLibcall(MI, MIRBuilder, Size, HLTy); + auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); + if (Status != Legalized) + return Status; + break; } } + + MI.eraseFromParent(); + return Legalized; } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, diff --git a/contrib/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/contrib/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 47c6214c0552..4636806c3f08 100644 --- a/contrib/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/contrib/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -166,19 +166,24 @@ MachineInstrBuilder MachineIRBuilder::buildGlobalValue(unsigned Res, .addGlobalAddress(GV); } -MachineInstrBuilder MachineIRBuilder::buildAdd(unsigned Res, unsigned Op0, +MachineInstrBuilder MachineIRBuilder::buildBinaryOp(unsigned Opcode, unsigned Res, unsigned Op0, unsigned Op1) { assert((MRI->getType(Res).isScalar() || MRI->getType(Res).isVector()) && "invalid operand type"); assert(MRI->getType(Res) == MRI->getType(Op0) && MRI->getType(Res) == MRI->getType(Op1) && "type mismatch"); - return buildInstr(TargetOpcode::G_ADD) + return buildInstr(Opcode) .addDef(Res) .addUse(Op0) .addUse(Op1); } +MachineInstrBuilder MachineIRBuilder::buildAdd(unsigned Res, unsigned Op0, + unsigned Op1) { + return buildBinaryOp(TargetOpcode::G_ADD, Res, Op0, Op1); +} + MachineInstrBuilder MachineIRBuilder::buildGEP(unsigned Res, unsigned Op0, unsigned Op1) { assert(MRI->getType(Res).isPointer() && @@ -222,41 +227,22 @@ MachineInstrBuilder MachineIRBuilder::buildPtrMask(unsigned Res, unsigned Op0, MachineInstrBuilder MachineIRBuilder::buildSub(unsigned Res, unsigned Op0, unsigned Op1) { - assert((MRI->getType(Res).isScalar() || MRI->getType(Res).isVector()) && - "invalid operand type"); - assert(MRI->getType(Res) == MRI->getType(Op0) && - MRI->getType(Res) == MRI->getType(Op1) && "type mismatch"); - - return buildInstr(TargetOpcode::G_SUB) - .addDef(Res) - .addUse(Op0) - .addUse(Op1); + return buildBinaryOp(TargetOpcode::G_SUB, Res, Op0, Op1); } MachineInstrBuilder MachineIRBuilder::buildMul(unsigned Res, unsigned Op0, unsigned Op1) { - assert((MRI->getType(Res).isScalar() || MRI->getType(Res).isVector()) && - "invalid operand type"); - assert(MRI->getType(Res) == MRI->getType(Op0) && - MRI->getType(Res) == MRI->getType(Op1) && "type mismatch"); - - return buildInstr(TargetOpcode::G_MUL) - .addDef(Res) - .addUse(Op0) - .addUse(Op1); + return buildBinaryOp(TargetOpcode::G_MUL, Res, Op0, Op1); } MachineInstrBuilder MachineIRBuilder::buildAnd(unsigned Res, unsigned Op0, unsigned Op1) { - assert((MRI->getType(Res).isScalar() || MRI->getType(Res).isVector()) && - "invalid operand type"); - assert(MRI->getType(Res) == MRI->getType(Op0) && - MRI->getType(Res) == MRI->getType(Op1) && "type mismatch"); + return buildBinaryOp(TargetOpcode::G_AND, Res, Op0, Op1); +} - return buildInstr(TargetOpcode::G_AND) - .addDef(Res) - .addUse(Op0) - .addUse(Op1); +MachineInstrBuilder MachineIRBuilder::buildOr(unsigned Res, unsigned Op0, + unsigned Op1) { + return buildBinaryOp(TargetOpcode::G_OR, Res, Op0, Op1); } MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) { diff --git a/contrib/llvm/lib/CodeGen/LiveRegUnits.cpp b/contrib/llvm/lib/CodeGen/LiveRegUnits.cpp index 3746b74e0528..f9ba4ffa6527 100644 --- a/contrib/llvm/lib/CodeGen/LiveRegUnits.cpp +++ b/contrib/llvm/lib/CodeGen/LiveRegUnits.cpp @@ -67,7 +67,7 @@ void LiveRegUnits::stepBackward(const MachineInstr &MI) { } } -void LiveRegUnits::accumulateBackward(const MachineInstr &MI) { +void LiveRegUnits::accumulate(const MachineInstr &MI) { // Add defs, uses and regmask clobbers to the set. for (ConstMIBundleOperands O(MI); O.isValid(); ++O) { if (O->isReg()) { diff --git a/contrib/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/contrib/llvm/lib/CodeGen/MIRParser/MILexer.cpp index 1f1ce6e8d725..58a655a4dee4 100644 --- a/contrib/llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ b/contrib/llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -365,6 +365,14 @@ static Cursor maybeLexIRValue(Cursor C, MIToken &Token, return lexName(C, Token, MIToken::NamedIRValue, Rule.size(), ErrorCallback); } +static Cursor maybeLexStringConstant(Cursor C, MIToken &Token, + ErrorCallbackType ErrorCallback) { + if (C.peek() != '"') + return None; + return lexName(C, Token, MIToken::StringConstant, /*PrefixLength=*/0, + ErrorCallback); +} + static Cursor lexVirtualRegister(Cursor C, MIToken &Token) { auto Range = C; C.advance(); // Skip '%' @@ -630,6 +638,8 @@ StringRef llvm::lexMIToken(StringRef Source, MIToken &Token, return R.remaining(); if (Cursor R = maybeLexEscapedIRValue(C, Token, ErrorCallback)) return R.remaining(); + if (Cursor R = maybeLexStringConstant(C, Token, ErrorCallback)) + return R.remaining(); Token.reset(MIToken::Error, C.remaining()); ErrorCallback(C.location(), diff --git a/contrib/llvm/lib/CodeGen/MIRParser/MILexer.h b/contrib/llvm/lib/CodeGen/MIRParser/MILexer.h index 3e9513111bf4..08b82e59c4fc 100644 --- a/contrib/llvm/lib/CodeGen/MIRParser/MILexer.h +++ b/contrib/llvm/lib/CodeGen/MIRParser/MILexer.h @@ -127,7 +127,8 @@ struct MIToken { NamedIRValue, IRValue, QuotedIRValue, // `` - SubRegisterIndex + SubRegisterIndex, + StringConstant }; private: @@ -168,7 +169,8 @@ struct MIToken { bool isMemoryOperandFlag() const { return Kind == kw_volatile || Kind == kw_non_temporal || - Kind == kw_dereferenceable || Kind == kw_invariant; + Kind == kw_dereferenceable || Kind == kw_invariant || + Kind == StringConstant; } bool is(TokenKind K) const { return Kind == K; } diff --git a/contrib/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/contrib/llvm/lib/CodeGen/MIRParser/MIParser.cpp index c58d192284dd..c68d87b15a31 100644 --- a/contrib/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/contrib/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -141,6 +141,8 @@ class MIParser { StringMap Names2DirectTargetFlags; /// Maps from direct target flag names to the bitmask target flag values. StringMap Names2BitmaskTargetFlags; + /// Maps from MMO target flag names to MMO target flag values. + StringMap Names2MMOTargetFlags; public: MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error, @@ -229,6 +231,7 @@ class MIParser { bool parseMemoryOperandFlag(MachineMemOperand::Flags &Flags); bool parseMemoryPseudoSourceValue(const PseudoSourceValue *&PSV); bool parseMachinePointerInfo(MachinePointerInfo &Dest); + bool parseOptionalScope(LLVMContext &Context, SyncScope::ID &SSID); bool parseOptionalAtomicOrdering(AtomicOrdering &Order); bool parseMachineMemoryOperand(MachineMemOperand *&Dest); @@ -318,6 +321,18 @@ class MIParser { /// /// Return true if the name isn't a name of a bitmask target flag. bool getBitmaskTargetFlag(StringRef Name, unsigned &Flag); + + void initNames2MMOTargetFlags(); + + /// Try to convert a name of a MachineMemOperand target flag to the + /// corresponding target flag. + /// + /// Return true if the name isn't a name of a target MMO flag. + bool getMMOTargetFlag(StringRef Name, MachineMemOperand::Flags &Flag); + + /// parseStringConstant + /// ::= StringConstant + bool parseStringConstant(std::string &Result); }; } // end anonymous namespace @@ -2034,7 +2049,14 @@ bool MIParser::parseMemoryOperandFlag(MachineMemOperand::Flags &Flags) { case MIToken::kw_invariant: Flags |= MachineMemOperand::MOInvariant; break; - // TODO: parse the target specific memory operand flags. + case MIToken::StringConstant: { + MachineMemOperand::Flags TF; + if (getMMOTargetFlag(Token.stringValue(), TF)) + return error("use of undefined target MMO flag '" + Token.stringValue() + + "'"); + Flags |= TF; + break; + } default: llvm_unreachable("The current token should be a memory operand flag"); } @@ -2135,6 +2157,26 @@ bool MIParser::parseMachinePointerInfo(MachinePointerInfo &Dest) { return false; } +bool MIParser::parseOptionalScope(LLVMContext &Context, + SyncScope::ID &SSID) { + SSID = SyncScope::System; + if (Token.is(MIToken::Identifier) && Token.stringValue() == "syncscope") { + lex(); + if (expectAndConsume(MIToken::lparen)) + return error("expected '(' in syncscope"); + + std::string SSN; + if (parseStringConstant(SSN)) + return true; + + SSID = Context.getOrInsertSyncScopeID(SSN); + if (expectAndConsume(MIToken::rparen)) + return error("expected ')' in syncscope"); + } + + return false; +} + bool MIParser::parseOptionalAtomicOrdering(AtomicOrdering &Order) { Order = AtomicOrdering::NotAtomic; if (Token.isNot(MIToken::Identifier)) @@ -2174,12 +2216,10 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) { Flags |= MachineMemOperand::MOStore; lex(); - // Optional "singlethread" scope. - SynchronizationScope Scope = SynchronizationScope::CrossThread; - if (Token.is(MIToken::Identifier) && Token.stringValue() == "singlethread") { - Scope = SynchronizationScope::SingleThread; - lex(); - } + // Optional synchronization scope. + SyncScope::ID SSID; + if (parseOptionalScope(MF.getFunction()->getContext(), SSID)) + return true; // Up to two atomic orderings (cmpxchg provides guarantees on failure). AtomicOrdering Order, FailureOrder; @@ -2244,7 +2284,7 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) { if (expectAndConsume(MIToken::rparen)) return true; Dest = MF.getMachineMemOperand(Ptr, Flags, Size, BaseAlignment, AAInfo, Range, - Scope, Order, FailureOrder); + SSID, Order, FailureOrder); return false; } @@ -2457,6 +2497,35 @@ bool MIParser::getBitmaskTargetFlag(StringRef Name, unsigned &Flag) { return false; } +void MIParser::initNames2MMOTargetFlags() { + if (!Names2MMOTargetFlags.empty()) + return; + const auto *TII = MF.getSubtarget().getInstrInfo(); + assert(TII && "Expected target instruction info"); + auto Flags = TII->getSerializableMachineMemOperandTargetFlags(); + for (const auto &I : Flags) + Names2MMOTargetFlags.insert( + std::make_pair(StringRef(I.second), I.first)); +} + +bool MIParser::getMMOTargetFlag(StringRef Name, + MachineMemOperand::Flags &Flag) { + initNames2MMOTargetFlags(); + auto FlagInfo = Names2MMOTargetFlags.find(Name); + if (FlagInfo == Names2MMOTargetFlags.end()) + return true; + Flag = FlagInfo->second; + return false; +} + +bool MIParser::parseStringConstant(std::string &Result) { + if (Token.isNot(MIToken::StringConstant)) + return error("expected string constant"); + Result = Token.stringValue(); + lex(); + return false; +} + bool llvm::parseMachineBasicBlockDefinitions(PerFunctionMIParsingState &PFS, StringRef Src, SMDiagnostic &Error) { diff --git a/contrib/llvm/lib/CodeGen/MIRPrinter.cpp b/contrib/llvm/lib/CodeGen/MIRPrinter.cpp index c524a9835f33..ddeacf1d1bfb 100644 --- a/contrib/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/contrib/llvm/lib/CodeGen/MIRPrinter.cpp @@ -18,6 +18,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/CodeGen/GlobalISel/RegisterBank.h" @@ -139,6 +140,8 @@ class MIPrinter { ModuleSlotTracker &MST; const DenseMap &RegisterMaskIds; const DenseMap &StackObjectOperandMapping; + /// Synchronization scope names registered with LLVMContext. + SmallVector SSNs; bool canPredictBranchProbabilities(const MachineBasicBlock &MBB) const; bool canPredictSuccessors(const MachineBasicBlock &MBB) const; @@ -162,7 +165,9 @@ class MIPrinter { void print(const MachineOperand &Op, const TargetRegisterInfo *TRI, unsigned I, bool ShouldPrintRegisterTies, LLT TypeToPrint, bool IsDef = false); - void print(const MachineMemOperand &Op); + void print(const LLVMContext &Context, const TargetInstrInfo &TII, + const MachineMemOperand &Op); + void printSyncScope(const LLVMContext &Context, SyncScope::ID SSID); void print(const MCCFIInstruction &CFI, const TargetRegisterInfo *TRI); }; @@ -731,11 +736,12 @@ void MIPrinter::print(const MachineInstr &MI) { if (!MI.memoperands_empty()) { OS << " :: "; + const LLVMContext &Context = MF->getFunction()->getContext(); bool NeedComma = false; for (const auto *Op : MI.memoperands()) { if (NeedComma) OS << ", "; - print(*Op); + print(Context, *TII, *Op); NeedComma = true; } } @@ -1031,9 +1037,20 @@ void MIPrinter::print(const MachineOperand &Op, const TargetRegisterInfo *TRI, } } -void MIPrinter::print(const MachineMemOperand &Op) { +static const char *getTargetMMOFlagName(const TargetInstrInfo &TII, + unsigned TMMOFlag) { + auto Flags = TII.getSerializableMachineMemOperandTargetFlags(); + for (const auto &I : Flags) { + if (I.first == TMMOFlag) { + return I.second; + } + } + return nullptr; +} + +void MIPrinter::print(const LLVMContext &Context, const TargetInstrInfo &TII, + const MachineMemOperand &Op) { OS << '('; - // TODO: Print operand's target specific flags. if (Op.isVolatile()) OS << "volatile "; if (Op.isNonTemporal()) @@ -1042,6 +1059,15 @@ void MIPrinter::print(const MachineMemOperand &Op) { OS << "dereferenceable "; if (Op.isInvariant()) OS << "invariant "; + if (Op.getFlags() & MachineMemOperand::MOTargetFlag1) + OS << '"' << getTargetMMOFlagName(TII, MachineMemOperand::MOTargetFlag1) + << "\" "; + if (Op.getFlags() & MachineMemOperand::MOTargetFlag2) + OS << '"' << getTargetMMOFlagName(TII, MachineMemOperand::MOTargetFlag2) + << "\" "; + if (Op.getFlags() & MachineMemOperand::MOTargetFlag3) + OS << '"' << getTargetMMOFlagName(TII, MachineMemOperand::MOTargetFlag3) + << "\" "; if (Op.isLoad()) OS << "load "; else { @@ -1049,8 +1075,7 @@ void MIPrinter::print(const MachineMemOperand &Op) { OS << "store "; } - if (Op.getSynchScope() == SynchronizationScope::SingleThread) - OS << "singlethread "; + printSyncScope(Context, Op.getSyncScopeID()); if (Op.getOrdering() != AtomicOrdering::NotAtomic) OS << toIRString(Op.getOrdering()) << ' '; @@ -1119,6 +1144,23 @@ void MIPrinter::print(const MachineMemOperand &Op) { OS << ')'; } +void MIPrinter::printSyncScope(const LLVMContext &Context, SyncScope::ID SSID) { + switch (SSID) { + case SyncScope::System: { + break; + } + default: { + if (SSNs.empty()) + Context.getSyncScopeNames(SSNs); + + OS << "syncscope(\""; + PrintEscapedString(SSNs[SSID], OS); + OS << "\") "; + break; + } + } +} + static void printCFIRegister(unsigned DwarfReg, raw_ostream &OS, const TargetRegisterInfo *TRI) { int Reg = TRI->getLLVMRegNum(DwarfReg, true); diff --git a/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp index 2d4b95974cc6..447ad629885b 100644 --- a/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp +++ b/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp @@ -1917,6 +1917,12 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, return; MachineBasicBlock *Top = *LoopChain.begin(); + MachineBasicBlock *Bottom = *std::prev(LoopChain.end()); + + // If ExitingBB is already the last one in a chain then nothing to do. + if (Bottom == ExitingBB) + return; + bool ViableTopFallthrough = false; for (MachineBasicBlock *Pred : Top->predecessors()) { BlockChain *PredChain = BlockToChain[Pred]; @@ -1931,7 +1937,6 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, // bottom is a viable exiting block. If so, bail out as rotating will // introduce an unnecessary branch. if (ViableTopFallthrough) { - MachineBasicBlock *Bottom = *std::prev(LoopChain.end()); for (MachineBasicBlock *Succ : Bottom->successors()) { BlockChain *SuccChain = BlockToChain[Succ]; if (!LoopBlockSet.count(Succ) && @@ -1944,6 +1949,36 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, if (ExitIt == LoopChain.end()) return; + // Rotating a loop exit to the bottom when there is a fallthrough to top + // trades the entry fallthrough for an exit fallthrough. + // If there is no bottom->top edge, but the chosen exit block does have + // a fallthrough, we break that fallthrough for nothing in return. + + // Let's consider an example. We have a built chain of basic blocks + // B1, B2, ..., Bn, where Bk is a ExitingBB - chosen exit block. + // By doing a rotation we get + // Bk+1, ..., Bn, B1, ..., Bk + // Break of fallthrough to B1 is compensated by a fallthrough from Bk. + // If we had a fallthrough Bk -> Bk+1 it is broken now. + // It might be compensated by fallthrough Bn -> B1. + // So we have a condition to avoid creation of extra branch by loop rotation. + // All below must be true to avoid loop rotation: + // If there is a fallthrough to top (B1) + // There was fallthrough from chosen exit block (Bk) to next one (Bk+1) + // There is no fallthrough from bottom (Bn) to top (B1). + // Please note that there is no exit fallthrough from Bn because we checked it + // above. + if (ViableTopFallthrough) { + assert(std::next(ExitIt) != LoopChain.end() && + "Exit should not be last BB"); + MachineBasicBlock *NextBlockInChain = *std::next(ExitIt); + if (ExitingBB->isSuccessor(NextBlockInChain)) + if (!Bottom->isSuccessor(Top)) + return; + } + + DEBUG(dbgs() << "Rotating loop to put exit " << getBlockName(ExitingBB) + << " at bottom\n"); std::rotate(LoopChain.begin(), std::next(ExitIt), LoopChain.end()); } diff --git a/contrib/llvm/lib/CodeGen/MachineFunction.cpp b/contrib/llvm/lib/CodeGen/MachineFunction.cpp index bbdae6e1a49e..f88e175a9776 100644 --- a/contrib/llvm/lib/CodeGen/MachineFunction.cpp +++ b/contrib/llvm/lib/CodeGen/MachineFunction.cpp @@ -305,11 +305,11 @@ MachineFunction::DeleteMachineBasicBlock(MachineBasicBlock *MBB) { MachineMemOperand *MachineFunction::getMachineMemOperand( MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges, - SynchronizationScope SynchScope, AtomicOrdering Ordering, + SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) { return new (Allocator) MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges, - SynchScope, Ordering, FailureOrdering); + SSID, Ordering, FailureOrdering); } MachineMemOperand * @@ -320,13 +320,13 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, MachineMemOperand(MachinePointerInfo(MMO->getValue(), MMO->getOffset()+Offset), MMO->getFlags(), Size, MMO->getBaseAlignment(), - AAMDNodes(), nullptr, MMO->getSynchScope(), + AAMDNodes(), nullptr, MMO->getSyncScopeID(), MMO->getOrdering(), MMO->getFailureOrdering()); return new (Allocator) MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(), MMO->getOffset()+Offset), MMO->getFlags(), Size, MMO->getBaseAlignment(), - AAMDNodes(), nullptr, MMO->getSynchScope(), + AAMDNodes(), nullptr, MMO->getSyncScopeID(), MMO->getOrdering(), MMO->getFailureOrdering()); } @@ -359,7 +359,7 @@ MachineFunction::extractLoadMemRefs(MachineInstr::mmo_iterator Begin, (*I)->getFlags() & ~MachineMemOperand::MOStore, (*I)->getSize(), (*I)->getBaseAlignment(), (*I)->getAAInfo(), nullptr, - (*I)->getSynchScope(), (*I)->getOrdering(), + (*I)->getSyncScopeID(), (*I)->getOrdering(), (*I)->getFailureOrdering()); Result[Index] = JustLoad; } @@ -393,7 +393,7 @@ MachineFunction::extractStoreMemRefs(MachineInstr::mmo_iterator Begin, (*I)->getFlags() & ~MachineMemOperand::MOLoad, (*I)->getSize(), (*I)->getBaseAlignment(), (*I)->getAAInfo(), nullptr, - (*I)->getSynchScope(), (*I)->getOrdering(), + (*I)->getSyncScopeID(), (*I)->getOrdering(), (*I)->getFailureOrdering()); Result[Index] = JustStore; } diff --git a/contrib/llvm/lib/CodeGen/MachineInstr.cpp b/contrib/llvm/lib/CodeGen/MachineInstr.cpp index 81c6dace92e0..afea5575a3ae 100644 --- a/contrib/llvm/lib/CodeGen/MachineInstr.cpp +++ b/contrib/llvm/lib/CodeGen/MachineInstr.cpp @@ -447,6 +447,14 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, SmallString<16> Str; getFPImm()->getValueAPF().toString(Str); OS << "quad " << Str; + } else if (getFPImm()->getType()->isX86_FP80Ty()) { + APFloat APF = getFPImm()->getValueAPF(); + OS << "x86_fp80 0xK"; + APInt API = APF.bitcastToAPInt(); + OS << format_hex_no_prefix(API.getHiBits(16).getZExtValue(), 4, + /*Upper=*/true); + OS << format_hex_no_prefix(API.getLoBits(64).getZExtValue(), 16, + /*Upper=*/true); } else { OS << getFPImm()->getValueAPF().convertToDouble(); } @@ -606,7 +614,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, uint64_t s, unsigned int a, const AAMDNodes &AAInfo, const MDNode *Ranges, - SynchronizationScope SynchScope, + SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlignLog2(Log2_32(a) + 1), @@ -617,8 +625,8 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, assert(getBaseAlignment() == a && "Alignment is not a power of 2!"); assert((isLoad() || isStore()) && "Not a load/store!"); - AtomicInfo.SynchScope = static_cast(SynchScope); - assert(getSynchScope() == SynchScope && "Value truncated"); + AtomicInfo.SSID = static_cast(SSID); + assert(getSyncScopeID() == SSID && "Value truncated"); AtomicInfo.Ordering = static_cast(Ordering); assert(getOrdering() == Ordering && "Value truncated"); AtomicInfo.FailureOrdering = static_cast(FailureOrdering); @@ -744,6 +752,12 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST) const { OS << "(dereferenceable)"; if (isInvariant()) OS << "(invariant)"; + if (getFlags() & MOTargetFlag1) + OS << "(flag1)"; + if (getFlags() & MOTargetFlag2) + OS << "(flag2)"; + if (getFlags() & MOTargetFlag3) + OS << "(flag3)"; } //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/CodeGen/MachineVerifier.cpp b/contrib/llvm/lib/CodeGen/MachineVerifier.cpp index e65c256c1bb5..fcb544806dda 100644 --- a/contrib/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/contrib/llvm/lib/CodeGen/MachineVerifier.cpp @@ -985,6 +985,14 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { report("Operand should be tied", MO, MONum); else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum)) report("Tied def doesn't match MCInstrDesc", MO, MONum); + else if (TargetRegisterInfo::isPhysicalRegister(MO->getReg())) { + const MachineOperand &MOTied = MI->getOperand(TiedTo); + if (!MOTied.isReg()) + report("Tied counterpart must be a register", &MOTied, TiedTo); + else if (TargetRegisterInfo::isPhysicalRegister(MOTied.getReg()) && + MO->getReg() != MOTied.getReg()) + report("Tied physical registers must match.", &MOTied, TiedTo); + } } else if (MO->isReg() && MO->isTied()) report("Explicit operand should not be tied", MO, MONum); } else { diff --git a/contrib/llvm/lib/CodeGen/MacroFusion.cpp b/contrib/llvm/lib/CodeGen/MacroFusion.cpp index 5e279b065bbd..633a853b2c74 100644 --- a/contrib/llvm/lib/CodeGen/MacroFusion.cpp +++ b/contrib/llvm/lib/CodeGen/MacroFusion.cpp @@ -24,7 +24,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" STATISTIC(NumFused, "Number of instr pairs fused"); diff --git a/contrib/llvm/lib/CodeGen/PostRAHazardRecognizer.cpp b/contrib/llvm/lib/CodeGen/PostRAHazardRecognizer.cpp index 425a59dc0375..4a50d895340a 100644 --- a/contrib/llvm/lib/CodeGen/PostRAHazardRecognizer.cpp +++ b/contrib/llvm/lib/CodeGen/PostRAHazardRecognizer.cpp @@ -23,7 +23,7 @@ /// This pass traverses all the instructions in a program in top-down order. /// In contrast to the instruction scheduling passes, this pass never resets /// the hazard recognizer to ensure it can correctly handles noop hazards at -/// the begining of blocks. +/// the beginning of blocks. // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/CodeGen/RegAllocFast.cpp b/contrib/llvm/lib/CodeGen/RegAllocFast.cpp index c606b7b83310..d5538be4bba2 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocFast.cpp @@ -203,6 +203,8 @@ namespace { char RAFast::ID = 0; } +INITIALIZE_PASS(RAFast, "regallocfast", "Fast Register Allocator", false, false) + /// getStackSpaceFor - This allocates space for the specified virtual register /// to be held on the stack. int RAFast::getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC) { @@ -244,8 +246,15 @@ void RAFast::addKillFlag(const LiveReg &LR) { if (MO.isUse() && !LR.LastUse->isRegTiedToDefOperand(LR.LastOpNum)) { if (MO.getReg() == LR.PhysReg) MO.setIsKill(); - else - LR.LastUse->addRegisterKilled(LR.PhysReg, TRI, true); + // else, don't do anything we are problably redefining a + // subreg of this register and given we don't track which + // lanes are actually dead, we cannot insert a kill flag here. + // Otherwise we may end up in a situation like this: + // ... = (MO) physreg:sub1, physreg + // ... <== Here we would allow later pass to reuse physreg:sub1 + // which is potentially wrong. + // LR:sub0 = ... + // ... = LR.sub1 <== This is going to use physreg:sub1 } } diff --git a/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp b/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp index 9562652556ac..020e81eca2dd 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -2458,7 +2458,7 @@ void RAGreedy::tryHintRecoloring(LiveInterval &VirtReg) { do { Reg = RecoloringCandidates.pop_back_val(); - // We cannot recolor physcal register. + // We cannot recolor physical register. if (TargetRegisterInfo::isPhysicalRegister(Reg)) continue; diff --git a/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp b/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp index e3baff4be4bc..9778103575fa 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp @@ -924,5 +924,3 @@ FunctionPass *llvm::createPBQPRegisterAllocator(char *customPassID) { FunctionPass* llvm::createDefaultPBQPRegisterAllocator() { return createPBQPRegisterAllocator(); } - -#undef DEBUG_TYPE diff --git a/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp b/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp index ff9bca092dbe..a67d07b36474 100644 --- a/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -1227,6 +1227,34 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP, SR->createDeadDef(DefIndex, Alloc); } } + + // Make sure that the subrange for resultant undef is removed + // For example: + // vreg1:sub1 = LOAD CONSTANT 1 + // vreg2 = COPY vreg1 + // ==> + // vreg2:sub1 = LOAD CONSTANT 1 + // ; Correct but need to remove the subrange for vreg2:sub0 + // ; as it is now undef + if (NewIdx != 0 && DstInt.hasSubRanges()) { + // The affected subregister segments can be removed. + SlotIndex CurrIdx = LIS->getInstructionIndex(NewMI); + LaneBitmask DstMask = TRI->getSubRegIndexLaneMask(NewIdx); + bool UpdatedSubRanges = false; + for (LiveInterval::SubRange &SR : DstInt.subranges()) { + if ((SR.LaneMask & DstMask).none()) { + DEBUG(dbgs() << "Removing undefined SubRange " + << PrintLaneMask(SR.LaneMask) << " : " << SR << "\n"); + // VNI is in ValNo - remove any segments in this SubRange that have this ValNo + if (VNInfo *RmValNo = SR.getVNInfoAt(CurrIdx.getRegSlot())) { + SR.removeValNo(RmValNo); + UpdatedSubRanges = true; + } + } + } + if (UpdatedSubRanges) + DstInt.removeEmptySubRanges(); + } } else if (NewMI.getOperand(0).getReg() != CopyDstReg) { // The New instruction may be defining a sub-register of what's actually // been asked for. If so it must implicitly define the whole thing. diff --git a/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp b/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp index 05e641d9489d..fc5105aadbff 100644 --- a/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp +++ b/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp @@ -375,7 +375,8 @@ unsigned RegScavenger::findSurvivorReg(MachineBasicBlock::iterator StartMI, static std::pair findSurvivorBackwards(const MachineRegisterInfo &MRI, MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, - const LiveRegUnits &LiveOut, ArrayRef AllocationOrder) { + const LiveRegUnits &LiveOut, ArrayRef AllocationOrder, + bool RestoreAfter) { bool FoundTo = false; MCPhysReg Survivor = 0; MachineBasicBlock::iterator Pos; @@ -388,7 +389,7 @@ findSurvivorBackwards(const MachineRegisterInfo &MRI, for (MachineBasicBlock::iterator I = From;; --I) { const MachineInstr &MI = *I; - Used.accumulateBackward(MI); + Used.accumulate(MI); if (I == To) { // See if one of the registers in RC wasn't used so far. @@ -401,6 +402,11 @@ findSurvivorBackwards(const MachineRegisterInfo &MRI, // the register which is not defined/used for the longest time. FoundTo = true; Pos = To; + // Note: It was fine so far to start our search at From, however now that + // we have to spill, and can only place the restore after From then + // add the regs used/defed by std::next(From) to the set. + if (RestoreAfter) + Used.accumulate(*std::next(From)); } if (FoundTo) { if (Survivor == 0 || !Used.available(Survivor)) { @@ -575,7 +581,8 @@ unsigned RegScavenger::scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator UseMI; ArrayRef AllocationOrder = RC.getRawAllocationOrder(MF); std::pair P = - findSurvivorBackwards(*MRI, MBBI, To, LiveUnits, AllocationOrder); + findSurvivorBackwards(*MRI, MBBI, To, LiveUnits, AllocationOrder, + RestoreAfter); MCPhysReg Reg = P.first; MachineBasicBlock::iterator SpillBefore = P.second; assert(Reg != 0 && "No register left to scavenge!"); @@ -626,7 +633,7 @@ static unsigned scavengeVReg(MachineRegisterInfo &MRI, RegScavenger &RS, assert(RealDef != nullptr && "Must have at least 1 Def"); #endif - // We should only have one definition of the register. However to accomodate + // We should only have one definition of the register. However to accommodate // the requirements of two address code we also allow definitions in // subsequent instructions provided they also read the register. That way // we get a single contiguous lifetime. diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp index 3cd270cec3a6..5e95f760aaa2 100644 --- a/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp +++ b/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp @@ -67,6 +67,41 @@ const MCInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const { return &TII->get(Node->getMachineOpcode()); } +LLVM_DUMP_METHOD +raw_ostream &SDep::print(raw_ostream &OS, const TargetRegisterInfo *TRI) const { + switch (getKind()) { + case Data: OS << "Data"; break; + case Anti: OS << "Anti"; break; + case Output: OS << "Out "; break; + case Order: OS << "Ord "; break; + } + + switch (getKind()) { + case Data: + OS << " Latency=" << getLatency(); + if (TRI && isAssignedRegDep()) + OS << " Reg=" << PrintReg(getReg(), TRI); + break; + case Anti: + case Output: + OS << " Latency=" << getLatency(); + break; + case Order: + OS << " Latency=" << getLatency(); + switch(Contents.OrdKind) { + case Barrier: OS << " Barrier"; break; + case MayAliasMem: + case MustAliasMem: OS << " Memory"; break; + case Artificial: OS << " Artificial"; break; + case Weak: OS << " Weak"; break; + case Cluster: OS << " Cluster"; break; + } + break; + } + + return OS; +} + bool SUnit::addPred(const SDep &D, bool Required) { // If this node already has this dependence, don't add a redundant one. for (SDep &PredDep : Preds) { @@ -302,16 +337,24 @@ void SUnit::biasCriticalPath() { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD -void SUnit::print(raw_ostream &OS, const ScheduleDAG *DAG) const { - if (this == &DAG->ExitSU) - OS << "ExitSU"; - else if (this == &DAG->EntrySU) +raw_ostream &SUnit::print(raw_ostream &OS, + const SUnit *Entry, const SUnit *Exit) const { + if (this == Entry) OS << "EntrySU"; + else if (this == Exit) + OS << "ExitSU"; else OS << "SU(" << NodeNum << ")"; + return OS; } -LLVM_DUMP_METHOD void SUnit::dump(const ScheduleDAG *G) const { +LLVM_DUMP_METHOD +raw_ostream &SUnit::print(raw_ostream &OS, const ScheduleDAG *G) const { + return print(OS, &G->EntrySU, &G->ExitSU); +} + +LLVM_DUMP_METHOD +void SUnit::dump(const ScheduleDAG *G) const { print(dbgs(), G); dbgs() << ": "; G->dumpNode(this); @@ -333,40 +376,18 @@ LLVM_DUMP_METHOD void SUnit::dumpAll(const ScheduleDAG *G) const { if (Preds.size() != 0) { dbgs() << " Predecessors:\n"; - for (const SDep &SuccDep : Preds) { - dbgs() << " "; - switch (SuccDep.getKind()) { - case SDep::Data: dbgs() << "data "; break; - case SDep::Anti: dbgs() << "anti "; break; - case SDep::Output: dbgs() << "out "; break; - case SDep::Order: dbgs() << "ord "; break; - } - SuccDep.getSUnit()->print(dbgs(), G); - if (SuccDep.isArtificial()) - dbgs() << " *"; - dbgs() << ": Latency=" << SuccDep.getLatency(); - if (SuccDep.isAssignedRegDep()) - dbgs() << " Reg=" << PrintReg(SuccDep.getReg(), G->TRI); - dbgs() << "\n"; + for (const SDep &Dep : Preds) { + dbgs() << " "; + Dep.getSUnit()->print(dbgs(), G); dbgs() << ": "; + Dep.print(dbgs(), G->TRI); dbgs() << '\n'; } } if (Succs.size() != 0) { dbgs() << " Successors:\n"; - for (const SDep &SuccDep : Succs) { - dbgs() << " "; - switch (SuccDep.getKind()) { - case SDep::Data: dbgs() << "data "; break; - case SDep::Anti: dbgs() << "anti "; break; - case SDep::Output: dbgs() << "out "; break; - case SDep::Order: dbgs() << "ord "; break; - } - SuccDep.getSUnit()->print(dbgs(), G); - if (SuccDep.isArtificial()) - dbgs() << " *"; - dbgs() << ": Latency=" << SuccDep.getLatency(); - if (SuccDep.isAssignedRegDep()) - dbgs() << " Reg=" << PrintReg(SuccDep.getReg(), G->TRI); - dbgs() << "\n"; + for (const SDep &Dep : Succs) { + dbgs() << " "; + Dep.getSUnit()->print(dbgs(), G); dbgs() << ": "; + Dep.print(dbgs(), G->TRI); dbgs() << '\n'; } } } diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp index 0f70b0e9ca07..ccd937950a74 100644 --- a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -63,7 +63,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" static cl::opt EnableAASchedMI("enable-aa-sched-mi", cl::Hidden, cl::ZeroOrMore, cl::init(false), diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index d901af727686..71382c18fdf9 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -400,6 +400,7 @@ namespace { SDValue reduceBuildVecExtToExtBuildVec(SDNode *N); SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N); SDValue reduceBuildVecToShuffle(SDNode *N); + SDValue reduceBuildVecToTrunc(SDNode *N); SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N, ArrayRef VectorMask, SDValue VecIn1, SDValue VecIn2, unsigned LeftIdx); @@ -5267,14 +5268,40 @@ SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) { } SDValue DAGCombiner::visitRotate(SDNode *N) { + SDLoc dl(N); + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + EVT VT = N->getValueType(0); + + // fold (rot x, 0) -> x + if (isNullConstantOrNullSplatConstant(N1)) + return N0; + // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))). - if (N->getOperand(1).getOpcode() == ISD::TRUNCATE && - N->getOperand(1).getOperand(0).getOpcode() == ISD::AND) { - if (SDValue NewOp1 = - distributeTruncateThroughAnd(N->getOperand(1).getNode())) - return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), - N->getOperand(0), NewOp1); + if (N1.getOpcode() == ISD::TRUNCATE && + N1.getOperand(0).getOpcode() == ISD::AND) { + if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode())) + return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); } + + unsigned NextOp = N0.getOpcode(); + // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize) + if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) + if (SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1)) + if (SDNode *C2 = + DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) { + bool SameSide = (N->getOpcode() == NextOp); + unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB; + if (SDValue CombinedShift = + DAG.FoldConstantArithmetic(CombineOp, dl, VT, C1, C2)) { + unsigned Bitsize = VT.getScalarSizeInBits(); + SDValue BitsizeC = DAG.getConstant(Bitsize, dl, VT); + SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic( + ISD::SREM, dl, VT, CombinedShift.getNode(), BitsizeC.getNode()); + return DAG.getNode( + N->getOpcode(), dl, VT, N0->getOperand(0), CombinedShiftNorm); + } + } return SDValue(); } @@ -6091,19 +6118,22 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { SDValue N2 = N->getOperand(2); EVT VT = N->getValueType(0); EVT VT0 = N0.getValueType(); + SDLoc DL(N); // fold (select C, X, X) -> X if (N1 == N2) return N1; + if (const ConstantSDNode *N0C = dyn_cast(N0)) { // fold (select true, X, Y) -> X // fold (select false, X, Y) -> Y return !N0C->isNullValue() ? N1 : N2; } + // fold (select X, X, Y) -> (or X, Y) // fold (select X, 1, Y) -> (or C, Y) if (VT == VT0 && VT == MVT::i1 && (N0 == N1 || isOneConstant(N1))) - return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2); + return DAG.getNode(ISD::OR, DL, VT, N0, N2); if (SDValue V = foldSelectOfConstants(N)) return V; @@ -6112,22 +6142,22 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) { SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT); AddToWorklist(NOTNode.getNode()); - return DAG.getNode(ISD::AND, SDLoc(N), VT, NOTNode, N2); + return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2); } // fold (select C, X, 1) -> (or (not C), X) if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) { SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT); AddToWorklist(NOTNode.getNode()); - return DAG.getNode(ISD::OR, SDLoc(N), VT, NOTNode, N1); + return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1); } // fold (select X, Y, X) -> (and X, Y) // fold (select X, Y, 0) -> (and X, Y) if (VT == VT0 && VT == MVT::i1 && (N0 == N2 || isNullConstant(N2))) - return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1); + return DAG.getNode(ISD::AND, DL, VT, N0, N1); // If we can fold this based on the true/false value, do so. if (SimplifySelectOps(N, N1, N2)) - return SDValue(N, 0); // Don't revisit N. + return SDValue(N, 0); // Don't revisit N. if (VT0 == MVT::i1) { // The code in this block deals with the following 2 equivalences: @@ -6138,27 +6168,27 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { // to the right anyway if we find the inner select exists in the DAG anyway // and we always transform to the left side if we know that we can further // optimize the combination of the conditions. - bool normalizeToSequence - = TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT); + bool normalizeToSequence = + TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT); // select (and Cond0, Cond1), X, Y // -> select Cond0, (select Cond1, X, Y), Y if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { SDValue Cond0 = N0->getOperand(0); SDValue Cond1 = N0->getOperand(1); - SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N), - N1.getValueType(), Cond1, N1, N2); + SDValue InnerSelect = + DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2); if (normalizeToSequence || !InnerSelect.use_empty()) - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, InnerSelect, N2); } // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y) if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { SDValue Cond0 = N0->getOperand(0); SDValue Cond1 = N0->getOperand(1); - SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N), - N1.getValueType(), Cond1, N1, N2); + SDValue InnerSelect = + DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2); if (normalizeToSequence || !InnerSelect.use_empty()) - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, N1, + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1, InnerSelect); } @@ -6170,15 +6200,13 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) { // Create the actual and node if we can generate good code for it. if (!normalizeToSequence) { - SDValue And = DAG.getNode(ISD::AND, SDLoc(N), N0.getValueType(), - N0, N1_0); - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), And, - N1_1, N2); + SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, N2); } // Otherwise see if we can optimize the "and" to a better pattern. if (SDValue Combined = visitANDLike(N0, N1_0, N)) - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined, - N1_1, N2); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1, + N2); } } // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y @@ -6189,15 +6217,13 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) { // Create the actual or node if we can generate good code for it. if (!normalizeToSequence) { - SDValue Or = DAG.getNode(ISD::OR, SDLoc(N), N0.getValueType(), - N0, N2_0); - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Or, - N1, N2_2); + SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, N2_2); } // Otherwise see if we can optimize to a better pattern. if (SDValue Combined = visitORLike(N0, N2_0, N)) - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined, - N1, N2_2); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1, + N2_2); } } } @@ -6208,8 +6234,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { if (auto *C = dyn_cast(N0->getOperand(1))) { SDValue Cond0 = N0->getOperand(0); if (C->isOne()) - return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), - Cond0, N2, N1); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N2, N1); } } } @@ -6226,24 +6251,21 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { // FIXME: Instead of testing for UnsafeFPMath, this should be checking for // no signed zeros as well as no nans. const TargetOptions &Options = DAG.getTarget().Options; - if (Options.UnsafeFPMath && - VT.isFloatingPoint() && N0.hasOneUse() && + if (Options.UnsafeFPMath && VT.isFloatingPoint() && N0.hasOneUse() && DAG.isKnownNeverNaN(N1) && DAG.isKnownNeverNaN(N2)) { ISD::CondCode CC = cast(N0.getOperand(2))->get(); - if (SDValue FMinMax = combineMinNumMaxNum(SDLoc(N), VT, N0.getOperand(0), - N0.getOperand(1), N1, N2, CC, - TLI, DAG)) + if (SDValue FMinMax = combineMinNumMaxNum( + DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG)) return FMinMax; } if ((!LegalOperations && TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) || TLI.isOperationLegal(ISD::SELECT_CC, VT)) - return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, - N0.getOperand(0), N0.getOperand(1), - N1, N2, N0.getOperand(2)); - return SimplifySelect(SDLoc(N), N0, N1, N2); + return DAG.getNode(ISD::SELECT_CC, DL, VT, N0.getOperand(0), + N0.getOperand(1), N1, N2, N0.getOperand(2)); + return SimplifySelect(DL, N0, N1, N2); } return SDValue(); @@ -11045,7 +11067,7 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { // x1 * offset1 + y1 * ptr0 = t1 (the indexed load/store) // // where x0, x1, y0 and y1 in {-1, 1} are given by the types of the - // indexed load/store and the expresion that needs to be re-written. + // indexed load/store and the expression that needs to be re-written. // // Therefore, we have: // t0 = (x0 * offset0 - x1 * y0 * y1 *offset1) + (y0 * y1) * t1 @@ -11379,7 +11401,7 @@ namespace { /// Shift = srl Ty1 Origin, CstTy Amount /// Inst = trunc Shift to Ty2 /// -/// Then, it will be rewriten into: +/// Then, it will be rewritten into: /// Slice = load SliceTy, Base + SliceOffset /// [Inst = zext Slice to Ty2], only if SliceTy <> Ty2 /// @@ -12694,7 +12716,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { EVT StoreTy = EVT::getIntegerVT(Context, SizeInBits); bool IsFast = false; if (TLI.isTypeLegal(StoreTy) && - TLI.canMergeStoresTo(FirstStoreAS, StoreTy) && + TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, FirstStoreAlign, &IsFast) && IsFast) { @@ -12706,7 +12728,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { EVT LegalizedStoredValueTy = TLI.getTypeToTransformTo(Context, StoredVal.getValueType()); if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) && - TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValueTy) && + TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValueTy, DAG) && TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy, FirstStoreAS, FirstStoreAlign, &IsFast) && IsFast) { @@ -12723,7 +12745,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { !NoVectors) { // Find a legal type for the vector store. EVT Ty = EVT::getVectorVT(Context, MemVT, i + 1); - if (TLI.isTypeLegal(Ty) && TLI.canMergeStoresTo(FirstStoreAS, Ty) && + if (TLI.isTypeLegal(Ty) && + TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) && TLI.allowsMemoryAccess(Context, DL, Ty, FirstStoreAS, FirstStoreAlign, &IsFast) && IsFast) @@ -12781,7 +12804,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts); bool IsFast; - if (TLI.isTypeLegal(Ty) && TLI.canMergeStoresTo(FirstStoreAS, Ty) && + if (TLI.isTypeLegal(Ty) && + TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) && TLI.allowsMemoryAccess(Context, DL, Ty, FirstStoreAS, FirstStoreAlign, &IsFast) && IsFast) @@ -12898,7 +12922,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { EVT StoreTy = EVT::getVectorVT(Context, MemVT, i + 1); bool IsFastSt, IsFastLd; if (TLI.isTypeLegal(StoreTy) && - TLI.canMergeStoresTo(FirstStoreAS, StoreTy) && + TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, FirstStoreAlign, &IsFastSt) && IsFastSt && @@ -12912,7 +12936,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8; StoreTy = EVT::getIntegerVT(Context, SizeInBits); if (TLI.isTypeLegal(StoreTy) && - TLI.canMergeStoresTo(FirstStoreAS, StoreTy) && + TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, FirstStoreAlign, &IsFastSt) && IsFastSt && @@ -12926,7 +12950,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { TargetLowering::TypePromoteInteger) { EVT LegalizedStoredValueTy = TLI.getTypeToTransformTo(Context, StoreTy); if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) && - TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValueTy) && + TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValueTy, DAG) && TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy, StoreTy) && TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy, @@ -14228,6 +14252,73 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) { return Shuffles[0]; } +// Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT +// operations which can be matched to a truncate. +SDValue DAGCombiner::reduceBuildVecToTrunc(SDNode *N) { + // TODO: Add support for big-endian. + if (DAG.getDataLayout().isBigEndian()) + return SDValue(); + if (N->getNumOperands() < 2) + return SDValue(); + SDLoc DL(N); + EVT VT = N->getValueType(0); + unsigned NumElems = N->getNumOperands(); + + if (!isTypeLegal(VT)) + return SDValue(); + + // If the input is something other than an EXTRACT_VECTOR_ELT with a constant + // index, bail out. + // TODO: Allow undef elements in some cases? + if (any_of(N->ops(), [VT](SDValue Op) { + return Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || + !isa(Op.getOperand(1)) || + Op.getValueType() != VT.getVectorElementType(); + })) + return SDValue(); + + // Helper for obtaining an EXTRACT_VECTOR_ELT's constant index + auto GetExtractIdx = [](SDValue Extract) { + return cast(Extract.getOperand(1))->getSExtValue(); + }; + + // The first BUILD_VECTOR operand must be an an extract from index zero + // (assuming no undef and little-endian). + if (GetExtractIdx(N->getOperand(0)) != 0) + return SDValue(); + + // Compute the stride from the first index. + int Stride = GetExtractIdx(N->getOperand(1)); + SDValue ExtractedFromVec = N->getOperand(0).getOperand(0); + + // Proceed only if the stride and the types can be matched to a truncate. + if ((Stride == 1 || !isPowerOf2_32(Stride)) || + (ExtractedFromVec.getValueType().getVectorNumElements() != + Stride * NumElems) || + (VT.getScalarSizeInBits() * Stride > 64)) + return SDValue(); + + // Check remaining operands are consistent with the computed stride. + for (unsigned i = 1; i != NumElems; ++i) { + SDValue Op = N->getOperand(i); + + if ((Op.getOperand(0) != ExtractedFromVec) || + (GetExtractIdx(Op) != Stride * i)) + return SDValue(); + } + + // All checks were ok, construct the truncate. + LLVMContext &Ctx = *DAG.getContext(); + EVT NewVT = VT.getVectorVT( + Ctx, EVT::getIntegerVT(Ctx, VT.getScalarSizeInBits() * Stride), NumElems); + EVT TruncVT = + VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT; + + SDValue Res = DAG.getBitcast(NewVT, ExtractedFromVec); + Res = DAG.getNode(ISD::TRUNCATE, SDLoc(N), TruncVT, Res); + return DAG.getBitcast(VT, Res); +} + SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { EVT VT = N->getValueType(0); @@ -14270,6 +14361,10 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { if (SDValue V = reduceBuildVecConvertToConvertBuildVec(N)) return V; + if (TLI.isDesirableToCombineBuildVectorToTruncate()) + if (SDValue V = reduceBuildVecToTrunc(N)) + return V; + if (SDValue V = reduceBuildVecToShuffle(N)) return V; diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index b235e19aaab2..b96c96f0b4df 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -589,7 +589,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, } else AddOperand(MIB, N0, 0, nullptr, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); - // Add the subregster being inserted + // Add the subregister being inserted AddOperand(MIB, N1, 0, nullptr, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); MIB.addImm(SubIdx); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 873b2bd48f1e..7e4bc3ccb5d3 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1991,7 +1991,8 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, std::move(Args)) .setTailCall(isTailCall) .setSExtResult(isSigned) - .setZExtResult(!isSigned); + .setZExtResult(!isSigned) + .setIsPostTypeLegalization(true); std::pair CallInfo = TLI.LowerCallTo(CLI); @@ -2029,7 +2030,8 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args)) .setSExtResult(isSigned) - .setZExtResult(!isSigned); + .setZExtResult(!isSigned) + .setIsPostTypeLegalization(true); std::pair CallInfo = TLI.LowerCallTo(CLI); @@ -3565,16 +3567,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { SDValue Args[] = { HiLHS, LHS, HiRHS, RHS }; Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); } - BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, - DAG.getIntPtrConstant(0, dl)); - TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, - DAG.getIntPtrConstant(1, dl)); - // Ret is a node with an illegal type. Because such things are not - // generally permitted during this phase of legalization, make sure the - // node has no more uses. The above EXTRACT_ELEMENT nodes should have been - // folded. - assert(Ret->use_empty() && - "Unexpected uses of illegally type from expanded lib call."); + assert(Ret.getOpcode() == ISD::MERGE_VALUES && + "Ret value is a collection of constituent nodes holding result."); + BottomHalf = Ret.getOperand(0); + TopHalf = Ret.getOperand(1); } if (isSigned) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index c1cb5d9b5235..eaf177d0661b 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -112,15 +112,15 @@ bool DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; } - // If R is null, the sub-method took care of registering the result. - if (R.getNode()) { + if (R.getNode() && R.getNode() != N) { SetSoftenedFloat(SDValue(N, ResNo), R); - ReplaceSoftenFloatResult(N, ResNo, R); + // Return true only if the node is changed, assuming that the operands + // are also converted when necessary. + return true; } - // Return true only if the node is changed, - // assuming that the operands are also converted when necessary. + // Otherwise, return false to tell caller to scan operands. - return R.getNode() && R.getNode() != N; + return false; } SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N, unsigned ResNo) { @@ -753,12 +753,17 @@ bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { llvm_unreachable("Do not know how to soften this operator's operand!"); case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; + case ISD::CopyToReg: Res = SoftenFloatOp_COPY_TO_REG(N); break; case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; + case ISD::FABS: Res = SoftenFloatOp_FABS(N); break; + case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; + case ISD::FNEG: Res = SoftenFloatOp_FNEG(N); break; case ISD::FP_EXTEND: Res = SoftenFloatOp_FP_EXTEND(N); break; case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; + case ISD::SELECT: Res = SoftenFloatOp_SELECT(N); break; case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; case ISD::STORE: @@ -791,9 +796,9 @@ bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { bool DAGTypeLegalizer::CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo) { if (!isLegalInHWReg(N->getOperand(OpNo).getValueType())) return false; - // When the operand type can be kept in registers, SoftenFloatResult - // will call ReplaceValueWith to replace all references and we can - // skip softening this operand. + + // When the operand type can be kept in registers there is nothing to do for + // the following opcodes. switch (N->getOperand(OpNo).getOpcode()) { case ISD::BITCAST: case ISD::ConstantFP: @@ -807,18 +812,12 @@ bool DAGTypeLegalizer::CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo) { case ISD::SELECT_CC: return true; } - // For some opcodes, SoftenFloatResult handles all conversion of softening - // and replacing operands, so that there is no need to soften operands - // again, although such opcode could be scanned for other illegal operands. + switch (N->getOpcode()) { - case ISD::ConstantFP: - case ISD::CopyFromReg: - case ISD::CopyToReg: - case ISD::FABS: - case ISD::FCOPYSIGN: - case ISD::FNEG: - case ISD::Register: - case ISD::SELECT: + case ISD::ConstantFP: // Leaf node. + case ISD::CopyFromReg: // Operand is a register that we know to be left + // unchanged by SoftenFloatResult(). + case ISD::Register: // Leaf node. return true; } return false; @@ -829,6 +828,21 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { GetSoftenedFloat(N->getOperand(0))); } +SDValue DAGTypeLegalizer::SoftenFloatOp_COPY_TO_REG(SDNode *N) { + SDValue Op1 = GetSoftenedFloat(N->getOperand(1)); + SDValue Op2 = GetSoftenedFloat(N->getOperand(2)); + + if (Op1 == N->getOperand(1) && Op2 == N->getOperand(2)) + return SDValue(); + + if (N->getNumOperands() == 3) + return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op1, Op2), 0); + + return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op1, Op2, + N->getOperand(3)), + 0); +} + SDValue DAGTypeLegalizer::SoftenFloatOp_FP_EXTEND(SDNode *N) { // If we get here, the result must be legal but the source illegal. EVT SVT = N->getOperand(0).getValueType(); @@ -884,6 +898,34 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 0); } +SDValue DAGTypeLegalizer::SoftenFloatOp_FABS(SDNode *N) { + SDValue Op = GetSoftenedFloat(N->getOperand(0)); + + if (Op == N->getOperand(0)) + return SDValue(); + + return SDValue(DAG.UpdateNodeOperands(N, Op), 0); +} + +SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { + SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); + SDValue Op1 = GetSoftenedFloat(N->getOperand(1)); + + if (Op0 == N->getOperand(0) && Op1 == N->getOperand(1)) + return SDValue(); + + return SDValue(DAG.UpdateNodeOperands(N, Op0, Op1), 0); +} + +SDValue DAGTypeLegalizer::SoftenFloatOp_FNEG(SDNode *N) { + SDValue Op = GetSoftenedFloat(N->getOperand(0)); + + if (Op == N->getOperand(0)) + return SDValue(); + + return SDValue(DAG.UpdateNodeOperands(N, Op), 0); +} + SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { bool Signed = N->getOpcode() == ISD::FP_TO_SINT; EVT SVT = N->getOperand(0).getValueType(); @@ -913,6 +955,17 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { return DAG.getNode(ISD::TRUNCATE, dl, RVT, Res); } +SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT(SDNode *N) { + SDValue Op1 = GetSoftenedFloat(N->getOperand(1)); + SDValue Op2 = GetSoftenedFloat(N->getOperand(2)); + + if (Op1 == N->getOperand(1) && Op2 == N->getOperand(2)) + return SDValue(); + + return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op1, Op2), + 0); +} + SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); ISD::CondCode CCCode = cast(N->getOperand(4))->get(); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index 154af46c9446..001eed9fb8f6 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -80,6 +80,7 @@ void DAGTypeLegalizer::PerformExpensiveChecks() { for (unsigned i = 0, e = Node.getNumValues(); i != e; ++i) { SDValue Res(&Node, i); + EVT VT = Res.getValueType(); bool Failed = false; unsigned Mapped = 0; @@ -129,13 +130,17 @@ void DAGTypeLegalizer::PerformExpensiveChecks() { dbgs() << "Unprocessed value in a map!"; Failed = true; } - } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(&Node)) { + } else if (isTypeLegal(VT) || IgnoreNodeResults(&Node)) { if (Mapped > 1) { dbgs() << "Value with legal type was transformed!"; Failed = true; } } else { - if (Mapped == 0) { + // If the value can be kept in HW registers, softening machinery can + // leave it unchanged and don't put it to any map. + if (Mapped == 0 && + !(getTypeAction(VT) == TargetLowering::TypeSoftenFloat && + isLegalInHWReg(VT))) { dbgs() << "Processed value not in any map!"; Failed = true; } else if (Mapped & (Mapped - 1)) { @@ -331,11 +336,6 @@ bool DAGTypeLegalizer::run() { if (NeedsReanalyzing) { assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?"); - // Remove any result values from SoftenedFloats as N will be revisited - // again. - for (unsigned i = 0, NumResults = N->getNumValues(); i < NumResults; ++i) - SoftenedFloats.erase(SDValue(N, i)); - N->setNodeId(NewNode); // Recompute the NodeId and correct processed operands, adding the node to // the worklist if ready. @@ -754,8 +754,6 @@ void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) { // new uses of From due to CSE. If this happens, replace the new uses of // From with To. } while (!From.use_empty()); - - SoftenedFloats.erase(From); } void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 8e999188d8e1..e102df5e913d 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -416,16 +416,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { } void SetSoftenedFloat(SDValue Op, SDValue Result); - // Call ReplaceValueWith(SDValue(N, ResNo), Res) if necessary. - void ReplaceSoftenFloatResult(SDNode *N, unsigned ResNo, SDValue &NewRes) { - // When the result type can be kept in HW registers, the converted - // NewRes node could have the same type. We can save the effort in - // cloning every user of N in SoftenFloatOperand or other legalization functions, - // by calling ReplaceValueWith here to update all users. - if (NewRes.getNode() != N && isLegalInHWReg(N->getValueType(ResNo))) - ReplaceValueWith(SDValue(N, ResNo), NewRes); - } - // Convert Float Results to Integer for Non-HW-supported Operations. bool SoftenFloatResult(SDNode *N, unsigned ResNo); SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo); @@ -471,17 +461,23 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N); // Return true if we can skip softening the given operand or SDNode because - // it was soften before by SoftenFloatResult and references to the operand - // were replaced by ReplaceValueWith. + // either it was soften before by SoftenFloatResult and references to the + // operand were replaced by ReplaceValueWith or it's value type is legal in HW + // registers and the operand can be left unchanged. bool CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo); // Convert Float Operand to Integer for Non-HW-supported Operations. bool SoftenFloatOperand(SDNode *N, unsigned OpNo); SDValue SoftenFloatOp_BITCAST(SDNode *N); + SDValue SoftenFloatOp_COPY_TO_REG(SDNode *N); SDValue SoftenFloatOp_BR_CC(SDNode *N); + SDValue SoftenFloatOp_FABS(SDNode *N); + SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N); + SDValue SoftenFloatOp_FNEG(SDNode *N); SDValue SoftenFloatOp_FP_EXTEND(SDNode *N); SDValue SoftenFloatOp_FP_ROUND(SDNode *N); SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N); + SDValue SoftenFloatOp_SELECT(SDNode *N); SDValue SoftenFloatOp_SELECT_CC(SDNode *N); SDValue SoftenFloatOp_SETCC(SDNode *N); SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index aa69e0e2adfc..f3306151d864 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -57,7 +57,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { // Expand the floating point operand only if it was converted to integers. // Otherwise, it is a legal type like f128 that can be saved in a register. auto SoftenedOp = GetSoftenedFloat(InOp); - if (SoftenedOp == InOp) + if (isLegalInHWReg(SoftenedOp.getValueType())) break; SplitInteger(SoftenedOp, Lo, Hi); Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index ff0e609803d8..d41054b15bbc 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -2977,7 +2977,11 @@ SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled. unsigned InMaskOpc = InMask->getOpcode(); + + // FIXME: This code seems to be too restrictive, we might consider + // generalizing it or dropping it. assert((InMaskOpc == ISD::SETCC || + ISD::isBuildVectorOfConstantSDNodes(InMask.getNode()) || (isLogicalMaskOp(InMaskOpc) && isSETCCorConvertedSETCC(InMask->getOperand(0)) && isSETCCorConvertedSETCC(InMask->getOperand(1)))) && diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 98553152117d..823e77850c4b 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -34,6 +34,7 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineValueType.h" #include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/ValueTypes.h" @@ -5442,7 +5443,7 @@ SDValue SelectionDAG::getAtomicCmpSwap( unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) { + AtomicOrdering FailureOrdering, SyncScope::ID SSID) { assert(Opcode == ISD::ATOMIC_CMP_SWAP || Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); @@ -5458,7 +5459,7 @@ SDValue SelectionDAG::getAtomicCmpSwap( MachineMemOperand::MOStore; MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, - AAMDNodes(), nullptr, SynchScope, SuccessOrdering, + AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering); return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO); @@ -5480,7 +5481,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, - SynchronizationScope SynchScope) { + SyncScope::ID SSID) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -5500,7 +5501,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, MemVT.getStoreSize(), Alignment, AAMDNodes(), - nullptr, SynchScope, Ordering); + nullptr, SSID, Ordering); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); } @@ -7630,45 +7631,13 @@ bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, SDValue Loc = LD->getOperand(1); SDValue BaseLoc = Base->getOperand(1); - if (Loc.getOpcode() == ISD::FrameIndex) { - if (BaseLoc.getOpcode() != ISD::FrameIndex) - return false; - const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); - int FI = cast(Loc)->getIndex(); - int BFI = cast(BaseLoc)->getIndex(); - int FS = MFI.getObjectSize(FI); - int BFS = MFI.getObjectSize(BFI); - if (FS != BFS || FS != (int)Bytes) return false; - return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes); - } - // Handle X + C. - if (isBaseWithConstantOffset(Loc)) { - int64_t LocOffset = cast(Loc.getOperand(1))->getSExtValue(); - if (Loc.getOperand(0) == BaseLoc) { - // If the base location is a simple address with no offset itself, then - // the second load's first add operand should be the base address. - if (LocOffset == Dist * (int)Bytes) - return true; - } else if (isBaseWithConstantOffset(BaseLoc)) { - // The base location itself has an offset, so subtract that value from the - // second load's offset before comparing to distance * size. - int64_t BOffset = - cast(BaseLoc.getOperand(1))->getSExtValue(); - if (Loc.getOperand(0) == BaseLoc.getOperand(0)) { - if ((LocOffset - BOffset) == Dist * (int)Bytes) - return true; - } - } - } - const GlobalValue *GV1 = nullptr; - const GlobalValue *GV2 = nullptr; - int64_t Offset1 = 0; - int64_t Offset2 = 0; - bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1); - bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); - if (isGA1 && isGA2 && GV1 == GV2) - return Offset1 == (Offset2 + Dist*Bytes); + auto BaseLocDecomp = BaseIndexOffset::match(BaseLoc, *this); + auto LocDecomp = BaseIndexOffset::match(Loc, *this); + + int64_t Offset = 0; + if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) + return (Dist * Bytes == Offset); return false; } diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 4e899ae6668e..0d69441ebb7f 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -37,13 +37,13 @@ bool BaseIndexOffset::equalBaseIndex(BaseIndexOffset &Other, const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - // Match non-equal FrameIndexes - a FrameIndex stemming from an - // alloca will not have it's ObjectOffset set until post-DAG and - // as such we must assume the two framesIndices are incomparable. + // Match non-equal FrameIndexes - If both frame indices are fixed + // we know their relative offsets and can compare them. Otherwise + // we must be conservative. if (auto *A = dyn_cast(Base)) if (auto *B = dyn_cast(Other.Base)) - if (!MFI.getObjectAllocation(A->getIndex()) && - !MFI.getObjectAllocation(B->getIndex())) { + if (MFI.isFixedObjectIndex(A->getIndex()) && + MFI.isFixedObjectIndex(B->getIndex())) { Off += MFI.getObjectOffset(B->getIndex()) - MFI.getObjectOffset(A->getIndex()); return true; @@ -60,12 +60,18 @@ BaseIndexOffset BaseIndexOffset::match(SDValue Ptr, const SelectionDAG &DAG) { int64_t Offset = 0; bool IsIndexSignExt = false; - // Consume constant adds - while (Base->getOpcode() == ISD::ADD && - isa(Base->getOperand(1))) { - int64_t POffset = cast(Base->getOperand(1))->getSExtValue(); - Offset += POffset; - Base = Base->getOperand(0); + // Consume constant adds & ors with appropriate masking. + while (Base->getOpcode() == ISD::ADD || Base->getOpcode() == ISD::OR) { + if (auto *C = dyn_cast(Base->getOperand(1))) { + // Only consider ORs which act as adds. + if (Base->getOpcode() == ISD::OR && + !DAG.MaskedValueIsZero(Base->getOperand(0), C->getAPIntValue())) + break; + Offset += C->getSExtValue(); + Base = Base->getOperand(0); + continue; + } + break; } if (Base->getOpcode() == ISD::ADD) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index acf68fbbdedf..41c3f5f235ea 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3220,7 +3220,13 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { setValue(&I, DAG.getBuildVector(VT, DL, Ops)); } -void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { +void SelectionDAGBuilder::visitInsertValue(const User &I) { + ArrayRef Indices; + if (const InsertValueInst *IV = dyn_cast(&I)) + Indices = IV->getIndices(); + else + Indices = cast(&I)->getIndices(); + const Value *Op0 = I.getOperand(0); const Value *Op1 = I.getOperand(1); Type *AggTy = I.getType(); @@ -3228,7 +3234,7 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { bool IntoUndef = isa(Op0); bool FromUndef = isa(Op1); - unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); + unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SmallVector AggValueVTs; @@ -3268,13 +3274,19 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { DAG.getVTList(AggValueVTs), Values)); } -void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { +void SelectionDAGBuilder::visitExtractValue(const User &I) { + ArrayRef Indices; + if (const ExtractValueInst *EV = dyn_cast(&I)) + Indices = EV->getIndices(); + else + Indices = cast(&I)->getIndices(); + const Value *Op0 = I.getOperand(0); Type *AggTy = Op0->getType(); Type *ValTy = I.getType(); bool OutOfUndef = isa(Op0); - unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); + unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SmallVector ValValueVTs; @@ -3559,6 +3571,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { MMOFlags |= MachineMemOperand::MOInvariant; if (isDereferenceable) MMOFlags |= MachineMemOperand::MODereferenceable; + MMOFlags |= TLI.getMMOFlags(I); SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A, MachinePointerInfo(SV, Offsets[i]), Alignment, @@ -3688,6 +3701,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { MMOFlags |= MachineMemOperand::MOVolatile; if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr) MMOFlags |= MachineMemOperand::MONonTemporal; + MMOFlags |= TLI.getMMOFlags(I); // An aggregate load cannot wrap around the address space, so offsets to its // parts don't wrap either. @@ -3978,7 +3992,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { SDLoc dl = getCurSDLoc(); AtomicOrdering SuccessOrder = I.getSuccessOrdering(); AtomicOrdering FailureOrder = I.getFailureOrdering(); - SynchronizationScope Scope = I.getSynchScope(); + SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); @@ -3988,7 +4002,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain, getValue(I.getPointerOperand()), getValue(I.getCompareOperand()), getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), - /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope); + /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID); SDValue OutChain = L.getValue(2); @@ -4014,7 +4028,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; } AtomicOrdering Order = I.getOrdering(); - SynchronizationScope Scope = I.getSynchScope(); + SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); @@ -4025,7 +4039,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { getValue(I.getPointerOperand()), getValue(I.getValOperand()), I.getPointerOperand(), - /* Alignment=*/ 0, Order, Scope); + /* Alignment=*/ 0, Order, SSID); SDValue OutChain = L.getValue(1); @@ -4040,7 +4054,7 @@ void SelectionDAGBuilder::visitFence(const FenceInst &I) { Ops[0] = getRoot(); Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl, TLI.getFenceOperandTy(DAG.getDataLayout())); - Ops[2] = DAG.getConstant(I.getSynchScope(), dl, + Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl, TLI.getFenceOperandTy(DAG.getDataLayout())); DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops)); } @@ -4048,7 +4062,7 @@ void SelectionDAGBuilder::visitFence(const FenceInst &I) { void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { SDLoc dl = getCurSDLoc(); AtomicOrdering Order = I.getOrdering(); - SynchronizationScope Scope = I.getSynchScope(); + SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); @@ -4066,7 +4080,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { VT.getStoreSize(), I.getAlignment() ? I.getAlignment() : DAG.getEVTAlignment(VT), - AAMDNodes(), nullptr, Scope, Order); + AAMDNodes(), nullptr, SSID, Order); InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); SDValue L = @@ -4083,7 +4097,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { SDLoc dl = getCurSDLoc(); AtomicOrdering Order = I.getOrdering(); - SynchronizationScope Scope = I.getSynchScope(); + SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); @@ -4100,7 +4114,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { getValue(I.getPointerOperand()), getValue(I.getValueOperand()), I.getPointerOperand(), I.getAlignment(), - Order, Scope); + Order, SSID); DAG.setRoot(OutChain); } @@ -4982,6 +4996,83 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.setRoot(CallResult.second); return nullptr; } + case Intrinsic::memmove_element_unordered_atomic: { + auto &MI = cast(I); + SDValue Dst = getValue(MI.getRawDest()); + SDValue Src = getValue(MI.getRawSource()); + SDValue Length = getValue(MI.getLength()); + + // Emit a library call. + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); + Entry.Node = Dst; + Args.push_back(Entry); + + Entry.Node = Src; + Args.push_back(Entry); + + Entry.Ty = MI.getLength()->getType(); + Entry.Node = Length; + Args.push_back(Entry); + + uint64_t ElementSizeConstant = MI.getElementSizeInBytes(); + RTLIB::Libcall LibraryCall = + RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSizeConstant); + if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) + report_fatal_error("Unsupported element size"); + + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee( + TLI.getLibcallCallingConv(LibraryCall), + Type::getVoidTy(*DAG.getContext()), + DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall), + TLI.getPointerTy(DAG.getDataLayout())), + std::move(Args)); + + std::pair CallResult = TLI.LowerCallTo(CLI); + DAG.setRoot(CallResult.second); + return nullptr; + } + case Intrinsic::memset_element_unordered_atomic: { + auto &MI = cast(I); + SDValue Dst = getValue(MI.getRawDest()); + SDValue Val = getValue(MI.getValue()); + SDValue Length = getValue(MI.getLength()); + + // Emit a library call. + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); + Entry.Node = Dst; + Args.push_back(Entry); + + Entry.Ty = Type::getInt8Ty(*DAG.getContext()); + Entry.Node = Val; + Args.push_back(Entry); + + Entry.Ty = MI.getLength()->getType(); + Entry.Node = Length; + Args.push_back(Entry); + + uint64_t ElementSizeConstant = MI.getElementSizeInBytes(); + RTLIB::Libcall LibraryCall = + RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSizeConstant); + if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) + report_fatal_error("Unsupported element size"); + + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee( + TLI.getLibcallCallingConv(LibraryCall), + Type::getVoidTy(*DAG.getContext()), + DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall), + TLI.getPointerTy(DAG.getDataLayout())), + std::move(Args)); + + std::pair CallResult = TLI.LowerCallTo(CLI); + DAG.setRoot(CallResult.second); + return nullptr; + } case Intrinsic::dbg_declare: { const DbgDeclareInst &DI = cast(I); DILocalVariable *Variable = DI.getVariable(); @@ -7842,6 +7933,22 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { auto &DL = CLI.DAG.getDataLayout(); ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); + if (CLI.IsPostTypeLegalization) { + // If we are lowering a libcall after legalization, split the return type. + SmallVector OldRetTys = std::move(RetTys); + SmallVector OldOffsets = std::move(Offsets); + for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) { + EVT RetVT = OldRetTys[i]; + uint64_t Offset = OldOffsets[i]; + MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT); + unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT); + unsigned RegisterVTSize = RegisterVT.getSizeInBits(); + RetTys.append(NumRegs, RegisterVT); + for (unsigned j = 0; j != NumRegs; ++j) + Offsets.push_back(Offset + j * RegisterVTSize); + } + } + SmallVector Outs; GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL); @@ -7924,6 +8031,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { for (unsigned i = 0, e = Args.size(); i != e; ++i) { SmallVector ValueVTs; ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs); + // FIXME: Split arguments if CLI.IsPostTypeLegalization Type *FinalType = Args[i].Ty; if (Args[i].IsByVal) FinalType = cast(Args[i].Ty)->getElementType(); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 431d52b4b9b9..ac1d6aae65a5 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -38,7 +38,6 @@ class BranchInst; class CallInst; class DbgValueInst; class ExtractElementInst; -class ExtractValueInst; class FCmpInst; class FPExtInst; class FPToSIInst; @@ -53,7 +52,6 @@ class IntToPtrInst; class IndirectBrInst; class InvokeInst; class InsertElementInst; -class InsertValueInst; class Instruction; class LoadInst; class MachineBasicBlock; @@ -859,8 +857,8 @@ class SelectionDAGBuilder { void visitInsertElement(const User &I); void visitShuffleVector(const User &I); - void visitExtractValue(const ExtractValueInst &I); - void visitInsertValue(const InsertValueInst &I); + void visitExtractValue(const User &I); + void visitInsertValue(const User &I); void visitLandingPad(const LandingPadInst &I); void visitGetElementPtr(const User &I); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index f711ca71f79f..bdf57e805842 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1483,7 +1483,6 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Try to select the instruction with FastISel. if (FastIS->selectInstruction(Inst)) { - FastISelFailed = true; --NumFastIselRemaining; ++NumFastIselSuccess; // If fast isel succeeded, skip over all the folded instructions, and @@ -1506,8 +1505,14 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { continue; } + FastISelFailed = true; + // Then handle certain instructions as single-LLVM-Instruction blocks. - if (isa(Inst)) { + // We cannot separate out GCrelocates to their own blocks since we need + // to keep track of gc-relocates for a particular gc-statepoint. This is + // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before + // visitGCRelocate. + if (isa(Inst) && !isStatepoint(Inst) && !isGCRelocate(Inst)) { OptimizationRemarkMissed R("sdagisel", "FastISelFailure", Inst->getDebugLoc(), LLVMBB); diff --git a/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp index 7886737b879c..17a3a84ecda5 100644 --- a/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp +++ b/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp @@ -125,8 +125,11 @@ static void MarkBlocksLiveIn(BasicBlock *BB, if (!LiveBBs.insert(BB).second) return; // already been here. - for (BasicBlock *PredBB : predecessors(BB)) - MarkBlocksLiveIn(PredBB, LiveBBs); + df_iterator_default_set Visited; + + for (BasicBlock *B : inverse_depth_first_ext(BB, Visited)) + LiveBBs.insert(B); + } /// substituteLPadValues - Substitute the values returned by the landingpad diff --git a/contrib/llvm/lib/CodeGen/SplitKit.cpp b/contrib/llvm/lib/CodeGen/SplitKit.cpp index 008b984dd961..323045fd2aaa 100644 --- a/contrib/llvm/lib/CodeGen/SplitKit.cpp +++ b/contrib/llvm/lib/CodeGen/SplitKit.cpp @@ -53,10 +53,10 @@ InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI, std::pair &LIP = LastInsertPoint[Num]; SlotIndex MBBEnd = LIS.getMBBEndIdx(&MBB); - SmallVector EHPadSucessors; + SmallVector EHPadSuccessors; for (const MachineBasicBlock *SMBB : MBB.successors()) if (SMBB->isEHPad()) - EHPadSucessors.push_back(SMBB); + EHPadSuccessors.push_back(SMBB); // Compute insert points on the first call. The pair is independent of the // current live interval. @@ -68,7 +68,7 @@ InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI, LIP.first = LIS.getInstructionIndex(*FirstTerm); // If there is a landing pad successor, also find the call instruction. - if (EHPadSucessors.empty()) + if (EHPadSuccessors.empty()) return LIP.first; // There may not be a call instruction (?) in which case we ignore LPad. LIP.second = LIP.first; @@ -87,7 +87,7 @@ InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI, if (!LIP.second) return LIP.first; - if (none_of(EHPadSucessors, [&](const MachineBasicBlock *EHPad) { + if (none_of(EHPadSuccessors, [&](const MachineBasicBlock *EHPad) { return LIS.isLiveInToMBB(CurLI, EHPad); })) return LIP.first; diff --git a/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp b/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp index e9d38c10c860..3914ee514712 100644 --- a/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -384,6 +384,26 @@ static void InitLibcallNames(const char **Names, const Triple &TT) { "__llvm_memcpy_element_unordered_atomic_8"; Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_16] = "__llvm_memcpy_element_unordered_atomic_16"; + Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1] = + "__llvm_memmove_element_unordered_atomic_1"; + Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2] = + "__llvm_memmove_element_unordered_atomic_2"; + Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4] = + "__llvm_memmove_element_unordered_atomic_4"; + Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8] = + "__llvm_memmove_element_unordered_atomic_8"; + Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16] = + "__llvm_memmove_element_unordered_atomic_16"; + Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_1] = + "__llvm_memset_element_unordered_atomic_1"; + Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_2] = + "__llvm_memset_element_unordered_atomic_2"; + Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_4] = + "__llvm_memset_element_unordered_atomic_4"; + Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_8] = + "__llvm_memset_element_unordered_atomic_8"; + Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_16] = + "__llvm_memset_element_unordered_atomic_16"; Names[RTLIB::UNWIND_RESUME] = "_Unwind_Resume"; Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1] = "__sync_val_compare_and_swap_1"; Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2] = "__sync_val_compare_and_swap_2"; @@ -803,6 +823,40 @@ RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { } } +RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { + switch (ElementSize) { + case 1: + return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1; + case 2: + return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2; + case 4: + return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4; + case 8: + return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8; + case 16: + return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16; + default: + return UNKNOWN_LIBCALL; + } +} + +RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { + switch (ElementSize) { + case 1: + return MEMSET_ELEMENT_UNORDERED_ATOMIC_1; + case 2: + return MEMSET_ELEMENT_UNORDERED_ATOMIC_2; + case 4: + return MEMSET_ELEMENT_UNORDERED_ATOMIC_4; + case 8: + return MEMSET_ELEMENT_UNORDERED_ATOMIC_8; + case 16: + return MEMSET_ELEMENT_UNORDERED_ATOMIC_16; + default: + return UNKNOWN_LIBCALL; + } +} + /// InitCmpLibcallCCs - Set default comparison libcall CC. /// static void InitCmpLibcallCCs(ISD::CondCode *CCs) { diff --git a/contrib/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp b/contrib/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp index b9fa9b6a6ad7..c2c02f8de03f 100644 --- a/contrib/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp +++ b/contrib/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp @@ -62,6 +62,18 @@ class CVSymbolDumperImpl : public SymbolVisitorCallbacks { }; } +static StringRef getSymbolKindName(SymbolKind Kind) { + switch (Kind) { +#define SYMBOL_RECORD(EnumName, EnumVal, Name) \ + case EnumName: \ + return #Name; +#include "llvm/DebugInfo/CodeView/CodeViewSymbols.def" + default: + break; + } + return "UnknownSym"; +} + void CVSymbolDumperImpl::printLocalVariableAddrRange( const LocalVariableAddrRange &Range, uint32_t RelocationOffset) { DictScope S(W, "LocalVariableAddrRange"); @@ -86,18 +98,23 @@ void CVSymbolDumperImpl::printTypeIndex(StringRef FieldName, TypeIndex TI) { } Error CVSymbolDumperImpl::visitSymbolBegin(CVSymbol &CVR) { + W.startLine() << getSymbolKindName(CVR.Type); + W.getOStream() << " {\n"; + W.indent(); + W.printEnum("Kind", unsigned(CVR.Type), getSymbolTypeNames()); return Error::success(); } Error CVSymbolDumperImpl::visitSymbolEnd(CVSymbol &CVR) { if (PrintRecordBytes && ObjDelegate) ObjDelegate->printBinaryBlockWithRelocs("SymData", CVR.content()); + + W.unindent(); + W.startLine() << "}\n"; return Error::success(); } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, BlockSym &Block) { - DictScope S(W, "BlockStart"); - StringRef LinkageName; W.printHex("PtrParent", Block.Parent); W.printHex("PtrEnd", Block.End); @@ -113,7 +130,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, BlockSym &Block) { } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Thunk32Sym &Thunk) { - DictScope S(W, "Thunk32"); W.printNumber("Parent", Thunk.Parent); W.printNumber("End", Thunk.End); W.printNumber("Next", Thunk.Next); @@ -126,7 +142,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Thunk32Sym &Thunk) { Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, TrampolineSym &Tramp) { - DictScope S(W, "Trampoline"); W.printEnum("Type", uint16_t(Tramp.Type), getTrampolineNames()); W.printNumber("Size", Tramp.Size); W.printNumber("ThunkOff", Tramp.ThunkOffset); @@ -137,7 +152,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, SectionSym &Section) { - DictScope S(W, "Section"); W.printNumber("SectionNumber", Section.SectionNumber); W.printNumber("Alignment", Section.Alignment); W.printNumber("Rva", Section.Rva); @@ -152,7 +166,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, SectionSym &Section) { Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, CoffGroupSym &CoffGroup) { - DictScope S(W, "COFF Group"); W.printNumber("Size", CoffGroup.Size); W.printFlags("Characteristics", CoffGroup.Characteristics, getImageSectionCharacteristicNames(), @@ -165,8 +178,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, BPRelativeSym &BPRel) { - DictScope S(W, "BPRelativeSym"); - W.printNumber("Offset", BPRel.Offset); printTypeIndex("Type", BPRel.Type); W.printString("VarName", BPRel.Name); @@ -175,16 +186,12 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, BuildInfoSym &BuildInfo) { - DictScope S(W, "BuildInfo"); - W.printNumber("BuildId", BuildInfo.BuildId); return Error::success(); } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, CallSiteInfoSym &CallSiteInfo) { - DictScope S(W, "CallSiteInfo"); - StringRef LinkageName; if (ObjDelegate) { ObjDelegate->printRelocatedField("CodeOffset", @@ -200,8 +207,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, EnvBlockSym &EnvBlock) { - DictScope S(W, "EnvBlock"); - ListScope L(W, "Entries"); for (auto Entry : EnvBlock.Fields) { W.printString(Entry); @@ -211,7 +216,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, FileStaticSym &FileStatic) { - DictScope S(W, "FileStatic"); printTypeIndex("Index", FileStatic.Index); W.printNumber("ModFilenameOffset", FileStatic.ModFilenameOffset); W.printFlags("Flags", uint16_t(FileStatic.Flags), getLocalFlagNames()); @@ -220,7 +224,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ExportSym &Export) { - DictScope S(W, "Export"); W.printNumber("Ordinal", Export.Ordinal); W.printFlags("Flags", uint16_t(Export.Flags), getExportSymFlagNames()); W.printString("Name", Export.Name); @@ -229,8 +232,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ExportSym &Export) { Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Compile2Sym &Compile2) { - DictScope S(W, "CompilerFlags2"); - W.printEnum("Language", Compile2.getLanguage(), getSourceLanguageNames()); W.printFlags("Flags", Compile2.getFlags(), getCompileSym2FlagNames()); W.printEnum("Machine", unsigned(Compile2.Machine), getCPUTypeNames()); @@ -254,8 +255,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Compile3Sym &Compile3) { - DictScope S(W, "CompilerFlags3"); - W.printEnum("Language", Compile3.getLanguage(), getSourceLanguageNames()); W.printFlags("Flags", Compile3.getFlags(), getCompileSym3FlagNames()); W.printEnum("Machine", unsigned(Compile3.Machine), getCPUTypeNames()); @@ -281,8 +280,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ConstantSym &Constant) { - DictScope S(W, "Constant"); - printTypeIndex("Type", Constant.Type); W.printNumber("Value", Constant.Value); W.printString("Name", Constant.Name); @@ -290,9 +287,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, DataSym &Data) { - DictScope S(W, "DataSym"); - - W.printEnum("Kind", uint16_t(CVR.kind()), getSymbolTypeNames()); StringRef LinkageName; if (ObjDelegate) { ObjDelegate->printRelocatedField("DataOffset", Data.getRelocationOffset(), @@ -308,15 +302,12 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, DataSym &Data) { Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeFramePointerRelFullScopeSym &DefRangeFramePointerRelFullScope) { - DictScope S(W, "DefRangeFramePointerRelFullScope"); W.printNumber("Offset", DefRangeFramePointerRelFullScope.Offset); return Error::success(); } Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeFramePointerRelSym &DefRangeFramePointerRel) { - DictScope S(W, "DefRangeFramePointerRel"); - W.printNumber("Offset", DefRangeFramePointerRel.Offset); printLocalVariableAddrRange(DefRangeFramePointerRel.Range, DefRangeFramePointerRel.getRelocationOffset()); @@ -326,8 +317,6 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeRegisterRelSym &DefRangeRegisterRel) { - DictScope S(W, "DefRangeRegisterRel"); - W.printNumber("BaseRegister", DefRangeRegisterRel.Hdr.Register); W.printBoolean("HasSpilledUDTMember", DefRangeRegisterRel.hasSpilledUDTMember()); @@ -341,8 +330,6 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeRegisterSym &DefRangeRegister) { - DictScope S(W, "DefRangeRegister"); - W.printNumber("Register", DefRangeRegister.Hdr.Register); W.printNumber("MayHaveNoName", DefRangeRegister.Hdr.MayHaveNoName); printLocalVariableAddrRange(DefRangeRegister.Range, @@ -353,8 +340,6 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeSubfieldRegisterSym &DefRangeSubfieldRegister) { - DictScope S(W, "DefRangeSubfieldRegister"); - W.printNumber("Register", DefRangeSubfieldRegister.Hdr.Register); W.printNumber("MayHaveNoName", DefRangeSubfieldRegister.Hdr.MayHaveNoName); W.printNumber("OffsetInParent", DefRangeSubfieldRegister.Hdr.OffsetInParent); @@ -366,8 +351,6 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeSubfieldSym &DefRangeSubfield) { - DictScope S(W, "DefRangeSubfield"); - if (ObjDelegate) { DebugStringTableSubsectionRef Strings = ObjDelegate->getStringTable(); auto ExpectedProgram = Strings.getString(DefRangeSubfield.Program); @@ -387,8 +370,6 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, DefRangeSym &DefRange) { - DictScope S(W, "DefRange"); - if (ObjDelegate) { DebugStringTableSubsectionRef Strings = ObjDelegate->getStringTable(); auto ExpectedProgram = Strings.getString(DefRange.Program); @@ -406,8 +387,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, FrameCookieSym &FrameCookie) { - DictScope S(W, "FrameCookie"); - StringRef LinkageName; if (ObjDelegate) { ObjDelegate->printRelocatedField("CodeOffset", @@ -423,8 +402,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, FrameProcSym &FrameProc) { - DictScope S(W, "FrameProc"); - W.printHex("TotalFrameBytes", FrameProc.TotalFrameBytes); W.printHex("PaddingFrameBytes", FrameProc.PaddingFrameBytes); W.printHex("OffsetToPadding", FrameProc.OffsetToPadding); @@ -440,8 +417,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, HeapAllocationSiteSym &HeapAllocSite) { - DictScope S(W, "HeapAllocationSite"); - StringRef LinkageName; if (ObjDelegate) { ObjDelegate->printRelocatedField("CodeOffset", @@ -458,8 +433,6 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, InlineSiteSym &InlineSite) { - DictScope S(W, "InlineSite"); - W.printHex("PtrParent", InlineSite.Parent); W.printHex("PtrEnd", InlineSite.End); printTypeIndex("Inlinee", InlineSite.Inlinee); @@ -515,7 +488,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, RegisterSym &Register) { - DictScope S(W, "RegisterSym"); printTypeIndex("Type", Register.Index); W.printEnum("Seg", uint16_t(Register.Register), getRegisterNames()); W.printString("Name", Register.Name); @@ -523,7 +495,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, PublicSym32 &Public) { - DictScope S(W, "PublicSym"); W.printFlags("Flags", uint32_t(Public.Flags), getPublicSymFlagNames()); W.printNumber("Seg", Public.Segment); W.printNumber("Off", Public.Offset); @@ -532,7 +503,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, PublicSym32 &Public) { } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ProcRefSym &ProcRef) { - DictScope S(W, "ProcRef"); W.printNumber("SumName", ProcRef.SumName); W.printNumber("SymOffset", ProcRef.SymOffset); W.printNumber("Mod", ProcRef.Module); @@ -541,8 +511,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ProcRefSym &ProcRef) { } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, LabelSym &Label) { - DictScope S(W, "Label"); - StringRef LinkageName; if (ObjDelegate) { ObjDelegate->printRelocatedField("CodeOffset", Label.getRelocationOffset(), @@ -558,8 +526,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, LabelSym &Label) { } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, LocalSym &Local) { - DictScope S(W, "Local"); - printTypeIndex("Type", Local.Type); W.printFlags("Flags", uint16_t(Local.Flags), getLocalFlagNames()); W.printString("VarName", Local.Name); @@ -567,16 +533,12 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, LocalSym &Local) { } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ObjNameSym &ObjName) { - DictScope S(W, "ObjectName"); - W.printHex("Signature", ObjName.Signature); W.printString("ObjectName", ObjName.Name); return Error::success(); } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ProcSym &Proc) { - DictScope S(W, "ProcStart"); - if (InFunctionScope) return llvm::make_error( "Visiting a ProcSym while inside function scope!"); @@ -584,7 +546,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ProcSym &Proc) { InFunctionScope = true; StringRef LinkageName; - W.printEnum("Kind", uint16_t(CVR.kind()), getSymbolTypeNames()); W.printHex("PtrParent", Proc.Parent); W.printHex("PtrEnd", Proc.End); W.printHex("PtrNext", Proc.Next); @@ -607,13 +568,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ProcSym &Proc) { Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ScopeEndSym &ScopeEnd) { - if (CVR.kind() == SymbolKind::S_END) - DictScope S(W, "BlockEnd"); - else if (CVR.kind() == SymbolKind::S_PROC_ID_END) - DictScope S(W, "ProcEnd"); - else if (CVR.kind() == SymbolKind::S_INLINESITE_END) - DictScope S(W, "InlineSiteEnd"); - InFunctionScope = false; return Error::success(); } @@ -627,8 +581,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, CallerSym &Caller) { Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, RegRelativeSym &RegRel) { - DictScope S(W, "RegRelativeSym"); - W.printHex("Offset", RegRel.Offset); printTypeIndex("Type", RegRel.Type); W.printEnum("Register", uint16_t(RegRel.Register), getRegisterNames()); @@ -638,8 +590,6 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, ThreadLocalDataSym &Data) { - DictScope S(W, "ThreadLocalDataSym"); - StringRef LinkageName; if (ObjDelegate) { ObjDelegate->printRelocatedField("DataOffset", Data.getRelocationOffset(), @@ -653,15 +603,12 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, } Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, UDTSym &UDT) { - DictScope S(W, "UDT"); printTypeIndex("Type", UDT.Type); W.printString("UDTName", UDT.Name); return Error::success(); } Error CVSymbolDumperImpl::visitUnknownSymbol(CVSymbol &CVR) { - DictScope S(W, "UnknownSym"); - W.printEnum("Kind", uint16_t(CVR.kind()), getSymbolTypeNames()); W.printNumber("Length", CVR.length()); return Error::success(); } diff --git a/contrib/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp b/contrib/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp index 72cb9e2e3544..0d935c4472ae 100644 --- a/contrib/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp +++ b/contrib/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp @@ -382,6 +382,13 @@ static bool discoverTypeIndices(ArrayRef Content, SymbolKind Kind, case SymbolKind::S_BUILDINFO: Refs.push_back({TiRefKind::IndexRef, 0, 1}); // Compile flags break; + case SymbolKind::S_LTHREAD32: + case SymbolKind::S_GTHREAD32: + Refs.push_back({TiRefKind::TypeRef, 0, 1}); // Type + break; + case SymbolKind::S_FILESTATIC: + Refs.push_back({TiRefKind::TypeRef, 0, 1}); // Type + break; case SymbolKind::S_LOCAL: Refs.push_back({TiRefKind::TypeRef, 0, 1}); // Type break; @@ -403,6 +410,10 @@ static bool discoverTypeIndices(ArrayRef Content, SymbolKind Kind, case SymbolKind::S_INLINESITE: Refs.push_back({TiRefKind::IndexRef, 8, 1}); // ID of inlinee break; + case SymbolKind::S_HEAPALLOCSITE: + // FIXME: It's not clear if this is a type or item reference. + Refs.push_back({TiRefKind::IndexRef, 8, 1}); // signature + break; // Defranges don't have types, just registers and code offsets. case SymbolKind::S_DEFRANGE_REGISTER: @@ -419,6 +430,7 @@ static bool discoverTypeIndices(ArrayRef Content, SymbolKind Kind, case SymbolKind::S_COMPILE: case SymbolKind::S_COMPILE2: case SymbolKind::S_COMPILE3: + case SymbolKind::S_ENVBLOCK: case SymbolKind::S_BLOCK32: case SymbolKind::S_FRAMEPROC: break; diff --git a/contrib/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/contrib/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp index a18d4efec07a..495e09fbae35 100644 --- a/contrib/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp +++ b/contrib/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp @@ -591,10 +591,10 @@ void DWARFContext::parseCompileUnits() { void DWARFContext::parseTypeUnits() { if (!TUs.empty()) return; - for (const auto &I : getTypesSections()) { + forEachTypesSections([&](const DWARFSection &S) { TUs.emplace_back(); - TUs.back().parse(*this, I.second); - } + TUs.back().parse(*this, S); + }); } void DWARFContext::parseDWOCompileUnits() { @@ -604,10 +604,10 @@ void DWARFContext::parseDWOCompileUnits() { void DWARFContext::parseDWOTypeUnits() { if (!DWOTUs.empty()) return; - for (const auto &I : getTypesDWOSections()) { + forEachTypesDWOSections([&](const DWARFSection &S) { DWOTUs.emplace_back(); - DWOTUs.back().parseDWO(*this, I.second); - } + DWOTUs.back().parseDWO(*this, S); + }); } DWARFCompileUnit *DWARFContext::getCompileUnitForOffset(uint32_t Offset) { @@ -937,27 +937,23 @@ DWARFContextInMemory::DWARFContextInMemory( : FileName(Obj.getFileName()), IsLittleEndian(Obj.isLittleEndian()), AddressSize(Obj.getBytesInAddress()) { for (const SectionRef &Section : Obj.sections()) { - StringRef name; - Section.getName(name); + StringRef Name; + Section.getName(Name); // Skip BSS and Virtual sections, they aren't interesting. - bool IsBSS = Section.isBSS(); - if (IsBSS) + if (Section.isBSS() || Section.isVirtual()) continue; - bool IsVirtual = Section.isVirtual(); - if (IsVirtual) - continue; - StringRef data; + StringRef Data; section_iterator RelocatedSection = Section.getRelocatedSection(); // Try to obtain an already relocated version of this section. // Else use the unrelocated section from the object file. We'll have to // apply relocations ourselves later. - if (!L || !L->getLoadedSectionContents(*RelocatedSection, data)) - Section.getContents(data); + if (!L || !L->getLoadedSectionContents(*RelocatedSection, Data)) + Section.getContents(Data); - if (auto Err = maybeDecompress(Section, name, data)) { + if (auto Err = maybeDecompress(Section, Name, Data)) { ErrorPolicy EP = HandleError( - createError("failed to decompress '" + name + "', ", std::move(Err))); + createError("failed to decompress '" + Name + "', ", std::move(Err))); if (EP == ErrorPolicy::Halt) return; continue; @@ -965,26 +961,26 @@ DWARFContextInMemory::DWARFContextInMemory( // Compressed sections names in GNU style starts from ".z", // at this point section is decompressed and we drop compression prefix. - name = name.substr( - name.find_first_not_of("._z")); // Skip ".", "z" and "_" prefixes. - - if (StringRef *SectionData = MapSectionToMember(name)) { - *SectionData = data; - if (name == "debug_ranges") { - // FIXME: Use the other dwo range section when we emit it. - RangeDWOSection.Data = data; - } - } else if (name == "debug_types") { - // Find debug_types data by section rather than name as there are - // multiple, comdat grouped, debug_types sections. - TypesSections[Section].Data = data; - } else if (name == "debug_types.dwo") { - TypesDWOSections[Section].Data = data; - } + Name = Name.substr( + Name.find_first_not_of("._z")); // Skip ".", "z" and "_" prefixes. // Map platform specific debug section names to DWARF standard section // names. - name = Obj.mapDebugSectionName(name); + Name = Obj.mapDebugSectionName(Name); + + if (StringRef *SectionData = mapSectionToMember(Name)) { + *SectionData = Data; + if (Name == "debug_ranges") { + // FIXME: Use the other dwo range section when we emit it. + RangeDWOSection.Data = Data; + } + } else if (Name == "debug_types") { + // Find debug_types data by section rather than name as there are + // multiple, comdat grouped, debug_types sections. + TypesSections[Section].Data = Data; + } else if (Name == "debug_types.dwo") { + TypesDWOSections[Section].Data = Data; + } if (RelocatedSection == Obj.section_end()) continue; @@ -1012,21 +1008,8 @@ DWARFContextInMemory::DWARFContextInMemory( // TODO: Add support for relocations in other sections as needed. // Record relocations for the debug_info and debug_line sections. - RelocAddrMap *Map = - StringSwitch(RelSecName) - .Case("debug_info", &InfoSection.Relocs) - .Case("debug_loc", &LocSection.Relocs) - .Case("debug_info.dwo", &InfoDWOSection.Relocs) - .Case("debug_line", &LineSection.Relocs) - .Case("debug_str_offsets", &StringOffsetSection.Relocs) - .Case("debug_ranges", &RangeSection.Relocs) - .Case("debug_addr", &AddrSection.Relocs) - .Case("apple_names", &AppleNamesSection.Relocs) - .Case("apple_types", &AppleTypesSection.Relocs) - .Case("apple_namespaces", &AppleNamespacesSection.Relocs) - .Case("apple_namespac", &AppleNamespacesSection.Relocs) - .Case("apple_objc", &AppleObjCSection.Relocs) - .Default(nullptr); + DWARFSection *Sec = mapNameToDWARFSection(RelSecName); + RelocAddrMap *Map = Sec ? &Sec->Relocs : nullptr; if (!Map) { // Find debug_types relocs by section rather than name as there are // multiple, comdat grouped, debug_types sections. @@ -1059,10 +1042,10 @@ DWARFContextInMemory::DWARFContextInMemory( object::RelocVisitor V(Obj); uint64_t Val = V.visit(Reloc.getType(), Reloc, SymInfoOrErr->Address); if (V.error()) { - SmallString<32> Name; - Reloc.getTypeName(Name); + SmallString<32> Type; + Reloc.getTypeName(Type); ErrorPolicy EP = HandleError( - createError("failed to compute relocation: " + Name + ", ", + createError("failed to compute relocation: " + Type + ", ", errorCodeToError(object_error::parse_failed))); if (EP == ErrorPolicy::Halt) return; @@ -1079,40 +1062,47 @@ DWARFContextInMemory::DWARFContextInMemory( bool isLittleEndian) : IsLittleEndian(isLittleEndian), AddressSize(AddrSize) { for (const auto &SecIt : Sections) { - if (StringRef *SectionData = MapSectionToMember(SecIt.first())) + if (StringRef *SectionData = mapSectionToMember(SecIt.first())) *SectionData = SecIt.second->getBuffer(); } } -StringRef *DWARFContextInMemory::MapSectionToMember(StringRef Name) { +DWARFSection *DWARFContextInMemory::mapNameToDWARFSection(StringRef Name) { + return StringSwitch(Name) + .Case("debug_info", &InfoSection) + .Case("debug_loc", &LocSection) + .Case("debug_line", &LineSection) + .Case("debug_str_offsets", &StringOffsetSection) + .Case("debug_ranges", &RangeSection) + .Case("debug_info.dwo", &InfoDWOSection) + .Case("debug_loc.dwo", &LocDWOSection) + .Case("debug_line.dwo", &LineDWOSection) + .Case("debug_str_offsets.dwo", &StringOffsetDWOSection) + .Case("debug_addr", &AddrSection) + .Case("apple_names", &AppleNamesSection) + .Case("apple_types", &AppleTypesSection) + .Case("apple_namespaces", &AppleNamespacesSection) + .Case("apple_namespac", &AppleNamespacesSection) + .Case("apple_objc", &AppleObjCSection) + .Default(nullptr); +} + +StringRef *DWARFContextInMemory::mapSectionToMember(StringRef Name) { + if (DWARFSection *Sec = mapNameToDWARFSection(Name)) + return &Sec->Data; return StringSwitch(Name) - .Case("debug_info", &InfoSection.Data) .Case("debug_abbrev", &AbbrevSection) - .Case("debug_loc", &LocSection.Data) - .Case("debug_line", &LineSection.Data) .Case("debug_aranges", &ARangeSection) .Case("debug_frame", &DebugFrameSection) .Case("eh_frame", &EHFrameSection) .Case("debug_str", &StringSection) - .Case("debug_str_offsets", &StringOffsetSection.Data) - .Case("debug_ranges", &RangeSection.Data) .Case("debug_macinfo", &MacinfoSection) .Case("debug_pubnames", &PubNamesSection) .Case("debug_pubtypes", &PubTypesSection) .Case("debug_gnu_pubnames", &GnuPubNamesSection) .Case("debug_gnu_pubtypes", &GnuPubTypesSection) - .Case("debug_info.dwo", &InfoDWOSection.Data) .Case("debug_abbrev.dwo", &AbbrevDWOSection) - .Case("debug_loc.dwo", &LocDWOSection.Data) - .Case("debug_line.dwo", &LineDWOSection.Data) .Case("debug_str.dwo", &StringDWOSection) - .Case("debug_str_offsets.dwo", &StringOffsetDWOSection.Data) - .Case("debug_addr", &AddrSection.Data) - .Case("apple_names", &AppleNamesSection.Data) - .Case("apple_types", &AppleTypesSection.Data) - .Case("apple_namespaces", &AppleNamespacesSection.Data) - .Case("apple_namespac", &AppleNamespacesSection.Data) - .Case("apple_objc", &AppleObjCSection.Data) .Case("debug_cu_index", &CUIndexSection) .Case("debug_tu_index", &TUIndexSection) .Case("gdb_index", &GdbIndexSection) diff --git a/contrib/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/contrib/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp index ef416f72ad17..111f0bbd4444 100644 --- a/contrib/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp +++ b/contrib/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp @@ -395,7 +395,7 @@ DWARFDie::attribute_iterator::attribute_iterator(DWARFDie D, bool End) : void DWARFDie::attribute_iterator::updateForIndex( const DWARFAbbreviationDeclaration &AbbrDecl, uint32_t I) { Index = I; - // AbbrDecl must be valid befor calling this function. + // AbbrDecl must be valid before calling this function. auto NumAttrs = AbbrDecl.getNumAttributes(); if (Index < NumAttrs) { AttrValue.Attr = AbbrDecl.getAttrByIndex(Index); diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp index 745dd742aadc..897f78c51032 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp @@ -65,6 +65,10 @@ void DbiModuleDescriptorBuilder::setObjFileName(StringRef Name) { ObjFileName = Name; } +void DbiModuleDescriptorBuilder::setPdbFilePathNI(uint32_t NI) { + PdbFilePathNI = NI; +} + void DbiModuleDescriptorBuilder::addSymbol(CVSymbol Symbol) { Symbols.push_back(Symbol); // Symbols written to a PDB file are required to be 4 byte aligned. The same @@ -111,7 +115,7 @@ void DbiModuleDescriptorBuilder::finalize() { (void)Layout.Mod; // Set in constructor (void)Layout.ModDiStream; // Set in finalizeMsfLayout Layout.NumFiles = SourceFiles.size(); - Layout.PdbFilePathNI = 0; + Layout.PdbFilePathNI = PdbFilePathNI; Layout.SrcFileNameNI = 0; // This value includes both the signature field as well as the record bytes diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStream.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStream.cpp index a1f0671dec3e..0eeac7e4c084 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStream.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStream.cpp @@ -225,6 +225,10 @@ void DbiStream::visitSectionContributions( } } +Expected DbiStream::getECName(uint32_t NI) const { + return ECNames.getStringForID(NI); +} + Error DbiStream::initializeSectionContributionData() { if (SecContrSubstream.empty()) return Error::success(); @@ -248,6 +252,9 @@ Error DbiStream::initializeSectionHeadersData() { return Error::success(); uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::SectionHdr); + if (StreamNum == kInvalidStreamIndex) + return Error::success(); + if (StreamNum >= Pdb.getNumStreams()) return make_error(raw_error_code::no_stream); diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp index aad247ea185f..25076e40fc98 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp @@ -49,9 +49,17 @@ void DbiStreamBuilder::setSectionMap(ArrayRef SecMap) { SectionMap = SecMap; } +void DbiStreamBuilder::setSymbolRecordStreamIndex(uint32_t Index) { + SymRecordStreamIndex = Index; +} + +void DbiStreamBuilder::setPublicsStreamIndex(uint32_t Index) { + PublicsStreamIndex = Index; +} + Error DbiStreamBuilder::addDbgStream(pdb::DbgHeaderType Type, ArrayRef Data) { - if (DbgStreams[(int)Type].StreamNumber) + if (DbgStreams[(int)Type].StreamNumber != kInvalidStreamIndex) return make_error(raw_error_code::duplicate_entry, "The specified stream type already exists"); auto ExpectedIndex = Msf.addStream(Data.size()); @@ -63,11 +71,16 @@ Error DbiStreamBuilder::addDbgStream(pdb::DbgHeaderType Type, return Error::success(); } +uint32_t DbiStreamBuilder::addECName(StringRef Name) { + return ECNamesBuilder.insert(Name); +} + uint32_t DbiStreamBuilder::calculateSerializedLength() const { // For now we only support serializing the header. return sizeof(DbiStreamHeader) + calculateFileInfoSubstreamSize() + calculateModiSubstreamSize() + calculateSectionContribsStreamSize() + - calculateSectionMapStreamSize() + calculateDbgStreamsSize(); + calculateSectionMapStreamSize() + calculateDbgStreamsSize() + + ECNamesBuilder.calculateSerializedSize(); } Expected @@ -247,15 +260,15 @@ Error DbiStreamBuilder::finalize() { H->PdbDllVersion = PdbDllVersion; H->MachineType = static_cast(MachineType); - H->ECSubstreamSize = 0; + H->ECSubstreamSize = ECNamesBuilder.calculateSerializedSize(); H->FileInfoSize = FileInfoBuffer.getLength(); H->ModiSubstreamSize = calculateModiSubstreamSize(); H->OptionalDbgHdrSize = DbgStreams.size() * sizeof(uint16_t); H->SecContrSubstreamSize = calculateSectionContribsStreamSize(); H->SectionMapSize = calculateSectionMapStreamSize(); H->TypeServerSize = 0; - H->SymRecordStreamIndex = kInvalidStreamIndex; - H->PublicSymbolStreamIndex = kInvalidStreamIndex; + H->SymRecordStreamIndex = SymRecordStreamIndex; + H->PublicSymbolStreamIndex = PublicsStreamIndex; H->MFCTypeServerIndex = kInvalidStreamIndex; H->GlobalSymbolStreamIndex = kInvalidStreamIndex; @@ -383,6 +396,9 @@ Error DbiStreamBuilder::commit(const msf::MSFLayout &Layout, if (auto EC = Writer.writeStreamRef(FileInfoBuffer)) return EC; + if (auto EC = ECNamesBuilder.commit(Writer)) + return EC; + for (auto &Stream : DbgStreams) if (auto EC = Writer.writeInteger(Stream.StreamNumber)) return EC; diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp index 354b8c0e07ff..6cdf6dde04d9 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp @@ -86,7 +86,8 @@ Error NamedStreamMap::commit(BinaryStreamWriter &Writer) const { for (const auto &Name : OrderedStreamNames) { auto Item = Mapping.find(Name); - assert(Item != Mapping.end()); + if (Item == Mapping.end()) + continue; if (auto EC = Writer.writeCString(Item->getKey())) return EC; } @@ -108,7 +109,8 @@ uint32_t NamedStreamMap::finalize() { for (const auto &Name : OrderedStreamNames) { auto Item = Mapping.find(Name); - assert(Item != Mapping.end()); + if (Item == Mapping.end()) + continue; FinalizedHashTable.set(FinalizedInfo->StringDataBytes, Item->getValue()); FinalizedInfo->StringDataBytes += Item->getKeyLength() + 1; } diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeBuiltinSymbol.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeBuiltinSymbol.cpp new file mode 100644 index 000000000000..60416f69e137 --- /dev/null +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeBuiltinSymbol.cpp @@ -0,0 +1,48 @@ +//===- NativeBuiltinSymbol.cpp ------------------------------------ C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h" + +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +NativeBuiltinSymbol::NativeBuiltinSymbol(NativeSession &PDBSession, + SymIndexId Id, PDB_BuiltinType T, + uint64_t L) + : NativeRawSymbol(PDBSession, Id), Session(PDBSession), Type(T), Length(L) { +} + +NativeBuiltinSymbol::~NativeBuiltinSymbol() {} + +std::unique_ptr NativeBuiltinSymbol::clone() const { + return llvm::make_unique(Session, SymbolId, Type, Length); +} + +void NativeBuiltinSymbol::dump(raw_ostream &OS, int Indent) const { + // TODO: Apparently nothing needs this yet. +} + +PDB_SymType NativeBuiltinSymbol::getSymTag() const { + return PDB_SymType::BuiltinType; +} + +PDB_BuiltinType NativeBuiltinSymbol::getBuiltinType() const { return Type; } + +bool NativeBuiltinSymbol::isConstType() const { return false; } + +uint64_t NativeBuiltinSymbol::getLength() const { return Length; } + +bool NativeBuiltinSymbol::isUnalignedType() const { return false; } + +bool NativeBuiltinSymbol::isVolatileType() const { return false; } + +} // namespace pdb +} // namespace llvm diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp index 180c169ec209..7132a99a9f16 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp @@ -15,7 +15,7 @@ namespace llvm { namespace pdb { NativeCompilandSymbol::NativeCompilandSymbol(NativeSession &Session, - uint32_t SymbolId, + SymIndexId SymbolId, DbiModuleDescriptor MI) : NativeRawSymbol(Session, SymbolId), Module(MI) {} diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp index 6206155b9fb6..cb0830f453c8 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp @@ -18,7 +18,7 @@ namespace llvm { namespace pdb { -NativeExeSymbol::NativeExeSymbol(NativeSession &Session, uint32_t SymbolId) +NativeExeSymbol::NativeExeSymbol(NativeSession &Session, SymIndexId SymbolId) : NativeRawSymbol(Session, SymbolId), File(Session.getPDBFile()) {} std::unique_ptr NativeExeSymbol::clone() const { diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp index b4f5c96ce66b..92612bcea4ac 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp @@ -13,7 +13,7 @@ using namespace llvm; using namespace llvm::pdb; -NativeRawSymbol::NativeRawSymbol(NativeSession &PDBSession, uint32_t SymbolId) +NativeRawSymbol::NativeRawSymbol(NativeSession &PDBSession, SymIndexId SymbolId) : Session(PDBSession), SymbolId(SymbolId) {} void NativeRawSymbol::dump(raw_ostream &OS, int Indent) const {} diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp index 93d43d9ef341..76de0d8f9e7e 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp @@ -10,9 +10,11 @@ #include "llvm/DebugInfo/PDB/Native/NativeSession.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/PDB/GenericError.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/IPDBSourceFile.h" +#include "llvm/DebugInfo/PDB/Native/NativeBuiltinSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeExeSymbol.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" @@ -33,6 +35,28 @@ using namespace llvm; using namespace llvm::msf; using namespace llvm::pdb; +namespace { +// Maps codeview::SimpleTypeKind of a built-in type to the parameters necessary +// to instantiate a NativeBuiltinSymbol for that type. +static const struct BuiltinTypeEntry { + codeview::SimpleTypeKind Kind; + PDB_BuiltinType Type; + uint32_t Size; +} BuiltinTypes[] = { + {codeview::SimpleTypeKind::Int32, PDB_BuiltinType::Int, 4}, + {codeview::SimpleTypeKind::UInt32, PDB_BuiltinType::UInt, 4}, + {codeview::SimpleTypeKind::UInt32Long, PDB_BuiltinType::UInt, 4}, + {codeview::SimpleTypeKind::UInt64Quad, PDB_BuiltinType::UInt, 8}, + {codeview::SimpleTypeKind::NarrowCharacter, PDB_BuiltinType::Char, 1}, + {codeview::SimpleTypeKind::SignedCharacter, PDB_BuiltinType::Char, 1}, + {codeview::SimpleTypeKind::UnsignedCharacter, PDB_BuiltinType::UInt, 1}, + {codeview::SimpleTypeKind::UInt16Short, PDB_BuiltinType::UInt, 2}, + {codeview::SimpleTypeKind::Boolean8, PDB_BuiltinType::Bool, 1} + // This table can be grown as necessary, but these are the only types we've + // needed so far. +}; +} // namespace + NativeSession::NativeSession(std::unique_ptr PdbFile, std::unique_ptr Allocator) : Pdb(std::move(PdbFile)), Allocator(std::move(Allocator)) {} @@ -71,19 +95,51 @@ Error NativeSession::createFromExe(StringRef Path, std::unique_ptr NativeSession::createCompilandSymbol(DbiModuleDescriptor MI) { - const auto Id = static_cast(SymbolCache.size()); + const auto Id = static_cast(SymbolCache.size()); SymbolCache.push_back( llvm::make_unique(*this, Id, MI)); return llvm::make_unique( *this, std::unique_ptr(SymbolCache[Id]->clone())); } +SymIndexId NativeSession::findSymbolByTypeIndex(codeview::TypeIndex Index) { + // First see if it's already in our cache. + const auto Entry = TypeIndexToSymbolId.find(Index); + if (Entry != TypeIndexToSymbolId.end()) + return Entry->second; + + // Symbols for built-in types are created on the fly. + if (Index.isSimple()) { + // FIXME: We will eventually need to handle pointers to other simple types, + // which are still simple types in the world of CodeView TypeIndexes. + if (Index.getSimpleMode() != codeview::SimpleTypeMode::Direct) + return 0; + const auto Kind = Index.getSimpleKind(); + const auto It = + std::find_if(std::begin(BuiltinTypes), std::end(BuiltinTypes), + [Kind](const BuiltinTypeEntry &Builtin) { + return Builtin.Kind == Kind; + }); + if (It == std::end(BuiltinTypes)) + return 0; + SymIndexId Id = SymbolCache.size(); + SymbolCache.emplace_back( + llvm::make_unique(*this, Id, It->Type, It->Size)); + TypeIndexToSymbolId[Index] = Id; + return Id; + } + + // TODO: Look up PDB type by type index + + return 0; +} + uint64_t NativeSession::getLoadAddress() const { return 0; } void NativeSession::setLoadAddress(uint64_t Address) {} std::unique_ptr NativeSession::getGlobalScope() { - const auto Id = static_cast(SymbolCache.size()); + const auto Id = static_cast(SymbolCache.size()); SymbolCache.push_back(llvm::make_unique(*this, Id)); auto RawSymbol = SymbolCache[Id]->clone(); auto PdbSymbol(PDBSymbol::create(*this, std::move(RawSymbol))); diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp index 4f6ebb0cb342..0b6492efc70f 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp @@ -385,8 +385,11 @@ bool PDBFile::hasPDBDbiStream() const { return StreamDBI < getNumStreams(); } bool PDBFile::hasPDBGlobalsStream() { auto DbiS = getPDBDbiStream(); - if (!DbiS) + if (!DbiS) { + consumeError(DbiS.takeError()); return false; + } + return DbiS->getGlobalSymbolStreamIndex() < getNumStreams(); } @@ -396,8 +399,10 @@ bool PDBFile::hasPDBIpiStream() const { return StreamIPI < getNumStreams(); } bool PDBFile::hasPDBPublicsStream() { auto DbiS = getPDBDbiStream(); - if (!DbiS) + if (!DbiS) { + consumeError(DbiS.takeError()); return false; + } return DbiS->getPublicSymbolStreamIndex() < getNumStreams(); } diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp index 12b0c3b36c1d..9f35fd73629c 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp @@ -18,6 +18,7 @@ #include "llvm/DebugInfo/PDB/Native/InfoStream.h" #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" +#include "llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h" #include "llvm/DebugInfo/PDB/Native/RawError.h" #include "llvm/DebugInfo/PDB/Native/TpiStream.h" #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h" @@ -33,6 +34,8 @@ using namespace llvm::support; PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator) : Allocator(Allocator) {} +PDBFileBuilder::~PDBFileBuilder() {} + Error PDBFileBuilder::initialize(uint32_t BlockSize) { auto ExpectedMsf = MSFBuilder::create(Allocator, BlockSize); if (!ExpectedMsf) @@ -71,6 +74,12 @@ PDBStringTableBuilder &PDBFileBuilder::getStringTableBuilder() { return Strings; } +PublicsStreamBuilder &PDBFileBuilder::getPublicsBuilder() { + if (!Publics) + Publics = llvm::make_unique(*Msf); + return *Publics; +} + Error PDBFileBuilder::addNamedStream(StringRef Name, uint32_t Size) { auto ExpectedStream = Msf->addStream(Size); if (!ExpectedStream) @@ -96,8 +105,6 @@ Expected PDBFileBuilder::finalizeMsfLayout() { return std::move(EC); if (auto EC = addNamedStream("/LinkInfo", 0)) return std::move(EC); - if (auto EC = addNamedStream("/src/headerblock", 0)) - return std::move(EC); if (Info) { if (auto EC = Info->finalizeMsfLayout()) @@ -115,6 +122,14 @@ Expected PDBFileBuilder::finalizeMsfLayout() { if (auto EC = Ipi->finalizeMsfLayout()) return std::move(EC); } + if (Publics) { + if (auto EC = Publics->finalizeMsfLayout()) + return std::move(EC); + if (Dbi) { + Dbi->setPublicsStreamIndex(Publics->getStreamIndex()); + Dbi->setSymbolRecordStreamIndex(Publics->getRecordStreamIdx()); + } + } return Msf->build(); } @@ -194,5 +209,13 @@ Error PDBFileBuilder::commit(StringRef Filename) { return EC; } + if (Publics) { + auto PS = WritableMappedBlockStream::createIndexedStream( + Layout, Buffer, Publics->getStreamIndex(), Allocator); + BinaryStreamWriter PSWriter(*PS); + if (auto EC = Publics->commit(PSWriter)) + return EC; + } + return Buffer.commit(); } diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/PDBStringTable.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/PDBStringTable.cpp index f9f8ac219d35..acd45f7a6219 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/PDBStringTable.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/PDBStringTable.cpp @@ -21,7 +21,7 @@ using namespace llvm; using namespace llvm::support; using namespace llvm::pdb; -uint32_t PDBStringTable::getByteSize() const { return ByteSize; } +uint32_t PDBStringTable::getByteSize() const { return Header->ByteSize; } uint32_t PDBStringTable::getNameCount() const { return NameCount; } uint32_t PDBStringTable::getHashVersion() const { return Header->HashVersion; } uint32_t PDBStringTable::getSignature() const { return Header->Signature; } diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp index 8f3474b9ce19..9c3e654f808b 100644 --- a/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp @@ -41,19 +41,6 @@ using namespace llvm::msf; using namespace llvm::support; using namespace llvm::pdb; -// This is PSGSIHDR struct defined in -// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h -struct PublicsStream::HeaderInfo { - ulittle32_t SymHash; - ulittle32_t AddrMap; - ulittle32_t NumThunks; - ulittle32_t SizeOfThunk; - ulittle16_t ISectThunkTable; - char Padding[2]; - ulittle32_t OffThunkTable; - ulittle32_t NumSections; -}; - PublicsStream::PublicsStream(PDBFile &File, std::unique_ptr Stream) : Pdb(File), Stream(std::move(Stream)) {} @@ -72,7 +59,8 @@ Error PublicsStream::reload() { BinaryStreamReader Reader(*Stream); // Check stream size. - if (Reader.bytesRemaining() < sizeof(HeaderInfo) + sizeof(GSIHashHeader)) + if (Reader.bytesRemaining() < + sizeof(PublicsStreamHeader) + sizeof(GSIHashHeader)) return make_error(raw_error_code::corrupt_file, "Publics Stream does not contain a header."); diff --git a/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStreamBuilder.cpp b/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStreamBuilder.cpp new file mode 100644 index 000000000000..28c4a8fc35d9 --- /dev/null +++ b/contrib/llvm/lib/DebugInfo/PDB/Native/PublicsStreamBuilder.cpp @@ -0,0 +1,89 @@ +//===- DbiStreamBuilder.cpp - PDB Dbi Stream Creation -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h" + +#include "llvm/DebugInfo/MSF/MSFBuilder.h" +#include "llvm/DebugInfo/MSF/MSFCommon.h" +#include "llvm/DebugInfo/MSF/MappedBlockStream.h" + +#include "GSI.h" + +using namespace llvm; +using namespace llvm::msf; +using namespace llvm::pdb; + +PublicsStreamBuilder::PublicsStreamBuilder(msf::MSFBuilder &Msf) : Msf(Msf) {} + +PublicsStreamBuilder::~PublicsStreamBuilder() {} + +uint32_t PublicsStreamBuilder::calculateSerializedLength() const { + uint32_t Size = 0; + Size += sizeof(PublicsStreamHeader); + Size += sizeof(GSIHashHeader); + Size += HashRecords.size() * sizeof(PSHashRecord); + size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32); + uint32_t NumBitmapEntries = BitmapSizeInBits / 8; + Size += NumBitmapEntries; + + // FIXME: Account for hash buckets. For now since we we write a zero-bitmap + // indicating that no hash buckets are valid, we also write zero byets of hash + // bucket data. + Size += 0; + return Size; +} + +Error PublicsStreamBuilder::finalizeMsfLayout() { + Expected Idx = Msf.addStream(calculateSerializedLength()); + if (!Idx) + return Idx.takeError(); + StreamIdx = *Idx; + + Expected RecordIdx = Msf.addStream(0); + if (!RecordIdx) + return RecordIdx.takeError(); + RecordStreamIdx = *RecordIdx; + return Error::success(); +} + +Error PublicsStreamBuilder::commit(BinaryStreamWriter &PublicsWriter) { + PublicsStreamHeader PSH; + GSIHashHeader GSH; + + // FIXME: Figure out what to put for these values. + PSH.AddrMap = 0; + PSH.ISectThunkTable = 0; + PSH.NumSections = 0; + PSH.NumThunks = 0; + PSH.OffThunkTable = 0; + PSH.SizeOfThunk = 0; + PSH.SymHash = 0; + + GSH.VerSignature = GSIHashHeader::HdrSignature; + GSH.VerHdr = GSIHashHeader::HdrVersion; + GSH.HrSize = 0; + GSH.NumBuckets = 0; + + if (auto EC = PublicsWriter.writeObject(PSH)) + return EC; + if (auto EC = PublicsWriter.writeObject(GSH)) + return EC; + if (auto EC = PublicsWriter.writeArray(makeArrayRef(HashRecords))) + return EC; + + size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32); + uint32_t NumBitmapEntries = BitmapSizeInBits / 8; + std::vector BitmapData(NumBitmapEntries); + // FIXME: Build an actual bitmap + if (auto EC = PublicsWriter.writeBytes(makeArrayRef(BitmapData))) + return EC; + + // FIXME: Write actual hash buckets. + return Error::success(); +} diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp index ff8749fbfed4..1164d60ffc10 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -317,7 +317,13 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, raw_string_ostream MangledNameStream(MangledName); Mangler::getNameWithPrefix(MangledNameStream, Name, getDataLayout()); } - return findSymbol(MangledName, CheckFunctionsOnly).getAddress(); + if (auto Sym = findSymbol(MangledName, CheckFunctionsOnly)) { + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + report_fatal_error(AddrOrErr.takeError()); + } else + report_fatal_error(Sym.takeError()); } JITSymbol MCJIT::findSymbol(const std::string &Name, @@ -599,11 +605,12 @@ GenericValue MCJIT::runFunction(Function *F, ArrayRef ArgValues) { void *MCJIT::getPointerToNamedFunction(StringRef Name, bool AbortOnFailure) { if (!isSymbolSearchingDisabled()) { - void *ptr = - reinterpret_cast( - static_cast(Resolver.findSymbol(Name).getAddress())); - if (ptr) - return ptr; + if (auto Sym = Resolver.findSymbol(Name)) { + if (auto AddrOrErr = Sym.getAddress()) + return reinterpret_cast( + static_cast(*AddrOrErr)); + } else if (auto Err = Sym.takeError()) + report_fatal_error(std::move(Err)); } /// If a LazyFunctionCreator is installed, use it to get/create the function. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp index 5fe259f80b6f..de80cb1d0dd4 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -60,12 +60,13 @@ void LLVMOrcGetMangledSymbol(LLVMOrcJITStackRef JITStack, char **MangledName, void LLVMOrcDisposeMangledSymbol(char *MangledName) { delete[] MangledName; } -LLVMOrcTargetAddress +LLVMOrcErrorCode LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.createLazyCompileCallback(Callback, CallbackCtx); + return J.createLazyCompileCallback(*RetAddr, Callback, CallbackCtx); } LLVMOrcErrorCode LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, @@ -82,38 +83,44 @@ LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, return J.setIndirectStubPointer(StubName, NewAddr); } -LLVMOrcModuleHandle +LLVMOrcErrorCode LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, LLVMSharedModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); std::shared_ptr *M(unwrap(Mod)); - return J.addIRModuleEager(*M, SymbolResolver, SymbolResolverCtx); + return J.addIRModuleEager(*RetHandle, *M, SymbolResolver, SymbolResolverCtx); } -LLVMOrcModuleHandle +LLVMOrcErrorCode LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, LLVMSharedModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); std::shared_ptr *M(unwrap(Mod)); - return J.addIRModuleLazy(*M, SymbolResolver, SymbolResolverCtx); + return J.addIRModuleLazy(*RetHandle, *M, SymbolResolver, SymbolResolverCtx); } -void LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, LLVMOrcModuleHandle H) { +LLVMOrcErrorCode LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle H) { OrcCBindingsStack &J = *unwrap(JITStack); - J.removeModule(H); + return J.removeModule(H); } -LLVMOrcTargetAddress LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, - const char *SymbolName) { +LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + const char *SymbolName) { OrcCBindingsStack &J = *unwrap(JITStack); - auto Sym = J.findSymbol(SymbolName, true); - return Sym.getAddress(); + return J.findSymbolAddress(*RetAddr, SymbolName, true); } -void LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { - delete unwrap(JITStack); +LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { + auto *J = unwrap(JITStack); + auto Err = J->shutdown(); + delete J; + return Err; } diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 931d0a9eb2ad..e38decf94f3e 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -70,7 +70,7 @@ class OrcCBindingsStack { virtual JITSymbol findSymbolIn(const std::string &Name, bool ExportedSymbolsOnly) = 0; - virtual void removeModule() = 0; + virtual Error removeModule() = 0; }; template class GenericHandleImpl : public GenericHandle { @@ -83,7 +83,7 @@ class OrcCBindingsStack { return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly); } - void removeModule() override { return Layer.removeModule(Handle); } + Error removeModule() override { return Layer.removeModule(Handle); } private: LayerT &Layer; @@ -105,6 +105,10 @@ class OrcCBindingsStack { IndirectStubsManagerBuilder IndirectStubsMgrBuilder) : DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), CCMgr(std::move(CCMgr)), + ObjectLayer( + []() { + return std::make_shared(); + }), CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), CODLayer(CompileLayer, [](Function &F) { return std::set({&F}); }, @@ -112,12 +116,14 @@ class OrcCBindingsStack { CXXRuntimeOverrides( [this](const std::string &S) { return mangle(S); }) {} - ~OrcCBindingsStack() { + LLVMOrcErrorCode shutdown() { // Run any destructors registered with __cxa_atexit. CXXRuntimeOverrides.runDestructors(); // Run any IR destructors. for (auto &DtorRunner : IRStaticDestructorRunners) - DtorRunner.runViaLayer(*this); + if (auto Err = DtorRunner.runViaLayer(*this)) + return mapError(std::move(Err)); + return LLVMOrcErrSuccess; } std::string mangle(StringRef Name) { @@ -134,14 +140,17 @@ class OrcCBindingsStack { return reinterpret_cast(static_cast(Addr)); } - JITTargetAddress - createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, + + LLVMOrcErrorCode + createLazyCompileCallback(JITTargetAddress &RetAddr, + LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { auto CCInfo = CCMgr->getCompileCallback(); CCInfo.setCompileAction([=]() -> JITTargetAddress { return Callback(wrap(this), CallbackCtx); }); - return CCInfo.getAddress(); + RetAddr = CCInfo.getAddress(); + return LLVMOrcErrSuccess; } LLVMOrcErrorCode createIndirectStub(StringRef StubName, @@ -155,12 +164,12 @@ class OrcCBindingsStack { return mapError(IndirectStubsMgr->updatePointer(Name, Addr)); } - std::unique_ptr + std::shared_ptr createResolver(LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { return orc::createLambdaResolver( [this, ExternalResolver, ExternalResolverCtx](const std::string &Name) - -> JITSymbol { + -> JITSymbol { // Search order: // 1. JIT'd symbols. // 2. Runtime overrides. @@ -168,6 +177,9 @@ class OrcCBindingsStack { if (auto Sym = CODLayer.findSymbol(Name, true)) return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + if (auto Sym = CXXRuntimeOverrides.searchOverrides(Name)) return Sym; @@ -178,16 +190,19 @@ class OrcCBindingsStack { return JITSymbol(nullptr); }, - [](const std::string &Name) { + [](const std::string &Name) -> JITSymbol { return JITSymbol(nullptr); }); } template - ModuleHandleT addIRModule(LayerT &Layer, std::shared_ptr M, - std::unique_ptr MemMgr, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { + LLVMOrcErrorCode + addIRModule(ModuleHandleT &RetHandle, LayerT &Layer, + std::shared_ptr M, + std::unique_ptr MemMgr, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + // Attach a data-layout if one isn't already present. if (M->getDataLayout().isDefault()) M->setDataLayout(DL); @@ -204,43 +219,52 @@ class OrcCBindingsStack { auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx); // Add the module to the JIT. - auto LH = Layer.addModule(std::move(M), std::move(MemMgr), - std::move(Resolver)); - ModuleHandleT H = createHandle(Layer, LH); + ModuleHandleT H; + if (auto LHOrErr = Layer.addModule(std::move(M), std::move(Resolver))) + H = createHandle(Layer, *LHOrErr); + else + return mapError(LHOrErr.takeError()); // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. orc::CtorDtorRunner CtorRunner(std::move(CtorNames), H); - CtorRunner.runViaLayer(*this); + if (auto Err = CtorRunner.runViaLayer(*this)) + return mapError(std::move(Err)); IRStaticDestructorRunners.emplace_back(std::move(DtorNames), H); - return H; + RetHandle = H; + return LLVMOrcErrSuccess; } - ModuleHandleT addIRModuleEager(std::shared_ptr M, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return addIRModule(CompileLayer, std::move(M), + LLVMOrcErrorCode addIRModuleEager(ModuleHandleT &RetHandle, + std::shared_ptr M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(RetHandle, CompileLayer, std::move(M), llvm::make_unique(), std::move(ExternalResolver), ExternalResolverCtx); } - ModuleHandleT addIRModuleLazy(std::shared_ptr M, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return addIRModule(CODLayer, std::move(M), + LLVMOrcErrorCode addIRModuleLazy(ModuleHandleT &RetHandle, + std::shared_ptr M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(RetHandle, CODLayer, std::move(M), llvm::make_unique(), std::move(ExternalResolver), ExternalResolverCtx); } - void removeModule(ModuleHandleT H) { - GenericHandles[H]->removeModule(); + LLVMOrcErrorCode removeModule(ModuleHandleT H) { + if (auto Err = GenericHandles[H]->removeModule()) + return mapError(std::move(Err)); GenericHandles[H] = nullptr; FreeHandleIndexes.push_back(H); + return LLVMOrcErrSuccess; } - JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { + JITSymbol findSymbol(const std::string &Name, + bool ExportedSymbolsOnly) { if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly)) return Sym; return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); @@ -251,6 +275,26 @@ class OrcCBindingsStack { return GenericHandles[H]->findSymbolIn(Name, ExportedSymbolsOnly); } + LLVMOrcErrorCode findSymbolAddress(JITTargetAddress &RetAddr, + const std::string &Name, + bool ExportedSymbolsOnly) { + RetAddr = 0; + if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) { + RetAddr = *AddrOrErr; + return LLVMOrcErrSuccess; + } else + return mapError(AddrOrErr.takeError()); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return mapError(std::move(Err)); + } + // Otherwise we had a successful lookup but got a null result. We already + // set RetAddr to '0' above, so just return success. + return LLVMOrcErrSuccess; + } + const std::string &getErrorMessage() const { return ErrMsg; } private: diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp index 9e70c4ac1dbf..df2d320e0f7a 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp @@ -45,6 +45,8 @@ class OrcErrorCategory : public std::error_category { return "Could not negotiate RPC function"; case OrcErrorCode::RPCResponseAbandoned: return "RPC response abandoned"; + case OrcErrorCode::JITSymbolNotFound: + return "JIT symbol not found"; case OrcErrorCode::UnexpectedRPCCall: return "Unexpected RPC call"; case OrcErrorCode::UnexpectedRPCResponse: @@ -63,10 +65,29 @@ static ManagedStatic OrcErrCat; namespace llvm { namespace orc { +char JITSymbolNotFound::ID = 0; + std::error_code orcError(OrcErrorCode ErrCode) { typedef std::underlying_type::type UT; return std::error_code(static_cast(ErrCode), *OrcErrCat); } +JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code JITSymbolNotFound::convertToErrorCode() const { + typedef std::underlying_type::type UT; + return std::error_code(static_cast(OrcErrorCode::JITSymbolNotFound), + *OrcErrCat); +} + +void JITSymbolNotFound::log(raw_ostream &OS) const { + OS << "Could not find symbol '" << SymbolName << "'"; +} + +const std::string &JITSymbolNotFound::getSymbolName() const { + return SymbolName; +} + } } diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 690276232a6f..346a40405ff1 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -172,10 +172,13 @@ class OrcMCJITReplacement : public ExecutionEngine { std::shared_ptr ClientResolver, std::unique_ptr TM) : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), - MemMgr(*this, std::move(MemMgr)), Resolver(*this), + MemMgr(std::make_shared(*this, + std::move(MemMgr))), + Resolver(std::make_shared(*this)), ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), NotifyFinalized(*this), - ObjectLayer(NotifyObjectLoaded, NotifyFinalized), + ObjectLayer([this]() { return this->MemMgr; }, NotifyObjectLoaded, + NotifyFinalized), CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)), LazyEmitLayer(CompileLayer) {} @@ -199,20 +202,20 @@ class OrcMCJITReplacement : public ExecutionEngine { delete Mod; }; LocalModules.push_back(std::shared_ptr(MPtr, std::move(Deleter))); - LazyEmitLayer.addModule(LocalModules.back(), &MemMgr, &Resolver); + cantFail(LazyEmitLayer.addModule(LocalModules.back(), Resolver)); } void addObjectFile(std::unique_ptr O) override { auto Obj = std::make_shared>(std::move(O), nullptr); - ObjectLayer.addObject(std::move(Obj), &MemMgr, &Resolver); + cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); } void addObjectFile(object::OwningBinary O) override { auto Obj = std::make_shared>(std::move(O)); - ObjectLayer.addObject(std::move(Obj), &MemMgr, &Resolver); + cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); } void addArchive(object::OwningBinary A) override { @@ -231,7 +234,7 @@ class OrcMCJITReplacement : public ExecutionEngine { } uint64_t getSymbolAddress(StringRef Name) { - return findSymbol(Name).getAddress(); + return cantFail(findSymbol(Name).getAddress()); } JITSymbol findSymbol(StringRef Name) { @@ -320,7 +323,7 @@ class OrcMCJITReplacement : public ExecutionEngine { auto Obj = std::make_shared>( std::move(ChildObj), nullptr); - ObjectLayer.addObject(std::move(Obj), &MemMgr, &Resolver); + cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); if (auto Sym = ObjectLayer.findSymbol(Name, true)) return Sym; } @@ -341,7 +344,7 @@ class OrcMCJITReplacement : public ExecutionEngine { const LoadedObjectInfo &Info) const { M.UnfinalizedSections[H] = std::move(M.SectionsAllocatedSinceLastLoad); M.SectionsAllocatedSinceLastLoad = SectionAddrSet(); - M.MemMgr.notifyObjectLoaded(&M, *Obj->getBinary()); + M.MemMgr->notifyObjectLoaded(&M, *Obj->getBinary()); } private: OrcMCJITReplacement &M; @@ -373,8 +376,8 @@ class OrcMCJITReplacement : public ExecutionEngine { using LazyEmitLayerT = LazyEmittingLayer; std::unique_ptr TM; - MCJITReplacementMemMgr MemMgr; - LinkingResolver Resolver; + std::shared_ptr MemMgr; + std::shared_ptr Resolver; std::shared_ptr ClientResolver; Mangler Mang; diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 2b69f1a0269f..8198836f7a0c 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -128,7 +128,10 @@ void RuntimeDyldImpl::resolveRelocations() { ); // First, resolve relocations associated with external symbols. - resolveExternalSymbols(); + if (auto Err = resolveExternalSymbols()) { + HasError = true; + ErrorStr = toString(std::move(Err)); + } // Iterate over all outstanding relocations for (auto it = Relocations.begin(), e = Relocations.end(); it != e; ++it) { @@ -243,9 +246,11 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { continue; // Then check the symbol resolver to see if there's a definition // elsewhere in this logical dylib. - if (auto Sym = Resolver.findSymbolInLogicalDylib(Name)) + if (auto Sym = Resolver.findSymbolInLogicalDylib(Name)) { if (Sym.getFlags().isStrongDefinition()) continue; + } else if (auto Err = Sym.takeError()) + return std::move(Err); // else JITSymFlags &= ~JITSymbolFlags::Weak; } @@ -953,7 +958,7 @@ void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, } } -void RuntimeDyldImpl::resolveExternalSymbols() { +Error RuntimeDyldImpl::resolveExternalSymbols() { while (!ExternalSymbolRelocations.empty()) { StringMap::iterator i = ExternalSymbolRelocations.begin(); @@ -971,10 +976,24 @@ void RuntimeDyldImpl::resolveExternalSymbols() { // This is an external symbol, try to get its address from the symbol // resolver. // First search for the symbol in this logical dylib. - Addr = Resolver.findSymbolInLogicalDylib(Name.data()).getAddress(); + if (auto Sym = Resolver.findSymbolInLogicalDylib(Name.data())) { + if (auto AddrOrErr = Sym.getAddress()) + Addr = *AddrOrErr; + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) + return Err; + // If that fails, try searching for an external symbol. - if (!Addr) - Addr = Resolver.findSymbol(Name.data()).getAddress(); + if (!Addr) { + if (auto Sym = Resolver.findSymbol(Name.data())) { + if (auto AddrOrErr = Sym.getAddress()) + Addr = *AddrOrErr; + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) + return Err; + } // The call to getSymbolAddress may have caused additional modules to // be loaded, which may have added new entries to the // ExternalSymbolRelocations map. Consquently, we need to update our @@ -1009,6 +1028,8 @@ void RuntimeDyldImpl::resolveExternalSymbols() { ExternalSymbolRelocations.erase(i); } + + return Error::success(); } //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 1bd28ef37ed1..1c54ad6fb03f 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -27,9 +27,12 @@ using namespace llvm::object; namespace { class LoadedCOFFObjectInfo final - : public RuntimeDyld::LoadedObjectInfoHelper { + : public LoadedObjectInfoHelper { public: - LoadedCOFFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) + LoadedCOFFObjectInfo( + RuntimeDyldImpl &RTDyld, + RuntimeDyld::LoadedObjectInfo::ObjSectionToIDMap ObjSecToIDMap) : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {} OwningBinary diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index e45fdc7aee18..5bc7434e703f 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -742,7 +742,7 @@ uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { if (auto InternalSymbol = getRTDyld().getSymbol(Symbol)) return InternalSymbol.getAddress(); - return getRTDyld().Resolver.findSymbol(Symbol).getAddress(); + return cantFail(getRTDyld().Resolver.findSymbol(Symbol).getAddress()); } uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 8b6f9bef66df..77c968401c16 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -123,7 +123,8 @@ void DyldELFObject::updateSymbolAddress(const SymbolRef &SymRef, } class LoadedELFObjectInfo final - : public RuntimeDyld::LoadedObjectInfoHelper { + : public LoadedObjectInfoHelper { public: LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {} diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 5268bc5a1868..95b04fd93251 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -417,7 +417,7 @@ class RuntimeDyldImpl { StubMap &Stubs) = 0; /// \brief Resolve relocations to external symbols. - void resolveExternalSymbols(); + Error resolveExternalSymbols(); // \brief Compute an upper bound of the memory that is required to load all // sections diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index 00541e8c06fe..80e9c7ac18aa 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -27,7 +27,8 @@ using namespace llvm::object; namespace { class LoadedMachOObjectInfo final - : public RuntimeDyld::LoadedObjectInfoHelper { + : public LoadedObjectInfoHelper { public: LoadedMachOObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) diff --git a/contrib/llvm/lib/IR/AsmWriter.cpp b/contrib/llvm/lib/IR/AsmWriter.cpp index c7f112887a30..80371780fb6d 100644 --- a/contrib/llvm/lib/IR/AsmWriter.cpp +++ b/contrib/llvm/lib/IR/AsmWriter.cpp @@ -2119,6 +2119,8 @@ class AssemblyWriter { bool ShouldPreserveUseListOrder; UseListOrderStack UseListOrders; SmallVector MDNames; + /// Synchronization scope names registered with LLVMContext. + SmallVector SSNs; public: /// Construct an AssemblyWriter with an external SlotTracker @@ -2134,10 +2136,15 @@ class AssemblyWriter { void writeOperand(const Value *Op, bool PrintType); void writeParamOperand(const Value *Operand, AttributeSet Attrs); void writeOperandBundles(ImmutableCallSite CS); - void writeAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope); - void writeAtomicCmpXchg(AtomicOrdering SuccessOrdering, + void writeSyncScope(const LLVMContext &Context, + SyncScope::ID SSID); + void writeAtomic(const LLVMContext &Context, + AtomicOrdering Ordering, + SyncScope::ID SSID); + void writeAtomicCmpXchg(const LLVMContext &Context, + AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); + SyncScope::ID SSID); void writeAllMDNodes(); void writeMDNode(unsigned Slot, const MDNode *Node); @@ -2199,30 +2206,42 @@ void AssemblyWriter::writeOperand(const Value *Operand, bool PrintType) { WriteAsOperandInternal(Out, Operand, &TypePrinter, &Machine, TheModule); } -void AssemblyWriter::writeAtomic(AtomicOrdering Ordering, - SynchronizationScope SynchScope) { +void AssemblyWriter::writeSyncScope(const LLVMContext &Context, + SyncScope::ID SSID) { + switch (SSID) { + case SyncScope::System: { + break; + } + default: { + if (SSNs.empty()) + Context.getSyncScopeNames(SSNs); + + Out << " syncscope(\""; + PrintEscapedString(SSNs[SSID], Out); + Out << "\")"; + break; + } + } +} + +void AssemblyWriter::writeAtomic(const LLVMContext &Context, + AtomicOrdering Ordering, + SyncScope::ID SSID) { if (Ordering == AtomicOrdering::NotAtomic) return; - switch (SynchScope) { - case SingleThread: Out << " singlethread"; break; - case CrossThread: break; - } - + writeSyncScope(Context, SSID); Out << " " << toIRString(Ordering); } -void AssemblyWriter::writeAtomicCmpXchg(AtomicOrdering SuccessOrdering, +void AssemblyWriter::writeAtomicCmpXchg(const LLVMContext &Context, + AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { + SyncScope::ID SSID) { assert(SuccessOrdering != AtomicOrdering::NotAtomic && FailureOrdering != AtomicOrdering::NotAtomic); - switch (SynchScope) { - case SingleThread: Out << " singlethread"; break; - case CrossThread: break; - } - + writeSyncScope(Context, SSID); Out << " " << toIRString(SuccessOrdering); Out << " " << toIRString(FailureOrdering); } @@ -3215,21 +3234,22 @@ void AssemblyWriter::printInstruction(const Instruction &I) { // Print atomic ordering/alignment for memory operations if (const LoadInst *LI = dyn_cast(&I)) { if (LI->isAtomic()) - writeAtomic(LI->getOrdering(), LI->getSynchScope()); + writeAtomic(LI->getContext(), LI->getOrdering(), LI->getSyncScopeID()); if (LI->getAlignment()) Out << ", align " << LI->getAlignment(); } else if (const StoreInst *SI = dyn_cast(&I)) { if (SI->isAtomic()) - writeAtomic(SI->getOrdering(), SI->getSynchScope()); + writeAtomic(SI->getContext(), SI->getOrdering(), SI->getSyncScopeID()); if (SI->getAlignment()) Out << ", align " << SI->getAlignment(); } else if (const AtomicCmpXchgInst *CXI = dyn_cast(&I)) { - writeAtomicCmpXchg(CXI->getSuccessOrdering(), CXI->getFailureOrdering(), - CXI->getSynchScope()); + writeAtomicCmpXchg(CXI->getContext(), CXI->getSuccessOrdering(), + CXI->getFailureOrdering(), CXI->getSyncScopeID()); } else if (const AtomicRMWInst *RMWI = dyn_cast(&I)) { - writeAtomic(RMWI->getOrdering(), RMWI->getSynchScope()); + writeAtomic(RMWI->getContext(), RMWI->getOrdering(), + RMWI->getSyncScopeID()); } else if (const FenceInst *FI = dyn_cast(&I)) { - writeAtomic(FI->getOrdering(), FI->getSynchScope()); + writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID()); } // Print Metadata info. diff --git a/contrib/llvm/lib/IR/ConstantFold.cpp b/contrib/llvm/lib/IR/ConstantFold.cpp index 3469026ad7ed..23ccd8d4cf42 100644 --- a/contrib/llvm/lib/IR/ConstantFold.cpp +++ b/contrib/llvm/lib/IR/ConstantFold.cpp @@ -242,7 +242,7 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, // X | -1 -> -1. if (ConstantInt *RHSC = dyn_cast(RHS)) - if (RHSC->isAllOnesValue()) + if (RHSC->isMinusOne()) return RHSC; Constant *LHS = ExtractConstantBytes(CE->getOperand(0), ByteStart,ByteSize); @@ -1015,33 +1015,33 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, if (ConstantInt *CI2 = dyn_cast(C2)) { switch (Opcode) { case Instruction::Add: - if (CI2->equalsInt(0)) return C1; // X + 0 == X + if (CI2->isZero()) return C1; // X + 0 == X break; case Instruction::Sub: - if (CI2->equalsInt(0)) return C1; // X - 0 == X + if (CI2->isZero()) return C1; // X - 0 == X break; case Instruction::Mul: - if (CI2->equalsInt(0)) return C2; // X * 0 == 0 - if (CI2->equalsInt(1)) + if (CI2->isZero()) return C2; // X * 0 == 0 + if (CI2->isOne()) return C1; // X * 1 == X break; case Instruction::UDiv: case Instruction::SDiv: - if (CI2->equalsInt(1)) + if (CI2->isOne()) return C1; // X / 1 == X - if (CI2->equalsInt(0)) + if (CI2->isZero()) return UndefValue::get(CI2->getType()); // X / 0 == undef break; case Instruction::URem: case Instruction::SRem: - if (CI2->equalsInt(1)) + if (CI2->isOne()) return Constant::getNullValue(CI2->getType()); // X % 1 == 0 - if (CI2->equalsInt(0)) + if (CI2->isZero()) return UndefValue::get(CI2->getType()); // X % 0 == undef break; case Instruction::And: if (CI2->isZero()) return C2; // X & 0 == 0 - if (CI2->isAllOnesValue()) + if (CI2->isMinusOne()) return C1; // X & -1 == X if (ConstantExpr *CE1 = dyn_cast(C1)) { @@ -1078,12 +1078,12 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, } break; case Instruction::Or: - if (CI2->equalsInt(0)) return C1; // X | 0 == X - if (CI2->isAllOnesValue()) + if (CI2->isZero()) return C1; // X | 0 == X + if (CI2->isMinusOne()) return C2; // X | -1 == -1 break; case Instruction::Xor: - if (CI2->equalsInt(0)) return C1; // X ^ 0 == X + if (CI2->isZero()) return C1; // X ^ 0 == X if (ConstantExpr *CE1 = dyn_cast(C1)) { switch (CE1->getOpcode()) { @@ -1091,7 +1091,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, case Instruction::ICmp: case Instruction::FCmp: // cmp pred ^ true -> cmp !pred - assert(CI2->equalsInt(1)); + assert(CI2->isOne()); CmpInst::Predicate pred = (CmpInst::Predicate)CE1->getPredicate(); pred = CmpInst::getInversePredicate(pred); return ConstantExpr::getCompare(pred, CE1->getOperand(0), @@ -1126,18 +1126,18 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, case Instruction::Mul: return ConstantInt::get(CI1->getContext(), C1V * C2V); case Instruction::UDiv: - assert(!CI2->isNullValue() && "Div by zero handled above"); + assert(!CI2->isZero() && "Div by zero handled above"); return ConstantInt::get(CI1->getContext(), C1V.udiv(C2V)); case Instruction::SDiv: - assert(!CI2->isNullValue() && "Div by zero handled above"); + assert(!CI2->isZero() && "Div by zero handled above"); if (C2V.isAllOnesValue() && C1V.isMinSignedValue()) return UndefValue::get(CI1->getType()); // MIN_INT / -1 -> undef return ConstantInt::get(CI1->getContext(), C1V.sdiv(C2V)); case Instruction::URem: - assert(!CI2->isNullValue() && "Div by zero handled above"); + assert(!CI2->isZero() && "Div by zero handled above"); return ConstantInt::get(CI1->getContext(), C1V.urem(C2V)); case Instruction::SRem: - assert(!CI2->isNullValue() && "Div by zero handled above"); + assert(!CI2->isZero() && "Div by zero handled above"); if (C2V.isAllOnesValue() && C1V.isMinSignedValue()) return UndefValue::get(CI1->getType()); // MIN_INT % -1 -> undef return ConstantInt::get(CI1->getContext(), C1V.srem(C2V)); @@ -1170,7 +1170,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, case Instruction::LShr: case Instruction::AShr: case Instruction::Shl: - if (CI1->equalsInt(0)) return C1; + if (CI1->isZero()) return C1; break; default: break; diff --git a/contrib/llvm/lib/IR/Constants.cpp b/contrib/llvm/lib/IR/Constants.cpp index d387a6f0ecb9..e31779c83e3a 100644 --- a/contrib/llvm/lib/IR/Constants.cpp +++ b/contrib/llvm/lib/IR/Constants.cpp @@ -512,7 +512,7 @@ ConstantInt *ConstantInt::getFalse(LLVMContext &Context) { } Constant *ConstantInt::getTrue(Type *Ty) { - assert(Ty->getScalarType()->isIntegerTy(1) && "Type not i1 or vector of i1."); + assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1."); ConstantInt *TrueC = ConstantInt::getTrue(Ty->getContext()); if (auto *VTy = dyn_cast(Ty)) return ConstantVector::getSplat(VTy->getNumElements(), TrueC); @@ -520,7 +520,7 @@ Constant *ConstantInt::getTrue(Type *Ty) { } Constant *ConstantInt::getFalse(Type *Ty) { - assert(Ty->getScalarType()->isIntegerTy(1) && "Type not i1 or vector of i1."); + assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1."); ConstantInt *FalseC = ConstantInt::getFalse(Ty->getContext()); if (auto *VTy = dyn_cast(Ty)) return ConstantVector::getSplat(VTy->getNumElements(), FalseC); @@ -1635,9 +1635,9 @@ Constant *ConstantExpr::getFPToSI(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy, bool OnlyIfReduced) { - assert(C->getType()->getScalarType()->isPointerTy() && + assert(C->getType()->isPtrOrPtrVectorTy() && "PtrToInt source must be pointer or pointer vector"); - assert(DstTy->getScalarType()->isIntegerTy() && + assert(DstTy->isIntOrIntVectorTy() && "PtrToInt destination must be integer or integer vector"); assert(isa(C->getType()) == isa(DstTy)); if (isa(C->getType())) @@ -1648,9 +1648,9 @@ Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy, Constant *ConstantExpr::getIntToPtr(Constant *C, Type *DstTy, bool OnlyIfReduced) { - assert(C->getType()->getScalarType()->isIntegerTy() && + assert(C->getType()->isIntOrIntVectorTy() && "IntToPtr source must be integer or integer vector"); - assert(DstTy->getScalarType()->isPointerTy() && + assert(DstTy->isPtrOrPtrVectorTy() && "IntToPtr destination must be a pointer or pointer vector"); assert(isa(C->getType()) == isa(DstTy)); if (isa(C->getType())) @@ -1914,8 +1914,8 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, Constant *ConstantExpr::getICmp(unsigned short pred, Constant *LHS, Constant *RHS, bool OnlyIfReduced) { assert(LHS->getType() == RHS->getType()); - assert(pred >= ICmpInst::FIRST_ICMP_PREDICATE && - pred <= ICmpInst::LAST_ICMP_PREDICATE && "Invalid ICmp Predicate"); + assert(CmpInst::isIntPredicate((CmpInst::Predicate)pred) && + "Invalid ICmp Predicate"); if (Constant *FC = ConstantFoldCompareInstruction(pred, LHS, RHS)) return FC; // Fold a few common cases... @@ -1939,7 +1939,8 @@ Constant *ConstantExpr::getICmp(unsigned short pred, Constant *LHS, Constant *ConstantExpr::getFCmp(unsigned short pred, Constant *LHS, Constant *RHS, bool OnlyIfReduced) { assert(LHS->getType() == RHS->getType()); - assert(pred <= FCmpInst::LAST_FCMP_PREDICATE && "Invalid FCmp Predicate"); + assert(CmpInst::isFPPredicate((CmpInst::Predicate)pred) && + "Invalid FCmp Predicate"); if (Constant *FC = ConstantFoldCompareInstruction(pred, LHS, RHS)) return FC; // Fold a few common cases... @@ -2379,32 +2380,32 @@ void ConstantDataSequential::destroyConstantImpl() { Constant *ConstantDataArray::get(LLVMContext &Context, ArrayRef Elts) { Type *Ty = ArrayType::get(Type::getInt8Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*1), Ty); + return getImpl(StringRef(Data, Elts.size() * 1), Ty); } Constant *ConstantDataArray::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = ArrayType::get(Type::getInt16Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*2), Ty); + return getImpl(StringRef(Data, Elts.size() * 2), Ty); } Constant *ConstantDataArray::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = ArrayType::get(Type::getInt32Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*4), Ty); + return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataArray::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = ArrayType::get(Type::getInt64Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*8), Ty); + return getImpl(StringRef(Data, Elts.size() * 8), Ty); } Constant *ConstantDataArray::get(LLVMContext &Context, ArrayRef Elts) { Type *Ty = ArrayType::get(Type::getFloatTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*4), Ty); + return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataArray::get(LLVMContext &Context, ArrayRef Elts) { Type *Ty = ArrayType::get(Type::getDoubleTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 8), Ty); + return getImpl(StringRef(Data, Elts.size() * 8), Ty); } /// getFP() constructors - Return a constant with array type with an element @@ -2416,27 +2417,26 @@ Constant *ConstantDataArray::getFP(LLVMContext &Context, ArrayRef Elts) { Type *Ty = ArrayType::get(Type::getHalfTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 2), Ty); + return getImpl(StringRef(Data, Elts.size() * 2), Ty); } Constant *ConstantDataArray::getFP(LLVMContext &Context, ArrayRef Elts) { Type *Ty = ArrayType::get(Type::getFloatTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 4), Ty); + return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataArray::getFP(LLVMContext &Context, ArrayRef Elts) { Type *Ty = ArrayType::get(Type::getDoubleTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 8), Ty); + return getImpl(StringRef(Data, Elts.size() * 8), Ty); } Constant *ConstantDataArray::getString(LLVMContext &Context, StringRef Str, bool AddNull) { if (!AddNull) { const uint8_t *Data = reinterpret_cast(Str.data()); - return get(Context, makeArrayRef(const_cast(Data), - Str.size())); + return get(Context, makeArrayRef(Data, Str.size())); } SmallVector ElementVals; @@ -2451,32 +2451,32 @@ Constant *ConstantDataArray::getString(LLVMContext &Context, Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = VectorType::get(Type::getInt8Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*1), Ty); + return getImpl(StringRef(Data, Elts.size() * 1), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = VectorType::get(Type::getInt16Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*2), Ty); + return getImpl(StringRef(Data, Elts.size() * 2), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = VectorType::get(Type::getInt32Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*4), Ty); + return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef Elts){ Type *Ty = VectorType::get(Type::getInt64Ty(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*8), Ty); + return getImpl(StringRef(Data, Elts.size() * 8), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef Elts) { Type *Ty = VectorType::get(Type::getFloatTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size()*4), Ty); + return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef Elts) { Type *Ty = VectorType::get(Type::getDoubleTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 8), Ty); + return getImpl(StringRef(Data, Elts.size() * 8), Ty); } /// getFP() constructors - Return a constant with vector type with an element @@ -2488,19 +2488,19 @@ Constant *ConstantDataVector::getFP(LLVMContext &Context, ArrayRef Elts) { Type *Ty = VectorType::get(Type::getHalfTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 2), Ty); + return getImpl(StringRef(Data, Elts.size() * 2), Ty); } Constant *ConstantDataVector::getFP(LLVMContext &Context, ArrayRef Elts) { Type *Ty = VectorType::get(Type::getFloatTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 4), Ty); + return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataVector::getFP(LLVMContext &Context, ArrayRef Elts) { Type *Ty = VectorType::get(Type::getDoubleTy(Context), Elts.size()); const char *Data = reinterpret_cast(Elts.data()); - return getImpl(StringRef(const_cast(Data), Elts.size() * 8), Ty); + return getImpl(StringRef(Data, Elts.size() * 8), Ty); } Constant *ConstantDataVector::getSplat(unsigned NumElts, Constant *V) { @@ -2555,13 +2555,13 @@ uint64_t ConstantDataSequential::getElementAsInteger(unsigned Elt) const { switch (getElementType()->getIntegerBitWidth()) { default: llvm_unreachable("Invalid bitwidth for CDS"); case 8: - return *const_cast(reinterpret_cast(EltPtr)); + return *reinterpret_cast(EltPtr); case 16: - return *const_cast(reinterpret_cast(EltPtr)); + return *reinterpret_cast(EltPtr); case 32: - return *const_cast(reinterpret_cast(EltPtr)); + return *reinterpret_cast(EltPtr); case 64: - return *const_cast(reinterpret_cast(EltPtr)); + return *reinterpret_cast(EltPtr); } } @@ -2589,16 +2589,13 @@ APFloat ConstantDataSequential::getElementAsAPFloat(unsigned Elt) const { float ConstantDataSequential::getElementAsFloat(unsigned Elt) const { assert(getElementType()->isFloatTy() && "Accessor can only be used when element is a 'float'"); - const float *EltPtr = reinterpret_cast(getElementPointer(Elt)); - return *const_cast(EltPtr); + return *reinterpret_cast(getElementPointer(Elt)); } double ConstantDataSequential::getElementAsDouble(unsigned Elt) const { assert(getElementType()->isDoubleTy() && "Accessor can only be used when element is a 'float'"); - const double *EltPtr = - reinterpret_cast(getElementPointer(Elt)); - return *const_cast(EltPtr); + return *reinterpret_cast(getElementPointer(Elt)); } Constant *ConstantDataSequential::getElementAsConstant(unsigned Elt) const { diff --git a/contrib/llvm/lib/IR/Core.cpp b/contrib/llvm/lib/IR/Core.cpp index 4ff0261a7f08..2165ae5a9470 100644 --- a/contrib/llvm/lib/IR/Core.cpp +++ b/contrib/llvm/lib/IR/Core.cpp @@ -50,6 +50,7 @@ void llvm::initializeCore(PassRegistry &Registry) { initializePrintModulePassWrapperPass(Registry); initializePrintFunctionPassWrapperPass(Registry); initializePrintBasicBlockPassPass(Registry); + initializeSafepointIRVerifierPass(Registry); initializeVerifierLegacyPassPass(Registry); } @@ -2755,11 +2756,14 @@ static LLVMAtomicOrdering mapToLLVMOrdering(AtomicOrdering Ordering) { llvm_unreachable("Invalid AtomicOrdering value!"); } +// TODO: Should this and other atomic instructions support building with +// "syncscope"? LLVMValueRef LLVMBuildFence(LLVMBuilderRef B, LLVMAtomicOrdering Ordering, LLVMBool isSingleThread, const char *Name) { return wrap( unwrap(B)->CreateFence(mapFromLLVMOrdering(Ordering), - isSingleThread ? SingleThread : CrossThread, + isSingleThread ? SyncScope::SingleThread + : SyncScope::System, Name)); } @@ -3041,7 +3045,8 @@ LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,LLVMAtomicRMWBinOp op, case LLVMAtomicRMWBinOpUMin: intop = AtomicRMWInst::UMin; break; } return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR), unwrap(Val), - mapFromLLVMOrdering(ordering), singleThread ? SingleThread : CrossThread)); + mapFromLLVMOrdering(ordering), singleThread ? SyncScope::SingleThread + : SyncScope::System)); } LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr, @@ -3053,7 +3058,7 @@ LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr, return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(Ptr), unwrap(Cmp), unwrap(New), mapFromLLVMOrdering(SuccessOrdering), mapFromLLVMOrdering(FailureOrdering), - singleThread ? SingleThread : CrossThread)); + singleThread ? SyncScope::SingleThread : SyncScope::System)); } @@ -3061,17 +3066,18 @@ LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) { Value *P = unwrap(AtomicInst); if (AtomicRMWInst *I = dyn_cast(P)) - return I->getSynchScope() == SingleThread; - return cast(P)->getSynchScope() == SingleThread; + return I->getSyncScopeID() == SyncScope::SingleThread; + return cast(P)->getSyncScopeID() == + SyncScope::SingleThread; } void LLVMSetAtomicSingleThread(LLVMValueRef AtomicInst, LLVMBool NewValue) { Value *P = unwrap(AtomicInst); - SynchronizationScope Sync = NewValue ? SingleThread : CrossThread; + SyncScope::ID SSID = NewValue ? SyncScope::SingleThread : SyncScope::System; if (AtomicRMWInst *I = dyn_cast(P)) - return I->setSynchScope(Sync); - return cast(P)->setSynchScope(Sync); + return I->setSyncScopeID(SSID); + return cast(P)->setSyncScopeID(SSID); } LLVMAtomicOrdering LLVMGetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst) { diff --git a/contrib/llvm/lib/IR/Instruction.cpp b/contrib/llvm/lib/IR/Instruction.cpp index 3dd653d2d047..365cb019aec4 100644 --- a/contrib/llvm/lib/IR/Instruction.cpp +++ b/contrib/llvm/lib/IR/Instruction.cpp @@ -362,13 +362,13 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, (LI->getAlignment() == cast(I2)->getAlignment() || IgnoreAlignment) && LI->getOrdering() == cast(I2)->getOrdering() && - LI->getSynchScope() == cast(I2)->getSynchScope(); + LI->getSyncScopeID() == cast(I2)->getSyncScopeID(); if (const StoreInst *SI = dyn_cast(I1)) return SI->isVolatile() == cast(I2)->isVolatile() && (SI->getAlignment() == cast(I2)->getAlignment() || IgnoreAlignment) && SI->getOrdering() == cast(I2)->getOrdering() && - SI->getSynchScope() == cast(I2)->getSynchScope(); + SI->getSyncScopeID() == cast(I2)->getSyncScopeID(); if (const CmpInst *CI = dyn_cast(I1)) return CI->getPredicate() == cast(I2)->getPredicate(); if (const CallInst *CI = dyn_cast(I1)) @@ -386,7 +386,7 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, return EVI->getIndices() == cast(I2)->getIndices(); if (const FenceInst *FI = dyn_cast(I1)) return FI->getOrdering() == cast(I2)->getOrdering() && - FI->getSynchScope() == cast(I2)->getSynchScope(); + FI->getSyncScopeID() == cast(I2)->getSyncScopeID(); if (const AtomicCmpXchgInst *CXI = dyn_cast(I1)) return CXI->isVolatile() == cast(I2)->isVolatile() && CXI->isWeak() == cast(I2)->isWeak() && @@ -394,12 +394,13 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, cast(I2)->getSuccessOrdering() && CXI->getFailureOrdering() == cast(I2)->getFailureOrdering() && - CXI->getSynchScope() == cast(I2)->getSynchScope(); + CXI->getSyncScopeID() == + cast(I2)->getSyncScopeID(); if (const AtomicRMWInst *RMWI = dyn_cast(I1)) return RMWI->getOperation() == cast(I2)->getOperation() && RMWI->isVolatile() == cast(I2)->isVolatile() && RMWI->getOrdering() == cast(I2)->getOrdering() && - RMWI->getSynchScope() == cast(I2)->getSynchScope(); + RMWI->getSyncScopeID() == cast(I2)->getSyncScopeID(); return true; } diff --git a/contrib/llvm/lib/IR/Instructions.cpp b/contrib/llvm/lib/IR/Instructions.cpp index a79b00be4ffe..2c49564e328b 100644 --- a/contrib/llvm/lib/IR/Instructions.cpp +++ b/contrib/llvm/lib/IR/Instructions.cpp @@ -1304,34 +1304,34 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, Instruction *InsertBef) : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic, - CrossThread, InsertBef) {} + SyncScope::System, InsertBef) {} LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, BasicBlock *InsertAE) : LoadInst(Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic, - CrossThread, InsertAE) {} + SyncScope::System, InsertAE) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, Instruction *InsertBef) + SyncScope::ID SSID, Instruction *InsertBef) : UnaryInstruction(Ty, Load, Ptr, InsertBef) { assert(Ty == cast(Ptr->getType())->getElementType()); setVolatile(isVolatile); setAlignment(Align); - setAtomic(Order, SynchScope); + setAtomic(Order, SSID); AssertOK(); setName(Name); } LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, + SyncScope::ID SSID, BasicBlock *InsertAE) : UnaryInstruction(cast(Ptr->getType())->getElementType(), Load, Ptr, InsertAE) { setVolatile(isVolatile); setAlignment(Align); - setAtomic(Order, SynchScope); + setAtomic(Order, SSID); AssertOK(); setName(Name); } @@ -1419,16 +1419,16 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, Instruction *InsertBefore) : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, - CrossThread, InsertBefore) {} + SyncScope::System, InsertBefore) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, BasicBlock *InsertAtEnd) : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, - CrossThread, InsertAtEnd) {} + SyncScope::System, InsertAtEnd) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, + SyncScope::ID SSID, Instruction *InsertBefore) : Instruction(Type::getVoidTy(val->getContext()), Store, OperandTraits::op_begin(this), @@ -1438,13 +1438,13 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Op<1>() = addr; setVolatile(isVolatile); setAlignment(Align); - setAtomic(Order, SynchScope); + setAtomic(Order, SSID); AssertOK(); } StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, AtomicOrdering Order, - SynchronizationScope SynchScope, + SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction(Type::getVoidTy(val->getContext()), Store, OperandTraits::op_begin(this), @@ -1454,7 +1454,7 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Op<1>() = addr; setVolatile(isVolatile); setAlignment(Align); - setAtomic(Order, SynchScope); + setAtomic(Order, SSID); AssertOK(); } @@ -1474,13 +1474,13 @@ void StoreInst::setAlignment(unsigned Align) { void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { + SyncScope::ID SSID) { Op<0>() = Ptr; Op<1>() = Cmp; Op<2>() = NewVal; setSuccessOrdering(SuccessOrdering); setFailureOrdering(FailureOrdering); - setSynchScope(SynchScope); + setSyncScopeID(SSID); assert(getOperand(0) && getOperand(1) && getOperand(2) && "All operands must be non-null!"); @@ -1507,25 +1507,25 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope, + SyncScope::ID SSID, Instruction *InsertBefore) : Instruction( StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())), AtomicCmpXchg, OperandTraits::op_begin(this), OperandTraits::operands(this), InsertBefore) { - Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope); + Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SSID); } AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope, + SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction( StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())), AtomicCmpXchg, OperandTraits::op_begin(this), OperandTraits::operands(this), InsertAtEnd) { - Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope); + Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SSID); } //===----------------------------------------------------------------------===// @@ -1534,12 +1534,12 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, AtomicOrdering Ordering, - SynchronizationScope SynchScope) { + SyncScope::ID SSID) { Op<0>() = Ptr; Op<1>() = Val; setOperation(Operation); setOrdering(Ordering); - setSynchScope(SynchScope); + setSyncScopeID(SSID); assert(getOperand(0) && getOperand(1) && "All operands must be non-null!"); @@ -1554,24 +1554,24 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, AtomicOrdering Ordering, - SynchronizationScope SynchScope, + SyncScope::ID SSID, Instruction *InsertBefore) : Instruction(Val->getType(), AtomicRMW, OperandTraits::op_begin(this), OperandTraits::operands(this), InsertBefore) { - Init(Operation, Ptr, Val, Ordering, SynchScope); + Init(Operation, Ptr, Val, Ordering, SSID); } AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, AtomicOrdering Ordering, - SynchronizationScope SynchScope, + SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction(Val->getType(), AtomicRMW, OperandTraits::op_begin(this), OperandTraits::operands(this), InsertAtEnd) { - Init(Operation, Ptr, Val, Ordering, SynchScope); + Init(Operation, Ptr, Val, Ordering, SSID); } //===----------------------------------------------------------------------===// @@ -1579,19 +1579,19 @@ AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, //===----------------------------------------------------------------------===// FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering, - SynchronizationScope SynchScope, + SyncScope::ID SSID, Instruction *InsertBefore) : Instruction(Type::getVoidTy(C), Fence, nullptr, 0, InsertBefore) { setOrdering(Ordering); - setSynchScope(SynchScope); + setSyncScopeID(SSID); } FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering, - SynchronizationScope SynchScope, + SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction(Type::getVoidTy(C), Fence, nullptr, 0, InsertAtEnd) { setOrdering(Ordering); - setSynchScope(SynchScope); + setSyncScopeID(SSID); } //===----------------------------------------------------------------------===// @@ -3064,16 +3064,14 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) { if (VectorType *VT = dyn_cast(SrcTy)) if (VT->getNumElements() != cast(DstTy)->getNumElements()) return false; - return SrcTy->getScalarType()->isPointerTy() && - DstTy->getScalarType()->isIntegerTy(); + return SrcTy->isPtrOrPtrVectorTy() && DstTy->isIntOrIntVectorTy(); case Instruction::IntToPtr: if (isa(SrcTy) != isa(DstTy)) return false; if (VectorType *VT = dyn_cast(SrcTy)) if (VT->getNumElements() != cast(DstTy)->getNumElements()) return false; - return SrcTy->getScalarType()->isIntegerTy() && - DstTy->getScalarType()->isPointerTy(); + return SrcTy->isIntOrIntVectorTy() && DstTy->isPtrOrPtrVectorTy(); case Instruction::BitCast: { PointerType *SrcPtrTy = dyn_cast(SrcTy->getScalarType()); PointerType *DstPtrTy = dyn_cast(DstTy->getScalarType()); @@ -3797,12 +3795,12 @@ AllocaInst *AllocaInst::cloneImpl() const { LoadInst *LoadInst::cloneImpl() const { return new LoadInst(getOperand(0), Twine(), isVolatile(), - getAlignment(), getOrdering(), getSynchScope()); + getAlignment(), getOrdering(), getSyncScopeID()); } StoreInst *StoreInst::cloneImpl() const { return new StoreInst(getOperand(0), getOperand(1), isVolatile(), - getAlignment(), getOrdering(), getSynchScope()); + getAlignment(), getOrdering(), getSyncScopeID()); } @@ -3810,7 +3808,7 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { AtomicCmpXchgInst *Result = new AtomicCmpXchgInst(getOperand(0), getOperand(1), getOperand(2), getSuccessOrdering(), getFailureOrdering(), - getSynchScope()); + getSyncScopeID()); Result->setVolatile(isVolatile()); Result->setWeak(isWeak()); return Result; @@ -3818,14 +3816,14 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { AtomicRMWInst *AtomicRMWInst::cloneImpl() const { AtomicRMWInst *Result = - new AtomicRMWInst(getOperation(),getOperand(0), getOperand(1), - getOrdering(), getSynchScope()); + new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1), + getOrdering(), getSyncScopeID()); Result->setVolatile(isVolatile()); return Result; } FenceInst *FenceInst::cloneImpl() const { - return new FenceInst(getContext(), getOrdering(), getSynchScope()); + return new FenceInst(getContext(), getOrdering(), getSyncScopeID()); } TruncInst *TruncInst::cloneImpl() const { diff --git a/contrib/llvm/lib/IR/LLVMContext.cpp b/contrib/llvm/lib/IR/LLVMContext.cpp index 2e13f362344d..c58459d6d5f5 100644 --- a/contrib/llvm/lib/IR/LLVMContext.cpp +++ b/contrib/llvm/lib/IR/LLVMContext.cpp @@ -81,6 +81,18 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { assert(GCTransitionEntry->second == LLVMContext::OB_gc_transition && "gc-transition operand bundle id drifted!"); (void)GCTransitionEntry; + + SyncScope::ID SingleThreadSSID = + pImpl->getOrInsertSyncScopeID("singlethread"); + assert(SingleThreadSSID == SyncScope::SingleThread && + "singlethread synchronization scope ID drifted!"); + (void)SingleThreadSSID; + + SyncScope::ID SystemSSID = + pImpl->getOrInsertSyncScopeID(""); + assert(SystemSSID == SyncScope::System && + "system synchronization scope ID drifted!"); + (void)SystemSSID; } LLVMContext::~LLVMContext() { delete pImpl; } @@ -255,6 +267,14 @@ uint32_t LLVMContext::getOperandBundleTagID(StringRef Tag) const { return pImpl->getOperandBundleTagID(Tag); } +SyncScope::ID LLVMContext::getOrInsertSyncScopeID(StringRef SSN) { + return pImpl->getOrInsertSyncScopeID(SSN); +} + +void LLVMContext::getSyncScopeNames(SmallVectorImpl &SSNs) const { + pImpl->getSyncScopeNames(SSNs); +} + void LLVMContext::setGC(const Function &Fn, std::string GCName) { auto It = pImpl->GCNames.find(&Fn); diff --git a/contrib/llvm/lib/IR/LLVMContextImpl.cpp b/contrib/llvm/lib/IR/LLVMContextImpl.cpp index c19e1be44fdc..57dd08b36fe7 100644 --- a/contrib/llvm/lib/IR/LLVMContextImpl.cpp +++ b/contrib/llvm/lib/IR/LLVMContextImpl.cpp @@ -205,6 +205,20 @@ uint32_t LLVMContextImpl::getOperandBundleTagID(StringRef Tag) const { return I->second; } +SyncScope::ID LLVMContextImpl::getOrInsertSyncScopeID(StringRef SSN) { + auto NewSSID = SSC.size(); + assert(NewSSID < std::numeric_limits::max() && + "Hit the maximum number of synchronization scopes allowed!"); + return SSC.insert(std::make_pair(SSN, SyncScope::ID(NewSSID))).first->second; +} + +void LLVMContextImpl::getSyncScopeNames( + SmallVectorImpl &SSNs) const { + SSNs.resize(SSC.size()); + for (const auto &SSE : SSC) + SSNs[SSE.second] = SSE.first(); +} + /// Singleton instance of the OptBisect class. /// /// This singleton is accessed via the LLVMContext::getOptBisect() function. It diff --git a/contrib/llvm/lib/IR/LLVMContextImpl.h b/contrib/llvm/lib/IR/LLVMContextImpl.h index 395beb57fe37..e413a4f34432 100644 --- a/contrib/llvm/lib/IR/LLVMContextImpl.h +++ b/contrib/llvm/lib/IR/LLVMContextImpl.h @@ -1297,6 +1297,20 @@ class LLVMContextImpl { void getOperandBundleTags(SmallVectorImpl &Tags) const; uint32_t getOperandBundleTagID(StringRef Tag) const; + /// A set of interned synchronization scopes. The StringMap maps + /// synchronization scope names to their respective synchronization scope IDs. + StringMap SSC; + + /// getOrInsertSyncScopeID - Maps synchronization scope name to + /// synchronization scope ID. Every synchronization scope registered with + /// LLVMContext has unique ID except pre-defined ones. + SyncScope::ID getOrInsertSyncScopeID(StringRef SSN); + + /// getSyncScopeNames - Populates client supplied SmallVector with + /// synchronization scope names registered with LLVMContext. Synchronization + /// scope names are ordered by increasing synchronization scope IDs. + void getSyncScopeNames(SmallVectorImpl &SSNs) const; + /// Maintain the GC name for each function. /// /// This saves allocating an additional word in Function for programs which diff --git a/contrib/llvm/lib/IR/Module.cpp b/contrib/llvm/lib/IR/Module.cpp index f8853ed169c5..fdc7de6eaa34 100644 --- a/contrib/llvm/lib/IR/Module.cpp +++ b/contrib/llvm/lib/IR/Module.cpp @@ -88,7 +88,7 @@ Module::~Module() { delete static_cast *>(NamedMDSymTab); } -RandomNumberGenerator *Module::createRNG(const Pass* P) const { +std::unique_ptr Module::createRNG(const Pass* P) const { SmallString<32> Salt(P->getPassName()); // This RNG is guaranteed to produce the same random stream only @@ -103,7 +103,7 @@ RandomNumberGenerator *Module::createRNG(const Pass* P) const { // store salt metadata from the Module constructor. Salt += sys::path::filename(getModuleIdentifier()); - return new RandomNumberGenerator(Salt); + return std::unique_ptr{new RandomNumberGenerator(Salt)}; } /// getNamedValue - Return the first global value in the module with diff --git a/contrib/llvm/lib/IR/SafepointIRVerifier.cpp b/contrib/llvm/lib/IR/SafepointIRVerifier.cpp new file mode 100644 index 000000000000..8b328c221da3 --- /dev/null +++ b/contrib/llvm/lib/IR/SafepointIRVerifier.cpp @@ -0,0 +1,437 @@ +//===-- SafepointIRVerifier.cpp - Verify gc.statepoint invariants ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Run a sanity check on the IR to ensure that Safepoints - if they've been +// inserted - were inserted correctly. In particular, look for use of +// non-relocated values after a safepoint. It's primary use is to check the +// correctness of safepoint insertion immediately after insertion, but it can +// also be used to verify that later transforms have not found a way to break +// safepoint semenatics. +// +// In its current form, this verify checks a property which is sufficient, but +// not neccessary for correctness. There are some cases where an unrelocated +// pointer can be used after the safepoint. Consider this example: +// +// a = ... +// b = ... +// (a',b') = safepoint(a,b) +// c = cmp eq a b +// br c, ..., .... +// +// Because it is valid to reorder 'c' above the safepoint, this is legal. In +// practice, this is a somewhat uncommon transform, but CodeGenPrep does create +// idioms like this. The verifier knows about these cases and avoids reporting +// false positives. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/SafepointIRVerifier.h" +#include "llvm/IR/Statepoint.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "safepoint-ir-verifier" + +using namespace llvm; + +/// This option is used for writing test cases. Instead of crashing the program +/// when verification fails, report a message to the console (for FileCheck +/// usage) and continue execution as if nothing happened. +static cl::opt PrintOnly("safepoint-ir-verifier-print-only", + cl::init(false)); + +static void Verify(const Function &F, const DominatorTree &DT); + +struct SafepointIRVerifier : public FunctionPass { + static char ID; // Pass identification, replacement for typeid + DominatorTree DT; + SafepointIRVerifier() : FunctionPass(ID) { + initializeSafepointIRVerifierPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override { + DT.recalculate(F); + Verify(F, DT); + return false; // no modifications + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } + + StringRef getPassName() const override { return "safepoint verifier"; } +}; + +void llvm::verifySafepointIR(Function &F) { + SafepointIRVerifier pass; + pass.runOnFunction(F); +} + +char SafepointIRVerifier::ID = 0; + +FunctionPass *llvm::createSafepointIRVerifierPass() { + return new SafepointIRVerifier(); +} + +INITIALIZE_PASS_BEGIN(SafepointIRVerifier, "verify-safepoint-ir", + "Safepoint IR Verifier", false, true) +INITIALIZE_PASS_END(SafepointIRVerifier, "verify-safepoint-ir", + "Safepoint IR Verifier", false, true) + +static bool isGCPointerType(Type *T) { + if (auto *PT = dyn_cast(T)) + // For the sake of this example GC, we arbitrarily pick addrspace(1) as our + // GC managed heap. We know that a pointer into this heap needs to be + // updated and that no other pointer does. + return (1 == PT->getAddressSpace()); + return false; +} + +static bool containsGCPtrType(Type *Ty) { + if (isGCPointerType(Ty)) + return true; + if (VectorType *VT = dyn_cast(Ty)) + return isGCPointerType(VT->getScalarType()); + if (ArrayType *AT = dyn_cast(Ty)) + return containsGCPtrType(AT->getElementType()); + if (StructType *ST = dyn_cast(Ty)) + return std::any_of(ST->subtypes().begin(), ST->subtypes().end(), + containsGCPtrType); + return false; +} + +// Debugging aid -- prints a [Begin, End) range of values. +template +static void PrintValueSet(raw_ostream &OS, IteratorTy Begin, IteratorTy End) { + OS << "[ "; + while (Begin != End) { + OS << **Begin << " "; + ++Begin; + } + OS << "]"; +} + +/// The verifier algorithm is phrased in terms of availability. The set of +/// values "available" at a given point in the control flow graph is the set of +/// correctly relocated value at that point, and is a subset of the set of +/// definitions dominating that point. + +/// State we compute and track per basic block. +struct BasicBlockState { + // Set of values available coming in, before the phi nodes + DenseSet AvailableIn; + + // Set of values available going out + DenseSet AvailableOut; + + // AvailableOut minus AvailableIn. + // All elements are Instructions + DenseSet Contribution; + + // True if this block contains a safepoint and thus AvailableIn does not + // contribute to AvailableOut. + bool Cleared = false; +}; + + +/// Gather all the definitions dominating the start of BB into Result. This is +/// simply the Defs introduced by every dominating basic block and the function +/// arguments. +static void GatherDominatingDefs(const BasicBlock *BB, + DenseSet &Result, + const DominatorTree &DT, + DenseMap &BlockMap) { + DomTreeNode *DTN = DT[const_cast(BB)]; + + while (DTN->getIDom()) { + DTN = DTN->getIDom(); + const auto &Defs = BlockMap[DTN->getBlock()]->Contribution; + Result.insert(Defs.begin(), Defs.end()); + // If this block is 'Cleared', then nothing LiveIn to this block can be + // available after this block completes. Note: This turns out to be + // really important for reducing memory consuption of the initial available + // sets and thus peak memory usage by this verifier. + if (BlockMap[DTN->getBlock()]->Cleared) + return; + } + + for (const Argument &A : BB->getParent()->args()) + if (containsGCPtrType(A.getType())) + Result.insert(&A); +} + +/// Model the effect of an instruction on the set of available values. +static void TransferInstruction(const Instruction &I, bool &Cleared, + DenseSet &Available) { + if (isStatepoint(I)) { + Cleared = true; + Available.clear(); + } else if (containsGCPtrType(I.getType())) + Available.insert(&I); +} + +/// Compute the AvailableOut set for BB, based on the +/// BasicBlockState BBS, which is the BasicBlockState for BB. FirstPass is set +/// when the verifier runs for the first time computing the AvailableOut set +/// for BB. +static void TransferBlock(const BasicBlock *BB, + BasicBlockState &BBS, bool FirstPass) { + + const DenseSet &AvailableIn = BBS.AvailableIn; + DenseSet &AvailableOut = BBS.AvailableOut; + + if (BBS.Cleared) { + // AvailableOut does not change no matter how the input changes, just + // leave it be. We need to force this calculation the first time so that + // we have a AvailableOut at all. + if (FirstPass) { + AvailableOut = BBS.Contribution; + } + } else { + // Otherwise, we need to reduce the AvailableOut set by things which are no + // longer in our AvailableIn + DenseSet Temp = BBS.Contribution; + set_union(Temp, AvailableIn); + AvailableOut = std::move(Temp); + } + + DEBUG(dbgs() << "Transfered block " << BB->getName() << " from "; + PrintValueSet(dbgs(), AvailableIn.begin(), AvailableIn.end()); + dbgs() << " to "; + PrintValueSet(dbgs(), AvailableOut.begin(), AvailableOut.end()); + dbgs() << "\n";); +} + +/// A given derived pointer can have multiple base pointers through phi/selects. +/// This type indicates when the base pointer is exclusively constant +/// (ExclusivelySomeConstant), and if that constant is proven to be exclusively +/// null, we record that as ExclusivelyNull. In all other cases, the BaseType is +/// NonConstant. +enum BaseType { + NonConstant = 1, // Base pointers is not exclusively constant. + ExclusivelyNull, + ExclusivelySomeConstant // Base pointers for a given derived pointer is from a + // set of constants, but they are not exclusively + // null. +}; + +/// Return the baseType for Val which states whether Val is exclusively +/// derived from constant/null, or not exclusively derived from constant. +/// Val is exclusively derived off a constant base when all operands of phi and +/// selects are derived off a constant base. +static enum BaseType getBaseType(const Value *Val) { + + SmallVector Worklist; + DenseSet Visited; + bool isExclusivelyDerivedFromNull = true; + Worklist.push_back(Val); + // Strip through all the bitcasts and geps to get base pointer. Also check for + // the exclusive value when there can be multiple base pointers (through phis + // or selects). + while(!Worklist.empty()) { + const Value *V = Worklist.pop_back_val(); + if (!Visited.insert(V).second) + continue; + + if (const auto *CI = dyn_cast(V)) { + Worklist.push_back(CI->stripPointerCasts()); + continue; + } + if (const auto *GEP = dyn_cast(V)) { + Worklist.push_back(GEP->getPointerOperand()); + continue; + } + // Push all the incoming values of phi node into the worklist for + // processing. + if (const auto *PN = dyn_cast(V)) { + for (Value *InV: PN->incoming_values()) + Worklist.push_back(InV); + continue; + } + if (const auto *SI = dyn_cast(V)) { + // Push in the true and false values + Worklist.push_back(SI->getTrueValue()); + Worklist.push_back(SI->getFalseValue()); + continue; + } + if (isa(V)) { + // We found at least one base pointer which is non-null, so this derived + // pointer is not exclusively derived from null. + if (V != Constant::getNullValue(V->getType())) + isExclusivelyDerivedFromNull = false; + // Continue processing the remaining values to make sure it's exclusively + // constant. + continue; + } + // At this point, we know that the base pointer is not exclusively + // constant. + return BaseType::NonConstant; + } + // Now, we know that the base pointer is exclusively constant, but we need to + // differentiate between exclusive null constant and non-null constant. + return isExclusivelyDerivedFromNull ? BaseType::ExclusivelyNull + : BaseType::ExclusivelySomeConstant; +} + +static void Verify(const Function &F, const DominatorTree &DT) { + SpecificBumpPtrAllocator BSAllocator; + DenseMap BlockMap; + + DEBUG(dbgs() << "Verifying gc pointers in function: " << F.getName() << "\n"); + if (PrintOnly) + dbgs() << "Verifying gc pointers in function: " << F.getName() << "\n"; + + + for (const BasicBlock &BB : F) { + BasicBlockState *BBS = new(BSAllocator.Allocate()) BasicBlockState; + for (const auto &I : BB) + TransferInstruction(I, BBS->Cleared, BBS->Contribution); + BlockMap[&BB] = BBS; + } + + for (auto &BBI : BlockMap) { + GatherDominatingDefs(BBI.first, BBI.second->AvailableIn, DT, BlockMap); + TransferBlock(BBI.first, *BBI.second, true); + } + + SetVector Worklist; + for (auto &BBI : BlockMap) + Worklist.insert(BBI.first); + + // This loop iterates the AvailableIn and AvailableOut sets to a fixed point. + // The AvailableIn and AvailableOut sets decrease as we iterate. + while (!Worklist.empty()) { + const BasicBlock *BB = Worklist.pop_back_val(); + BasicBlockState *BBS = BlockMap[BB]; + + size_t OldInCount = BBS->AvailableIn.size(); + for (const BasicBlock *PBB : predecessors(BB)) + set_intersect(BBS->AvailableIn, BlockMap[PBB]->AvailableOut); + + if (OldInCount == BBS->AvailableIn.size()) + continue; + + assert(OldInCount > BBS->AvailableIn.size() && "invariant!"); + + size_t OldOutCount = BBS->AvailableOut.size(); + TransferBlock(BB, *BBS, false); + if (OldOutCount != BBS->AvailableOut.size()) { + assert(OldOutCount > BBS->AvailableOut.size() && "invariant!"); + Worklist.insert(succ_begin(BB), succ_end(BB)); + } + } + + // We now have all the information we need to decide if the use of a heap + // reference is legal or not, given our safepoint semantics. + + bool AnyInvalidUses = false; + + auto ReportInvalidUse = [&AnyInvalidUses](const Value &V, + const Instruction &I) { + errs() << "Illegal use of unrelocated value found!\n"; + errs() << "Def: " << V << "\n"; + errs() << "Use: " << I << "\n"; + if (!PrintOnly) + abort(); + AnyInvalidUses = true; + }; + + auto isNotExclusivelyConstantDerived = [](const Value *V) { + return getBaseType(V) == BaseType::NonConstant; + }; + + for (const BasicBlock &BB : F) { + // We destructively modify AvailableIn as we traverse the block instruction + // by instruction. + DenseSet &AvailableSet = BlockMap[&BB]->AvailableIn; + for (const Instruction &I : BB) { + if (const PHINode *PN = dyn_cast(&I)) { + if (containsGCPtrType(PN->getType())) + for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { + const BasicBlock *InBB = PN->getIncomingBlock(i); + const Value *InValue = PN->getIncomingValue(i); + + if (isNotExclusivelyConstantDerived(InValue) && + !BlockMap[InBB]->AvailableOut.count(InValue)) + ReportInvalidUse(*InValue, *PN); + } + } else if (isa(I) && + containsGCPtrType(I.getOperand(0)->getType())) { + Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); + enum BaseType baseTyLHS = getBaseType(LHS), + baseTyRHS = getBaseType(RHS); + + // Returns true if LHS and RHS are unrelocated pointers and they are + // valid unrelocated uses. + auto hasValidUnrelocatedUse = [&AvailableSet, baseTyLHS, baseTyRHS, &LHS, &RHS] () { + // A cmp instruction has valid unrelocated pointer operands only if + // both operands are unrelocated pointers. + // In the comparison between two pointers, if one is an unrelocated + // use, the other *should be* an unrelocated use, for this + // instruction to contain valid unrelocated uses. This unrelocated + // use can be a null constant as well, or another unrelocated + // pointer. + if (AvailableSet.count(LHS) || AvailableSet.count(RHS)) + return false; + // Constant pointers (that are not exclusively null) may have + // meaning in different VMs, so we cannot reorder the compare + // against constant pointers before the safepoint. In other words, + // comparison of an unrelocated use against a non-null constant + // maybe invalid. + if ((baseTyLHS == BaseType::ExclusivelySomeConstant && + baseTyRHS == BaseType::NonConstant) || + (baseTyLHS == BaseType::NonConstant && + baseTyRHS == BaseType::ExclusivelySomeConstant)) + return false; + // All other cases are valid cases enumerated below: + // 1. Comparison between an exlusively derived null pointer and a + // constant base pointer. + // 2. Comparison between an exlusively derived null pointer and a + // non-constant unrelocated base pointer. + // 3. Comparison between 2 unrelocated pointers. + return true; + }; + if (!hasValidUnrelocatedUse()) { + // Print out all non-constant derived pointers that are unrelocated + // uses, which are invalid. + if (baseTyLHS == BaseType::NonConstant && !AvailableSet.count(LHS)) + ReportInvalidUse(*LHS, I); + if (baseTyRHS == BaseType::NonConstant && !AvailableSet.count(RHS)) + ReportInvalidUse(*RHS, I); + } + } else { + for (const Value *V : I.operands()) + if (containsGCPtrType(V->getType()) && + isNotExclusivelyConstantDerived(V) && !AvailableSet.count(V)) + ReportInvalidUse(*V, I); + } + + bool Cleared = false; + TransferInstruction(I, Cleared, AvailableSet); + (void)Cleared; + } + } + + if (PrintOnly && !AnyInvalidUses) { + dbgs() << "No illegal uses found by SafepointIRVerifier in: " << F.getName() + << "\n"; + } +} diff --git a/contrib/llvm/lib/IR/Type.cpp b/contrib/llvm/lib/IR/Type.cpp index 44fe5e48c720..20e9c2b5fff2 100644 --- a/contrib/llvm/lib/IR/Type.cpp +++ b/contrib/llvm/lib/IR/Type.cpp @@ -538,7 +538,7 @@ bool CompositeType::indexValid(const Value *V) const { if (auto *STy = dyn_cast(this)) { // Structure indexes require (vectors of) 32-bit integer constants. In the // vector case all of the indices must be equal. - if (!V->getType()->getScalarType()->isIntegerTy(32)) + if (!V->getType()->isIntOrIntVectorTy(32)) return false; const Constant *C = dyn_cast(V); if (C && V->getType()->isVectorTy()) diff --git a/contrib/llvm/lib/IR/Verifier.cpp b/contrib/llvm/lib/IR/Verifier.cpp index 819f63520c74..454a56a76923 100644 --- a/contrib/llvm/lib/IR/Verifier.cpp +++ b/contrib/llvm/lib/IR/Verifier.cpp @@ -2504,15 +2504,13 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) { Type *SrcTy = I.getOperand(0)->getType(); Type *DestTy = I.getType(); - Assert(SrcTy->getScalarType()->isPointerTy(), - "PtrToInt source must be pointer", &I); + Assert(SrcTy->isPtrOrPtrVectorTy(), "PtrToInt source must be pointer", &I); if (auto *PTy = dyn_cast(SrcTy->getScalarType())) Assert(!DL.isNonIntegralPointerType(PTy), "ptrtoint not supported for non-integral pointers"); - Assert(DestTy->getScalarType()->isIntegerTy(), - "PtrToInt result must be integral", &I); + Assert(DestTy->isIntOrIntVectorTy(), "PtrToInt result must be integral", &I); Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "PtrToInt type mismatch", &I); @@ -2531,10 +2529,9 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) { Type *SrcTy = I.getOperand(0)->getType(); Type *DestTy = I.getType(); - Assert(SrcTy->getScalarType()->isIntegerTy(), + Assert(SrcTy->isIntOrIntVectorTy(), "IntToPtr source must be an integral", &I); - Assert(DestTy->getScalarType()->isPointerTy(), - "IntToPtr result must be a pointer", &I); + Assert(DestTy->isPtrOrPtrVectorTy(), "IntToPtr result must be a pointer", &I); if (auto *PTy = dyn_cast(DestTy->getScalarType())) Assert(!DL.isNonIntegralPointerType(PTy), @@ -2952,11 +2949,10 @@ void Verifier::visitICmpInst(ICmpInst &IC) { Assert(Op0Ty == Op1Ty, "Both operands to ICmp instruction are not of the same type!", &IC); // Check that the operands are the right type - Assert(Op0Ty->isIntOrIntVectorTy() || Op0Ty->getScalarType()->isPointerTy(), + Assert(Op0Ty->isIntOrIntVectorTy() || Op0Ty->isPtrOrPtrVectorTy(), "Invalid operand types for ICmp instruction", &IC); // Check that the predicate is valid. - Assert(IC.getPredicate() >= CmpInst::FIRST_ICMP_PREDICATE && - IC.getPredicate() <= CmpInst::LAST_ICMP_PREDICATE, + Assert(IC.isIntPredicate(), "Invalid predicate in ICmp instruction!", &IC); visitInstruction(IC); @@ -2972,8 +2968,7 @@ void Verifier::visitFCmpInst(FCmpInst &FC) { Assert(Op0Ty->isFPOrFPVectorTy(), "Invalid operand types for FCmp instruction", &FC); // Check that the predicate is valid. - Assert(FC.getPredicate() >= CmpInst::FIRST_FCMP_PREDICATE && - FC.getPredicate() <= CmpInst::LAST_FCMP_PREDICATE, + Assert(FC.isFPPredicate(), "Invalid predicate in FCmp instruction!", &FC); visitInstruction(FC); @@ -3011,7 +3006,7 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { GetElementPtrInst::getIndexedType(GEP.getSourceElementType(), Idxs); Assert(ElTy, "Invalid indices for GEP pointer type!", &GEP); - Assert(GEP.getType()->getScalarType()->isPointerTy() && + Assert(GEP.getType()->isPtrOrPtrVectorTy() && GEP.getResultElementType() == ElTy, "GEP is not of right type for indices!", &GEP, ElTy); @@ -3027,7 +3022,7 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { unsigned IndexWidth = IndexTy->getVectorNumElements(); Assert(IndexWidth == GEPWidth, "Invalid GEP index vector width", &GEP); } - Assert(IndexTy->getScalarType()->isIntegerTy(), + Assert(IndexTy->isIntOrIntVectorTy(), "All GEP indices should be of integer type"); } } @@ -3113,7 +3108,7 @@ void Verifier::visitLoadInst(LoadInst &LI) { ElTy, &LI); checkAtomicMemAccessSize(ElTy, &LI); } else { - Assert(LI.getSynchScope() == CrossThread, + Assert(LI.getSyncScopeID() == SyncScope::System, "Non-atomic load cannot have SynchronizationScope specified", &LI); } @@ -3142,7 +3137,7 @@ void Verifier::visitStoreInst(StoreInst &SI) { ElTy, &SI); checkAtomicMemAccessSize(ElTy, &SI); } else { - Assert(SI.getSynchScope() == CrossThread, + Assert(SI.getSyncScopeID() == SyncScope::System, "Non-atomic store cannot have SynchronizationScope specified", &SI); } visitInstruction(SI); @@ -4049,6 +4044,73 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { "incorrect alignment of the source argument", CS); break; } + case Intrinsic::memmove_element_unordered_atomic: { + auto *MI = cast(CS.getInstruction()); + + ConstantInt *ElementSizeCI = + dyn_cast(MI->getRawElementSizeInBytes()); + Assert(ElementSizeCI, + "element size of the element-wise unordered atomic memory " + "intrinsic must be a constant int", + CS); + const APInt &ElementSizeVal = ElementSizeCI->getValue(); + Assert(ElementSizeVal.isPowerOf2(), + "element size of the element-wise atomic memory intrinsic " + "must be a power of 2", + CS); + + if (auto *LengthCI = dyn_cast(MI->getLength())) { + uint64_t Length = LengthCI->getZExtValue(); + uint64_t ElementSize = MI->getElementSizeInBytes(); + Assert((Length % ElementSize) == 0, + "constant length must be a multiple of the element size in the " + "element-wise atomic memory intrinsic", + CS); + } + + auto IsValidAlignment = [&](uint64_t Alignment) { + return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment); + }; + uint64_t DstAlignment = CS.getParamAlignment(0), + SrcAlignment = CS.getParamAlignment(1); + Assert(IsValidAlignment(DstAlignment), + "incorrect alignment of the destination argument", CS); + Assert(IsValidAlignment(SrcAlignment), + "incorrect alignment of the source argument", CS); + break; + } + case Intrinsic::memset_element_unordered_atomic: { + auto *MI = cast(CS.getInstruction()); + + ConstantInt *ElementSizeCI = + dyn_cast(MI->getRawElementSizeInBytes()); + Assert(ElementSizeCI, + "element size of the element-wise unordered atomic memory " + "intrinsic must be a constant int", + CS); + const APInt &ElementSizeVal = ElementSizeCI->getValue(); + Assert(ElementSizeVal.isPowerOf2(), + "element size of the element-wise atomic memory intrinsic " + "must be a power of 2", + CS); + + if (auto *LengthCI = dyn_cast(MI->getLength())) { + uint64_t Length = LengthCI->getZExtValue(); + uint64_t ElementSize = MI->getElementSizeInBytes(); + Assert((Length % ElementSize) == 0, + "constant length must be a multiple of the element size in the " + "element-wise atomic memory intrinsic", + CS); + } + + auto IsValidAlignment = [&](uint64_t Alignment) { + return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment); + }; + uint64_t DstAlignment = CS.getParamAlignment(0); + Assert(IsValidAlignment(DstAlignment), + "incorrect alignment of the destination argument", CS); + break; + } case Intrinsic::gcroot: case Intrinsic::gcwrite: case Intrinsic::gcread: @@ -4253,7 +4315,7 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { // relocated pointer. It can be casted to the correct type later if it's // desired. However, they must have the same address space and 'vectorness' GCRelocateInst &Relocate = cast(*CS.getInstruction()); - Assert(Relocate.getDerivedPtr()->getType()->getScalarType()->isPointerTy(), + Assert(Relocate.getDerivedPtr()->getType()->isPtrOrPtrVectorTy(), "gc.relocate: relocated value must be a gc pointer", CS); auto ResultType = CS.getType(); diff --git a/contrib/llvm/lib/LTO/LTO.cpp b/contrib/llvm/lib/LTO/LTO.cpp index 68b8c9fcb939..19973946ac5a 100644 --- a/contrib/llvm/lib/LTO/LTO.cpp +++ b/contrib/llvm/lib/LTO/LTO.cpp @@ -665,6 +665,15 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef Syms, auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier( Sym.getIRName(), GlobalValue::ExternalLinkage, "")); ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier(); + + // For linker redefined symbols (via --wrap or --defsym) we want to + // switch the linkage to `weak` to prevent IPOs from happening. + // Find the summary in the module for this very GV and record the new + // linkage so that we can switch it when we import the GV. + if (Res.LinkerRedefined) + if (auto S = ThinLTO.CombinedIndex.findSummaryInModule( + GUID, BM.getModuleIdentifier())) + S->setLinkage(GlobalValue::WeakAnyLinkage); } } } @@ -1021,7 +1030,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, // Collect for each module the list of function it defines (GUID -> // Summary). - StringMap> + StringMap ModuleToDefinedGVSummaries(ThinLTO.ModuleMap.size()); ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule( ModuleToDefinedGVSummaries); diff --git a/contrib/llvm/lib/Linker/IRMover.cpp b/contrib/llvm/lib/Linker/IRMover.cpp index defad1904989..f486e525b5e7 100644 --- a/contrib/llvm/lib/Linker/IRMover.cpp +++ b/contrib/llvm/lib/Linker/IRMover.cpp @@ -1256,6 +1256,18 @@ Error IRLinker::linkModuleFlagsMetadata() { return Error::success(); } +/// Return InlineAsm adjusted with target-specific directives if required. +/// For ARM and Thumb, we have to add directives to select the appropriate ISA +/// to support mixing module-level inline assembly from ARM and Thumb modules. +static std::string adjustInlineAsm(const std::string &InlineAsm, + const Triple &Triple) { + if (Triple.getArch() == Triple::thumb || Triple.getArch() == Triple::thumbeb) + return ".text\n.balign 2\n.thumb\n" + InlineAsm; + if (Triple.getArch() == Triple::arm || Triple.getArch() == Triple::armeb) + return ".text\n.balign 4\n.arm\n" + InlineAsm; + return InlineAsm; +} + Error IRLinker::run() { // Ensure metadata materialized before value mapping. if (SrcM->getMaterializer()) @@ -1293,11 +1305,13 @@ Error IRLinker::run() { // Append the module inline asm string. if (!IsPerformingImport && !SrcM->getModuleInlineAsm().empty()) { + std::string SrcModuleInlineAsm = adjustInlineAsm(SrcM->getModuleInlineAsm(), + SrcTriple); if (DstM.getModuleInlineAsm().empty()) - DstM.setModuleInlineAsm(SrcM->getModuleInlineAsm()); + DstM.setModuleInlineAsm(SrcModuleInlineAsm); else DstM.setModuleInlineAsm(DstM.getModuleInlineAsm() + "\n" + - SrcM->getModuleInlineAsm()); + SrcModuleInlineAsm); } // Loop over all of the linked values to compute type mappings. diff --git a/contrib/llvm/lib/MC/ELFObjectWriter.cpp b/contrib/llvm/lib/MC/ELFObjectWriter.cpp index 30f357826805..c8dd63011943 100644 --- a/contrib/llvm/lib/MC/ELFObjectWriter.cpp +++ b/contrib/llvm/lib/MC/ELFObjectWriter.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCAsmLayout.h" #include "llvm/MC/MCAssembler.h" @@ -26,6 +27,7 @@ #include "llvm/MC/MCELFObjectWriter.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCFixup.h" +#include "llvm/MC/MCFixupKindInfo.h" #include "llvm/MC/MCFragment.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCSection.h" @@ -204,8 +206,7 @@ class ELFObjectWriter : public MCObjectWriter { void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, - MCValue Target, bool &IsPCRel, - uint64_t &FixedValue) override; + MCValue Target, uint64_t &FixedValue) override; // Map from a signature symbol to the group section index using RevGroupMapTy = DenseMap; @@ -626,7 +627,10 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, - bool &IsPCRel, uint64_t &FixedValue) { + uint64_t &FixedValue) { + MCAsmBackend &Backend = Asm.getBackend(); + bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags & + MCFixupKindInfo::FKF_IsPCRel; const MCSectionELF &FixupSection = cast(*Fragment->getParent()); uint64_t C = Target.getConstant(); uint64_t FixupOffset = Layout.getFragmentOffset(Fragment) + Fixup.getOffset(); diff --git a/contrib/llvm/lib/MC/MCAssembler.cpp b/contrib/llvm/lib/MC/MCAssembler.cpp index 0318d916aa49..eaf6f19326eb 100644 --- a/contrib/llvm/lib/MC/MCAssembler.cpp +++ b/contrib/llvm/lib/MC/MCAssembler.cpp @@ -653,16 +653,14 @@ MCAssembler::handleFixup(const MCAsmLayout &Layout, MCFragment &F, // Evaluate the fixup. MCValue Target; uint64_t FixedValue; - bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags & - MCFixupKindInfo::FKF_IsPCRel; - if (!evaluateFixup(Layout, Fixup, &F, Target, FixedValue)) { + bool IsResolved = evaluateFixup(Layout, Fixup, &F, Target, FixedValue); + if (!IsResolved) { // The fixup was unresolved, we need a relocation. Inform the object // writer of the relocation, and give it an opportunity to adjust the // fixup value if need be. - getWriter().recordRelocation(*this, Layout, &F, Fixup, Target, IsPCRel, - FixedValue); + getWriter().recordRelocation(*this, Layout, &F, Fixup, Target, FixedValue); } - return std::make_tuple(Target, FixedValue, IsPCRel); + return std::make_tuple(Target, FixedValue, IsResolved); } void MCAssembler::layout(MCAsmLayout &Layout) { @@ -738,12 +736,12 @@ void MCAssembler::layout(MCAsmLayout &Layout) { llvm_unreachable("Unknown fragment with fixups!"); for (const MCFixup &Fixup : Fixups) { uint64_t FixedValue; - bool IsPCRel; + bool IsResolved; MCValue Target; - std::tie(Target, FixedValue, IsPCRel) = + std::tie(Target, FixedValue, IsResolved) = handleFixup(Layout, Frag, Fixup); getBackend().applyFixup(*this, Fixup, Target, Contents, FixedValue, - IsPCRel); + IsResolved); } } } diff --git a/contrib/llvm/lib/MC/MachObjectWriter.cpp b/contrib/llvm/lib/MC/MachObjectWriter.cpp index c4e7cdbe095e..62bf0a58fdfa 100644 --- a/contrib/llvm/lib/MC/MachObjectWriter.cpp +++ b/contrib/llvm/lib/MC/MachObjectWriter.cpp @@ -449,7 +449,7 @@ void MachObjectWriter::recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, - bool &IsPCRel, uint64_t &FixedValue) { + uint64_t &FixedValue) { TargetObjectWriter->recordRelocation(this, Asm, Layout, Fragment, Fixup, Target, FixedValue); } diff --git a/contrib/llvm/lib/MC/WasmObjectWriter.cpp b/contrib/llvm/lib/MC/WasmObjectWriter.cpp index 82352cb50c70..0d31f65c49d9 100644 --- a/contrib/llvm/lib/MC/WasmObjectWriter.cpp +++ b/contrib/llvm/lib/MC/WasmObjectWriter.cpp @@ -36,8 +36,7 @@ using namespace llvm; -#undef DEBUG_TYPE -#define DEBUG_TYPE "reloc-info" +#define DEBUG_TYPE "mc" namespace { @@ -153,7 +152,7 @@ struct WasmRelocationEntry { } void print(raw_ostream &Out) const { - Out << "Off=" << Offset << ", Sym=" << Symbol << ", Addend=" << Addend + Out << "Off=" << Offset << ", Sym=" << *Symbol << ", Addend=" << Addend << ", Type=" << Type << ", FixupSection=" << FixupSection; } @@ -199,6 +198,7 @@ class WasmObjectWriter : public MCObjectWriter { DenseMap FunctionTypeIndices; + SmallVector FunctionTypes; // TargetObjectWriter wrappers. bool is64Bit() const { return TargetObjectWriter->is64Bit(); } @@ -224,6 +224,7 @@ class WasmObjectWriter : public MCObjectWriter { SymbolIndices.clear(); IndirectSymbolIndices.clear(); FunctionTypeIndices.clear(); + FunctionTypes.clear(); MCObjectWriter::reset(); } @@ -231,8 +232,7 @@ class WasmObjectWriter : public MCObjectWriter { void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, - MCValue Target, bool &IsPCRel, - uint64_t &FixedValue) override; + MCValue Target, uint64_t &FixedValue) override; void executePostLayoutBinding(MCAssembler &Asm, const MCAsmLayout &Layout) override; @@ -276,6 +276,8 @@ class WasmObjectWriter : public MCObjectWriter { void writeRelocations(ArrayRef Relocations, uint64_t HeaderSize); uint32_t getRelocationIndexValue(const WasmRelocationEntry &RelEntry); + uint32_t getFunctionType(const MCSymbolWasm& Symbol); + uint32_t registerFunctionType(const MCSymbolWasm& Symbol); }; } // end anonymous namespace @@ -350,7 +352,10 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, - bool &IsPCRel, uint64_t &FixedValue) { + uint64_t &FixedValue) { + MCAsmBackend &Backend = Asm.getBackend(); + bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags & + MCFixupKindInfo::FKF_IsPCRel; const auto &FixupSection = cast(*Fragment->getParent()); uint64_t C = Target.getConstant(); uint64_t FixupOffset = Layout.getFragmentOffset(Fragment) + Fixup.getOffset(); @@ -401,15 +406,11 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm, const MCSymbolRefExpr *RefA = Target.getSymA(); const auto *SymA = RefA ? cast(&RefA->getSymbol()) : nullptr; - bool ViaWeakRef = false; if (SymA && SymA->isVariable()) { const MCExpr *Expr = SymA->getVariableValue(); - if (const auto *Inner = dyn_cast(Expr)) { - if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF) { - SymA = cast(&Inner->getSymbol()); - ViaWeakRef = true; - } - } + const auto *Inner = cast(Expr); + if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF) + llvm_unreachable("weakref used in reloc not yet implemented"); } // Put any constant offset in an addend. Offsets can be negative, and @@ -417,12 +418,8 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm, // be negative and don't wrap. FixedValue = 0; - if (SymA) { - if (ViaWeakRef) - llvm_unreachable("weakref used in reloc not yet implemented"); - else - SymA->setUsedInReloc(); - } + if (SymA) + SymA->setUsedInReloc(); assert(!IsPCRel); assert(SymA); @@ -493,7 +490,7 @@ uint32_t WasmObjectWriter::getRelocationIndexValue( case wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB: case wasm::R_WEBASSEMBLY_TABLE_INDEX_I32: if (!IndirectSymbolIndices.count(RelEntry.Symbol)) - report_fatal_error("symbol not found table index space:" + + report_fatal_error("symbol not found table index space: " + RelEntry.Symbol->getName()); return IndirectSymbolIndices[RelEntry.Symbol]; case wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB: @@ -502,12 +499,12 @@ uint32_t WasmObjectWriter::getRelocationIndexValue( case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_SLEB: case wasm::R_WEBASSEMBLY_GLOBAL_ADDR_I32: if (!SymbolIndices.count(RelEntry.Symbol)) - report_fatal_error("symbol not found function/global index space:" + + report_fatal_error("symbol not found function/global index space: " + RelEntry.Symbol->getName()); return SymbolIndices[RelEntry.Symbol]; case wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB: if (!TypeIndices.count(RelEntry.Symbol)) - report_fatal_error("symbol not found in type index space:" + + report_fatal_error("symbol not found in type index space: " + RelEntry.Symbol->getName()); return TypeIndices[RelEntry.Symbol]; default: @@ -913,6 +910,38 @@ void WasmObjectWriter::writeLinkingMetaDataSection( endSection(Section); } +uint32_t WasmObjectWriter::getFunctionType(const MCSymbolWasm& Symbol) { + assert(Symbol.isFunction()); + assert(TypeIndices.count(&Symbol)); + return TypeIndices[&Symbol]; +} + +uint32_t WasmObjectWriter::registerFunctionType(const MCSymbolWasm& Symbol) { + assert(Symbol.isFunction()); + + WasmFunctionType F; + if (Symbol.isVariable()) { + const MCExpr *Expr = Symbol.getVariableValue(); + auto *Inner = cast(Expr); + const auto *ResolvedSym = cast(&Inner->getSymbol()); + F.Returns = ResolvedSym->getReturns(); + F.Params = ResolvedSym->getParams(); + } else { + F.Returns = Symbol.getReturns(); + F.Params = Symbol.getParams(); + } + + auto Pair = + FunctionTypeIndices.insert(std::make_pair(F, FunctionTypes.size())); + if (Pair.second) + FunctionTypes.push_back(F); + TypeIndices[&Symbol] = Pair.first->second; + + DEBUG(dbgs() << "registerFunctionType: " << Symbol << " new:" << Pair.second << "\n"); + DEBUG(dbgs() << " -> type index: " << Pair.first->second << "\n"); + return Pair.first->second; +} + void WasmObjectWriter::writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) { DEBUG(dbgs() << "WasmObjectWriter::writeObject\n"); @@ -920,7 +949,6 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, wasm::ValType PtrType = is64Bit() ? wasm::ValType::I64 : wasm::ValType::I32; // Collect information from the available symbols. - SmallVector FunctionTypes; SmallVector Functions; SmallVector TableElems; SmallVector Globals; @@ -960,37 +988,27 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, // Populate the Imports set. for (const MCSymbol &S : Asm.symbols()) { const auto &WS = static_cast(S); - int32_t Type; - if (WS.isFunction()) { - // Prepare the function's type, if we haven't seen it yet. - WasmFunctionType F; - F.Returns = WS.getReturns(); - F.Params = WS.getParams(); - auto Pair = - FunctionTypeIndices.insert(std::make_pair(F, FunctionTypes.size())); - if (Pair.second) - FunctionTypes.push_back(F); + if (WS.isTemporary()) + continue; - Type = Pair.first->second; - } else { - Type = int32_t(PtrType); - } + if (WS.isFunction()) + registerFunctionType(WS); // If the symbol is not defined in this translation unit, import it. - if (!WS.isTemporary() && !WS.isDefined(/*SetUsed=*/false)) { + if (!WS.isDefined(/*SetUsed=*/false) || WS.isVariable()) { WasmImport Import; Import.ModuleName = WS.getModuleName(); Import.FieldName = WS.getName(); if (WS.isFunction()) { Import.Kind = wasm::WASM_EXTERNAL_FUNCTION; - Import.Type = Type; + Import.Type = getFunctionType(WS); SymbolIndices[&WS] = NumFuncImports; ++NumFuncImports; } else { Import.Kind = wasm::WASM_EXTERNAL_GLOBAL; - Import.Type = Type; + Import.Type = int32_t(PtrType); SymbolIndices[&WS] = NumGlobalImports; ++NumGlobalImports; } @@ -1082,10 +1100,6 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, if (S.isTemporary() && S.getName().empty()) continue; - // Variable references (weak references) are handled in a second pass - if (S.isVariable()) - continue; - const auto &WS = static_cast(S); DEBUG(dbgs() << "MCSymbol: '" << S << "'" << " isDefined=" << S.isDefined() << " isExternal=" @@ -1097,20 +1111,12 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, if (WS.isWeak()) WeakSymbols.push_back(WS.getName()); + if (WS.isVariable()) + continue; + unsigned Index; if (WS.isFunction()) { - // Prepare the function's type, if we haven't seen it yet. - WasmFunctionType F; - F.Returns = WS.getReturns(); - F.Params = WS.getParams(); - auto Pair = - FunctionTypeIndices.insert(std::make_pair(F, FunctionTypes.size())); - if (Pair.second) - FunctionTypes.push_back(F); - - int32_t Type = Pair.first->second; - if (WS.isDefined(/*SetUsed=*/false)) { if (WS.getOffset() != 0) report_fatal_error( @@ -1125,21 +1131,21 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, // Prepare the function. WasmFunction Func; - Func.Type = Type; + Func.Type = getFunctionType(WS); Func.Sym = &WS; SymbolIndices[&WS] = Index; Functions.push_back(Func); } else { - // Should be no such thing as weak undefined symbol - assert(!WS.isVariable()); - // An import; the index was assigned above. Index = SymbolIndices.find(&WS)->second; } + DEBUG(dbgs() << " -> function index: " << Index << "\n"); + // If needed, prepare the function to be called indirectly. - if (IsAddressTaken.count(&WS)) { + if (IsAddressTaken.count(&WS) != 0) { IndirectSymbolIndices[&WS] = TableElems.size(); + DEBUG(dbgs() << " -> adding to table: " << TableElems.size() << "\n"); TableElems.push_back(Index); } } else { @@ -1185,7 +1191,7 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, Align->getMaxBytesToEmit()); DataBytes.resize(Size, Value); } else if (auto *Fill = dyn_cast(&Frag)) { - DataBytes.insert(DataBytes.end(), Size, Fill->getValue()); + DataBytes.insert(DataBytes.end(), Fill->getSize(), Fill->getValue()); } else { const auto &DataFrag = cast(Frag); const SmallVectorImpl &Contents = DataFrag.getContents(); @@ -1205,11 +1211,12 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, Global.InitialValue = DataSection.getSectionOffset(); Global.ImportIndex = 0; SymbolIndices[&WS] = Index; + DEBUG(dbgs() << " -> global index: " << Index << "\n"); Globals.push_back(Global); } // If the symbol is visible outside this translation unit, export it. - if (WS.isExternal() && WS.isDefined(/*SetUsed=*/false)) { + if ((WS.isExternal() && WS.isDefined(/*SetUsed=*/false))) { WasmExport Export; Export.FieldName = WS.getName(); Export.Index = Index; @@ -1217,26 +1224,28 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, Export.Kind = wasm::WASM_EXTERNAL_FUNCTION; else Export.Kind = wasm::WASM_EXTERNAL_GLOBAL; + DEBUG(dbgs() << " -> export " << Exports.size() << "\n"); Exports.push_back(Export); } } - // Handle weak aliases + // Handle weak aliases. We need to process these in a separate pass because + // we need to have processed the target of the alias before the alias itself + // and the symbols are not necessarily ordered in this way. for (const MCSymbol &S : Asm.symbols()) { if (!S.isVariable()) continue; - assert(S.isExternal()); assert(S.isDefined(/*SetUsed=*/false)); const auto &WS = static_cast(S); - - // Find the target symbol of this weak alias + // Find the target symbol of this weak alias and export that index const MCExpr *Expr = WS.getVariableValue(); - auto *Inner = dyn_cast(Expr); + auto *Inner = cast(Expr); const auto *ResolvedSym = cast(&Inner->getSymbol()); + DEBUG(dbgs() << WS.getName() << ": weak alias of '" << *ResolvedSym << "'\n"); + assert(SymbolIndices.count(ResolvedSym) > 0); uint32_t Index = SymbolIndices.find(ResolvedSym)->second; - DEBUG(dbgs() << "Weak alias: '" << WS << "' -> '" << ResolvedSym << "' = " << Index << "\n"); - SymbolIndices[&WS] = Index; + DEBUG(dbgs() << " -> index:" << Index << "\n"); WasmExport Export; Export.FieldName = WS.getName(); @@ -1245,7 +1254,7 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, Export.Kind = wasm::WASM_EXTERNAL_FUNCTION; else Export.Kind = wasm::WASM_EXTERNAL_GLOBAL; - WeakSymbols.push_back(Export.FieldName); + DEBUG(dbgs() << " -> export " << Exports.size() << "\n"); Exports.push_back(Export); } @@ -1254,15 +1263,7 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, if (Fixup.Type != wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB) continue; - WasmFunctionType F; - F.Returns = Fixup.Symbol->getReturns(); - F.Params = Fixup.Symbol->getParams(); - auto Pair = - FunctionTypeIndices.insert(std::make_pair(F, FunctionTypes.size())); - if (Pair.second) - FunctionTypes.push_back(F); - - TypeIndices[Fixup.Symbol] = Pair.first->second; + registerFunctionType(*Fixup.Symbol); } // Write out the Wasm header. diff --git a/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp b/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp index fc5234950391..956ae70b38d1 100644 --- a/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/contrib/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -197,8 +197,7 @@ class WinCOFFObjectWriter : public MCObjectWriter { void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, - MCValue Target, bool &IsPCRel, - uint64_t &FixedValue) override; + MCValue Target, uint64_t &FixedValue) override; void createFileSymbols(MCAssembler &Asm); void assignSectionNumbers(); @@ -708,9 +707,11 @@ bool WinCOFFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl( InSet, IsPCRel); } -void WinCOFFObjectWriter::recordRelocation( - MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, - const MCFixup &Fixup, MCValue Target, bool &IsPCRel, uint64_t &FixedValue) { +void WinCOFFObjectWriter::recordRelocation(MCAssembler &Asm, + const MCAsmLayout &Layout, + const MCFragment *Fragment, + const MCFixup &Fixup, MCValue Target, + uint64_t &FixedValue) { assert(Target.getSymA() && "Relocation must reference a symbol!"); const MCSymbol &A = Target.getSymA()->getSymbol(); diff --git a/contrib/llvm/lib/Object/WasmObjectFile.cpp b/contrib/llvm/lib/Object/WasmObjectFile.cpp index fff497ba5564..7f80bf0b83a0 100644 --- a/contrib/llvm/lib/Object/WasmObjectFile.cpp +++ b/contrib/llvm/lib/Object/WasmObjectFile.cpp @@ -567,20 +567,16 @@ Error WasmObjectFile::parseExportSection(const uint8_t *Ptr, const uint8_t *End) Ex.Name = readString(Ptr); Ex.Kind = readUint8(Ptr); Ex.Index = readVaruint32(Ptr); + WasmSymbol::SymbolType ExportType; + bool MakeSymbol = false; switch (Ex.Kind) { case wasm::WASM_EXTERNAL_FUNCTION: - SymbolMap.try_emplace(Ex.Name, Symbols.size()); - Symbols.emplace_back(Ex.Name, WasmSymbol::SymbolType::FUNCTION_EXPORT, - Sections.size(), i); - DEBUG(dbgs() << "Adding export: " << Symbols.back() - << " sym index:" << Symbols.size() << "\n"); + ExportType = WasmSymbol::SymbolType::FUNCTION_EXPORT; + MakeSymbol = true; break; case wasm::WASM_EXTERNAL_GLOBAL: - SymbolMap.try_emplace(Ex.Name, Symbols.size()); - Symbols.emplace_back(Ex.Name, WasmSymbol::SymbolType::GLOBAL_EXPORT, - Sections.size(), i); - DEBUG(dbgs() << "Adding export: " << Symbols.back() - << " sym index:" << Symbols.size() << "\n"); + ExportType = WasmSymbol::SymbolType::GLOBAL_EXPORT; + MakeSymbol = true; break; case wasm::WASM_EXTERNAL_MEMORY: case wasm::WASM_EXTERNAL_TABLE: @@ -589,6 +585,20 @@ Error WasmObjectFile::parseExportSection(const uint8_t *Ptr, const uint8_t *End) return make_error( "Unexpected export kind", object_error::parse_failed); } + if (MakeSymbol) { + auto Pair = SymbolMap.try_emplace(Ex.Name, Symbols.size()); + if (Pair.second) { + Symbols.emplace_back(Ex.Name, ExportType, + Sections.size(), i); + DEBUG(dbgs() << "Adding export: " << Symbols.back() + << " sym index:" << Symbols.size() << "\n"); + } else { + uint32_t SymIndex = Pair.first->second; + Symbols[SymIndex] = WasmSymbol(Ex.Name, ExportType, Sections.size(), i); + DEBUG(dbgs() << "Replacing existing symbol: " << Symbols[SymIndex] + << " sym index:" << SymIndex << "\n"); + } + } Exports.push_back(Ex); } if (Ptr != End) @@ -665,15 +675,17 @@ Error WasmObjectFile::parseElemSection(const uint8_t *Ptr, const uint8_t *End) { } Error WasmObjectFile::parseDataSection(const uint8_t *Ptr, const uint8_t *End) { + const uint8_t *Start = Ptr; uint32_t Count = readVaruint32(Ptr); DataSegments.reserve(Count); while (Count--) { - wasm::WasmDataSegment Segment; - Segment.Index = readVaruint32(Ptr); - if (Error Err = readInitExpr(Segment.Offset, Ptr)) + WasmSegment Segment; + Segment.Data.MemoryIndex = readVaruint32(Ptr); + if (Error Err = readInitExpr(Segment.Data.Offset, Ptr)) return Err; uint32_t Size = readVaruint32(Ptr); - Segment.Content = ArrayRef(Ptr, Size); + Segment.Data.Content = ArrayRef(Ptr, Size); + Segment.SectionOffset = Ptr - Start; Ptr += Size; DataSegments.push_back(Segment); } diff --git a/contrib/llvm/lib/Object/WindowsResource.cpp b/contrib/llvm/lib/Object/WindowsResource.cpp index 1371eacdf8f2..246eee5ddb31 100644 --- a/contrib/llvm/lib/Object/WindowsResource.cpp +++ b/contrib/llvm/lib/Object/WindowsResource.cpp @@ -609,8 +609,8 @@ void WindowsResourceCOFFWriter::writeDirectoryTree() { for (auto const &Child : StringChildren) { auto *Entry = reinterpret_cast(BufferStart + CurrentOffset); - Entry->Identifier.NameOffset = - StringTableOffsets[Child.second->getStringIndex()]; + Entry->Identifier.setNameOffset( + StringTableOffsets[Child.second->getStringIndex()]); if (Child.second->checkIsDataNode()) { Entry->Offset.DataEntryOffset = NextLevelOffset; NextLevelOffset += sizeof(coff_resource_data_entry); diff --git a/contrib/llvm/lib/ObjectYAML/WasmYAML.cpp b/contrib/llvm/lib/ObjectYAML/WasmYAML.cpp index 2040efdc9d11..6a68cd265ad8 100644 --- a/contrib/llvm/lib/ObjectYAML/WasmYAML.cpp +++ b/contrib/llvm/lib/ObjectYAML/WasmYAML.cpp @@ -345,7 +345,8 @@ void MappingTraits::mapping(IO &IO, void MappingTraits::mapping( IO &IO, WasmYAML::DataSegment &Segment) { - IO.mapRequired("Index", Segment.Index); + IO.mapOptional("SectionOffset", Segment.SectionOffset); + IO.mapRequired("MemoryIndex", Segment.MemoryIndex); IO.mapRequired("Offset", Segment.Offset); IO.mapRequired("Content", Segment.Content); } diff --git a/contrib/llvm/lib/Option/OptTable.cpp b/contrib/llvm/lib/Option/OptTable.cpp index acb9e8d015bc..bcd365236e46 100644 --- a/contrib/llvm/lib/Option/OptTable.cpp +++ b/contrib/llvm/lib/Option/OptTable.cpp @@ -225,11 +225,15 @@ OptTable::suggestValueCompletions(StringRef Option, StringRef Arg) const { return {}; } -std::vector OptTable::findByPrefix(StringRef Cur) const { +std::vector +OptTable::findByPrefix(StringRef Cur, unsigned short DisableFlags) const { std::vector Ret; for (const Info &In : OptionInfos.slice(FirstSearchableIndex)) { - if (!In.Prefixes) + if (!In.Prefixes || (!In.HelpText && !In.GroupID)) continue; + if (In.Flags & DisableFlags) + continue; + for (int I = 0; In.Prefixes[I]; I++) { std::string S = std::string(In.Prefixes[I]) + std::string(In.Name); if (StringRef(S).startswith(Cur)) diff --git a/contrib/llvm/lib/Passes/PassBuilder.cpp b/contrib/llvm/lib/Passes/PassBuilder.cpp index 0380bd991d71..9e0cf27aa17b 100644 --- a/contrib/llvm/lib/Passes/PassBuilder.cpp +++ b/contrib/llvm/lib/Passes/PassBuilder.cpp @@ -281,33 +281,52 @@ AnalysisKey NoOpLoopAnalysis::Key; } // End anonymous namespace. +void PassBuilder::invokePeepholeEPCallbacks( + FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { + for (auto &C : PeepholeEPCallbacks) + C(FPM, Level); +} + void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) { #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ MAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" + + for (auto &C : ModuleAnalysisRegistrationCallbacks) + C(MAM); } void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) { #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ CGAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" + + for (auto &C : CGSCCAnalysisRegistrationCallbacks) + C(CGAM); } void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ FAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" + + for (auto &C : FunctionAnalysisRegistrationCallbacks) + C(FAM); } void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) { #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ LAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" + + for (auto &C : LoopAnalysisRegistrationCallbacks) + C(LAM); } FunctionPassManager PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging) { + bool DebugLogging, + bool PrepareForThinLTO) { assert(Level != O0 && "Must request optimizations!"); FunctionPassManager FPM(DebugLogging); @@ -340,6 +359,8 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, if (!isOptimizingForSize(Level)) FPM.addPass(LibCallsShrinkWrapPass()); + invokePeepholeEPCallbacks(FPM, Level); + FPM.addPass(TailCallElimPass()); FPM.addPass(SimplifyCFGPass()); @@ -363,11 +384,19 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, LPM1.addPass(SimpleLoopUnswitchPass()); LPM2.addPass(IndVarSimplifyPass()); LPM2.addPass(LoopIdiomRecognizePass()); + + for (auto &C : LateLoopOptimizationsEPCallbacks) + C(LPM2, Level); + LPM2.addPass(LoopDeletionPass()); - // FIXME: The old pass manager has a hack to disable loop unrolling during - // ThinLTO when using sample PGO. Need to either fix it or port some - // workaround. - LPM2.addPass(LoopUnrollPass::createFull(Level)); + // Do not enable unrolling in PrepareForThinLTO phase during sample PGO + // because it changes IR to makes profile annotation in back compile + // inaccurate. + if (!PrepareForThinLTO || !PGOOpt || PGOOpt->SampleProfileFile.empty()) + LPM2.addPass(LoopUnrollPass::createFull(Level)); + + for (auto &C : LoopOptimizerEndEPCallbacks) + C(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. @@ -403,6 +432,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, // Run instcombine after redundancy and dead bit elimination to exploit // opportunities opened up by them. FPM.addPass(InstCombinePass()); + invokePeepholeEPCallbacks(FPM, Level); // Re-consider control flow based optimizations after redundancy elimination, // redo DCE, etc. @@ -411,19 +441,24 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, FPM.addPass(DSEPass()); FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass())); + for (auto &C : ScalarOptimizerLateEPCallbacks) + C(FPM, Level); + // Finally, do an expensive DCE pass to catch all the dead code exposed by // the simplifications and basic cleanup after all the simplifications. FPM.addPass(ADCEPass()); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); + invokePeepholeEPCallbacks(FPM, Level); return FPM; } -static void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, - PassBuilder::OptimizationLevel Level, - bool RunProfileGen, std::string ProfileGenFile, - std::string ProfileUseFile) { +void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, + PassBuilder::OptimizationLevel Level, + bool RunProfileGen, + std::string ProfileGenFile, + std::string ProfileUseFile) { // Generally running simplification passes and the inliner with an high // threshold results in smaller executables, but there may be cases where // the size grows, so let's be conservative here and skip this simplification @@ -448,9 +483,8 @@ static void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies. FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks. FPM.addPass(InstCombinePass()); // Combine silly sequences. + invokePeepholeEPCallbacks(FPM, Level); - // FIXME: Here the old pass manager inserts peephole extensions. - // Add them when they're supported. CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline))); @@ -490,7 +524,8 @@ getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) { ModulePassManager PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging) { + bool DebugLogging, + bool PrepareForThinLTO) { ModulePassManager MPM(DebugLogging); // Do basic inference of function attributes from known properties of system @@ -530,6 +565,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // optimizations. FunctionPassManager GlobalCleanupPM(DebugLogging); GlobalCleanupPM.addPass(InstCombinePass()); + invokePeepholeEPCallbacks(GlobalCleanupPM, Level); + GlobalCleanupPM.addPass(SimplifyCFGPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); @@ -544,8 +581,11 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile)); // Indirect call promotion that promotes intra-module targes only. - MPM.addPass(PGOIndirectCallPromotion( - false, PGOOpt && !PGOOpt->SampleProfileFile.empty())); + // Do not enable it in PrepareForThinLTO phase during sample PGO because + // it changes IR to makes profile annotation in back compile inaccurate. + if (!PrepareForThinLTO || PGOOpt->SampleProfileFile.empty()) + MPM.addPass(PGOIndirectCallPromotion( + false, PGOOpt && !PGOOpt->SampleProfileFile.empty())); } // Require the GlobalsAA analysis for the module so we can query it within @@ -570,7 +610,12 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // Run the inliner first. The theory is that we are walking bottom-up and so // the callees have already been fully optimized, and we want to inline them // into the callers so that our optimizations can reflect that. - MainCGPipeline.addPass(InlinerPass(getInlineParamsFromOptLevel(Level))); + // For PrepareForThinLTO pass, we disable hot-caller heuristic for sample PGO + // because it makes profile annotation in the backend inaccurate. + InlineParams IP = getInlineParamsFromOptLevel(Level); + if (PrepareForThinLTO && PGOOpt && !PGOOpt->SampleProfileFile.empty()) + IP.HotCallSiteThreshold = 0; + MainCGPipeline.addPass(InlinerPass(IP)); // Now deduce any function attributes based in the current code. MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); @@ -583,7 +628,11 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // Lastly, add the core function simplification pipeline nested inside the // CGSCC walk. MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( - buildFunctionSimplificationPipeline(Level, DebugLogging))); + buildFunctionSimplificationPipeline(Level, DebugLogging, + PrepareForThinLTO))); + + for (auto &C : CGSCCOptimizerLateEPCallbacks) + C(MainCGPipeline, Level); // We wrap the CGSCC pipeline in a devirtualization repeater. This will try // to detect when we devirtualize indirect calls and iterate the SCC passes @@ -643,6 +692,9 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, // rather than on each loop in an inside-out manner, and so they are actually // function passes. + for (auto &C : VectorizerStartEPCallbacks) + C(OptimizePM, Level); + // First rotate loops that may have been un-rotated by prior passes. OptimizePM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass())); @@ -726,7 +778,8 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, MPM.addPass(ForceFunctionAttrsPass()); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging)); + MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging, + /*PrepareForThinLTO=*/false)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); @@ -747,7 +800,8 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level, // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we // can. - MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging)); + MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging, + /*PrepareForThinLTO=*/true)); // Run partial inlining pass to partially inline functions that have // large bodies. @@ -785,7 +839,8 @@ PassBuilder::buildThinLTODefaultPipeline(OptimizationLevel Level, !PGOOpt->ProfileUseFile.empty())); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging)); + MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging, + /*PrepareForThinLTO=*/false)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); @@ -868,8 +923,11 @@ ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, // simplification opportunities, and both can propagate functions through // function pointers. When this happens, we often have to resolve varargs // calls, etc, so let instcombine do this. - // FIXME: add peephole extensions here as the legacy PM does. - MPM.addPass(createModuleToFunctionPassAdaptor(InstCombinePass())); + FunctionPassManager PeepholeFPM(DebugLogging); + PeepholeFPM.addPass(InstCombinePass()); + invokePeepholeEPCallbacks(PeepholeFPM, Level); + + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is @@ -887,10 +945,10 @@ ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, MPM.addPass(GlobalDCEPass()); FunctionPassManager FPM(DebugLogging); - // The IPO Passes may leave cruft around. Clean up after them. - // FIXME: add peephole extensions here as the legacy PM does. FPM.addPass(InstCombinePass()); + invokePeepholeEPCallbacks(FPM, Level); + FPM.addPass(JumpThreadingPass()); // Break up allocas @@ -937,8 +995,11 @@ ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, MainFPM.add(AlignmentFromAssumptionsPass()); #endif - // FIXME: add peephole extensions to the PM here. + // FIXME: Conditionally run LoadCombine here, after it's ported + // (in case we still have this pass, given its questionable usefulness). + MainFPM.addPass(InstCombinePass()); + invokePeepholeEPCallbacks(MainFPM, Level); MainFPM.addPass(JumpThreadingPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); @@ -1021,7 +1082,27 @@ static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) { Name.startswith("lto"); } -static bool isModulePassName(StringRef Name) { +/// Tests whether registered callbacks will accept a given pass name. +/// +/// When parsing a pipeline text, the type of the outermost pipeline may be +/// omitted, in which case the type is automatically determined from the first +/// pass name in the text. This may be a name that is handled through one of the +/// callbacks. We check this through the oridinary parsing callbacks by setting +/// up a dummy PassManager in order to not force the client to also handle this +/// type of query. +template +static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) { + if (!Callbacks.empty()) { + PassManagerT DummyPM; + for (auto &CB : Callbacks) + if (CB(Name, DummyPM, {})) + return true; + } + return false; +} + +template +static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { // Manually handle aliases for pre-configured pipeline fragments. if (startsWithDefaultPipelineAliasPrefix(Name)) return DefaultAliasRegex.match(Name); @@ -1046,10 +1127,11 @@ static bool isModulePassName(StringRef Name) { return true; #include "PassRegistry.def" - return false; + return callbacksAcceptPassName(Name, Callbacks); } -static bool isCGSCCPassName(StringRef Name) { +template +static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "cgscc") return true; @@ -1070,10 +1152,11 @@ static bool isCGSCCPassName(StringRef Name) { return true; #include "PassRegistry.def" - return false; + return callbacksAcceptPassName(Name, Callbacks); } -static bool isFunctionPassName(StringRef Name) { +template +static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "function") return true; @@ -1092,10 +1175,11 @@ static bool isFunctionPassName(StringRef Name) { return true; #include "PassRegistry.def" - return false; + return callbacksAcceptPassName(Name, Callbacks); } -static bool isLoopPassName(StringRef Name) { +template +static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "loop") return true; @@ -1112,7 +1196,7 @@ static bool isLoopPassName(StringRef Name) { return true; #include "PassRegistry.def" - return false; + return callbacksAcceptPassName(Name, Callbacks); } Optional> @@ -1213,6 +1297,11 @@ bool PassBuilder::parseModulePass(ModulePassManager &MPM, MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM))); return true; } + + for (auto &C : ModulePipelineParsingCallbacks) + if (C(Name, MPM, InnerPipeline)) + return true; + // Normal passes can't have pipelines. return false; } @@ -1225,12 +1314,12 @@ bool PassBuilder::parseModulePass(ModulePassManager &MPM, assert(Matches.size() == 3 && "Must capture two matched strings!"); OptimizationLevel L = StringSwitch(Matches[2]) - .Case("O0", O0) - .Case("O1", O1) - .Case("O2", O2) - .Case("O3", O3) - .Case("Os", Os) - .Case("Oz", Oz); + .Case("O0", O0) + .Case("O1", O1) + .Case("O2", O2) + .Case("O3", O3) + .Case("Os", Os) + .Case("Oz", Oz); if (L == O0) // At O0 we do nothing at all! return true; @@ -1270,6 +1359,9 @@ bool PassBuilder::parseModulePass(ModulePassManager &MPM, } #include "PassRegistry.def" + for (auto &C : ModulePipelineParsingCallbacks) + if (C(Name, MPM, InnerPipeline)) + return true; return false; } @@ -1317,11 +1409,16 @@ bool PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, *MaxRepetitions, DebugLogging)); return true; } + + for (auto &C : CGSCCPipelineParsingCallbacks) + if (C(Name, CGPM, InnerPipeline)) + return true; + // Normal passes can't have pipelines. return false; } - // Now expand the basic registered passes from the .inc file. +// Now expand the basic registered passes from the .inc file. #define CGSCC_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ CGPM.addPass(CREATE_PASS); \ @@ -1342,6 +1439,9 @@ bool PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, } #include "PassRegistry.def" + for (auto &C : CGSCCPipelineParsingCallbacks) + if (C(Name, CGPM, InnerPipeline)) + return true; return false; } @@ -1379,11 +1479,16 @@ bool PassBuilder::parseFunctionPass(FunctionPassManager &FPM, FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM))); return true; } + + for (auto &C : FunctionPipelineParsingCallbacks) + if (C(Name, FPM, InnerPipeline)) + return true; + // Normal passes can't have pipelines. return false; } - // Now expand the basic registered passes from the .inc file. +// Now expand the basic registered passes from the .inc file. #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ FPM.addPass(CREATE_PASS); \ @@ -1403,6 +1508,9 @@ bool PassBuilder::parseFunctionPass(FunctionPassManager &FPM, } #include "PassRegistry.def" + for (auto &C : FunctionPipelineParsingCallbacks) + if (C(Name, FPM, InnerPipeline)) + return true; return false; } @@ -1430,11 +1538,16 @@ bool PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E, LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM))); return true; } + + for (auto &C : LoopPipelineParsingCallbacks) + if (C(Name, LPM, InnerPipeline)) + return true; + // Normal passes can't have pipelines. return false; } - // Now expand the basic registered passes from the .inc file. +// Now expand the basic registered passes from the .inc file. #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ LPM.addPass(CREATE_PASS); \ @@ -1455,6 +1568,9 @@ bool PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E, } #include "PassRegistry.def" + for (auto &C : LoopPipelineParsingCallbacks) + if (C(Name, LPM, InnerPipeline)) + return true; return false; } @@ -1473,6 +1589,9 @@ bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) { } #include "PassRegistry.def" + for (auto &C : AAParsingCallbacks) + if (C(Name, AA)) + return true; return false; } @@ -1539,7 +1658,7 @@ bool PassBuilder::parseModulePassPipeline(ModulePassManager &MPM, return true; } -// Primary pass pipeline description parsing routine. +// Primary pass pipeline description parsing routine for a \c ModulePassManager // FIXME: Should this routine accept a TargetMachine or require the caller to // pre-populate the analysis managers with target-specific stuff? bool PassBuilder::parsePassPipeline(ModulePassManager &MPM, @@ -1553,21 +1672,70 @@ bool PassBuilder::parsePassPipeline(ModulePassManager &MPM, // automatically. StringRef FirstName = Pipeline->front().Name; - if (!isModulePassName(FirstName)) { - if (isCGSCCPassName(FirstName)) + if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) { + if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) { Pipeline = {{"cgscc", std::move(*Pipeline)}}; - else if (isFunctionPassName(FirstName)) + } else if (isFunctionPassName(FirstName, + FunctionPipelineParsingCallbacks)) { Pipeline = {{"function", std::move(*Pipeline)}}; - else if (isLoopPassName(FirstName)) + } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) { Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}}; - else + } else { + for (auto &C : TopLevelPipelineParsingCallbacks) + if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging)) + return true; + // Unknown pass name! return false; + } } return parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging); } +// Primary pass pipeline description parsing routine for a \c CGSCCPassManager +bool PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM, + StringRef PipelineText, bool VerifyEachPass, + bool DebugLogging) { + auto Pipeline = parsePipelineText(PipelineText); + if (!Pipeline || Pipeline->empty()) + return false; + + StringRef FirstName = Pipeline->front().Name; + if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) + return false; + + return parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging); +} + +// Primary pass pipeline description parsing routine for a \c +// FunctionPassManager +bool PassBuilder::parsePassPipeline(FunctionPassManager &FPM, + StringRef PipelineText, bool VerifyEachPass, + bool DebugLogging) { + auto Pipeline = parsePipelineText(PipelineText); + if (!Pipeline || Pipeline->empty()) + return false; + + StringRef FirstName = Pipeline->front().Name; + if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) + return false; + + return parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass, + DebugLogging); +} + +// Primary pass pipeline description parsing routine for a \c LoopPassManager +bool PassBuilder::parsePassPipeline(LoopPassManager &CGPM, + StringRef PipelineText, bool VerifyEachPass, + bool DebugLogging) { + auto Pipeline = parsePipelineText(PipelineText); + if (!Pipeline || Pipeline->empty()) + return false; + + return parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging); +} + bool PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { // If the pipeline just consists of the word 'default' just replace the AA // manager with our default one. diff --git a/contrib/llvm/lib/ProfileData/InstrProf.cpp b/contrib/llvm/lib/ProfileData/InstrProf.cpp index a1d18724fcd5..48c1643cb13c 100644 --- a/contrib/llvm/lib/ProfileData/InstrProf.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProf.cpp @@ -460,9 +460,9 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { return Error::success(); } -void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE, - InstrProfValueSiteRecord &Input, - uint64_t Weight) { +void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, + uint64_t Weight, + function_ref Warn) { this->sortByTargetValues(); Input.sortByTargetValues(); auto I = ValueData.begin(); @@ -475,7 +475,7 @@ void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE, bool Overflowed; I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed); if (Overflowed) - SIPE.addError(instrprof_error::counter_overflow); + Warn(instrprof_error::counter_overflow); ++I; continue; } @@ -483,25 +483,25 @@ void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE, } } -void InstrProfValueSiteRecord::scale(SoftInstrProfErrors &SIPE, - uint64_t Weight) { +void InstrProfValueSiteRecord::scale(uint64_t Weight, + function_ref Warn) { for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) { bool Overflowed; I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed); if (Overflowed) - SIPE.addError(instrprof_error::counter_overflow); + Warn(instrprof_error::counter_overflow); } } // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. -void InstrProfRecord::mergeValueProfData(uint32_t ValueKind, - InstrProfRecord &Src, - uint64_t Weight) { +void InstrProfRecord::mergeValueProfData( + uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight, + function_ref Warn) { uint32_t ThisNumValueSites = getNumValueSites(ValueKind); uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); if (ThisNumValueSites != OtherNumValueSites) { - SIPE.addError(instrprof_error::value_site_count_mismatch); + Warn(instrprof_error::value_site_count_mismatch); return; } if (!ThisNumValueSites) @@ -511,14 +511,15 @@ void InstrProfRecord::mergeValueProfData(uint32_t ValueKind, MutableArrayRef OtherSiteRecords = Src.getValueSitesForKind(ValueKind); for (uint32_t I = 0; I < ThisNumValueSites; I++) - ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight); + ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn); } -void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) { +void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, + function_ref Warn) { // If the number of counters doesn't match we either have bad data // or a hash collision. if (Counts.size() != Other.Counts.size()) { - SIPE.addError(instrprof_error::count_mismatch); + Warn(instrprof_error::count_mismatch); return; } @@ -527,27 +528,30 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) { Counts[I] = SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); if (Overflowed) - SIPE.addError(instrprof_error::counter_overflow); + Warn(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - mergeValueProfData(Kind, Other, Weight); + mergeValueProfData(Kind, Other, Weight, Warn); } -void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) { +void InstrProfRecord::scaleValueProfData( + uint32_t ValueKind, uint64_t Weight, + function_ref Warn) { for (auto &R : getValueSitesForKind(ValueKind)) - R.scale(SIPE, Weight); + R.scale(Weight, Warn); } -void InstrProfRecord::scale(uint64_t Weight) { +void InstrProfRecord::scale(uint64_t Weight, + function_ref Warn) { for (auto &Count : this->Counts) { bool Overflowed; Count = SaturatingMultiply(Count, Weight, &Overflowed); if (Overflowed) - SIPE.addError(instrprof_error::counter_overflow); + Warn(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - scaleValueProfData(Kind, Weight); + scaleValueProfData(Kind, Weight, Warn); } // Map indirect call target name hash to name string. diff --git a/contrib/llvm/lib/ProfileData/InstrProfReader.cpp b/contrib/llvm/lib/ProfileData/InstrProfReader.cpp index 1ed1fb8b6f0b..1b39a0695aac 100644 --- a/contrib/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProfReader.cpp @@ -221,7 +221,7 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { #undef VP_READ_ADVANCE } -Error TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { +Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { // Skip empty lines and comments. while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) ++Line; @@ -377,13 +377,13 @@ Error RawInstrProfReader::readHeader( } template -Error RawInstrProfReader::readName(InstrProfRecord &Record) { +Error RawInstrProfReader::readName(NamedInstrProfRecord &Record) { Record.Name = getName(Data->NameRef); return success(); } template -Error RawInstrProfReader::readFuncHash(InstrProfRecord &Record) { +Error RawInstrProfReader::readFuncHash(NamedInstrProfRecord &Record) { Record.Hash = swap(Data->FuncHash); return success(); } @@ -445,7 +445,7 @@ Error RawInstrProfReader::readValueProfilingData( } template -Error RawInstrProfReader::readNextRecord(InstrProfRecord &Record) { +Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { if (atEnd()) // At this point, ValueDataStart field points to the next header. if (Error E = readNextHeader(getNextHeaderPos())) @@ -550,7 +550,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, template Error InstrProfReaderIndex::getRecords( - StringRef FuncName, ArrayRef &Data) { + StringRef FuncName, ArrayRef &Data) { auto Iter = HashTable->find(FuncName); if (Iter == HashTable->end()) return make_error(instrprof_error::unknown_function); @@ -564,7 +564,7 @@ Error InstrProfReaderIndex::getRecords( template Error InstrProfReaderIndex::getRecords( - ArrayRef &Data) { + ArrayRef &Data) { if (atEnd()) return make_error(instrprof_error::eof); @@ -644,7 +644,7 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version, InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); // FIXME: This only computes an empty summary. Need to call addRecord for - // all InstrProfRecords to get the correct summary. + // all NamedInstrProfRecords to get the correct summary. this->Summary = Builder.getSummary(); return Cur; } @@ -707,7 +707,7 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() { Expected IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName, uint64_t FuncHash) { - ArrayRef Data; + ArrayRef Data; Error Err = Index->getRecords(FuncName, Data); if (Err) return std::move(Err); @@ -732,10 +732,10 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, return success(); } -Error IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) { +Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { static unsigned RecordIndex = 0; - ArrayRef Data; + ArrayRef Data; Error E = Index->getRecords(Data); if (E) diff --git a/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp b/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp index 9efea78ed2a8..ce3f8806e12e 100644 --- a/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -176,38 +176,46 @@ void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; } -Error InstrProfWriter::addRecord(InstrProfRecord &&I, uint64_t Weight) { - auto &ProfileDataMap = FunctionData[I.Name]; +void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight, + function_ref Warn) { + auto Name = I.Name; + auto Hash = I.Hash; + addRecord(Name, Hash, std::move(I), Weight, Warn); +} + +void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash, + InstrProfRecord &&I, uint64_t Weight, + function_ref Warn) { + auto &ProfileDataMap = FunctionData[Name]; bool NewFunc; ProfilingData::iterator Where; std::tie(Where, NewFunc) = - ProfileDataMap.insert(std::make_pair(I.Hash, InstrProfRecord())); + ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord())); InstrProfRecord &Dest = Where->second; + auto MapWarn = [&](instrprof_error E) { + Warn(make_error(E)); + }; + if (NewFunc) { // We've never seen a function with this name and hash, add it. Dest = std::move(I); - // Fix up the name to avoid dangling reference. - Dest.Name = FunctionData.find(Dest.Name)->getKey(); if (Weight > 1) - Dest.scale(Weight); + Dest.scale(Weight, MapWarn); } else { // We're updating a function we've seen before. - Dest.merge(I, Weight); + Dest.merge(I, Weight, MapWarn); } Dest.sortValueData(); - - return Dest.takeError(); } -Error InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW) { +void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW, + function_ref Warn) { for (auto &I : IPW.FunctionData) for (auto &Func : I.getValue()) - if (Error E = addRecord(std::move(Func.second), 1)) - return E; - return Error::success(); + addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn); } bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { @@ -323,11 +331,12 @@ static const char *ValueProfKindStr[] = { #include "llvm/ProfileData/InstrProfData.inc" }; -void InstrProfWriter::writeRecordInText(const InstrProfRecord &Func, +void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, + const InstrProfRecord &Func, InstrProfSymtab &Symtab, raw_fd_ostream &OS) { - OS << Func.Name << "\n"; - OS << "# Func Hash:\n" << Func.Hash << "\n"; + OS << Name << "\n"; + OS << "# Func Hash:\n" << Hash << "\n"; OS << "# Num Counters:\n" << Func.Counts.size() << "\n"; OS << "# Counter Values:\n"; for (uint64_t Count : Func.Counts) @@ -375,6 +384,6 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { for (const auto &I : FunctionData) if (shouldEncodeData(I.getValue())) for (const auto &Func : I.getValue()) - writeRecordInText(Func.second, Symtab, OS); + writeRecordInText(I.getKey(), Func.first, Func.second, Symtab, OS); return Error::success(); } diff --git a/contrib/llvm/lib/Support/CommandLine.cpp b/contrib/llvm/lib/Support/CommandLine.cpp index 0345a5e3d2a1..50173f5256bf 100644 --- a/contrib/llvm/lib/Support/CommandLine.cpp +++ b/contrib/llvm/lib/Support/CommandLine.cpp @@ -1236,7 +1236,7 @@ bool CommandLineParser::ParseCommandLineOptions(int argc, << ": Not enough positional command line arguments specified!\n" << "Must specify at least " << NumPositionalRequired << " positional argument" << (NumPositionalRequired > 1 ? "s" : "") - << ": See: " << argv[0] << " - help\n"; + << ": See: " << argv[0] << " -help\n"; ErrorParsing = true; } else if (!HasUnlimitedPositionals && diff --git a/contrib/llvm/lib/Support/DynamicLibrary.cpp b/contrib/llvm/lib/Support/DynamicLibrary.cpp index 9398789cea87..d8422115eae8 100644 --- a/contrib/llvm/lib/Support/DynamicLibrary.cpp +++ b/contrib/llvm/lib/Support/DynamicLibrary.cpp @@ -14,6 +14,7 @@ #include "llvm/Support/DynamicLibrary.h" #include "llvm-c/Support.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/Config/config.h" #include "llvm/Support/ManagedStatic.h" @@ -73,19 +74,37 @@ class DynamicLibrary::HandleSet { return true; } - void *Lookup(const char *Symbol) { - // Process handle gets first try. + void *LibLookup(const char *Symbol, DynamicLibrary::SearchOrdering Order) { + if (Order & SO_LoadOrder) { + for (void *Handle : Handles) { + if (void *Ptr = DLSym(Handle, Symbol)) + return Ptr; + } + } else { + for (void *Handle : llvm::reverse(Handles)) { + if (void *Ptr = DLSym(Handle, Symbol)) + return Ptr; + } + } + return nullptr; + } + + void *Lookup(const char *Symbol, DynamicLibrary::SearchOrdering Order) { + assert(!((Order & SO_LoadedFirst) && (Order & SO_LoadedLast)) && + "Invalid Ordering"); + + if (!Process || (Order & SO_LoadedFirst)) { + if (void *Ptr = LibLookup(Symbol, Order)) + return Ptr; + } if (Process) { + // Use OS facilities to search the current binary and all loaded libs. if (void *Ptr = DLSym(Process, Symbol)) return Ptr; -#ifndef NDEBUG - for (void *Handle : Handles) - assert(!DLSym(Handle, Symbol) && "Symbol exists in non process handle"); -#endif - } else { - // Iterate in reverse, so newer libraries/symbols override older. - for (auto &&I = Handles.rbegin(), E = Handles.rend(); I != E; ++I) { - if (void *Ptr = DLSym(*I, Symbol)) + + // Search any libs that might have been skipped because of RTLD_LOCAL. + if (Order & SO_LoadedLast) { + if (void *Ptr = LibLookup(Symbol, Order)) return Ptr; } } @@ -113,6 +132,8 @@ static llvm::ManagedStatic> SymbolsMutex; #endif char DynamicLibrary::Invalid; +DynamicLibrary::SearchOrdering DynamicLibrary::SearchOrder = + DynamicLibrary::SO_Linker; namespace llvm { void *SearchForAddressOfSpecialSymbol(const char *SymbolName) { @@ -170,7 +191,7 @@ void *DynamicLibrary::SearchForAddressOfSymbol(const char *SymbolName) { // Now search the libraries. if (OpenedHandles.isConstructed()) { - if (void *Ptr = OpenedHandles->Lookup(SymbolName)) + if (void *Ptr = OpenedHandles->Lookup(SymbolName, SearchOrder)) return Ptr; } } diff --git a/contrib/llvm/lib/Support/ErrorHandling.cpp b/contrib/llvm/lib/Support/ErrorHandling.cpp index a7d3a18003ee..fe69151665c6 100644 --- a/contrib/llvm/lib/Support/ErrorHandling.cpp +++ b/contrib/llvm/lib/Support/ErrorHandling.cpp @@ -20,15 +20,14 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Mutex.h" -#include "llvm/Support/MutexGuard.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" #include "llvm/Support/WindowsError.h" #include "llvm/Support/raw_ostream.h" #include #include +#include +#include #if defined(HAVE_UNISTD_H) # include @@ -43,18 +42,25 @@ using namespace llvm; static fatal_error_handler_t ErrorHandler = nullptr; static void *ErrorHandlerUserData = nullptr; -static ManagedStatic ErrorHandlerMutex; +static fatal_error_handler_t BadAllocErrorHandler = nullptr; +static void *BadAllocErrorHandlerUserData = nullptr; + +// Mutexes to synchronize installing error handlers and calling error handlers. +// Do not use ManagedStatic, or that may allocate memory while attempting to +// report an OOM. +static std::mutex ErrorHandlerMutex; +static std::mutex BadAllocErrorHandlerMutex; void llvm::install_fatal_error_handler(fatal_error_handler_t handler, void *user_data) { - llvm::MutexGuard Lock(*ErrorHandlerMutex); + std::lock_guard Lock(ErrorHandlerMutex); assert(!ErrorHandler && "Error handler already registered!\n"); ErrorHandler = handler; ErrorHandlerUserData = user_data; } void llvm::remove_fatal_error_handler() { - llvm::MutexGuard Lock(*ErrorHandlerMutex); + std::lock_guard Lock(ErrorHandlerMutex); ErrorHandler = nullptr; ErrorHandlerUserData = nullptr; } @@ -77,7 +83,7 @@ void llvm::report_fatal_error(const Twine &Reason, bool GenCrashDiag) { { // Only acquire the mutex while reading the handler, so as not to invoke a // user-supplied callback under a lock. - llvm::MutexGuard Lock(*ErrorHandlerMutex); + std::lock_guard Lock(ErrorHandlerMutex); handler = ErrorHandler; handlerData = ErrorHandlerUserData; } @@ -104,6 +110,48 @@ void llvm::report_fatal_error(const Twine &Reason, bool GenCrashDiag) { exit(1); } +void llvm::install_bad_alloc_error_handler(fatal_error_handler_t handler, + void *user_data) { + std::lock_guard Lock(BadAllocErrorHandlerMutex); + assert(!ErrorHandler && "Bad alloc error handler already registered!\n"); + BadAllocErrorHandler = handler; + BadAllocErrorHandlerUserData = user_data; +} + +void llvm::remove_bad_alloc_error_handler() { + std::lock_guard Lock(BadAllocErrorHandlerMutex); + BadAllocErrorHandler = nullptr; + BadAllocErrorHandlerUserData = nullptr; +} + +void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { + fatal_error_handler_t Handler = nullptr; + void *HandlerData = nullptr; + { + // Only acquire the mutex while reading the handler, so as not to invoke a + // user-supplied callback under a lock. + std::lock_guard Lock(BadAllocErrorHandlerMutex); + Handler = BadAllocErrorHandler; + HandlerData = BadAllocErrorHandlerUserData; + } + + if (Handler) { + Handler(HandlerData, Reason, GenCrashDiag); + llvm_unreachable("bad alloc handler should not return"); + } + +#ifdef LLVM_ENABLE_EXCEPTIONS + // If exceptions are enabled, make OOM in malloc look like OOM in new. + throw std::bad_alloc(); +#else + // Don't call the normal error handler. It may allocate memory. Directly write + // an OOM to stderr and abort. + char OOMMessage[] = "LLVM ERROR: out of memory\n"; + (void)::write(2, OOMMessage, strlen(OOMMessage)); + abort(); +#endif +} + void llvm::llvm_unreachable_internal(const char *msg, const char *file, unsigned line) { // This code intentionally doesn't call the ErrorHandler callback, because diff --git a/contrib/llvm/lib/Support/Host.cpp b/contrib/llvm/lib/Support/Host.cpp index 232efe648b03..9f22f89b3c9e 100644 --- a/contrib/llvm/lib/Support/Host.cpp +++ b/contrib/llvm/lib/Support/Host.cpp @@ -281,11 +281,17 @@ enum ProcessorVendors { }; enum ProcessorTypes { - INTEL_ATOM = 1, + INTEL_BONNELL = 1, INTEL_CORE2, INTEL_COREI7, AMDFAM10H, AMDFAM15H, + INTEL_SILVERMONT, + INTEL_KNL, + AMD_BTVER1, + AMD_BTVER2, + AMDFAM17H, + // Entries below this are not in libgcc/compiler-rt. INTEL_i386, INTEL_i486, INTEL_PENTIUM, @@ -295,16 +301,13 @@ enum ProcessorTypes { INTEL_PENTIUM_IV, INTEL_PENTIUM_M, INTEL_CORE_DUO, - INTEL_XEONPHI, INTEL_X86_64, INTEL_NOCONA, INTEL_PRESCOTT, AMD_i486, AMDPENTIUM, AMDATHLON, - AMDFAM14H, - AMDFAM16H, - AMDFAM17H, + INTEL_GOLDMONT, CPU_TYPE_MAX }; @@ -317,34 +320,26 @@ enum ProcessorSubtypes { AMDFAM10H_ISTANBUL, AMDFAM15H_BDVER1, AMDFAM15H_BDVER2, - INTEL_PENTIUM_MMX, - INTEL_CORE2_65, - INTEL_CORE2_45, + AMDFAM15H_BDVER3, + AMDFAM15H_BDVER4, + AMDFAM17H_ZNVER1, INTEL_COREI7_IVYBRIDGE, INTEL_COREI7_HASWELL, INTEL_COREI7_BROADWELL, INTEL_COREI7_SKYLAKE, INTEL_COREI7_SKYLAKE_AVX512, - INTEL_ATOM_BONNELL, - INTEL_ATOM_SILVERMONT, - INTEL_ATOM_GOLDMONT, - INTEL_KNIGHTS_LANDING, + // Entries below this are not in libgcc/compiler-rt. + INTEL_PENTIUM_MMX, + INTEL_CORE2_65, + INTEL_CORE2_45, AMDPENTIUM_K6, AMDPENTIUM_K62, AMDPENTIUM_K63, AMDPENTIUM_GEODE, - AMDATHLON_TBIRD, - AMDATHLON_MP, + AMDATHLON_CLASSIC, AMDATHLON_XP, + AMDATHLON_K8, AMDATHLON_K8SSE3, - AMDATHLON_OPTERON, - AMDATHLON_FX, - AMDATHLON_64, - AMD_BTVER1, - AMD_BTVER2, - AMDFAM15H_BDVER3, - AMDFAM15H_BDVER4, - AMDFAM17H_ZNVER1, CPU_SUBTYPE_MAX }; @@ -360,9 +355,28 @@ enum ProcessorFeatures { FEATURE_SSE4_2, FEATURE_AVX, FEATURE_AVX2, - FEATURE_AVX512, - FEATURE_AVX512SAVE, - FEATURE_MOVBE, + FEATURE_SSE4_A, + FEATURE_FMA4, + FEATURE_XOP, + FEATURE_FMA, + FEATURE_AVX512F, + FEATURE_BMI, + FEATURE_BMI2, + FEATURE_AES, + FEATURE_PCLMUL, + FEATURE_AVX512VL, + FEATURE_AVX512BW, + FEATURE_AVX512DQ, + FEATURE_AVX512CD, + FEATURE_AVX512ER, + FEATURE_AVX512PF, + FEATURE_AVX512VBMI, + FEATURE_AVX512IFMA, + FEATURE_AVX5124VNNIW, + FEATURE_AVX5124FMAPS, + FEATURE_AVX512VPOPCNTDQ, + // Only one bit free left in the first 32 features. + FEATURE_MOVBE = 32, FEATURE_ADX, FEATURE_EM64T }; @@ -406,7 +420,6 @@ static bool isCpuIdSupported() { /// the specified arguments. If we can't run cpuid on the host, return true. static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, unsigned *rECX, unsigned *rEDX) { -#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) #if defined(__GNUC__) || defined(__clang__) #if defined(__x86_64__) // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually. @@ -416,14 +429,16 @@ static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, "xchgq\t%%rbx, %%rsi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value)); + return false; #elif defined(__i386__) __asm__("movl\t%%ebx, %%esi\n\t" "cpuid\n\t" "xchgl\t%%ebx, %%esi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value)); + return false; #else - assert(0 && "This method is defined only for x86."); + return true; #endif #elif defined(_MSC_VER) // The MSVC intrinsic is portable across x86 and x64. @@ -433,7 +448,6 @@ static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, *rEBX = registers[1]; *rECX = registers[2]; *rEDX = registers[3]; -#endif return false; #else return true; @@ -446,16 +460,16 @@ static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, unsigned *rEAX, unsigned *rEBX, unsigned *rECX, unsigned *rEDX) { -#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) #if defined(__x86_64__) || defined(_M_X64) #if defined(__GNUC__) || defined(__clang__) - // gcc doesn't know cpuid would clobber ebx/rbx. Preseve it manually. + // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually. // FIXME: should we save this for Clang? __asm__("movq\t%%rbx, %%rsi\n\t" "cpuid\n\t" "xchgq\t%%rbx, %%rsi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value), "c"(subleaf)); + return false; #elif defined(_MSC_VER) int registers[4]; __cpuidex(registers, value, subleaf); @@ -463,6 +477,9 @@ static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, *rEBX = registers[1]; *rECX = registers[2]; *rEDX = registers[3]; + return false; +#else + return true; #endif #elif defined(__i386__) || defined(_M_IX86) #if defined(__GNUC__) || defined(__clang__) @@ -471,6 +488,7 @@ static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, "xchgl\t%%ebx, %%esi\n\t" : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX) : "a"(value), "c"(subleaf)); + return false; #elif defined(_MSC_VER) __asm { mov eax,value @@ -485,16 +503,16 @@ static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, mov esi,rEDX mov dword ptr [esi],edx } -#endif -#else - assert(0 && "This method is defined only for x86."); -#endif return false; #else return true; #endif +#else + return true; +#endif } +// Read control register 0 (XCR0). Used to detect features such as AVX. static bool getX86XCR0(unsigned *rEAX, unsigned *rEDX) { #if defined(__GNUC__) || defined(__clang__) // Check xgetbv; this uses a .byte sequence instead of the instruction @@ -526,9 +544,10 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family, } static void -getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, - unsigned int Brand_id, unsigned int Features, - unsigned *Type, unsigned *Subtype) { +getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, + unsigned Brand_id, unsigned Features, + unsigned Features2, unsigned *Type, + unsigned *Subtype) { if (Brand_id != 0) return; switch (Family) { @@ -681,12 +700,7 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, // Skylake Xeon: case 0x55: *Type = INTEL_COREI7; - // Check that we really have AVX512 - if (Features & (1 << FEATURE_AVX512)) { - *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" - } else { - *Subtype = INTEL_COREI7_SKYLAKE; // "skylake" - } + *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512" break; case 0x1c: // Most 45 nm Intel Atom processors @@ -694,8 +708,7 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, case 0x27: // 32 nm Atom Medfield case 0x35: // 32 nm Atom Midview case 0x36: // 32 nm Atom Midview - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_BONNELL; + *Type = INTEL_BONNELL; break; // "bonnell" // Atom Silvermont codes from the Intel software optimization guide. @@ -705,27 +718,23 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, case 0x5a: case 0x5d: case 0x4c: // really airmont - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_SILVERMONT; + *Type = INTEL_SILVERMONT; break; // "silvermont" // Goldmont: case 0x5c: case 0x5f: - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_GOLDMONT; + *Type = INTEL_GOLDMONT; break; // "goldmont" case 0x57: - *Type = INTEL_XEONPHI; // knl - *Subtype = INTEL_KNIGHTS_LANDING; + *Type = INTEL_KNL; // knl break; default: // Unknown family 6 CPU, try to guess. - if (Features & (1 << FEATURE_AVX512)) { - *Type = INTEL_XEONPHI; // knl - *Subtype = INTEL_KNIGHTS_LANDING; + if (Features & (1 << FEATURE_AVX512F)) { + *Type = INTEL_KNL; // knl break; } - if (Features & (1 << FEATURE_ADX)) { + if (Features2 & (1 << (FEATURE_ADX - 32))) { *Type = INTEL_COREI7; *Subtype = INTEL_COREI7_BROADWELL; break; @@ -741,9 +750,8 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, break; } if (Features & (1 << FEATURE_SSE4_2)) { - if (Features & (1 << FEATURE_MOVBE)) { - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_SILVERMONT; + if (Features2 & (1 << (FEATURE_MOVBE - 32))) { + *Type = INTEL_SILVERMONT; } else { *Type = INTEL_COREI7; *Subtype = INTEL_COREI7_NEHALEM; @@ -756,16 +764,15 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, break; } if (Features & (1 << FEATURE_SSSE3)) { - if (Features & (1 << FEATURE_MOVBE)) { - *Type = INTEL_ATOM; - *Subtype = INTEL_ATOM_BONNELL; // "bonnell" + if (Features2 & (1 << (FEATURE_MOVBE - 32))) { + *Type = INTEL_BONNELL; // "bonnell" } else { *Type = INTEL_CORE2; // "core2" *Subtype = INTEL_CORE2_65; } break; } - if (Features & (1 << FEATURE_EM64T)) { + if (Features2 & (1 << (FEATURE_EM64T - 32))) { *Type = INTEL_X86_64; break; // x86-64 } @@ -796,8 +803,8 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, // Intel Xeon processor, Intel Xeon processor MP, Intel Celeron // processor, and Mobile Intel Celeron processor. All processors // are model 02h and manufactured using the 0.13 micron process. - *Type = - ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV); + *Type = ((Features2 & (1 << (FEATURE_EM64T - 32))) ? INTEL_X86_64 + : INTEL_PENTIUM_IV); break; case 3: // Pentium 4 processor, Intel Xeon processor, Intel Celeron D @@ -811,13 +818,13 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, // Extreme Edition, Intel Xeon processor, Intel Xeon processor // MP, Intel Celeron D processor. All processors are model 06h // and manufactured using the 65 nm process. - *Type = - ((Features & (1 << FEATURE_EM64T)) ? INTEL_NOCONA : INTEL_PRESCOTT); + *Type = ((Features2 & (1 << (FEATURE_EM64T - 32))) ? INTEL_NOCONA + : INTEL_PRESCOTT); break; default: - *Type = - ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV); + *Type = ((Features2 & (1 << (FEATURE_EM64T - 32))) ? INTEL_X86_64 + : INTEL_PENTIUM_IV); break; } break; @@ -827,10 +834,8 @@ getIntelProcessorTypeAndSubtype(unsigned int Family, unsigned int Model, } } -static void getAMDProcessorTypeAndSubtype(unsigned int Family, - unsigned int Model, - unsigned int Features, - unsigned *Type, +static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, + unsigned Features, unsigned *Type, unsigned *Subtype) { // FIXME: this poorly matches the generated SubtargetFeatureKV table. There // appears to be no way to generate the wide variety of AMD-specific targets @@ -860,38 +865,20 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family, break; case 6: *Type = AMDATHLON; - switch (Model) { - case 4: - *Subtype = AMDATHLON_TBIRD; - break; // "athlon-tbird" - case 6: - case 7: - case 8: - *Subtype = AMDATHLON_MP; - break; // "athlon-mp" - case 10: + if (Features & (1 << FEATURE_SSE)) { *Subtype = AMDATHLON_XP; break; // "athlon-xp" } - break; + *Subtype = AMDATHLON_CLASSIC; + break; // "athlon" case 15: *Type = AMDATHLON; if (Features & (1 << FEATURE_SSE3)) { *Subtype = AMDATHLON_K8SSE3; break; // "k8-sse3" } - switch (Model) { - case 1: - *Subtype = AMDATHLON_OPTERON; - break; // "opteron" - case 5: - *Subtype = AMDATHLON_FX; - break; // "athlon-fx"; also opteron - default: - *Subtype = AMDATHLON_64; - break; // "athlon64" - } - break; + *Subtype = AMDATHLON_K8; + break; // "k8" case 16: *Type = AMDFAM10H; // "amdfam10" switch (Model) { @@ -907,19 +894,13 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family, } break; case 20: - *Type = AMDFAM14H; - *Subtype = AMD_BTVER1; + *Type = AMD_BTVER1; break; // "btver1"; case 21: *Type = AMDFAM15H; - if (!(Features & - (1 << FEATURE_AVX))) { // If no AVX support, provide a sane fallback. - *Subtype = AMD_BTVER1; - break; // "btver1" - } - if (Model >= 0x50 && Model <= 0x6f) { + if (Model >= 0x60 && Model <= 0x7f) { *Subtype = AMDFAM15H_BDVER4; - break; // "bdver4"; 50h-6Fh: Excavator + break; // "bdver4"; 60h-7Fh: Excavator } if (Model >= 0x30 && Model <= 0x3f) { *Subtype = AMDFAM15H_BDVER3; @@ -935,39 +916,52 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family, } break; case 22: - *Type = AMDFAM16H; - if (!(Features & - (1 << FEATURE_AVX))) { // If no AVX support provide a sane fallback. - *Subtype = AMD_BTVER1; - break; // "btver1"; - } - *Subtype = AMD_BTVER2; + *Type = AMD_BTVER2; break; // "btver2" case 23: *Type = AMDFAM17H; - if (Features & (1 << FEATURE_ADX)) { - *Subtype = AMDFAM17H_ZNVER1; - break; // "znver1" - } - *Subtype = AMD_BTVER1; + *Subtype = AMDFAM17H_ZNVER1; break; default: break; // "generic" } } -static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX, - unsigned MaxLeaf) { +static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, + unsigned *FeaturesOut, + unsigned *Features2Out) { unsigned Features = 0; - unsigned int EAX, EBX; - Features |= (((EDX >> 23) & 1) << FEATURE_MMX); - Features |= (((EDX >> 25) & 1) << FEATURE_SSE); - Features |= (((EDX >> 26) & 1) << FEATURE_SSE2); - Features |= (((ECX >> 0) & 1) << FEATURE_SSE3); - Features |= (((ECX >> 9) & 1) << FEATURE_SSSE3); - Features |= (((ECX >> 19) & 1) << FEATURE_SSE4_1); - Features |= (((ECX >> 20) & 1) << FEATURE_SSE4_2); - Features |= (((ECX >> 22) & 1) << FEATURE_MOVBE); + unsigned Features2 = 0; + unsigned EAX, EBX; + + if ((EDX >> 15) & 1) + Features |= 1 << FEATURE_CMOV; + if ((EDX >> 23) & 1) + Features |= 1 << FEATURE_MMX; + if ((EDX >> 25) & 1) + Features |= 1 << FEATURE_SSE; + if ((EDX >> 26) & 1) + Features |= 1 << FEATURE_SSE2; + + if ((ECX >> 0) & 1) + Features |= 1 << FEATURE_SSE3; + if ((ECX >> 1) & 1) + Features |= 1 << FEATURE_PCLMUL; + if ((ECX >> 9) & 1) + Features |= 1 << FEATURE_SSSE3; + if ((ECX >> 12) & 1) + Features |= 1 << FEATURE_FMA; + if ((ECX >> 19) & 1) + Features |= 1 << FEATURE_SSE4_1; + if ((ECX >> 20) & 1) + Features |= 1 << FEATURE_SSE4_2; + if ((ECX >> 23) & 1) + Features |= 1 << FEATURE_POPCNT; + if ((ECX >> 25) & 1) + Features |= 1 << FEATURE_AES; + + if ((ECX >> 22) & 1) + Features2 |= 1 << (FEATURE_MOVBE - 32); // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV // indicates that the AVX registers will be saved and restored on context @@ -976,20 +970,65 @@ static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX, bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) && ((EAX & 0x6) == 0x6); bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0); + + if (HasAVX) + Features |= 1 << FEATURE_AVX; + bool HasLeaf7 = MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); - bool HasADX = HasLeaf7 && ((EBX >> 19) & 1); - bool HasAVX2 = HasAVX && HasLeaf7 && (EBX & 0x20); - bool HasAVX512 = HasLeaf7 && HasAVX512Save && ((EBX >> 16) & 1); - Features |= (HasAVX << FEATURE_AVX); - Features |= (HasAVX2 << FEATURE_AVX2); - Features |= (HasAVX512 << FEATURE_AVX512); - Features |= (HasAVX512Save << FEATURE_AVX512SAVE); - Features |= (HasADX << FEATURE_ADX); - getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); - Features |= (((EDX >> 29) & 0x1) << FEATURE_EM64T); - return Features; + if (HasLeaf7 && ((EBX >> 3) & 1)) + Features |= 1 << FEATURE_BMI; + if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX) + Features |= 1 << FEATURE_AVX2; + if (HasLeaf7 && ((EBX >> 9) & 1)) + Features |= 1 << FEATURE_BMI2; + if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512F; + if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512DQ; + if (HasLeaf7 && ((EBX >> 19) & 1)) + Features2 |= 1 << (FEATURE_ADX - 32); + if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512IFMA; + if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512PF; + if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512ER; + if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512CD; + if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512BW; + if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512VL; + + if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512VBMI; + if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX512VPOPCNTDQ; + + if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX5124VNNIW; + if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save) + Features |= 1 << FEATURE_AVX5124FMAPS; + + unsigned MaxExtLevel; + getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX); + + bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 && + !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); + if (HasExtLeaf1 && ((ECX >> 6) & 1)) + Features |= 1 << FEATURE_SSE4_A; + if (HasExtLeaf1 && ((ECX >> 11) & 1)) + Features |= 1 << FEATURE_XOP; + if (HasExtLeaf1 && ((ECX >> 16) & 1)) + Features |= 1 << FEATURE_FMA4; + + if (HasExtLeaf1 && ((EDX >> 29) & 1)) + Features2 |= 1 << (FEATURE_EM64T - 32); + + *FeaturesOut = Features; + *Features2Out = Features2; } StringRef sys::getHostCPUName() { @@ -1004,23 +1043,22 @@ StringRef sys::getHostCPUName() { if(!isCpuIdSupported()) return "generic"; #endif - if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX)) - return "generic"; - if (getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX)) + if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) return "generic"; + getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX); unsigned Brand_id = EBX & 0xff; unsigned Family = 0, Model = 0; - unsigned Features = 0; + unsigned Features = 0, Features2 = 0; detectX86FamilyModel(EAX, &Family, &Model); - Features = getAvailableFeatures(ECX, EDX, MaxLeaf); + getAvailableFeatures(ECX, EDX, MaxLeaf, &Features, &Features2); unsigned Type; unsigned Subtype; if (Vendor == SIG_INTEL) { - getIntelProcessorTypeAndSubtype(Family, Model, Brand_id, Features, &Type, - &Subtype); + getIntelProcessorTypeAndSubtype(Family, Model, Brand_id, Features, + Features2, &Type, &Subtype); switch (Type) { case INTEL_i386: return "i386"; @@ -1049,7 +1087,7 @@ StringRef sys::getHostCPUName() { case INTEL_CORE2_45: return "penryn"; default: - return "core2"; + llvm_unreachable("Unexpected subtype!"); } case INTEL_COREI7: switch (Subtype) { @@ -1070,21 +1108,16 @@ StringRef sys::getHostCPUName() { case INTEL_COREI7_SKYLAKE_AVX512: return "skylake-avx512"; default: - return "corei7"; + llvm_unreachable("Unexpected subtype!"); } - case INTEL_ATOM: - switch (Subtype) { - case INTEL_ATOM_BONNELL: - return "bonnell"; - case INTEL_ATOM_GOLDMONT: - return "goldmont"; - case INTEL_ATOM_SILVERMONT: - return "silvermont"; - default: - return "atom"; - } - case INTEL_XEONPHI: - return "knl"; /*update for more variants added*/ + case INTEL_BONNELL: + return "bonnell"; + case INTEL_SILVERMONT: + return "silvermont"; + case INTEL_GOLDMONT: + return "goldmont"; + case INTEL_KNL: + return "knl"; case INTEL_X86_64: return "x86-64"; case INTEL_NOCONA: @@ -1092,7 +1125,7 @@ StringRef sys::getHostCPUName() { case INTEL_PRESCOTT: return "prescott"; default: - return "generic"; + break; } } else if (Vendor == SIG_AMD) { getAMDProcessorTypeAndSubtype(Family, Model, Features, &Type, &Subtype); @@ -1114,31 +1147,24 @@ StringRef sys::getHostCPUName() { } case AMDATHLON: switch (Subtype) { - case AMDATHLON_TBIRD: - return "athlon-tbird"; - case AMDATHLON_MP: - return "athlon-mp"; + case AMDATHLON_CLASSIC: + return "athlon"; case AMDATHLON_XP: return "athlon-xp"; + case AMDATHLON_K8: + return "k8"; case AMDATHLON_K8SSE3: return "k8-sse3"; - case AMDATHLON_OPTERON: - return "opteron"; - case AMDATHLON_FX: - return "athlon-fx"; - case AMDATHLON_64: - return "athlon64"; default: - return "athlon"; + llvm_unreachable("Unexpected subtype!"); } case AMDFAM10H: - if(Subtype == AMDFAM10H_BARCELONA) - return "barcelona"; return "amdfam10"; - case AMDFAM14H: + case AMD_BTVER1: return "btver1"; case AMDFAM15H: switch (Subtype) { + default: // There are gaps in the subtype detection. case AMDFAM15H_BDVER1: return "bdver1"; case AMDFAM15H_BDVER2: @@ -1147,31 +1173,13 @@ StringRef sys::getHostCPUName() { return "bdver3"; case AMDFAM15H_BDVER4: return "bdver4"; - case AMD_BTVER1: - return "btver1"; - default: - return "amdfam15"; - } - case AMDFAM16H: - switch (Subtype) { - case AMD_BTVER1: - return "btver1"; - case AMD_BTVER2: - return "btver2"; - default: - return "amdfam16"; } + case AMD_BTVER2: + return "btver2"; case AMDFAM17H: - switch (Subtype) { - case AMD_BTVER1: - return "btver1"; - case AMDFAM17H_ZNVER1: - return "znver1"; - default: - return "amdfam17"; - } + return "znver1"; default: - return "generic"; + break; } } return "generic"; @@ -1494,7 +1502,8 @@ bool sys::getHostCPUFeatures(StringMap &Features) { return false; } #endif std::string sys::getProcessTriple() { - Triple PT(Triple::normalize(LLVM_HOST_TRIPLE)); + std::string TargetTripleString = updateTripleOSVersion(LLVM_HOST_TRIPLE); + Triple PT(Triple::normalize(TargetTripleString)); if (sizeof(void *) == 8 && PT.isArch32Bit()) PT = PT.get64BitArchVariant(); diff --git a/contrib/llvm/lib/Support/Mutex.cpp b/contrib/llvm/lib/Support/Mutex.cpp index bdd02105f6f0..b1d5e7c0d991 100644 --- a/contrib/llvm/lib/Support/Mutex.cpp +++ b/contrib/llvm/lib/Support/Mutex.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/Mutex.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Config/config.h" //===----------------------------------------------------------------------===// @@ -47,6 +48,10 @@ MutexImpl::MutexImpl( bool recursive) // Declare the pthread_mutex data structures pthread_mutex_t* mutex = static_cast(malloc(sizeof(pthread_mutex_t))); + + if (mutex == nullptr) + report_bad_alloc_error("Mutex allocation failed"); + pthread_mutexattr_t attr; // Initialize the mutex attributes diff --git a/contrib/llvm/lib/Support/Unix/DynamicLibrary.inc b/contrib/llvm/lib/Support/Unix/DynamicLibrary.inc index aad77f19c35a..f05103ccd1eb 100644 --- a/contrib/llvm/lib/Support/Unix/DynamicLibrary.inc +++ b/contrib/llvm/lib/Support/Unix/DynamicLibrary.inc @@ -20,6 +20,9 @@ DynamicLibrary::HandleSet::~HandleSet() { ::dlclose(Handle); if (Process) ::dlclose(Process); + + // llvm_shutdown called, Return to default + DynamicLibrary::SearchOrder = DynamicLibrary::SO_Linker; } void *DynamicLibrary::HandleSet::DLOpen(const char *File, std::string *Err) { diff --git a/contrib/llvm/lib/Support/Unix/Host.inc b/contrib/llvm/lib/Support/Unix/Host.inc index 0ba6a25aa198..5580e63893c6 100644 --- a/contrib/llvm/lib/Support/Unix/Host.inc +++ b/contrib/llvm/lib/Support/Unix/Host.inc @@ -34,18 +34,31 @@ static std::string getOSVersion() { return info.release; } -std::string sys::getDefaultTargetTriple() { - std::string TargetTripleString(LLVM_DEFAULT_TARGET_TRIPLE); - - // On darwin, we want to update the version to match that of the - // target. +static std::string updateTripleOSVersion(std::string TargetTripleString) { + // On darwin, we want to update the version to match that of the target. std::string::size_type DarwinDashIdx = TargetTripleString.find("-darwin"); if (DarwinDashIdx != std::string::npos) { TargetTripleString.resize(DarwinDashIdx + strlen("-darwin")); TargetTripleString += getOSVersion(); + return TargetTripleString; } + std::string::size_type MacOSDashIdx = TargetTripleString.find("-macos"); + if (MacOSDashIdx != std::string::npos) { + TargetTripleString.resize(MacOSDashIdx); + // Reset the OS to darwin as the OS version from `uname` doesn't use the + // macOS version scheme. + TargetTripleString += "-darwin"; + TargetTripleString += getOSVersion(); + } + return TargetTripleString; +} - // Override the default target with an environment variable named by LLVM_TARGET_TRIPLE_ENV. +std::string sys::getDefaultTargetTriple() { + std::string TargetTripleString = + updateTripleOSVersion(LLVM_DEFAULT_TARGET_TRIPLE); + + // Override the default target with an environment variable named by + // LLVM_TARGET_TRIPLE_ENV. #if defined(LLVM_TARGET_TRIPLE_ENV) if (const char *EnvTriple = std::getenv(LLVM_TARGET_TRIPLE_ENV)) TargetTripleString = EnvTriple; diff --git a/contrib/llvm/lib/Support/Unix/Program.inc b/contrib/llvm/lib/Support/Unix/Program.inc index 1704fa479942..c866d5b5a84e 100644 --- a/contrib/llvm/lib/Support/Unix/Program.inc +++ b/contrib/llvm/lib/Support/Unix/Program.inc @@ -40,9 +40,6 @@ #include #endif #ifdef HAVE_POSIX_SPAWN -#ifdef __sun__ -#define _RESTRICT_KYWD -#endif #include #if defined(__APPLE__) diff --git a/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc b/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc index caf1a0a658de..083ea902eeb2 100644 --- a/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc +++ b/contrib/llvm/lib/Support/Windows/DynamicLibrary.inc @@ -28,6 +28,8 @@ DynamicLibrary::HandleSet::~HandleSet() { // 'Process' should not be released on Windows. assert((!Process || Process==this) && "Bad Handle"); + // llvm_shutdown called, Return to default + DynamicLibrary::SearchOrder = DynamicLibrary::SO_Linker; } void *DynamicLibrary::HandleSet::DLOpen(const char *File, std::string *Err) { diff --git a/contrib/llvm/lib/Support/Windows/Host.inc b/contrib/llvm/lib/Support/Windows/Host.inc index 7e196cf0ce18..90a6fb316703 100644 --- a/contrib/llvm/lib/Support/Windows/Host.inc +++ b/contrib/llvm/lib/Support/Windows/Host.inc @@ -17,6 +17,10 @@ using namespace llvm; +static std::string updateTripleOSVersion(std::string Triple) { + return Triple; +} + std::string sys::getDefaultTargetTriple() { const char *Triple = LLVM_DEFAULT_TARGET_TRIPLE; diff --git a/contrib/llvm/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/contrib/llvm/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp index 4a7e0b2b803e..db1fbe069f4d 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp @@ -509,7 +509,7 @@ int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C, assert(ChainBegin != ChainEnd && "Chain should contain instructions"); do { --I; - Units.accumulateBackward(*I); + Units.accumulate(*I); } while (I != ChainBegin); // Make sure we allocate in-order, to get the cheapest registers first. diff --git a/contrib/llvm/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp b/contrib/llvm/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp index 6f8dd3e3ac0c..b3b738584b40 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp @@ -113,7 +113,7 @@ struct LDTLSCleanup : public MachineFunctionPass { return Copy; } - // Create a virtal register in *TLSBaseAddrReg, and populate it by + // Create a virtual register in *TLSBaseAddrReg, and populate it by // inserting a copy instruction after I. Returns the new instruction. MachineInstr *setRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { MachineFunction *MF = I.getParent()->getParent(); diff --git a/contrib/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp b/contrib/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp index 0a948812ff33..51700f905979 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp @@ -167,6 +167,7 @@ bool AArch64CondBrTuning::tryToTuneBranch(MachineInstr &MI, case AArch64::SUBWrs: case AArch64::SUBWrx: IsFlagSetting = false; + LLVM_FALLTHROUGH; case AArch64::ADDSWri: case AArch64::ADDSWrr: case AArch64::ADDSWrs: @@ -226,6 +227,7 @@ bool AArch64CondBrTuning::tryToTuneBranch(MachineInstr &MI, case AArch64::SUBXrs: case AArch64::SUBXrx: IsFlagSetting = false; + LLVM_FALLTHROUGH; case AArch64::ADDSXri: case AArch64::ADDSXrr: case AArch64::ADDSXrs: diff --git a/contrib/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/contrib/llvm/lib/Target/AArch64/AArch64FastISel.cpp index 7bf2097c17ce..3682b62d2b84 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -2114,7 +2114,7 @@ bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr, switch (VT.SimpleTy) { default: llvm_unreachable("Unexpected value type."); - case MVT::i1: VTIsi1 = true; + case MVT::i1: VTIsi1 = true; LLVM_FALLTHROUGH; case MVT::i8: Opc = OpcTable[Idx][0]; break; case MVT::i16: Opc = OpcTable[Idx][1]; break; case MVT::i32: Opc = OpcTable[Idx][2]; break; diff --git a/contrib/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/contrib/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index aaf32a499bc3..60fde5caa339 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -8364,9 +8364,9 @@ static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount, /// EXTR instruction extracts a contiguous chunk of bits from two existing /// registers viewed as a high/low pair. This function looks for the pattern: -/// (or (shl VAL1, #N), (srl VAL2, #RegWidth-N)) and replaces it with an -/// EXTR. Can't quite be done in TableGen because the two immediates aren't -/// independent. +/// (or (shl VAL1, \#N), (srl VAL2, \#RegWidth-N)) and replaces it +/// with an EXTR. Can't quite be done in TableGen because the two immediates +/// aren't independent. static SDValue tryCombineToEXTR(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { SelectionDAG &DAG = DCI.DAG; @@ -9531,7 +9531,7 @@ static SDValue performPostLD1Combine(SDNode *N, return SDValue(); } -/// Simplify \Addr given that the top byte of it is ignored by HW during +/// Simplify ``Addr`` given that the top byte of it is ignored by HW during /// address translation. static bool performTBISimplification(SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, diff --git a/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 314e89bbca86..dba3e4bdf82f 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -1282,6 +1282,7 @@ static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) { case AArch64CC::HI: // Z clear and C set case AArch64CC::LS: // Z set or C clear UsedFlags.Z = true; + LLVM_FALLTHROUGH; case AArch64CC::HS: // C set case AArch64CC::LO: // C clear UsedFlags.C = true; @@ -1300,6 +1301,7 @@ static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) { case AArch64CC::GT: // Z clear, N and V the same case AArch64CC::LE: // Z set, N and V differ UsedFlags.Z = true; + LLVM_FALLTHROUGH; case AArch64CC::GE: // N and V the same case AArch64CC::LT: // N and V differ UsedFlags.N = true; @@ -3669,12 +3671,17 @@ enum class FMAInstKind { Default, Indexed, Accumulator }; /// F|MUL I=A,B,0 /// F|ADD R,I,C /// ==> F|MADD R,A,B,C +/// \param MF Containing MachineFunction +/// \param MRI Register information +/// \param TII Target information /// \param Root is the F|ADD instruction /// \param [out] InsInstrs is a vector of machine instructions and will /// contain the generated madd instruction /// \param IdxMulOpd is index of operand in Root that is the result of /// the F|MUL. In the example above IdxMulOpd is 1. /// \param MaddOpc the opcode fo the f|madd instruction +/// \param RC Register class of operands +/// \param kind of fma instruction (addressing mode) to be generated static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, @@ -3733,6 +3740,9 @@ genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, /// ADD R,I,Imm /// ==> ORR V, ZR, Imm /// ==> MADD R,A,B,V +/// \param MF Containing MachineFunction +/// \param MRI Register information +/// \param TII Target information /// \param Root is the ADD instruction /// \param [out] InsInstrs is a vector of machine instructions and will /// contain the generated madd instruction @@ -3741,6 +3751,7 @@ genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, /// \param MaddOpc the opcode fo the madd instruction /// \param VR is a virtual register that holds the value of an ADD operand /// (V in the example above). +/// \param RC Register class of operands static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl &InsInstrs, @@ -4216,26 +4227,36 @@ void AArch64InstrInfo::genAlternativeCodeSequence( /// \brief Replace csincr-branch sequence by simple conditional branch /// /// Examples: -/// 1. +/// 1. \code /// csinc w9, wzr, wzr, /// tbnz w9, #0, 0x44 +/// \endcode /// to +/// \code /// b. +/// \endcode /// -/// 2. +/// 2. \code /// csinc w9, wzr, wzr, /// tbz w9, #0, 0x44 +/// \endcode /// to +/// \code /// b. +/// \endcode /// /// Replace compare and branch sequence by TBZ/TBNZ instruction when the /// compare's constant operand is power of 2. /// /// Examples: +/// \code /// and w8, w8, #0x400 /// cbnz w8, L1 +/// \endcode /// to +/// \code /// tbnz w8, #10, L1 +/// \endcode /// /// \param MI Conditional Branch /// \return True when the simple conditional branch is generated @@ -4409,6 +4430,13 @@ AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { return makeArrayRef(TargetFlags); } +ArrayRef> +AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const { + static const std::pair TargetFlags[] = + {{MOSuppressPair, "aarch64-suppress-pair"}}; + return makeArrayRef(TargetFlags); +} + unsigned AArch64InstrInfo::getOutliningBenefit(size_t SequenceSize, size_t Occurrences, bool CanBeTailCall) const { diff --git a/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 58e9ce583d44..0809ede4df2a 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -263,8 +263,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { /// \param Pattern - combiner pattern bool isThroughputPattern(MachineCombinerPattern Pattern) const override; /// Return true when there is potentially a faster code sequence - /// for an instruction chain ending in . All potential patterns are - /// listed in the array. + /// for an instruction chain ending in ``Root``. All potential patterns are + /// listed in the ``Patterns`` array. bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl &Patterns) const override; @@ -289,6 +289,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { getSerializableDirectMachineOperandTargetFlags() const override; ArrayRef> getSerializableBitmaskMachineOperandTargetFlags() const override; + ArrayRef> + getSerializableMachineMemOperandTargetFlags() const override; bool isFunctionSafeToOutlineFrom(MachineFunction &MF) const override; unsigned getOutliningBenefit(size_t SequenceSize, size_t Occurrences, diff --git a/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 6cb723d187af..0be14673eb20 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/contrib/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -313,9 +313,6 @@ def AArch64umaxv : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>; //===----------------------------------------------------------------------===// // AArch64 Instruction Predicate Definitions. -def IsDarwin : Predicate<"Subtarget->isTargetDarwin()">; -def IsNotDarwin: Predicate<"!Subtarget->isTargetDarwin()">; - // We could compute these on a per-module basis but doing so requires accessing // the Function object through the Subtarget and objections were raised // to that (see post-commit review comments for r301750). @@ -714,10 +711,10 @@ def : InstAlias<"negs $dst, $src$shift", defm UDIV : Div<0, "udiv", udiv>; defm SDIV : Div<1, "sdiv", sdiv>; -def : Pat<(int_aarch64_udiv GPR32:$Rn, GPR32:$Rm), (UDIVWr $Rn, $Rm)>; -def : Pat<(int_aarch64_udiv GPR64:$Rn, GPR64:$Rm), (UDIVXr $Rn, $Rm)>; -def : Pat<(int_aarch64_sdiv GPR32:$Rn, GPR32:$Rm), (SDIVWr $Rn, $Rm)>; -def : Pat<(int_aarch64_sdiv GPR64:$Rn, GPR64:$Rm), (SDIVXr $Rn, $Rm)>; +def : Pat<(int_aarch64_udiv GPR32:$Rn, GPR32:$Rm), (UDIVWr GPR32:$Rn, GPR32:$Rm)>; +def : Pat<(int_aarch64_udiv GPR64:$Rn, GPR64:$Rm), (UDIVXr GPR64:$Rn, GPR64:$Rm)>; +def : Pat<(int_aarch64_sdiv GPR32:$Rn, GPR32:$Rm), (SDIVWr GPR32:$Rn, GPR32:$Rm)>; +def : Pat<(int_aarch64_sdiv GPR64:$Rn, GPR64:$Rm), (SDIVXr GPR64:$Rn, GPR64:$Rm)>; // Variable shift defm ASRV : Shift<0b10, "asr", sra>; diff --git a/contrib/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp b/contrib/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp index 07ce0e863c5e..7e275e4d2f46 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -33,6 +33,8 @@ #define DEBUG_TYPE "aarch64-isel" +#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" + using namespace llvm; #ifndef LLVM_BUILD_GLOBAL_ISEL @@ -212,6 +214,7 @@ static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, return GenericOpc; } } + break; case AArch64::FPRRegBankID: switch (OpSize) { case 32: @@ -243,7 +246,8 @@ static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, return GenericOpc; } } - }; + break; + } return GenericOpc; } @@ -267,6 +271,7 @@ static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, case 64: return isStore ? AArch64::STRXui : AArch64::LDRXui; } + break; case AArch64::FPRRegBankID: switch (OpSize) { case 8: @@ -278,7 +283,8 @@ static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, case 64: return isStore ? AArch64::STRDui : AArch64::LDRDui; } - }; + break; + } return GenericOpc; } @@ -1319,6 +1325,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I) const { case TargetOpcode::G_VASTART: return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI) : selectVaStartAAPCS(I, MF, MRI); + case TargetOpcode::G_IMPLICIT_DEF: + I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF)); + return true; } return false; diff --git a/contrib/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/contrib/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp index 4b568f3fba2b..4a0a7c36baf8 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -291,11 +291,10 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, unsigned DstPtr; if (Align > PtrSize) { // Realign the list to the actual required alignment. - unsigned AlignMinus1 = MRI.createGenericVirtualRegister(IntPtrTy); - MIRBuilder.buildConstant(AlignMinus1, Align - 1); + auto AlignMinus1 = MIRBuilder.buildConstant(IntPtrTy, Align - 1); unsigned ListTmp = MRI.createGenericVirtualRegister(PtrTy); - MIRBuilder.buildGEP(ListTmp, List, AlignMinus1); + MIRBuilder.buildGEP(ListTmp, List, AlignMinus1->getOperand(0).getReg()); DstPtr = MRI.createGenericVirtualRegister(PtrTy); MIRBuilder.buildPtrMask(DstPtr, ListTmp, Log2_64(Align)); diff --git a/contrib/llvm/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp b/contrib/llvm/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp index f3c8e7e9bdc2..4e65c0ab6011 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp @@ -163,6 +163,7 @@ AArch64RedundantCopyElimination::knownRegValInBlock( case AArch64::ADDSWri: case AArch64::ADDSXri: IsCMN = true; + LLVM_FALLTHROUGH; // CMP is an alias for SUBS with a dead destination register. case AArch64::SUBSWri: case AArch64::SUBSXri: { diff --git a/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index a9a9d5ce8429..a3238cf3b60f 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -81,6 +81,7 @@ void AArch64Subtarget::initializeProperties() { break; case CortexA57: MaxInterleaveFactor = 4; + PrefFunctionAlignment = 4; break; case ExynosM1: MaxInterleaveFactor = 4; @@ -130,7 +131,9 @@ void AArch64Subtarget::initializeProperties() { break; case CortexA35: break; case CortexA53: break; - case CortexA72: break; + case CortexA72: + PrefFunctionAlignment = 4; + break; case CortexA73: break; case Others: break; } diff --git a/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.h b/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.h index 7933e58c49ee..db53946cbc77 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/contrib/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -218,6 +218,13 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo { bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; } bool hasFuseAES() const { return HasFuseAES; } bool hasFuseLiterals() const { return HasFuseLiterals; } + + /// \brief Return true if the CPU supports any kind of instruction fusion. + bool hasFusion() const { + return hasArithmeticBccFusion() || hasArithmeticCbzFusion() || + hasFuseAES() || hasFuseLiterals(); + } + bool useRSqrt() const { return UseRSqrt; } unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; } unsigned getVectorInsertExtractBaseCost() const { diff --git a/contrib/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/contrib/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index 1252f9403812..6237b8f3e7b9 100644 --- a/contrib/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/contrib/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -277,17 +277,19 @@ class AArch64PassConfig : public TargetPassConfig { ScheduleDAGInstrs * createMachineScheduler(MachineSchedContext *C) const override { + const AArch64Subtarget &ST = C->MF->getSubtarget(); ScheduleDAGMILive *DAG = createGenericSchedLive(C); DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); - DAG->addMutation(createAArch64MacroFusionDAGMutation()); + if (ST.hasFusion()) + DAG->addMutation(createAArch64MacroFusionDAGMutation()); return DAG; } ScheduleDAGInstrs * createPostMachineScheduler(MachineSchedContext *C) const override { const AArch64Subtarget &ST = C->MF->getSubtarget(); - if (ST.hasFuseAES() || ST.hasFuseLiterals()) { + if (ST.hasFusion()) { // Run the Macro Fusion after RA again since literals are expanded from // pseudos then (v. addPreSched2()). ScheduleDAGMI *DAG = createGenericSchedPostRA(C); diff --git a/contrib/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/contrib/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index 0d860a7eef79..7870dce5c9c0 100644 --- a/contrib/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/contrib/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -756,7 +756,7 @@ static DecodeStatus DecodeThreeAddrSRegInstruction(llvm::MCInst &Inst, // if shift == '11' then ReservedValue() if (shiftHi == 0x3) return Fail; - // Deliberate fallthrough + LLVM_FALLTHROUGH; case AArch64::ANDWrs: case AArch64::ANDSWrs: case AArch64::BICWrs: @@ -780,7 +780,7 @@ static DecodeStatus DecodeThreeAddrSRegInstruction(llvm::MCInst &Inst, // if shift == '11' then ReservedValue() if (shiftHi == 0x3) return Fail; - // Deliberate fallthrough + LLVM_FALLTHROUGH; case AArch64::ANDXrs: case AArch64::ANDSXrs: case AArch64::BICXrs: diff --git a/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index 475f91016840..a7a7daf4b4a5 100644 --- a/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -73,7 +73,7 @@ class AArch64AsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; bool mayNeedRelaxation(const MCInst &Inst) const override; bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, @@ -264,7 +264,7 @@ unsigned AArch64AsmBackend::getFixupKindContainereSizeInBytes(unsigned Kind) con void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind()); if (!Value) return; // Doesn't change encoding. diff --git a/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp index fc808ee0cdd6..c25bd8c8f6cc 100644 --- a/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp +++ b/contrib/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp @@ -103,4 +103,6 @@ AArch64MCAsmInfoELF::AArch64MCAsmInfoELF(const Triple &T) { AArch64MCAsmInfoCOFF::AArch64MCAsmInfoCOFF() { CommentString = ";"; + PrivateGlobalPrefix = ".L"; + PrivateLabelPrefix = ".L"; } diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp index 6f002860044c..ed5370826647 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp @@ -108,10 +108,11 @@ bool AMDGPUAnnotateUniformValues::isClobberedInFunction(LoadInst * Load) { DFS(Start, Checklist); for (auto &BB : Checklist) { BasicBlock::iterator StartIt = (!L && (BB == Load->getParent())) ? - BasicBlock::iterator(Load) : BB->end(); - if (MDR->getPointerDependencyFrom(MemoryLocation(Ptr), - true, StartIt, BB, Load).isClobber()) - return true; + BasicBlock::iterator(Load) : BB->end(); + auto Q = MDR->getPointerDependencyFrom(MemoryLocation(Ptr), true, + StartIt, BB, Load); + if (Q.isClobber() || Q.isUnknown()) + return true; } return false; } diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index b312dbc8d14d..31ee9206ae27 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -380,7 +380,9 @@ bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) { FastMathFlags FMF = FPOp->getFastMathFlags(); bool UnsafeDiv = HasUnsafeFPMath || FMF.unsafeAlgebra() || FMF.allowReciprocal(); - if (ST->hasFP32Denormals() && !UnsafeDiv) + + // With UnsafeDiv node will be optimized to just rcp and mul. + if (ST->hasFP32Denormals() || UnsafeDiv) return false; IRBuilder<> Builder(FDiv.getParent(), std::next(FDiv.getIterator()), FPMath); diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 96f819fd0e68..2553cf4da0fe 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -2651,8 +2651,11 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N, SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0)); return DAG.getZExtOrTrunc(Shl, SL, VT); } - case ISD::OR: if (!isOrEquivalentToAdd(DAG, LHS)) break; - case ISD::ADD: { // Fall through from above + case ISD::OR: + if (!isOrEquivalentToAdd(DAG, LHS)) + break; + LLVM_FALLTHROUGH; + case ISD::ADD: { // shl (or|add x, c2), c1 => or|add (shl x, c1), (c2 << c1) if (ConstantSDNode *C2 = dyn_cast(LHS->getOperand(1))) { SDValue Shl = DAG.getNode(ISD::SHL, SL, VT, LHS->getOperand(0), diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp index 846e7dff5f8c..7e0e9802c0e6 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp @@ -10,6 +10,7 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -34,9 +35,14 @@ class AMDGPULowerIntrinsics : public ModulePass { AMDGPULowerIntrinsics() : ModulePass(ID) {} bool runOnModule(Module &M) override; + bool expandMemIntrinsicUses(Function &F); StringRef getPassName() const override { return "AMDGPU Lower Intrinsics"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + } }; } @@ -55,7 +61,7 @@ static bool shouldExpandOperationWithSize(Value *Size) { return !CI || (CI->getZExtValue() > MaxStaticSize); } -static bool expandMemIntrinsicUses(Function &F) { +bool AMDGPULowerIntrinsics::expandMemIntrinsicUses(Function &F) { Intrinsic::ID ID = F.getIntrinsicID(); bool Changed = false; @@ -67,7 +73,10 @@ static bool expandMemIntrinsicUses(Function &F) { case Intrinsic::memcpy: { auto *Memcpy = cast(Inst); if (shouldExpandOperationWithSize(Memcpy->getLength())) { - expandMemCpyAsLoop(Memcpy); + Function *ParentFunc = Memcpy->getParent()->getParent(); + const TargetTransformInfo &TTI = + getAnalysis().getTTI(*ParentFunc); + expandMemCpyAsLoop(Memcpy, TTI); Changed = true; Memcpy->eraseFromParent(); } diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp new file mode 100644 index 000000000000..7263ba73d155 --- /dev/null +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp @@ -0,0 +1,64 @@ +//===--- AMDGPUMacroFusion.cpp - AMDGPU Macro Fusion ----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file This file contains the AMDGPU implementation of the DAG scheduling +/// mutation to pair instructions back to back. +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUMacroFusion.h" +#include "AMDGPUSubtarget.h" +#include "SIInstrInfo.h" + +#include "llvm/CodeGen/MacroFusion.h" + +using namespace llvm; + +namespace { + +/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused +/// together. Given SecondMI, when FirstMI is unspecified, then check if +/// SecondMI may be part of a fused pair at all. +static bool shouldScheduleAdjacent(const TargetInstrInfo &TII_, + const TargetSubtargetInfo &TSI, + const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { + const SIInstrInfo &TII = static_cast(TII_); + + switch (SecondMI.getOpcode()) { + case AMDGPU::V_ADDC_U32_e64: + case AMDGPU::V_SUBB_U32_e64: + case AMDGPU::V_CNDMASK_B32_e64: { + // Try to cluster defs of condition registers to their uses. This improves + // the chance VCC will be available which will allow shrinking to VOP2 + // encodings. + if (!FirstMI) + return true; + + const MachineOperand *Src2 = TII.getNamedOperand(SecondMI, + AMDGPU::OpName::src2); + return FirstMI->definesRegister(Src2->getReg()); + } + default: + return false; + } + + return false; +} + +} // end namespace + + +namespace llvm { + +std::unique_ptr createAMDGPUMacroFusionDAGMutation () { + return createMacroFusionDAGMutation(shouldScheduleAdjacent); +} + +} // end namespace llvm diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h b/contrib/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h new file mode 100644 index 000000000000..844958580a65 --- /dev/null +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h @@ -0,0 +1,19 @@ +//===- AMDGPUMacroFusion.h - AMDGPU Macro Fusion ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineScheduler.h" + +namespace llvm { + +/// Note that you have to add: +/// DAG.addMutation(createAMDGPUMacroFusionDAGMutation()); +/// to AMDGPUPassConfig::createMachineScheduler() to have an effect. +std::unique_ptr createAMDGPUMacroFusionDAGMutation(); + +} // llvm diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp index be47b900c6f0..1bc5a52053ec 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -13,6 +13,14 @@ //===----------------------------------------------------------------------===// #include "AMDGPUSubtarget.h" +#include "AMDGPU.h" +#include "AMDGPUTargetMachine.h" +#ifdef LLVM_BUILD_GLOBAL_ISEL +#include "AMDGPUCallLowering.h" +#include "AMDGPUInstructionSelector.h" +#include "AMDGPULegalizerInfo.h" +#include "AMDGPURegisterBankInfo.h" +#endif #include "SIMachineFunctionInfo.h" #include "llvm/ADT/SmallString.h" #include "llvm/CodeGen/MachineScheduler.h" @@ -72,6 +80,31 @@ AMDGPUSubtarget::initializeSubtargetDependencies(const Triple &TT, return *this; } +#ifdef LLVM_BUILD_GLOBAL_ISEL +namespace { + +struct SIGISelActualAccessor : public GISelAccessor { + std::unique_ptr CallLoweringInfo; + std::unique_ptr InstSelector; + std::unique_ptr Legalizer; + std::unique_ptr RegBankInfo; + const AMDGPUCallLowering *getCallLowering() const override { + return CallLoweringInfo.get(); + } + const InstructionSelector *getInstructionSelector() const override { + return InstSelector.get(); + } + const LegalizerInfo *getLegalizerInfo() const override { + return Legalizer.get(); + } + const RegisterBankInfo *getRegBankInfo() const override { + return RegBankInfo.get(); + } +}; + +} // end anonymous namespace +#endif + AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS, const TargetMachine &TM) : AMDGPUGenSubtargetInfo(TT, GPU, FS), @@ -265,18 +298,21 @@ bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const { case Intrinsic::amdgcn_workitem_id_x: case Intrinsic::r600_read_tidig_x: IdQuery = true; + LLVM_FALLTHROUGH; case Intrinsic::r600_read_local_size_x: Dim = 0; break; case Intrinsic::amdgcn_workitem_id_y: case Intrinsic::r600_read_tidig_y: IdQuery = true; + LLVM_FALLTHROUGH; case Intrinsic::r600_read_local_size_y: Dim = 1; break; case Intrinsic::amdgcn_workitem_id_z: case Intrinsic::r600_read_tidig_z: IdQuery = true; + LLVM_FALLTHROUGH; case Intrinsic::r600_read_local_size_z: Dim = 2; break; @@ -317,11 +353,23 @@ R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS, TLInfo(TM, *this) {} SISubtarget::SISubtarget(const Triple &TT, StringRef GPU, StringRef FS, - const TargetMachine &TM) : - AMDGPUSubtarget(TT, GPU, FS, TM), - InstrInfo(*this), - FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0), - TLInfo(TM, *this) {} + const TargetMachine &TM) + : AMDGPUSubtarget(TT, GPU, FS, TM), InstrInfo(*this), + FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0), + TLInfo(TM, *this) { +#ifndef LLVM_BUILD_GLOBAL_ISEL + GISelAccessor *GISel = new GISelAccessor(); +#else + SIGISelActualAccessor *GISel = new SIGISelActualAccessor(); + GISel->CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering())); + GISel->Legalizer.reset(new AMDGPULegalizerInfo()); + + GISel->RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo())); + GISel->InstSelector.reset(new AMDGPUInstructionSelector( + *this, *static_cast(GISel->RegBankInfo.get()))); +#endif + setGISelAccessor(*GISel); +} void SISubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy, unsigned NumRegionInstrs) const { diff --git a/contrib/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/contrib/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 425fd35d47de..dc868f010d85 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -19,9 +19,7 @@ #include "AMDGPUCallLowering.h" #include "AMDGPUInstructionSelector.h" #include "AMDGPULegalizerInfo.h" -#ifdef LLVM_BUILD_GLOBAL_ISEL -#include "AMDGPURegisterBankInfo.h" -#endif +#include "AMDGPUMacroFusion.h" #include "AMDGPUTargetObjectFile.h" #include "AMDGPUTargetTransformInfo.h" #include "GCNIterativeScheduler.h" @@ -85,7 +83,7 @@ static cl::opt EnableLoadStoreVectorizer( static cl::opt ScalarizeGlobal( "amdgpu-scalarize-global-loads", cl::desc("Enable global load scalarization"), - cl::init(false), + cl::init(true), cl::Hidden); // Option to run internalize pass. @@ -176,6 +174,7 @@ createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) { new GCNScheduleDAGMILive(C, make_unique(C)); DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); + DAG->addMutation(createAMDGPUMacroFusionDAGMutation()); return DAG; } @@ -389,31 +388,6 @@ const R600Subtarget *R600TargetMachine::getSubtargetImpl( // GCN Target Machine (SI+) //===----------------------------------------------------------------------===// -#ifdef LLVM_BUILD_GLOBAL_ISEL -namespace { - -struct SIGISelActualAccessor : public GISelAccessor { - std::unique_ptr CallLoweringInfo; - std::unique_ptr InstSelector; - std::unique_ptr Legalizer; - std::unique_ptr RegBankInfo; - const AMDGPUCallLowering *getCallLowering() const override { - return CallLoweringInfo.get(); - } - const InstructionSelector *getInstructionSelector() const override { - return InstSelector.get(); - } - const LegalizerInfo *getLegalizerInfo() const override { - return Legalizer.get(); - } - const RegisterBankInfo *getRegBankInfo() const override { - return RegBankInfo.get(); - } -}; - -} // end anonymous namespace -#endif - GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, TargetOptions Options, @@ -435,21 +409,6 @@ const SISubtarget *GCNTargetMachine::getSubtargetImpl(const Function &F) const { // function that reside in TargetOptions. resetTargetOptions(F); I = llvm::make_unique(TargetTriple, GPU, FS, *this); - -#ifndef LLVM_BUILD_GLOBAL_ISEL - GISelAccessor *GISel = new GISelAccessor(); -#else - SIGISelActualAccessor *GISel = new SIGISelActualAccessor(); - GISel->CallLoweringInfo.reset( - new AMDGPUCallLowering(*I->getTargetLowering())); - GISel->Legalizer.reset(new AMDGPULegalizerInfo()); - - GISel->RegBankInfo.reset(new AMDGPURegisterBankInfo(*I->getRegisterInfo())); - GISel->InstSelector.reset(new AMDGPUInstructionSelector(*I, - *static_cast(GISel->RegBankInfo.get()))); -#endif - - I->setGISelAccessor(*GISel); } I->setScalarizeGlobalBehavior(ScalarizeGlobal); diff --git a/contrib/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/contrib/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 7b8756050b75..e3c90f250600 100644 --- a/contrib/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -1058,17 +1058,13 @@ class AMDGPUAsmParser : public MCTargetAsmParser { OperandMatchResultTy parseOModOperand(OperandVector &Operands); - void cvtId(MCInst &Inst, const OperandVector &Operands); - void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands); - - void cvtVOP3Impl(MCInst &Inst, - const OperandVector &Operands, - OptionalImmIndexMap &OptionalIdx); + void cvtVOP3(MCInst &Inst, const OperandVector &Operands, + OptionalImmIndexMap &OptionalIdx); void cvtVOP3(MCInst &Inst, const OperandVector &Operands); - void cvtVOP3OMod(MCInst &Inst, const OperandVector &Operands); void cvtVOP3P(MCInst &Inst, const OperandVector &Operands); - void cvtMIMG(MCInst &Inst, const OperandVector &Operands); + void cvtMIMG(MCInst &Inst, const OperandVector &Operands, + bool IsAtomic = false); void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands); OperandMatchResultTy parseDPPCtrl(OperandVector &Operands); @@ -3870,13 +3866,19 @@ void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) { // mimg //===----------------------------------------------------------------------===// -void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) { +void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands, + bool IsAtomic) { unsigned I = 1; const MCInstrDesc &Desc = MII.get(Inst.getOpcode()); for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1); } + if (IsAtomic) { + // Add src, same as dst + ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1); + } + OptionalImmIndexMap OptionalIdx; for (unsigned E = Operands.size(); I != E; ++I) { @@ -3904,39 +3906,7 @@ void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) { } void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) { - unsigned I = 1; - const MCInstrDesc &Desc = MII.get(Inst.getOpcode()); - for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { - ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1); - } - - // Add src, same as dst - ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1); - - OptionalImmIndexMap OptionalIdx; - - for (unsigned E = Operands.size(); I != E; ++I) { - AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); - - // Add the register arguments - if (Op.isRegOrImm()) { - Op.addRegOrImmOperands(Inst, 1); - continue; - } else if (Op.isImmModifier()) { - OptionalIdx[Op.getImmTy()] = I; - } else { - llvm_unreachable("unexpected operand type"); - } - } - - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC); + cvtMIMG(Inst, Operands, true); } AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const { @@ -4118,25 +4088,6 @@ OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) return MatchOperand_NoMatch; } -void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) { - unsigned I = 1; - const MCInstrDesc &Desc = MII.get(Inst.getOpcode()); - for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { - ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1); - } - for (unsigned E = Operands.size(); I != E; ++I) - ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1); -} - -void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) { - uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags; - if (TSFlags & SIInstrFlags::VOP3) { - cvtVOP3(Inst, Operands); - } else { - cvtId(Inst, Operands); - } -} - static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) { // 1. This operand is input modifiers return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS @@ -4148,91 +4099,78 @@ static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) { && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1; } -void AMDGPUAsmParser::cvtVOP3Impl(MCInst &Inst, const OperandVector &Operands, - OptionalImmIndexMap &OptionalIdx) { +void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands, + OptionalImmIndexMap &OptionalIdx) { + unsigned Opc = Inst.getOpcode(); + unsigned I = 1; const MCInstrDesc &Desc = MII.get(Inst.getOpcode()); for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1); } - for (unsigned E = Operands.size(); I != E; ++I) { - AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); - if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) { - Op.addRegOrImmWithFPInputModsOperands(Inst, 2); - } else if (Op.isImmModifier()) { - OptionalIdx[Op.getImmTy()] = I; - } else if (Op.isRegOrImm()) { - Op.addRegOrImmOperands(Inst, 1); - } else { - llvm_unreachable("unhandled operand type"); + if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) { + // This instruction has src modifiers + for (unsigned E = Operands.size(); I != E; ++I) { + AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); + if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) { + Op.addRegOrImmWithFPInputModsOperands(Inst, 2); + } else if (Op.isImmModifier()) { + OptionalIdx[Op.getImmTy()] = I; + } else if (Op.isRegOrImm()) { + Op.addRegOrImmOperands(Inst, 1); + } else { + llvm_unreachable("unhandled operand type"); + } + } + } else { + // No src modifiers + for (unsigned E = Operands.size(); I != E; ++I) { + AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); + if (Op.isMod()) { + OptionalIdx[Op.getImmTy()] = I; + } else { + Op.addRegOrImmOperands(Inst, 1); + } } } -} -void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) { - OptionalImmIndexMap OptionalIdx; + if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) { + addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI); + } - cvtVOP3Impl(Inst, Operands, OptionalIdx); - - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI); + if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) { + addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI); + } // special case v_mac_{f16, f32}: // it has src2 register operand that is tied to dst operand // we don't allow modifiers for this operand in assembler so src2_modifiers // should be 0 - if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si || - Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi || - Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) { + if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi || + Opc == AMDGPU::V_MAC_F16_e64_vi) { auto it = Inst.begin(); - std::advance( - it, - AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ? - AMDGPU::V_MAC_F16_e64 : - AMDGPU::V_MAC_F32_e64, - AMDGPU::OpName::src2_modifiers)); + std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers)); it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2 ++it; Inst.insert(it, Inst.getOperand(0)); // src2 = dst } } -void AMDGPUAsmParser::cvtVOP3OMod(MCInst &Inst, const OperandVector &Operands) { +void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) { OptionalImmIndexMap OptionalIdx; - - unsigned I = 1; - const MCInstrDesc &Desc = MII.get(Inst.getOpcode()); - for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { - ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1); - } - - for (unsigned E = Operands.size(); I != E; ++I) { - AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); - if (Op.isMod()) { - OptionalIdx[Op.getImmTy()] = I; - } else { - Op.addRegOrImmOperands(Inst, 1); - } - } - - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI); - addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI); + cvtVOP3(Inst, Operands, OptionalIdx); } void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) { OptionalImmIndexMap OptIdx; - cvtVOP3Impl(Inst, Operands, OptIdx); + cvtVOP3(Inst, Operands, OptIdx); // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3 // instruction, and then figure out where to actually put the modifiers int Opc = Inst.getOpcode(); - if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) { - addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyClampSI); - } - addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel); addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1); @@ -4284,7 +4222,7 @@ void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) { int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]); - Inst.getOperand(ModIdx).setImm(ModVal); + Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal); } } diff --git a/contrib/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/contrib/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp index 8ead48067336..2e7641cda375 100644 --- a/contrib/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp @@ -17,7 +17,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" namespace llvm { std::vector makeMinRegSchedule(ArrayRef TopRoots, diff --git a/contrib/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp b/contrib/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp index d378df674be9..0657f67b217d 100644 --- a/contrib/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp @@ -15,7 +15,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" namespace { class GCNMinRegScheduler { diff --git a/contrib/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/contrib/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp index 390a8286c76a..1d02c7fdffbf 100644 --- a/contrib/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp @@ -16,7 +16,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD diff --git a/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 8ec46665daf5..155b400ba022 100644 --- a/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -20,7 +20,7 @@ #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/Support/MathExtras.h" -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" using namespace llvm; diff --git a/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h index 3ed3cd5b3b1c..060d2ca72d93 100644 --- a/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ b/contrib/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -66,7 +66,7 @@ class GCNScheduleDAGMILive : public ScheduleDAGMILive { const SIMachineFunctionInfo &MFI; - // Occupancy target at the begining of function scheduling cycle. + // Occupancy target at the beginning of function scheduling cycle. unsigned StartingOccupancy; // Minimal real occupancy recorder for the function. diff --git a/contrib/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/contrib/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp index 2b408ff10caa..a50e3eb8d9ce 100644 --- a/contrib/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp @@ -32,7 +32,7 @@ class AMDGPUAsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, const MCAsmLayout &Layout) const override { @@ -100,7 +100,7 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value, void AMDGPUAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { Value = adjustFixupValue(Fixup, Value, &Asm.getContext()); if (!Value) return; // Doesn't change encoding. diff --git a/contrib/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/contrib/llvm/lib/Target/AMDGPU/MIMGInstructions.td index a515eecc222a..06e2c11b0193 100644 --- a/contrib/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/contrib/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -26,6 +26,7 @@ class MIMG_Helper op, string asm, diff --git a/contrib/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp b/contrib/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp index 6993e8a62a9c..00cbd24b84fb 100644 --- a/contrib/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp @@ -555,7 +555,7 @@ class R600ControlFlowFinalizer : public MachineFunctionPass { CFStack.pushBranch(AMDGPU::CF_PUSH_EG); } else CFStack.pushBranch(AMDGPU::CF_ALU_PUSH_BEFORE); - + LLVM_FALLTHROUGH; case AMDGPU::CF_ALU: I = MI; AluClauses.push_back(MakeALUClause(MBB, I)); diff --git a/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp index 215791f4f92d..69a63b6941ef 100644 --- a/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp @@ -1618,7 +1618,8 @@ EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, return VT.changeVectorElementTypeToInteger(); } -bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT) const { +bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT, + const SelectionDAG &DAG) const { // Local and Private addresses do not handle vectors. Limit to i32 if ((AS == AMDGPUASI.LOCAL_ADDRESS || AS == AMDGPUASI.PRIVATE_ADDRESS)) { return (MemVT.getSizeInBits() <= 32); diff --git a/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.h b/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.h index d6a0876a6ee7..2a774693f02b 100644 --- a/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.h +++ b/contrib/llvm/lib/Target/AMDGPU/R600ISelLowering.h @@ -44,7 +44,8 @@ class R600TargetLowering final : public AMDGPUTargetLowering { EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT VT) const override; - bool canMergeStoresTo(unsigned AS, EVT MemVT) const override; + bool canMergeStoresTo(unsigned AS, EVT MemVT, + const SelectionDAG &DAG) const override; bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, unsigned Align, diff --git a/contrib/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp b/contrib/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp index 47fda1c8fa82..a7e540f9d14d 100644 --- a/contrib/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp @@ -22,7 +22,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" void R600SchedStrategy::initialize(ScheduleDAGMI *dag) { assert(dag->hasVRegLiveness() && "R600SchedStrategy needs vreg liveness"); diff --git a/contrib/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/contrib/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp index f391f67a241f..3af242d9ea66 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -137,6 +137,7 @@ static bool isInlineConstantIfFolded(const SIInstrInfo *TII, = TII->get(IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16); return TII->isInlineConstant(OpToFold, MadDesc.OpInfo[OpNo].OperandType); } + return false; } default: return false; diff --git a/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d39b345bdf03..2ba570b9ebbb 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -547,7 +547,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.align = 0; const ConstantInt *Vol = dyn_cast(CI.getOperand(4)); - Info.vol = !Vol || !Vol->isNullValue(); + Info.vol = !Vol || !Vol->isZero(); Info.readMem = true; Info.writeMem = true; return true; @@ -713,7 +713,8 @@ bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL, } } -bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT) const { +bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT, + const SelectionDAG &DAG) const { if (AS == AMDGPUASI.GLOBAL_ADDRESS || AS == AMDGPUASI.FLAT_ADDRESS) { return (MemVT.getSizeInBits() <= 4 * 32); } else if (AS == AMDGPUASI.PRIVATE_ADDRESS) { @@ -2374,20 +2375,16 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N, } case ISD::INTRINSIC_WO_CHAIN: { unsigned IID = cast(N->getOperand(0))->getZExtValue(); - switch (IID) { - case Intrinsic::amdgcn_cvt_pkrtz: { + if (IID == Intrinsic::amdgcn_cvt_pkrtz) { SDValue Src0 = N->getOperand(1); SDValue Src1 = N->getOperand(2); SDLoc SL(N); SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, Src0, Src1); - Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); return; } - default: - break; - } + break; } case ISD::SELECT: { SDLoc SL(N); @@ -3736,7 +3733,9 @@ SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op, SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); EVT VT = Op.getValueType(); - bool Unsafe = DAG.getTarget().Options.UnsafeFPMath; + const SDNodeFlags Flags = Op->getFlags(); + bool Unsafe = DAG.getTarget().Options.UnsafeFPMath || + Flags.hasUnsafeAlgebra() || Flags.hasAllowReciprocal(); if (!Unsafe && VT == MVT::f32 && Subtarget->hasFP32Denormals()) return SDValue(); @@ -3771,15 +3770,11 @@ SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op, } } - const SDNodeFlags Flags = Op->getFlags(); - - if (Unsafe || Flags.hasAllowReciprocal()) { + if (Unsafe) { // Turn into multiply by the reciprocal. // x / y -> x * (1.0 / y) - SDNodeFlags NewFlags; - NewFlags.setUnsafeAlgebra(true); SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); - return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, NewFlags); + return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags); } return SDValue(); @@ -4622,15 +4617,99 @@ SDValue SITargetLowering::performClassCombine(SDNode *N, return SDValue(); } +static bool isKnownNeverSNan(SelectionDAG &DAG, SDValue Op) { + if (!DAG.getTargetLoweringInfo().hasFloatingPointExceptions()) + return true; + + return DAG.isKnownNeverNaN(Op); +} + +static bool isCanonicalized(SDValue Op, const SISubtarget *ST, + unsigned MaxDepth=5) { + // If source is a result of another standard FP operation it is already in + // canonical form. + + switch (Op.getOpcode()) { + default: + break; + + // These will flush denorms if required. + case ISD::FADD: + case ISD::FSUB: + case ISD::FMUL: + case ISD::FSQRT: + case ISD::FCEIL: + case ISD::FFLOOR: + case ISD::FMA: + case ISD::FMAD: + + case ISD::FCANONICALIZE: + return true; + + case ISD::FP_ROUND: + return Op.getValueType().getScalarType() != MVT::f16 || + ST->hasFP16Denormals(); + + case ISD::FP_EXTEND: + return Op.getOperand(0).getValueType().getScalarType() != MVT::f16 || + ST->hasFP16Denormals(); + + case ISD::FP16_TO_FP: + case ISD::FP_TO_FP16: + return ST->hasFP16Denormals(); + + // It can/will be lowered or combined as a bit operation. + // Need to check their input recursively to handle. + case ISD::FNEG: + case ISD::FABS: + return (MaxDepth > 0) && + isCanonicalized(Op.getOperand(0), ST, MaxDepth - 1); + + case ISD::FSIN: + case ISD::FCOS: + case ISD::FSINCOS: + return Op.getValueType().getScalarType() != MVT::f16; + + // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. + // For such targets need to check their input recursively. + // TODO: on GFX9+ we could return true without checking provided no-nan + // mode, since canonicalization is also used to quiet sNaNs. + case ISD::FMINNUM: + case ISD::FMAXNUM: + case ISD::FMINNAN: + case ISD::FMAXNAN: + + return (MaxDepth > 0) && + isCanonicalized(Op.getOperand(0), ST, MaxDepth - 1) && + isCanonicalized(Op.getOperand(1), ST, MaxDepth - 1); + + case ISD::ConstantFP: { + auto F = cast(Op)->getValueAPF(); + return !F.isDenormal() && !(F.isNaN() && F.isSignaling()); + } + } + return false; +} + // Constant fold canonicalize. SDValue SITargetLowering::performFCanonicalizeCombine( SDNode *N, DAGCombinerInfo &DCI) const { - ConstantFPSDNode *CFP = isConstOrConstSplatFP(N->getOperand(0)); - if (!CFP) - return SDValue(); - SelectionDAG &DAG = DCI.DAG; + ConstantFPSDNode *CFP = isConstOrConstSplatFP(N->getOperand(0)); + + if (!CFP) { + SDValue N0 = N->getOperand(0); + + bool IsIEEEMode = Subtarget->enableIEEEBit(DAG.getMachineFunction()); + + if ((IsIEEEMode || isKnownNeverSNan(DAG, N0)) && + isCanonicalized(N0, getSubtarget())) + return N0; + + return SDValue(); + } + const APFloat &C = CFP->getValueAPF(); // Flush denormals to 0 if not enabled. @@ -4723,13 +4802,6 @@ SDValue SITargetLowering::performIntMed3ImmCombine( return DAG.getNode(ISD::TRUNCATE, SL, VT, Med3); } -static bool isKnownNeverSNan(SelectionDAG &DAG, SDValue Op) { - if (!DAG.getTargetLoweringInfo().hasFloatingPointExceptions()) - return true; - - return DAG.isKnownNeverNaN(Op); -} - SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL, SDValue Op0, diff --git a/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.h b/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.h index 24f88e632d38..83392a7ab1b2 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/contrib/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -153,7 +153,8 @@ class SITargetLowering final : public AMDGPUTargetLowering { bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS) const override; - bool canMergeStoresTo(unsigned AS, EVT MemVT) const override; + bool canMergeStoresTo(unsigned AS, EVT MemVT, + const SelectionDAG &DAG) const override; bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, unsigned Align, diff --git a/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index b6784ec14e9f..160f8837d49c 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -2022,10 +2022,12 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB, return nullptr; case AMDGPU::V_MAC_F16_e64: IsF16 = true; + LLVM_FALLTHROUGH; case AMDGPU::V_MAC_F32_e64: break; case AMDGPU::V_MAC_F16_e32: IsF16 = true; + LLVM_FALLTHROUGH; case AMDGPU::V_MAC_F32_e32: { int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); @@ -4320,6 +4322,24 @@ SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const return new GCNHazardRecognizer(MF); } +std::pair +SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { + return std::make_pair(TF & MO_MASK, TF & ~MO_MASK); +} + +ArrayRef> +SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { + static const std::pair TargetFlags[] = { + { MO_GOTPCREL, "amdgpu-gotprel" }, + { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, + { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, + { MO_REL32_LO, "amdgpu-rel32-lo" }, + { MO_REL32_HI, "amdgpu-rel32-hi" } + }; + + return makeArrayRef(TargetFlags); +} + bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const { return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && MI.modifiesRegister(AMDGPU::EXEC, &RI); diff --git a/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 74b48c761808..d00c0d4a7f4e 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -100,6 +100,8 @@ class SIInstrInfo final : public AMDGPUInstrInfo { public: enum TargetOperandFlags { + MO_MASK = 0x7, + MO_NONE = 0, // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL. MO_GOTPCREL = 1, @@ -781,9 +783,15 @@ class SIInstrInfo final : public AMDGPUInstrInfo { void convertNonUniformLoopRegion(MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const; + std::pair + decomposeMachineOperandsTargetFlags(unsigned TF) const override; + ArrayRef> getSerializableTargetIndices() const override; + ArrayRef> + getSerializableDirectMachineOperandTargetFlags() const override; + ScheduleHazardRecognizer * CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const override; diff --git a/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 4a81fb3b463a..ffb01363e131 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/contrib/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -1502,6 +1502,8 @@ def VOP_B32_F16_F16 : VOPProfile <[i32, f16, f16, untyped]>; def VOP_V2F16_V2F16_V2F16_V2F16 : VOPProfile <[v2f16, v2f16, v2f16, v2f16]>; def VOP_V2I16_V2I16_V2I16_V2I16 : VOPProfile <[v2i16, v2i16, v2i16, v2i16]>; +def VOP_F32_V2F16_V2F16_V2F16 : VOPProfile <[f32, v2f16, v2f16, v2f16]>; + def VOP_NONE : VOPProfile <[untyped, untyped, untyped, untyped]>; def VOP_F32_F32 : VOPProfile <[f32, f32, untyped, untyped]>; diff --git a/contrib/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/contrib/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp index bb17dbbdfbd6..34886c48f461 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp @@ -38,7 +38,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" // This scheduler implements a different scheduling algorithm than // GenericScheduler. diff --git a/contrib/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/contrib/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp index 96a18544f02a..874fbadca7f3 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp @@ -110,10 +110,8 @@ static bool canShrink(MachineInstr &MI, const SIInstrInfo *TII, } const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); - const MachineOperand *Src1Mod = - TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); - - if (Src1 && (!isVGPR(Src1, TRI, MRI) || (Src1Mod && Src1Mod->getImm() != 0))) + if (Src1 && (!isVGPR(Src1, TRI, MRI) || + TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) return false; // We don't need to check src0, all input types are legal, so just make sure @@ -122,58 +120,64 @@ static bool canShrink(MachineInstr &MI, const SIInstrInfo *TII, return false; // Check output modifiers - if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) - return false; - - return !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp); + return !TII->hasModifiersSet(MI, AMDGPU::OpName::omod) && + !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp); } /// \brief This function checks \p MI for operands defined by a move immediate /// instruction and then folds the literal constant into the instruction if it -/// can. This function assumes that \p MI is a VOP1, VOP2, or VOPC instruction -/// and will only fold literal constants if we are still in SSA. -static void foldImmediates(MachineInstr &MI, const SIInstrInfo *TII, +/// can. This function assumes that \p MI is a VOP1, VOP2, or VOPC instructions. +static bool foldImmediates(MachineInstr &MI, const SIInstrInfo *TII, MachineRegisterInfo &MRI, bool TryToCommute = true) { - - if (!MRI.isSSA()) - return; - assert(TII->isVOP1(MI) || TII->isVOP2(MI) || TII->isVOPC(MI)); int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); - // Only one literal constant is allowed per instruction, so if src0 is a - // literal constant then we can't do any folding. - if (TII->isLiteralConstant(MI, Src0Idx)) - return; - // Try to fold Src0 MachineOperand &Src0 = MI.getOperand(Src0Idx); - if (Src0.isReg() && MRI.hasOneUse(Src0.getReg())) { + if (Src0.isReg()) { unsigned Reg = Src0.getReg(); - MachineInstr *Def = MRI.getUniqueVRegDef(Reg); - if (Def && Def->isMoveImmediate()) { - MachineOperand &MovSrc = Def->getOperand(1); - bool ConstantFolded = false; + if (TargetRegisterInfo::isVirtualRegister(Reg) && MRI.hasOneUse(Reg)) { + MachineInstr *Def = MRI.getUniqueVRegDef(Reg); + if (Def && Def->isMoveImmediate()) { + MachineOperand &MovSrc = Def->getOperand(1); + bool ConstantFolded = false; - if (MovSrc.isImm() && (isInt<32>(MovSrc.getImm()) || - isUInt<32>(MovSrc.getImm()))) { - Src0.ChangeToImmediate(MovSrc.getImm()); - ConstantFolded = true; - } - if (ConstantFolded) { - if (MRI.use_empty(Reg)) + if (MovSrc.isImm() && (isInt<32>(MovSrc.getImm()) || + isUInt<32>(MovSrc.getImm()))) { + // It's possible to have only one component of a super-reg defined by + // a single mov, so we need to clear any subregister flag. + Src0.setSubReg(0); + Src0.ChangeToImmediate(MovSrc.getImm()); + ConstantFolded = true; + } else if (MovSrc.isFI()) { + Src0.setSubReg(0); + Src0.ChangeToFrameIndex(MovSrc.getIndex()); + ConstantFolded = true; + } + + if (ConstantFolded) { + assert(MRI.use_empty(Reg)); Def->eraseFromParent(); - ++NumLiteralConstantsFolded; - return; + ++NumLiteralConstantsFolded; + return true; + } } } } // We have failed to fold src0, so commute the instruction and try again. - if (TryToCommute && MI.isCommutable() && TII->commuteInstruction(MI)) - foldImmediates(MI, TII, MRI, false); + if (TryToCommute && MI.isCommutable()) { + if (TII->commuteInstruction(MI)) { + if (foldImmediates(MI, TII, MRI, false)) + return true; + // Commute back. + TII->commuteInstruction(MI); + } + } + + return false; } // Copy MachineOperand with all flags except setting it as implicit. diff --git a/contrib/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp b/contrib/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp index 9908fc003ce7..92fb762ebd73 100644 --- a/contrib/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp @@ -16,7 +16,7 @@ using namespace llvm; -/// \brief The target which suports all AMD GPUs. This will eventually +/// \brief The target which supports all AMD GPUs. This will eventually /// be deprecated and there will be a R600 target and a GCN target. Target &llvm::getTheAMDGPUTarget() { static Target TheAMDGPUTarget; diff --git a/contrib/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/contrib/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index 96d343099132..f2de1f995726 100644 --- a/contrib/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/contrib/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -16,12 +16,21 @@ class VOP3PInst !if(P.HasModifiers, getVOP3PModPat.ret, getVOP3Pat.ret) >; -// Non-packed instructions that use the VOP3P encoding. i.e. where -// omod/abs are used. +// Non-packed instructions that use the VOP3P encoding. +// VOP3 neg/abs and VOP3P opsel/opsel_hi modifiers are allowed. class VOP3_VOP3PInst : - VOP3P_Pseudo.ret, getVOP3Pat.ret) ->; + VOP3P_Pseudo { + let InOperandList = + (ins + FP32InputMods:$src0_modifiers, VCSrc_f32:$src0, + FP32InputMods:$src1_modifiers, VCSrc_f32:$src1, + FP32InputMods:$src2_modifiers, VCSrc_f32:$src2, + clampmod:$clamp, + op_sel:$op_sel, + op_sel_hi:$op_sel_hi); + let AsmOperands = + " $vdst, $src0_modifiers, $src1_modifiers, $src2_modifiers$op_sel$op_sel_hi$clamp"; +} let isCommutable = 1 in { def V_PK_FMA_F16 : VOP3PInst<"v_pk_fma_f16", VOP3_Profile, fma>; @@ -46,9 +55,12 @@ def V_PK_ASHRREV_I16 : VOP3PInst<"v_pk_ashrrev_i16", VOP3_Profile, lshr_rev>; // XXX - Commutable? -def V_MAD_MIX_F32 : VOP3_VOP3PInst<"v_mad_mix_f32", VOP3_Profile>; -def V_MAD_MIXLO_F16 : VOP3_VOP3PInst<"v_mad_mixlo_f16", VOP3_Profile>; -def V_MAD_MIXHI_F16 : VOP3_VOP3PInst<"v_mad_mixhi_f16", VOP3_Profile>; +// These are VOP3a-like opcodes which accept no omod. +// Size of src arguments (16/32) is controlled by op_sel. +// For 16-bit src arguments their location (hi/lo) are controlled by op_sel_hi. +def V_MAD_MIX_F32 : VOP3_VOP3PInst<"v_mad_mix_f32", VOP3_Profile>; +def V_MAD_MIXLO_F16 : VOP3_VOP3PInst<"v_mad_mixlo_f16", VOP3_Profile>; +def V_MAD_MIXHI_F16 : VOP3_VOP3PInst<"v_mad_mixhi_f16", VOP3_Profile>; multiclass VOP3P_Real_vi op> { diff --git a/contrib/llvm/lib/Target/AMDGPU/VOPInstructions.td b/contrib/llvm/lib/Target/AMDGPU/VOPInstructions.td index e386f21c2ba4..77b7952b22a8 100644 --- a/contrib/llvm/lib/Target/AMDGPU/VOPInstructions.td +++ b/contrib/llvm/lib/Target/AMDGPU/VOPInstructions.td @@ -51,12 +51,8 @@ class VOP3Common pattern = [], let AsmVariantName = AMDGPUAsmVariants.VOP3; let AsmMatchConverter = - !if(!eq(VOP3Only,1), - !if(!and(P.IsPacked, isVOP3P), "cvtVOP3P", "cvtVOP3"), - !if(!eq(P.HasModifiers, 1), - "cvtVOP3_2_mod", - !if(!eq(P.HasOMod, 1), "cvtVOP3OMod", "") - ) - ); + !if(!and(P.IsPacked, isVOP3P), + "cvtVOP3P", + !if(!or(P.HasModifiers, P.HasOMod), + "cvtVOP3", + "")); VOPProfile Pfl = P; } diff --git a/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp index 90f635c81254..582153daebde 100644 --- a/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMAsmPrinter.cpp @@ -1103,6 +1103,7 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { case ARM::tPUSH: // Special case here: no src & dst reg, but two extra imp ops. StartOp = 2; NumOffset = 2; + LLVM_FALLTHROUGH; case ARM::STMDB_UPD: case ARM::t2STMDB_UPD: case ARM::VSTMDDB_UPD: diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp index 1ec6b24b2ed6..3cf5950a1918 100644 --- a/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -1880,6 +1880,9 @@ isProfitableToIfCvt(MachineBasicBlock &TBB, // Diamond: TBB is the block that is branched to, FBB is the fallthrough TUnpredCycles = TCycles + TakenBranchCost; FUnpredCycles = FCycles + NotTakenBranchCost; + // The branch at the end of FBB will disappear when it's predicated, so + // discount it from PredCost. + PredCost -= 1 * ScalingUpFactor; } // The total cost is the cost of each path scaled by their probabilites unsigned TUnpredCost = Probability.scale(TUnpredCycles * ScalingUpFactor); diff --git a/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp index b4fb292c0116..e97a7ce5067f 100644 --- a/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -193,10 +193,11 @@ getReservedRegs(const MachineFunction &MF) const { for (unsigned R = 0; R < 16; ++R) markSuperRegs(Reserved, ARM::D16 + R); } - const TargetRegisterClass *RC = &ARM::GPRPairRegClass; - for(TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I!=E; ++I) - for (MCSubRegIterator SI(*I, this); SI.isValid(); ++SI) - if (Reserved.test(*SI)) markSuperRegs(Reserved, *I); + const TargetRegisterClass &RC = ARM::GPRPairRegClass; + for (unsigned Reg : RC) + for (MCSubRegIterator SI(Reg, this); SI.isValid(); ++SI) + if (Reserved.test(*SI)) + markSuperRegs(Reserved, Reg); assert(checkAllSuperRegsMarked(Reserved)); return Reserved; @@ -315,8 +316,7 @@ ARMBaseRegisterInfo::getRegAllocationHints(unsigned VirtReg, Hints.push_back(PairedPhys); // Then prefer even or odd registers. - for (unsigned I = 0, E = Order.size(); I != E; ++I) { - unsigned Reg = Order[I]; + for (unsigned Reg : Order) { if (Reg == PairedPhys || (getEncodingValue(Reg) & 1) != Odd) continue; // Don't provide hints that are paired to a reserved register. @@ -659,11 +659,8 @@ bool ARMBaseRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, unsigned Ba const MCInstrDesc &Desc = MI->getDesc(); unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask); unsigned i = 0; - - while (!MI->getOperand(i).isFI()) { - ++i; - assert(i < MI->getNumOperands() &&"Instr doesn't have FrameIndex operand!"); - } + for (; !MI->getOperand(i).isFI(); ++i) + assert(i+1 < MI->getNumOperands() && "Instr doesn't have FrameIndex operand!"); // AddrMode4 and AddrMode6 cannot handle any offset. if (AddrMode == ARMII::AddrMode4 || AddrMode == ARMII::AddrMode6) diff --git a/contrib/llvm/lib/Target/ARM/ARMCallLowering.cpp b/contrib/llvm/lib/Target/ARM/ARMCallLowering.cpp index e498f70b820d..051827a6a6a2 100644 --- a/contrib/llvm/lib/Target/ARM/ARMCallLowering.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMCallLowering.cpp @@ -321,7 +321,7 @@ struct IncomingValueHandler : public CallLowering::ValueHandler { assert(VA.getValVT().getSizeInBits() <= 64 && "Unsupported value size"); assert(VA.getLocVT().getSizeInBits() <= 64 && "Unsupported location size"); - // The necesary extensions are handled on the other side of the ABI + // The necessary extensions are handled on the other side of the ABI // boundary. markPhysRegUsed(PhysReg); MIRBuilder.buildCopy(ValVReg, PhysReg); diff --git a/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp b/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp index e42514acd76f..6ba7593543a9 100644 --- a/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -3398,9 +3398,9 @@ ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG, static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { SDLoc dl(Op); - ConstantSDNode *ScopeN = cast(Op.getOperand(2)); - auto Scope = static_cast(ScopeN->getZExtValue()); - if (Scope == SynchronizationScope::SingleThread) + ConstantSDNode *SSIDNode = cast(Op.getOperand(2)); + auto SSID = static_cast(SSIDNode->getZExtValue()); + if (SSID == SyncScope::SingleThread) return Op; if (!Subtarget->hasDataBarrier()) { @@ -5356,15 +5356,15 @@ static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) { // Integer comparisons. switch (SetCCOpcode) { default: llvm_unreachable("Illegal integer comparison"); - case ISD::SETNE: Invert = true; + case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH; case ISD::SETEQ: Opc = ARMISD::VCEQ; break; - case ISD::SETLT: Swap = true; + case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH; case ISD::SETGT: Opc = ARMISD::VCGT; break; - case ISD::SETLE: Swap = true; + case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH; case ISD::SETGE: Opc = ARMISD::VCGE; break; - case ISD::SETULT: Swap = true; + case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH; case ISD::SETUGT: Opc = ARMISD::VCGTU; break; - case ISD::SETULE: Swap = true; + case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH; case ISD::SETUGE: Opc = ARMISD::VCGEU; break; } @@ -13779,7 +13779,9 @@ bool ARMTargetLowering::lowerInterleavedLoad( // Convert the integer vector to pointer vector if the element is pointer. if (EltTy->isPointerTy()) - SubVec = Builder.CreateIntToPtr(SubVec, SV->getType()); + SubVec = Builder.CreateIntToPtr( + SubVec, VectorType::get(SV->getType()->getVectorElementType(), + VecTy->getVectorNumElements())); SubVecs[SV].push_back(SubVec); } diff --git a/contrib/llvm/lib/Target/ARM/ARMISelLowering.h b/contrib/llvm/lib/Target/ARM/ARMISelLowering.h index 5044134f5b1e..f05b14255236 100644 --- a/contrib/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/contrib/llvm/lib/Target/ARM/ARMISelLowering.h @@ -510,7 +510,8 @@ class InstrItineraryData; bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override; - bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT) const override { + bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, + const SelectionDAG &DAG) const override { // Do not merge to larger than i32. return (MemVT.getSizeInBits() <= 32); } diff --git a/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td b/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td index 53db5acbe805..42eac12e457b 100644 --- a/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td +++ b/contrib/llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -4799,7 +4799,7 @@ def : t2InstAlias<"add${p} $Rd, pc, $imm", // Pseudo instruction ldr Rt, =immediate def t2LDRConstPool : t2AsmPseudo<"ldr${p} $Rt, $immediate", - (ins GPRnopc:$Rt, const_pool_asm_imm:$immediate, pred:$p)>; + (ins GPR:$Rt, const_pool_asm_imm:$immediate, pred:$p)>; // Version w/ the .w suffix. def : t2InstAlias<"ldr${p}.w $Rt, $immediate", (t2LDRConstPool GPRnopc:$Rt, diff --git a/contrib/llvm/lib/Target/ARM/ARMInstructionSelector.cpp b/contrib/llvm/lib/Target/ARM/ARMInstructionSelector.cpp index 374176d1d737..29ef69ad0010 100644 --- a/contrib/llvm/lib/Target/ARM/ARMInstructionSelector.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMInstructionSelector.cpp @@ -20,6 +20,8 @@ #define DEBUG_TYPE "arm-isel" +#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" + using namespace llvm; #ifndef LLVM_BUILD_GLOBAL_ISEL @@ -42,13 +44,32 @@ class ARMInstructionSelector : public InstructionSelector { private: bool selectImpl(MachineInstr &I) const; - bool selectICmp(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, - MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI) const; + struct CmpConstants; + struct InsertInfo; - bool selectSelect(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, - MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI) const; + bool selectCmp(CmpConstants Helper, MachineInstrBuilder &MIB, + MachineRegisterInfo &MRI) const; + + // Helper for inserting a comparison sequence that sets \p ResReg to either 1 + // if \p LHSReg and \p RHSReg are in the relationship defined by \p Cond, or + // \p PrevRes otherwise. In essence, it computes PrevRes OR (LHS Cond RHS). + bool insertComparison(CmpConstants Helper, InsertInfo I, unsigned ResReg, + ARMCC::CondCodes Cond, unsigned LHSReg, unsigned RHSReg, + unsigned PrevRes) const; + + // Set \p DestReg to \p Constant. + void putConstant(InsertInfo I, unsigned DestReg, unsigned Constant) const; + + bool selectSelect(MachineInstrBuilder &MIB, MachineRegisterInfo &MRI) const; + + // Check if the types match and both operands have the expected size and + // register bank. + bool validOpRegPair(MachineRegisterInfo &MRI, unsigned LHS, unsigned RHS, + unsigned ExpectedSize, unsigned ExpectedRegBankID) const; + + // Check if the register has the expected size and register bank. + bool validReg(MachineRegisterInfo &MRI, unsigned Reg, unsigned ExpectedSize, + unsigned ExpectedRegBankID) const; const ARMBaseInstrInfo &TII; const ARMBaseRegisterInfo &TRI; @@ -251,120 +272,233 @@ static unsigned selectLoadStoreOpCode(unsigned Opc, unsigned RegBank, return Opc; } -static ARMCC::CondCodes getComparePred(CmpInst::Predicate Pred) { +// When lowering comparisons, we sometimes need to perform two compares instead +// of just one. Get the condition codes for both comparisons. If only one is +// needed, the second member of the pair is ARMCC::AL. +static std::pair +getComparePreds(CmpInst::Predicate Pred) { + std::pair Preds = {ARMCC::AL, ARMCC::AL}; switch (Pred) { - // Needs two compares... case CmpInst::FCMP_ONE: + Preds = {ARMCC::GT, ARMCC::MI}; + break; case CmpInst::FCMP_UEQ: - default: - // AL is our "false" for now. The other two need more compares. - return ARMCC::AL; + Preds = {ARMCC::EQ, ARMCC::VS}; + break; case CmpInst::ICMP_EQ: case CmpInst::FCMP_OEQ: - return ARMCC::EQ; + Preds.first = ARMCC::EQ; + break; case CmpInst::ICMP_SGT: case CmpInst::FCMP_OGT: - return ARMCC::GT; + Preds.first = ARMCC::GT; + break; case CmpInst::ICMP_SGE: case CmpInst::FCMP_OGE: - return ARMCC::GE; + Preds.first = ARMCC::GE; + break; case CmpInst::ICMP_UGT: case CmpInst::FCMP_UGT: - return ARMCC::HI; + Preds.first = ARMCC::HI; + break; case CmpInst::FCMP_OLT: - return ARMCC::MI; + Preds.first = ARMCC::MI; + break; case CmpInst::ICMP_ULE: case CmpInst::FCMP_OLE: - return ARMCC::LS; + Preds.first = ARMCC::LS; + break; case CmpInst::FCMP_ORD: - return ARMCC::VC; + Preds.first = ARMCC::VC; + break; case CmpInst::FCMP_UNO: - return ARMCC::VS; + Preds.first = ARMCC::VS; + break; case CmpInst::FCMP_UGE: - return ARMCC::PL; + Preds.first = ARMCC::PL; + break; case CmpInst::ICMP_SLT: case CmpInst::FCMP_ULT: - return ARMCC::LT; + Preds.first = ARMCC::LT; + break; case CmpInst::ICMP_SLE: case CmpInst::FCMP_ULE: - return ARMCC::LE; + Preds.first = ARMCC::LE; + break; case CmpInst::FCMP_UNE: case CmpInst::ICMP_NE: - return ARMCC::NE; + Preds.first = ARMCC::NE; + break; case CmpInst::ICMP_UGE: - return ARMCC::HS; + Preds.first = ARMCC::HS; + break; case CmpInst::ICMP_ULT: - return ARMCC::LO; + Preds.first = ARMCC::LO; + break; + default: + break; } + assert(Preds.first != ARMCC::AL && "No comparisons needed?"); + return Preds; } -bool ARMInstructionSelector::selectICmp(MachineInstrBuilder &MIB, - const ARMBaseInstrInfo &TII, - MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI) const { - auto &MBB = *MIB->getParent(); - auto InsertBefore = std::next(MIB->getIterator()); - auto &DebugLoc = MIB->getDebugLoc(); +struct ARMInstructionSelector::CmpConstants { + CmpConstants(unsigned CmpOpcode, unsigned FlagsOpcode, unsigned OpRegBank, + unsigned OpSize) + : ComparisonOpcode(CmpOpcode), ReadFlagsOpcode(FlagsOpcode), + OperandRegBankID(OpRegBank), OperandSize(OpSize) {} - // Move 0 into the result register. - auto Mov0I = BuildMI(MBB, InsertBefore, DebugLoc, TII.get(ARM::MOVi)) - .addDef(MRI.createVirtualRegister(&ARM::GPRRegClass)) - .addImm(0) - .add(predOps(ARMCC::AL)) - .add(condCodeOp()); - if (!constrainSelectedInstRegOperands(*Mov0I, TII, TRI, RBI)) + // The opcode used for performing the comparison. + const unsigned ComparisonOpcode; + + // The opcode used for reading the flags set by the comparison. May be + // ARM::INSTRUCTION_LIST_END if we don't need to read the flags. + const unsigned ReadFlagsOpcode; + + // The assumed register bank ID for the operands. + const unsigned OperandRegBankID; + + // The assumed size in bits for the operands. + const unsigned OperandSize; +}; + +struct ARMInstructionSelector::InsertInfo { + InsertInfo(MachineInstrBuilder &MIB) + : MBB(*MIB->getParent()), InsertBefore(std::next(MIB->getIterator())), + DbgLoc(MIB->getDebugLoc()) {} + + MachineBasicBlock &MBB; + const MachineBasicBlock::instr_iterator InsertBefore; + const DebugLoc &DbgLoc; +}; + +void ARMInstructionSelector::putConstant(InsertInfo I, unsigned DestReg, + unsigned Constant) const { + (void)BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, TII.get(ARM::MOVi)) + .addDef(DestReg) + .addImm(Constant) + .add(predOps(ARMCC::AL)) + .add(condCodeOp()); +} + +bool ARMInstructionSelector::validOpRegPair(MachineRegisterInfo &MRI, + unsigned LHSReg, unsigned RHSReg, + unsigned ExpectedSize, + unsigned ExpectedRegBankID) const { + return MRI.getType(LHSReg) == MRI.getType(RHSReg) && + validReg(MRI, LHSReg, ExpectedSize, ExpectedRegBankID) && + validReg(MRI, RHSReg, ExpectedSize, ExpectedRegBankID); +} + +bool ARMInstructionSelector::validReg(MachineRegisterInfo &MRI, unsigned Reg, + unsigned ExpectedSize, + unsigned ExpectedRegBankID) const { + if (MRI.getType(Reg).getSizeInBits() != ExpectedSize) { + DEBUG(dbgs() << "Unexpected size for register"); return false; + } - // Perform the comparison. - auto LHSReg = MIB->getOperand(2).getReg(); - auto RHSReg = MIB->getOperand(3).getReg(); - assert(MRI.getType(LHSReg) == MRI.getType(RHSReg) && - MRI.getType(LHSReg).getSizeInBits() == 32 && - MRI.getType(RHSReg).getSizeInBits() == 32 && - "Unsupported types for comparison operation"); - auto CmpI = BuildMI(MBB, InsertBefore, DebugLoc, TII.get(ARM::CMPrr)) - .addUse(LHSReg) - .addUse(RHSReg) - .add(predOps(ARMCC::AL)); - if (!constrainSelectedInstRegOperands(*CmpI, TII, TRI, RBI)) + if (RBI.getRegBank(Reg, MRI, TRI)->getID() != ExpectedRegBankID) { + DEBUG(dbgs() << "Unexpected register bank for register"); return false; + } + + return true; +} + +bool ARMInstructionSelector::selectCmp(CmpConstants Helper, + MachineInstrBuilder &MIB, + MachineRegisterInfo &MRI) const { + const InsertInfo I(MIB); - // Move 1 into the result register if the flags say so. auto ResReg = MIB->getOperand(0).getReg(); + if (!validReg(MRI, ResReg, 1, ARM::GPRRegBankID)) + return false; + auto Cond = static_cast(MIB->getOperand(1).getPredicate()); - auto ARMCond = getComparePred(Cond); - if (ARMCond == ARMCC::AL) + if (Cond == CmpInst::FCMP_TRUE || Cond == CmpInst::FCMP_FALSE) { + putConstant(I, ResReg, Cond == CmpInst::FCMP_TRUE ? 1 : 0); + MIB->eraseFromParent(); + return true; + } + + auto LHSReg = MIB->getOperand(2).getReg(); + auto RHSReg = MIB->getOperand(3).getReg(); + if (!validOpRegPair(MRI, LHSReg, RHSReg, Helper.OperandSize, + Helper.OperandRegBankID)) return false; - auto Mov1I = BuildMI(MBB, InsertBefore, DebugLoc, TII.get(ARM::MOVCCi)) - .addDef(ResReg) - .addUse(Mov0I->getOperand(0).getReg()) - .addImm(1) - .add(predOps(ARMCond, ARM::CPSR)); - if (!constrainSelectedInstRegOperands(*Mov1I, TII, TRI, RBI)) - return false; + auto ARMConds = getComparePreds(Cond); + auto ZeroReg = MRI.createVirtualRegister(&ARM::GPRRegClass); + putConstant(I, ZeroReg, 0); + + if (ARMConds.second == ARMCC::AL) { + // Simple case, we only need one comparison and we're done. + if (!insertComparison(Helper, I, ResReg, ARMConds.first, LHSReg, RHSReg, + ZeroReg)) + return false; + } else { + // Not so simple, we need two successive comparisons. + auto IntermediateRes = MRI.createVirtualRegister(&ARM::GPRRegClass); + if (!insertComparison(Helper, I, IntermediateRes, ARMConds.first, LHSReg, + RHSReg, ZeroReg)) + return false; + if (!insertComparison(Helper, I, ResReg, ARMConds.second, LHSReg, RHSReg, + IntermediateRes)) + return false; + } MIB->eraseFromParent(); return true; } +bool ARMInstructionSelector::insertComparison(CmpConstants Helper, InsertInfo I, + unsigned ResReg, + ARMCC::CondCodes Cond, + unsigned LHSReg, unsigned RHSReg, + unsigned PrevRes) const { + // Perform the comparison. + auto CmpI = + BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, TII.get(Helper.ComparisonOpcode)) + .addUse(LHSReg) + .addUse(RHSReg) + .add(predOps(ARMCC::AL)); + if (!constrainSelectedInstRegOperands(*CmpI, TII, TRI, RBI)) + return false; + + // Read the comparison flags (if necessary). + if (Helper.ReadFlagsOpcode != ARM::INSTRUCTION_LIST_END) { + auto ReadI = BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, + TII.get(Helper.ReadFlagsOpcode)) + .add(predOps(ARMCC::AL)); + if (!constrainSelectedInstRegOperands(*ReadI, TII, TRI, RBI)) + return false; + } + + // Select either 1 or the previous result based on the value of the flags. + auto Mov1I = BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, TII.get(ARM::MOVCCi)) + .addDef(ResReg) + .addUse(PrevRes) + .addImm(1) + .add(predOps(Cond, ARM::CPSR)); + if (!constrainSelectedInstRegOperands(*Mov1I, TII, TRI, RBI)) + return false; + + return true; +} + bool ARMInstructionSelector::selectSelect(MachineInstrBuilder &MIB, - const ARMBaseInstrInfo &TII, - MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI) const { + MachineRegisterInfo &MRI) const { auto &MBB = *MIB->getParent(); auto InsertBefore = std::next(MIB->getIterator()); - auto &DebugLoc = MIB->getDebugLoc(); + auto &DbgLoc = MIB->getDebugLoc(); // Compare the condition to 0. auto CondReg = MIB->getOperand(1).getReg(); - assert(MRI.getType(CondReg).getSizeInBits() == 1 && - RBI.getRegBank(CondReg, MRI, TRI)->getID() == ARM::GPRRegBankID && + assert(validReg(MRI, CondReg, 1, ARM::GPRRegBankID) && "Unsupported types for select operation"); - auto CmpI = BuildMI(MBB, InsertBefore, DebugLoc, TII.get(ARM::CMPri)) + auto CmpI = BuildMI(MBB, InsertBefore, DbgLoc, TII.get(ARM::CMPri)) .addUse(CondReg) .addImm(0) .add(predOps(ARMCC::AL)); @@ -376,13 +510,10 @@ bool ARMInstructionSelector::selectSelect(MachineInstrBuilder &MIB, auto ResReg = MIB->getOperand(0).getReg(); auto TrueReg = MIB->getOperand(2).getReg(); auto FalseReg = MIB->getOperand(3).getReg(); - assert(MRI.getType(ResReg) == MRI.getType(TrueReg) && - MRI.getType(TrueReg) == MRI.getType(FalseReg) && - MRI.getType(FalseReg).getSizeInBits() == 32 && - RBI.getRegBank(TrueReg, MRI, TRI)->getID() == ARM::GPRRegBankID && - RBI.getRegBank(FalseReg, MRI, TRI)->getID() == ARM::GPRRegBankID && + assert(validOpRegPair(MRI, ResReg, TrueReg, 32, ARM::GPRRegBankID) && + validOpRegPair(MRI, TrueReg, FalseReg, 32, ARM::GPRRegBankID) && "Unsupported types for select operation"); - auto Mov1I = BuildMI(MBB, InsertBefore, DebugLoc, TII.get(ARM::MOVCCr)) + auto Mov1I = BuildMI(MBB, InsertBefore, DbgLoc, TII.get(ARM::MOVCCr)) .addDef(ResReg) .addUse(TrueReg) .addUse(FalseReg) @@ -494,10 +625,32 @@ bool ARMInstructionSelector::select(MachineInstr &I) const { I.setDesc(TII.get(COPY)); return selectCopy(I, TII, MRI, TRI, RBI); } - case G_ICMP: - return selectICmp(MIB, TII, MRI, TRI, RBI); case G_SELECT: - return selectSelect(MIB, TII, MRI, TRI, RBI); + return selectSelect(MIB, MRI); + case G_ICMP: { + CmpConstants Helper(ARM::CMPrr, ARM::INSTRUCTION_LIST_END, + ARM::GPRRegBankID, 32); + return selectCmp(Helper, MIB, MRI); + } + case G_FCMP: { + assert(TII.getSubtarget().hasVFP2() && "Can't select fcmp without VFP"); + + unsigned OpReg = I.getOperand(2).getReg(); + unsigned Size = MRI.getType(OpReg).getSizeInBits(); + + if (Size == 64 && TII.getSubtarget().isFPOnlySP()) { + DEBUG(dbgs() << "Subtarget only supports single precision"); + return false; + } + if (Size != 32 && Size != 64) { + DEBUG(dbgs() << "Unsupported size for G_FCMP operand"); + return false; + } + + CmpConstants Helper(Size == 32 ? ARM::VCMPS : ARM::VCMPD, ARM::FMSTAT, + ARM::FPRRegBankID, Size); + return selectCmp(Helper, MIB, MRI); + } case G_GEP: I.setDesc(TII.get(ARM::ADDrr)); MIB.add(predOps(ARMCC::AL)).add(condCodeOp()); @@ -510,11 +663,10 @@ bool ARMInstructionSelector::select(MachineInstr &I) const { break; case G_CONSTANT: { unsigned Reg = I.getOperand(0).getReg(); - if (MRI.getType(Reg).getSizeInBits() != 32) + + if (!validReg(MRI, Reg, 32, ARM::GPRRegBankID)) return false; - assert(RBI.getRegBank(Reg, MRI, TRI)->getID() == ARM::GPRRegBankID && - "Expected constant to live in a GPR"); I.setDesc(TII.get(ARM::MOVi)); MIB.add(predOps(ARMCC::AL)).add(condCodeOp()); diff --git a/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp index f3e62d09cc30..f23e62595d2e 100644 --- a/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp @@ -28,6 +28,10 @@ using namespace llvm; #error "You shouldn't build this" #endif +static bool AEABI(const ARMSubtarget &ST) { + return ST.isTargetAEABI() || ST.isTargetGNUAEABI() || ST.isTargetMuslAEABI(); +} + ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) { using namespace TargetOpcode; @@ -66,8 +70,7 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) { for (unsigned Op : {G_SREM, G_UREM}) if (ST.hasDivideInARMMode()) setAction({Op, s32}, Lower); - else if (ST.isTargetAEABI() || ST.isTargetGNUAEABI() || - ST.isTargetMuslAEABI()) + else if (AEABI(ST)) setAction({Op, s32}, Custom); else setAction({Op, s32}, Libcall); @@ -86,6 +89,8 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) { setAction({G_SELECT, 1, s1}, Legal); setAction({G_CONSTANT, s32}, Legal); + for (auto Ty : {s1, s8, s16}) + setAction({G_CONSTANT, Ty}, WidenScalar); setAction({G_ICMP, s1}, Legal); for (auto Ty : {s8, s16}) @@ -99,9 +104,22 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) { setAction({G_LOAD, s64}, Legal); setAction({G_STORE, s64}, Legal); + + setAction({G_FCMP, s1}, Legal); + setAction({G_FCMP, 1, s32}, Legal); + setAction({G_FCMP, 1, s64}, Legal); } else { for (auto Ty : {s32, s64}) setAction({G_FADD, Ty}, Libcall); + + setAction({G_FCMP, s1}, Legal); + setAction({G_FCMP, 1, s32}, Custom); + setAction({G_FCMP, 1, s64}, Custom); + + if (AEABI(ST)) + setFCmpLibcallsAEABI(); + else + setFCmpLibcallsGNU(); } for (unsigned Op : {G_FREM, G_FPOW}) @@ -111,11 +129,120 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) { computeTables(); } +void ARMLegalizerInfo::setFCmpLibcallsAEABI() { + // FCMP_TRUE and FCMP_FALSE don't need libcalls, they should be + // default-initialized. + FCmp32Libcalls.resize(CmpInst::LAST_FCMP_PREDICATE + 1); + FCmp32Libcalls[CmpInst::FCMP_OEQ] = { + {RTLIB::OEQ_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_OGE] = { + {RTLIB::OGE_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_OGT] = { + {RTLIB::OGT_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_OLE] = { + {RTLIB::OLE_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_OLT] = { + {RTLIB::OLT_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_ORD] = {{RTLIB::O_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_UGE] = {{RTLIB::OLT_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_UGT] = {{RTLIB::OLE_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_ULE] = {{RTLIB::OGT_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_ULT] = {{RTLIB::OGE_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_UNE] = {{RTLIB::UNE_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_UNO] = { + {RTLIB::UO_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_ONE] = { + {RTLIB::OGT_F32, CmpInst::BAD_ICMP_PREDICATE}, + {RTLIB::OLT_F32, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp32Libcalls[CmpInst::FCMP_UEQ] = { + {RTLIB::OEQ_F32, CmpInst::BAD_ICMP_PREDICATE}, + {RTLIB::UO_F32, CmpInst::BAD_ICMP_PREDICATE}}; + + FCmp64Libcalls.resize(CmpInst::LAST_FCMP_PREDICATE + 1); + FCmp64Libcalls[CmpInst::FCMP_OEQ] = { + {RTLIB::OEQ_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_OGE] = { + {RTLIB::OGE_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_OGT] = { + {RTLIB::OGT_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_OLE] = { + {RTLIB::OLE_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_OLT] = { + {RTLIB::OLT_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_ORD] = {{RTLIB::O_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_UGE] = {{RTLIB::OLT_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_UGT] = {{RTLIB::OLE_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_ULE] = {{RTLIB::OGT_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_ULT] = {{RTLIB::OGE_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_UNE] = {{RTLIB::UNE_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_UNO] = { + {RTLIB::UO_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_ONE] = { + {RTLIB::OGT_F64, CmpInst::BAD_ICMP_PREDICATE}, + {RTLIB::OLT_F64, CmpInst::BAD_ICMP_PREDICATE}}; + FCmp64Libcalls[CmpInst::FCMP_UEQ] = { + {RTLIB::OEQ_F64, CmpInst::BAD_ICMP_PREDICATE}, + {RTLIB::UO_F64, CmpInst::BAD_ICMP_PREDICATE}}; +} + +void ARMLegalizerInfo::setFCmpLibcallsGNU() { + // FCMP_TRUE and FCMP_FALSE don't need libcalls, they should be + // default-initialized. + FCmp32Libcalls.resize(CmpInst::LAST_FCMP_PREDICATE + 1); + FCmp32Libcalls[CmpInst::FCMP_OEQ] = {{RTLIB::OEQ_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_OGE] = {{RTLIB::OGE_F32, CmpInst::ICMP_SGE}}; + FCmp32Libcalls[CmpInst::FCMP_OGT] = {{RTLIB::OGT_F32, CmpInst::ICMP_SGT}}; + FCmp32Libcalls[CmpInst::FCMP_OLE] = {{RTLIB::OLE_F32, CmpInst::ICMP_SLE}}; + FCmp32Libcalls[CmpInst::FCMP_OLT] = {{RTLIB::OLT_F32, CmpInst::ICMP_SLT}}; + FCmp32Libcalls[CmpInst::FCMP_ORD] = {{RTLIB::O_F32, CmpInst::ICMP_EQ}}; + FCmp32Libcalls[CmpInst::FCMP_UGE] = {{RTLIB::OLT_F32, CmpInst::ICMP_SGE}}; + FCmp32Libcalls[CmpInst::FCMP_UGT] = {{RTLIB::OLE_F32, CmpInst::ICMP_SGT}}; + FCmp32Libcalls[CmpInst::FCMP_ULE] = {{RTLIB::OGT_F32, CmpInst::ICMP_SLE}}; + FCmp32Libcalls[CmpInst::FCMP_ULT] = {{RTLIB::OGE_F32, CmpInst::ICMP_SLT}}; + FCmp32Libcalls[CmpInst::FCMP_UNE] = {{RTLIB::UNE_F32, CmpInst::ICMP_NE}}; + FCmp32Libcalls[CmpInst::FCMP_UNO] = {{RTLIB::UO_F32, CmpInst::ICMP_NE}}; + FCmp32Libcalls[CmpInst::FCMP_ONE] = {{RTLIB::OGT_F32, CmpInst::ICMP_SGT}, + {RTLIB::OLT_F32, CmpInst::ICMP_SLT}}; + FCmp32Libcalls[CmpInst::FCMP_UEQ] = {{RTLIB::OEQ_F32, CmpInst::ICMP_EQ}, + {RTLIB::UO_F32, CmpInst::ICMP_NE}}; + + FCmp64Libcalls.resize(CmpInst::LAST_FCMP_PREDICATE + 1); + FCmp64Libcalls[CmpInst::FCMP_OEQ] = {{RTLIB::OEQ_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_OGE] = {{RTLIB::OGE_F64, CmpInst::ICMP_SGE}}; + FCmp64Libcalls[CmpInst::FCMP_OGT] = {{RTLIB::OGT_F64, CmpInst::ICMP_SGT}}; + FCmp64Libcalls[CmpInst::FCMP_OLE] = {{RTLIB::OLE_F64, CmpInst::ICMP_SLE}}; + FCmp64Libcalls[CmpInst::FCMP_OLT] = {{RTLIB::OLT_F64, CmpInst::ICMP_SLT}}; + FCmp64Libcalls[CmpInst::FCMP_ORD] = {{RTLIB::O_F64, CmpInst::ICMP_EQ}}; + FCmp64Libcalls[CmpInst::FCMP_UGE] = {{RTLIB::OLT_F64, CmpInst::ICMP_SGE}}; + FCmp64Libcalls[CmpInst::FCMP_UGT] = {{RTLIB::OLE_F64, CmpInst::ICMP_SGT}}; + FCmp64Libcalls[CmpInst::FCMP_ULE] = {{RTLIB::OGT_F64, CmpInst::ICMP_SLE}}; + FCmp64Libcalls[CmpInst::FCMP_ULT] = {{RTLIB::OGE_F64, CmpInst::ICMP_SLT}}; + FCmp64Libcalls[CmpInst::FCMP_UNE] = {{RTLIB::UNE_F64, CmpInst::ICMP_NE}}; + FCmp64Libcalls[CmpInst::FCMP_UNO] = {{RTLIB::UO_F64, CmpInst::ICMP_NE}}; + FCmp64Libcalls[CmpInst::FCMP_ONE] = {{RTLIB::OGT_F64, CmpInst::ICMP_SGT}, + {RTLIB::OLT_F64, CmpInst::ICMP_SLT}}; + FCmp64Libcalls[CmpInst::FCMP_UEQ] = {{RTLIB::OEQ_F64, CmpInst::ICMP_EQ}, + {RTLIB::UO_F64, CmpInst::ICMP_NE}}; +} + +ARMLegalizerInfo::FCmpLibcallsList +ARMLegalizerInfo::getFCmpLibcalls(CmpInst::Predicate Predicate, + unsigned Size) const { + assert(CmpInst::isFPPredicate(Predicate) && "Unsupported FCmp predicate"); + if (Size == 32) + return FCmp32Libcalls[Predicate]; + if (Size == 64) + return FCmp64Libcalls[Predicate]; + llvm_unreachable("Unsupported size for FCmp predicate"); +} + bool ARMLegalizerInfo::legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const { using namespace TargetOpcode; + MIRBuilder.setInstr(MI); + switch (MI.getOpcode()) { default: return false; @@ -137,9 +264,9 @@ bool ARMLegalizerInfo::legalizeCustom(MachineInstr &MI, auto RetVal = MRI.createGenericVirtualRegister( getLLTForType(*RetTy, MIRBuilder.getMF().getDataLayout())); - auto Status = replaceWithLibcall(MI, MIRBuilder, Libcall, {RetVal, RetTy}, - {{MI.getOperand(1).getReg(), ArgTy}, - {MI.getOperand(2).getReg(), ArgTy}}); + auto Status = createLibcall(MIRBuilder, Libcall, {RetVal, RetTy}, + {{MI.getOperand(1).getReg(), ArgTy}, + {MI.getOperand(2).getReg(), ArgTy}}); if (Status != LegalizerHelper::Legalized) return false; @@ -149,8 +276,76 @@ bool ARMLegalizerInfo::legalizeCustom(MachineInstr &MI, MIRBuilder.buildUnmerge( {MRI.createGenericVirtualRegister(LLT::scalar(32)), OriginalResult}, RetVal); + break; + } + case G_FCMP: { + assert(MRI.getType(MI.getOperand(2).getReg()) == + MRI.getType(MI.getOperand(3).getReg()) && + "Mismatched operands for G_FCMP"); + auto OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); - return LegalizerHelper::Legalized; + auto OriginalResult = MI.getOperand(0).getReg(); + auto Predicate = + static_cast(MI.getOperand(1).getPredicate()); + auto Libcalls = getFCmpLibcalls(Predicate, OpSize); + + if (Libcalls.empty()) { + assert((Predicate == CmpInst::FCMP_TRUE || + Predicate == CmpInst::FCMP_FALSE) && + "Predicate needs libcalls, but none specified"); + MIRBuilder.buildConstant(OriginalResult, + Predicate == CmpInst::FCMP_TRUE ? 1 : 0); + MI.eraseFromParent(); + return true; + } + + auto &Ctx = MIRBuilder.getMF().getFunction()->getContext(); + assert((OpSize == 32 || OpSize == 64) && "Unsupported operand size"); + auto *ArgTy = OpSize == 32 ? Type::getFloatTy(Ctx) : Type::getDoubleTy(Ctx); + auto *RetTy = Type::getInt32Ty(Ctx); + + SmallVector Results; + for (auto Libcall : Libcalls) { + auto LibcallResult = MRI.createGenericVirtualRegister(LLT::scalar(32)); + auto Status = + createLibcall(MIRBuilder, Libcall.LibcallID, {LibcallResult, RetTy}, + {{MI.getOperand(2).getReg(), ArgTy}, + {MI.getOperand(3).getReg(), ArgTy}}); + + if (Status != LegalizerHelper::Legalized) + return false; + + auto ProcessedResult = + Libcalls.size() == 1 + ? OriginalResult + : MRI.createGenericVirtualRegister(MRI.getType(OriginalResult)); + + // We have a result, but we need to transform it into a proper 1-bit 0 or + // 1, taking into account the different peculiarities of the values + // returned by the comparison functions. + CmpInst::Predicate ResultPred = Libcall.Predicate; + if (ResultPred == CmpInst::BAD_ICMP_PREDICATE) { + // We have a nice 0 or 1, and we just need to truncate it back to 1 bit + // to keep the types consistent. + MIRBuilder.buildTrunc(ProcessedResult, LibcallResult); + } else { + // We need to compare against 0. + assert(CmpInst::isIntPredicate(ResultPred) && "Unsupported predicate"); + auto Zero = MRI.createGenericVirtualRegister(LLT::scalar(32)); + MIRBuilder.buildConstant(Zero, 0); + MIRBuilder.buildICmp(ResultPred, ProcessedResult, LibcallResult, Zero); + } + Results.push_back(ProcessedResult); + } + + if (Results.size() != 1) { + assert(Results.size() == 2 && "Unexpected number of results"); + MIRBuilder.buildOr(OriginalResult, Results[0], Results[1]); + } + break; } } + + MI.eraseFromParent(); + return true; } diff --git a/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.h b/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.h index a9bdd367737e..78ab9412c04b 100644 --- a/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.h +++ b/contrib/llvm/lib/Target/ARM/ARMLegalizerInfo.h @@ -14,7 +14,10 @@ #ifndef LLVM_LIB_TARGET_ARM_ARMMACHINELEGALIZER_H #define LLVM_LIB_TARGET_ARM_ARMMACHINELEGALIZER_H +#include "llvm/ADT/IndexedMap.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/IR/Instructions.h" namespace llvm { @@ -27,6 +30,36 @@ class ARMLegalizerInfo : public LegalizerInfo { bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const override; + +private: + void setFCmpLibcallsGNU(); + void setFCmpLibcallsAEABI(); + + struct FCmpLibcallInfo { + // Which libcall this is. + RTLIB::Libcall LibcallID; + + // The predicate to be used when comparing the value returned by the + // function with a relevant constant (currently hard-coded to zero). This is + // necessary because often the libcall will return e.g. a value greater than + // 0 to represent 'true' and anything negative to represent 'false', or + // maybe 0 to represent 'true' and non-zero for 'false'. If no comparison is + // needed, this should be CmpInst::BAD_ICMP_PREDICATE. + CmpInst::Predicate Predicate; + }; + using FCmpLibcallsList = SmallVector; + + // Map from each FCmp predicate to the corresponding libcall infos. A FCmp + // instruction may be lowered to one or two libcalls, which is why we need a + // list. If two libcalls are needed, their results will be OR'ed. + using FCmpLibcallsMapTy = IndexedMap; + + FCmpLibcallsMapTy FCmp32Libcalls; + FCmpLibcallsMapTy FCmp64Libcalls; + + // Get the libcall(s) corresponding to \p Predicate for operands of \p Size + // bits. + FCmpLibcallsList getFCmpLibcalls(CmpInst::Predicate, unsigned Size) const; }; } // End llvm namespace. #endif diff --git a/contrib/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp index 11fb81a4f9fe..c0c09e8c15af 100644 --- a/contrib/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp @@ -212,8 +212,6 @@ ARMRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { const MachineFunction &MF = *MI.getParent()->getParent(); const MachineRegisterInfo &MRI = MF.getRegInfo(); - LLT Ty = MRI.getType(MI.getOperand(0).getReg()); - unsigned NumOperands = MI.getNumOperands(); const ValueMapping *OperandsMapping = &ARM::ValueMappings[ARM::GPR3OpsIdx]; @@ -236,26 +234,31 @@ ARMRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OperandsMapping = &ARM::ValueMappings[ARM::GPR3OpsIdx]; break; case G_LOAD: - case G_STORE: + case G_STORE: { + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); OperandsMapping = Ty.getSizeInBits() == 64 ? getOperandsMapping({&ARM::ValueMappings[ARM::DPR3OpsIdx], &ARM::ValueMappings[ARM::GPR3OpsIdx]}) : &ARM::ValueMappings[ARM::GPR3OpsIdx]; break; - case G_FADD: + } + case G_FADD: { + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); assert((Ty.getSizeInBits() == 32 || Ty.getSizeInBits() == 64) && "Unsupported size for G_FADD"); OperandsMapping = Ty.getSizeInBits() == 64 ? &ARM::ValueMappings[ARM::DPR3OpsIdx] : &ARM::ValueMappings[ARM::SPR3OpsIdx]; break; + } case G_CONSTANT: case G_FRAME_INDEX: OperandsMapping = getOperandsMapping({&ARM::ValueMappings[ARM::GPR3OpsIdx], nullptr}); break; case G_SELECT: { + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); LLT Ty2 = MRI.getType(MI.getOperand(1).getReg()); (void)Ty2; assert(Ty.getSizeInBits() == 32 && "Unsupported size for G_SELECT"); @@ -277,9 +280,29 @@ ARMRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { &ARM::ValueMappings[ARM::GPR3OpsIdx]}); break; } + case G_FCMP: { + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); + LLT Ty1 = MRI.getType(MI.getOperand(2).getReg()); + LLT Ty2 = MRI.getType(MI.getOperand(3).getReg()); + (void)Ty2; + assert(Ty.getSizeInBits() == 1 && "Unsupported size for G_FCMP"); + assert(Ty1.getSizeInBits() == Ty2.getSizeInBits() && + "Mismatched operand sizes for G_FCMP"); + + unsigned Size = Ty1.getSizeInBits(); + assert((Size == 32 || Size == 64) && "Unsupported size for G_FCMP"); + + auto FPRValueMapping = Size == 32 ? &ARM::ValueMappings[ARM::SPR3OpsIdx] + : &ARM::ValueMappings[ARM::DPR3OpsIdx]; + OperandsMapping = + getOperandsMapping({&ARM::ValueMappings[ARM::GPR3OpsIdx], nullptr, + FPRValueMapping, FPRValueMapping}); + break; + } case G_MERGE_VALUES: { // We only support G_MERGE_VALUES for creating a double precision floating // point value out of two GPRs. + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); LLT Ty1 = MRI.getType(MI.getOperand(1).getReg()); LLT Ty2 = MRI.getType(MI.getOperand(2).getReg()); if (Ty.getSizeInBits() != 64 || Ty1.getSizeInBits() != 32 || @@ -294,6 +317,7 @@ ARMRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case G_UNMERGE_VALUES: { // We only support G_UNMERGE_VALUES for splitting a double precision // floating point value into two GPRs. + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); LLT Ty1 = MRI.getType(MI.getOperand(1).getReg()); LLT Ty2 = MRI.getType(MI.getOperand(2).getReg()); if (Ty.getSizeInBits() != 32 || Ty1.getSizeInBits() != 32 || diff --git a/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index 8eb9dbf5f9de..51b0fedd2b54 100644 --- a/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -15,6 +15,24 @@ using namespace llvm; #define DEBUG_TYPE "armtti" +bool ARMTTIImpl::areInlineCompatible(const Function *Caller, + const Function *Callee) const { + const TargetMachine &TM = getTLI()->getTargetMachine(); + const FeatureBitset &CallerBits = + TM.getSubtargetImpl(*Caller)->getFeatureBits(); + const FeatureBitset &CalleeBits = + TM.getSubtargetImpl(*Callee)->getFeatureBits(); + + // To inline a callee, all features not in the whitelist must match exactly. + bool MatchExact = (CallerBits & ~InlineFeatureWhitelist) == + (CalleeBits & ~InlineFeatureWhitelist); + // For features in the whitelist, the callee's features must be a subset of + // the callers'. + bool MatchSubset = ((CallerBits & CalleeBits) & InlineFeatureWhitelist) == + (CalleeBits & InlineFeatureWhitelist); + return MatchExact && MatchSubset; +} + int ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty) { assert(Ty->isIntegerTy()); diff --git a/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 8a1a37863877..0695a4e63346 100644 --- a/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/contrib/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -33,6 +33,39 @@ class ARMTTIImpl : public BasicTTIImplBase { const ARMSubtarget *ST; const ARMTargetLowering *TLI; + // Currently the following features are excluded from InlineFeatureWhitelist. + // ModeThumb, FeatureNoARM, ModeSoftFloat, FeatureVFPOnlySP, FeatureD16 + // Depending on whether they are set or unset, different + // instructions/registers are available. For example, inlining a callee with + // -thumb-mode in a caller with +thumb-mode, may cause the assembler to + // fail if the callee uses ARM only instructions, e.g. in inline asm. + const FeatureBitset InlineFeatureWhitelist = { + ARM::FeatureVFP2, ARM::FeatureVFP3, ARM::FeatureNEON, ARM::FeatureThumb2, + ARM::FeatureFP16, ARM::FeatureVFP4, ARM::FeatureFPARMv8, + ARM::FeatureFullFP16, ARM::FeatureHWDivThumb, + ARM::FeatureHWDivARM, ARM::FeatureDB, ARM::FeatureV7Clrex, + ARM::FeatureAcquireRelease, ARM::FeatureSlowFPBrcc, + ARM::FeaturePerfMon, ARM::FeatureTrustZone, ARM::Feature8MSecExt, + ARM::FeatureCrypto, ARM::FeatureCRC, ARM::FeatureRAS, + ARM::FeatureFPAO, ARM::FeatureFuseAES, ARM::FeatureZCZeroing, + ARM::FeatureProfUnpredicate, ARM::FeatureSlowVGETLNi32, + ARM::FeatureSlowVDUP32, ARM::FeaturePreferVMOVSR, + ARM::FeaturePrefISHSTBarrier, ARM::FeatureMuxedUnits, + ARM::FeatureSlowOddRegister, ARM::FeatureSlowLoadDSubreg, + ARM::FeatureDontWidenVMOVS, ARM::FeatureExpandMLx, + ARM::FeatureHasVMLxHazards, ARM::FeatureNEONForFPMovs, + ARM::FeatureNEONForFP, ARM::FeatureCheckVLDnAlign, + ARM::FeatureHasSlowFPVMLx, ARM::FeatureVMLxForwarding, + ARM::FeaturePref32BitThumb, ARM::FeatureAvoidPartialCPSR, + ARM::FeatureCheapPredicableCPSR, ARM::FeatureAvoidMOVsShOp, + ARM::FeatureHasRetAddrStack, ARM::FeatureHasNoBranchPredictor, + ARM::FeatureDSP, ARM::FeatureMP, ARM::FeatureVirtualization, + ARM::FeatureMClass, ARM::FeatureRClass, ARM::FeatureAClass, + ARM::FeatureNaClTrap, ARM::FeatureStrictAlign, ARM::FeatureLongCalls, + ARM::FeatureExecuteOnly, ARM::FeatureReserveR9, ARM::FeatureNoMovt, + ARM::FeatureNoNegativeImmediates + }; + const ARMSubtarget *getST() const { return ST; } const ARMTargetLowering *getTLI() const { return TLI; } @@ -41,6 +74,9 @@ class ARMTTIImpl : public BasicTTIImplBase { : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {} + bool areInlineCompatible(const Function *Caller, + const Function *Callee) const; + bool enableInterleavedAccessVectorization() { return true; } /// Floating-point computation using ARMv8 AArch32 Advanced diff --git a/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 891b5c60e1fd..1129826f21f6 100644 --- a/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/contrib/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -5249,6 +5249,7 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) { // Fall though for the Identifier case that is not a register or a // special name. + LLVM_FALLTHROUGH; } case AsmToken::LParen: // parenthesized expressions like (_strcmp-4) case AsmToken::Integer: // things like 1f and 2b as a branch targets @@ -8992,6 +8993,8 @@ unsigned ARMAsmParser::MatchInstruction(OperandVector &Operands, MCInst &Inst, return PlainMatchResult; } +std::string ARMMnemonicSpellCheck(StringRef S, uint64_t FBS); + static const char *getSubtargetFeatureName(uint64_t Val); bool ARMAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, OperandVector &Operands, @@ -9085,9 +9088,13 @@ bool ARMAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, return Error(ErrorLoc, "invalid operand for instruction"); } - case Match_MnemonicFail: - return Error(IDLoc, "invalid instruction", + case Match_MnemonicFail: { + uint64_t FBS = ComputeAvailableFeatures(getSTI().getFeatureBits()); + std::string Suggestion = ARMMnemonicSpellCheck( + ((ARMOperand &)*Operands[0]).getToken(), FBS); + return Error(IDLoc, "invalid instruction" + Suggestion, ((ARMOperand &)*Operands[0]).getLocRange()); + } case Match_RequiresNotITBlock: return Error(IDLoc, "flag setting instruction only valid outside IT block"); case Match_RequiresITBlock: diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp index 22de728fe06e..a77df7a2598f 100644 --- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp +++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp @@ -361,9 +361,8 @@ static uint32_t joinHalfWords(uint32_t FirstHalf, uint32_t SecondHalf, unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, uint64_t Value, - bool IsPCRel, MCContext &Ctx, - bool IsLittleEndian, - bool IsResolved) const { + bool IsResolved, MCContext &Ctx, + bool IsLittleEndian) const { unsigned Kind = Fixup.getKind(); // MachO tries to make .o files that look vaguely pre-linked, so for MOVW/MOVT @@ -392,7 +391,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm, case FK_SecRel_4: return Value; case ARM::fixup_arm_movt_hi16: - if (!IsPCRel) + if (IsResolved || !STI->getTargetTriple().isOSBinFormatELF()) Value >>= 16; LLVM_FALLTHROUGH; case ARM::fixup_arm_movw_lo16: { @@ -404,7 +403,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm, return Value; } case ARM::fixup_t2_movt_hi16: - if (!IsPCRel) + if (IsResolved || !STI->getTargetTriple().isOSBinFormatELF()) Value >>= 16; LLVM_FALLTHROUGH; case ARM::fixup_t2_movw_lo16: { @@ -885,11 +884,11 @@ static unsigned getFixupKindContainerSizeBytes(unsigned Kind) { void ARMAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind()); MCContext &Ctx = Asm.getContext(); - Value = adjustFixupValue(Asm, Fixup, Target, Value, IsPCRel, Ctx, - IsLittleEndian, true); + Value = adjustFixupValue(Asm, Fixup, Target, Value, IsResolved, Ctx, + IsLittleEndian); if (!Value) return; // Doesn't change encoding. diff --git a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h index 84b54bbb9a49..02374966dafe 100644 --- a/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h +++ b/contrib/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h @@ -42,13 +42,13 @@ class ARMAsmBackend : public MCAsmBackend { const MCValue &Target) override; unsigned adjustFixupValue(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target, uint64_t Value, bool IsPCRel, - MCContext &Ctx, bool IsLittleEndian, - bool IsResolved) const; + const MCValue &Target, uint64_t Value, + bool IsResolved, MCContext &Ctx, + bool IsLittleEndian) const; void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; unsigned getRelaxedOpcode(unsigned Op) const; diff --git a/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index 0b6574c37de1..5709b4e61798 100644 --- a/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/contrib/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -236,7 +236,7 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF, case ARM::R12: if (STI.splitFramePushPop(MF)) break; - // fallthough + LLVM_FALLTHROUGH; case ARM::R0: case ARM::R1: case ARM::R2: diff --git a/contrib/llvm/lib/Target/AVR/AVRAsmPrinter.cpp b/contrib/llvm/lib/Target/AVR/AVRAsmPrinter.cpp index f0c7b11895b4..c058c9e1f534 100644 --- a/contrib/llvm/lib/Target/AVR/AVRAsmPrinter.cpp +++ b/contrib/llvm/lib/Target/AVR/AVRAsmPrinter.cpp @@ -149,7 +149,10 @@ bool AVRAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, (void)MO; assert(MO.isReg() && "Unexpected inline asm memory operand"); - // TODO: We can look up the alternative name for the register if it's given. + // TODO: We should be able to look up the alternative name for + // the register if it's given. + // TableGen doesn't expose a way of getting retrieving names + // for registers. if (MI->getOperand(OpNum).getReg() == AVR::R31R30) { O << "Z"; } else { diff --git a/contrib/llvm/lib/Target/AVR/AVRDevices.td b/contrib/llvm/lib/Target/AVR/AVRDevices.td index 9224af613d14..62def4574437 100644 --- a/contrib/llvm/lib/Target/AVR/AVRDevices.td +++ b/contrib/llvm/lib/Target/AVR/AVRDevices.td @@ -6,7 +6,6 @@ // :TODO: We define all devices with SRAM to have all variants of LD/ST/LDD/STD. // In reality, avr1 (no SRAM) has one variant each of `LD` and `ST`. // avr2 (with SRAM) adds the rest of the variants. -// :TODO: s/AVRTiny/Tiny // A feature set aggregates features, grouping them. We don't want to create a @@ -136,7 +135,7 @@ def ELFArchAVR4 : ELFArch<"EF_AVR_ARCH_AVR4">; def ELFArchAVR5 : ELFArch<"EF_AVR_ARCH_AVR5">; def ELFArchAVR51 : ELFArch<"EF_AVR_ARCH_AVR51">; def ELFArchAVR6 : ELFArch<"EF_AVR_ARCH_AVR6">; -def ELFArchAVRTiny : ELFArch<"EF_AVR_ARCH_AVRTINY">; +def ELFArchTiny : ELFArch<"EF_AVR_ARCH_AVRTINY">; def ELFArchXMEGA1 : ELFArch<"EF_AVR_ARCH_XMEGA1">; def ELFArchXMEGA2 : ELFArch<"EF_AVR_ARCH_XMEGA2">; def ELFArchXMEGA3 : ELFArch<"EF_AVR_ARCH_XMEGA3">; @@ -189,7 +188,7 @@ def FamilyAVR51 : Family<"avr51", def FamilyAVR6 : Family<"avr6", [FamilyAVR51]>; -def FamilyAVRTiny : Family<"avrtiny", +def FamilyTiny : Family<"avrtiny", [FamilyAVR0, FeatureBREAK, FeatureSRAM, FeatureTinyEncoding]>; @@ -240,7 +239,7 @@ def : Device<"avrxmega4", FamilyXMEGA, ELFArchXMEGA4>; def : Device<"avrxmega5", FamilyXMEGA, ELFArchXMEGA5>; def : Device<"avrxmega6", FamilyXMEGA, ELFArchXMEGA6>; def : Device<"avrxmega7", FamilyXMEGA, ELFArchXMEGA7>; -def : Device<"avrtiny", FamilyAVRTiny, ELFArchAVRTiny>; +def : Device<"avrtiny", FamilyTiny, ELFArchTiny>; // Specific MCUs def : Device<"at90s1200", FamilyAVR0, ELFArchAVR1>; @@ -480,12 +479,12 @@ def : Device<"atxmega384d3", FamilyXMEGA, ELFArchXMEGA6>; def : Device<"atxmega128a1", FamilyXMEGA, ELFArchXMEGA7>; def : Device<"atxmega128a1u", FamilyXMEGAU, ELFArchXMEGA7>; def : Device<"atxmega128a4u", FamilyXMEGAU, ELFArchXMEGA7>; -def : Device<"attiny4", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny5", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny9", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny10", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny20", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny40", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny102", FamilyAVRTiny, ELFArchAVRTiny>; -def : Device<"attiny104", FamilyAVRTiny, ELFArchAVRTiny>; +def : Device<"attiny4", FamilyTiny, ELFArchTiny>; +def : Device<"attiny5", FamilyTiny, ELFArchTiny>; +def : Device<"attiny9", FamilyTiny, ELFArchTiny>; +def : Device<"attiny10", FamilyTiny, ELFArchTiny>; +def : Device<"attiny20", FamilyTiny, ELFArchTiny>; +def : Device<"attiny40", FamilyTiny, ELFArchTiny>; +def : Device<"attiny102", FamilyTiny, ELFArchTiny>; +def : Device<"attiny104", FamilyTiny, ELFArchTiny>; diff --git a/contrib/llvm/lib/Target/AVR/AVRInstrInfo.cpp b/contrib/llvm/lib/Target/AVR/AVRInstrInfo.cpp index afba66b2e69b..744aa723c416 100644 --- a/contrib/llvm/lib/Target/AVR/AVRInstrInfo.cpp +++ b/contrib/llvm/lib/Target/AVR/AVRInstrInfo.cpp @@ -402,7 +402,7 @@ unsigned AVRInstrInfo::insertBranch(MachineBasicBlock &MBB, ArrayRef Cond, const DebugLoc &DL, int *BytesAdded) const { - assert(!BytesAdded && "code size not handled"); + if (BytesAdded) *BytesAdded = 0; // Shouldn't be a fall through. assert(TBB && "insertBranch must not be told to insert a fallthrough"); @@ -411,19 +411,24 @@ unsigned AVRInstrInfo::insertBranch(MachineBasicBlock &MBB, if (Cond.empty()) { assert(!FBB && "Unconditional branch with multiple successors!"); - BuildMI(&MBB, DL, get(AVR::RJMPk)).addMBB(TBB); + auto &MI = *BuildMI(&MBB, DL, get(AVR::RJMPk)).addMBB(TBB); + if (BytesAdded) + *BytesAdded += getInstSizeInBytes(MI); return 1; } // Conditional branch. unsigned Count = 0; AVRCC::CondCodes CC = (AVRCC::CondCodes)Cond[0].getImm(); - BuildMI(&MBB, DL, getBrCond(CC)).addMBB(TBB); + auto &CondMI = *BuildMI(&MBB, DL, getBrCond(CC)).addMBB(TBB); + + if (BytesAdded) *BytesAdded += getInstSizeInBytes(CondMI); ++Count; if (FBB) { // Two-way Conditional branch. Insert the second branch. - BuildMI(&MBB, DL, get(AVR::RJMPk)).addMBB(FBB); + auto &MI = *BuildMI(&MBB, DL, get(AVR::RJMPk)).addMBB(FBB); + if (BytesAdded) *BytesAdded += getInstSizeInBytes(MI); ++Count; } @@ -432,7 +437,7 @@ unsigned AVRInstrInfo::insertBranch(MachineBasicBlock &MBB, unsigned AVRInstrInfo::removeBranch(MachineBasicBlock &MBB, int *BytesRemoved) const { - assert(!BytesRemoved && "code size not handled"); + if (BytesRemoved) *BytesRemoved = 0; MachineBasicBlock::iterator I = MBB.end(); unsigned Count = 0; @@ -450,6 +455,7 @@ unsigned AVRInstrInfo::removeBranch(MachineBasicBlock &MBB, } // Remove the branch. + if (BytesRemoved) *BytesRemoved += getInstSizeInBytes(*I); I->eraseFromParent(); I = MBB.end(); ++Count; @@ -494,5 +500,61 @@ unsigned AVRInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { } } +MachineBasicBlock * +AVRInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { + switch (MI.getOpcode()) { + default: + llvm_unreachable("unexpected opcode!"); + case AVR::JMPk: + case AVR::CALLk: + case AVR::RCALLk: + case AVR::RJMPk: + case AVR::BREQk: + case AVR::BRNEk: + case AVR::BRSHk: + case AVR::BRLOk: + case AVR::BRMIk: + case AVR::BRPLk: + case AVR::BRGEk: + case AVR::BRLTk: + return MI.getOperand(0).getMBB(); + case AVR::BRBSsk: + case AVR::BRBCsk: + return MI.getOperand(1).getMBB(); + case AVR::SBRCRrB: + case AVR::SBRSRrB: + case AVR::SBICAb: + case AVR::SBISAb: + llvm_unreachable("unimplemented branch instructions"); + } +} + +bool AVRInstrInfo::isBranchOffsetInRange(unsigned BranchOp, + int64_t BrOffset) const { + + switch (BranchOp) { + default: + llvm_unreachable("unexpected opcode!"); + case AVR::JMPk: + case AVR::CALLk: + assert(BrOffset >= 0 && "offset must be absolute address"); + return isUIntN(16, BrOffset); + case AVR::RCALLk: + case AVR::RJMPk: + return isIntN(13, BrOffset); + case AVR::BRBSsk: + case AVR::BRBCsk: + case AVR::BREQk: + case AVR::BRNEk: + case AVR::BRSHk: + case AVR::BRLOk: + case AVR::BRMIk: + case AVR::BRPLk: + case AVR::BRGEk: + case AVR::BRLTk: + return isIntN(7, BrOffset); + } +} + } // end of namespace llvm diff --git a/contrib/llvm/lib/Target/AVR/AVRInstrInfo.h b/contrib/llvm/lib/Target/AVR/AVRInstrInfo.h index c5105dafe5eb..f42d34fb2848 100644 --- a/contrib/llvm/lib/Target/AVR/AVRInstrInfo.h +++ b/contrib/llvm/lib/Target/AVR/AVRInstrInfo.h @@ -103,6 +103,10 @@ class AVRInstrInfo : public AVRGenInstrInfo { bool reverseBranchCondition(SmallVectorImpl &Cond) const override; + MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override; + + bool isBranchOffsetInRange(unsigned BranchOpc, + int64_t BrOffset) const override; private: const AVRRegisterInfo RI; }; diff --git a/contrib/llvm/lib/Target/AVR/AVRInstrInfo.td b/contrib/llvm/lib/Target/AVR/AVRInstrInfo.td index 5dd8b2c27b21..184e4d53f7c8 100644 --- a/contrib/llvm/lib/Target/AVR/AVRInstrInfo.td +++ b/contrib/llvm/lib/Target/AVR/AVRInstrInfo.td @@ -1411,17 +1411,11 @@ hasSideEffects = 0 in def LPMRdZ : FLPMX<0, 0, (outs GPR8:$dst), - (ins ZREGS:$z), + (ins ZREG:$z), "lpm\t$dst, $z", []>, Requires<[HasLPMX]>; - def LPMWRdZ : Pseudo<(outs DREGS:$dst), - (ins ZREGS:$z), - "lpmw\t$dst, $z", - []>, - Requires<[HasLPMX]>; - // Load program memory, while postincrementing the Z register. let mayLoad = 1, Defs = [R31R30] in @@ -1429,13 +1423,19 @@ hasSideEffects = 0 in def LPMRdZPi : FLPMX<0, 1, (outs GPR8:$dst), - (ins ZREGS:$z), + (ins ZREG:$z), "lpm\t$dst, $z+", []>, Requires<[HasLPMX]>; + def LPMWRdZ : Pseudo<(outs DREGS:$dst), + (ins ZREG:$z), + "lpmw\t$dst, $z", + []>, + Requires<[HasLPMX]>; + def LPMWRdZPi : Pseudo<(outs DREGS:$dst), - (ins ZREGS:$z), + (ins ZREG:$z), "lpmw\t$dst, $z+", []>, Requires<[HasLPMX]>; @@ -1458,7 +1458,7 @@ hasSideEffects = 0 in def ELPMRdZ : FLPMX<1, 0, (outs GPR8:$dst), - (ins ZREGS:$z), + (ins ZREG:$z), "elpm\t$dst, $z", []>, Requires<[HasELPMX]>; @@ -1467,7 +1467,7 @@ hasSideEffects = 0 in def ELPMRdZPi : FLPMX<1, 1, (outs GPR8:$dst), - (ins ZREGS: $z), + (ins ZREG: $z), "elpm\t$dst, $z+", []>, Requires<[HasELPMX]>; @@ -1487,7 +1487,7 @@ let Uses = [R1, R0] in let Defs = [R31R30] in def SPMZPi : F16<0b1001010111111000, (outs), - (ins ZREGS:$z), + (ins ZREG:$z), "spm $z+", []>, Requires<[HasSPMX]>; @@ -1564,28 +1564,28 @@ hasSideEffects = 0 in // Read-Write-Modify (RMW) instructions. def XCHZRd : FZRd<0b100, (outs GPR8:$rd), - (ins ZREGS:$z), + (ins ZREG:$z), "xch\t$z, $rd", []>, Requires<[SupportsRMW]>; def LASZRd : FZRd<0b101, (outs GPR8:$rd), - (ins ZREGS:$z), + (ins ZREG:$z), "las\t$z, $rd", []>, Requires<[SupportsRMW]>; def LACZRd : FZRd<0b110, (outs GPR8:$rd), - (ins ZREGS:$z), + (ins ZREG:$z), "lac\t$z, $rd", []>, Requires<[SupportsRMW]>; def LATZRd : FZRd<0b111, (outs GPR8:$rd), - (ins ZREGS:$z), + (ins ZREG:$z), "lat\t$z, $rd", []>, Requires<[SupportsRMW]>; diff --git a/contrib/llvm/lib/Target/AVR/AVRMCInstLower.cpp b/contrib/llvm/lib/Target/AVR/AVRMCInstLower.cpp index 475dda420e89..dfefd09bc4b8 100644 --- a/contrib/llvm/lib/Target/AVR/AVRMCInstLower.cpp +++ b/contrib/llvm/lib/Target/AVR/AVRMCInstLower.cpp @@ -37,10 +37,22 @@ MCOperand AVRMCInstLower::lowerSymbolOperand(const MachineOperand &MO, Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx); } + bool IsFunction = MO.isGlobal() && isa(MO.getGlobal()); + if (TF & AVRII::MO_LO) { - Expr = AVRMCExpr::create(AVRMCExpr::VK_AVR_LO8, Expr, IsNegated, Ctx); + if (IsFunction) { + // N.B. Should we use _GS fixups here to cope with >128k progmem? + Expr = AVRMCExpr::create(AVRMCExpr::VK_AVR_PM_LO8, Expr, IsNegated, Ctx); + } else { + Expr = AVRMCExpr::create(AVRMCExpr::VK_AVR_LO8, Expr, IsNegated, Ctx); + } } else if (TF & AVRII::MO_HI) { - Expr = AVRMCExpr::create(AVRMCExpr::VK_AVR_HI8, Expr, IsNegated, Ctx); + if (IsFunction) { + // N.B. Should we use _GS fixups here to cope with >128k progmem? + Expr = AVRMCExpr::create(AVRMCExpr::VK_AVR_PM_HI8, Expr, IsNegated, Ctx); + } else { + Expr = AVRMCExpr::create(AVRMCExpr::VK_AVR_HI8, Expr, IsNegated, Ctx); + } } else if (TF != 0) { llvm_unreachable("Unknown target flag on symbol operand"); } diff --git a/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.cpp b/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.cpp index 55f3f5cf428a..249dc5512c28 100644 --- a/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.cpp +++ b/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.cpp @@ -95,7 +95,8 @@ AVRRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC, } /// Fold a frame offset shared between two add instructions into a single one. -static void foldFrameOffset(MachineInstr &MI, int &Offset, unsigned DstReg) { +static void foldFrameOffset(MachineBasicBlock::iterator &II, int &Offset, unsigned DstReg) { + MachineInstr &MI = *II; int Opcode = MI.getOpcode(); // Don't bother trying if the next instruction is not an add or a sub. @@ -120,6 +121,7 @@ static void foldFrameOffset(MachineInstr &MI, int &Offset, unsigned DstReg) { } // Finally remove the instruction. + II++; MI.eraseFromParent(); } @@ -158,6 +160,8 @@ void AVRRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, unsigned DstReg = MI.getOperand(0).getReg(); assert(DstReg != AVR::R29R28 && "Dest reg cannot be the frame pointer"); + II++; // Skip over the FRMIDX (and now MOVW) instruction. + // Generally, to load a frame address two add instructions are emitted that // could get folded into a single one: // movw r31:r30, r29:r28 @@ -166,7 +170,8 @@ void AVRRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, // to: // movw r31:r30, r29:r28 // adiw r31:r30, 45 - foldFrameOffset(*std::next(II), Offset, DstReg); + if (II != MBB.end()) + foldFrameOffset(II, Offset, DstReg); // Select the best opcode based on DstReg and the offset size. switch (DstReg) { @@ -187,7 +192,7 @@ void AVRRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, } } - MachineInstr *New = BuildMI(MBB, std::next(II), dl, TII.get(Opcode), DstReg) + MachineInstr *New = BuildMI(MBB, II, dl, TII.get(Opcode), DstReg) .addReg(DstReg, RegState::Kill) .addImm(Offset); New->getOperand(3).setIsDead(); diff --git a/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.td b/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.td index 32650fc66751..8162f12052be 100644 --- a/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.td +++ b/contrib/llvm/lib/Target/AVR/AVRRegisterInfo.td @@ -110,8 +110,6 @@ CoveredBySubRegs = 1 in // Register Classes //===----------------------------------------------------------------------===// -//:TODO: use proper set instructions instead of using always "add" - // Main 8-bit register class. def GPR8 : RegisterClass<"AVR", [i8], 8, ( @@ -199,14 +197,11 @@ def PTRDISPREGS : RegisterClass<"AVR", [i16], 8, // We have a bunch of instructions with an explicit Z register argument. We // model this using a register class containing only the Z register. -// :TODO: Rename to 'ZREG'. -def ZREGS : RegisterClass<"AVR", [i16], 8, (add R31R30)>; +def ZREG : RegisterClass<"AVR", [i16], 8, (add R31R30)>; // Register class used for the stack read pseudo instruction. def GPRSP: RegisterClass<"AVR", [i16], 8, (add SP)>; -//:TODO: if we remove this we get an error in tablegen -//:TODO: this is just a hack, remove it once add16 works! // Status register. def SREG : AVRReg<14, "FLAGS">, DwarfRegNum<[88]>; def CCR : RegisterClass<"AVR", [i8], 8, (add SREG)> diff --git a/contrib/llvm/lib/Target/AVR/AVRTargetMachine.cpp b/contrib/llvm/lib/Target/AVR/AVRTargetMachine.cpp index 91d2a8737b87..a9d61ffc952c 100644 --- a/contrib/llvm/lib/Target/AVR/AVRTargetMachine.cpp +++ b/contrib/llvm/lib/Target/AVR/AVRTargetMachine.cpp @@ -66,6 +66,7 @@ class AVRPassConfig : public TargetPassConfig { bool addInstSelector() override; void addPreSched2() override; + void addPreEmitPass() override; void addPreRegAlloc() override; }; } // namespace @@ -115,4 +116,9 @@ void AVRPassConfig::addPreSched2() { addPass(createAVRExpandPseudoPass()); } +void AVRPassConfig::addPreEmitPass() { + // Must run branch selection immediately preceding the asm printer. + addPass(&BranchRelaxationPassID); +} + } // end of namespace llvm diff --git a/contrib/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/contrib/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp index cf52e552978f..5004736365c7 100644 --- a/contrib/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp +++ b/contrib/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp @@ -466,6 +466,7 @@ bool AVRAsmParser::parseOperand(OperandVector &Operands) { if (!tryParseRegisterOperand(Operands)) { return false; } + LLVM_FALLTHROUGH; case AsmToken::LParen: case AsmToken::Integer: case AsmToken::Dot: diff --git a/contrib/llvm/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp b/contrib/llvm/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp index 316b7836df0d..0f34b8e18ff9 100644 --- a/contrib/llvm/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp +++ b/contrib/llvm/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp @@ -106,7 +106,7 @@ void AVRInstPrinter::printOperand(const MCInst *MI, unsigned OpNo, if (Op.isReg()) { bool isPtrReg = (MOI.RegClass == AVR::PTRREGSRegClassID) || (MOI.RegClass == AVR::PTRDISPREGSRegClassID) || - (MOI.RegClass == AVR::ZREGSRegClassID); + (MOI.RegClass == AVR::ZREGRegClassID); if (isPtrReg) { O << getRegisterName(Op.getReg(), AVR::ptr); diff --git a/contrib/llvm/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.cpp b/contrib/llvm/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.cpp index 1e61eccf775f..6d126ed622aa 100644 --- a/contrib/llvm/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.cpp +++ b/contrib/llvm/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.cpp @@ -33,7 +33,7 @@ static unsigned getEFlagsForFeatureSet(const FeatureBitset &Features) { EFlags |= ELF::EF_AVR_ARCH_AVR51; else if (Features[AVR::ELFArchAVR6]) EFlags |= ELF::EF_AVR_ARCH_AVR6; - else if (Features[AVR::ELFArchAVRTiny]) + else if (Features[AVR::ELFArchTiny]) EFlags |= ELF::EF_AVR_ARCH_AVRTINY; else if (Features[AVR::ELFArchXMEGA1]) EFlags |= ELF::EF_AVR_ARCH_XMEGA1; diff --git a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp index 15e89fb2a261..9fc812cdef14 100644 --- a/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp +++ b/contrib/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp @@ -29,7 +29,7 @@ class BPFAsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override; @@ -65,7 +65,7 @@ bool BPFAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const { void BPFAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { if (Fixup.getKind() == FK_SecRel_4 || Fixup.getKind() == FK_SecRel_8) { assert(Value == 0); } else if (Fixup.getKind() == FK_Data_4 || Fixup.getKind() == FK_Data_8) { diff --git a/contrib/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/contrib/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp index c19e636d79ca..d901abbd1692 100644 --- a/contrib/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp +++ b/contrib/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp @@ -1413,6 +1413,7 @@ int HexagonAsmParser::processInstruction(MCInst &Inst, // Translate a "$Rx = CONST32(#imm)" to "$Rx = memw(gp+#LABEL) " case Hexagon::CONST32: is32bit = true; + LLVM_FALLTHROUGH; // Translate a "$Rx:y = CONST64(#imm)" to "$Rx:y = memd(gp+#LABEL) " case Hexagon::CONST64: // FIXME: need better way to detect AsmStreamer (upstream removed getKind()) diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp index 14c682c6df4b..b064778c4bbd 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp @@ -1947,8 +1947,10 @@ bool BitSimplification::genStoreImmediate(MachineInstr *MI) { switch (Opc) { case Hexagon::S2_storeri_io: Align++; + LLVM_FALLTHROUGH; case Hexagon::S2_storerh_io: Align++; + LLVM_FALLTHROUGH; case Hexagon::S2_storerb_io: break; default: diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp index 730026121d3b..3de531088240 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp @@ -937,6 +937,7 @@ bool HexagonEvaluator::evaluate(const MachineInstr &BI, case Hexagon::J2_jumpfnew: case Hexagon::J2_jumpfnewpt: Negated = true; + LLVM_FALLTHROUGH; case Hexagon::J2_jumpt: case Hexagon::J2_jumptpt: case Hexagon::J2_jumptnew: diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp index aa68f6cfdfc1..49ddd6961f8a 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp @@ -2244,6 +2244,7 @@ bool HexagonConstEvaluator::evaluate(const MachineInstr &BrI, case Hexagon::J2_jumpfnew: case Hexagon::J2_jumpfnewpt: Negated = true; + LLVM_FALLTHROUGH; case Hexagon::J2_jumpt: case Hexagon::J2_jumptnew: case Hexagon::J2_jumptnewpt: diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp index 97a53dcbaed7..c790579ccebc 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp @@ -979,18 +979,6 @@ bool HexagonFrameLowering::hasFP(const MachineFunction &MF) const { if (MFI.hasCalls() || HMFI.hasClobberLR()) return true; - // Frame pointer elimination is a possiblility at this point, but - // to know if FP is necessary we need to know if spill/restore - // functions will be used (they require FP to be valid). - // This means that hasFP shouldn't really be called before CSI is - // calculated, and some measures are taken to make sure of that - // (e.g. default implementations of virtual functions that call it - // are overridden apropriately). - assert(MFI.isCalleeSavedInfoValid() && "Need to know CSI"); - const std::vector &CSI = MFI.getCalleeSavedInfo(); - if (useSpillFunction(MF, CSI) || useRestoreFunction(MF, CSI)) - return true; - return false; } @@ -2437,6 +2425,8 @@ bool HexagonFrameLowering::shouldInlineCSR(const MachineFunction &MF, const CSIVect &CSI) const { if (MF.getInfo()->hasEHReturn()) return true; + if (!hasFP(MF)) + return true; if (!isOptSize(MF) && !isMinSize(MF)) if (MF.getTarget().getOptLevel() > CodeGenOpt::Default) return true; diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp index f14c733dcf51..3470480d607d 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonGenPredicate.cpp @@ -334,6 +334,7 @@ bool HexagonGenPredicate::isScalarPred(Register PredReg) { if (MRI->getRegClass(PR.R) != PredRC) return false; // If it is a copy between two predicate registers, fall through. + LLVM_FALLTHROUGH; } case Hexagon::C2_and: case Hexagon::C2_andn: diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp index e5f49ca77a91..0163b2e2bdc4 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp @@ -241,22 +241,31 @@ void HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, const SDLoc &dl) { case MVT::v32i16: case MVT::v16i32: case MVT::v8i64: - if (isAlignedMemNode(LD)) - Opcode = IsValidInc ? Hexagon::V6_vL32b_pi : Hexagon::V6_vL32b_ai; - else + if (isAlignedMemNode(LD)) { + if (LD->isNonTemporal()) + Opcode = IsValidInc ? Hexagon::V6_vL32b_nt_pi : Hexagon::V6_vL32b_nt_ai; + else + Opcode = IsValidInc ? Hexagon::V6_vL32b_pi : Hexagon::V6_vL32b_ai; + } else { Opcode = IsValidInc ? Hexagon::V6_vL32Ub_pi : Hexagon::V6_vL32Ub_ai; + } break; // 128B case MVT::v128i8: case MVT::v64i16: case MVT::v32i32: case MVT::v16i64: - if (isAlignedMemNode(LD)) - Opcode = IsValidInc ? Hexagon::V6_vL32b_pi_128B - : Hexagon::V6_vL32b_ai_128B; - else + if (isAlignedMemNode(LD)) { + if (LD->isNonTemporal()) + Opcode = IsValidInc ? Hexagon::V6_vL32b_nt_pi_128B + : Hexagon::V6_vL32b_nt_ai_128B; + else + Opcode = IsValidInc ? Hexagon::V6_vL32b_pi_128B + : Hexagon::V6_vL32b_ai_128B; + } else { Opcode = IsValidInc ? Hexagon::V6_vL32Ub_pi_128B : Hexagon::V6_vL32Ub_ai_128B; + } break; default: llvm_unreachable("Unexpected memory type in indexed load"); @@ -529,22 +538,31 @@ void HexagonDAGToDAGISel::SelectIndexedStore(StoreSDNode *ST, const SDLoc &dl) { case MVT::v32i16: case MVT::v16i32: case MVT::v8i64: - if (isAlignedMemNode(ST)) - Opcode = IsValidInc ? Hexagon::V6_vS32b_pi : Hexagon::V6_vS32b_ai; - else + if (isAlignedMemNode(ST)) { + if (ST->isNonTemporal()) + Opcode = IsValidInc ? Hexagon::V6_vS32b_nt_pi : Hexagon::V6_vS32b_nt_ai; + else + Opcode = IsValidInc ? Hexagon::V6_vS32b_pi : Hexagon::V6_vS32b_ai; + } else { Opcode = IsValidInc ? Hexagon::V6_vS32Ub_pi : Hexagon::V6_vS32Ub_ai; + } break; // 128B case MVT::v128i8: case MVT::v64i16: case MVT::v32i32: case MVT::v16i64: - if (isAlignedMemNode(ST)) - Opcode = IsValidInc ? Hexagon::V6_vS32b_pi_128B - : Hexagon::V6_vS32b_ai_128B; - else + if (isAlignedMemNode(ST)) { + if (ST->isNonTemporal()) + Opcode = IsValidInc ? Hexagon::V6_vS32b_nt_pi_128B + : Hexagon::V6_vS32b_nt_ai_128B; + else + Opcode = IsValidInc ? Hexagon::V6_vS32b_pi_128B + : Hexagon::V6_vS32b_ai_128B; + } else { Opcode = IsValidInc ? Hexagon::V6_vS32Ub_pi_128B : Hexagon::V6_vS32Ub_ai_128B; + } break; default: llvm_unreachable("Unexpected memory type in indexed store"); diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index 2daacf795555..67242764d453 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -716,6 +716,7 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, bool IsStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet(); MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo &MFI = MF.getFrameInfo(); auto PtrVT = getPointerTy(MF.getDataLayout()); // Check for varargs. @@ -832,7 +833,6 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (NeedsArgAlign && Subtarget.hasV60TOps()) { DEBUG(dbgs() << "Function needs byte stack align due to call args\n"); - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); // V6 vectors passed by value have 64 or 128 byte alignment depending // on whether we are 64 byte vector mode or 128 byte. bool UseHVXDbl = Subtarget.useHVXDblOps(); @@ -916,10 +916,15 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, Ops.push_back(Glue); if (IsTailCall) { - MF.getFrameInfo().setHasTailCall(); + MFI.setHasTailCall(); return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops); } + // Set this here because we need to know this for "hasFP" in frame lowering. + // The target-independent code calls getFrameRegister before setting it, and + // getFrameRegister uses hasFP to determine whether the function has FP. + MFI.setHasCalls(true); + unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL; Chain = DAG.getNode(OpCode, dl, NodeTys, Ops); Glue = Chain.getValue(1); @@ -1284,11 +1289,9 @@ HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { // Creates a SPLAT instruction for a constant value VAL. static SDValue createSplat(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue Val) { - if (VT.getSimpleVT() == MVT::v4i8) - return DAG.getNode(HexagonISD::VSPLATB, dl, VT, Val); - - if (VT.getSimpleVT() == MVT::v4i16) - return DAG.getNode(HexagonISD::VSPLATH, dl, VT, Val); + EVT T = VT.getVectorElementType(); + if (T == MVT::i8 || T == MVT::i16) + return DAG.getNode(HexagonISD::VSPLAT, dl, VT, Val); return SDValue(); } @@ -2296,32 +2299,13 @@ const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { case HexagonISD::JT: return "HexagonISD::JT"; case HexagonISD::PACKHL: return "HexagonISD::PACKHL"; case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG"; - case HexagonISD::SHUFFEB: return "HexagonISD::SHUFFEB"; - case HexagonISD::SHUFFEH: return "HexagonISD::SHUFFEH"; - case HexagonISD::SHUFFOB: return "HexagonISD::SHUFFOB"; - case HexagonISD::SHUFFOH: return "HexagonISD::SHUFFOH"; case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN"; - case HexagonISD::VCMPBEQ: return "HexagonISD::VCMPBEQ"; - case HexagonISD::VCMPBGT: return "HexagonISD::VCMPBGT"; - case HexagonISD::VCMPBGTU: return "HexagonISD::VCMPBGTU"; - case HexagonISD::VCMPHEQ: return "HexagonISD::VCMPHEQ"; - case HexagonISD::VCMPHGT: return "HexagonISD::VCMPHGT"; - case HexagonISD::VCMPHGTU: return "HexagonISD::VCMPHGTU"; - case HexagonISD::VCMPWEQ: return "HexagonISD::VCMPWEQ"; - case HexagonISD::VCMPWGT: return "HexagonISD::VCMPWGT"; - case HexagonISD::VCMPWGTU: return "HexagonISD::VCMPWGTU"; case HexagonISD::VCOMBINE: return "HexagonISD::VCOMBINE"; case HexagonISD::VPACK: return "HexagonISD::VPACK"; - case HexagonISD::VSHLH: return "HexagonISD::VSHLH"; - case HexagonISD::VSHLW: return "HexagonISD::VSHLW"; - case HexagonISD::VSPLATB: return "HexagonISD::VSPLTB"; - case HexagonISD::VSPLATH: return "HexagonISD::VSPLATH"; - case HexagonISD::VSRAH: return "HexagonISD::VSRAH"; - case HexagonISD::VSRAW: return "HexagonISD::VSRAW"; - case HexagonISD::VSRLH: return "HexagonISD::VSRLH"; - case HexagonISD::VSRLW: return "HexagonISD::VSRLW"; - case HexagonISD::VSXTBH: return "HexagonISD::VSXTBH"; - case HexagonISD::VSXTBW: return "HexagonISD::VSXTBW"; + case HexagonISD::VASL: return "HexagonISD::VASL"; + case HexagonISD::VASR: return "HexagonISD::VASR"; + case HexagonISD::VLSR: return "HexagonISD::VLSR"; + case HexagonISD::VSPLAT: return "HexagonISD::VSPLAT"; case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE"; case HexagonISD::OP_END: break; } @@ -2503,13 +2487,13 @@ HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { if (VT.getSimpleVT() == MVT::v4i16) { switch (Op.getOpcode()) { case ISD::SRA: - Result = DAG.getNode(HexagonISD::VSRAH, dl, VT, V3, CommonSplat); + Result = DAG.getNode(HexagonISD::VASR, dl, VT, V3, CommonSplat); break; case ISD::SHL: - Result = DAG.getNode(HexagonISD::VSHLH, dl, VT, V3, CommonSplat); + Result = DAG.getNode(HexagonISD::VASL, dl, VT, V3, CommonSplat); break; case ISD::SRL: - Result = DAG.getNode(HexagonISD::VSRLH, dl, VT, V3, CommonSplat); + Result = DAG.getNode(HexagonISD::VLSR, dl, VT, V3, CommonSplat); break; default: return SDValue(); @@ -2517,13 +2501,13 @@ HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { } else if (VT.getSimpleVT() == MVT::v2i32) { switch (Op.getOpcode()) { case ISD::SRA: - Result = DAG.getNode(HexagonISD::VSRAW, dl, VT, V3, CommonSplat); + Result = DAG.getNode(HexagonISD::VASR, dl, VT, V3, CommonSplat); break; case ISD::SHL: - Result = DAG.getNode(HexagonISD::VSHLW, dl, VT, V3, CommonSplat); + Result = DAG.getNode(HexagonISD::VASL, dl, VT, V3, CommonSplat); break; case ISD::SRL: - Result = DAG.getNode(HexagonISD::VSRLW, dl, VT, V3, CommonSplat); + Result = DAG.getNode(HexagonISD::VLSR, dl, VT, V3, CommonSplat); break; default: return SDValue(); diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.h index 1415156487c0..bfd2c94eeaba 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/contrib/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -52,29 +52,10 @@ namespace HexagonISD { COMBINE, PACKHL, - VSPLATB, - VSPLATH, - SHUFFEB, - SHUFFEH, - SHUFFOB, - SHUFFOH, - VSXTBH, - VSXTBW, - VSRAW, - VSRAH, - VSRLW, - VSRLH, - VSHLW, - VSHLH, - VCMPBEQ, - VCMPBGT, - VCMPBGTU, - VCMPHEQ, - VCMPHGT, - VCMPHGTU, - VCMPWEQ, - VCMPWGT, - VCMPWGTU, + VSPLAT, + VASL, + VASR, + VLSR, INSERT, INSERTRP, diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp index 1eac2d3dd8e2..c77c669f4ca7 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp @@ -250,15 +250,19 @@ unsigned HexagonInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, case Hexagon::L2_loadri_io: case Hexagon::L2_loadrd_io: case Hexagon::V6_vL32b_ai: + case Hexagon::V6_vL32b_nt_ai: case Hexagon::V6_vL32b_ai_128B: + case Hexagon::V6_vL32b_nt_ai_128B: case Hexagon::V6_vL32Ub_ai: case Hexagon::V6_vL32Ub_ai_128B: case Hexagon::LDriw_pred: case Hexagon::LDriw_mod: case Hexagon::PS_vloadrq_ai: case Hexagon::PS_vloadrw_ai: + case Hexagon::PS_vloadrw_nt_ai: case Hexagon::PS_vloadrq_ai_128B: - case Hexagon::PS_vloadrw_ai_128B: { + case Hexagon::PS_vloadrw_ai_128B: + case Hexagon::PS_vloadrw_nt_ai_128B: { const MachineOperand OpFI = MI.getOperand(1); if (!OpFI.isFI()) return 0; @@ -1726,6 +1730,39 @@ bool HexagonInstrInfo::getIncrementValue(const MachineInstr &MI, return false; } +std::pair +HexagonInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { + return std::make_pair(TF & ~HexagonII::MO_Bitmasks, + TF & HexagonII::MO_Bitmasks); +} + +ArrayRef> +HexagonInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { + using namespace HexagonII; + static const std::pair Flags[] = { + {MO_PCREL, "hexagon-pcrel"}, + {MO_GOT, "hexagon-got"}, + {MO_LO16, "hexagon-lo16"}, + {MO_HI16, "hexagon-hi16"}, + {MO_GPREL, "hexagon-gprel"}, + {MO_GDGOT, "hexagon-gdgot"}, + {MO_GDPLT, "hexagon-gdplt"}, + {MO_IE, "hexagon-ie"}, + {MO_IEGOT, "hexagon-iegot"}, + {MO_TPREL, "hexagon-tprel"} + }; + return makeArrayRef(Flags); +} + +ArrayRef> +HexagonInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { + using namespace HexagonII; + static const std::pair Flags[] = { + {HMOTF_ConstExtended, "hexagon-ext"} + }; + return makeArrayRef(Flags); +} + unsigned HexagonInstrInfo::createVR(MachineFunction *MF, MVT VT) const { MachineRegisterInfo &MRI = MF->getRegInfo(); const TargetRegisterClass *TRC; @@ -1797,7 +1834,7 @@ bool HexagonInstrInfo::isConstExtended(const MachineInstr &MI) const { const MachineOperand &MO = MI.getOperand(ExtOpNum); // Use MO operand flags to determine if MO // has the HMOTF_ConstExtended flag set. - if (MO.getTargetFlags() && HexagonII::HMOTF_ConstExtended) + if (MO.getTargetFlags() & HexagonII::HMOTF_ConstExtended) return true; // If this is a Machine BB address we are talking about, and it is // not marked as extended, say so. @@ -1807,9 +1844,6 @@ bool HexagonInstrInfo::isConstExtended(const MachineInstr &MI) const { // We could be using an instruction with an extendable immediate and shoehorn // a global address into it. If it is a global address it will be constant // extended. We do this for COMBINE. - // We currently only handle isGlobal() because it is the only kind of - // object we are going to end up with here for now. - // In the future we probably should add isSymbol(), etc. if (MO.isGlobal() || MO.isSymbol() || MO.isBlockAddress() || MO.isJTI() || MO.isCPI() || MO.isFPImm()) return true; @@ -1961,11 +1995,9 @@ bool HexagonInstrInfo::isExtended(const MachineInstr &MI) const { return true; // Use MO operand flags to determine if one of MI's operands // has HMOTF_ConstExtended flag set. - for (MachineInstr::const_mop_iterator I = MI.operands_begin(), - E = MI.operands_end(); I != E; ++I) { - if (I->getTargetFlags() && HexagonII::HMOTF_ConstExtended) + for (const MachineOperand &MO : MI.operands()) + if (MO.getTargetFlags() & HexagonII::HMOTF_ConstExtended) return true; - } return false; } @@ -2445,20 +2477,28 @@ bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset, switch (Opcode) { case Hexagon::PS_vstorerq_ai: case Hexagon::PS_vstorerw_ai: + case Hexagon::PS_vstorerw_nt_ai: case Hexagon::PS_vloadrq_ai: case Hexagon::PS_vloadrw_ai: + case Hexagon::PS_vloadrw_nt_ai: case Hexagon::V6_vL32b_ai: case Hexagon::V6_vS32b_ai: + case Hexagon::V6_vL32b_nt_ai: + case Hexagon::V6_vS32b_nt_ai: case Hexagon::V6_vL32Ub_ai: case Hexagon::V6_vS32Ub_ai: return isShiftedInt<4,6>(Offset); case Hexagon::PS_vstorerq_ai_128B: case Hexagon::PS_vstorerw_ai_128B: + case Hexagon::PS_vstorerw_nt_ai_128B: case Hexagon::PS_vloadrq_ai_128B: case Hexagon::PS_vloadrw_ai_128B: + case Hexagon::PS_vloadrw_nt_ai_128B: case Hexagon::V6_vL32b_ai_128B: case Hexagon::V6_vS32b_ai_128B: + case Hexagon::V6_vL32b_nt_ai_128B: + case Hexagon::V6_vS32b_nt_ai_128B: case Hexagon::V6_vL32Ub_ai_128B: case Hexagon::V6_vS32Ub_ai_128B: return isShiftedInt<4,7>(Offset); @@ -3170,11 +3210,19 @@ int HexagonInstrInfo::getDotCurOp(const MachineInstr &MI) const { return Hexagon::V6_vL32b_cur_pi; case Hexagon::V6_vL32b_ai: return Hexagon::V6_vL32b_cur_ai; + case Hexagon::V6_vL32b_nt_pi: + return Hexagon::V6_vL32b_nt_cur_pi; + case Hexagon::V6_vL32b_nt_ai: + return Hexagon::V6_vL32b_nt_cur_ai; //128B case Hexagon::V6_vL32b_pi_128B: return Hexagon::V6_vL32b_cur_pi_128B; case Hexagon::V6_vL32b_ai_128B: return Hexagon::V6_vL32b_cur_ai_128B; + case Hexagon::V6_vL32b_nt_pi_128B: + return Hexagon::V6_vL32b_nt_cur_pi_128B; + case Hexagon::V6_vL32b_nt_ai_128B: + return Hexagon::V6_vL32b_nt_cur_ai_128B; } return 0; } @@ -3187,11 +3235,19 @@ int HexagonInstrInfo::getNonDotCurOp(const MachineInstr &MI) const { return Hexagon::V6_vL32b_pi; case Hexagon::V6_vL32b_cur_ai: return Hexagon::V6_vL32b_ai; + case Hexagon::V6_vL32b_nt_cur_pi: + return Hexagon::V6_vL32b_nt_pi; + case Hexagon::V6_vL32b_nt_cur_ai: + return Hexagon::V6_vL32b_nt_ai; //128B case Hexagon::V6_vL32b_cur_pi_128B: return Hexagon::V6_vL32b_pi_128B; case Hexagon::V6_vL32b_cur_ai_128B: return Hexagon::V6_vL32b_ai_128B; + case Hexagon::V6_vL32b_nt_cur_pi_128B: + return Hexagon::V6_vL32b_nt_pi_128B; + case Hexagon::V6_vL32b_nt_cur_ai_128B: + return Hexagon::V6_vL32b_nt_ai_128B; } return 0; } diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.h index 944d0161a7c8..0436ce3ac475 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.h +++ b/contrib/llvm/lib/Target/Hexagon/HexagonInstrInfo.h @@ -301,6 +301,27 @@ class HexagonInstrInfo : public HexagonGenInstrInfo { const MachineInstr &UseMI, unsigned UseIdx) const override; + /// Decompose the machine operand's target flags into two values - the direct + /// target flag value and any of bit flags that are applied. + std::pair + decomposeMachineOperandsTargetFlags(unsigned TF) const override; + + /// Return an array that contains the direct target flag values and their + /// names. + /// + /// MIR Serialization is able to serialize only the target flags that are + /// defined by this method. + ArrayRef> + getSerializableDirectMachineOperandTargetFlags() const override; + + /// Return an array that contains the bitmask target flag values and their + /// names. + /// + /// MIR Serialization is able to serialize only the target flags that are + /// defined by this method. + ArrayRef> + getSerializableBitmaskMachineOperandTargetFlags() const override; + bool isTailCall(const MachineInstr &MI) const override; /// HexagonInstrInfo specifics. diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp index 4602de979024..1a26805d190d 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp @@ -49,7 +49,7 @@ static cl::opt CheckEarlyAvail("check-early-avail", cl::Hidden, using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" namespace { class HexagonCallMutation : public ScheduleDAGMutation { diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonPatterns.td b/contrib/llvm/lib/Target/Hexagon/HexagonPatterns.td index 689419638f54..ba98b8994937 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonPatterns.td +++ b/contrib/llvm/lib/Target/Hexagon/HexagonPatterns.td @@ -2770,6 +2770,9 @@ def unalignedstore : PatFrag<(ops node:$val, node:$addr), (store $val, $addr), [ multiclass vS32b_ai_pats { // Aligned stores + def : Pat<(alignednontemporalstore (VTSgl VectorRegs:$src1), IntRegs:$addr), + (V6_vS32b_nt_ai IntRegs:$addr, 0, (VTSgl VectorRegs:$src1))>, + Requires<[UseHVXSgl]>; def : Pat<(alignedstore (VTSgl VectorRegs:$src1), IntRegs:$addr), (V6_vS32b_ai IntRegs:$addr, 0, (VTSgl VectorRegs:$src1))>, Requires<[UseHVXSgl]>; @@ -2778,6 +2781,9 @@ multiclass vS32b_ai_pats { Requires<[UseHVXSgl]>; // 128B Aligned stores + def : Pat<(alignednontemporalstore (VTDbl VectorRegs128B:$src1), IntRegs:$addr), + (V6_vS32b_nt_ai_128B IntRegs:$addr, 0, (VTDbl VectorRegs128B:$src1))>, + Requires<[UseHVXDbl]>; def : Pat<(alignedstore (VTDbl VectorRegs128B:$src1), IntRegs:$addr), (V6_vS32b_ai_128B IntRegs:$addr, 0, (VTDbl VectorRegs128B:$src1))>, Requires<[UseHVXDbl]>; @@ -2787,6 +2793,11 @@ multiclass vS32b_ai_pats { // Fold Add R+OFF into vector store. let AddedComplexity = 10 in { + def : Pat<(alignednontemporalstore (VTSgl VectorRegs:$src1), + (add IntRegs:$src2, Iss4_6:$offset)), + (V6_vS32b_nt_ai IntRegs:$src2, Iss4_6:$offset, + (VTSgl VectorRegs:$src1))>, + Requires<[UseHVXSgl]>; def : Pat<(alignedstore (VTSgl VectorRegs:$src1), (add IntRegs:$src2, Iss4_6:$offset)), (V6_vS32b_ai IntRegs:$src2, Iss4_6:$offset, @@ -2799,6 +2810,11 @@ multiclass vS32b_ai_pats { Requires<[UseHVXSgl]>; // Fold Add R+OFF into vector store 128B. + def : Pat<(alignednontemporalstore (VTDbl VectorRegs128B:$src1), + (add IntRegs:$src2, Iss4_7:$offset)), + (V6_vS32b_nt_ai_128B IntRegs:$src2, Iss4_7:$offset, + (VTDbl VectorRegs128B:$src1))>, + Requires<[UseHVXDbl]>; def : Pat<(alignedstore (VTDbl VectorRegs128B:$src1), (add IntRegs:$src2, Iss4_7:$offset)), (V6_vS32b_ai_128B IntRegs:$src2, Iss4_7:$offset, @@ -2820,6 +2836,9 @@ defm : vS32b_ai_pats ; multiclass vL32b_ai_pats { // Aligned loads + def : Pat < (VTSgl (alignednontemporalload IntRegs:$addr)), + (V6_vL32b_nt_ai IntRegs:$addr, 0) >, + Requires<[UseHVXSgl]>; def : Pat < (VTSgl (alignedload IntRegs:$addr)), (V6_vL32b_ai IntRegs:$addr, 0) >, Requires<[UseHVXSgl]>; @@ -2828,6 +2847,9 @@ multiclass vL32b_ai_pats { Requires<[UseHVXSgl]>; // 128B Load + def : Pat < (VTDbl (alignednontemporalload IntRegs:$addr)), + (V6_vL32b_nt_ai_128B IntRegs:$addr, 0) >, + Requires<[UseHVXDbl]>; def : Pat < (VTDbl (alignedload IntRegs:$addr)), (V6_vL32b_ai_128B IntRegs:$addr, 0) >, Requires<[UseHVXDbl]>; @@ -2837,6 +2859,9 @@ multiclass vL32b_ai_pats { // Fold Add R+OFF into vector load. let AddedComplexity = 10 in { + def : Pat<(VTDbl (alignednontemporalload (add IntRegs:$src2, Iss4_7:$offset))), + (V6_vL32b_nt_ai_128B IntRegs:$src2, Iss4_7:$offset)>, + Requires<[UseHVXDbl]>; def : Pat<(VTDbl (alignedload (add IntRegs:$src2, Iss4_7:$offset))), (V6_vL32b_ai_128B IntRegs:$src2, Iss4_7:$offset)>, Requires<[UseHVXDbl]>; @@ -2844,6 +2869,9 @@ multiclass vL32b_ai_pats { (V6_vL32Ub_ai_128B IntRegs:$src2, Iss4_7:$offset)>, Requires<[UseHVXDbl]>; + def : Pat<(VTSgl (alignednontemporalload (add IntRegs:$src2, Iss4_6:$offset))), + (V6_vL32b_nt_ai IntRegs:$src2, Iss4_6:$offset)>, + Requires<[UseHVXSgl]>; def : Pat<(VTSgl (alignedload (add IntRegs:$src2, Iss4_6:$offset))), (V6_vL32b_ai IntRegs:$src2, Iss4_6:$offset)>, Requires<[UseHVXSgl]>; @@ -2859,6 +2887,9 @@ defm : vL32b_ai_pats ; defm : vL32b_ai_pats ; multiclass STrivv_pats { + def : Pat<(alignednontemporalstore (VTSgl VecDblRegs:$src1), IntRegs:$addr), + (PS_vstorerw_nt_ai IntRegs:$addr, 0, (VTSgl VecDblRegs:$src1))>, + Requires<[UseHVXSgl]>; def : Pat<(alignedstore (VTSgl VecDblRegs:$src1), IntRegs:$addr), (PS_vstorerw_ai IntRegs:$addr, 0, (VTSgl VecDblRegs:$src1))>, Requires<[UseHVXSgl]>; @@ -2866,6 +2897,10 @@ multiclass STrivv_pats { (PS_vstorerwu_ai IntRegs:$addr, 0, (VTSgl VecDblRegs:$src1))>, Requires<[UseHVXSgl]>; + def : Pat<(alignednontemporalstore (VTDbl VecDblRegs128B:$src1), IntRegs:$addr), + (PS_vstorerw_nt_ai_128B IntRegs:$addr, 0, + (VTDbl VecDblRegs128B:$src1))>, + Requires<[UseHVXDbl]>; def : Pat<(alignedstore (VTDbl VecDblRegs128B:$src1), IntRegs:$addr), (PS_vstorerw_ai_128B IntRegs:$addr, 0, (VTDbl VecDblRegs128B:$src1))>, @@ -2882,6 +2917,9 @@ defm : STrivv_pats ; defm : STrivv_pats ; multiclass LDrivv_pats { + def : Pat<(VTSgl (alignednontemporalload I32:$addr)), + (PS_vloadrw_nt_ai I32:$addr, 0)>, + Requires<[UseHVXSgl]>; def : Pat<(VTSgl (alignedload I32:$addr)), (PS_vloadrw_ai I32:$addr, 0)>, Requires<[UseHVXSgl]>; @@ -2889,6 +2927,9 @@ multiclass LDrivv_pats { (PS_vloadrwu_ai I32:$addr, 0)>, Requires<[UseHVXSgl]>; + def : Pat<(VTDbl (alignednontemporalload I32:$addr)), + (PS_vloadrw_nt_ai_128B I32:$addr, 0)>, + Requires<[UseHVXDbl]>; def : Pat<(VTDbl (alignedload I32:$addr)), (PS_vloadrw_ai_128B I32:$addr, 0)>, Requires<[UseHVXDbl]>; @@ -3021,16 +3062,16 @@ def : Pat<(v2i16 (add (v2i16 IntRegs:$src1), (v2i16 IntRegs:$src2))), def : Pat<(v2i16 (sub (v2i16 IntRegs:$src1), (v2i16 IntRegs:$src2))), (A2_svsubh IntRegs:$src1, IntRegs:$src2)>; -def HexagonVSPLATB: SDNode<"HexagonISD::VSPLATB", SDTUnaryOp>; -def HexagonVSPLATH: SDNode<"HexagonISD::VSPLATH", SDTUnaryOp>; +def SDTHexagonVSPLAT: SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; +def HexagonVSPLAT: SDNode<"HexagonISD::VSPLAT", SDTHexagonVSPLAT>; // Replicate the low 8-bits from 32-bits input register into each of the // four bytes of 32-bits destination register. -def: Pat<(v4i8 (HexagonVSPLATB I32:$Rs)), (S2_vsplatrb I32:$Rs)>; +def: Pat<(v4i8 (HexagonVSPLAT I32:$Rs)), (S2_vsplatrb I32:$Rs)>; // Replicate the low 16-bits from 32-bits input register into each of the // four halfwords of 64-bits destination register. -def: Pat<(v4i16 (HexagonVSPLATH I32:$Rs)), (S2_vsplatrh I32:$Rs)>; +def: Pat<(v4i16 (HexagonVSPLAT I32:$Rs)), (S2_vsplatrh I32:$Rs)>; class VArith_pat @@ -3068,84 +3109,44 @@ def: Pat<(v2i32 (shl V2I32:$b, (i64 (HexagonCOMBINE (i32 u5_0ImmPred:$c), (i32 u5_0ImmPred:$c))))), (S2_asl_i_vw V2I32:$b, imm:$c)>; -def: Pat<(v4i16 (sra V4I16:$b, (v4i16 (HexagonVSPLATH (i32 (u4_0ImmPred:$c)))))), +def: Pat<(v4i16 (sra V4I16:$b, (v4i16 (HexagonVSPLAT u4_0ImmPred:$c)))), (S2_asr_i_vh V4I16:$b, imm:$c)>; -def: Pat<(v4i16 (srl V4I16:$b, (v4i16 (HexagonVSPLATH (i32 (u4_0ImmPred:$c)))))), +def: Pat<(v4i16 (srl V4I16:$b, (v4i16 (HexagonVSPLAT u4_0ImmPred:$c)))), (S2_lsr_i_vh V4I16:$b, imm:$c)>; -def: Pat<(v4i16 (shl V4I16:$b, (v4i16 (HexagonVSPLATH (i32 (u4_0ImmPred:$c)))))), +def: Pat<(v4i16 (shl V4I16:$b, (v4i16 (HexagonVSPLAT u4_0ImmPred:$c)))), (S2_asl_i_vh V4I16:$b, imm:$c)>; -def SDTHexagon_v2i32_v2i32_i32 : SDTypeProfile<1, 2, - [SDTCisSameAs<0, 1>, SDTCisVT<0, v2i32>, SDTCisInt<2>]>; -def SDTHexagon_v4i16_v4i16_i32 : SDTypeProfile<1, 2, - [SDTCisSameAs<0, 1>, SDTCisVT<0, v4i16>, SDTCisInt<2>]>; +def SDTHexagonVShift + : SDTypeProfile<1, 2, [SDTCisSameAs<0, 1>, SDTCisVec<0>, SDTCisVT<2, i32>]>; -def HexagonVSRAW: SDNode<"HexagonISD::VSRAW", SDTHexagon_v2i32_v2i32_i32>; -def HexagonVSRAH: SDNode<"HexagonISD::VSRAH", SDTHexagon_v4i16_v4i16_i32>; -def HexagonVSRLW: SDNode<"HexagonISD::VSRLW", SDTHexagon_v2i32_v2i32_i32>; -def HexagonVSRLH: SDNode<"HexagonISD::VSRLH", SDTHexagon_v4i16_v4i16_i32>; -def HexagonVSHLW: SDNode<"HexagonISD::VSHLW", SDTHexagon_v2i32_v2i32_i32>; -def HexagonVSHLH: SDNode<"HexagonISD::VSHLH", SDTHexagon_v4i16_v4i16_i32>; +def HexagonVASL: SDNode<"HexagonISD::VASL", SDTHexagonVShift>; +def HexagonVASR: SDNode<"HexagonISD::VASR", SDTHexagonVShift>; +def HexagonVLSR: SDNode<"HexagonISD::VLSR", SDTHexagonVShift>; -def: Pat<(v2i32 (HexagonVSRAW V2I32:$Rs, u5_0ImmPred:$u5)), - (S2_asr_i_vw V2I32:$Rs, imm:$u5)>; -def: Pat<(v4i16 (HexagonVSRAH V4I16:$Rs, u4_0ImmPred:$u4)), - (S2_asr_i_vh V4I16:$Rs, imm:$u4)>; -def: Pat<(v2i32 (HexagonVSRLW V2I32:$Rs, u5_0ImmPred:$u5)), - (S2_lsr_i_vw V2I32:$Rs, imm:$u5)>; -def: Pat<(v4i16 (HexagonVSRLH V4I16:$Rs, u4_0ImmPred:$u4)), - (S2_lsr_i_vh V4I16:$Rs, imm:$u4)>; -def: Pat<(v2i32 (HexagonVSHLW V2I32:$Rs, u5_0ImmPred:$u5)), +def: Pat<(v2i32 (HexagonVASL V2I32:$Rs, u5_0ImmPred:$u5)), (S2_asl_i_vw V2I32:$Rs, imm:$u5)>; -def: Pat<(v4i16 (HexagonVSHLH V4I16:$Rs, u4_0ImmPred:$u4)), +def: Pat<(v4i16 (HexagonVASL V4I16:$Rs, u4_0ImmPred:$u4)), (S2_asl_i_vh V4I16:$Rs, imm:$u4)>; +def: Pat<(v2i32 (HexagonVASR V2I32:$Rs, u5_0ImmPred:$u5)), + (S2_asr_i_vw V2I32:$Rs, imm:$u5)>; +def: Pat<(v4i16 (HexagonVASR V4I16:$Rs, u4_0ImmPred:$u4)), + (S2_asr_i_vh V4I16:$Rs, imm:$u4)>; +def: Pat<(v2i32 (HexagonVLSR V2I32:$Rs, u5_0ImmPred:$u5)), + (S2_lsr_i_vw V2I32:$Rs, imm:$u5)>; +def: Pat<(v4i16 (HexagonVLSR V4I16:$Rs, u4_0ImmPred:$u4)), + (S2_lsr_i_vh V4I16:$Rs, imm:$u4)>; class vshift_rr_pat : Pat <(Op Value:$Rs, I32:$Rt), (MI Value:$Rs, I32:$Rt)>; -def: vshift_rr_pat ; -def: vshift_rr_pat ; -def: vshift_rr_pat ; -def: vshift_rr_pat ; -def: vshift_rr_pat ; -def: vshift_rr_pat ; - - -def SDTHexagonVecCompare_v8i8 : SDTypeProfile<1, 2, - [SDTCisSameAs<1, 2>, SDTCisVT<0, i1>, SDTCisVT<1, v8i8>]>; -def SDTHexagonVecCompare_v4i16 : SDTypeProfile<1, 2, - [SDTCisSameAs<1, 2>, SDTCisVT<0, i1>, SDTCisVT<1, v4i16>]>; -def SDTHexagonVecCompare_v2i32 : SDTypeProfile<1, 2, - [SDTCisSameAs<1, 2>, SDTCisVT<0, i1>, SDTCisVT<1, v2i32>]>; - -def HexagonVCMPBEQ: SDNode<"HexagonISD::VCMPBEQ", SDTHexagonVecCompare_v8i8>; -def HexagonVCMPBGT: SDNode<"HexagonISD::VCMPBGT", SDTHexagonVecCompare_v8i8>; -def HexagonVCMPBGTU: SDNode<"HexagonISD::VCMPBGTU", SDTHexagonVecCompare_v8i8>; -def HexagonVCMPHEQ: SDNode<"HexagonISD::VCMPHEQ", SDTHexagonVecCompare_v4i16>; -def HexagonVCMPHGT: SDNode<"HexagonISD::VCMPHGT", SDTHexagonVecCompare_v4i16>; -def HexagonVCMPHGTU: SDNode<"HexagonISD::VCMPHGTU", SDTHexagonVecCompare_v4i16>; -def HexagonVCMPWEQ: SDNode<"HexagonISD::VCMPWEQ", SDTHexagonVecCompare_v2i32>; -def HexagonVCMPWGT: SDNode<"HexagonISD::VCMPWGT", SDTHexagonVecCompare_v2i32>; -def HexagonVCMPWGTU: SDNode<"HexagonISD::VCMPWGTU", SDTHexagonVecCompare_v2i32>; - - -class vcmp_i1_pat - : Pat <(i1 (Op Value:$Rs, Value:$Rt)), - (MI Value:$Rs, Value:$Rt)>; - -def: vcmp_i1_pat; -def: vcmp_i1_pat; -def: vcmp_i1_pat; - -def: vcmp_i1_pat; -def: vcmp_i1_pat; -def: vcmp_i1_pat; - -def: vcmp_i1_pat; -def: vcmp_i1_pat; -def: vcmp_i1_pat; +def: vshift_rr_pat ; +def: vshift_rr_pat ; +def: vshift_rr_pat ; +def: vshift_rr_pat ; +def: vshift_rr_pat ; +def: vshift_rr_pat ; class vcmp_vi1_pat @@ -3255,13 +3256,6 @@ def: Pat<(v4i8 (trunc V4I16:$Rs)), def: Pat<(v2i16 (trunc V2I32:$Rs)), (LoReg (S2_packhl (HiReg $Rs), (LoReg $Rs)))>; - -def HexagonVSXTBH : SDNode<"HexagonISD::VSXTBH", SDTUnaryOp>; -def HexagonVSXTBW : SDNode<"HexagonISD::VSXTBW", SDTUnaryOp>; - -def: Pat<(i64 (HexagonVSXTBH I32:$Rs)), (S2_vsxtbh I32:$Rs)>; -def: Pat<(i64 (HexagonVSXTBW I32:$Rs)), (S2_vsxthw I32:$Rs)>; - def: Pat<(v4i16 (zext V4I8:$Rs)), (S2_vzxtbh V4I8:$Rs)>; def: Pat<(v2i32 (zext V2I16:$Rs)), (S2_vzxthw V2I16:$Rs)>; def: Pat<(v4i16 (anyext V4I8:$Rs)), (S2_vzxtbh V4I8:$Rs)>; @@ -3322,31 +3316,6 @@ def: Pat<(v8i8 (mul V8I8:$Rs, V8I8:$Rt)), (A2_combinew (S2_vtrunehb (VMPYB_no_V5 (HiReg $Rs), (HiReg $Rt))), (S2_vtrunehb (VMPYB_no_V5 (LoReg $Rs), (LoReg $Rt))))>; -def SDTHexagonBinOp64 : SDTypeProfile<1, 2, - [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisVT<0, i64>]>; - -def HexagonSHUFFEB: SDNode<"HexagonISD::SHUFFEB", SDTHexagonBinOp64>; -def HexagonSHUFFEH: SDNode<"HexagonISD::SHUFFEH", SDTHexagonBinOp64>; -def HexagonSHUFFOB: SDNode<"HexagonISD::SHUFFOB", SDTHexagonBinOp64>; -def HexagonSHUFFOH: SDNode<"HexagonISD::SHUFFOH", SDTHexagonBinOp64>; - -class ShufflePat - : Pat<(i64 (Op DoubleRegs:$src1, DoubleRegs:$src2)), - (i64 (MI DoubleRegs:$src1, DoubleRegs:$src2))>; - -// Shuffles even bytes for i=0..3: A[2*i].b = C[2*i].b; A[2*i+1].b = B[2*i].b -def: ShufflePat; - -// Shuffles odd bytes for i=0..3: A[2*i].b = C[2*i+1].b; A[2*i+1].b = B[2*i+1].b -def: ShufflePat; - -// Shuffles even half for i=0,1: A[2*i].h = C[2*i].h; A[2*i+1].h = B[2*i].h -def: ShufflePat; - -// Shuffles odd half for i=0,1: A[2*i].h = C[2*i+1].h; A[2*i+1].h = B[2*i+1].h -def: ShufflePat; - - // Truncated store from v4i16 to v4i8. def truncstorev4i8: PatFrag<(ops node:$val, node:$ptr), (truncstore node:$val, node:$ptr), diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonPseudo.td b/contrib/llvm/lib/Target/Hexagon/HexagonPseudo.td index 93fb688fc1c0..b42c1ab975a8 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonPseudo.td +++ b/contrib/llvm/lib/Target/Hexagon/HexagonPseudo.td @@ -407,6 +407,11 @@ def PS_vstorerw_ai: STrivv_template, def PS_vstorerw_ai_128B: STrivv_template, Requires<[HasV60T,UseHVXDbl]>; +def PS_vstorerw_nt_ai: STrivv_template, + Requires<[HasV60T,UseHVXSgl]>; +def PS_vstorerw_nt_ai_128B: STrivv_template, + Requires<[HasV60T,UseHVXDbl]>; + def PS_vstorerwu_ai: STrivv_template, Requires<[HasV60T,UseHVXSgl]>; def PS_vstorerwu_ai_128B: STrivv_template, @@ -433,6 +438,11 @@ def PS_vloadrw_ai: LDrivv_template, def PS_vloadrw_ai_128B: LDrivv_template, Requires<[HasV60T,UseHVXDbl]>; +def PS_vloadrw_nt_ai: LDrivv_template, + Requires<[HasV60T,UseHVXSgl]>; +def PS_vloadrw_nt_ai_128B: LDrivv_template, + Requires<[HasV60T,UseHVXDbl]>; + def PS_vloadrwu_ai: LDrivv_template, Requires<[HasV60T,UseHVXSgl]>; def PS_vloadrwu_ai_128B: LDrivv_template, diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp index db268b78cd73..4fa929a20810 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp @@ -350,6 +350,8 @@ int32_t HexagonSplitDoubleRegs::profit(const MachineInstr *MI) const { MI->getOperand(2).getImm()); case Hexagon::A4_combineri: ImmX++; + // Fall through into A4_combineir. + LLVM_FALLTHROUGH; case Hexagon::A4_combineir: { ImmX++; int64_t V = MI->getOperand(ImmX).getImm(); diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 76d9b31b005f..7d88b51f32dd 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -110,10 +110,11 @@ SchedCustomRegistry("hexagon", "Run Hexagon's custom scheduler", namespace llvm { extern char &HexagonExpandCondsetsID; void initializeHexagonExpandCondsetsPass(PassRegistry&); - void initializeHexagonLoopIdiomRecognizePass(PassRegistry&); void initializeHexagonGenMuxPass(PassRegistry&); - void initializeHexagonOptAddrModePass(PassRegistry&); + void initializeHexagonLoopIdiomRecognizePass(PassRegistry&); void initializeHexagonNewValueJumpPass(PassRegistry&); + void initializeHexagonOptAddrModePass(PassRegistry&); + void initializeHexagonPacketizerPass(PassRegistry&); Pass *createHexagonLoopIdiomPass(); FunctionPass *createHexagonBitSimplify(); @@ -156,10 +157,11 @@ extern "C" void LLVMInitializeHexagonTarget() { RegisterTargetMachine X(getTheHexagonTarget()); PassRegistry &PR = *PassRegistry::getPassRegistry(); - initializeHexagonLoopIdiomRecognizePass(PR); initializeHexagonGenMuxPass(PR); - initializeHexagonOptAddrModePass(PR); + initializeHexagonLoopIdiomRecognizePass(PR); initializeHexagonNewValueJumpPass(PR); + initializeHexagonOptAddrModePass(PR); + initializeHexagonPacketizerPass(PR); } HexagonTargetMachine::HexagonTargetMachine(const Target &T, const Triple &TT, diff --git a/contrib/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/contrib/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp index 7667bfb7a0eb..a3021e3dfe43 100644 --- a/contrib/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp +++ b/contrib/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp @@ -60,9 +60,7 @@ namespace { class HexagonPacketizer : public MachineFunctionPass { public: static char ID; - HexagonPacketizer() : MachineFunctionPass(ID) { - initializeHexagonPacketizerPass(*PassRegistry::getPassRegistry()); - } + HexagonPacketizer() : MachineFunctionPass(ID) {} void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); @@ -89,14 +87,14 @@ namespace { char HexagonPacketizer::ID = 0; } -INITIALIZE_PASS_BEGIN(HexagonPacketizer, "packets", "Hexagon Packetizer", - false, false) +INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer", + "Hexagon Packetizer", false, false) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) -INITIALIZE_PASS_END(HexagonPacketizer, "packets", "Hexagon Packetizer", - false, false) +INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer", + "Hexagon Packetizer", false, false) HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF, MachineLoopInfo &MLI, AliasAnalysis *AA, diff --git a/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp index 34d0b55aa22a..2a0edda8dcee 100644 --- a/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp +++ b/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp @@ -412,7 +412,7 @@ class HexagonAsmBackend : public MCAsmBackend { /// fixup kind as appropriate. void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t FixupValue, bool IsPCRel) const override { + uint64_t FixupValue, bool IsResolved) const override { // When FixupValue is 0 the relocation is external and there // is nothing for us to do. @@ -442,6 +442,7 @@ class HexagonAsmBackend : public MCAsmBackend { case fixup_Hexagon_B7_PCREL: if (!(isIntN(7, sValue))) HandleFixupError(7, 2, (int64_t)FixupValue, "B7_PCREL"); + LLVM_FALLTHROUGH; case fixup_Hexagon_B7_PCREL_X: InstMask = 0x00001f18; // Word32_B7 Reloc = (((Value >> 2) & 0x1f) << 8) | // Value 6-2 = Target 12-8 @@ -451,6 +452,7 @@ class HexagonAsmBackend : public MCAsmBackend { case fixup_Hexagon_B9_PCREL: if (!(isIntN(9, sValue))) HandleFixupError(9, 2, (int64_t)FixupValue, "B9_PCREL"); + LLVM_FALLTHROUGH; case fixup_Hexagon_B9_PCREL_X: InstMask = 0x003000fe; // Word32_B9 Reloc = (((Value >> 7) & 0x3) << 20) | // Value 8-7 = Target 21-20 @@ -462,6 +464,7 @@ class HexagonAsmBackend : public MCAsmBackend { case fixup_Hexagon_B13_PCREL: if (!(isIntN(13, sValue))) HandleFixupError(13, 2, (int64_t)FixupValue, "B13_PCREL"); + LLVM_FALLTHROUGH; case fixup_Hexagon_B13_PCREL_X: InstMask = 0x00202ffe; // Word32_B13 Reloc = (((Value >> 12) & 0x1) << 21) | // Value 12 = Target 21 @@ -472,6 +475,7 @@ class HexagonAsmBackend : public MCAsmBackend { case fixup_Hexagon_B15_PCREL: if (!(isIntN(15, sValue))) HandleFixupError(15, 2, (int64_t)FixupValue, "B15_PCREL"); + LLVM_FALLTHROUGH; case fixup_Hexagon_B15_PCREL_X: InstMask = 0x00df20fe; // Word32_B15 Reloc = (((Value >> 13) & 0x3) << 22) | // Value 14-13 = Target 23-22 @@ -483,6 +487,7 @@ class HexagonAsmBackend : public MCAsmBackend { case fixup_Hexagon_B22_PCREL: if (!(isIntN(22, sValue))) HandleFixupError(22, 2, (int64_t)FixupValue, "B22_PCREL"); + LLVM_FALLTHROUGH; case fixup_Hexagon_B22_PCREL_X: InstMask = 0x01ff3ffe; // Word32_B22 Reloc = (((Value >> 13) & 0x1ff) << 16) | // Value 21-13 = Target 24-16 diff --git a/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h b/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h index d8009c5da08e..7f90e83fc8e9 100644 --- a/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h +++ b/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h @@ -169,8 +169,11 @@ namespace HexagonII { // Hexagon specific MO operand flag mask. enum HexagonMOTargetFlagVal { - //===------------------------------------------------------------------===// - // Hexagon Specific MachineOperand flags. + // Hexagon-specific MachineOperand target flags. + // + // When chaning these, make sure to update + // getSerializableDirectMachineOperandTargetFlags and + // getSerializableBitmaskMachineOperandTargetFlags if needed. MO_NO_FLAG, /// MO_PCREL - On a symbol operand, indicates a PC-relative relocation @@ -207,10 +210,12 @@ namespace HexagonII { MO_TPREL, // HMOTF_ConstExtended - // Addendum to abovem, indicates a const extended op + // Addendum to above, indicates a const extended op // Can be used as a mask. - HMOTF_ConstExtended = 0x80 + HMOTF_ConstExtended = 0x80, + // Union of all bitmasks (currently only HMOTF_ConstExtended). + MO_Bitmasks = HMOTF_ConstExtended }; // Hexagon Sub-instruction classes. diff --git a/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp b/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp index 564d43b45cb8..1604e7c8dc54 100644 --- a/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp +++ b/contrib/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp @@ -259,6 +259,7 @@ bool HexagonShuffler::check() { break; case HexagonII::TypeCVI_VM_VP_LDU: ++onlyNo1; + LLVM_FALLTHROUGH; case HexagonII::TypeCVI_VM_LD: case HexagonII::TypeCVI_VM_TMP_LD: case HexagonII::TypeLD: @@ -274,6 +275,7 @@ bool HexagonShuffler::check() { break; case HexagonII::TypeCVI_VM_STU: ++onlyNo1; + LLVM_FALLTHROUGH; case HexagonII::TypeCVI_VM_ST: case HexagonII::TypeCVI_VM_NEW_ST: case HexagonII::TypeST: diff --git a/contrib/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/contrib/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp index 72e471f5766e..1394ac7210f2 100644 --- a/contrib/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp +++ b/contrib/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp @@ -787,6 +787,7 @@ std::unique_ptr LanaiAsmParser::parseImmediate() { case AsmToken::Dot: if (!Parser.parseExpression(ExprVal)) return LanaiOperand::createImm(ExprVal, Start, End); + LLVM_FALLTHROUGH; default: return nullptr; } diff --git a/contrib/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/contrib/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp index c212726113ab..bbce5f670c99 100644 --- a/contrib/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp +++ b/contrib/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp @@ -51,7 +51,7 @@ class LanaiAsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override; @@ -92,7 +92,7 @@ bool LanaiAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const { void LanaiAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool /*IsPCRel*/) const { + bool /*IsResolved*/) const { MCFixupKind Kind = Fixup.getKind(); Value = adjustFixupValue(static_cast(Kind), Value); diff --git a/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 69b1ba1528d0..b72c9d534478 100644 --- a/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/contrib/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -304,6 +304,9 @@ class MipsAsmParser : public MCTargetAsmParser { bool expandSeqI(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, const MCSubtargetInfo *STI); + bool expandMXTRAlias(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, + const MCSubtargetInfo *STI); + bool reportParseError(Twine ErrorMsg); bool reportParseError(SMLoc Loc, Twine ErrorMsg); @@ -343,6 +346,8 @@ class MipsAsmParser : public MCTargetAsmParser { bool parseSetPushDirective(); bool parseSetSoftFloatDirective(); bool parseSetHardFloatDirective(); + bool parseSetMtDirective(); + bool parseSetNoMtDirective(); bool parseSetAssignment(); @@ -628,6 +633,9 @@ class MipsAsmParser : public MCTargetAsmParser { bool useSoftFloat() const { return getSTI().getFeatureBits()[Mips::FeatureSoftFloat]; } + bool hasMT() const { + return getSTI().getFeatureBits()[Mips::FeatureMT]; + } /// Warn if RegIndex is the same as the current AT. void warnIfRegIndexIsAT(unsigned RegIndex, SMLoc Loc); @@ -1966,6 +1974,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, case Mips::SDIV_MM: FirstOp = 0; SecondOp = 1; + LLVM_FALLTHROUGH; case Mips::SDivMacro: case Mips::DSDivMacro: case Mips::UDivMacro: @@ -2505,6 +2514,16 @@ MipsAsmParser::tryExpandInstruction(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, return expandSeq(Inst, IDLoc, Out, STI) ? MER_Fail : MER_Success; case Mips::SEQIMacro: return expandSeqI(Inst, IDLoc, Out, STI) ? MER_Fail : MER_Success; + case Mips::MFTC0: case Mips::MTTC0: + case Mips::MFTGPR: case Mips::MTTGPR: + case Mips::MFTLO: case Mips::MTTLO: + case Mips::MFTHI: case Mips::MTTHI: + case Mips::MFTACX: case Mips::MTTACX: + case Mips::MFTDSP: case Mips::MTTDSP: + case Mips::MFTC1: case Mips::MTTC1: + case Mips::MFTHC1: case Mips::MTTHC1: + case Mips::CFTC1: case Mips::CTTC1: + return expandMXTRAlias(Inst, IDLoc, Out, STI) ? MER_Fail : MER_Success; } } @@ -4876,6 +4895,212 @@ bool MipsAsmParser::expandSeqI(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, return false; } +// Map the DSP accumulator and control register to the corresponding gpr +// operand. Unlike the other alias, the m(f|t)t(lo|hi|acx) instructions +// do not map the DSP registers contigously to gpr registers. +static unsigned getRegisterForMxtrDSP(MCInst &Inst, bool IsMFDSP) { + switch (Inst.getOpcode()) { + case Mips::MFTLO: + case Mips::MTTLO: + switch (Inst.getOperand(IsMFDSP ? 1 : 0).getReg()) { + case Mips::AC0: + return Mips::ZERO; + case Mips::AC1: + return Mips::A0; + case Mips::AC2: + return Mips::T0; + case Mips::AC3: + return Mips::T4; + default: + llvm_unreachable("Unknown register for 'mttr' alias!"); + } + case Mips::MFTHI: + case Mips::MTTHI: + switch (Inst.getOperand(IsMFDSP ? 1 : 0).getReg()) { + case Mips::AC0: + return Mips::AT; + case Mips::AC1: + return Mips::A1; + case Mips::AC2: + return Mips::T1; + case Mips::AC3: + return Mips::T5; + default: + llvm_unreachable("Unknown register for 'mttr' alias!"); + } + case Mips::MFTACX: + case Mips::MTTACX: + switch (Inst.getOperand(IsMFDSP ? 1 : 0).getReg()) { + case Mips::AC0: + return Mips::V0; + case Mips::AC1: + return Mips::A2; + case Mips::AC2: + return Mips::T2; + case Mips::AC3: + return Mips::T6; + default: + llvm_unreachable("Unknown register for 'mttr' alias!"); + } + case Mips::MFTDSP: + case Mips::MTTDSP: + return Mips::S0; + default: + llvm_unreachable("Unknown instruction for 'mttr' dsp alias!"); + } +} + +// Map the floating point register operand to the corresponding register +// operand. +static unsigned getRegisterForMxtrFP(MCInst &Inst, bool IsMFTC1) { + switch (Inst.getOperand(IsMFTC1 ? 1 : 0).getReg()) { + case Mips::F0: return Mips::ZERO; + case Mips::F1: return Mips::AT; + case Mips::F2: return Mips::V0; + case Mips::F3: return Mips::V1; + case Mips::F4: return Mips::A0; + case Mips::F5: return Mips::A1; + case Mips::F6: return Mips::A2; + case Mips::F7: return Mips::A3; + case Mips::F8: return Mips::T0; + case Mips::F9: return Mips::T1; + case Mips::F10: return Mips::T2; + case Mips::F11: return Mips::T3; + case Mips::F12: return Mips::T4; + case Mips::F13: return Mips::T5; + case Mips::F14: return Mips::T6; + case Mips::F15: return Mips::T7; + case Mips::F16: return Mips::S0; + case Mips::F17: return Mips::S1; + case Mips::F18: return Mips::S2; + case Mips::F19: return Mips::S3; + case Mips::F20: return Mips::S4; + case Mips::F21: return Mips::S5; + case Mips::F22: return Mips::S6; + case Mips::F23: return Mips::S7; + case Mips::F24: return Mips::T8; + case Mips::F25: return Mips::T9; + case Mips::F26: return Mips::K0; + case Mips::F27: return Mips::K1; + case Mips::F28: return Mips::GP; + case Mips::F29: return Mips::SP; + case Mips::F30: return Mips::FP; + case Mips::F31: return Mips::RA; + default: llvm_unreachable("Unknown register for mttc1 alias!"); + } +} + +// Map the coprocessor operand the corresponding gpr register operand. +static unsigned getRegisterForMxtrC0(MCInst &Inst, bool IsMFTC0) { + switch (Inst.getOperand(IsMFTC0 ? 1 : 0).getReg()) { + case Mips::COP00: return Mips::ZERO; + case Mips::COP01: return Mips::AT; + case Mips::COP02: return Mips::V0; + case Mips::COP03: return Mips::V1; + case Mips::COP04: return Mips::A0; + case Mips::COP05: return Mips::A1; + case Mips::COP06: return Mips::A2; + case Mips::COP07: return Mips::A3; + case Mips::COP08: return Mips::T0; + case Mips::COP09: return Mips::T1; + case Mips::COP010: return Mips::T2; + case Mips::COP011: return Mips::T3; + case Mips::COP012: return Mips::T4; + case Mips::COP013: return Mips::T5; + case Mips::COP014: return Mips::T6; + case Mips::COP015: return Mips::T7; + case Mips::COP016: return Mips::S0; + case Mips::COP017: return Mips::S1; + case Mips::COP018: return Mips::S2; + case Mips::COP019: return Mips::S3; + case Mips::COP020: return Mips::S4; + case Mips::COP021: return Mips::S5; + case Mips::COP022: return Mips::S6; + case Mips::COP023: return Mips::S7; + case Mips::COP024: return Mips::T8; + case Mips::COP025: return Mips::T9; + case Mips::COP026: return Mips::K0; + case Mips::COP027: return Mips::K1; + case Mips::COP028: return Mips::GP; + case Mips::COP029: return Mips::SP; + case Mips::COP030: return Mips::FP; + case Mips::COP031: return Mips::RA; + default: llvm_unreachable("Unknown register for mttc0 alias!"); + } +} + +/// Expand an alias of 'mftr' or 'mttr' into the full instruction, by producing +/// an mftr or mttr with the correctly mapped gpr register, u, sel and h bits. +bool MipsAsmParser::expandMXTRAlias(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, + const MCSubtargetInfo *STI) { + MipsTargetStreamer &TOut = getTargetStreamer(); + unsigned rd = 0; + unsigned u = 1; + unsigned sel = 0; + unsigned h = 0; + bool IsMFTR = false; + switch (Inst.getOpcode()) { + case Mips::MFTC0: + IsMFTR = true; + LLVM_FALLTHROUGH; + case Mips::MTTC0: + u = 0; + rd = getRegisterForMxtrC0(Inst, IsMFTR); + sel = Inst.getOperand(2).getImm(); + break; + case Mips::MFTGPR: + IsMFTR = true; + LLVM_FALLTHROUGH; + case Mips::MTTGPR: + rd = Inst.getOperand(IsMFTR ? 1 : 0).getReg(); + break; + case Mips::MFTLO: + case Mips::MFTHI: + case Mips::MFTACX: + case Mips::MFTDSP: + IsMFTR = true; + LLVM_FALLTHROUGH; + case Mips::MTTLO: + case Mips::MTTHI: + case Mips::MTTACX: + case Mips::MTTDSP: + rd = getRegisterForMxtrDSP(Inst, IsMFTR); + sel = 1; + break; + case Mips::MFTHC1: + h = 1; + LLVM_FALLTHROUGH; + case Mips::MFTC1: + IsMFTR = true; + rd = getRegisterForMxtrFP(Inst, IsMFTR); + sel = 2; + break; + case Mips::MTTHC1: + h = 1; + LLVM_FALLTHROUGH; + case Mips::MTTC1: + rd = getRegisterForMxtrFP(Inst, IsMFTR); + sel = 2; + break; + case Mips::CFTC1: + IsMFTR = true; + LLVM_FALLTHROUGH; + case Mips::CTTC1: + rd = getRegisterForMxtrFP(Inst, IsMFTR); + sel = 3; + break; + } + unsigned Op0 = IsMFTR ? Inst.getOperand(0).getReg() : rd; + unsigned Op1 = + IsMFTR ? rd + : (Inst.getOpcode() != Mips::MTTDSP ? Inst.getOperand(1).getReg() + : Inst.getOperand(0).getReg()); + + TOut.emitRRIII(IsMFTR ? Mips::MFTR : Mips::MTTR, Op0, Op1, u, sel, h, IDLoc, + STI); + return false; +} + unsigned MipsAsmParser::checkEarlyTargetMatchPredicate(MCInst &Inst, const OperandVector &Operands) { @@ -6329,6 +6554,39 @@ bool MipsAsmParser::parseSetNoOddSPRegDirective() { return false; } +bool MipsAsmParser::parseSetMtDirective() { + MCAsmParser &Parser = getParser(); + Parser.Lex(); // Eat "mt". + + // If this is not the end of the statement, report an error. + if (getLexer().isNot(AsmToken::EndOfStatement)) { + reportParseError("unexpected token, expected end of statement"); + return false; + } + + setFeatureBits(Mips::FeatureMT, "mt"); + getTargetStreamer().emitDirectiveSetMt(); + Parser.Lex(); // Consume the EndOfStatement. + return false; +} + +bool MipsAsmParser::parseSetNoMtDirective() { + MCAsmParser &Parser = getParser(); + Parser.Lex(); // Eat "nomt". + + // If this is not the end of the statement, report an error. + if (getLexer().isNot(AsmToken::EndOfStatement)) { + reportParseError("unexpected token, expected end of statement"); + return false; + } + + clearFeatureBits(Mips::FeatureMT, "mt"); + + getTargetStreamer().emitDirectiveSetNoMt(); + Parser.Lex(); // Consume the EndOfStatement. + return false; +} + bool MipsAsmParser::parseSetPopDirective() { MCAsmParser &Parser = getParser(); SMLoc Loc = getLexer().getLoc(); @@ -6829,6 +7087,10 @@ bool MipsAsmParser::parseDirectiveSet() { return parseSetMsaDirective(); } else if (Tok.getString() == "nomsa") { return parseSetNoMsaDirective(); + } else if (Tok.getString() == "mt") { + return parseSetMtDirective(); + } else if (Tok.getString() == "nomt") { + return parseSetNoMtDirective(); } else if (Tok.getString() == "softfloat") { return parseSetSoftFloatDirective(); } else if (Tok.getString() == "hardfloat") { @@ -7078,6 +7340,7 @@ bool MipsAsmParser::parseSSectionDirective(StringRef Section, unsigned Type) { /// ::= .module fp=value /// ::= .module softfloat /// ::= .module hardfloat +/// ::= .module mt bool MipsAsmParser::parseDirectiveModule() { MCAsmParser &Parser = getParser(); MCAsmLexer &Lexer = getLexer(); @@ -7176,6 +7439,25 @@ bool MipsAsmParser::parseDirectiveModule() { return false; } + return false; // parseDirectiveModule has finished successfully. + } else if (Option == "mt") { + setModuleFeatureBits(Mips::FeatureMT, "mt"); + + // Synchronize the ABI Flags information with the FeatureBits information we + // updated above. + getTargetStreamer().updateABIInfo(*this); + + // If printing assembly, use the recently updated ABI Flags information. + // If generating ELF, don't do anything (the .MIPS.abiflags section gets + // emitted later). + getTargetStreamer().emitDirectiveModuleMT(); + + // If this is not the end of the statement, report an error. + if (getLexer().isNot(AsmToken::EndOfStatement)) { + reportParseError("unexpected token, expected end of statement"); + return false; + } + return false; // parseDirectiveModule has finished successfully. } else { return Error(L, "'" + Twine(Option) + "' is not a valid .module option."); diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h index f38541027023..9abd4f1d6b08 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h @@ -159,6 +159,8 @@ struct MipsABIFlagsSection { ASESet |= Mips::AFL_ASE_MICROMIPS; if (P.inMips16Mode()) ASESet |= Mips::AFL_ASE_MIPS16; + if (P.hasMT()) + ASESet |= Mips::AFL_ASE_MT; } template diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp index ae48d6e38fa0..a1ed0ea4d7f3 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp @@ -238,7 +238,7 @@ static unsigned calculateMMLEIndex(unsigned i) { void MipsAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { MCFixupKind Kind = Fixup.getKind(); MCContext &Ctx = Asm.getContext(); Value = adjustFixupValue(Fixup, Value, Ctx); diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h index bf3b290b7ed5..8ebde3b9b7a4 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h @@ -40,7 +40,7 @@ class MipsAsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; Optional getFixupKind(StringRef Name) const override; const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; diff --git a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp index 0cd4aebe4d16..7caeb08589af 100644 --- a/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp +++ b/contrib/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp @@ -50,6 +50,8 @@ void MipsTargetStreamer::emitDirectiveSetMacro() { forbidModuleDirective(); } void MipsTargetStreamer::emitDirectiveSetNoMacro() { forbidModuleDirective(); } void MipsTargetStreamer::emitDirectiveSetMsa() { forbidModuleDirective(); } void MipsTargetStreamer::emitDirectiveSetNoMsa() { forbidModuleDirective(); } +void MipsTargetStreamer::emitDirectiveSetMt() {} +void MipsTargetStreamer::emitDirectiveSetNoMt() { forbidModuleDirective(); } void MipsTargetStreamer::emitDirectiveSetAt() { forbidModuleDirective(); } void MipsTargetStreamer::emitDirectiveSetAtWithArg(unsigned RegNo) { forbidModuleDirective(); @@ -118,6 +120,7 @@ void MipsTargetStreamer::emitDirectiveModuleOddSPReg() { } void MipsTargetStreamer::emitDirectiveModuleSoftFloat() {} void MipsTargetStreamer::emitDirectiveModuleHardFloat() {} +void MipsTargetStreamer::emitDirectiveModuleMT() {} void MipsTargetStreamer::emitDirectiveSetFp( MipsABIFlagsSection::FpABIKind Value) { forbidModuleDirective(); @@ -190,6 +193,21 @@ void MipsTargetStreamer::emitRRI(unsigned Opcode, unsigned Reg0, unsigned Reg1, emitRRX(Opcode, Reg0, Reg1, MCOperand::createImm(Imm), IDLoc, STI); } +void MipsTargetStreamer::emitRRIII(unsigned Opcode, unsigned Reg0, + unsigned Reg1, int16_t Imm0, int16_t Imm1, + int16_t Imm2, SMLoc IDLoc, + const MCSubtargetInfo *STI) { + MCInst TmpInst; + TmpInst.setOpcode(Opcode); + TmpInst.addOperand(MCOperand::createReg(Reg0)); + TmpInst.addOperand(MCOperand::createReg(Reg1)); + TmpInst.addOperand(MCOperand::createImm(Imm0)); + TmpInst.addOperand(MCOperand::createImm(Imm1)); + TmpInst.addOperand(MCOperand::createImm(Imm2)); + TmpInst.setLoc(IDLoc); + getStreamer().EmitInstruction(TmpInst, *STI); +} + void MipsTargetStreamer::emitAddu(unsigned DstReg, unsigned SrcReg, unsigned TrgReg, bool Is64Bit, const MCSubtargetInfo *STI) { @@ -392,6 +410,16 @@ void MipsTargetAsmStreamer::emitDirectiveSetNoMsa() { MipsTargetStreamer::emitDirectiveSetNoMsa(); } +void MipsTargetAsmStreamer::emitDirectiveSetMt() { + OS << "\t.set\tmt\n"; + MipsTargetStreamer::emitDirectiveSetMt(); +} + +void MipsTargetAsmStreamer::emitDirectiveSetNoMt() { + OS << "\t.set\tnomt\n"; + MipsTargetStreamer::emitDirectiveSetNoMt(); +} + void MipsTargetAsmStreamer::emitDirectiveSetAt() { OS << "\t.set\tat\n"; MipsTargetStreamer::emitDirectiveSetAt(); @@ -656,6 +684,10 @@ void MipsTargetAsmStreamer::emitDirectiveModuleHardFloat() { OS << "\t.module\thardfloat\n"; } +void MipsTargetAsmStreamer::emitDirectiveModuleMT() { + OS << "\t.module\tmt\n"; +} + // This part is for ELF object output. MipsTargetELFStreamer::MipsTargetELFStreamer(MCStreamer &S, const MCSubtargetInfo &STI) diff --git a/contrib/llvm/lib/Target/Mips/Mips.td b/contrib/llvm/lib/Target/Mips/Mips.td index f24761d7d101..d2f0fdcc6cc1 100644 --- a/contrib/llvm/lib/Target/Mips/Mips.td +++ b/contrib/llvm/lib/Target/Mips/Mips.td @@ -188,6 +188,8 @@ def FeatureUseTCCInDIV : SubtargetFeature< def FeatureMadd4 : SubtargetFeature<"nomadd4", "DisableMadd4", "true", "Disable 4-operand madd.fmt and related instructions">; +def FeatureMT : SubtargetFeature<"mt", "HasMT", "true", "Mips MT ASE">; + //===----------------------------------------------------------------------===// // Mips processors supported. //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td index 40078fb77144..89a5854bede0 100644 --- a/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td +++ b/contrib/llvm/lib/Target/Mips/MipsInstrInfo.td @@ -240,7 +240,8 @@ def HasMSA : Predicate<"Subtarget->hasMSA()">, AssemblerPredicate<"FeatureMSA">; def HasMadd4 : Predicate<"!Subtarget->disableMadd4()">, AssemblerPredicate<"!FeatureMadd4">; - +def HasMT : Predicate<"Subtarget->hasMT()">, + AssemblerPredicate<"FeatureMT">; //===----------------------------------------------------------------------===// // Mips GPR size adjectives. @@ -382,6 +383,10 @@ class ASE_MSA64 { list InsnPredicates = [HasMSA, HasMips64]; } +class ASE_MT { + list InsnPredicates = [HasMT]; +} + // Class used for separating microMIPSr6 and microMIPS (r3) instruction. // It can be used only on instructions that doesn't inherit PredicateControl. class ISA_MICROMIPS_NOT_32R6_64R6 : PredicateControl { @@ -2919,6 +2924,10 @@ include "MipsMSAInstrInfo.td" include "MipsEVAInstrFormats.td" include "MipsEVAInstrInfo.td" +// MT +include "MipsMTInstrFormats.td" +include "MipsMTInstrInfo.td" + // Micromips include "MicroMipsInstrFormats.td" include "MicroMipsInstrInfo.td" diff --git a/contrib/llvm/lib/Target/Mips/MipsMTInstrFormats.td b/contrib/llvm/lib/Target/Mips/MipsMTInstrFormats.td new file mode 100644 index 000000000000..edc0981e6278 --- /dev/null +++ b/contrib/llvm/lib/Target/Mips/MipsMTInstrFormats.td @@ -0,0 +1,99 @@ +//===-- MipsMTInstrFormats.td - Mips Instruction Formats ---*- tablegen -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Describe the MIPS MT instructions format +// +// opcode - operation code. +// rt - destination register +// +//===----------------------------------------------------------------------===// + +class MipsMTInst : MipsInst<(outs), (ins), "", [], NoItinerary, FrmOther>, + PredicateControl { + let DecoderNamespace = "Mips"; + let EncodingPredicates = [HasStdEnc]; +} + +class OPCODE1 Val> { + bits<1> Value = Val; +} + +def OPCODE_SC_D : OPCODE1<0b0>; +def OPCODE_SC_E : OPCODE1<0b1>; + +class FIELD5 Val> { + bits<5> Value = Val; +} + +def FIELD5_1_DMT_EMT : FIELD5<0b00001>; +def FIELD5_2_DMT_EMT : FIELD5<0b01111>; +def FIELD5_1_2_DVPE_EVPE : FIELD5<0b00000>; +def FIELD5_MFTR : FIELD5<0b01000>; +def FIELD5_MTTR : FIELD5<0b01100>; + +class COP0_MFMC0_MT : MipsMTInst { + bits<32> Inst; + + bits<5> rt; + let Inst{31-26} = 0b010000; // COP0 + let Inst{25-21} = 0b01011; // MFMC0 + let Inst{20-16} = rt; + let Inst{15-11} = Op1.Value; + let Inst{10-6} = Op2.Value; + let Inst{5} = sc.Value; + let Inst{4-3} = 0b00; + let Inst{2-0} = 0b001; +} + +class COP0_MFTTR_MT : MipsMTInst { + bits<32> Inst; + + bits<5> rt; + bits<5> rd; + bits<1> u; + bits<1> h; + bits<3> sel; + let Inst{31-26} = 0b010000; // COP0 + let Inst{25-21} = Op.Value; // MFMC0 + let Inst{20-16} = rt; + let Inst{15-11} = rd; + let Inst{10-6} = 0b00000; // rx - currently unsupported. + let Inst{5} = u; + let Inst{4} = h; + let Inst{3} = 0b0; + let Inst{2-0} = sel; +} + +class SPECIAL3_MT_FORK : MipsMTInst { + bits<32> Inst; + + bits<5> rs; + bits<5> rt; + bits<5> rd; + let Inst{31-26} = 0b011111; // SPECIAL3 + let Inst{25-21} = rs; + let Inst{20-16} = rt; + let Inst{15-11} = rd; + let Inst{10-6} = 0b00000; + let Inst{5-0} = 0b001000; // FORK +} + +class SPECIAL3_MT_YIELD : MipsMTInst { + bits<32> Inst; + + bits<5> rs; + bits<5> rd; + let Inst{31-26} = 0b011111; // SPECIAL3 + let Inst{25-21} = rs; + let Inst{20-16} = 0b00000; + let Inst{15-11} = rd; + let Inst{10-6} = 0b00000; + let Inst{5-0} = 0b001001; // FORK +} diff --git a/contrib/llvm/lib/Target/Mips/MipsMTInstrInfo.td b/contrib/llvm/lib/Target/Mips/MipsMTInstrInfo.td new file mode 100644 index 000000000000..72e626cbec40 --- /dev/null +++ b/contrib/llvm/lib/Target/Mips/MipsMTInstrInfo.td @@ -0,0 +1,208 @@ +//===-- MipsMTInstrInfo.td - Mips MT Instruction Infos -----*- tablegen -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file describes the MIPS MT ASE as defined by MD00378 1.12. +// +// TODO: Add support for the microMIPS encodings for the MT ASE and add the +// instruction mappings. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// MIPS MT Instruction Encodings +//===----------------------------------------------------------------------===// + +class DMT_ENC : COP0_MFMC0_MT; + +class EMT_ENC : COP0_MFMC0_MT; + +class DVPE_ENC : COP0_MFMC0_MT; + +class EVPE_ENC : COP0_MFMC0_MT; + +class FORK_ENC : SPECIAL3_MT_FORK; + +class YIELD_ENC : SPECIAL3_MT_YIELD; + +class MFTR_ENC : COP0_MFTTR_MT; + +class MTTR_ENC : COP0_MFTTR_MT; + +//===----------------------------------------------------------------------===// +// MIPS MT Instruction Descriptions +//===----------------------------------------------------------------------===// + +class MT_1R_DESC_BASE { + dag OutOperandList = (outs GPR32Opnd:$rt); + dag InOperandList = (ins); + string AsmString = !strconcat(instr_asm, "\t$rt"); + list Pattern = []; + InstrItinClass Itinerary = Itin; +} + +class MFTR_DESC { + dag OutOperandList = (outs GPR32Opnd:$rd); + dag InOperandList = (ins GPR32Opnd:$rt, uimm1:$u, uimm3:$sel, uimm1:$h); + string AsmString = "mftr\t$rd, $rt, $u, $sel, $h"; + list Pattern = []; + InstrItinClass Itinerary = II_MFTR; +} + +class MTTR_DESC { + dag OutOperandList = (outs GPR32Opnd:$rd); + dag InOperandList = (ins GPR32Opnd:$rt, uimm1:$u, uimm3:$sel, uimm1:$h); + string AsmString = "mttr\t$rt, $rd, $u, $sel, $h"; + list Pattern = []; + InstrItinClass Itinerary = II_MTTR; +} + +class FORK_DESC { + dag OutOperandList = (outs GPR32Opnd:$rs, GPR32Opnd:$rd); + dag InOperandList = (ins GPR32Opnd:$rt); + string AsmString = "fork\t$rd, $rs, $rt"; + list Pattern = []; + InstrItinClass Itinerary = II_FORK; +} + +class YIELD_DESC { + dag OutOperandList = (outs GPR32Opnd:$rd); + dag InOperandList = (ins GPR32Opnd:$rs); + string AsmString = "yield\t$rd, $rs"; + list Pattern = []; + InstrItinClass Itinerary = II_YIELD; +} + +class DMT_DESC : MT_1R_DESC_BASE<"dmt", II_DMT>; + +class EMT_DESC : MT_1R_DESC_BASE<"emt", II_EMT>; + +class DVPE_DESC : MT_1R_DESC_BASE<"dvpe", II_DVPE>; + +class EVPE_DESC : MT_1R_DESC_BASE<"evpe", II_EVPE>; + +//===----------------------------------------------------------------------===// +// MIPS MT Instruction Definitions +//===----------------------------------------------------------------------===// +let hasSideEffects = 1, isNotDuplicable = 1, + AdditionalPredicates = [NotInMicroMips] in { + def DMT : DMT_ENC, DMT_DESC, ASE_MT; + + def EMT : EMT_ENC, EMT_DESC, ASE_MT; + + def DVPE : DVPE_ENC, DVPE_DESC, ASE_MT; + + def EVPE : EVPE_ENC, EVPE_DESC, ASE_MT; + + def FORK : FORK_ENC, FORK_DESC, ASE_MT; + + def YIELD : YIELD_ENC, YIELD_DESC, ASE_MT; + + def MFTR : MFTR_ENC, MFTR_DESC, ASE_MT; + + def MTTR : MTTR_ENC, MTTR_DESC, ASE_MT; +} + +//===----------------------------------------------------------------------===// +// MIPS MT Pseudo Instructions - used to support mtfr & mttr aliases. +//===----------------------------------------------------------------------===// +def MFTC0 : MipsAsmPseudoInst<(outs GPR32Opnd:$rd), (ins COP0Opnd:$rt, + uimm3:$sel), + "mftc0 $rd, $rt, $sel">, ASE_MT; + +def MFTGPR : MipsAsmPseudoInst<(outs GPR32Opnd:$rd), (ins GPR32Opnd:$rt, + uimm3:$sel), + "mftgpr $rd, $rt">, ASE_MT; + +def MFTLO : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins ACC64DSPOpnd:$ac), + "mftlo $rt, $ac">, ASE_MT; + +def MFTHI : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins ACC64DSPOpnd:$ac), + "mfthi $rt, $ac">, ASE_MT; + +def MFTACX : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins ACC64DSPOpnd:$ac), + "mftacx $rt, $ac">, ASE_MT; + +def MFTDSP : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins), + "mftdsp $rt">, ASE_MT; + +def MFTC1 : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins FGR32Opnd:$ft), + "mftc1 $rt, $ft">, ASE_MT; + +def MFTHC1 : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins FGR32Opnd:$ft), + "mfthc1 $rt, $ft">, ASE_MT; + +def CFTC1 : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins FGRCCOpnd:$ft), + "cftc1 $rt, $ft">, ASE_MT; + + +def MTTC0 : MipsAsmPseudoInst<(outs COP0Opnd:$rd), (ins GPR32Opnd:$rt, + uimm3:$sel), + "mttc0 $rt, $rd, $sel">, ASE_MT; + +def MTTGPR : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), (ins GPR32Opnd:$rd), + "mttgpr $rd, $rt">, ASE_MT; + +def MTTLO : MipsAsmPseudoInst<(outs ACC64DSPOpnd:$ac), (ins GPR32Opnd:$rt), + "mttlo $rt, $ac">, ASE_MT; + +def MTTHI : MipsAsmPseudoInst<(outs ACC64DSPOpnd:$ac), (ins GPR32Opnd:$rt), + "mtthi $rt, $ac">, ASE_MT; + +def MTTACX : MipsAsmPseudoInst<(outs ACC64DSPOpnd:$ac), (ins GPR32Opnd:$rt), + "mttacx $rt, $ac">, ASE_MT; + +def MTTDSP : MipsAsmPseudoInst<(outs), (ins GPR32Opnd:$rt), + "mttdsp $rt">, ASE_MT; + +def MTTC1 : MipsAsmPseudoInst<(outs FGR32Opnd:$ft), (ins GPR32Opnd:$rt), + "mttc1 $rt, $ft">, ASE_MT; + +def MTTHC1 : MipsAsmPseudoInst<(outs FGR32Opnd:$ft), (ins GPR32Opnd:$rt), + "mtthc1 $rt, $ft">, ASE_MT; + +def CTTC1 : MipsAsmPseudoInst<(outs FGRCCOpnd:$ft), (ins GPR32Opnd:$rt), + "cttc1 $rt, $ft">, ASE_MT; + +//===----------------------------------------------------------------------===// +// MIPS MT Instruction Definitions +//===----------------------------------------------------------------------===// + +let AdditionalPredicates = [NotInMicroMips] in { + def : MipsInstAlias<"dmt", (DMT ZERO), 1>, ASE_MT; + + def : MipsInstAlias<"emt", (EMT ZERO), 1>, ASE_MT; + + def : MipsInstAlias<"dvpe", (DVPE ZERO), 1>, ASE_MT; + + def : MipsInstAlias<"evpe", (EVPE ZERO), 1>, ASE_MT; + + def : MipsInstAlias<"yield $rs", (YIELD ZERO, GPR32Opnd:$rs), 1>, ASE_MT; + + def : MipsInstAlias<"mftc0 $rd, $rt", (MFTC0 GPR32Opnd:$rd, COP0Opnd:$rt, 0), + 1>, ASE_MT; + + def : MipsInstAlias<"mftlo $rt", (MFTLO GPR32Opnd:$rt, AC0), 1>, ASE_MT; + + def : MipsInstAlias<"mfthi $rt", (MFTHI GPR32Opnd:$rt, AC0), 1>, ASE_MT; + + def : MipsInstAlias<"mftacx $rt", (MFTACX GPR32Opnd:$rt, AC0), 1>, ASE_MT; + + def : MipsInstAlias<"mttc0 $rd, $rt", (MTTC0 COP0Opnd:$rt, GPR32Opnd:$rd, 0), + 1>, ASE_MT; + + def : MipsInstAlias<"mttlo $rt", (MTTLO AC0, GPR32Opnd:$rt), 1>, ASE_MT; + + def : MipsInstAlias<"mtthi $rt", (MTTHI AC0, GPR32Opnd:$rt), 1>, ASE_MT; + + def : MipsInstAlias<"mttacx $rt", (MTTACX AC0, GPR32Opnd:$rt), 1>, ASE_MT; +} diff --git a/contrib/llvm/lib/Target/Mips/MipsSchedule.td b/contrib/llvm/lib/Target/Mips/MipsSchedule.td index c0de59ba15f5..8ec55ab6284d 100644 --- a/contrib/llvm/lib/Target/Mips/MipsSchedule.td +++ b/contrib/llvm/lib/Target/Mips/MipsSchedule.td @@ -84,6 +84,7 @@ def II_DIVU : InstrItinClass; def II_DIV_D : InstrItinClass; def II_DIV_S : InstrItinClass; def II_DMFC0 : InstrItinClass; +def II_DMT : InstrItinClass; def II_DMTC0 : InstrItinClass; def II_DMFC1 : InstrItinClass; def II_DMTC1 : InstrItinClass; @@ -113,8 +114,12 @@ def II_DSBH : InstrItinClass; def II_DSHD : InstrItinClass; def II_DSUBU : InstrItinClass; def II_DSUB : InstrItinClass; +def II_DVPE : InstrItinClass; +def II_EMT : InstrItinClass; +def II_EVPE : InstrItinClass; def II_EXT : InstrItinClass; // Any EXT instruction def II_FLOOR : InstrItinClass; +def II_FORK : InstrItinClass; def II_INS : InstrItinClass; // Any INS instruction def II_IndirectBranchPseudo : InstrItinClass; // Indirect branch pseudo. def II_J : InstrItinClass; @@ -221,6 +226,7 @@ def II_MFC1 : InstrItinClass; def II_MFHC1 : InstrItinClass; def II_MFC2 : InstrItinClass; def II_MFHI_MFLO : InstrItinClass; // mfhi and mflo +def II_MFTR : InstrItinClass; def II_MOD : InstrItinClass; def II_MODU : InstrItinClass; def II_MOVE : InstrItinClass; @@ -250,6 +256,7 @@ def II_MTC1 : InstrItinClass; def II_MTHC1 : InstrItinClass; def II_MTC2 : InstrItinClass; def II_MTHI_MTLO : InstrItinClass; // mthi and mtlo +def II_MTTR : InstrItinClass; def II_MUL : InstrItinClass; def II_MUH : InstrItinClass; def II_MUHU : InstrItinClass; @@ -345,6 +352,7 @@ def II_WRPGPR : InstrItinClass; def II_RDPGPR : InstrItinClass; def II_DVP : InstrItinClass; def II_EVP : InstrItinClass; +def II_YIELD : InstrItinClass; //===----------------------------------------------------------------------===// // Mips Generic instruction itineraries. @@ -386,6 +394,7 @@ def MipsGenericItineraries : ProcessorItineraries<[ALU, IMULDIV], [], [ InstrItinData]>, InstrItinData]>, InstrItinData]>, + InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, @@ -404,7 +413,11 @@ def MipsGenericItineraries : ProcessorItineraries<[ALU, IMULDIV], [], [ InstrItinData]>, InstrItinData]>, InstrItinData]>, + InstrItinData]>, + InstrItinData]>, + InstrItinData]>, InstrItinData]>, + InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, @@ -653,12 +666,14 @@ def MipsGenericItineraries : ProcessorItineraries<[ALU, IMULDIV], [], [ InstrItinData]>, InstrItinData]>, InstrItinData]>, + InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, + InstrItinData]>, InstrItinData]>, InstrItinData]>, InstrItinData]>, @@ -670,5 +685,6 @@ def MipsGenericItineraries : ProcessorItineraries<[ALU, IMULDIV], [], [ InstrItinData]>, InstrItinData]>, InstrItinData]>, - InstrItinData]> + InstrItinData]>, + InstrItinData]> ]>; diff --git a/contrib/llvm/lib/Target/Mips/MipsScheduleGeneric.td b/contrib/llvm/lib/Target/Mips/MipsScheduleGeneric.td index 15a0401b781e..89cda676441e 100644 --- a/contrib/llvm/lib/Target/Mips/MipsScheduleGeneric.td +++ b/contrib/llvm/lib/Target/Mips/MipsScheduleGeneric.td @@ -187,7 +187,11 @@ def GenericIssueCOP0 : ProcResource<1> { let Super = GenericCOP0; } def GenericWriteCOP0TLB : SchedWriteRes<[GenericIssueCOP0]> { let Latency = 4; } def GenericWriteCOP0 : SchedWriteRes<[GenericIssueCOP0]> { let Latency = 3; } def GenericReadCOP0 : SchedWriteRes<[GenericIssueCOP0]> { let Latency = 2; } -def GnereicReadWritePGPR : SchedWriteRes<[GenericIssueCOP0]>; +def GenericReadWritePGPR : SchedWriteRes<[GenericIssueCOP0]>; +def GenericReadWriteCOP0Long : SchedWriteRes<[GenericIssueCOP0]> { + let Latency = 5; +} +def GenericWriteCOP0Short : SchedWriteRes<[GenericIssueCOP0]>; def : ItinRW<[GenericWriteCOP0TLB], [II_TLBP, II_TLBR, II_TLBWI, II_TLBWR]>; def : ItinRW<[GenericWriteCOP0TLB], [II_TLBINV, II_TLBINVF]>; @@ -261,6 +265,14 @@ def : ItinRW<[GenericWriteLoad], [II_LBE, II_LBUE, II_LHE, II_LHUE, II_LWE, def : ItinRW<[GenericWriteLoad], [II_LWLE, II_LWRE]>; +// MIPS MT instructions +// ==================== + +def : ItinRW<[GenericWriteMove], [II_DMT, II_DVPE, II_EMT, II_EVPE]>; + +def : ItinRW<[GenericReadWriteCOP0Long], [II_YIELD]>; +def : ItinRW<[GenericWriteCOP0Short], [II_FORK]>; + // MIPS32R6 and MIPS16e // ==================== diff --git a/contrib/llvm/lib/Target/Mips/MipsScheduleP5600.td b/contrib/llvm/lib/Target/Mips/MipsScheduleP5600.td index 882a241d1426..fedfac24e4e7 100644 --- a/contrib/llvm/lib/Target/Mips/MipsScheduleP5600.td +++ b/contrib/llvm/lib/Target/Mips/MipsScheduleP5600.td @@ -19,7 +19,7 @@ def MipsP5600Model : SchedMachineModel { HasMips64, HasMips64r2, HasCnMips, InMicroMips, InMips16Mode, HasMicroMips32r6, HasMicroMips64r6, - HasDSP, HasDSPR2]; + HasDSP, HasDSPR2, HasMT]; } diff --git a/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp b/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp index 154d5825427b..eba21e0a1c67 100644 --- a/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp +++ b/contrib/llvm/lib/Target/Mips/MipsSubtarget.cpp @@ -70,7 +70,8 @@ MipsSubtarget::MipsSubtarget(const Triple &TT, StringRef CPU, StringRef FS, InMips16HardFloat(Mips16HardFloat), InMicroMipsMode(false), HasDSP(false), HasDSPR2(false), HasDSPR3(false), AllowMixed16_32(Mixed16_32 | Mips_Os16), Os16(Mips_Os16), HasMSA(false), UseTCCInDIV(false), HasSym32(false), - HasEVA(false), DisableMadd4(false), TM(TM), TargetTriple(TT), TSInfo(), + HasEVA(false), DisableMadd4(false), HasMT(false), TM(TM), + TargetTriple(TT), TSInfo(), InstrInfo( MipsInstrInfo::create(initializeSubtargetDependencies(CPU, FS, TM))), FrameLowering(MipsFrameLowering::create(*this)), diff --git a/contrib/llvm/lib/Target/Mips/MipsSubtarget.h b/contrib/llvm/lib/Target/Mips/MipsSubtarget.h index ccd47f00c0d3..7619e7b08612 100644 --- a/contrib/llvm/lib/Target/Mips/MipsSubtarget.h +++ b/contrib/llvm/lib/Target/Mips/MipsSubtarget.h @@ -149,6 +149,9 @@ class MipsSubtarget : public MipsGenSubtargetInfo { // related instructions. bool DisableMadd4; + // HasMT -- support MT ASE. + bool HasMT; + InstrItineraryData InstrItins; // We can override the determination of whether we are in mips16 mode @@ -259,6 +262,7 @@ class MipsSubtarget : public MipsGenSubtargetInfo { bool hasMSA() const { return HasMSA; } bool disableMadd4() const { return DisableMadd4; } bool hasEVA() const { return HasEVA; } + bool hasMT() const { return HasMT; } bool useSmallSection() const { return UseSmallSection; } bool hasStandardEncoding() const { return !inMips16Mode(); } diff --git a/contrib/llvm/lib/Target/Mips/MipsTargetStreamer.h b/contrib/llvm/lib/Target/Mips/MipsTargetStreamer.h index 41ebe411b98d..af24838665e1 100644 --- a/contrib/llvm/lib/Target/Mips/MipsTargetStreamer.h +++ b/contrib/llvm/lib/Target/Mips/MipsTargetStreamer.h @@ -40,6 +40,8 @@ class MipsTargetStreamer : public MCTargetStreamer { virtual void emitDirectiveSetNoMacro(); virtual void emitDirectiveSetMsa(); virtual void emitDirectiveSetNoMsa(); + virtual void emitDirectiveSetMt(); + virtual void emitDirectiveSetNoMt(); virtual void emitDirectiveSetAt(); virtual void emitDirectiveSetAtWithArg(unsigned RegNo); virtual void emitDirectiveSetNoAt(); @@ -96,6 +98,7 @@ class MipsTargetStreamer : public MCTargetStreamer { virtual void emitDirectiveModuleOddSPReg(); virtual void emitDirectiveModuleSoftFloat(); virtual void emitDirectiveModuleHardFloat(); + virtual void emitDirectiveModuleMT(); virtual void emitDirectiveSetFp(MipsABIFlagsSection::FpABIKind Value); virtual void emitDirectiveSetOddSPReg(); virtual void emitDirectiveSetNoOddSPReg(); @@ -116,6 +119,9 @@ class MipsTargetStreamer : public MCTargetStreamer { SMLoc IDLoc, const MCSubtargetInfo *STI); void emitRRI(unsigned Opcode, unsigned Reg0, unsigned Reg1, int16_t Imm, SMLoc IDLoc, const MCSubtargetInfo *STI); + void emitRRIII(unsigned Opcode, unsigned Reg0, unsigned Reg1, int16_t Imm0, + int16_t Imm1, int16_t Imm2, SMLoc IDLoc, + const MCSubtargetInfo *STI); void emitAddu(unsigned DstReg, unsigned SrcReg, unsigned TrgReg, bool Is64Bit, const MCSubtargetInfo *STI); void emitDSLL(unsigned DstReg, unsigned SrcReg, int16_t ShiftAmount, @@ -204,6 +210,8 @@ class MipsTargetAsmStreamer : public MipsTargetStreamer { void emitDirectiveSetNoMacro() override; void emitDirectiveSetMsa() override; void emitDirectiveSetNoMsa() override; + void emitDirectiveSetMt() override; + void emitDirectiveSetNoMt() override; void emitDirectiveSetAt() override; void emitDirectiveSetAtWithArg(unsigned RegNo) override; void emitDirectiveSetNoAt() override; @@ -267,6 +275,7 @@ class MipsTargetAsmStreamer : public MipsTargetStreamer { void emitDirectiveModuleOddSPReg() override; void emitDirectiveModuleSoftFloat() override; void emitDirectiveModuleHardFloat() override; + void emitDirectiveModuleMT() override; void emitDirectiveSetFp(MipsABIFlagsSection::FpABIKind Value) override; void emitDirectiveSetOddSPReg() override; void emitDirectiveSetNoOddSPReg() override; diff --git a/contrib/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/contrib/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index f26b9a7cb8dd..f800d91f4093 100644 --- a/contrib/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/contrib/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -62,7 +62,6 @@ #include #include -#undef DEBUG_TYPE #define DEBUG_TYPE "nvptx-lower" using namespace llvm; @@ -2456,7 +2455,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments( // v2f16 was loaded as an i32. Now we must bitcast it back. else if (EltVT == MVT::v2f16) Elt = DAG.getNode(ISD::BITCAST, dl, MVT::v2f16, Elt); - // Extend the element if necesary (e.g. an i8 is loaded + // Extend the element if necessary (e.g. an i8 is loaded // into an i16 register) if (Ins[InsIdx].VT.isInteger() && Ins[InsIdx].VT.getSizeInBits() > LoadVT.getSizeInBits()) { diff --git a/contrib/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp b/contrib/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp index 3be291b48b8f..989f0a3aba2f 100644 --- a/contrib/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp +++ b/contrib/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// #include "NVPTXLowerAggrCopies.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/StackProtector.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -42,6 +43,7 @@ struct NVPTXLowerAggrCopies : public FunctionPass { void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addPreserved(); + AU.addRequired(); } bool runOnFunction(Function &F) override; @@ -61,6 +63,8 @@ bool NVPTXLowerAggrCopies::runOnFunction(Function &F) { const DataLayout &DL = F.getParent()->getDataLayout(); LLVMContext &Context = F.getParent()->getContext(); + const TargetTransformInfo &TTI = + getAnalysis().getTTI(F); // Collect all aggregate loads and mem* calls. for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) { @@ -104,15 +108,26 @@ bool NVPTXLowerAggrCopies::runOnFunction(Function &F) { Value *SrcAddr = LI->getOperand(0); Value *DstAddr = SI->getOperand(1); unsigned NumLoads = DL.getTypeStoreSize(LI->getType()); - Value *CopyLen = ConstantInt::get(Type::getInt32Ty(Context), NumLoads); + ConstantInt *CopyLen = + ConstantInt::get(Type::getInt32Ty(Context), NumLoads); - createMemCpyLoop(/* ConvertedInst */ SI, - /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr, - /* CopyLen */ CopyLen, - /* SrcAlign */ LI->getAlignment(), - /* DestAlign */ SI->getAlignment(), - /* SrcIsVolatile */ LI->isVolatile(), - /* DstIsVolatile */ SI->isVolatile()); + if (!TTI.useWideIRMemcpyLoopLowering()) { + createMemCpyLoop(/* ConvertedInst */ SI, + /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr, + /* CopyLen */ CopyLen, + /* SrcAlign */ LI->getAlignment(), + /* DestAlign */ SI->getAlignment(), + /* SrcIsVolatile */ LI->isVolatile(), + /* DstIsVolatile */ SI->isVolatile()); + } else { + createMemCpyLoopKnownSize(/* ConvertedInst */ SI, + /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr, + /* CopyLen */ CopyLen, + /* SrcAlign */ LI->getAlignment(), + /* DestAlign */ SI->getAlignment(), + /* SrcIsVolatile */ LI->isVolatile(), + /* DstIsVolatile */ SI->isVolatile(), TTI); + } SI->eraseFromParent(); LI->eraseFromParent(); @@ -121,7 +136,7 @@ bool NVPTXLowerAggrCopies::runOnFunction(Function &F) { // Transform mem* intrinsic calls. for (MemIntrinsic *MemCall : MemCalls) { if (MemCpyInst *Memcpy = dyn_cast(MemCall)) { - expandMemCpyAsLoop(Memcpy); + expandMemCpyAsLoop(Memcpy, TTI); } else if (MemMoveInst *Memmove = dyn_cast(MemCall)) { expandMemMoveAsLoop(Memmove); } else if (MemSetInst *Memset = dyn_cast(MemCall)) { diff --git a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp index 7393f3d7a08a..bdad2fe8714f 100644 --- a/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp +++ b/contrib/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp @@ -115,7 +115,7 @@ class PPCAsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override { + uint64_t Value, bool IsResolved) const override { Value = adjustFixupValue(Fixup.getKind(), Value); if (!Value) return; // Doesn't change encoding. diff --git a/contrib/llvm/lib/Target/PowerPC/PPCCTRLoops.cpp b/contrib/llvm/lib/Target/PowerPC/PPCCTRLoops.cpp index 094d3e6a61b5..53f33ac1fc0e 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCCTRLoops.cpp +++ b/contrib/llvm/lib/Target/PowerPC/PPCCTRLoops.cpp @@ -607,7 +607,10 @@ bool PPCCTRLoops::convertToCTRLoop(Loop *L) { // The old condition may be dead now, and may have even created a dead PHI // (the original induction variable). RecursivelyDeleteTriviallyDeadInstructions(OldCond); - DeleteDeadPHIs(CountedExitBlock); + // Run through the basic blocks of the loop and see if any of them have dead + // PHIs that can be removed. + for (auto I : L->blocks()) + DeleteDeadPHIs(I); ++NumCTRLoops; return MadeChange; diff --git a/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp index c2c115cb6daf..b49c3345a17d 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp +++ b/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp @@ -435,22 +435,19 @@ unsigned PPCFrameLowering::determineFrameLayout(MachineFunction &MF, const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo(); - // If we are a leaf function, and use up to 224 bytes of stack space, - // don't have a frame pointer, calls, or dynamic alloca then we do not need - // to adjust the stack pointer (we fit in the Red Zone). - // The 32-bit SVR4 ABI has no Red Zone. However, it can still generate - // stackless code if all local vars are reg-allocated. - bool DisableRedZone = MF.getFunction()->hasFnAttribute(Attribute::NoRedZone); unsigned LR = RegInfo->getRARegister(); - if (!DisableRedZone && - (Subtarget.isPPC64() || // 32-bit SVR4, no stack- - !Subtarget.isSVR4ABI() || // allocated locals. - FrameSize == 0) && - FrameSize <= 224 && // Fits in red zone. - !MFI.hasVarSizedObjects() && // No dynamic alloca. - !MFI.adjustsStack() && // No calls. - !MustSaveLR(MF, LR) && - !RegInfo->hasBasePointer(MF)) { // No special alignment. + bool DisableRedZone = MF.getFunction()->hasFnAttribute(Attribute::NoRedZone); + bool CanUseRedZone = !MFI.hasVarSizedObjects() && // No dynamic alloca. + !MFI.adjustsStack() && // No calls. + !MustSaveLR(MF, LR) && // No need to save LR. + !RegInfo->hasBasePointer(MF); // No special alignment. + + // Note: for PPC32 SVR4ABI (Non-DarwinABI), we can still generate stackless + // code if all local vars are reg-allocated. + bool FitsInRedZone = FrameSize <= Subtarget.getRedZoneSize(); + + // Check whether we can skip adjusting the stack pointer (by using red zone) + if (!DisableRedZone && CanUseRedZone && FitsInRedZone) { // No need for frame if (UpdateMF) MFI.setStackSize(0); @@ -1869,8 +1866,13 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF, } if (HasVRSaveArea) { - // Insert alignment padding, we need 16-byte alignment. - LowerBound = (LowerBound - 15) & ~(15); + // Insert alignment padding, we need 16-byte alignment. Note: for postive + // number the alignment formula is : y = (x + (n-1)) & (~(n-1)). But since + // we are using negative number here (the stack grows downward). We should + // use formula : y = x & (~(n-1)). Where x is the size before aligning, n + // is the alignment size ( n = 16 here) and y is the size after aligning. + assert(LowerBound <= 0 && "Expect LowerBound have a non-positive value!"); + LowerBound &= ~(15); for (unsigned i = 0, e = VRegs.size(); i != e; ++i) { int FI = VRegs[i].getFrameIdx(); diff --git a/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index 535b9deaefac..3aaf7ef2c2a0 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/contrib/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -419,25 +419,6 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { .getNode(); } -/// isIntS16Immediate - This method tests to see if the node is either a 32-bit -/// or 64-bit immediate, and if the value can be accurately represented as a -/// sign extension from a 16-bit value. If so, this returns true and the -/// immediate. -static bool isIntS16Immediate(SDNode *N, short &Imm) { - if (N->getOpcode() != ISD::Constant) - return false; - - Imm = (short)cast(N)->getZExtValue(); - if (N->getValueType(0) == MVT::i32) - return Imm == (int32_t)cast(N)->getZExtValue(); - else - return Imm == (int64_t)cast(N)->getZExtValue(); -} - -static bool isIntS16Immediate(SDValue Op, short &Imm) { - return isIntS16Immediate(Op.getNode(), Imm); -} - /// isInt32Immediate - This method tests to see if the node is a 32-bit constant /// operand. If so Imm will receive the 32-bit value. static bool isInt32Immediate(SDNode *N, unsigned &Imm) { @@ -728,7 +709,10 @@ static uint64_t Rot64(uint64_t Imm, unsigned R) { static unsigned getInt64Count(int64_t Imm) { unsigned Count = getInt64CountDirect(Imm); - if (Count == 1) + + // If the instruction count is 1 or 2, we do not need further analysis + // since rotate + load constant requires at least 2 instructions. + if (Count <= 2) return Count; for (unsigned r = 1; r < 63; ++r) { @@ -838,7 +822,10 @@ static SDNode *getInt64Direct(SelectionDAG *CurDAG, const SDLoc &dl, static SDNode *getInt64(SelectionDAG *CurDAG, const SDLoc &dl, int64_t Imm) { unsigned Count = getInt64CountDirect(Imm); - if (Count == 1) + + // If the instruction count is 1 or 2, we do not need further analysis + // since rotate + load constant requires at least 2 instructions. + if (Count <= 2) return getInt64Direct(CurDAG, dl, Imm); unsigned RMin = 0; @@ -2126,7 +2113,7 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, getI32Imm(Imm & 0xFFFF, dl)), 0); Opc = PPC::CMPLW; } else { - short SImm; + int16_t SImm; if (isIntS16Immediate(RHS, SImm)) return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, getI32Imm((int)SImm & 0xFFFF, @@ -2173,7 +2160,7 @@ SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, getI64Imm(Imm & 0xFFFF, dl)), 0); Opc = PPC::CMPLD; } else { - short SImm; + int16_t SImm; if (isIntS16Immediate(RHS, SImm)) return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, getI64Imm(SImm & 0xFFFF, dl)), @@ -3323,7 +3310,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { if (tryLogicOpOfCompares(N)) return; - short Imm; + int16_t Imm; if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && isIntS16Immediate(N->getOperand(1), Imm)) { KnownBits LHSKnown; @@ -3346,7 +3333,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { break; } case ISD::ADD: { - short Imm; + int16_t Imm; if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && isIntS16Immediate(N->getOperand(1), Imm)) { selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm); @@ -4034,11 +4021,13 @@ void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { O0.getNode(), O1.getNode()); }; + // FIXME: When the semantics of the interaction between select and undef + // are clearly defined, it may turn out to be unnecessary to break here. SDValue TrueRes = TryFold(ConstTrue); - if (!TrueRes) + if (!TrueRes || TrueRes.isUndef()) break; SDValue FalseRes = TryFold(ConstFalse); - if (!FalseRes) + if (!FalseRes || FalseRes.isUndef()) break; // For us to materialize these using one instruction, we must be able to diff --git a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 72f14e969138..0e069ec1665f 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -136,6 +136,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, addRegisterClass(MVT::f64, &PPC::F8RCRegClass); } + // Match BITREVERSE to customized fast code sequence in the td file. + setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); + setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); + // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. for (MVT VT : MVT::integer_valuetypes()) { setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); @@ -1168,6 +1172,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::LXSIZX: return "PPCISD::LXSIZX"; case PPCISD::STXSIX: return "PPCISD::STXSIX"; case PPCISD::VEXTS: return "PPCISD::VEXTS"; + case PPCISD::SExtVElems: return "PPCISD::SExtVElems"; case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; @@ -2028,17 +2033,17 @@ int PPC::isQVALIGNIShuffleMask(SDNode *N) { /// or 64-bit immediate, and if the value can be accurately represented as a /// sign extension from a 16-bit value. If so, this returns true and the /// immediate. -static bool isIntS16Immediate(SDNode *N, short &Imm) { +bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) { if (!isa(N)) return false; - Imm = (short)cast(N)->getZExtValue(); + Imm = (int16_t)cast(N)->getZExtValue(); if (N->getValueType(0) == MVT::i32) return Imm == (int32_t)cast(N)->getZExtValue(); else return Imm == (int64_t)cast(N)->getZExtValue(); } -static bool isIntS16Immediate(SDValue Op, short &Imm) { +bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { return isIntS16Immediate(Op.getNode(), Imm); } @@ -2048,7 +2053,7 @@ static bool isIntS16Immediate(SDValue Op, short &Imm) { bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const { - short imm = 0; + int16_t imm = 0; if (N.getOpcode() == ISD::ADD) { if (isIntS16Immediate(N.getOperand(1), imm)) return false; // r+i @@ -2138,7 +2143,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, return false; if (N.getOpcode() == ISD::ADD) { - short imm = 0; + int16_t imm = 0; if (isIntS16Immediate(N.getOperand(1), imm) && (!Aligned || (imm & 3) == 0)) { Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); @@ -2162,7 +2167,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, return true; // [&g+r] } } else if (N.getOpcode() == ISD::OR) { - short imm = 0; + int16_t imm = 0; if (isIntS16Immediate(N.getOperand(1), imm) && (!Aligned || (imm & 3) == 0)) { // If this is an or of disjoint bitfields, we can codegen this as an add @@ -2190,7 +2195,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, // If this address fits entirely in a 16-bit sext immediate field, codegen // this as "d, 0" - short Imm; + int16_t Imm; if (isIntS16Immediate(CN, Imm) && (!Aligned || (Imm & 3) == 0)) { Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, @@ -2235,10 +2240,15 @@ bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, if (SelectAddressRegReg(N, Base, Index, DAG)) return true; - // If the operand is an addition, always emit this as [r+r], since this is - // better (for code size, and execution, as the memop does the add for free) - // than emitting an explicit add. - if (N.getOpcode() == ISD::ADD) { + // If the address is the result of an add, we will utilize the fact that the + // address calculation includes an implicit add. However, we can reduce + // register pressure if we do not materialize a constant just for use as the + // index register. We only get rid of the add if it is not an add of a + // value and a 16-bit signed constant and both have a single use. + int16_t imm = 0; + if (N.getOpcode() == ISD::ADD && + (!isIntS16Immediate(N.getOperand(1), imm) || + !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { Base = N.getOperand(0); Index = N.getOperand(1); return true; @@ -6422,7 +6432,7 @@ PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); - // Get the corect type for integers. + // Get the correct type for integers. EVT IntVT = Op.getValueType(); // Get the inputs. @@ -6439,7 +6449,7 @@ SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, // When we pop the dynamic allocation we need to restore the SP link. SDLoc dl(Op); - // Get the corect type for pointers. + // Get the correct type for pointers. EVT PtrVT = getPointerTy(DAG.getDataLayout()); // Construct the stack pointer operand. @@ -6514,7 +6524,7 @@ SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SDValue Size = Op.getOperand(1); SDLoc dl(Op); - // Get the corect type for pointers. + // Get the correct type for pointers. EVT PtrVT = getPointerTy(DAG.getDataLayout()); // Negate the size. SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, @@ -6645,6 +6655,7 @@ SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { default: break; // SETUO etc aren't handled by fsel. case ISD::SETNE: std::swap(TV, FV); + LLVM_FALLTHROUGH; case ISD::SETEQ: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); @@ -6656,6 +6667,7 @@ SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { case ISD::SETULT: case ISD::SETLT: std::swap(TV, FV); // fsel is natively setge, swap operands for setlt + LLVM_FALLTHROUGH; case ISD::SETOGE: case ISD::SETGE: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits @@ -6664,6 +6676,7 @@ SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { case ISD::SETUGT: case ISD::SETGT: std::swap(TV, FV); // fsel is natively setge, swap operands for setlt + LLVM_FALLTHROUGH; case ISD::SETOLE: case ISD::SETLE: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits @@ -6677,6 +6690,7 @@ SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { default: break; // SETUO etc aren't handled by fsel. case ISD::SETNE: std::swap(TV, FV); + LLVM_FALLTHROUGH; case ISD::SETEQ: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits @@ -11311,6 +11325,132 @@ static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) { return SDValue(); } +// This function adds the required vector_shuffle needed to get +// the elements of the vector extract in the correct position +// as specified by the CorrectElems encoding. +static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG, + SDValue Input, uint64_t Elems, + uint64_t CorrectElems) { + SDLoc dl(N); + + unsigned NumElems = Input.getValueType().getVectorNumElements(); + SmallVector ShuffleMask(NumElems, -1); + + // Knowing the element indices being extracted from the original + // vector and the order in which they're being inserted, just put + // them at element indices required for the instruction. + for (unsigned i = 0; i < N->getNumOperands(); i++) { + if (DAG.getDataLayout().isLittleEndian()) + ShuffleMask[CorrectElems & 0xF] = Elems & 0xF; + else + ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4; + CorrectElems = CorrectElems >> 8; + Elems = Elems >> 8; + } + + SDValue Shuffle = + DAG.getVectorShuffle(Input.getValueType(), dl, Input, + DAG.getUNDEF(Input.getValueType()), ShuffleMask); + + EVT Ty = N->getValueType(0); + SDValue BV = DAG.getNode(PPCISD::SExtVElems, dl, Ty, Shuffle); + return BV; +} + +// Look for build vector patterns where input operands come from sign +// extended vector_extract elements of specific indices. If the correct indices +// aren't used, add a vector shuffle to fix up the indices and create a new +// PPCISD:SExtVElems node which selects the vector sign extend instructions +// during instruction selection. +static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { + // This array encodes the indices that the vector sign extend instructions + // extract from when extending from one type to another for both BE and LE. + // The right nibble of each byte corresponds to the LE incides. + // and the left nibble of each byte corresponds to the BE incides. + // For example: 0x3074B8FC byte->word + // For LE: the allowed indices are: 0x0,0x4,0x8,0xC + // For BE: the allowed indices are: 0x3,0x7,0xB,0xF + // For example: 0x000070F8 byte->double word + // For LE: the allowed indices are: 0x0,0x8 + // For BE: the allowed indices are: 0x7,0xF + uint64_t TargetElems[] = { + 0x3074B8FC, // b->w + 0x000070F8, // b->d + 0x10325476, // h->w + 0x00003074, // h->d + 0x00001032, // w->d + }; + + uint64_t Elems = 0; + int Index; + SDValue Input; + + auto isSExtOfVecExtract = [&](SDValue Op) -> bool { + if (!Op) + return false; + if (Op.getOpcode() != ISD::SIGN_EXTEND) + return false; + + SDValue Extract = Op.getOperand(0); + if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT) + return false; + + ConstantSDNode *ExtOp = dyn_cast(Extract.getOperand(1)); + if (!ExtOp) + return false; + + Index = ExtOp->getZExtValue(); + if (Input && Input != Extract.getOperand(0)) + return false; + + if (!Input) + Input = Extract.getOperand(0); + + Elems = Elems << 8; + Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4; + Elems |= Index; + + return true; + }; + + // If the build vector operands aren't sign extended vector extracts, + // of the same input vector, then return. + for (unsigned i = 0; i < N->getNumOperands(); i++) { + if (!isSExtOfVecExtract(N->getOperand(i))) { + return SDValue(); + } + } + + // If the vector extract indicies are not correct, add the appropriate + // vector_shuffle. + int TgtElemArrayIdx; + int InputSize = Input.getValueType().getScalarSizeInBits(); + int OutputSize = N->getValueType(0).getScalarSizeInBits(); + if (InputSize + OutputSize == 40) + TgtElemArrayIdx = 0; + else if (InputSize + OutputSize == 72) + TgtElemArrayIdx = 1; + else if (InputSize + OutputSize == 48) + TgtElemArrayIdx = 2; + else if (InputSize + OutputSize == 80) + TgtElemArrayIdx = 3; + else if (InputSize + OutputSize == 96) + TgtElemArrayIdx = 4; + else + return SDValue(); + + uint64_t CorrectElems = TargetElems[TgtElemArrayIdx]; + CorrectElems = DAG.getDataLayout().isLittleEndian() + ? CorrectElems & 0x0F0F0F0F0F0F0F0F + : CorrectElems & 0xF0F0F0F0F0F0F0F0; + if (Elems != CorrectElems) { + return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems); + } + + // Regular lowering will catch cases where a shuffle is not needed. + return SDValue(); +} + SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::BUILD_VECTOR && @@ -11338,6 +11478,15 @@ SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, if (Reduced) return Reduced; + // If we're building a vector out of extended elements from another vector + // we have P9 vector integer extend instructions. + if (Subtarget.hasP9Altivec()) { + Reduced = combineBVOfVecSExt(N, DAG); + if (Reduced) + return Reduced; + } + + if (N->getValueType(0) != MVT::v2f64) return SDValue(); diff --git a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h index a5108727bb4b..821927d3b157 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/contrib/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -67,6 +67,10 @@ namespace llvm { /// VSFRC that is sign-extended from ByteWidth to a 64-byte integer. VEXTS, + /// SExtVElems, takes an input vector of a smaller type and sign + /// extends to an output vector of a larger type. + SExtVElems, + /// Reciprocal estimate instructions (unary FP ops). FRE, FRSQRTE, @@ -1092,6 +1096,9 @@ namespace llvm { ISD::ArgFlagsTy &ArgFlags, CCState &State); + bool isIntS16Immediate(SDNode *N, int16_t &Imm); + bool isIntS16Immediate(SDValue Op, int16_t &Imm); + } // end namespace llvm #endif // LLVM_TARGET_POWERPC_PPC32ISELLOWERING_H diff --git a/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 47d59c25392a..6d9f55206b6a 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/contrib/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -32,6 +32,9 @@ def SDT_PPCstxsix : SDTypeProfile<0, 3, [ def SDT_PPCVexts : SDTypeProfile<1, 2, [ SDTCisVT<0, f64>, SDTCisVT<1, f64>, SDTCisPtrTy<2> ]>; +def SDT_PPCSExtVElems : SDTypeProfile<1, 1, [ + SDTCisVec<0>, SDTCisVec<1> +]>; def SDT_PPCCallSeqStart : SDCallSeqStart<[ SDTCisVT<0, i32>, SDTCisVT<1, i32> ]>; @@ -131,6 +134,7 @@ def PPClxsizx : SDNode<"PPCISD::LXSIZX", SDT_PPCLxsizx, def PPCstxsix : SDNode<"PPCISD::STXSIX", SDT_PPCstxsix, [SDNPHasChain, SDNPMayStore]>; def PPCVexts : SDNode<"PPCISD::VEXTS", SDT_PPCVexts, []>; +def PPCSExtVElems : SDNode<"PPCISD::SExtVElems", SDT_PPCSExtVElems, []>; // Extract FPSCR (not modeled at the DAG level). def PPCmffs : SDNode<"PPCISD::MFFS", @@ -4450,3 +4454,190 @@ def MSGSYNC : XForm_0<31, 886, (outs), (ins), "msgsync", IIC_SprMSGSYNC, []>; def STOP : XForm_0<19, 370, (outs), (ins), "stop", IIC_SprSTOP, []>; } // IsISA3_0 + +// Fast 32-bit reverse bits algorithm: +// Step 1: 1-bit swap (swap odd 1-bit and even 1-bit): +// n = ((n >> 1) & 0x55555555) | ((n << 1) & 0xAAAAAAAA); +// Step 2: 2-bit swap (swap odd 2-bit and even 2-bit): +// n = ((n >> 2) & 0x33333333) | ((n << 2) & 0xCCCCCCCC); +// Step 3: 4-bit swap (swap odd 4-bit and even 4-bit): +// n = ((n >> 4) & 0x0F0F0F0F) | ((n << 4) & 0xF0F0F0F0); +// Step 4: byte reverse (Suppose n = [B1,B2,B3,B4]): +// Step 4.1: Put B4,B2 in the right position (rotate left 3 bytes): +// n' = (n rotl 24); After which n' = [B4, B1, B2, B3] +// Step 4.2: Insert B3 to the right position: +// n' = rlwimi n', n, 8, 8, 15; After which n' = [B4, B3, B2, B3] +// Step 4.3: Insert B1 to the right position: +// n' = rlwimi n', n, 8, 24, 31; After which n' = [B4, B3, B2, B1] +def MaskValues { + dag Lo1 = (ORI (LIS 0x5555), 0x5555); + dag Hi1 = (ORI (LIS 0xAAAA), 0xAAAA); + dag Lo2 = (ORI (LIS 0x3333), 0x3333); + dag Hi2 = (ORI (LIS 0xCCCC), 0xCCCC); + dag Lo4 = (ORI (LIS 0x0F0F), 0x0F0F); + dag Hi4 = (ORI (LIS 0xF0F0), 0xF0F0); +} + +def Shift1 { + dag Right = (RLWINM $A, 31, 1, 31); + dag Left = (RLWINM $A, 1, 0, 30); +} + +def Swap1 { + dag Bit = (OR (AND Shift1.Right, MaskValues.Lo1), + (AND Shift1.Left, MaskValues.Hi1)); +} + +def Shift2 { + dag Right = (RLWINM Swap1.Bit, 30, 2, 31); + dag Left = (RLWINM Swap1.Bit, 2, 0, 29); +} + +def Swap2 { + dag Bits = (OR (AND Shift2.Right, MaskValues.Lo2), + (AND Shift2.Left, MaskValues.Hi2)); +} + +def Shift4 { + dag Right = (RLWINM Swap2.Bits, 28, 4, 31); + dag Left = (RLWINM Swap2.Bits, 4, 0, 27); +} + +def Swap4 { + dag Bits = (OR (AND Shift4.Right, MaskValues.Lo4), + (AND Shift4.Left, MaskValues.Hi4)); +} + +def Rotate { + dag Left3Bytes = (RLWINM Swap4.Bits, 24, 0, 31); +} + +def RotateInsertByte3 { + dag Left = (RLWIMI Rotate.Left3Bytes, Swap4.Bits, 8, 8, 15); +} + +def RotateInsertByte1 { + dag Left = (RLWIMI RotateInsertByte3.Left, Swap4.Bits, 8, 24, 31); +} + +def : Pat<(i32 (bitreverse i32:$A)), + (RLDICL_32 RotateInsertByte1.Left, 0, 32)>; + +// Fast 64-bit reverse bits algorithm: +// Step 1: 1-bit swap (swap odd 1-bit and even 1-bit): +// n = ((n >> 1) & 0x5555555555555555) | ((n << 1) & 0xAAAAAAAAAAAAAAAA); +// Step 2: 2-bit swap (swap odd 2-bit and even 2-bit): +// n = ((n >> 2) & 0x3333333333333333) | ((n << 2) & 0xCCCCCCCCCCCCCCCC); +// Step 3: 4-bit swap (swap odd 4-bit and even 4-bit): +// n = ((n >> 4) & 0x0F0F0F0F0F0F0F0F) | ((n << 4) & 0xF0F0F0F0F0F0F0F0); +// Step 4: byte reverse (Suppose n = [B1,B2,B3,B4,B5,B6,B7,B8]): +// Apply the same byte reverse algorithm mentioned above for the fast 32-bit +// reverse to both the high 32 bit and low 32 bit of the 64 bit value. And +// then OR them together to get the final result. +def MaskValues64 { + dag Lo1 = (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), MaskValues.Lo1, sub_32)); + dag Hi1 = (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), MaskValues.Hi1, sub_32)); + dag Lo2 = (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), MaskValues.Lo2, sub_32)); + dag Hi2 = (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), MaskValues.Hi2, sub_32)); + dag Lo4 = (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), MaskValues.Lo4, sub_32)); + dag Hi4 = (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), MaskValues.Hi4, sub_32)); +} + +def DWMaskValues { + dag Lo1 = (ORI8 (ORIS8 (RLDICR MaskValues64.Lo1, 32, 31), 0x5555), 0x5555); + dag Hi1 = (ORI8 (ORIS8 (RLDICR MaskValues64.Hi1, 32, 31), 0xAAAA), 0xAAAA); + dag Lo2 = (ORI8 (ORIS8 (RLDICR MaskValues64.Lo2, 32, 31), 0x3333), 0x3333); + dag Hi2 = (ORI8 (ORIS8 (RLDICR MaskValues64.Hi2, 32, 31), 0xCCCC), 0xCCCC); + dag Lo4 = (ORI8 (ORIS8 (RLDICR MaskValues64.Lo4, 32, 31), 0x0F0F), 0x0F0F); + dag Hi4 = (ORI8 (ORIS8 (RLDICR MaskValues64.Hi4, 32, 31), 0xF0F0), 0xF0F0); +} + +def DWShift1 { + dag Right = (RLDICL $A, 63, 1); + dag Left = (RLDICR $A, 1, 62); +} + +def DWSwap1 { + dag Bit = (OR8 (AND8 DWShift1.Right, DWMaskValues.Lo1), + (AND8 DWShift1.Left, DWMaskValues.Hi1)); +} + +def DWShift2 { + dag Right = (RLDICL DWSwap1.Bit, 62, 2); + dag Left = (RLDICR DWSwap1.Bit, 2, 61); +} + +def DWSwap2 { + dag Bits = (OR8 (AND8 DWShift2.Right, DWMaskValues.Lo2), + (AND8 DWShift2.Left, DWMaskValues.Hi2)); +} + +def DWShift4 { + dag Right = (RLDICL DWSwap2.Bits, 60, 4); + dag Left = (RLDICR DWSwap2.Bits, 4, 59); +} + +def DWSwap4 { + dag Bits = (OR8 (AND8 DWShift4.Right, DWMaskValues.Lo4), + (AND8 DWShift4.Left, DWMaskValues.Hi4)); +} + +// Bit swap is done, now start byte swap. +def DWExtractLo32 { + dag SubReg = (i32 (EXTRACT_SUBREG DWSwap4.Bits, sub_32)); +} + +def DWRotateLo32 { + dag Left24 = (RLWINM DWExtractLo32.SubReg, 24, 0, 31); +} + +def DWLo32RotateInsertByte3 { + dag Left = (RLWIMI DWRotateLo32.Left24, DWExtractLo32.SubReg, 8, 8, 15); +} + +// Lower 32 bits in the right order +def DWLo32RotateInsertByte1 { + dag Left = + (RLWIMI DWLo32RotateInsertByte3.Left, DWExtractLo32.SubReg, 8, 24, 31); +} + +def ExtendLo32 { + dag To64Bit = + (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), + DWLo32RotateInsertByte1.Left, sub_32)); +} + +def DWShiftHi32 { // SRDI DWSwap4.Bits, 32) + dag ToLo32 = (RLDICL DWSwap4.Bits, 32, 32); +} + +def DWExtractHi32 { + dag SubReg = (i32 (EXTRACT_SUBREG DWShiftHi32.ToLo32, sub_32)); +} + +def DWRotateHi32 { + dag Left24 = (RLWINM DWExtractHi32.SubReg, 24, 0, 31); +} + +def DWHi32RotateInsertByte3 { + dag Left = (RLWIMI DWRotateHi32.Left24, DWExtractHi32.SubReg, 8, 8, 15); +} + +// High 32 bits in the right order, but in the low 32-bit position +def DWHi32RotateInsertByte1 { + dag Left = + (RLWIMI DWHi32RotateInsertByte3.Left, DWExtractHi32.SubReg, 8, 24, 31); +} + +def ExtendHi32 { + dag To64Bit = + (i64 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), + DWHi32RotateInsertByte1.Left, sub_32)); +} + +def DWShiftLo32 { // SLDI ExtendHi32.To64Bit, 32 + dag ToHi32 = (RLDICR ExtendHi32.To64Bit, 32, 31); +} + +def : Pat<(i64 (bitreverse i64:$A)), + (OR8 DWShiftLo32.ToHi32, ExtendLo32.To64Bit)>; diff --git a/contrib/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/contrib/llvm/lib/Target/PowerPC/PPCInstrVSX.td index 9cfc897cdb3f..43635a8919e2 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCInstrVSX.td +++ b/contrib/llvm/lib/Target/PowerPC/PPCInstrVSX.td @@ -1901,6 +1901,98 @@ let Predicates = [IsLittleEndian, HasVSX] in def : Pat<(v4i32 (int_ppc_vsx_lxvw4x_be xoaddr:$src)), (LXVW4X xoaddr:$src)>; def : Pat<(v2f64 (int_ppc_vsx_lxvd2x_be xoaddr:$src)), (LXVD2X xoaddr:$src)>; +// Variable index unsigned vector_extract on Power9 +let Predicates = [HasP9Altivec, IsLittleEndian] in { + def : Pat<(i64 (anyext (i32 (vector_extract v16i8:$S, i64:$Idx)))), + (VEXTUBRX $Idx, $S)>; + + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, i64:$Idx)))), + (VEXTUHRX (RLWINM8 $Idx, 1, 28, 30), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 0)))), + (VEXTUHRX (LI8 0), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 1)))), + (VEXTUHRX (LI8 2), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 2)))), + (VEXTUHRX (LI8 4), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 3)))), + (VEXTUHRX (LI8 6), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 4)))), + (VEXTUHRX (LI8 8), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 5)))), + (VEXTUHRX (LI8 10), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 6)))), + (VEXTUHRX (LI8 12), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 7)))), + (VEXTUHRX (LI8 14), $S)>; + + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, i64:$Idx)))), + (VEXTUWRX (RLWINM8 $Idx, 2, 28, 29), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 0)))), + (VEXTUWRX (LI8 0), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 1)))), + (VEXTUWRX (LI8 4), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 2)))), + (VEXTUWRX (LI8 8), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 3)))), + (VEXTUWRX (LI8 12), $S)>; + + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, i64:$Idx)))), + (EXTSW (VEXTUWRX (RLWINM8 $Idx, 2, 28, 29), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 0)))), + (EXTSW (VEXTUWRX (LI8 0), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 1)))), + (EXTSW (VEXTUWRX (LI8 4), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 2)))), + (EXTSW (VEXTUWRX (LI8 8), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 3)))), + (EXTSW (VEXTUWRX (LI8 12), $S))>; +} +let Predicates = [HasP9Altivec, IsBigEndian] in { + def : Pat<(i64 (anyext (i32 (vector_extract v16i8:$S, i64:$Idx)))), + (VEXTUBLX $Idx, $S)>; + + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, i64:$Idx)))), + (VEXTUHLX (RLWINM8 $Idx, 1, 28, 30), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 0)))), + (VEXTUHLX (LI8 0), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 1)))), + (VEXTUHLX (LI8 2), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 2)))), + (VEXTUHLX (LI8 4), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 3)))), + (VEXTUHLX (LI8 6), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 4)))), + (VEXTUHLX (LI8 8), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 5)))), + (VEXTUHLX (LI8 10), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 6)))), + (VEXTUHLX (LI8 12), $S)>; + def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 7)))), + (VEXTUHLX (LI8 14), $S)>; + + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, i64:$Idx)))), + (VEXTUWLX (RLWINM8 $Idx, 2, 28, 29), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 0)))), + (VEXTUWLX (LI8 0), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 1)))), + (VEXTUWLX (LI8 4), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 2)))), + (VEXTUWLX (LI8 8), $S)>; + def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 3)))), + (VEXTUWLX (LI8 12), $S)>; + + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, i64:$Idx)))), + (EXTSW (VEXTUWLX (RLWINM8 $Idx, 2, 28, 29), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 0)))), + (EXTSW (VEXTUWLX (LI8 0), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 1)))), + (EXTSW (VEXTUWLX (LI8 4), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 2)))), + (EXTSW (VEXTUWLX (LI8 8), $S))>; + def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 3)))), + (EXTSW (VEXTUWLX (LI8 12), $S))>; +} + let Predicates = [IsLittleEndian, HasDirectMove] in { // v16i8 scalar <-> vector conversions (LE) def : Pat<(v16i8 (scalar_to_vector i32:$A)), @@ -2729,36 +2821,54 @@ def DblToFlt { } def ByteToWord { - dag A0 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 0)), i8)); - dag A1 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 4)), i8)); - dag A2 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 8)), i8)); - dag A3 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 12)), i8)); + dag LE_A0 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 0)), i8)); + dag LE_A1 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 4)), i8)); + dag LE_A2 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 8)), i8)); + dag LE_A3 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 12)), i8)); + dag BE_A0 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 3)), i8)); + dag BE_A1 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 7)), i8)); + dag BE_A2 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 11)), i8)); + dag BE_A3 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 15)), i8)); } def ByteToDWord { - dag A0 = (i64 (sext_inreg - (i64 (anyext (i32 (vector_extract v16i8:$A, 0)))), i8)); - dag A1 = (i64 (sext_inreg - (i64 (anyext (i32 (vector_extract v16i8:$A, 8)))), i8)); + dag LE_A0 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v16i8:$A, 0)))), i8)); + dag LE_A1 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v16i8:$A, 8)))), i8)); + dag BE_A0 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v16i8:$A, 7)))), i8)); + dag BE_A1 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v16i8:$A, 15)))), i8)); } def HWordToWord { - dag A0 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 0)), i16)); - dag A1 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 2)), i16)); - dag A2 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 4)), i16)); - dag A3 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 6)), i16)); + dag LE_A0 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 0)), i16)); + dag LE_A1 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 2)), i16)); + dag LE_A2 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 4)), i16)); + dag LE_A3 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 6)), i16)); + dag BE_A0 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 1)), i16)); + dag BE_A1 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 3)), i16)); + dag BE_A2 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 5)), i16)); + dag BE_A3 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 7)), i16)); } def HWordToDWord { - dag A0 = (i64 (sext_inreg - (i64 (anyext (i32 (vector_extract v8i16:$A, 0)))), i16)); - dag A1 = (i64 (sext_inreg - (i64 (anyext (i32 (vector_extract v8i16:$A, 4)))), i16)); + dag LE_A0 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v8i16:$A, 0)))), i16)); + dag LE_A1 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v8i16:$A, 4)))), i16)); + dag BE_A0 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v8i16:$A, 3)))), i16)); + dag BE_A1 = (i64 (sext_inreg + (i64 (anyext (i32 (vector_extract v8i16:$A, 7)))), i16)); } def WordToDWord { - dag A0 = (i64 (sext (i32 (vector_extract v4i32:$A, 0)))); - dag A1 = (i64 (sext (i32 (vector_extract v4i32:$A, 2)))); + dag LE_A0 = (i64 (sext (i32 (vector_extract v4i32:$A, 0)))); + dag LE_A1 = (i64 (sext (i32 (vector_extract v4i32:$A, 2)))); + dag BE_A0 = (i64 (sext (i32 (vector_extract v4i32:$A, 1)))); + dag BE_A1 = (i64 (sext (i32 (vector_extract v4i32:$A, 3)))); } def FltToIntLoad { @@ -3016,18 +3126,46 @@ let AddedComplexity = 400 in { // P9 Altivec instructions that can be used to build vectors. // Adding them to PPCInstrVSX.td rather than PPCAltivecVSX.td to compete // with complexities of existing build vector patterns in this file. - let Predicates = [HasP9Altivec] in { - def : Pat<(v2i64 (build_vector WordToDWord.A0, WordToDWord.A1)), + let Predicates = [HasP9Altivec, IsLittleEndian] in { + def : Pat<(v2i64 (build_vector WordToDWord.LE_A0, WordToDWord.LE_A1)), (v2i64 (VEXTSW2D $A))>; - def : Pat<(v2i64 (build_vector HWordToDWord.A0, HWordToDWord.A1)), + def : Pat<(v2i64 (build_vector HWordToDWord.LE_A0, HWordToDWord.LE_A1)), (v2i64 (VEXTSH2D $A))>; - def : Pat<(v4i32 (build_vector HWordToWord.A0, HWordToWord.A1, - HWordToWord.A2, HWordToWord.A3)), + def : Pat<(v4i32 (build_vector HWordToWord.LE_A0, HWordToWord.LE_A1, + HWordToWord.LE_A2, HWordToWord.LE_A3)), (v4i32 (VEXTSH2W $A))>; - def : Pat<(v4i32 (build_vector ByteToWord.A0, ByteToWord.A1, - ByteToWord.A2, ByteToWord.A3)), + def : Pat<(v4i32 (build_vector ByteToWord.LE_A0, ByteToWord.LE_A1, + ByteToWord.LE_A2, ByteToWord.LE_A3)), (v4i32 (VEXTSB2W $A))>; - def : Pat<(v2i64 (build_vector ByteToDWord.A0, ByteToDWord.A1)), + def : Pat<(v2i64 (build_vector ByteToDWord.LE_A0, ByteToDWord.LE_A1)), (v2i64 (VEXTSB2D $A))>; } + + let Predicates = [HasP9Altivec, IsBigEndian] in { + def : Pat<(v2i64 (build_vector WordToDWord.BE_A0, WordToDWord.BE_A1)), + (v2i64 (VEXTSW2D $A))>; + def : Pat<(v2i64 (build_vector HWordToDWord.BE_A0, HWordToDWord.BE_A1)), + (v2i64 (VEXTSH2D $A))>; + def : Pat<(v4i32 (build_vector HWordToWord.BE_A0, HWordToWord.BE_A1, + HWordToWord.BE_A2, HWordToWord.BE_A3)), + (v4i32 (VEXTSH2W $A))>; + def : Pat<(v4i32 (build_vector ByteToWord.BE_A0, ByteToWord.BE_A1, + ByteToWord.BE_A2, ByteToWord.BE_A3)), + (v4i32 (VEXTSB2W $A))>; + def : Pat<(v2i64 (build_vector ByteToDWord.BE_A0, ByteToDWord.BE_A1)), + (v2i64 (VEXTSB2D $A))>; + } + + let Predicates = [HasP9Altivec] in { + def: Pat<(v2i64 (PPCSExtVElems v16i8:$A)), + (v2i64 (VEXTSB2D $A))>; + def: Pat<(v2i64 (PPCSExtVElems v8i16:$A)), + (v2i64 (VEXTSH2D $A))>; + def: Pat<(v2i64 (PPCSExtVElems v4i32:$A)), + (v2i64 (VEXTSW2D $A))>; + def: Pat<(v4i32 (PPCSExtVElems v16i8:$A)), + (v4i32 (VEXTSB2W $A))>; + def: Pat<(v4i32 (PPCSExtVElems v8i16:$A)), + (v4i32 (VEXTSH2W $A))>; + } } diff --git a/contrib/llvm/lib/Target/PowerPC/PPCScheduleP9.td b/contrib/llvm/lib/Target/PowerPC/PPCScheduleP9.td index a9c1bd78b05e..a01995a629c2 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCScheduleP9.td +++ b/contrib/llvm/lib/Target/PowerPC/PPCScheduleP9.td @@ -260,8 +260,8 @@ let SchedModel = P9Model in { // ***************** Defining Itinerary Class Resources ***************** - def : ItinRW<[P9_DFU_76C, IP_EXEC_1C, DISP_1C, DISP_1C], [IIC_IntSimple, - IIC_IntGeneral]>; + def : ItinRW<[P9_ALU_2C, IP_EXEC_1C, DISP_1C, DISP_1C], + [IIC_IntSimple, IIC_IntGeneral]>; def : ItinRW<[P9_ALU_2C, IP_EXEC_1C, DISP_1C, DISP_1C, DISP_1C], [IIC_IntISEL, IIC_IntRotate, IIC_IntShift]>; diff --git a/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.h b/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.h index 5a97f595ad8c..90d11f46a384 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.h +++ b/contrib/llvm/lib/Target/PowerPC/PPCSubtarget.h @@ -272,6 +272,13 @@ class PPCSubtarget : public PPCGenSubtargetInfo { return 16; } + + // DarwinABI has a 224-byte red zone. PPC32 SVR4ABI(Non-DarwinABI) has no + // red zone and PPC64 SVR4ABI has a 288-byte red zone. + unsigned getRedZoneSize() const { + return isDarwinABI() ? 224 : (isPPC64() ? 288 : 0); + } + bool hasHTM() const { return HasHTM; } bool hasFusion() const { return HasFusion; } bool hasFloat128() const { return HasFloat128; } diff --git a/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp b/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp index 491eaf326a50..7d34efd4af3e 100644 --- a/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp +++ b/contrib/llvm/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp @@ -195,8 +195,10 @@ struct PPCVSXSwapRemoval : public MachineFunctionPass { return false; // If we don't have VSX on the subtarget, don't do anything. + // Also, on Power 9 the load and store ops preserve element order and so + // the swaps are not required. const PPCSubtarget &STI = MF.getSubtarget(); - if (!STI.hasVSX()) + if (!STI.hasVSX() || !STI.needsSwapsForVSXMemOps()) return false; bool Changed = false; diff --git a/contrib/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/contrib/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp index f85c0cf111c4..be83efc02d27 100644 --- a/contrib/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp +++ b/contrib/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp @@ -34,7 +34,7 @@ class RISCVAsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override; @@ -73,7 +73,7 @@ bool RISCVAsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const { void RISCVAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { return; } diff --git a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp index d4454c271f5a..0d021d67033e 100644 --- a/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp +++ b/contrib/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp @@ -211,6 +211,7 @@ namespace { case Sparc::fixup_sparc_wplt30: if (Target.getSymA()->getSymbol().isTemporary()) return false; + LLVM_FALLTHROUGH; case Sparc::fixup_sparc_tls_gd_hi22: case Sparc::fixup_sparc_tls_gd_lo10: case Sparc::fixup_sparc_tls_gd_add: @@ -275,7 +276,7 @@ namespace { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override { + uint64_t Value, bool IsResolved) const override { Value = adjustFixupValue(Fixup.getKind(), Value); if (!Value) return; // Doesn't change encoding. diff --git a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp index 6b32a7926437..51ac410a9c81 100644 --- a/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp +++ b/contrib/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp @@ -52,7 +52,7 @@ class SystemZMCAsmBackend : public MCAsmBackend { const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override; + uint64_t Value, bool IsResolved) const override; bool mayNeedRelaxation(const MCInst &Inst) const override { return false; } @@ -94,7 +94,7 @@ void SystemZMCAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, - bool IsPCRel) const { + bool IsResolved) const { MCFixupKind Kind = Fixup.getKind(); unsigned Offset = Fixup.getOffset(); unsigned BitSize = getFixupKindInfo(Kind).TargetSize; diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZHazardRecognizer.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZHazardRecognizer.cpp index fe4b52b515e0..73a1036f88e0 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZHazardRecognizer.cpp +++ b/contrib/llvm/lib/Target/SystemZ/SystemZHazardRecognizer.cpp @@ -26,7 +26,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" // This is the limit of processor resource usage at which the // scheduler should try to look for other instructions (not using the diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index fef4a8c92a36..2801141cd951 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -2224,15 +2224,12 @@ static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, // Lower a binary operation that produces two VT results, one in each // half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, -// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation -// on the extended Op0 and (unextended) Op1. Store the even register result +// and Opcode performs the GR128 operation. Store the even register result // in Even and the odd register result in Odd. static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, - unsigned Extend, unsigned Opcode, SDValue Op0, - SDValue Op1, SDValue &Even, SDValue &Odd) { - SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); - SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, - SDValue(In128, 0), Op1); + unsigned Opcode, SDValue Op0, SDValue Op1, + SDValue &Even, SDValue &Odd) { + SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1); bool Is32Bit = is32Bit(VT); Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); @@ -2347,6 +2344,7 @@ static SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL, EVT VT, // Handle tests for order using (or (ogt y x) (oge x y)). case ISD::SETUO: Invert = true; + LLVM_FALLTHROUGH; case ISD::SETO: { assert(IsFP && "Unexpected integer comparison"); SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0); @@ -2358,6 +2356,7 @@ static SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL, EVT VT, // Handle <> tests using (or (ogt y x) (ogt x y)). case ISD::SETUEQ: Invert = true; + LLVM_FALLTHROUGH; case ISD::SETONE: { assert(IsFP && "Unexpected integer comparison"); SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0); @@ -2962,7 +2961,7 @@ SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); else { - // Do a full 128-bit multiplication based on UMUL_LOHI64: + // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI: // // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) // @@ -2980,10 +2979,10 @@ SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, SDValue RL = Op.getOperand(1); SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); - // UMUL_LOHI64 returns the low result in the odd register and the high - // result in the even register. SMUL_LOHI is defined to return the - // low half first, so the results are in reverse order. - lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, + // SystemZISD::UMUL_LOHI returns the low result in the odd register and + // the high result in the even register. ISD::SMUL_LOHI is defined to + // return the low half first, so the results are in reverse order. + lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI, LL, RL, Ops[1], Ops[0]); SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); @@ -3004,10 +3003,10 @@ SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); else - // UMUL_LOHI64 returns the low result in the odd register and the high - // result in the even register. UMUL_LOHI is defined to return the - // low half first, so the results are in reverse order. - lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, + // SystemZISD::UMUL_LOHI returns the low result in the odd register and + // the high result in the even register. ISD::UMUL_LOHI is defined to + // return the low half first, so the results are in reverse order. + lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI, Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); return DAG.getMergeValues(Ops, DL); } @@ -3018,24 +3017,19 @@ SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, SDValue Op1 = Op.getOperand(1); EVT VT = Op.getValueType(); SDLoc DL(Op); - unsigned Opcode; - // We use DSGF for 32-bit division. - if (is32Bit(VT)) { + // We use DSGF for 32-bit division. This means the first operand must + // always be 64-bit, and the second operand should be 32-bit whenever + // that is possible, to improve performance. + if (is32Bit(VT)) Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); - Opcode = SystemZISD::SDIVREM32; - } else if (DAG.ComputeNumSignBits(Op1) > 32) { + else if (DAG.ComputeNumSignBits(Op1) > 32) Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); - Opcode = SystemZISD::SDIVREM32; - } else - Opcode = SystemZISD::SDIVREM64; - // DSG(F) takes a 64-bit dividend, so the even register in the GR128 - // input is "don't care". The instruction returns the remainder in - // the even register and the quotient in the odd register. + // DSG(F) returns the remainder in the even register and the + // quotient in the odd register. SDValue Ops[2]; - lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, - Op0, Op1, Ops[1], Ops[0]); + lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]); return DAG.getMergeValues(Ops, DL); } @@ -3044,16 +3038,11 @@ SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, EVT VT = Op.getValueType(); SDLoc DL(Op); - // DL(G) uses a double-width dividend, so we need to clear the even - // register in the GR128 input. The instruction returns the remainder - // in the even register and the quotient in the odd register. + // DL(G) returns the remainder in the even register and the + // quotient in the odd register. SDValue Ops[2]; - if (is32Bit(VT)) - lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, - Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); - else - lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, - Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); + lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM, + Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); return DAG.getMergeValues(Ops, DL); } @@ -3193,13 +3182,13 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op, SDLoc DL(Op); AtomicOrdering FenceOrdering = static_cast( cast(Op.getOperand(1))->getZExtValue()); - SynchronizationScope FenceScope = static_cast( + SyncScope::ID FenceSSID = static_cast( cast(Op.getOperand(2))->getZExtValue()); // The only fence that needs an instruction is a sequentially-consistent // cross-thread fence. if (FenceOrdering == AtomicOrdering::SequentiallyConsistent && - FenceScope == CrossThread) { + FenceSSID == SyncScope::System) { return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other, Op.getOperand(0)), 0); @@ -4669,11 +4658,9 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { OPCODE(SELECT_CCMASK); OPCODE(ADJDYNALLOC); OPCODE(POPCNT); - OPCODE(UMUL_LOHI64); - OPCODE(SDIVREM32); - OPCODE(SDIVREM64); - OPCODE(UDIVREM32); - OPCODE(UDIVREM64); + OPCODE(UMUL_LOHI); + OPCODE(SDIVREM); + OPCODE(UDIVREM); OPCODE(MVC); OPCODE(MVC_LOOP); OPCODE(NC); @@ -5778,14 +5765,12 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, return DoneMBB; } -// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true +// Emit an extension from a GR64 to a GR128. ClearEven is true // if the high register of the GR128 value must be cleared or false if -// it's "don't care". SubReg is subreg_l32 when extending a GR32 -// and subreg_l64 when extending a GR64. +// it's "don't care". MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI, MachineBasicBlock *MBB, - bool ClearEven, - unsigned SubReg) const { + bool ClearEven) const { MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = static_cast(Subtarget.getInstrInfo()); @@ -5808,7 +5793,7 @@ MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI, In128 = NewIn128; } BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) - .addReg(In128).addReg(Src).addImm(SubReg); + .addReg(In128).addReg(Src).addImm(SystemZ::subreg_l64); MI.eraseFromParent(); return MBB; @@ -6172,12 +6157,10 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter( case SystemZ::CondStoreF64Inv: return emitCondStore(MI, MBB, SystemZ::STD, 0, true); - case SystemZ::AEXT128_64: - return emitExt128(MI, MBB, false, SystemZ::subreg_l64); - case SystemZ::ZEXT128_32: - return emitExt128(MI, MBB, true, SystemZ::subreg_l32); - case SystemZ::ZEXT128_64: - return emitExt128(MI, MBB, true, SystemZ::subreg_l64); + case SystemZ::AEXT128: + return emitExt128(MI, MBB, false); + case SystemZ::ZEXT128: + return emitExt128(MI, MBB, true); case SystemZ::ATOMIC_SWAPW: return emitAtomicLoadBinary(MI, MBB, 0, 0); diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 5dcb19c0a35d..6c9c404816f0 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -86,14 +86,11 @@ enum NodeType : unsigned { // Count number of bits set in operand 0 per byte. POPCNT, - // Wrappers around the ISD opcodes of the same name. The output and - // first input operands are GR128s. The trailing numbers are the - // widths of the second operand in bits. - UMUL_LOHI64, - SDIVREM32, - SDIVREM64, - UDIVREM32, - UDIVREM64, + // Wrappers around the ISD opcodes of the same name. The output is GR128. + // Input operands may be GR64 or GR32, depending on the instruction. + UMUL_LOHI, + SDIVREM, + UDIVREM, // Use a series of MVCs to copy bytes from one memory location to another. // The operands are: @@ -562,7 +559,7 @@ class SystemZTargetLowering : public TargetLowering { unsigned StoreOpcode, unsigned STOCOpcode, bool Invert) const; MachineBasicBlock *emitExt128(MachineInstr &MI, MachineBasicBlock *MBB, - bool ClearEven, unsigned SubReg) const; + bool ClearEven) const; MachineBasicBlock *emitAtomicLoadBinary(MachineInstr &MI, MachineBasicBlock *BB, unsigned BinOpcode, unsigned BitSize, diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td index 98f66c29ae64..4569be7602e4 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td +++ b/contrib/llvm/lib/Target/SystemZ/SystemZInstrInfo.td @@ -677,6 +677,22 @@ let Predicates = [FeatureLoadAndTrap] in { def LLGTAT : UnaryRXY<"llgtat", 0xE39C, null_frag, GR64, 4>; } +// Extend GR64s to GR128s. +let usesCustomInserter = 1 in + def ZEXT128 : Pseudo<(outs GR128:$dst), (ins GR64:$src), []>; + +//===----------------------------------------------------------------------===// +// "Any" extensions +//===----------------------------------------------------------------------===// + +// Use subregs to populate the "don't care" bits in a 32-bit to 64-bit anyext. +def : Pat<(i64 (anyext GR32:$src)), + (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$src, subreg_l32)>; + +// Extend GR64s to GR128s. +let usesCustomInserter = 1 in + def AEXT128 : Pseudo<(outs GR128:$dst), (ins GR64:$src), []>; + //===----------------------------------------------------------------------===// // Truncations //===----------------------------------------------------------------------===// @@ -1216,13 +1232,17 @@ def MSG : BinaryRXY<"msg", 0xE30C, mul, GR64, load, 8>; // Multiplication of a register, producing two results. def MR : BinaryRR <"mr", 0x1C, null_frag, GR128, GR32>; def MLR : BinaryRRE<"mlr", 0xB996, null_frag, GR128, GR32>; -def MLGR : BinaryRRE<"mlgr", 0xB986, z_umul_lohi64, GR128, GR64>; +def MLGR : BinaryRRE<"mlgr", 0xB986, null_frag, GR128, GR64>; +def : Pat<(z_umul_lohi GR64:$src1, GR64:$src2), + (MLGR (AEXT128 GR64:$src1), GR64:$src2)>; // Multiplication of memory, producing two results. def M : BinaryRX <"m", 0x5C, null_frag, GR128, load, 4>; def MFY : BinaryRXY<"mfy", 0xE35C, null_frag, GR128, load, 4>; def ML : BinaryRXY<"ml", 0xE396, null_frag, GR128, load, 4>; -def MLG : BinaryRXY<"mlg", 0xE386, z_umul_lohi64, GR128, load, 8>; +def MLG : BinaryRXY<"mlg", 0xE386, null_frag, GR128, load, 8>; +def : Pat<(z_umul_lohi GR64:$src1, (i64 (load bdxaddr20only:$src2))), + (MLG (AEXT128 GR64:$src1), bdxaddr20only:$src2)>; //===----------------------------------------------------------------------===// // Division and remainder @@ -1230,19 +1250,38 @@ def MLG : BinaryRXY<"mlg", 0xE386, z_umul_lohi64, GR128, load, 8>; let hasSideEffects = 1 in { // Do not speculatively execute. // Division and remainder, from registers. - def DR : BinaryRR <"dr", 0x1D, null_frag, GR128, GR32>; - def DSGFR : BinaryRRE<"dsgfr", 0xB91D, z_sdivrem32, GR128, GR32>; - def DSGR : BinaryRRE<"dsgr", 0xB90D, z_sdivrem64, GR128, GR64>; - def DLR : BinaryRRE<"dlr", 0xB997, z_udivrem32, GR128, GR32>; - def DLGR : BinaryRRE<"dlgr", 0xB987, z_udivrem64, GR128, GR64>; + def DR : BinaryRR <"dr", 0x1D, null_frag, GR128, GR32>; + def DSGFR : BinaryRRE<"dsgfr", 0xB91D, null_frag, GR128, GR32>; + def DSGR : BinaryRRE<"dsgr", 0xB90D, null_frag, GR128, GR64>; + def DLR : BinaryRRE<"dlr", 0xB997, null_frag, GR128, GR32>; + def DLGR : BinaryRRE<"dlgr", 0xB987, null_frag, GR128, GR64>; // Division and remainder, from memory. - def D : BinaryRX <"d", 0x5D, null_frag, GR128, load, 4>; - def DSGF : BinaryRXY<"dsgf", 0xE31D, z_sdivrem32, GR128, load, 4>; - def DSG : BinaryRXY<"dsg", 0xE30D, z_sdivrem64, GR128, load, 8>; - def DL : BinaryRXY<"dl", 0xE397, z_udivrem32, GR128, load, 4>; - def DLG : BinaryRXY<"dlg", 0xE387, z_udivrem64, GR128, load, 8>; + def D : BinaryRX <"d", 0x5D, null_frag, GR128, load, 4>; + def DSGF : BinaryRXY<"dsgf", 0xE31D, null_frag, GR128, load, 4>; + def DSG : BinaryRXY<"dsg", 0xE30D, null_frag, GR128, load, 8>; + def DL : BinaryRXY<"dl", 0xE397, null_frag, GR128, load, 4>; + def DLG : BinaryRXY<"dlg", 0xE387, null_frag, GR128, load, 8>; } +def : Pat<(z_sdivrem GR64:$src1, GR32:$src2), + (DSGFR (AEXT128 GR64:$src1), GR32:$src2)>; +def : Pat<(z_sdivrem GR64:$src1, (i32 (load bdxaddr20only:$src2))), + (DSGF (AEXT128 GR64:$src1), bdxaddr20only:$src2)>; +def : Pat<(z_sdivrem GR64:$src1, GR64:$src2), + (DSGR (AEXT128 GR64:$src1), GR64:$src2)>; +def : Pat<(z_sdivrem GR64:$src1, (i64 (load bdxaddr20only:$src2))), + (DSG (AEXT128 GR64:$src1), bdxaddr20only:$src2)>; + +def : Pat<(z_udivrem GR32:$src1, GR32:$src2), + (DLR (ZEXT128 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$src1, + subreg_l32)), GR32:$src2)>; +def : Pat<(z_udivrem GR32:$src1, (i32 (load bdxaddr20only:$src2))), + (DL (ZEXT128 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$src1, + subreg_l32)), bdxaddr20only:$src2)>; +def : Pat<(z_udivrem GR64:$src1, GR64:$src2), + (DLGR (ZEXT128 GR64:$src1), GR64:$src2)>; +def : Pat<(z_udivrem GR64:$src1, (i64 (load bdxaddr20only:$src2))), + (DLG (ZEXT128 GR64:$src1), bdxaddr20only:$src2)>; //===----------------------------------------------------------------------===// // Shifts @@ -1894,17 +1933,6 @@ def : Pat<(ctlz GR64:$src), let Predicates = [FeaturePopulationCount], Defs = [CC] in def POPCNT : UnaryRRE<"popcnt", 0xB9E1, z_popcnt, GR64, GR64>; -// Use subregs to populate the "don't care" bits in a 32-bit to 64-bit anyext. -def : Pat<(i64 (anyext GR32:$src)), - (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$src, subreg_l32)>; - -// Extend GR32s and GR64s to GR128s. -let usesCustomInserter = 1 in { - def AEXT128_64 : Pseudo<(outs GR128:$dst), (ins GR64:$src), []>; - def ZEXT128_32 : Pseudo<(outs GR128:$dst), (ins GR32:$src), []>; - def ZEXT128_64 : Pseudo<(outs GR128:$dst), (ins GR64:$src), []>; -} - // Search a block of memory for a character. let mayLoad = 1, Defs = [CC] in defm SRST : StringRRE<"srst", 0xB25E, z_search_string>; diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp index 3a0e01da42f0..d4cd89ce590f 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp +++ b/contrib/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp @@ -127,7 +127,7 @@ MachineInstr *SystemZLDCleanup::ReplaceTLSCall(MachineInstr *I, return Copy; } -// Create a virtal register in *TLSBaseAddrReg, and populate it by +// Create a virtual register in *TLSBaseAddrReg, and populate it by // inserting a copy instruction after I. Returns the new instruction. MachineInstr *SystemZLDCleanup::SetRegister(MachineInstr *I, unsigned *TLSBaseAddrReg) { diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp index b6feaa49d858..8342463c1086 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp +++ b/contrib/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp @@ -18,7 +18,7 @@ using namespace llvm; -#define DEBUG_TYPE "misched" +#define DEBUG_TYPE "machine-scheduler" #ifndef NDEBUG // Print the set of SUs diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZOperators.td b/contrib/llvm/lib/Target/SystemZ/SystemZOperators.td index ab2392809f3b..9c6d5819f8a7 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZOperators.td +++ b/contrib/llvm/lib/Target/SystemZ/SystemZOperators.td @@ -36,14 +36,10 @@ def SDT_ZWrapOffset : SDTypeProfile<1, 2, SDTCisSameAs<0, 2>, SDTCisPtrTy<0>]>; def SDT_ZAdjDynAlloc : SDTypeProfile<1, 0, [SDTCisVT<0, i64>]>; -def SDT_ZGR128Binary32 : SDTypeProfile<1, 2, +def SDT_ZGR128Binary : SDTypeProfile<1, 2, [SDTCisVT<0, untyped>, - SDTCisVT<1, untyped>, - SDTCisVT<2, i32>]>; -def SDT_ZGR128Binary64 : SDTypeProfile<1, 2, - [SDTCisVT<0, untyped>, - SDTCisVT<1, untyped>, - SDTCisVT<2, i64>]>; + SDTCisInt<1>, + SDTCisInt<2>]>; def SDT_ZAtomicLoadBinaryW : SDTypeProfile<1, 5, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, @@ -185,11 +181,9 @@ def z_select_ccmask : SDNode<"SystemZISD::SELECT_CCMASK", SDT_ZSelectCCMask, [SDNPInGlue]>; def z_adjdynalloc : SDNode<"SystemZISD::ADJDYNALLOC", SDT_ZAdjDynAlloc>; def z_popcnt : SDNode<"SystemZISD::POPCNT", SDTIntUnaryOp>; -def z_umul_lohi64 : SDNode<"SystemZISD::UMUL_LOHI64", SDT_ZGR128Binary64>; -def z_sdivrem32 : SDNode<"SystemZISD::SDIVREM32", SDT_ZGR128Binary32>; -def z_sdivrem64 : SDNode<"SystemZISD::SDIVREM64", SDT_ZGR128Binary64>; -def z_udivrem32 : SDNode<"SystemZISD::UDIVREM32", SDT_ZGR128Binary32>; -def z_udivrem64 : SDNode<"SystemZISD::UDIVREM64", SDT_ZGR128Binary64>; +def z_umul_lohi : SDNode<"SystemZISD::UMUL_LOHI", SDT_ZGR128Binary>; +def z_sdivrem : SDNode<"SystemZISD::SDIVREM", SDT_ZGR128Binary>; +def z_udivrem : SDNode<"SystemZISD::UDIVREM", SDT_ZGR128Binary>; def z_membarrier : SDNode<"SystemZISD::MEMBARRIER", SDTNone, [SDNPHasChain, SDNPSideEffect]>; diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td b/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td index adc9f2976f87..72543c1eaee2 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td +++ b/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td @@ -15,7 +15,7 @@ def Z13Model : SchedMachineModel { let UnsupportedFeatures = Arch11UnsupportedFeatures.List; - + let IssueWidth = 8; let MicroOpBufferSize = 60; // Issue queues let LoadLatency = 1; // Optimistic load latency. @@ -159,7 +159,7 @@ def : InstRW<[FXb], (instregex "CondReturn$")>; // Select instructions //===----------------------------------------------------------------------===// -// Select pseudo +// Select pseudo def : InstRW<[FXa], (instregex "Select(32|64|32Mux)$")>; // CondStore pseudos @@ -226,7 +226,7 @@ def : InstRW<[LSU, Lat30, GroupAlone], (instregex "MVST$")>; def : InstRW<[FXa, Lat2], (instregex "LOCRMux$")>; def : InstRW<[FXa, Lat2], (instregex "LOC(G|FH)?R(Asm.*)?$")>; -def : InstRW<[FXa, Lat2], (instregex "LOC(G|H)?HI(Asm.*)?$")>; +def : InstRW<[FXa, Lat2], (instregex "LOC(G|H)?HI(Mux|(Asm.*))?$")>; def : InstRW<[FXa, LSU, Lat6], (instregex "LOC(G|FH|Mux)?(Asm.*)?$")>; def : InstRW<[FXb, LSU, Lat5], (instregex "STOC(G|FH|Mux)?(Asm.*)?$")>; @@ -282,7 +282,7 @@ def : InstRW<[LSU, LSU, LSU, LSU, LSU, Lat10, GroupAlone], (instregex "LM(H|Y|G)?$")>; // Load multiple disjoint -def : InstRW<[FXb, Lat30, GroupAlone], (instregex "LMD$")>; +def : InstRW<[LSU, Lat30, GroupAlone], (instregex "LMD$")>; // Store multiple (estimated average of ceil(5/2) FXb ops) def : InstRW<[LSU, LSU, FXb, FXb, FXb, Lat10, @@ -446,13 +446,13 @@ def : InstRW<[FXa, Lat6], (instregex "MS(R|FI)$")>; def : InstRW<[FXa, LSU, Lat12], (instregex "MSG$")>; def : InstRW<[FXa, Lat8], (instregex "MSGR$")>; def : InstRW<[FXa, Lat6], (instregex "MSGF(I|R)$")>; -def : InstRW<[FXa, LSU, Lat15, GroupAlone], (instregex "MLG$")>; -def : InstRW<[FXa, Lat9, GroupAlone], (instregex "MLGR$")>; +def : InstRW<[FXa2, LSU, Lat15, GroupAlone], (instregex "MLG$")>; +def : InstRW<[FXa2, Lat9, GroupAlone], (instregex "MLGR$")>; def : InstRW<[FXa, Lat5], (instregex "MGHI$")>; def : InstRW<[FXa, Lat5], (instregex "MHI$")>; def : InstRW<[FXa, LSU, Lat9], (instregex "MH(Y)?$")>; -def : InstRW<[FXa, Lat7, GroupAlone], (instregex "M(L)?R$")>; -def : InstRW<[FXa, LSU, Lat7, GroupAlone], (instregex "M(FY|L)?$")>; +def : InstRW<[FXa2, Lat7, GroupAlone], (instregex "M(L)?R$")>; +def : InstRW<[FXa2, LSU, Lat7, GroupAlone], (instregex "M(FY|L)?$")>; //===----------------------------------------------------------------------===// // Division and remainder @@ -460,8 +460,8 @@ def : InstRW<[FXa, LSU, Lat7, GroupAlone], (instregex "M(FY|L)?$")>; def : InstRW<[FXa2, FXa2, Lat20, GroupAlone], (instregex "DR$")>; def : InstRW<[FXa2, FXa2, LSU, Lat30, GroupAlone], (instregex "D$")>; -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "DSG(F)?R$")>; -def : InstRW<[LSU, FXa, Lat30, GroupAlone], (instregex "DSG(F)?$")>; +def : InstRW<[FXa2, Lat30, GroupAlone], (instregex "DSG(F)?R$")>; +def : InstRW<[LSU, FXa2, Lat30, GroupAlone], (instregex "DSG(F)?$")>; def : InstRW<[FXa2, FXa2, Lat20, GroupAlone], (instregex "DLR$")>; def : InstRW<[FXa2, FXa2, Lat30, GroupAlone], (instregex "DLGR$")>; def : InstRW<[FXa2, FXa2, LSU, Lat30, GroupAlone], (instregex "DL(G)?$")>; @@ -474,7 +474,8 @@ def : InstRW<[FXa], (instregex "SLL(G|K)?$")>; def : InstRW<[FXa], (instregex "SRL(G|K)?$")>; def : InstRW<[FXa], (instregex "SRA(G|K)?$")>; def : InstRW<[FXa], (instregex "SLA(G|K)?$")>; -def : InstRW<[FXa, FXa, FXa, FXa, Lat8], (instregex "S(L|R)D(A|L)$")>; +def : InstRW<[FXa, FXa, FXa, FXa, LSU, Lat8, GroupAlone], + (instregex "S(L|R)D(A|L)$")>; // Rotate def : InstRW<[FXa, LSU, Lat6], (instregex "RLL(G)?$")>; @@ -537,7 +538,7 @@ def : InstRW<[FXb], (instregex "TMLH(64)?$")>; def : InstRW<[FXb], (instregex "TMLL(64)?$")>; // Compare logical characters under mask -def : InstRW<[FXb, LSU, Lat5], (instregex "CLM(H|Y)?$")>; +def : InstRW<[FXb, LSU, Lat6], (instregex "CLM(H|Y)?$")>; //===----------------------------------------------------------------------===// // Prefetch and execution hint @@ -573,7 +574,7 @@ def : InstRW<[FXa, FXa, FXb, FXb, LSU, FXb, FXb, LSU, LSU, Lat20, GroupAlone], (instregex "CDSG$")>; // Compare and swap and store -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "CSST$")>; +def : InstRW<[FXa, LSU, Lat30], (instregex "CSST$")>; // Perform locked operation def : InstRW<[LSU, Lat30, GroupAlone], (instregex "PLO$")>; @@ -589,36 +590,45 @@ def : InstRW<[LSU, LSU, Lat5, GroupAlone], (instregex "LPD(G)?$")>; // Translate and convert //===----------------------------------------------------------------------===// -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "TR(T|TR)?(E|EOpt)?$")>; -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "TR(T|O)(T|O)(Opt)?$")>; -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "CU(12|14|21|24|41|42)(Opt)?$")>; -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "(CUUTF|CUTFU)(Opt)?$")>; +def : InstRW<[LSU, Lat30, GroupAlone], (instregex "TR$")>; +def : InstRW<[FXa, FXa, FXa, LSU, LSU, Lat30, GroupAlone], (instregex "TRT$")>; +def : InstRW<[FXa, LSU, Lat30], (instregex "TRTR$")>; +def : InstRW<[FXa, Lat30], (instregex "TR(TR)?(T)?(E|EOpt)?$")>; +def : InstRW<[LSU, Lat30], (instregex "TR(T|O)(T|O)(Opt)?$")>; +def : InstRW<[FXa, Lat30], (instregex "CU(12|14|21|24|41|42)(Opt)?$")>; +def : InstRW<[FXa, Lat30], (instregex "(CUUTF|CUTFU)(Opt)?$")>; //===----------------------------------------------------------------------===// // Message-security assist //===----------------------------------------------------------------------===// -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "KM(C|F|O|CTR)?$")>; -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "(KIMD|KLMD|KMAC|PCC|PPNO)$")>; +def : InstRW<[FXa, Lat30], (instregex "KM(C|F|O|CTR)?$")>; +def : InstRW<[FXa, Lat30], (instregex "(KIMD|KLMD|KMAC|PCC|PPNO)$")>; //===----------------------------------------------------------------------===// // Decimal arithmetic //===----------------------------------------------------------------------===// -def : InstRW<[FXb, VecDF, LSU, Lat30, GroupAlone], (instregex "CVB(Y|G)?$")>; -def : InstRW<[FXb, VecDF, FXb, Lat30, GroupAlone], (instregex "CVD(Y|G)?$")>; -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "MV(N|Z|O)$")>; +def : InstRW<[FXb, VecDF, VecDF, LSU, LSU, Lat30, GroupAlone], + (instregex "CVBG$")>; +def : InstRW<[FXb, VecDF, LSU, Lat30, GroupAlone], (instregex "CVB(Y)?$")>; +def : InstRW<[FXb, FXb, FXb, VecDF2, VecDF2, LSU, Lat30, GroupAlone], + (instregex "CVDG$")>; +def : InstRW<[FXb, VecDF, FXb, LSU, Lat30, GroupAlone], (instregex "CVD(Y)?$")>; +def : InstRW<[LSU, Lat10, GroupAlone], (instregex "MVO$")>; +def : InstRW<[LSU, Lat30, GroupAlone], (instregex "MV(N|Z)$")>; def : InstRW<[LSU, Lat30, GroupAlone], (instregex "(PACK|PKA|PKU)$")>; -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "UNPK(A|U)?$")>; +def : InstRW<[LSU, Lat12, GroupAlone], (instregex "UNPK(A|U)$")>; +def : InstRW<[FXb, LSU, LSU, Lat9, BeginGroup], (instregex "UNPK$")>; -def : InstRW<[FXb, VecDFX, LSU, LSU, Lat9, GroupAlone], +def : InstRW<[FXb, VecDFX, LSU, LSU, LSU, Lat9, GroupAlone], (instregex "(A|S|ZA)P$")>; -def : InstRW<[FXb, VecDFX2, LSU, LSU, Lat30, GroupAlone], +def : InstRW<[FXb, VecDFX2, VecDFX2, LSU, LSU, LSU, Lat30, GroupAlone], (instregex "(M|D)P$")>; -def : InstRW<[FXb, FXb, VecDFX2, LSU, LSU, LSU, Lat15, GroupAlone], +def : InstRW<[FXb, VecDFX, VecDFX, LSU, LSU, Lat15, GroupAlone], (instregex "SRP$")>; def : InstRW<[VecDFX, LSU, LSU, Lat5, GroupAlone], (instregex "CP$")>; -def : InstRW<[VecDFX, LSU, Lat4, GroupAlone], (instregex "TP$")>; +def : InstRW<[VecDFX, LSU, Lat4, BeginGroup], (instregex "TP$")>; def : InstRW<[LSU, Lat30, GroupAlone], (instregex "ED(MK)?$")>; //===----------------------------------------------------------------------===// @@ -688,25 +698,25 @@ def : InstRW<[FXb], (instregex "PPA$")>; //===----------------------------------------------------------------------===// // Find leftmost one -def : InstRW<[FXa, Lat6, GroupAlone], (instregex "FLOGR$")>; +def : InstRW<[FXa, FXa, Lat6, GroupAlone], (instregex "FLOGR$")>; // Population count def : InstRW<[FXa, Lat3], (instregex "POPCNT$")>; // Extend -def : InstRW<[FXa], (instregex "AEXT128_64$")>; -def : InstRW<[FXa], (instregex "ZEXT128_(32|64)$")>; +def : InstRW<[FXa], (instregex "AEXT128$")>; +def : InstRW<[FXa], (instregex "ZEXT128$")>; // String instructions def : InstRW<[FXa, LSU, Lat30], (instregex "SRST$")>; -def : InstRW<[LSU, Lat30], (instregex "SRSTU$")>; +def : InstRW<[FXa, Lat30], (instregex "SRSTU$")>; def : InstRW<[LSU, Lat30, GroupAlone], (instregex "CUSE$")>; // Various complex instructions -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "CFC$")>; -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "UPT$")>; -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "CKSM$")>; -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "CMPSC$")>; +def : InstRW<[LSU, Lat30], (instregex "CFC$")>; +def : InstRW<[FXb, LSU, Lat30], (instregex "UPT$")>; +def : InstRW<[LSU, Lat30], (instregex "CKSM$")>; +def : InstRW<[FXa, Lat30], (instregex "CMPSC$")>; // Execute def : InstRW<[FXb, GroupAlone], (instregex "EX(RL)?$")>; @@ -833,7 +843,7 @@ def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "FIXBR(A)?$")>; // Addition def : InstRW<[VecBF, LSU, Lat12], (instregex "A(E|D)B$")>; def : InstRW<[VecBF], (instregex "A(E|D)BR$")>; -def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "AXBR$")>; +def : InstRW<[VecDF2, VecDF2, Lat10, GroupAlone], (instregex "AXBR$")>; // Subtraction def : InstRW<[VecBF, LSU, Lat12], (instregex "S(E|D)B$")>; @@ -848,9 +858,9 @@ def : InstRW<[VecBF2, VecBF2, GroupAlone], (instregex "MXDBR$")>; def : InstRW<[VecDF2, VecDF2, Lat20, GroupAlone], (instregex "MXBR$")>; // Multiply and add / subtract -def : InstRW<[VecBF, LSU, Lat12, GroupAlone], (instregex "M(A|S)EB$")>; +def : InstRW<[VecBF2, LSU, Lat12, GroupAlone], (instregex "M(A|S)EB$")>; def : InstRW<[VecBF, GroupAlone], (instregex "M(A|S)EBR$")>; -def : InstRW<[VecBF, LSU, Lat12, GroupAlone], (instregex "M(A|S)DB$")>; +def : InstRW<[VecBF2, LSU, Lat12, GroupAlone], (instregex "M(A|S)DB$")>; def : InstRW<[VecBF], (instregex "M(A|S)DBR$")>; // Division @@ -859,7 +869,7 @@ def : InstRW<[VecFPd], (instregex "D(E|D)BR$")>; def : InstRW<[VecFPd, VecFPd, GroupAlone], (instregex "DXBR$")>; // Divide to integer -def : InstRW<[VecFPd, Lat30, GroupAlone], (instregex "DI(E|D)BR$")>; +def : InstRW<[VecFPd, Lat30], (instregex "DI(E|D)BR$")>; //===----------------------------------------------------------------------===// // FP: Comparisons @@ -882,8 +892,8 @@ def : InstRW<[FXa, LSU, Lat4, GroupAlone], (instregex "EFPC$")>; def : InstRW<[FXb, LSU, Lat5, GroupAlone], (instregex "STFPC$")>; def : InstRW<[LSU, Lat3, GroupAlone], (instregex "SFPC$")>; def : InstRW<[LSU, LSU, Lat6, GroupAlone], (instregex "LFPC$")>; -def : InstRW<[FXa, Lat30, GroupAlone], (instregex "SFASR$")>; -def : InstRW<[FXa, LSU, Lat30, GroupAlone], (instregex "LFAS$")>; +def : InstRW<[FXa, Lat30], (instregex "SFASR$")>; +def : InstRW<[FXa, LSU, Lat30], (instregex "LFAS$")>; def : InstRW<[FXb, Lat3, GroupAlone], (instregex "SRNM(B|T)?$")>; @@ -904,7 +914,7 @@ def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "LTXR$")>; // Load rounded def : InstRW<[VecBF], (instregex "(LEDR|LRER)$")>; def : InstRW<[VecBF], (instregex "LEXR$")>; -def : InstRW<[VecDF2, VecDF2], (instregex "(LDXR|LRDR)$")>; +def : InstRW<[VecDF2], (instregex "(LDXR|LRDR)$")>; // Load lengthened def : InstRW<[LSU], (instregex "LDE$")>; @@ -955,7 +965,7 @@ def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "FIXR$")>; // Addition def : InstRW<[VecBF, LSU, Lat12], (instregex "A(E|D|U|W)$")>; def : InstRW<[VecBF], (instregex "A(E|D|U|W)R$")>; -def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "AXR$")>; +def : InstRW<[VecDF2, VecDF2, Lat10, GroupAlone], (instregex "AXR$")>; // Subtraction def : InstRW<[VecBF, LSU, Lat12], (instregex "S(E|D|U|W)$")>; @@ -968,16 +978,20 @@ def : InstRW<[VecBF], (instregex "M(D|DE|E|EE)R$")>; def : InstRW<[VecBF2, VecBF2, LSU, Lat12, GroupAlone], (instregex "MXD$")>; def : InstRW<[VecBF2, VecBF2, GroupAlone], (instregex "MXDR$")>; def : InstRW<[VecDF2, VecDF2, Lat20, GroupAlone], (instregex "MXR$")>; -def : InstRW<[VecBF2, VecBF2, LSU, Lat12, GroupAlone], (instregex "MY(H|L)?$")>; -def : InstRW<[VecBF2, VecBF2, GroupAlone], (instregex "MY(H|L)?R$")>; +def : InstRW<[VecBF2, VecBF2, LSU, Lat12, GroupAlone], (instregex "MY$")>; +def : InstRW<[VecBF2, LSU, Lat12, GroupAlone], (instregex "MY(H|L)$")>; +def : InstRW<[VecBF2, VecBF2, GroupAlone], (instregex "MYR$")>; +def : InstRW<[VecBF, GroupAlone], (instregex "MY(H|L)R$")>; // Multiply and add / subtract -def : InstRW<[VecBF, LSU, Lat12, GroupAlone], (instregex "M(A|S)E$")>; +def : InstRW<[VecBF2, LSU, Lat12, GroupAlone], (instregex "M(A|S)E$")>; def : InstRW<[VecBF, GroupAlone], (instregex "M(A|S)ER$")>; -def : InstRW<[VecBF, LSU, Lat12, GroupAlone], (instregex "M(A|S)D$")>; -def : InstRW<[VecBF], (instregex "M(A|S)DR$")>; -def : InstRW<[VecBF2, VecBF2, LSU, Lat12, GroupAlone], (instregex "MAY(H|L)?$")>; -def : InstRW<[VecBF2, VecBF2, GroupAlone], (instregex "MAY(H|L)?R$")>; +def : InstRW<[VecBF2, LSU, Lat12, GroupAlone], (instregex "M(A|S)D$")>; +def : InstRW<[VecBF, GroupAlone], (instregex "M(A|S)DR$")>; +def : InstRW<[VecBF2, LSU, Lat12, GroupAlone], (instregex "MAY(H|L)$")>; +def : InstRW<[VecBF2, VecBF2, LSU, Lat12, GroupAlone], (instregex "MAY$")>; +def : InstRW<[VecBF, GroupAlone], (instregex "MAY(H|L)R$")>; +def : InstRW<[VecBF2, VecBF2, GroupAlone], (instregex "MAYR$")>; // Division def : InstRW<[VecFPd, LSU], (instregex "D(E|D)$")>; @@ -989,8 +1003,8 @@ def : InstRW<[VecFPd, VecFPd, GroupAlone], (instregex "DXR$")>; //===----------------------------------------------------------------------===// // Compare -def : InstRW<[VecXsPm, LSU, Lat8], (instregex "C(E|D)$")>; -def : InstRW<[VecXsPm, Lat4], (instregex "C(E|D)R$")>; +def : InstRW<[VecBF, LSU, Lat12], (instregex "C(E|D)$")>; +def : InstRW<[VecBF], (instregex "C(E|D)R$")>; def : InstRW<[VecDF, VecDF, Lat20, GroupAlone], (instregex "CXR$")>; @@ -1032,7 +1046,7 @@ def : InstRW<[FXb, VecDF, VecDF, Lat30, BeginGroup], (instregex "CL(F|G)XTR$")>; def : InstRW<[FXb, VecDF, Lat9, BeginGroup], (instregex "CD(S|U)TR$")>; def : InstRW<[FXb, FXb, VecDF2, VecDF2, Lat15, GroupAlone], (instregex "CX(S|U)TR$")>; def : InstRW<[FXb, VecDF, Lat12, BeginGroup], (instregex "C(S|U)DTR$")>; -def : InstRW<[FXb, FXb, VecDF2, VecDF2, Lat15, BeginGroup], (instregex "C(S|U)XTR$")>; +def : InstRW<[FXb, FXb, VecDF2, VecDF2, Lat15, GroupAlone], (instregex "C(S|U)XTR$")>; // Convert from / to zoned def : InstRW<[LSU, VecDF, Lat11, BeginGroup], (instregex "CDZT$")>; @@ -1047,7 +1061,7 @@ def : InstRW<[FXb, LSU, VecDF, Lat11, BeginGroup], (instregex "CPDT$")>; def : InstRW<[FXb, LSU, VecDF, VecDF, Lat15, GroupAlone], (instregex "CPXT$")>; // Perform floating-point operation -def : InstRW<[LSU, Lat30, GroupAlone], (instregex "PFPO$")>; +def : InstRW<[FXb, Lat30], (instregex "PFPO$")>; //===----------------------------------------------------------------------===// // DFP: Unary arithmetic @@ -1071,7 +1085,7 @@ def : InstRW<[FXb, VecDF, VecDF, Lat15, BeginGroup], (instregex "ESXTR$")>; // Addition def : InstRW<[VecDF], (instregex "ADTR(A)?$")>; -def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "AXTR(A)?$")>; +def : InstRW<[VecDF2, VecDF2, Lat10, GroupAlone], (instregex "AXTR(A)?$")>; // Subtraction def : InstRW<[VecDF], (instregex "SDTR(A)?$")>; @@ -1090,15 +1104,15 @@ def : InstRW<[VecDF], (instregex "QADTR$")>; def : InstRW<[VecDF2, VecDF2, Lat11, GroupAlone], (instregex "QAXTR$")>; // Reround -def : InstRW<[FXb, VecDF, Lat11], (instregex "RRDTR$")>; +def : InstRW<[FXb, VecDF, Lat11, BeginGroup], (instregex "RRDTR$")>; def : InstRW<[FXb, VecDF2, VecDF2, Lat15, GroupAlone], (instregex "RRXTR$")>; // Shift significand left/right -def : InstRW<[LSU, VecDF, Lat11], (instregex "S(L|R)DT$")>; +def : InstRW<[LSU, VecDF, Lat11, GroupAlone], (instregex "S(L|R)DT$")>; def : InstRW<[LSU, VecDF2, VecDF2, Lat15, GroupAlone], (instregex "S(L|R)XT$")>; // Insert biased exponent -def : InstRW<[FXb, VecDF, Lat11], (instregex "IEDTR$")>; +def : InstRW<[FXb, VecDF, Lat11, BeginGroup], (instregex "IEDTR$")>; def : InstRW<[FXb, VecDF2, VecDF2, Lat15, GroupAlone], (instregex "IEXTR$")>; //===----------------------------------------------------------------------===// @@ -1115,7 +1129,7 @@ def : InstRW<[VecDF], (instregex "CEXTR$")>; // Test Data Class/Group def : InstRW<[LSU, VecDF, Lat11], (instregex "TD(C|G)(E|D)T$")>; -def : InstRW<[LSU, VecDF2, VecDF2, Lat15, GroupAlone], (instregex "TD(C|G)XT$")>; +def : InstRW<[LSU, VecDF, VecDF, Lat15, GroupAlone], (instregex "TD(C|G)XT$")>; // --------------------------------- Vector --------------------------------- // @@ -1271,32 +1285,43 @@ def : InstRW<[VecStr, Lat5], (instregex "VTM$")>; // Vector: Floating-point arithmetic //===----------------------------------------------------------------------===// -def : InstRW<[VecBF2], (instregex "VCD(G|GB|LG|LGB)$")>; -def : InstRW<[VecBF], (instregex "WCD(GB|LGB)$")>; +// Conversion and rounding +def : InstRW<[VecBF2], (instregex "VCD(L)?G$")>; +def : InstRW<[VecBF2], (instregex "VCD(L)?GB$")>; +def : InstRW<[VecBF], (instregex "WCD(L)?GB$")>; def : InstRW<[VecBF2], (instregex "VC(L)?GD$")>; -def : InstRW<[VecBF2], (instregex "VFADB$")>; -def : InstRW<[VecBF], (instregex "WFADB$")>; -def : InstRW<[VecBF2], (instregex "VCGDB$")>; -def : InstRW<[VecBF], (instregex "WCGDB$")>; -def : InstRW<[VecBF2], (instregex "VF(I|M|A|S)$")>; -def : InstRW<[VecBF2], (instregex "VF(I|M|S)DB$")>; -def : InstRW<[VecBF], (instregex "WF(I|M|S)DB$")>; -def : InstRW<[VecBF2], (instregex "VCLGDB$")>; -def : InstRW<[VecBF], (instregex "WCLGDB$")>; -def : InstRW<[VecXsPm], (instregex "VFL(C|N|P)DB$")>; -def : InstRW<[VecXsPm], (instregex "WFL(C|N|P)DB$")>; -def : InstRW<[VecBF2], (instregex "VFM(A|S)$")>; -def : InstRW<[VecBF2], (instregex "VFM(A|S)DB$")>; -def : InstRW<[VecBF], (instregex "WFM(A|S)DB$")>; -def : InstRW<[VecXsPm], (instregex "VFPSO$")>; -def : InstRW<[VecXsPm], (instregex "(V|W)FPSODB$")>; -def : InstRW<[VecXsPm, Lat4], (instregex "VFTCI(DB)?$")>; -def : InstRW<[VecXsPm, Lat4], (instregex "WFTCIDB$")>; +def : InstRW<[VecBF2], (instregex "VC(L)?GDB$")>; +def : InstRW<[VecBF], (instregex "WC(L)?GDB$")>; def : InstRW<[VecBF2], (instregex "VL(DE|ED)$")>; def : InstRW<[VecBF2], (instregex "VL(DE|ED)B$")>; def : InstRW<[VecBF], (instregex "WL(DE|ED)B$")>; +def : InstRW<[VecBF2], (instregex "VFI$")>; +def : InstRW<[VecBF2], (instregex "VFIDB$")>; +def : InstRW<[VecBF], (instregex "WFIDB$")>; -// divide / square root +// Sign operations +def : InstRW<[VecXsPm], (instregex "VFPSO$")>; +def : InstRW<[VecXsPm], (instregex "(V|W)FPSODB$")>; +def : InstRW<[VecXsPm], (instregex "(V|W)FL(C|N|P)DB$")>; + +// Test data class +def : InstRW<[VecXsPm, Lat4], (instregex "VFTCI$")>; +def : InstRW<[VecXsPm, Lat4], (instregex "(V|W)FTCIDB$")>; + +// Add / subtract +def : InstRW<[VecBF2], (instregex "VF(A|S)$")>; +def : InstRW<[VecBF2], (instregex "VF(A|S)DB$")>; +def : InstRW<[VecBF], (instregex "WF(A|S)DB$")>; + +// Multiply / multiply-and-add/subtract +def : InstRW<[VecBF2], (instregex "VFM$")>; +def : InstRW<[VecBF2], (instregex "VFMDB$")>; +def : InstRW<[VecBF], (instregex "WFMDB$")>; +def : InstRW<[VecBF2], (instregex "VFM(A|S)$")>; +def : InstRW<[VecBF2], (instregex "VFM(A|S)DB$")>; +def : InstRW<[VecBF], (instregex "WFM(A|S)DB$")>; + +// Divide / square root def : InstRW<[VecFPd], (instregex "VFD$")>; def : InstRW<[VecFPd], (instregex "(V|W)FDDB$")>; def : InstRW<[VecFPd], (instregex "VFSQ$")>; @@ -1308,10 +1333,10 @@ def : InstRW<[VecFPd], (instregex "(V|W)FSQDB$")>; def : InstRW<[VecXsPm], (instregex "VFC(E|H|HE)$")>; def : InstRW<[VecXsPm], (instregex "VFC(E|H|HE)DB$")>; -def : InstRW<[VecXsPm, Lat4], (instregex "WF(C|K)$")>; def : InstRW<[VecXsPm], (instregex "WFC(E|H|HE)DB$")>; def : InstRW<[VecXsPm, Lat4], (instregex "VFC(E|H|HE)DBS$")>; def : InstRW<[VecXsPm, Lat4], (instregex "WFC(E|H|HE)DBS$")>; +def : InstRW<[VecXsPm, Lat4], (instregex "WF(C|K)$")>; def : InstRW<[VecXsPm, Lat4], (instregex "WF(C|K)DB$")>; //===----------------------------------------------------------------------===// @@ -1351,12 +1376,12 @@ def : InstRW<[VecStr, Lat5], (instregex "VSTRCZ(B|F|H)S$")>; def : InstRW<[FXb, Lat30], (instregex "EPSW$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "LPSW(E)?$")>; -def : InstRW<[FXa, Lat3], (instregex "IPK$")>; -def : InstRW<[LSU], (instregex "SPKA$")>; -def : InstRW<[LSU], (instregex "SSM$")>; -def : InstRW<[FXb], (instregex "ST(N|O)SM$")>; +def : InstRW<[FXa, Lat3, GroupAlone], (instregex "IPK$")>; +def : InstRW<[LSU, EndGroup], (instregex "SPKA$")>; +def : InstRW<[LSU, EndGroup], (instregex "SSM$")>; +def : InstRW<[FXb, LSU, GroupAlone], (instregex "ST(N|O)SM$")>; def : InstRW<[FXa, Lat3], (instregex "IAC$")>; -def : InstRW<[LSU], (instregex "SAC(F)?$")>; +def : InstRW<[LSU, EndGroup], (instregex "SAC(F)?$")>; //===----------------------------------------------------------------------===// // System: Control Register Instructions @@ -1411,14 +1436,14 @@ def : InstRW<[FXb, LSU, Lat30], (instregex "TPROT$")>; def : InstRW<[FXa, FXa, FXb, LSU, Lat8, GroupAlone], (instregex "MVC(K|P|S)$")>; def : InstRW<[FXa, LSU, Lat6, GroupAlone], (instregex "MVC(S|D)K$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "MVCOS$")>; -def : InstRW<[FXb, LSU, Lat30], (instregex "MVPG$")>; +def : InstRW<[LSU, Lat30, GroupAlone], (instregex "MVPG$")>; //===----------------------------------------------------------------------===// // System: Address-Space Instructions //===----------------------------------------------------------------------===// def : InstRW<[FXb, LSU, Lat30], (instregex "LASP$")>; -def : InstRW<[LSU], (instregex "PALB$")>; +def : InstRW<[LSU, GroupAlone], (instregex "PALB$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "PC$")>; def : InstRW<[FXb, Lat30], (instregex "PR$")>; def : InstRW<[FXb, Lat30], (instregex "PT(I)?$")>; @@ -1430,7 +1455,7 @@ def : InstRW<[FXb, Lat20], (instregex "TAR$")>; // System: Linkage-Stack Instructions //===----------------------------------------------------------------------===// -def : InstRW<[FXb, Lat30], (instregex "BAKR$")>; +def : InstRW<[FXb, Lat30, EndGroup], (instregex "BAKR$")>; def : InstRW<[FXb, Lat30], (instregex "EREG(G)?$")>; def : InstRW<[FXb, Lat30], (instregex "(E|M)STA$")>; @@ -1442,13 +1467,13 @@ def : InstRW<[FXb, Lat30], (instregex "PTFF$")>; def : InstRW<[FXb, LSU, Lat20], (instregex "SCK$")>; def : InstRW<[FXb, Lat30], (instregex "SCKPF$")>; def : InstRW<[FXb, LSU, Lat20], (instregex "SCKC$")>; -def : InstRW<[LSU, GroupAlone], (instregex "SPT$")>; +def : InstRW<[LSU, LSU, GroupAlone], (instregex "SPT$")>; def : InstRW<[LSU, LSU, LSU, FXa, FXa, FXb, Lat9, GroupAlone], (instregex "STCK(F)?$")>; def : InstRW<[LSU, LSU, LSU, LSU, FXa, FXa, FXb, FXb, Lat11, GroupAlone], (instregex "STCKE$")>; def : InstRW<[FXb, LSU, Lat9], (instregex "STCKC$")>; -def : InstRW<[LSU, LSU, FXb, Lat3], (instregex "STPT$")>; +def : InstRW<[LSU, LSU, FXb, Lat5, BeginGroup], (instregex "STPT$")>; //===----------------------------------------------------------------------===// // System: CPU-Related Instructions @@ -1459,7 +1484,7 @@ def : InstRW<[FXb, LSU, Lat30], (instregex "STIDP$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "STSI$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "STFL(E)?$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "ECAG$")>; -def : InstRW<[FXb, LSU, Lat30], (instregex "ECTG$")>; +def : InstRW<[FXa, LSU, Lat30], (instregex "ECTG$")>; def : InstRW<[FXb, Lat30], (instregex "PTF$")>; def : InstRW<[FXb, Lat30], (instregex "PCKMO$")>; @@ -1468,7 +1493,7 @@ def : InstRW<[FXb, Lat30], (instregex "PCKMO$")>; //===----------------------------------------------------------------------===// def : InstRW<[FXb, Lat30], (instregex "SVC$")>; -def : InstRW<[FXb], (instregex "MC$")>; +def : InstRW<[FXb, GroupAlone], (instregex "MC$")>; def : InstRW<[FXb, Lat30], (instregex "DIAG$")>; def : InstRW<[FXb], (instregex "TRAC(E|G)$")>; def : InstRW<[FXb, Lat30], (instregex "TRAP(2|4)$")>; @@ -1483,7 +1508,8 @@ def : InstRW<[FXb, LSU, Lat30], (instregex "SIE$")>; def : InstRW<[FXb], (instregex "LPP$")>; def : InstRW<[FXb, Lat30], (instregex "ECPGA$")>; def : InstRW<[FXb, Lat30], (instregex "E(C|P)CTR$")>; -def : InstRW<[FXb, LSU, Lat30], (instregex "L(C|P|S)CTL$")>; +def : InstRW<[FXb, Lat30], (instregex "LCCTL$")>; +def : InstRW<[FXb, LSU, Lat30], (instregex "L(P|S)CTL$")>; def : InstRW<[FXb, LSU, Lat30], (instregex "Q(S|CTR)I$")>; def : InstRW<[FXb, Lat30], (instregex "S(C|P)CTR$")>; diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td b/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td index 128049a09086..e3e1999d8ad8 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td +++ b/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td @@ -627,8 +627,8 @@ def : InstRW<[FXU, Lat7, GroupAlone], (instregex "FLOGR$")>; def : InstRW<[FXU, Lat3], (instregex "POPCNT$")>; // Extend -def : InstRW<[FXU], (instregex "AEXT128_64$")>; -def : InstRW<[FXU], (instregex "ZEXT128_(32|64)$")>; +def : InstRW<[FXU], (instregex "AEXT128$")>; +def : InstRW<[FXU], (instregex "ZEXT128$")>; // String instructions def : InstRW<[FXU, LSU, Lat30], (instregex "SRST$")>; diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td b/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td index 76b378454631..59f37205f412 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td +++ b/contrib/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td @@ -665,8 +665,8 @@ def : InstRW<[FXU, Lat7, GroupAlone], (instregex "FLOGR$")>; def : InstRW<[FXU, Lat3], (instregex "POPCNT$")>; // Extend -def : InstRW<[FXU], (instregex "AEXT128_64$")>; -def : InstRW<[FXU], (instregex "ZEXT128_(32|64)$")>; +def : InstRW<[FXU], (instregex "AEXT128$")>; +def : InstRW<[FXU], (instregex "ZEXT128$")>; // String instructions def : InstRW<[FXU, LSU, Lat30], (instregex "SRST$")>; diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp index ce5c57e0f519..9ac768b2189d 100644 --- a/contrib/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/contrib/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -779,15 +779,14 @@ int SystemZTTIImpl:: getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) { // vlvgp will insert two grs into a vector register, so only count half the // number of instructions. - if (Opcode == Instruction::InsertElement && - Val->getScalarType()->isIntegerTy(64)) + if (Opcode == Instruction::InsertElement && Val->isIntOrIntVectorTy(64)) return ((Index % 2 == 0) ? 1 : 0); if (Opcode == Instruction::ExtractElement) { int Cost = ((Val->getScalarSizeInBits() == 1) ? 2 /*+test-under-mask*/ : 1); // Give a slight penalty for moving out of vector pipeline to FXU unit. - if (Index == 0 && Val->getScalarType()->isIntegerTy()) + if (Index == 0 && Val->isIntOrIntVectorTy()) Cost += 1; return Cost; diff --git a/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp b/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp index ad59f2f40587..00bf02469bdd 100644 --- a/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp +++ b/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp @@ -115,8 +115,8 @@ void WebAssemblyTargetAsmStreamer::emitStackPointer(uint32_t Index) { void WebAssemblyTargetAsmStreamer::emitEndFunc() { OS << "\t.endfunc\n"; } void WebAssemblyTargetAsmStreamer::emitIndirectFunctionType( - StringRef name, SmallVectorImpl &Params, SmallVectorImpl &Results) { - OS << "\t.functype\t" << name; + MCSymbol *Symbol, SmallVectorImpl &Params, SmallVectorImpl &Results) { + OS << "\t.functype\t" << Symbol->getName(); if (Results.empty()) OS << ", void"; else { @@ -171,7 +171,7 @@ void WebAssemblyTargetELFStreamer::emitIndIdx(const MCExpr *Value) { } void WebAssemblyTargetELFStreamer::emitIndirectFunctionType( - StringRef name, SmallVectorImpl &Params, SmallVectorImpl &Results) { + MCSymbol *Symbol, SmallVectorImpl &Params, SmallVectorImpl &Results) { // Nothing to emit here. TODO: Re-design how linking works and re-evaluate // whether it's necessary for .o files to declare indirect function types. } @@ -255,9 +255,25 @@ void WebAssemblyTargetWasmStreamer::emitIndIdx(const MCExpr *Value) { } void WebAssemblyTargetWasmStreamer::emitIndirectFunctionType( - StringRef name, SmallVectorImpl &Params, SmallVectorImpl &Results) { - // Nothing to emit here. TODO: Re-design how linking works and re-evaluate - // whether it's necessary for .o files to declare indirect function types. + MCSymbol *Symbol, SmallVectorImpl &Params, + SmallVectorImpl &Results) { + MCSymbolWasm *WasmSym = cast(Symbol); + if (WasmSym->isFunction()) { + // Symbol already has its arguments and result set. + return; + } + + SmallVector ValParams; + for (MVT Ty : Params) + ValParams.push_back(WebAssembly::toValType(Ty)); + + SmallVector ValResults; + for (MVT Ty : Results) + ValResults.push_back(WebAssembly::toValType(Ty)); + + WasmSym->setParams(std::move(ValParams)); + WasmSym->setReturns(std::move(ValResults)); + WasmSym->setIsFunction(true); } void WebAssemblyTargetWasmStreamer::emitGlobalImport(StringRef name) { diff --git a/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h b/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h index 5ad147e5e596..102d7219a1e7 100644 --- a/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h +++ b/contrib/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h @@ -44,7 +44,7 @@ class WebAssemblyTargetStreamer : public MCTargetStreamer { /// .endfunc virtual void emitEndFunc() = 0; /// .functype - virtual void emitIndirectFunctionType(StringRef name, + virtual void emitIndirectFunctionType(MCSymbol *Symbol, SmallVectorImpl &Params, SmallVectorImpl &Results) = 0; /// .indidx @@ -69,7 +69,7 @@ class WebAssemblyTargetAsmStreamer final : public WebAssemblyTargetStreamer { void emitGlobal(ArrayRef Globals) override; void emitStackPointer(uint32_t Index) override; void emitEndFunc() override; - void emitIndirectFunctionType(StringRef name, + void emitIndirectFunctionType(MCSymbol *Symbol, SmallVectorImpl &Params, SmallVectorImpl &Results) override; void emitIndIdx(const MCExpr *Value) override; @@ -87,7 +87,7 @@ class WebAssemblyTargetELFStreamer final : public WebAssemblyTargetStreamer { void emitGlobal(ArrayRef Globals) override; void emitStackPointer(uint32_t Index) override; void emitEndFunc() override; - void emitIndirectFunctionType(StringRef name, + void emitIndirectFunctionType(MCSymbol *Symbol, SmallVectorImpl &Params, SmallVectorImpl &Results) override; void emitIndIdx(const MCExpr *Value) override; @@ -105,7 +105,7 @@ class WebAssemblyTargetWasmStreamer final : public WebAssemblyTargetStreamer { void emitGlobal(ArrayRef Globals) override; void emitStackPointer(uint32_t Index) override; void emitEndFunc() override; - void emitIndirectFunctionType(StringRef name, + void emitIndirectFunctionType(MCSymbol *Symbol, SmallVectorImpl &Params, SmallVectorImpl &Results) override; void emitIndIdx(const MCExpr *Value) override; diff --git a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index f51585a10ca1..211358ad66cd 100644 --- a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -84,7 +84,7 @@ void WebAssemblyAsmPrinter::EmitEndOfAsmFile(Module &M) { SmallVector Results; SmallVector Params; ComputeSignatureVTs(F, TM, Params, Results); - getTargetStreamer()->emitIndirectFunctionType(F.getName(), Params, + getTargetStreamer()->emitIndirectFunctionType(getSymbol(&F), Params, Results); } } @@ -214,11 +214,8 @@ void WebAssemblyAsmPrinter::EmitInstruction(const MachineInstr *MI) { const MCExpr *WebAssemblyAsmPrinter::lowerConstant(const Constant *CV) { if (const GlobalValue *GV = dyn_cast(CV)) if (GV->getValueType()->isFunctionTy()) { - MCSymbol* Sym = getSymbol(GV); - if (!isa(Sym)) - cast(Sym)->setIsFunction(true); return MCSymbolRefExpr::create( - Sym, MCSymbolRefExpr::VK_WebAssembly_FUNCTION, OutContext); + getSymbol(GV), MCSymbolRefExpr::VK_WebAssembly_FUNCTION, OutContext); } return AsmPrinter::lowerConstant(CV); } diff --git a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp index 1691808d05a0..700111743ee8 100644 --- a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp +++ b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp @@ -132,7 +132,7 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI, // no blocks not dominated by the loop header. // - It's desirable to preserve the original block order when possible. // We use two ready lists; Preferred and Ready. Preferred has recently - // processed sucessors, to help preserve block sequences from the original + // processed successors, to help preserve block sequences from the original // order. Ready has the remaining ready blocks. PriorityQueue, CompareBlockNumbers> diff --git a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp index ff186eb91503..8880539804ca 100644 --- a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp +++ b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp @@ -112,8 +112,6 @@ MCOperand WebAssemblyMCInstLower::LowerSymbolOperand(MCSymbol *Sym, MCSymbolRefExpr::VariantKind VK = IsFunc ? MCSymbolRefExpr::VK_WebAssembly_FUNCTION : MCSymbolRefExpr::VK_None; - if (!isa(Sym)) - cast(Sym)->setIsFunction(IsFunc); const MCExpr *Expr = MCSymbolRefExpr::create(Sym, VK, Ctx); diff --git a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp index c02ef4a1c399..2599064334ee 100644 --- a/contrib/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp +++ b/contrib/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp @@ -394,11 +394,22 @@ RuntimeLibcallSignatures[RTLIB::UNKNOWN_LIBCALL] = { /* MEMMOVE */ iPTR_func_iPTR_iPTR_iPTR, // ELEMENT-WISE ATOMIC MEMORY -/* MEMCPY_ELEMENT_ATOMIC_1 */ iPTR_func_iPTR_iPTR_iPTR, -/* MEMCPY_ELEMENT_ATOMIC_2 */ iPTR_func_iPTR_iPTR_iPTR, -/* MEMCPY_ELEMENT_ATOMIC_4 */ iPTR_func_iPTR_iPTR_iPTR, -/* MEMCPY_ELEMENT_ATOMIC_8 */ iPTR_func_iPTR_iPTR_iPTR, -/* MEMCPY_ELEMENT_ATOMIC_16 */ iPTR_func_iPTR_iPTR_iPTR, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_1 */ unsupported, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_2 */ unsupported, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_4 */ unsupported, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_8 */ unsupported, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_16 */ unsupported, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1 */ unsupported, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2 */ unsupported, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4 */ unsupported, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8 */ unsupported, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16 */ unsupported, + +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_1 */ unsupported, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_2 */ unsupported, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_4 */ unsupported, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_8 */ unsupported, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_16 */ unsupported, // EXCEPTION HANDLING /* UNWIND_RESUME */ unsupported, @@ -839,11 +850,21 @@ RuntimeLibcallNames[RTLIB::UNKNOWN_LIBCALL] = { /* MEMCPY */ "memcpy", /* MEMMOVE */ "memset", /* MEMSET */ "memmove", -/* MEMCPY_ELEMENT_ATOMIC_1 */ "MEMCPY_ELEMENT_ATOMIC_1", -/* MEMCPY_ELEMENT_ATOMIC_2 */ "MEMCPY_ELEMENT_ATOMIC_2", -/* MEMCPY_ELEMENT_ATOMIC_4 */ "MEMCPY_ELEMENT_ATOMIC_4", -/* MEMCPY_ELEMENT_ATOMIC_8 */ "MEMCPY_ELEMENT_ATOMIC_8", -/* MEMCPY_ELEMENT_ATOMIC_16 */ "MEMCPY_ELEMENT_ATOMIC_16", +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_1 */ nullptr, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_2 */ nullptr, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_4 */ nullptr, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_8 */ nullptr, +/* MEMCPY_ELEMENT_UNORDERED_ATOMIC_16 */ nullptr, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1 */ nullptr, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2 */ nullptr, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4 */ nullptr, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8 */ nullptr, +/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16 */ nullptr, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_1 */ nullptr, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_2 */ nullptr, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_4 */ nullptr, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_8 */ nullptr, +/* MEMSET_ELEMENT_UNORDERED_ATOMIC_16 */ nullptr, /* UNWIND_RESUME */ "_Unwind_Resume", /* SYNC_VAL_COMPARE_AND_SWAP_1 */ "__sync_val_compare_and_swap_1", /* SYNC_VAL_COMPARE_AND_SWAP_2 */ "__sync_val_compare_and_swap_2", diff --git a/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index 825f23dc52d9..c1d216c8b7af 100644 --- a/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/contrib/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -2453,8 +2453,8 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, break; } - // In MS inline asm curly braces mark the begining/end of a block, therefore - // they should be interepreted as end of statement + // In MS inline asm curly braces mark the beginning/end of a block, + // therefore they should be interepreted as end of statement CurlyAsEndOfStatement = isParsingIntelSyntax() && isParsingInlineAsm() && (getLexer().is(AsmToken::LCurly) || getLexer().is(AsmToken::RCurly)); diff --git a/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp b/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp index 5e809c34325e..f5f3a4cc83dc 100644 --- a/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp +++ b/contrib/llvm/lib/Target/X86/InstPrinter/X86InstComments.cpp @@ -1038,7 +1038,7 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::EXTRQI: if (MI->getOperand(2).isImm() && MI->getOperand(3).isImm()) - DecodeEXTRQIMask(MI->getOperand(2).getImm(), + DecodeEXTRQIMask(MVT::v16i8, MI->getOperand(2).getImm(), MI->getOperand(3).getImm(), ShuffleMask); @@ -1049,7 +1049,7 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, case X86::INSERTQI: if (MI->getOperand(3).isImm() && MI->getOperand(4).isImm()) - DecodeINSERTQIMask(MI->getOperand(3).getImm(), + DecodeINSERTQIMask(MVT::v16i8, MI->getOperand(3).getImm(), MI->getOperand(4).getImm(), ShuffleMask); diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index 914fb36f91a7..733eac7c0321 100644 --- a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -110,7 +110,7 @@ class X86AsmBackend : public MCAsmBackend { void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, - uint64_t Value, bool IsPCRel) const override { + uint64_t Value, bool IsResolved) const override { unsigned Size = 1 << getFixupKindLog2Size(Fixup.getKind()); assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!"); diff --git a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp index 1be5aec849fc..8a0fbfb45b22 100644 --- a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp +++ b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.cpp @@ -452,15 +452,20 @@ void DecodeScalarMoveMask(MVT VT, bool IsLoad, SmallVectorImpl &Mask) { Mask.push_back(IsLoad ? static_cast(SM_SentinelZero) : i); } -void DecodeEXTRQIMask(int Len, int Idx, +void DecodeEXTRQIMask(MVT VT, int Len, int Idx, SmallVectorImpl &ShuffleMask) { + assert(VT.is128BitVector() && "Expected 128-bit vector"); + unsigned NumElts = VT.getVectorNumElements(); + unsigned EltSize = VT.getScalarSizeInBits(); + unsigned HalfElts = NumElts / 2; + // Only the bottom 6 bits are valid for each immediate. Len &= 0x3F; Idx &= 0x3F; // We can only decode this bit extraction instruction as a shuffle if both the - // length and index work with whole bytes. - if (0 != (Len % 8) || 0 != (Idx % 8)) + // length and index work with whole elements. + if (0 != (Len % EltSize) || 0 != (Idx % EltSize)) return; // A length of zero is equivalent to a bit length of 64. @@ -469,33 +474,38 @@ void DecodeEXTRQIMask(int Len, int Idx, // If the length + index exceeds the bottom 64 bits the result is undefined. if ((Len + Idx) > 64) { - ShuffleMask.append(16, SM_SentinelUndef); + ShuffleMask.append(NumElts, SM_SentinelUndef); return; } - // Convert index and index to work with bytes. - Len /= 8; - Idx /= 8; + // Convert index and index to work with elements. + Len /= EltSize; + Idx /= EltSize; - // EXTRQ: Extract Len bytes starting from Idx. Zero pad the remaining bytes - // of the lower 64-bits. The upper 64-bits are undefined. + // EXTRQ: Extract Len elements starting from Idx. Zero pad the remaining + // elements of the lower 64-bits. The upper 64-bits are undefined. for (int i = 0; i != Len; ++i) ShuffleMask.push_back(i + Idx); - for (int i = Len; i != 8; ++i) + for (int i = Len; i != (int)HalfElts; ++i) ShuffleMask.push_back(SM_SentinelZero); - for (int i = 8; i != 16; ++i) + for (int i = HalfElts; i != (int)NumElts; ++i) ShuffleMask.push_back(SM_SentinelUndef); } -void DecodeINSERTQIMask(int Len, int Idx, +void DecodeINSERTQIMask(MVT VT, int Len, int Idx, SmallVectorImpl &ShuffleMask) { + assert(VT.is128BitVector() && "Expected 128-bit vector"); + unsigned NumElts = VT.getVectorNumElements(); + unsigned EltSize = VT.getScalarSizeInBits(); + unsigned HalfElts = NumElts / 2; + // Only the bottom 6 bits are valid for each immediate. Len &= 0x3F; Idx &= 0x3F; // We can only decode this bit insertion instruction as a shuffle if both the - // length and index work with whole bytes. - if (0 != (Len % 8) || 0 != (Idx % 8)) + // length and index work with whole elements. + if (0 != (Len % EltSize) || 0 != (Idx % EltSize)) return; // A length of zero is equivalent to a bit length of 64. @@ -504,24 +514,24 @@ void DecodeINSERTQIMask(int Len, int Idx, // If the length + index exceeds the bottom 64 bits the result is undefined. if ((Len + Idx) > 64) { - ShuffleMask.append(16, SM_SentinelUndef); + ShuffleMask.append(NumElts, SM_SentinelUndef); return; } - // Convert index and index to work with bytes. - Len /= 8; - Idx /= 8; + // Convert index and index to work with elements. + Len /= EltSize; + Idx /= EltSize; - // INSERTQ: Extract lowest Len bytes from lower half of second source and - // insert over first source starting at Idx byte. The upper 64-bits are + // INSERTQ: Extract lowest Len elements from lower half of second source and + // insert over first source starting at Idx element. The upper 64-bits are // undefined. for (int i = 0; i != Idx; ++i) ShuffleMask.push_back(i); for (int i = 0; i != Len; ++i) - ShuffleMask.push_back(i + 16); - for (int i = Idx + Len; i != 8; ++i) + ShuffleMask.push_back(i + NumElts); + for (int i = Idx + Len; i != (int)HalfElts; ++i) ShuffleMask.push_back(i); - for (int i = 8; i != 16; ++i) + for (int i = HalfElts; i != (int)NumElts; ++i) ShuffleMask.push_back(SM_SentinelUndef); } diff --git a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h index 17619d09d059..251c9f7558ec 100644 --- a/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h +++ b/contrib/llvm/lib/Target/X86/Utils/X86ShuffleDecode.h @@ -134,12 +134,12 @@ void DecodeZeroMoveLowMask(MVT VT, SmallVectorImpl &ShuffleMask); void DecodeScalarMoveMask(MVT VT, bool IsLoad, SmallVectorImpl &ShuffleMask); -/// Decode a SSE4A EXTRQ instruction as a v16i8 shuffle mask. -void DecodeEXTRQIMask(int Len, int Idx, +/// Decode a SSE4A EXTRQ instruction as a shuffle mask. +void DecodeEXTRQIMask(MVT VT, int Len, int Idx, SmallVectorImpl &ShuffleMask); -/// Decode a SSE4A INSERTQ instruction as a v16i8 shuffle mask. -void DecodeINSERTQIMask(int Len, int Idx, +/// Decode a SSE4A INSERTQ instruction as a shuffle mask. +void DecodeINSERTQIMask(MVT VT, int Len, int Idx, SmallVectorImpl &ShuffleMask); /// Decode a VPERMILPD/VPERMILPS variable mask from a raw array of constants. diff --git a/contrib/llvm/lib/Target/X86/X86.td b/contrib/llvm/lib/Target/X86/X86.td index 7437ebacfac3..4ca57fe9fb00 100644 --- a/contrib/llvm/lib/Target/X86/X86.td +++ b/contrib/llvm/lib/Target/X86/X86.td @@ -451,6 +451,7 @@ class GoldmontProc : ProcessorModel &SplitArgs, const DataLayout &DL, MachineRegisterInfo &MRI, @@ -43,14 +44,24 @@ void X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, const X86TargetLowering &TLI = *getTLI(); LLVMContext &Context = OrigArg.Ty->getContext(); - EVT VT = TLI.getValueType(DL, OrigArg.Ty); + + SmallVector SplitVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); + + if (SplitVTs.size() != 1) { + // TODO: support struct/array split + return false; + } + + EVT VT = SplitVTs[0]; unsigned NumParts = TLI.getNumRegisters(Context, VT); if (NumParts == 1) { // replace the original type ( pointer -> GPR ). SplitArgs.emplace_back(OrigArg.Reg, VT.getTypeForEVT(Context), OrigArg.Flags, OrigArg.IsFixed); - return; + return true; } SmallVector SplitRegs; @@ -67,6 +78,7 @@ void X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, } PerformArgSplit(SplitRegs); + return true; } namespace { @@ -113,9 +125,11 @@ bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, setArgFlags(OrigArg, AttributeList::ReturnIndex, DL, F); SmallVector SplitArgs; - splitToValueTypes( - OrigArg, SplitArgs, DL, MRI, - [&](ArrayRef Regs) { MIRBuilder.buildUnmerge(Regs, VReg); }); + if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, + [&](ArrayRef Regs) { + MIRBuilder.buildUnmerge(Regs, VReg); + })) + return false; FuncReturnHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86); if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) @@ -181,12 +195,23 @@ bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, SmallVector SplitArgs; unsigned Idx = 0; for (auto &Arg : F.args()) { + + // TODO: handle not simple cases. + if (Arg.hasAttribute(Attribute::ByVal) || + Arg.hasAttribute(Attribute::InReg) || + Arg.hasAttribute(Attribute::StructRet) || + Arg.hasAttribute(Attribute::SwiftSelf) || + Arg.hasAttribute(Attribute::SwiftError) || + Arg.hasAttribute(Attribute::Nest)) + return false; + ArgInfo OrigArg(VRegs[Idx], Arg.getType()); - setArgFlags(OrigArg, Idx + 1, DL, F); - splitToValueTypes(OrigArg, SplitArgs, DL, MRI, - [&](ArrayRef Regs) { - MIRBuilder.buildMerge(VRegs[Idx], Regs); - }); + setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); + if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, + [&](ArrayRef Regs) { + MIRBuilder.buildMerge(VRegs[Idx], Regs); + })) + return false; Idx++; } diff --git a/contrib/llvm/lib/Target/X86/X86CallLowering.h b/contrib/llvm/lib/Target/X86/X86CallLowering.h index 8a8afb568298..6a5dabf33a0a 100644 --- a/contrib/llvm/lib/Target/X86/X86CallLowering.h +++ b/contrib/llvm/lib/Target/X86/X86CallLowering.h @@ -39,7 +39,7 @@ class X86CallLowering : public CallLowering { /// A function of this type is used to perform value split action. typedef std::function)> SplitArgTy; - void splitToValueTypes(const ArgInfo &OrigArgInfo, + bool splitToValueTypes(const ArgInfo &OrigArgInfo, SmallVectorImpl &SplitArgs, const DataLayout &DL, MachineRegisterInfo &MRI, SplitArgTy SplitArg) const; diff --git a/contrib/llvm/lib/Target/X86/X86CallingConv.td b/contrib/llvm/lib/Target/X86/X86CallingConv.td index 7d146d050a5c..6decb550ad5f 100644 --- a/contrib/llvm/lib/Target/X86/X86CallingConv.td +++ b/contrib/llvm/lib/Target/X86/X86CallingConv.td @@ -651,7 +651,15 @@ def CC_X86_64_GHC : CallingConv<[ // Pass in STG registers: F1, F2, F3, F4, D1, D2 CCIfType<[f32, f64, v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], CCIfSubtarget<"hasSSE1()", - CCAssignToReg<[XMM1, XMM2, XMM3, XMM4, XMM5, XMM6]>>> + CCAssignToReg<[XMM1, XMM2, XMM3, XMM4, XMM5, XMM6]>>>, + // AVX + CCIfType<[v32i8, v16i16, v8i32, v4i64, v8f32, v4f64], + CCIfSubtarget<"hasAVX()", + CCAssignToReg<[YMM1, YMM2, YMM3, YMM4, YMM5, YMM6]>>>, + // AVX-512 + CCIfType<[v64i8, v32i16, v16i32, v8i64, v16f32, v8f64], + CCIfSubtarget<"hasAVX512()", + CCAssignToReg<[ZMM1, ZMM2, ZMM3, ZMM4, ZMM5, ZMM6]>>> ]>; def CC_X86_64_HiPE : CallingConv<[ diff --git a/contrib/llvm/lib/Target/X86/X86FastISel.cpp b/contrib/llvm/lib/Target/X86/X86FastISel.cpp index 621505aaded9..ee9e78146305 100644 --- a/contrib/llvm/lib/Target/X86/X86FastISel.cpp +++ b/contrib/llvm/lib/Target/X86/X86FastISel.cpp @@ -3039,6 +3039,9 @@ bool X86FastISel::fastLowerArguments() { if (!Subtarget->is64Bit()) return false; + if (Subtarget->useSoftFloat()) + return false; + // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments. unsigned GPRCnt = 0; unsigned FPRCnt = 0; diff --git a/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp b/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp index e3aa227702be..f294e819090b 100644 --- a/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -972,7 +972,6 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, X86FI->setCalleeSavedFrameSize( X86FI->getCalleeSavedFrameSize() - TailCallReturnAddrDelta); - bool UseRedZone = false; bool UseStackProbe = !STI.getTargetLowering()->getStackProbeSymbolName(MF).empty(); // The default stack probe size is 4096 if the function has no stackprobesize @@ -1011,7 +1010,6 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0); StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0); MFI.setStackSize(StackSize); - UseRedZone = true; } // Insert stack pointer adjustment for later moving of return addr. Only @@ -1189,7 +1187,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, if (IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF)) AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign); if (AlignedNumBytes >= StackProbeSize && UseStackProbe) { - assert(!UseRedZone && "The Red Zone is not accounted for in stack probes"); + assert(!X86FI->getUsesRedZone() && + "The Red Zone is not accounted for in stack probes"); // Check whether EAX is livein for this block. bool isEAXAlive = isEAXLiveIn(MBB); diff --git a/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp b/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp index b89914f8893e..65486cf7f529 100644 --- a/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/contrib/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -4217,6 +4217,8 @@ static bool isTargetShuffle(unsigned Opcode) { case X86ISD::PSHUFLW: case X86ISD::SHUFP: case X86ISD::INSERTPS: + case X86ISD::EXTRQI: + case X86ISD::INSERTQI: case X86ISD::PALIGNR: case X86ISD::VSHLDQ: case X86ISD::VSRLDQ: @@ -5554,6 +5556,24 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero, DecodeINSERTPSMask(cast(ImmN)->getZExtValue(), Mask); IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); break; + case X86ISD::EXTRQI: + if (isa(N->getOperand(1)) && + isa(N->getOperand(2))) { + int BitLen = N->getConstantOperandVal(1); + int BitIdx = N->getConstantOperandVal(2); + DecodeEXTRQIMask(VT, BitLen, BitIdx, Mask); + IsUnary = true; + } + break; + case X86ISD::INSERTQI: + if (isa(N->getOperand(2)) && + isa(N->getOperand(3))) { + int BitLen = N->getConstantOperandVal(2); + int BitIdx = N->getConstantOperandVal(3); + DecodeINSERTQIMask(VT, BitLen, BitIdx, Mask); + IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); + } + break; case X86ISD::UNPCKH: DecodeUNPCKHMask(VT, Mask); IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); @@ -9317,11 +9337,11 @@ static SDValue lowerVectorShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1, return DAG.getBitcast(VT, V); } -/// \brief Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. -static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - const APInt &Zeroable, - SelectionDAG &DAG) { +// EXTRQ: Extract Len elements from lower half of source, starting at Idx. +// Remainder of lower half result is zero and upper half is all undef. +static bool matchVectorShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2, + ArrayRef Mask, uint64_t &BitLen, + uint64_t &BitIdx, const APInt &Zeroable) { int Size = Mask.size(); int HalfSize = Size / 2; assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); @@ -9329,120 +9349,133 @@ static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, // Upper half must be undefined. if (!isUndefInRange(Mask, HalfSize, HalfSize)) - return SDValue(); + return false; - // EXTRQ: Extract Len elements from lower half of source, starting at Idx. - // Remainder of lower half result is zero and upper half is all undef. - auto LowerAsEXTRQ = [&]() { - // Determine the extraction length from the part of the - // lower half that isn't zeroable. - int Len = HalfSize; - for (; Len > 0; --Len) - if (!Zeroable[Len - 1]) - break; - assert(Len > 0 && "Zeroable shuffle mask"); + // Determine the extraction length from the part of the + // lower half that isn't zeroable. + int Len = HalfSize; + for (; Len > 0; --Len) + if (!Zeroable[Len - 1]) + break; + assert(Len > 0 && "Zeroable shuffle mask"); - // Attempt to match first Len sequential elements from the lower half. - SDValue Src; - int Idx = -1; - for (int i = 0; i != Len; ++i) { - int M = Mask[i]; - if (M < 0) - continue; - SDValue &V = (M < Size ? V1 : V2); - M = M % Size; + // Attempt to match first Len sequential elements from the lower half. + SDValue Src; + int Idx = -1; + for (int i = 0; i != Len; ++i) { + int M = Mask[i]; + if (M == SM_SentinelUndef) + continue; + SDValue &V = (M < Size ? V1 : V2); + M = M % Size; - // The extracted elements must start at a valid index and all mask - // elements must be in the lower half. - if (i > M || M >= HalfSize) - return SDValue(); + // The extracted elements must start at a valid index and all mask + // elements must be in the lower half. + if (i > M || M >= HalfSize) + return false; - if (Idx < 0 || (Src == V && Idx == (M - i))) { - Src = V; - Idx = M - i; - continue; - } - return SDValue(); + if (Idx < 0 || (Src == V && Idx == (M - i))) { + Src = V; + Idx = M - i; + continue; + } + return false; + } + + if (!Src || Idx < 0) + return false; + + assert((Idx + Len) <= HalfSize && "Illegal extraction mask"); + BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; + BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; + V1 = Src; + return true; +} + +// INSERTQ: Extract lowest Len elements from lower half of second source and +// insert over first source, starting at Idx. +// { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... } +static bool matchVectorShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2, + ArrayRef Mask, uint64_t &BitLen, + uint64_t &BitIdx) { + int Size = Mask.size(); + int HalfSize = Size / 2; + assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); + + // Upper half must be undefined. + if (!isUndefInRange(Mask, HalfSize, HalfSize)) + return false; + + for (int Idx = 0; Idx != HalfSize; ++Idx) { + SDValue Base; + + // Attempt to match first source from mask before insertion point. + if (isUndefInRange(Mask, 0, Idx)) { + /* EMPTY */ + } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) { + Base = V1; + } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) { + Base = V2; + } else { + continue; } - if (Idx < 0) - return SDValue(); + // Extend the extraction length looking to match both the insertion of + // the second source and the remaining elements of the first. + for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) { + SDValue Insert; + int Len = Hi - Idx; - assert((Idx + Len) <= HalfSize && "Illegal extraction mask"); - int BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; - int BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; - return DAG.getNode(X86ISD::EXTRQI, DL, VT, Src, - DAG.getConstant(BitLen, DL, MVT::i8), - DAG.getConstant(BitIdx, DL, MVT::i8)); - }; + // Match insertion. + if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) { + Insert = V1; + } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) { + Insert = V2; + } else { + continue; + } - if (SDValue ExtrQ = LowerAsEXTRQ()) - return ExtrQ; - - // INSERTQ: Extract lowest Len elements from lower half of second source and - // insert over first source, starting at Idx. - // { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... } - auto LowerAsInsertQ = [&]() { - for (int Idx = 0; Idx != HalfSize; ++Idx) { - SDValue Base; - - // Attempt to match first source from mask before insertion point. - if (isUndefInRange(Mask, 0, Idx)) { + // Match the remaining elements of the lower half. + if (isUndefInRange(Mask, Hi, HalfSize - Hi)) { /* EMPTY */ - } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) { + } else if ((!Base || (Base == V1)) && + isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) { Base = V1; - } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) { + } else if ((!Base || (Base == V2)) && + isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, + Size + Hi)) { Base = V2; } else { continue; } - // Extend the extraction length looking to match both the insertion of - // the second source and the remaining elements of the first. - for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) { - SDValue Insert; - int Len = Hi - Idx; - - // Match insertion. - if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) { - Insert = V1; - } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) { - Insert = V2; - } else { - continue; - } - - // Match the remaining elements of the lower half. - if (isUndefInRange(Mask, Hi, HalfSize - Hi)) { - /* EMPTY */ - } else if ((!Base || (Base == V1)) && - isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) { - Base = V1; - } else if ((!Base || (Base == V2)) && - isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, - Size + Hi)) { - Base = V2; - } else { - continue; - } - - // We may not have a base (first source) - this can safely be undefined. - if (!Base) - Base = DAG.getUNDEF(VT); - - int BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; - int BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; - return DAG.getNode(X86ISD::INSERTQI, DL, VT, Base, Insert, - DAG.getConstant(BitLen, DL, MVT::i8), - DAG.getConstant(BitIdx, DL, MVT::i8)); - } + BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; + BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; + V1 = Base; + V2 = Insert; + return true; } + } - return SDValue(); - }; + return false; +} - if (SDValue InsertQ = LowerAsInsertQ()) - return InsertQ; +/// \brief Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. +static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + const APInt &Zeroable, + SelectionDAG &DAG) { + uint64_t BitLen, BitIdx; + if (matchVectorShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable)) + return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1, + DAG.getConstant(BitLen, DL, MVT::i8), + DAG.getConstant(BitIdx, DL, MVT::i8)); + + if (matchVectorShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx)) + return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT), + V2 ? V2 : DAG.getUNDEF(VT), + DAG.getConstant(BitLen, DL, MVT::i8), + DAG.getConstant(BitIdx, DL, MVT::i8)); return SDValue(); } @@ -22817,7 +22850,7 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const { auto Builder = IRBuilder<>(AI); Module *M = Builder.GetInsertBlock()->getParent()->getParent(); - auto SynchScope = AI->getSynchScope(); + auto SSID = AI->getSyncScopeID(); // We must restrict the ordering to avoid generating loads with Release or // ReleaseAcquire orderings. auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering()); @@ -22839,7 +22872,7 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const { // otherwise, we might be able to be more aggressive on relaxed idempotent // rmw. In practice, they do not look useful, so we don't try to be // especially clever. - if (SynchScope == SingleThread) + if (SSID == SyncScope::SingleThread) // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at // the IR level, so we must wrap it in an intrinsic. return nullptr; @@ -22858,7 +22891,7 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const { // Finally we can emit the atomic load. LoadInst *Loaded = Builder.CreateAlignedLoad(Ptr, AI->getType()->getPrimitiveSizeInBits()); - Loaded->setAtomic(Order, SynchScope); + Loaded->setAtomic(Order, SSID); AI->replaceAllUsesWith(Loaded); AI->eraseFromParent(); return Loaded; @@ -22869,13 +22902,13 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget, SDLoc dl(Op); AtomicOrdering FenceOrdering = static_cast( cast(Op.getOperand(1))->getZExtValue()); - SynchronizationScope FenceScope = static_cast( + SyncScope::ID FenceSSID = static_cast( cast(Op.getOperand(2))->getZExtValue()); // The only fence that needs an instruction is a sequentially-consistent // cross-thread fence. if (FenceOrdering == AtomicOrdering::SequentiallyConsistent && - FenceScope == CrossThread) { + FenceSSID == SyncScope::System) { if (Subtarget.hasMFence()) return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); @@ -23203,6 +23236,20 @@ static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget, SDLoc DL(Op.getNode()); SDValue Op0 = Op.getOperand(0); + // TRUNC(CTPOP(ZEXT(X))) to make use of vXi32/vXi64 VPOPCNT instructions. + if (Subtarget.hasVPOPCNTDQ()) { + if (VT == MVT::v8i16) { + Op = DAG.getNode(X86ISD::VZEXT, DL, MVT::v8i64, Op0); + Op = DAG.getNode(ISD::CTPOP, DL, MVT::v8i64, Op); + return DAG.getNode(X86ISD::VTRUNC, DL, VT, Op); + } + if (VT == MVT::v16i8 || VT == MVT::v16i16) { + Op = DAG.getNode(X86ISD::VZEXT, DL, MVT::v16i32, Op0); + Op = DAG.getNode(ISD::CTPOP, DL, MVT::v16i32, Op); + return DAG.getNode(X86ISD::VTRUNC, DL, VT, Op); + } + } + if (!Subtarget.hasSSSE3()) { // We can't use the fast LUT approach, so fall back on vectorized bitmath. assert(VT.is128BitVector() && "Only 128-bit vectors supported in SSE!"); @@ -27101,6 +27148,7 @@ static bool matchUnaryVectorShuffle(MVT MaskVT, ArrayRef Mask, // permute instructions. // TODO: Investigate sharing more of this with shuffle lowering. static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef Mask, + const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const X86Subtarget &Subtarget, @@ -27111,38 +27159,67 @@ static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef Mask, unsigned MaskScalarSizeInBits = InputSizeInBits / NumMaskElts; MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits); - bool ContainsZeros = false; - APInt Zeroable(NumMaskElts, false); - for (unsigned i = 0; i != NumMaskElts; ++i) { - int M = Mask[i]; - if (isUndefOrZero(M)) - Zeroable.setBit(i); - ContainsZeros |= (M == SM_SentinelZero); - } + bool ContainsZeros = + llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; }); - // Attempt to match against byte/bit shifts. - // FIXME: Add 512-bit support. - if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || - (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) { - int ShiftAmt = matchVectorShuffleAsShift(ShuffleVT, Shuffle, - MaskScalarSizeInBits, Mask, - 0, Zeroable, Subtarget); - if (0 < ShiftAmt) { - PermuteImm = (unsigned)ShiftAmt; + // Handle VPERMI/VPERMILPD vXi64/vXi64 patterns. + if (!ContainsZeros && MaskScalarSizeInBits == 64) { + // Check for lane crossing permutes. + if (is128BitLaneCrossingShuffleMask(MaskEltVT, Mask)) { + // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+). + if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) { + Shuffle = X86ISD::VPERMI; + ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64); + PermuteImm = getV4X86ShuffleImm(Mask); + return true; + } + if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) { + SmallVector RepeatedMask; + if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) { + Shuffle = X86ISD::VPERMI; + ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64); + PermuteImm = getV4X86ShuffleImm(RepeatedMask); + return true; + } + } + } else if (AllowFloatDomain && Subtarget.hasAVX()) { + // VPERMILPD can permute with a non-repeating shuffle. + Shuffle = X86ISD::VPERMILPI; + ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size()); + PermuteImm = 0; + for (int i = 0, e = Mask.size(); i != e; ++i) { + int M = Mask[i]; + if (M == SM_SentinelUndef) + continue; + assert(((M / 2) == (i / 2)) && "Out of range shuffle mask index"); + PermuteImm |= (M & 1) << i; + } return true; } } - // Ensure we don't contain any zero elements. - if (ContainsZeros) - return false; + // Handle PSHUFD/VPERMILPI vXi32/vXf32 repeated patterns. + // AVX introduced the VPERMILPD/VPERMILPS float permutes, before then we + // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here). + if ((MaskScalarSizeInBits == 64 || MaskScalarSizeInBits == 32) && + !ContainsZeros && (AllowIntDomain || Subtarget.hasAVX())) { + SmallVector RepeatedMask; + if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) { + // Narrow the repeated mask to create 32-bit element permutes. + SmallVector WordMask = RepeatedMask; + if (MaskScalarSizeInBits == 64) + scaleShuffleMask(2, RepeatedMask, WordMask); - assert(llvm::all_of(Mask, [&](int M) { - return SM_SentinelUndef <= M && M < (int)NumMaskElts; - }) && "Expected unary shuffle"); + Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI); + ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32); + ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32); + PermuteImm = getV4X86ShuffleImm(WordMask); + return true; + } + } - // Handle PSHUFLW/PSHUFHW repeated patterns. - if (MaskScalarSizeInBits == 16) { + // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns. + if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16) { SmallVector RepeatedMask; if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) { ArrayRef LoMask(Mask.data() + 0, 4); @@ -27170,78 +27247,23 @@ static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef Mask, PermuteImm = getV4X86ShuffleImm(OffsetHiMask); return true; } - - return false; } - return false; } - // We only support permutation of 32/64 bit elements after this. - if (MaskScalarSizeInBits != 32 && MaskScalarSizeInBits != 64) - return false; - - // AVX introduced the VPERMILPD/VPERMILPS float permutes, before then we - // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here). - if ((AllowFloatDomain && !AllowIntDomain) && !Subtarget.hasAVX()) - return false; - - // Pre-AVX2 we must use float shuffles on 256-bit vectors. - if (MaskVT.is256BitVector() && !Subtarget.hasAVX2()) { - AllowFloatDomain = true; - AllowIntDomain = false; - } - - // Check for lane crossing permutes. - if (is128BitLaneCrossingShuffleMask(MaskEltVT, Mask)) { - // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+). - if (Subtarget.hasAVX2() && MaskVT.is256BitVector() && Mask.size() == 4) { - Shuffle = X86ISD::VPERMI; - ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64); - PermuteImm = getV4X86ShuffleImm(Mask); + // Attempt to match against byte/bit shifts. + // FIXME: Add 512-bit support. + if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || + (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) { + int ShiftAmt = matchVectorShuffleAsShift(ShuffleVT, Shuffle, + MaskScalarSizeInBits, Mask, + 0, Zeroable, Subtarget); + if (0 < ShiftAmt) { + PermuteImm = (unsigned)ShiftAmt; return true; } - if (Subtarget.hasAVX512() && MaskVT.is512BitVector() && Mask.size() == 8) { - SmallVector RepeatedMask; - if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) { - Shuffle = X86ISD::VPERMI; - ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64); - PermuteImm = getV4X86ShuffleImm(RepeatedMask); - return true; - } - } - return false; } - // VPERMILPD can permute with a non-repeating shuffle. - if (AllowFloatDomain && MaskScalarSizeInBits == 64) { - Shuffle = X86ISD::VPERMILPI; - ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size()); - PermuteImm = 0; - for (int i = 0, e = Mask.size(); i != e; ++i) { - int M = Mask[i]; - if (M == SM_SentinelUndef) - continue; - assert(((M / 2) == (i / 2)) && "Out of range shuffle mask index"); - PermuteImm |= (M & 1) << i; - } - return true; - } - - // We need a repeating shuffle mask for VPERMILPS/PSHUFD. - SmallVector RepeatedMask; - if (!is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) - return false; - - // Narrow the repeated mask for 32-bit element permutes. - SmallVector WordMask = RepeatedMask; - if (MaskScalarSizeInBits == 64) - scaleShuffleMask(2, RepeatedMask, WordMask); - - Shuffle = (AllowFloatDomain ? X86ISD::VPERMILPI : X86ISD::PSHUFD); - ShuffleVT = (AllowFloatDomain ? MVT::f32 : MVT::i32); - ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32); - PermuteImm = getV4X86ShuffleImm(WordMask); - return true; + return false; } // Attempt to match a combined unary shuffle mask against supported binary @@ -27303,6 +27325,7 @@ static bool matchBinaryVectorShuffle(MVT MaskVT, ArrayRef Mask, } static bool matchBinaryPermuteVectorShuffle(MVT MaskVT, ArrayRef Mask, + const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, SDLoc &DL, @@ -27388,11 +27411,6 @@ static bool matchBinaryPermuteVectorShuffle(MVT MaskVT, ArrayRef Mask, // Attempt to combine to INSERTPS. if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() && MaskVT.is128BitVector()) { - APInt Zeroable(4, 0); - for (unsigned i = 0; i != NumMaskElts; ++i) - if (Mask[i] < 0) - Zeroable.setBit(i); - if (Zeroable.getBoolValue() && matchVectorShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) { Shuffle = X86ISD::INSERTPS; @@ -27578,7 +27596,14 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, // Which shuffle domains are permitted? // Permit domain crossing at higher combine depths. bool AllowFloatDomain = FloatDomain || (Depth > 3); - bool AllowIntDomain = !FloatDomain || (Depth > 3); + bool AllowIntDomain = (!FloatDomain || (Depth > 3)) && + (!MaskVT.is256BitVector() || Subtarget.hasAVX2()); + + // Determine zeroable mask elements. + APInt Zeroable(NumMaskElts, 0); + for (unsigned i = 0; i != NumMaskElts; ++i) + if (isUndefOrZero(Mask[i])) + Zeroable.setBit(i); if (UnaryShuffle) { // If we are shuffling a X86ISD::VZEXT_LOAD then we can use the load @@ -27612,7 +27637,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, return true; } - if (matchUnaryPermuteVectorShuffle(MaskVT, Mask, AllowFloatDomain, + if (matchUnaryPermuteVectorShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, Subtarget, Shuffle, ShuffleVT, PermuteImm)) { if (Depth == 1 && Root.getOpcode() == Shuffle) @@ -27648,7 +27673,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, return true; } - if (matchBinaryPermuteVectorShuffle(MaskVT, Mask, AllowFloatDomain, + if (matchBinaryPermuteVectorShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, V1, V2, DL, DAG, Subtarget, Shuffle, ShuffleVT, PermuteImm)) { @@ -27668,6 +27693,45 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, return true; } + // Typically from here on, we need an integer version of MaskVT. + MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits); + IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts); + + // Annoyingly, SSE4A instructions don't map into the above match helpers. + if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) { + uint64_t BitLen, BitIdx; + if (matchVectorShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx, + Zeroable)) { + if (Depth == 1 && Root.getOpcode() == X86ISD::EXTRQI) + return false; // Nothing to do! + V1 = DAG.getBitcast(IntMaskVT, V1); + DCI.AddToWorklist(V1.getNode()); + Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1, + DAG.getConstant(BitLen, DL, MVT::i8), + DAG.getConstant(BitIdx, DL, MVT::i8)); + DCI.AddToWorklist(Res.getNode()); + DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Res), + /*AddTo*/ true); + return true; + } + + if (matchVectorShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) { + if (Depth == 1 && Root.getOpcode() == X86ISD::INSERTQI) + return false; // Nothing to do! + V1 = DAG.getBitcast(IntMaskVT, V1); + DCI.AddToWorklist(V1.getNode()); + V2 = DAG.getBitcast(IntMaskVT, V2); + DCI.AddToWorklist(V2.getNode()); + Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2, + DAG.getConstant(BitLen, DL, MVT::i8), + DAG.getConstant(BitIdx, DL, MVT::i8)); + DCI.AddToWorklist(Res.getNode()); + DCI.CombineTo(Root.getNode(), DAG.getBitcast(RootVT, Res), + /*AddTo*/ true); + return true; + } + } + // Don't try to re-form single instruction chains under any circumstances now // that we've done encoding canonicalization for them. if (Depth < 2) @@ -27688,9 +27752,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { - MVT VPermMaskSVT = MVT::getIntegerVT(MaskEltSizeInBits); - MVT VPermMaskVT = MVT::getVectorVT(VPermMaskSVT, NumMaskElts); - SDValue VPermMask = getConstVector(Mask, VPermMaskVT, DAG, DL, true); + SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); DCI.AddToWorklist(VPermMask.getNode()); Res = DAG.getBitcast(MaskVT, V1); DCI.AddToWorklist(Res.getNode()); @@ -27719,9 +27781,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, if (Mask[i] == SM_SentinelZero) Mask[i] = NumMaskElts + i; - MVT VPermMaskSVT = MVT::getIntegerVT(MaskEltSizeInBits); - MVT VPermMaskVT = MVT::getVectorVT(VPermMaskSVT, NumMaskElts); - SDValue VPermMask = getConstVector(Mask, VPermMaskVT, DAG, DL, true); + SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); DCI.AddToWorklist(VPermMask.getNode()); Res = DAG.getBitcast(MaskVT, V1); DCI.AddToWorklist(Res.getNode()); @@ -27746,9 +27806,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { - MVT VPermMaskSVT = MVT::getIntegerVT(MaskEltSizeInBits); - MVT VPermMaskVT = MVT::getVectorVT(VPermMaskSVT, NumMaskElts); - SDValue VPermMask = getConstVector(Mask, VPermMaskVT, DAG, DL, true); + SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); DCI.AddToWorklist(VPermMask.getNode()); V1 = DAG.getBitcast(MaskVT, V1); DCI.AddToWorklist(V1.getNode()); @@ -27807,8 +27865,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32); VPermIdx.push_back(Idx); } - MVT VPermMaskVT = MVT::getVectorVT(MVT::i32, NumMaskElts); - SDValue VPermMask = DAG.getBuildVector(VPermMaskVT, DL, VPermIdx); + SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx); DCI.AddToWorklist(VPermMask.getNode()); Res = DAG.getBitcast(MaskVT, V1); DCI.AddToWorklist(Res.getNode()); @@ -27831,8 +27888,6 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, unsigned NumLanes = MaskVT.getSizeInBits() / 128; unsigned NumEltsPerLane = NumMaskElts / NumLanes; SmallVector VPerm2Idx; - MVT MaskIdxSVT = MVT::getIntegerVT(MaskVT.getScalarSizeInBits()); - MVT MaskIdxVT = MVT::getVectorVT(MaskIdxSVT, NumMaskElts); unsigned M2ZImm = 0; for (int M : Mask) { if (M == SM_SentinelUndef) { @@ -27852,7 +27907,7 @@ static bool combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, DCI.AddToWorklist(V1.getNode()); V2 = DAG.getBitcast(MaskVT, V2); DCI.AddToWorklist(V2.getNode()); - SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, MaskIdxVT, DAG, DL, true); + SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true); DCI.AddToWorklist(VPerm2MaskOp.getNode()); Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp, DAG.getConstant(M2ZImm, DL, MVT::i8)); @@ -29163,9 +29218,9 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast, // v8i16 and v16i16. // For these two cases, we can shuffle the upper element bytes to a // consecutive sequence at the start of the vector and treat the results as - // v16i8 or v32i8, and for v61i8 this is the prefferable solution. However, + // v16i8 or v32i8, and for v61i8 this is the preferable solution. However, // for v16i16 this is not the case, because the shuffle is expensive, so we - // avoid sign-exteding to this type entirely. + // avoid sign-extending to this type entirely. // For example, t0 := (v8i16 sext(v8i1 x)) needs to be shuffled as: // (v16i8 shuffle <0,2,4,6,8,10,12,14,u,u,...,u> (v16i8 bitcast t0), undef) MVT SExtVT; @@ -29207,7 +29262,7 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast, SExtVT = MVT::v16i8; // For the case (i16 bitcast (v16i1 setcc v16i16 v1, v2)), // it is not profitable to sign-extend to 256-bit because this will - // require an extra cross-lane shuffle which is more exprensive than + // require an extra cross-lane shuffle which is more expensive than // truncating the result of the compare to 128-bits. break; case MVT::v32i1: @@ -29580,8 +29635,8 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG, // (extends the sign bit which is zero). // So it is correct to skip the sign/zero extend instruction. if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND || - Root.getOpcode() == ISD::ZERO_EXTEND || - Root.getOpcode() == ISD::ANY_EXTEND)) + Root.getOpcode() == ISD::ZERO_EXTEND || + Root.getOpcode() == ISD::ANY_EXTEND)) Root = Root.getOperand(0); // If there was a match, we want Root to be a select that is the root of an @@ -34950,6 +35005,40 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) { EVT VT = N->getValueType(0); X86::CondCode CC = (X86::CondCode)Y.getConstantOperandVal(0); + // If X is -1 or 0, then we have an opportunity to avoid constants required in + // the general case below. + auto *ConstantX = dyn_cast(X); + if (ConstantX) { + if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnesValue()) || + (IsSub && CC == X86::COND_B && ConstantX->isNullValue())) { + // This is a complicated way to get -1 or 0 from the carry flag: + // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax + // 0 - SETB --> 0 - (CF) --> CF ? -1 : 0 --> SBB %eax, %eax + return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, + DAG.getConstant(X86::COND_B, DL, MVT::i8), + Y.getOperand(1)); + } + + if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnesValue()) || + (IsSub && CC == X86::COND_A && ConstantX->isNullValue())) { + SDValue EFLAGS = Y->getOperand(1); + if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() && + EFLAGS.getValueType().isInteger() && + !isa(EFLAGS.getOperand(1))) { + // Swap the operands of a SUB, and we have the same pattern as above. + // -1 + SETBE (SUB A, B) --> -1 + SETAE (SUB B, A) --> SUB + SBB + // 0 - SETA (SUB A, B) --> 0 - SETB (SUB B, A) --> SUB + SBB + SDValue NewSub = DAG.getNode( + X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), + EFLAGS.getOperand(1), EFLAGS.getOperand(0)); + SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo()); + return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, + DAG.getConstant(X86::COND_B, DL, MVT::i8), + NewEFLAGS); + } + } + } + if (CC == X86::COND_B) { // X + SETB Z --> X + (mask SBB Z, Z) // X - SETB Z --> X - (mask SBB Z, Z) @@ -34996,7 +35085,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) { // If X is -1 or 0, then we have an opportunity to avoid constants required in // the general case below. - if (auto *ConstantX = dyn_cast(X)) { + if (ConstantX) { // 'neg' sets the carry flag when Z != 0, so create 0 or -1 using 'sbb' with // fake operands: // 0 - (Z != 0) --> sbb %eax, %eax, (neg Z) @@ -35549,6 +35638,8 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget); case X86ISD::SHUFP: // Handle all target specific shuffles case X86ISD::INSERTPS: + case X86ISD::EXTRQI: + case X86ISD::INSERTQI: case X86ISD::PALIGNR: case X86ISD::VSHLDQ: case X86ISD::VSRLDQ: diff --git a/contrib/llvm/lib/Target/X86/X86ISelLowering.h b/contrib/llvm/lib/Target/X86/X86ISelLowering.h index e1ade92979dc..dbbc2bbba6a4 100644 --- a/contrib/llvm/lib/Target/X86/X86ISelLowering.h +++ b/contrib/llvm/lib/Target/X86/X86ISelLowering.h @@ -767,6 +767,19 @@ namespace llvm { SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; + // Return true if it is profitable to combine a BUILD_VECTOR to a TRUNCATE + // for given operand and result types. + // Example of such a combine: + // v4i32 build_vector((extract_elt V, 0), + // (extract_elt V, 2), + // (extract_elt V, 4), + // (extract_elt V, 6)) + // --> + // v4i32 truncate (bitcast V to v4i64) + bool isDesirableToCombineBuildVectorToTruncate() const override { + return true; + } + /// Return true if the target has native support for /// the specified value type and it is 'desirable' to use the type for the /// given node type. e.g. On x86 i16 is legal, but undesirable since i16 diff --git a/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp b/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp index f3094b781c49..34d4816a2518 100644 --- a/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/contrib/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -10488,7 +10488,7 @@ namespace { return Copy; } - // Create a virtal register in *TLSBaseAddrReg, and populate it by + // Create a virtual register in *TLSBaseAddrReg, and populate it by // inserting a copy instruction after I. Returns the new instruction. MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { MachineFunction *MF = I.getParent()->getParent(); diff --git a/contrib/llvm/lib/Target/X86/X86InstructionSelector.cpp b/contrib/llvm/lib/Target/X86/X86InstructionSelector.cpp index e34a90e975b8..859d3288db89 100644 --- a/contrib/llvm/lib/Target/X86/X86InstructionSelector.cpp +++ b/contrib/llvm/lib/Target/X86/X86InstructionSelector.cpp @@ -32,6 +32,8 @@ #define DEBUG_TYPE "X86-isel" +#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" + using namespace llvm; #ifndef LLVM_BUILD_GLOBAL_ISEL @@ -56,7 +58,7 @@ class X86InstructionSelector : public InstructionSelector { /// the patterns that don't require complex C++. bool selectImpl(MachineInstr &I) const; - // TODO: remove after suported by Tablegen-erated instruction selection. + // TODO: remove after supported by Tablegen-erated instruction selection. unsigned getLoadStoreOp(LLT &Ty, const RegisterBank &RB, unsigned Opc, uint64_t Alignment) const; @@ -64,6 +66,8 @@ class X86InstructionSelector : public InstructionSelector { MachineFunction &MF) const; bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; + bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI, + MachineFunction &MF) const; bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; bool selectTrunc(MachineInstr &I, MachineRegisterInfo &MRI, @@ -75,6 +79,8 @@ class X86InstructionSelector : public InstructionSelector { bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; + bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI, + MachineFunction &MF) const; bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, @@ -262,6 +268,8 @@ bool X86InstructionSelector::select(MachineInstr &I) const { return true; if (selectFrameIndexOrGep(I, MRI, MF)) return true; + if (selectGlobalValue(I, MRI, MF)) + return true; if (selectConstant(I, MRI, MF)) return true; if (selectTrunc(I, MRI, MF)) @@ -272,6 +280,8 @@ bool X86InstructionSelector::select(MachineInstr &I) const { return true; if (selectUadde(I, MRI, MF)) return true; + if (selectUnmergeValues(I, MRI, MF)) + return true; if (selectMergeValues(I, MRI, MF)) return true; if (selectExtract(I, MRI, MF)) @@ -423,6 +433,15 @@ bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } +static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) { + if (Ty == LLT::pointer(0, 64)) + return X86::LEA64r; + else if (Ty == LLT::pointer(0, 32)) + return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; + else + llvm_unreachable("Can't get LEA opcode. Unsupported type."); +} + bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { @@ -435,14 +454,7 @@ bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, LLT Ty = MRI.getType(DefReg); // Use LEA to calculate frame index and GEP - unsigned NewOpc; - if (Ty == LLT::pointer(0, 64)) - NewOpc = X86::LEA64r; - else if (Ty == LLT::pointer(0, 32)) - NewOpc = STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; - else - llvm_unreachable("Can't select G_FRAME_INDEX/G_GEP, unsupported type."); - + unsigned NewOpc = getLeaOP(Ty, STI); I.setDesc(TII.get(NewOpc)); MachineInstrBuilder MIB(MF, I); @@ -458,6 +470,54 @@ bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } +bool X86InstructionSelector::selectGlobalValue(MachineInstr &I, + MachineRegisterInfo &MRI, + MachineFunction &MF) const { + unsigned Opc = I.getOpcode(); + + if (Opc != TargetOpcode::G_GLOBAL_VALUE) + return false; + + auto GV = I.getOperand(1).getGlobal(); + if (GV->isThreadLocal()) { + return false; // TODO: we don't support TLS yet. + } + + // Can't handle alternate code models yet. + if (TM.getCodeModel() != CodeModel::Small) + return 0; + + X86AddressMode AM; + AM.GV = GV; + AM.GVOpFlags = STI.classifyGlobalReference(GV); + + // TODO: The ABI requires an extra load. not supported yet. + if (isGlobalStubReference(AM.GVOpFlags)) + return false; + + // TODO: This reference is relative to the pic base. not supported yet. + if (isGlobalRelativeToPICBase(AM.GVOpFlags)) + return false; + + if (STI.isPICStyleRIPRel()) { + // Use rip-relative addressing. + assert(AM.Base.Reg == 0 && AM.IndexReg == 0); + AM.Base.Reg = X86::RIP; + } + + const unsigned DefReg = I.getOperand(0).getReg(); + LLT Ty = MRI.getType(DefReg); + unsigned NewOpc = getLeaOP(Ty, STI); + + I.setDesc(TII.get(NewOpc)); + MachineInstrBuilder MIB(MF, I); + + I.RemoveOperand(1); + addFullAddress(MIB, AM); + + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); +} + bool X86InstructionSelector::selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { @@ -467,7 +527,8 @@ bool X86InstructionSelector::selectConstant(MachineInstr &I, const unsigned DefReg = I.getOperand(0).getReg(); LLT Ty = MRI.getType(DefReg); - assert(Ty.isScalar() && "invalid element type."); + if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID) + return false; uint64_t Val = 0; if (I.getOperand(1).isCImm()) { @@ -576,37 +637,40 @@ bool X86InstructionSelector::selectZext(MachineInstr &I, const LLT DstTy = MRI.getType(DstReg); const LLT SrcTy = MRI.getType(SrcReg); - if (SrcTy == LLT::scalar(1)) { + if (SrcTy != LLT::scalar(1)) + return false; - unsigned AndOpc; - if (DstTy == LLT::scalar(32)) - AndOpc = X86::AND32ri8; - else if (DstTy == LLT::scalar(64)) - AndOpc = X86::AND64ri8; - else - return false; - - unsigned DefReg = - MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); + unsigned AndOpc; + if (DstTy == LLT::scalar(8)) + AndOpc = X86::AND8ri; + else if (DstTy == LLT::scalar(16)) + AndOpc = X86::AND16ri8; + else if (DstTy == LLT::scalar(32)) + AndOpc = X86::AND32ri8; + else if (DstTy == LLT::scalar(64)) + AndOpc = X86::AND64ri8; + else + return false; + unsigned DefReg = SrcReg; + if (DstTy != LLT::scalar(8)) { + DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::SUBREG_TO_REG), DefReg) .addImm(0) .addReg(SrcReg) .addImm(X86::sub_8bit); - - MachineInstr &AndInst = - *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) - .addReg(DefReg) - .addImm(1); - - constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); - - I.eraseFromParent(); - return true; } - return false; + MachineInstr &AndInst = + *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) + .addReg(DefReg) + .addImm(1); + + constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); + + I.eraseFromParent(); + return true; } bool X86InstructionSelector::selectCmp(MachineInstr &I, @@ -918,6 +982,33 @@ bool X86InstructionSelector::selectInsert(MachineInstr &I, return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } +bool X86InstructionSelector::selectUnmergeValues(MachineInstr &I, + MachineRegisterInfo &MRI, + MachineFunction &MF) const { + if (I.getOpcode() != TargetOpcode::G_UNMERGE_VALUES) + return false; + + // Split to extracts. + unsigned NumDefs = I.getNumOperands() - 1; + unsigned SrcReg = I.getOperand(NumDefs).getReg(); + unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); + + for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { + + MachineInstr &ExtrInst = + *BuildMI(*I.getParent(), I, I.getDebugLoc(), + TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg()) + .addReg(SrcReg) + .addImm(Idx * DefSize); + + if (!select(ExtrInst)) + return false; + } + + I.eraseFromParent(); + return true; +} + bool X86InstructionSelector::selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { diff --git a/contrib/llvm/lib/Target/X86/X86LegalizerInfo.cpp b/contrib/llvm/lib/Target/X86/X86LegalizerInfo.cpp index a5fa3340c3f1..744ba21011af 100644 --- a/contrib/llvm/lib/Target/X86/X86LegalizerInfo.cpp +++ b/contrib/llvm/lib/Target/X86/X86LegalizerInfo.cpp @@ -69,12 +69,14 @@ void X86LegalizerInfo::setLegalizerInfo32bit() { for (auto Ty : {s8, s16, s32, p0}) setAction({MemOp, Ty}, Legal); + setAction({MemOp, s1}, WidenScalar); // And everything's fine in addrspace 0. setAction({MemOp, 1, p0}, Legal); } // Pointer-handling setAction({G_FRAME_INDEX, p0}, Legal); + setAction({G_GLOBAL_VALUE, p0}, Legal); setAction({G_GEP, p0}, Legal); setAction({G_GEP, 1, s32}, Legal); @@ -90,8 +92,10 @@ void X86LegalizerInfo::setLegalizerInfo32bit() { setAction({TargetOpcode::G_CONSTANT, s64}, NarrowScalar); // Extensions - setAction({G_ZEXT, s32}, Legal); - setAction({G_SEXT, s32}, Legal); + for (auto Ty : {s8, s16, s32}) { + setAction({G_ZEXT, Ty}, Legal); + setAction({G_SEXT, Ty}, Legal); + } for (auto Ty : {s1, s8, s16}) { setAction({G_ZEXT, 1, Ty}, Legal); @@ -125,12 +129,14 @@ void X86LegalizerInfo::setLegalizerInfo64bit() { for (auto Ty : {s8, s16, s32, s64, p0}) setAction({MemOp, Ty}, Legal); + setAction({MemOp, s1}, WidenScalar); // And everything's fine in addrspace 0. setAction({MemOp, 1, p0}, Legal); } // Pointer-handling setAction({G_FRAME_INDEX, p0}, Legal); + setAction({G_GLOBAL_VALUE, p0}, Legal); setAction({G_GEP, p0}, Legal); setAction({G_GEP, 1, s32}, Legal); @@ -146,7 +152,7 @@ void X86LegalizerInfo::setLegalizerInfo64bit() { setAction({TargetOpcode::G_CONSTANT, s1}, WidenScalar); // Extensions - for (auto Ty : {s32, s64}) { + for (auto Ty : {s8, s16, s32, s64}) { setAction({G_ZEXT, Ty}, Legal); setAction({G_SEXT, Ty}, Legal); } diff --git a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp index 33bc8e11a572..fd2837b79103 100644 --- a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1042,7 +1042,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, X86MCInstLower &MCIL) { - assert(Subtarget->is64Bit() && "XRay custom events only suports X86-64"); + assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64"); // We want to emit the following pattern, which follows the x86 calling // convention to prepare for the trampoline call to be patched in. @@ -1332,6 +1332,32 @@ static std::string getShuffleComment(const MachineInstr *MI, return Comment; } +static void printConstant(const Constant *COp, raw_ostream &CS) { + if (isa(COp)) { + CS << "u"; + } else if (auto *CI = dyn_cast(COp)) { + if (CI->getBitWidth() <= 64) { + CS << CI->getZExtValue(); + } else { + // print multi-word constant as (w0,w1) + const auto &Val = CI->getValue(); + CS << "("; + for (int i = 0, N = Val.getNumWords(); i < N; ++i) { + if (i > 0) + CS << ","; + CS << Val.getRawData()[i]; + } + CS << ")"; + } + } else if (auto *CF = dyn_cast(COp)) { + SmallString<32> Str; + CF->getValueAPF().toString(Str); + CS << Str; + } else { + CS << "?"; + } +} + void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { X86MCInstLower MCInstLowering(*MF, *this); const X86RegisterInfo *RI = MF->getSubtarget().getRegisterInfo(); @@ -1766,59 +1792,73 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { // For loads from a constant pool to a vector register, print the constant // loaded. CASE_ALL_MOV_RM() + case X86::VBROADCASTF128: + case X86::VBROADCASTI128: + case X86::VBROADCASTF32X4Z256rm: + case X86::VBROADCASTF32X4rm: + case X86::VBROADCASTF32X8rm: + case X86::VBROADCASTF64X2Z128rm: + case X86::VBROADCASTF64X2rm: + case X86::VBROADCASTF64X4rm: + case X86::VBROADCASTI32X4Z256rm: + case X86::VBROADCASTI32X4rm: + case X86::VBROADCASTI32X8rm: + case X86::VBROADCASTI64X2Z128rm: + case X86::VBROADCASTI64X2rm: + case X86::VBROADCASTI64X4rm: if (!OutStreamer->isVerboseAsm()) break; if (MI->getNumOperands() <= 4) break; if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { + int NumLanes = 1; + // Override NumLanes for the broadcast instructions. + switch (MI->getOpcode()) { + case X86::VBROADCASTF128: NumLanes = 2; break; + case X86::VBROADCASTI128: NumLanes = 2; break; + case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; break; + case X86::VBROADCASTF32X4rm: NumLanes = 4; break; + case X86::VBROADCASTF32X8rm: NumLanes = 2; break; + case X86::VBROADCASTF64X2Z128rm: NumLanes = 2; break; + case X86::VBROADCASTF64X2rm: NumLanes = 4; break; + case X86::VBROADCASTF64X4rm: NumLanes = 2; break; + case X86::VBROADCASTI32X4Z256rm: NumLanes = 2; break; + case X86::VBROADCASTI32X4rm: NumLanes = 4; break; + case X86::VBROADCASTI32X8rm: NumLanes = 2; break; + case X86::VBROADCASTI64X2Z128rm: NumLanes = 2; break; + case X86::VBROADCASTI64X2rm: NumLanes = 4; break; + case X86::VBROADCASTI64X4rm: NumLanes = 2; break; + } + std::string Comment; raw_string_ostream CS(Comment); const MachineOperand &DstOp = MI->getOperand(0); CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; if (auto *CDS = dyn_cast(C)) { CS << "["; - for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements; ++i) { - if (i != 0) - CS << ","; - if (CDS->getElementType()->isIntegerTy()) - CS << CDS->getElementAsInteger(i); - else if (CDS->getElementType()->isFloatTy()) - CS << CDS->getElementAsFloat(i); - else if (CDS->getElementType()->isDoubleTy()) - CS << CDS->getElementAsDouble(i); - else - CS << "?"; + for (int l = 0; l != NumLanes; ++l) { + for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements; ++i) { + if (i != 0 || l != 0) + CS << ","; + if (CDS->getElementType()->isIntegerTy()) + CS << CDS->getElementAsInteger(i); + else if (CDS->getElementType()->isFloatTy()) + CS << CDS->getElementAsFloat(i); + else if (CDS->getElementType()->isDoubleTy()) + CS << CDS->getElementAsDouble(i); + else + CS << "?"; + } } CS << "]"; OutStreamer->AddComment(CS.str(), !EnablePrintSchedInfo); } else if (auto *CV = dyn_cast(C)) { CS << "<"; - for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands; ++i) { - if (i != 0) - CS << ","; - Constant *COp = CV->getOperand(i); - if (isa(COp)) { - CS << "u"; - } else if (auto *CI = dyn_cast(COp)) { - if (CI->getBitWidth() <= 64) { - CS << CI->getZExtValue(); - } else { - // print multi-word constant as (w0,w1) - const auto &Val = CI->getValue(); - CS << "("; - for (int i = 0, N = Val.getNumWords(); i < N; ++i) { - if (i > 0) - CS << ","; - CS << Val.getRawData()[i]; - } - CS << ")"; - } - } else if (auto *CF = dyn_cast(COp)) { - SmallString<32> Str; - CF->getValueAPF().toString(Str); - CS << Str; - } else { - CS << "?"; + for (int l = 0; l != NumLanes; ++l) { + for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands; ++i) { + if (i != 0 || l != 0) + CS << ","; + printConstant(CV->getOperand(i), CS); } } CS << ">"; @@ -1826,6 +1866,85 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { } } break; + case X86::VBROADCASTSSrm: + case X86::VBROADCASTSSYrm: + case X86::VBROADCASTSSZ128m: + case X86::VBROADCASTSSZ256m: + case X86::VBROADCASTSSZm: + case X86::VBROADCASTSDYrm: + case X86::VBROADCASTSDZ256m: + case X86::VBROADCASTSDZm: + case X86::VPBROADCASTBrm: + case X86::VPBROADCASTBYrm: + case X86::VPBROADCASTBZ128m: + case X86::VPBROADCASTBZ256m: + case X86::VPBROADCASTBZm: + case X86::VPBROADCASTDrm: + case X86::VPBROADCASTDYrm: + case X86::VPBROADCASTDZ128m: + case X86::VPBROADCASTDZ256m: + case X86::VPBROADCASTDZm: + case X86::VPBROADCASTQrm: + case X86::VPBROADCASTQYrm: + case X86::VPBROADCASTQZ128m: + case X86::VPBROADCASTQZ256m: + case X86::VPBROADCASTQZm: + case X86::VPBROADCASTWrm: + case X86::VPBROADCASTWYrm: + case X86::VPBROADCASTWZ128m: + case X86::VPBROADCASTWZ256m: + case X86::VPBROADCASTWZm: + if (!OutStreamer->isVerboseAsm()) + break; + if (MI->getNumOperands() <= 4) + break; + if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { + int NumElts; + switch (MI->getOpcode()) { + default: llvm_unreachable("Invalid opcode"); + case X86::VBROADCASTSSrm: NumElts = 4; break; + case X86::VBROADCASTSSYrm: NumElts = 8; break; + case X86::VBROADCASTSSZ128m: NumElts = 4; break; + case X86::VBROADCASTSSZ256m: NumElts = 8; break; + case X86::VBROADCASTSSZm: NumElts = 16; break; + case X86::VBROADCASTSDYrm: NumElts = 4; break; + case X86::VBROADCASTSDZ256m: NumElts = 4; break; + case X86::VBROADCASTSDZm: NumElts = 8; break; + case X86::VPBROADCASTBrm: NumElts = 16; break; + case X86::VPBROADCASTBYrm: NumElts = 32; break; + case X86::VPBROADCASTBZ128m: NumElts = 16; break; + case X86::VPBROADCASTBZ256m: NumElts = 32; break; + case X86::VPBROADCASTBZm: NumElts = 64; break; + case X86::VPBROADCASTDrm: NumElts = 4; break; + case X86::VPBROADCASTDYrm: NumElts = 8; break; + case X86::VPBROADCASTDZ128m: NumElts = 4; break; + case X86::VPBROADCASTDZ256m: NumElts = 8; break; + case X86::VPBROADCASTDZm: NumElts = 16; break; + case X86::VPBROADCASTQrm: NumElts = 2; break; + case X86::VPBROADCASTQYrm: NumElts = 4; break; + case X86::VPBROADCASTQZ128m: NumElts = 2; break; + case X86::VPBROADCASTQZ256m: NumElts = 4; break; + case X86::VPBROADCASTQZm: NumElts = 8; break; + case X86::VPBROADCASTWrm: NumElts = 8; break; + case X86::VPBROADCASTWYrm: NumElts = 16; break; + case X86::VPBROADCASTWZ128m: NumElts = 8; break; + case X86::VPBROADCASTWZ256m: NumElts = 16; break; + case X86::VPBROADCASTWZm: NumElts = 32; break; + } + + std::string Comment; + raw_string_ostream CS(Comment); + const MachineOperand &DstOp = MI->getOperand(0); + CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; + CS << "["; + for (int i = 0; i != NumElts; ++i) { + if (i != 0) + CS << ","; + printConstant(C, CS); + } + CS << "]"; + OutStreamer->AddComment(CS.str(), !EnablePrintSchedInfo); + } } MCInst TmpInst; diff --git a/contrib/llvm/lib/Target/X86/X86SchedSandyBridge.td b/contrib/llvm/lib/Target/X86/X86SchedSandyBridge.td index b8ec5883152c..6d85ca6cad64 100644 --- a/contrib/llvm/lib/Target/X86/X86SchedSandyBridge.td +++ b/contrib/llvm/lib/Target/X86/X86SchedSandyBridge.td @@ -24,8 +24,8 @@ def SandyBridgeModel : SchedMachineModel { // Based on the LSD (loop-stream detector) queue size. let LoopMicroOpBufferSize = 28; - // FIXME: SSE4 and AVX are unimplemented. This flag is set to allow - // the scheduler to assign a default model to unrecognized opcodes. + // This flag is set to allow the scheduler to assign + // a default model to unrecognized opcodes. let CompleteModel = 0; } @@ -48,6 +48,7 @@ def SBPort23 : ProcResource<2>; def SBPort4 : ProcResource<1>; // Many micro-ops are capable of issuing on multiple ports. +def SBPort01 : ProcResGroup<[SBPort0, SBPort1]>; def SBPort05 : ProcResGroup<[SBPort0, SBPort5]>; def SBPort15 : ProcResGroup<[SBPort1, SBPort5]>; def SBPort015 : ProcResGroup<[SBPort0, SBPort1, SBPort5]>; @@ -115,10 +116,10 @@ def : WriteRes { // Scalar and vector floating point. defm : SBWriteResPair; defm : SBWriteResPair; -defm : SBWriteResPair; // 10-14 cycles. +defm : SBWriteResPair; defm : SBWriteResPair; defm : SBWriteResPair; -defm : SBWriteResPair; +defm : SBWriteResPair; defm : SBWriteResPair; defm : SBWriteResPair; defm : SBWriteResPair; @@ -134,11 +135,11 @@ def : WriteRes { } // Vector integer operations. -defm : SBWriteResPair; -defm : SBWriteResPair; -defm : SBWriteResPair; +defm : SBWriteResPair; +defm : SBWriteResPair; +defm : SBWriteResPair; defm : SBWriteResPair; -defm : SBWriteResPair; +defm : SBWriteResPair; defm : SBWriteResPair; def : WriteRes { let Latency = 2; @@ -148,13 +149,15 @@ def : WriteRes { let Latency = 6; let ResourceCycles = [1, 1, 1]; } -def : WriteRes { - let Latency = 6; - let ResourceCycles = [1, 1, 1]; +def : WriteRes { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; } -def : WriteRes { - let Latency = 6; - let ResourceCycles = [1, 1, 1, 1]; +def : WriteRes { + let Latency = 11; + let NumMicroOps = 4; + let ResourceCycles = [1,1,2]; } //////////////////////////////////////////////////////////////////////////////// @@ -204,13 +207,15 @@ def : WriteRes { } // Packed Compare Implicit Length Strings, Return Index -def : WriteRes { - let Latency = 3; +def : WriteRes { + let Latency = 11; + let NumMicroOps = 3; let ResourceCycles = [3]; } -def : WriteRes { - let Latency = 3; - let ResourceCycles = [3, 1]; +def : WriteRes { + let Latency = 17; + let NumMicroOps = 4; + let ResourceCycles = [3,1]; } // Packed Compare Explicit Length Strings, Return Index @@ -224,22 +229,26 @@ def : WriteRes { } // AES Instructions. -def : WriteRes { - let Latency = 8; - let ResourceCycles = [2]; +def : WriteRes { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; } -def : WriteRes { - let Latency = 8; - let ResourceCycles = [2, 1]; +def : WriteRes { + let Latency = 13; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; } -def : WriteRes { - let Latency = 8; +def : WriteRes { + let Latency = 12; + let NumMicroOps = 2; let ResourceCycles = [2]; } -def : WriteRes { - let Latency = 8; - let ResourceCycles = [2, 1]; +def : WriteRes { + let Latency = 18; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; } def : WriteRes { @@ -272,4 +281,2407 @@ def : WriteRes; defm : SBWriteResPair; defm : SBWriteResPair; defm : SBWriteResPair; + +// Remaining SNB instrs. + +def SBWriteResGroup0 : SchedWriteRes<[SBPort0]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup0], (instregex "CVTSS2SDrr")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSLLDri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSLLQri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSLLWri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSRADri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSRAWri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSRLDri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSRLQri")>; +def: InstRW<[SBWriteResGroup0], (instregex "PSRLWri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VCVTSS2SDrr")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPMOVMSKBrr")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSLLDri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSLLQri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSLLWri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSRADri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSRAWri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSRLDri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSRLQri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VPSRLWri")>; +def: InstRW<[SBWriteResGroup0], (instregex "VTESTPDYrr")>; +def: InstRW<[SBWriteResGroup0], (instregex "VTESTPDrr")>; +def: InstRW<[SBWriteResGroup0], (instregex "VTESTPSYrr")>; +def: InstRW<[SBWriteResGroup0], (instregex "VTESTPSrr")>; + +def SBWriteResGroup1 : SchedWriteRes<[SBPort1]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup1], (instregex "COMP_FST0r")>; +def: InstRW<[SBWriteResGroup1], (instregex "COM_FST0r")>; +def: InstRW<[SBWriteResGroup1], (instregex "UCOM_FPr")>; +def: InstRW<[SBWriteResGroup1], (instregex "UCOM_Fr")>; + +def SBWriteResGroup2 : SchedWriteRes<[SBPort5]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup2], (instregex "ANDNPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "ANDNPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "ANDPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "ANDPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "FDECSTP")>; +def: InstRW<[SBWriteResGroup2], (instregex "FFREE")>; +def: InstRW<[SBWriteResGroup2], (instregex "FINCSTP")>; +def: InstRW<[SBWriteResGroup2], (instregex "FNOP")>; +def: InstRW<[SBWriteResGroup2], (instregex "INSERTPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "JMP64r")>; +def: InstRW<[SBWriteResGroup2], (instregex "LD_Frr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOV64toPQIrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVAPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVAPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVDDUPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVDI2PDIrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVHLPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVLHPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVSDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVSHDUPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVSLDUPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVSSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVUPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "MOVUPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "ORPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "ORPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "RETQ")>; +def: InstRW<[SBWriteResGroup2], (instregex "SHUFPDrri")>; +def: InstRW<[SBWriteResGroup2], (instregex "SHUFPSrri")>; +def: InstRW<[SBWriteResGroup2], (instregex "ST_FPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "ST_Frr")>; +def: InstRW<[SBWriteResGroup2], (instregex "UNPCKHPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "UNPCKHPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "UNPCKLPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "UNPCKLPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDNPDYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDNPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDNPSYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDNPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VANDPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VEXTRACTF128rr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VINSERTF128rr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VINSERTPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOV64toPQIrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOV64toPQIrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVAPDYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVAPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVAPSYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVAPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVDDUPYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVDDUPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVHLPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVHLPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVSDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVSHDUPYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVSHDUPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVSLDUPYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVSLDUPrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVSSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVUPDYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVUPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVUPSYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VMOVUPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VORPDYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VORPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VORPSYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VORPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPDri")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPDrm")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPSri")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPSrm")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VPERMILPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VSHUFPDYrri")>; +def: InstRW<[SBWriteResGroup2], (instregex "VSHUFPDrri")>; +def: InstRW<[SBWriteResGroup2], (instregex "VSHUFPSYrri")>; +def: InstRW<[SBWriteResGroup2], (instregex "VSHUFPSrri")>; +def: InstRW<[SBWriteResGroup2], (instregex "VUNPCKHPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VUNPCKHPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VUNPCKLPDYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VUNPCKLPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VUNPCKLPSYrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VUNPCKLPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VXORPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "VXORPSrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "XORPDrr")>; +def: InstRW<[SBWriteResGroup2], (instregex "XORPSrr")>; + +def SBWriteResGroup3 : SchedWriteRes<[SBPort01]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup3], (instregex "LEA64_32r")>; + +def SBWriteResGroup4 : SchedWriteRes<[SBPort0]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup4], (instregex "BLENDPDrri")>; +def: InstRW<[SBWriteResGroup4], (instregex "BLENDPSrri")>; +def: InstRW<[SBWriteResGroup4], (instregex "BT32ri8")>; +def: InstRW<[SBWriteResGroup4], (instregex "BT32rr")>; +def: InstRW<[SBWriteResGroup4], (instregex "BTC32ri8")>; +def: InstRW<[SBWriteResGroup4], (instregex "BTC32rr")>; +def: InstRW<[SBWriteResGroup4], (instregex "BTR32ri8")>; +def: InstRW<[SBWriteResGroup4], (instregex "BTR32rr")>; +def: InstRW<[SBWriteResGroup4], (instregex "BTS32ri8")>; +def: InstRW<[SBWriteResGroup4], (instregex "BTS32rr")>; +def: InstRW<[SBWriteResGroup4], (instregex "CDQ")>; +def: InstRW<[SBWriteResGroup4], (instregex "CQO")>; +def: InstRW<[SBWriteResGroup4], (instregex "LAHF")>; +def: InstRW<[SBWriteResGroup4], (instregex "SAHF")>; +def: InstRW<[SBWriteResGroup4], (instregex "SAR32ri")>; +def: InstRW<[SBWriteResGroup4], (instregex "SAR8ri")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETAEr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETBr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETEr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETGEr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETGr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETLEr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETLr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETNEr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETNOr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETNPr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETNSr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETOr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETPr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SETSr")>; +def: InstRW<[SBWriteResGroup4], (instregex "SHL32ri")>; +def: InstRW<[SBWriteResGroup4], (instregex "SHL64r1")>; +def: InstRW<[SBWriteResGroup4], (instregex "SHL8r1")>; +def: InstRW<[SBWriteResGroup4], (instregex "SHL8ri")>; +def: InstRW<[SBWriteResGroup4], (instregex "SHR32ri")>; +def: InstRW<[SBWriteResGroup4], (instregex "SHR8ri")>; +def: InstRW<[SBWriteResGroup4], (instregex "VBLENDPDYrri")>; +def: InstRW<[SBWriteResGroup4], (instregex "VBLENDPDrri")>; +def: InstRW<[SBWriteResGroup4], (instregex "VBLENDPSYrri")>; +def: InstRW<[SBWriteResGroup4], (instregex "VBLENDPSrri")>; +def: InstRW<[SBWriteResGroup4], (instregex "VMOVDQAYrr")>; +def: InstRW<[SBWriteResGroup4], (instregex "VMOVDQArr")>; +def: InstRW<[SBWriteResGroup4], (instregex "VMOVDQUYrr")>; +def: InstRW<[SBWriteResGroup4], (instregex "VMOVDQUrr")>; + +def SBWriteResGroup5 : SchedWriteRes<[SBPort15]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup5], (instregex "KORTESTBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PABSBrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PABSDrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PABSWrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PADDQirr")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PALIGNR64irr")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PSHUFBrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PSIGNBrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PSIGNDrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "MMX_PSIGNWrr64")>; +def: InstRW<[SBWriteResGroup5], (instregex "PABSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PABSDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PABSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PACKSSDWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PACKSSWBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PACKUSDWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PACKUSWBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDUSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDUSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PADDWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PALIGNRrri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PAVGBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PAVGWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PBLENDWrri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPEQBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPEQDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPEQQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPEQWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPGTBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPGTDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PCMPGTWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMAXSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMAXSDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMAXSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMAXUBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMAXUDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMAXUWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMINSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMINSDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMINSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMINUBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMINUDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMINUWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVSXBDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVSXBQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVSXBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVSXDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVSXWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVSXWQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVZXBDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVZXBQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVZXBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVZXDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVZXWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PMOVZXWQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSHUFBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSHUFDri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSHUFHWri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSHUFLWri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSIGNBrr128")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSIGNDrr128")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSIGNWrr128")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSLLDQri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSRLDQri")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBUSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBUSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PSUBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKHBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKHDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKHQDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKHWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKLBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKLDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKLQDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "PUNPCKLWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VMASKMOVPSYrm")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPABSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPABSDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPABSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPACKSSDWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPACKSSWBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPACKUSDWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPACKUSWBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPADDBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPADDDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPADDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPADDUSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPADDUSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPALIGNRrri")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPAVGBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPAVGWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPBLENDWrri")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPCMPEQBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPCMPEQDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPCMPEQWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPCMPGTBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPCMPGTDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPCMPGTWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMAXSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMAXSDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMAXSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMAXUBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMAXUDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMAXUWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMINSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMINSDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMINSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMINUBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMINUDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMINUWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVSXBDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVSXBQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVSXBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVSXDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVSXWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVSXWQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVZXBDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVZXBQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVZXBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVZXDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVZXWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPMOVZXWQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSHUFBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSHUFDri")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSHUFLWri")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSIGNBrr128")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSIGNDrr128")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSIGNWrr128")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSLLDQri")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSRLDQri")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBUSBrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBUSWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPSUBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPUNPCKHBWrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPUNPCKHDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPUNPCKHWDrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPUNPCKLDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPUNPCKLQDQrr")>; +def: InstRW<[SBWriteResGroup5], (instregex "VPUNPCKLWDrr")>; + +def SBWriteResGroup6 : SchedWriteRes<[SBPort015]> { + let Latency = 1; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup6], (instregex "ADD32ri8")>; +def: InstRW<[SBWriteResGroup6], (instregex "ADD32rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "ADD8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "ADD8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "AND32ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "AND64ri8")>; +def: InstRW<[SBWriteResGroup6], (instregex "AND64rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "AND8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "AND8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "CBW")>; +def: InstRW<[SBWriteResGroup6], (instregex "CMC")>; +def: InstRW<[SBWriteResGroup6], (instregex "CMP16ri8")>; +def: InstRW<[SBWriteResGroup6], (instregex "CMP32i32")>; +def: InstRW<[SBWriteResGroup6], (instregex "CMP64rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "CMP8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "CMP8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "CWDE")>; +def: InstRW<[SBWriteResGroup6], (instregex "DEC64r")>; +def: InstRW<[SBWriteResGroup6], (instregex "DEC8r")>; +def: InstRW<[SBWriteResGroup6], (instregex "INC64r")>; +def: InstRW<[SBWriteResGroup6], (instregex "INC8r")>; +def: InstRW<[SBWriteResGroup6], (instregex "MMX_MOVD64from64rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MMX_MOVQ2DQrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOV32rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOV8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOV8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVDQArr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVDQUrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVPQI2QIrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVSX32rr16")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVSX32rr8")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVZX32rr16")>; +def: InstRW<[SBWriteResGroup6], (instregex "MOVZX32rr8")>; +def: InstRW<[SBWriteResGroup6], (instregex "NEG64r")>; +def: InstRW<[SBWriteResGroup6], (instregex "NEG8r")>; +def: InstRW<[SBWriteResGroup6], (instregex "NOT64r")>; +def: InstRW<[SBWriteResGroup6], (instregex "NOT8r")>; +def: InstRW<[SBWriteResGroup6], (instregex "OR64ri8")>; +def: InstRW<[SBWriteResGroup6], (instregex "OR64rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "OR8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "OR8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "PANDNrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "PANDrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "PORrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "PXORrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "STC")>; +def: InstRW<[SBWriteResGroup6], (instregex "SUB64ri8")>; +def: InstRW<[SBWriteResGroup6], (instregex "SUB64rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "SUB8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "SUB8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "TEST64rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "TEST8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "TEST8rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "VMOVPQI2QIrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "VMOVZPQILo2PQIrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "VPANDNrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "VPANDrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "VPORrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "VPXORrr")>; +def: InstRW<[SBWriteResGroup6], (instregex "XOR32rr")>; +def: InstRW<[SBWriteResGroup6], (instregex "XOR64ri8")>; +def: InstRW<[SBWriteResGroup6], (instregex "XOR8ri")>; +def: InstRW<[SBWriteResGroup6], (instregex "XOR8rr")>; + +def SBWriteResGroup7 : SchedWriteRes<[SBPort0]> { + let Latency = 2; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup7], (instregex "MOVMSKPDrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "MOVMSKPSrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "MOVPDI2DIrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "MOVPQIto64rr")>; +def: InstRW<[SBWriteResGroup7], (instregex "PMOVMSKBrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "VMOVMSKPDYrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "VMOVMSKPDrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "VMOVMSKPSrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "VMOVPDI2DIrr")>; +def: InstRW<[SBWriteResGroup7], (instregex "VMOVPQIto64rr")>; + +def SBWriteResGroup9 : SchedWriteRes<[SBPort0]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [2]; +} +def: InstRW<[SBWriteResGroup9], (instregex "BLENDVPDrr0")>; +def: InstRW<[SBWriteResGroup9], (instregex "BLENDVPSrr0")>; +def: InstRW<[SBWriteResGroup9], (instregex "ROL32ri")>; +def: InstRW<[SBWriteResGroup9], (instregex "ROL8ri")>; +def: InstRW<[SBWriteResGroup9], (instregex "ROR32ri")>; +def: InstRW<[SBWriteResGroup9], (instregex "ROR8ri")>; +def: InstRW<[SBWriteResGroup9], (instregex "SETAr")>; +def: InstRW<[SBWriteResGroup9], (instregex "SETBEr")>; +def: InstRW<[SBWriteResGroup9], (instregex "VBLENDVPDYrr")>; +def: InstRW<[SBWriteResGroup9], (instregex "VBLENDVPDrr")>; +def: InstRW<[SBWriteResGroup9], (instregex "VBLENDVPSYrr")>; +def: InstRW<[SBWriteResGroup9], (instregex "VBLENDVPSrr")>; + +def SBWriteResGroup10 : SchedWriteRes<[SBPort15]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [2]; +} +def: InstRW<[SBWriteResGroup10], (instregex "VPBLENDVBrr")>; + +def SBWriteResGroup11 : SchedWriteRes<[SBPort015]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [2]; +} +def: InstRW<[SBWriteResGroup11], (instregex "SCASB")>; +def: InstRW<[SBWriteResGroup11], (instregex "SCASL")>; +def: InstRW<[SBWriteResGroup11], (instregex "SCASQ")>; +def: InstRW<[SBWriteResGroup11], (instregex "SCASW")>; + +def SBWriteResGroup12 : SchedWriteRes<[SBPort0,SBPort1]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup12], (instregex "COMISDrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "COMISSrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "UCOMISDrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "UCOMISSrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "VCOMISDrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "VCOMISSrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "VUCOMISDrr")>; +def: InstRW<[SBWriteResGroup12], (instregex "VUCOMISSrr")>; + +def SBWriteResGroup13 : SchedWriteRes<[SBPort0,SBPort5]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup13], (instregex "CVTPS2PDrr")>; +def: InstRW<[SBWriteResGroup13], (instregex "PTESTrr")>; +def: InstRW<[SBWriteResGroup13], (instregex "VCVTPS2PDYrr")>; +def: InstRW<[SBWriteResGroup13], (instregex "VCVTPS2PDrr")>; +def: InstRW<[SBWriteResGroup13], (instregex "VPTESTYrr")>; +def: InstRW<[SBWriteResGroup13], (instregex "VPTESTrr")>; + +def SBWriteResGroup14 : SchedWriteRes<[SBPort0,SBPort15]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup14], (instregex "PSLLDrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSLLQrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSLLWrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSRADrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSRAWrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSRLDrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSRLQrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "PSRLWrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "VPSRADrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "VPSRAWrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "VPSRLDrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "VPSRLQrr")>; +def: InstRW<[SBWriteResGroup14], (instregex "VPSRLWrr")>; + +def SBWriteResGroup15 : SchedWriteRes<[SBPort0,SBPort015]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup15], (instregex "FNSTSW16r")>; + +def SBWriteResGroup16 : SchedWriteRes<[SBPort1,SBPort0]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup16], (instregex "BSWAP32r")>; + +def SBWriteResGroup17 : SchedWriteRes<[SBPort5,SBPort15]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup17], (instregex "PINSRBrr")>; +def: InstRW<[SBWriteResGroup17], (instregex "PINSRDrr")>; +def: InstRW<[SBWriteResGroup17], (instregex "PINSRQrr")>; +def: InstRW<[SBWriteResGroup17], (instregex "PINSRWrri")>; +def: InstRW<[SBWriteResGroup17], (instregex "VPINSRBrr")>; +def: InstRW<[SBWriteResGroup17], (instregex "VPINSRDrr")>; +def: InstRW<[SBWriteResGroup17], (instregex "VPINSRQrr")>; +def: InstRW<[SBWriteResGroup17], (instregex "VPINSRWrri")>; + +def SBWriteResGroup18 : SchedWriteRes<[SBPort5,SBPort015]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup18], (instregex "MMX_MOVDQ2Qrr")>; + +def SBWriteResGroup19 : SchedWriteRes<[SBPort0,SBPort015]> { + let Latency = 2; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup19], (instregex "ADC64ri8")>; +def: InstRW<[SBWriteResGroup19], (instregex "ADC64rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "ADC8ri")>; +def: InstRW<[SBWriteResGroup19], (instregex "ADC8rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVAE32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVB32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVE32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVG32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVGE32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVL32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVLE32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVNE32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVNO32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVNP32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVNS32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVO32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVP32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "CMOVS32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "SBB32rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "SBB64ri8")>; +def: InstRW<[SBWriteResGroup19], (instregex "SBB8ri")>; +def: InstRW<[SBWriteResGroup19], (instregex "SBB8rr")>; +def: InstRW<[SBWriteResGroup19], (instregex "SHLD32rri8")>; +def: InstRW<[SBWriteResGroup19], (instregex "SHRD32rri8")>; + +def SBWriteResGroup20 : SchedWriteRes<[SBPort0]> { + let Latency = 3; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup20], (instregex "MMX_PMADDUBSWrr64")>; +def: InstRW<[SBWriteResGroup20], (instregex "MMX_PMULHRSWrr64")>; +def: InstRW<[SBWriteResGroup20], (instregex "MMX_PMULUDQirr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMADDUBSWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMADDWDrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULDQrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULHRSWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULHUWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULHWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULLDrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULLWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PMULUDQrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "PSADBWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VMOVMSKPSYrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMADDUBSWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMADDWDrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMULDQrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMULHRSWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMULHWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMULLDrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPMULLWrr")>; +def: InstRW<[SBWriteResGroup20], (instregex "VPSADBWrr")>; + +def SBWriteResGroup21 : SchedWriteRes<[SBPort1]> { + let Latency = 3; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup21], (instregex "ADDPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADDPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADDSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADDSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADDSUBPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADDSUBPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADD_FPrST0")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADD_FST0r")>; +def: InstRW<[SBWriteResGroup21], (instregex "ADD_FrST0")>; +def: InstRW<[SBWriteResGroup21], (instregex "BSF32rr")>; +def: InstRW<[SBWriteResGroup21], (instregex "BSR32rr")>; +def: InstRW<[SBWriteResGroup21], (instregex "CMPPDrri")>; +def: InstRW<[SBWriteResGroup21], (instregex "CMPPSrri")>; +def: InstRW<[SBWriteResGroup21], (instregex "CMPSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "CMPSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "CRC32r32r32")>; +def: InstRW<[SBWriteResGroup21], (instregex "CRC32r32r8")>; +def: InstRW<[SBWriteResGroup21], (instregex "CVTDQ2PSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "CVTPS2DQrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "CVTTPS2DQrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MAXPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MAXPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MAXSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MAXSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MINPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MINPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MINSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MINSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MMX_CVTPI2PSirr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MMX_CVTPS2PIirr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MMX_CVTTPS2PIirr")>; +def: InstRW<[SBWriteResGroup21], (instregex "MUL8r")>; +def: InstRW<[SBWriteResGroup21], (instregex "POPCNT32rr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ROUNDPDr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ROUNDPSr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ROUNDSDr")>; +def: InstRW<[SBWriteResGroup21], (instregex "ROUNDSSr")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBR_FPrST0")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBR_FST0r")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBR_FrST0")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUBSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUB_FPrST0")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUB_FST0r")>; +def: InstRW<[SBWriteResGroup21], (instregex "SUB_FrST0")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDPDYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDPSYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDSUBPDYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDSUBPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDSUBPSYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VADDSUBPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VBROADCASTF128")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCMPPDYrri")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCMPPDrri")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCMPPSYrri")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCMPPSrri")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCMPSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCMPSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCVTDQ2PSYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCVTDQ2PSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCVTPS2DQYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCVTPS2DQrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VCVTTPS2DQrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMAXPDYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMAXPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMAXPSYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMAXPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMAXSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMAXSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMINPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMINPSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMINSDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VMINSSrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VROUNDPDr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VROUNDPSr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VROUNDSDr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VSUBPDYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VSUBPDrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VSUBPSYrr")>; +def: InstRW<[SBWriteResGroup21], (instregex "VSUBPSrr")>; + +def SBWriteResGroup22 : SchedWriteRes<[SBPort0,SBPort5]> { + let Latency = 3; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup22], (instregex "EXTRACTPSrr")>; +def: InstRW<[SBWriteResGroup22], (instregex "VEXTRACTPSrr")>; + +def SBWriteResGroup23 : SchedWriteRes<[SBPort0,SBPort15]> { + let Latency = 3; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup23], (instregex "PEXTRBrr")>; +def: InstRW<[SBWriteResGroup23], (instregex "PEXTRDrr")>; +def: InstRW<[SBWriteResGroup23], (instregex "PEXTRQrr")>; +def: InstRW<[SBWriteResGroup23], (instregex "PEXTRWri")>; +def: InstRW<[SBWriteResGroup23], (instregex "VPEXTRBrr")>; +def: InstRW<[SBWriteResGroup23], (instregex "VPEXTRDrr")>; +def: InstRW<[SBWriteResGroup23], (instregex "VPEXTRQrr")>; +def: InstRW<[SBWriteResGroup23], (instregex "VPEXTRWri")>; +def: InstRW<[SBWriteResGroup23], (instregex "SHL64rCL")>; +def: InstRW<[SBWriteResGroup23], (instregex "SHL8rCL")>; + +def SBWriteResGroup24 : SchedWriteRes<[SBPort15]> { + let Latency = 3; + let NumMicroOps = 3; + let ResourceCycles = [3]; +} +def: InstRW<[SBWriteResGroup24], (instregex "MMX_PHADDSWrr64")>; +def: InstRW<[SBWriteResGroup24], (instregex "MMX_PHADDWrr64")>; +def: InstRW<[SBWriteResGroup24], (instregex "MMX_PHADDrr64")>; +def: InstRW<[SBWriteResGroup24], (instregex "MMX_PHSUBDrr64")>; +def: InstRW<[SBWriteResGroup24], (instregex "MMX_PHSUBSWrr64")>; +def: InstRW<[SBWriteResGroup24], (instregex "MMX_PHSUBWrr64")>; +def: InstRW<[SBWriteResGroup24], (instregex "PHADDDrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "PHADDSWrr128")>; +def: InstRW<[SBWriteResGroup24], (instregex "PHADDWrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "PHSUBDrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "PHSUBSWrr128")>; +def: InstRW<[SBWriteResGroup24], (instregex "PHSUBWrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "VPHADDDrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "VPHADDSWrr128")>; +def: InstRW<[SBWriteResGroup24], (instregex "VPHADDWrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "VPHSUBDrr")>; +def: InstRW<[SBWriteResGroup24], (instregex "VPHSUBSWrr128")>; +def: InstRW<[SBWriteResGroup24], (instregex "VPHSUBWrr")>; + +def SBWriteResGroup25 : SchedWriteRes<[SBPort015]> { + let Latency = 3; + let NumMicroOps = 3; + let ResourceCycles = [3]; +} +def: InstRW<[SBWriteResGroup25], (instregex "LEAVE64")>; +def: InstRW<[SBWriteResGroup25], (instregex "XADD32rr")>; +def: InstRW<[SBWriteResGroup25], (instregex "XADD8rr")>; + +def SBWriteResGroup26 : SchedWriteRes<[SBPort0,SBPort015]> { + let Latency = 3; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup26], (instregex "CMOVA32rr")>; +def: InstRW<[SBWriteResGroup26], (instregex "CMOVBE32rr")>; + +def SBWriteResGroup27 : SchedWriteRes<[SBPort0,SBPort1]> { + let Latency = 4; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup27], (instregex "MUL64r")>; + +def SBWriteResGroup28 : SchedWriteRes<[SBPort1,SBPort5]> { + let Latency = 4; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup28], (instregex "CVTDQ2PDrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "CVTPD2DQrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "CVTPD2PSrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "CVTSD2SSrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "CVTSI2SD64rr")>; +def: InstRW<[SBWriteResGroup28], (instregex "CVTSI2SDrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "CVTTPD2DQrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "MMX_CVTPD2PIirr")>; +def: InstRW<[SBWriteResGroup28], (instregex "MMX_CVTPI2PDirr")>; +def: InstRW<[SBWriteResGroup28], (instregex "MMX_CVTTPD2PIirr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTDQ2PDYrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTDQ2PDrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTPD2DQYrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTPD2DQrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTPD2PSYrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTPD2PSrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTSI2SD64rr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTSI2SDrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTTPD2DQYrr")>; +def: InstRW<[SBWriteResGroup28], (instregex "VCVTTPD2DQrr")>; + +def SBWriteResGroup29 : SchedWriteRes<[SBPort1,SBPort015]> { + let Latency = 4; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup29], (instregex "MOV64sr")>; +def: InstRW<[SBWriteResGroup29], (instregex "PAUSE")>; + +def SBWriteResGroup30 : SchedWriteRes<[SBPort0]> { + let Latency = 5; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup30], (instregex "MULPDrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "MULPSrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "MULSDrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "MULSSrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "MUL_FPrST0")>; +def: InstRW<[SBWriteResGroup30], (instregex "MUL_FST0r")>; +def: InstRW<[SBWriteResGroup30], (instregex "MUL_FrST0")>; +def: InstRW<[SBWriteResGroup30], (instregex "PCMPGTQrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "PHMINPOSUWrr128")>; +def: InstRW<[SBWriteResGroup30], (instregex "RCPPSr")>; +def: InstRW<[SBWriteResGroup30], (instregex "RCPSSr")>; +def: InstRW<[SBWriteResGroup30], (instregex "RSQRTPSr")>; +def: InstRW<[SBWriteResGroup30], (instregex "RSQRTSSr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VMULPDYrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VMULPDrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VMULPSYrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VMULPSrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VMULSDrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VMULSSrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VPCMPGTQrr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VPHMINPOSUWrr128")>; +def: InstRW<[SBWriteResGroup30], (instregex "VRSQRTPSr")>; +def: InstRW<[SBWriteResGroup30], (instregex "VRSQRTSSr")>; + +def SBWriteResGroup31 : SchedWriteRes<[SBPort23]> { + let Latency = 5; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup31], (instregex "MOV32rm")>; +def: InstRW<[SBWriteResGroup31], (instregex "MOV8rm")>; +def: InstRW<[SBWriteResGroup31], (instregex "MOVSX32rm16")>; +def: InstRW<[SBWriteResGroup31], (instregex "MOVSX32rm8")>; +def: InstRW<[SBWriteResGroup31], (instregex "MOVZX32rm16")>; +def: InstRW<[SBWriteResGroup31], (instregex "MOVZX32rm8")>; +def: InstRW<[SBWriteResGroup31], (instregex "PREFETCH")>; + +def SBWriteResGroup32 : SchedWriteRes<[SBPort0,SBPort1]> { + let Latency = 5; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup32], (instregex "CVTSD2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTSD2SIrr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTSS2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTSS2SIrr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTTSD2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTTSD2SIrr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTTSS2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "CVTTSS2SIrr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTSD2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTSS2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTSS2SIrr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTTSD2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTTSD2SIrr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTTSS2SI64rr")>; +def: InstRW<[SBWriteResGroup32], (instregex "VCVTTSS2SIrr")>; + +def SBWriteResGroup33 : SchedWriteRes<[SBPort4,SBPort23]> { + let Latency = 5; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup33], (instregex "MOV64mr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOV8mr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVAPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVAPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVDQAmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVDQUmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVHPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVHPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVLPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVLPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVNTDQmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVNTI_64mr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVNTImr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVNTPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVNTPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVPDI2DImr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVPQI2QImr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVPQIto64mr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVSSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVUPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "MOVUPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "PUSH64i8")>; +def: InstRW<[SBWriteResGroup33], (instregex "PUSH64r")>; +def: InstRW<[SBWriteResGroup33], (instregex "VEXTRACTF128mr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVAPDYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVAPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVAPSYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVAPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVDQAYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVDQAmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVDQUYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVDQUmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVHPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVHPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVLPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVLPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVNTDQYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVNTDQmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVNTPDYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVNTPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVNTPSYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVNTPSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVPDI2DImr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVPQI2QImr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVPQIto64mr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVSDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVSSmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVUPDYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVUPDmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVUPSYmr")>; +def: InstRW<[SBWriteResGroup33], (instregex "VMOVUPSmr")>; + +def SBWriteResGroup34 : SchedWriteRes<[SBPort0,SBPort15]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup34], (instregex "MPSADBWrri")>; +def: InstRW<[SBWriteResGroup34], (instregex "VMPSADBWrri")>; + +def SBWriteResGroup35 : SchedWriteRes<[SBPort1,SBPort5]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup35], (instregex "CLI")>; +def: InstRW<[SBWriteResGroup35], (instregex "CVTSI2SS64rr")>; +def: InstRW<[SBWriteResGroup35], (instregex "CVTSI2SSrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "HADDPDrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "HADDPSrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "HSUBPDrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "HSUBPSrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VCVTSI2SS64rr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VCVTSI2SSrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHADDPDrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHADDPSYrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHADDPSrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHSUBPDYrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHSUBPDrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHSUBPSYrr")>; +def: InstRW<[SBWriteResGroup35], (instregex "VHSUBPSrr")>; + +def SBWriteResGroup36 : SchedWriteRes<[SBPort4,SBPort5,SBPort23]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup36], (instregex "CALL64r")>; +def: InstRW<[SBWriteResGroup36], (instregex "EXTRACTPSmr")>; +def: InstRW<[SBWriteResGroup36], (instregex "VEXTRACTPSmr")>; + +def SBWriteResGroup37 : SchedWriteRes<[SBPort4,SBPort01,SBPort23]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup37], (instregex "VMASKMOVPDYrm")>; +def: InstRW<[SBWriteResGroup37], (instregex "VMASKMOVPDmr")>; +def: InstRW<[SBWriteResGroup37], (instregex "VMASKMOVPSmr")>; + +def SBWriteResGroup38 : SchedWriteRes<[SBPort4,SBPort23,SBPort0]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup38], (instregex "SETAEm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETBm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETEm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETGEm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETGm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETLEm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETLm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETNEm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETNOm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETNPm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETNSm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETOm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETPm")>; +def: InstRW<[SBWriteResGroup38], (instregex "SETSm")>; + +def SBWriteResGroup39 : SchedWriteRes<[SBPort4,SBPort23,SBPort15]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup39], (instregex "PEXTRBmr")>; +def: InstRW<[SBWriteResGroup39], (instregex "VPEXTRBmr")>; +def: InstRW<[SBWriteResGroup39], (instregex "VPEXTRDmr")>; +def: InstRW<[SBWriteResGroup39], (instregex "VPEXTRWmr")>; + +def SBWriteResGroup40 : SchedWriteRes<[SBPort4,SBPort23,SBPort015]> { + let Latency = 5; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup40], (instregex "MOV8mi")>; +def: InstRW<[SBWriteResGroup40], (instregex "STOSB")>; +def: InstRW<[SBWriteResGroup40], (instregex "STOSL")>; +def: InstRW<[SBWriteResGroup40], (instregex "STOSQ")>; +def: InstRW<[SBWriteResGroup40], (instregex "STOSW")>; + +def SBWriteResGroup41 : SchedWriteRes<[SBPort5,SBPort015]> { + let Latency = 5; + let NumMicroOps = 4; + let ResourceCycles = [1,3]; +} +def: InstRW<[SBWriteResGroup41], (instregex "FNINIT")>; + +def SBWriteResGroup42 : SchedWriteRes<[SBPort0,SBPort015]> { + let Latency = 5; + let NumMicroOps = 4; + let ResourceCycles = [1,3]; +} +def: InstRW<[SBWriteResGroup42], (instregex "CMPXCHG32rr")>; +def: InstRW<[SBWriteResGroup42], (instregex "CMPXCHG8rr")>; + +def SBWriteResGroup43 : SchedWriteRes<[SBPort4,SBPort23,SBPort0]> { + let Latency = 5; + let NumMicroOps = 4; + let ResourceCycles = [1,1,2]; +} +def: InstRW<[SBWriteResGroup43], (instregex "SETAm")>; +def: InstRW<[SBWriteResGroup43], (instregex "SETBEm")>; + +def SBWriteResGroup44 : SchedWriteRes<[SBPort0,SBPort4,SBPort5,SBPort23]> { + let Latency = 5; + let NumMicroOps = 4; + let ResourceCycles = [1,1,1,1]; +} +def: InstRW<[SBWriteResGroup44], (instregex "LDMXCSR")>; +def: InstRW<[SBWriteResGroup44], (instregex "STMXCSR")>; +def: InstRW<[SBWriteResGroup44], (instregex "VLDMXCSR")>; +def: InstRW<[SBWriteResGroup44], (instregex "VSTMXCSR")>; + +def SBWriteResGroup45 : SchedWriteRes<[SBPort0,SBPort4,SBPort23,SBPort15]> { + let Latency = 5; + let NumMicroOps = 4; + let ResourceCycles = [1,1,1,1]; +} +def: InstRW<[SBWriteResGroup45], (instregex "PEXTRDmr")>; +def: InstRW<[SBWriteResGroup45], (instregex "PEXTRQmr")>; +def: InstRW<[SBWriteResGroup45], (instregex "VPEXTRQmr")>; +def: InstRW<[SBWriteResGroup45], (instregex "PUSHF16")>; +def: InstRW<[SBWriteResGroup45], (instregex "PUSHF64")>; + +def SBWriteResGroup46 : SchedWriteRes<[SBPort4,SBPort5,SBPort01,SBPort23]> { + let Latency = 5; + let NumMicroOps = 4; + let ResourceCycles = [1,1,1,1]; +} +def: InstRW<[SBWriteResGroup46], (instregex "CLFLUSH")>; + +def SBWriteResGroup47 : SchedWriteRes<[SBPort4,SBPort5,SBPort01,SBPort23]> { + let Latency = 5; + let NumMicroOps = 5; + let ResourceCycles = [1,2,1,1]; +} +def: InstRW<[SBWriteResGroup47], (instregex "FXRSTOR")>; + +def SBWriteResGroup48 : SchedWriteRes<[SBPort23]> { + let Latency = 6; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup48], (instregex "LDDQUrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MMX_MOVD64from64rm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOV64toPQIrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVAPDrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVAPSrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVDDUPrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVDI2PDIrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVDQArm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVDQUrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVNTDQArm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVSHDUPrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVSLDUPrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVSSrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVUPDrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "MOVUPSrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "POP64r")>; +def: InstRW<[SBWriteResGroup48], (instregex "VBROADCASTSSrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VLDDQUYrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VLDDQUrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOV64toPQIrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVAPDrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVAPSrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVDDUPrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVDI2PDIrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVDQArm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVDQUrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVNTDQArm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVQI2PQIrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVSDrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVSHDUPrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVSLDUPrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVSSrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVUPDrm")>; +def: InstRW<[SBWriteResGroup48], (instregex "VMOVUPSrm")>; + +def SBWriteResGroup49 : SchedWriteRes<[SBPort5,SBPort23]> { + let Latency = 6; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup49], (instregex "JMP64m")>; +def: InstRW<[SBWriteResGroup49], (instregex "MOV64sm")>; + +def SBWriteResGroup50 : SchedWriteRes<[SBPort23,SBPort0]> { + let Latency = 6; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup50], (instregex "BT64mi8")>; + +def SBWriteResGroup51 : SchedWriteRes<[SBPort23,SBPort15]> { + let Latency = 6; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PABSBrm64")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PABSDrm64")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PABSWrm64")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PALIGNR64irm")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PSHUFBrm64")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PSIGNBrm64")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PSIGNDrm64")>; +def: InstRW<[SBWriteResGroup51], (instregex "MMX_PSIGNWrm64")>; + +def SBWriteResGroup52 : SchedWriteRes<[SBPort23,SBPort015]> { + let Latency = 6; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup52], (instregex "ADD64rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "ADD8rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "AND64rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "AND8rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "CMP64mi8")>; +def: InstRW<[SBWriteResGroup52], (instregex "CMP64mr")>; +def: InstRW<[SBWriteResGroup52], (instregex "CMP64rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "CMP8mi")>; +def: InstRW<[SBWriteResGroup52], (instregex "CMP8mr")>; +def: InstRW<[SBWriteResGroup52], (instregex "CMP8rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "LODSL")>; +def: InstRW<[SBWriteResGroup52], (instregex "LODSQ")>; +def: InstRW<[SBWriteResGroup52], (instregex "OR64rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "OR8rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "SUB64rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "SUB8rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "XOR64rm")>; +def: InstRW<[SBWriteResGroup52], (instregex "XOR8rm")>; + +def SBWriteResGroup53 : SchedWriteRes<[SBPort4,SBPort23]> { + let Latency = 6; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup53], (instregex "POP64rmm")>; +def: InstRW<[SBWriteResGroup53], (instregex "PUSH64rmm")>; +def: InstRW<[SBWriteResGroup53], (instregex "ST_F32m")>; +def: InstRW<[SBWriteResGroup53], (instregex "ST_F64m")>; +def: InstRW<[SBWriteResGroup53], (instregex "ST_FP32m")>; +def: InstRW<[SBWriteResGroup53], (instregex "ST_FP64m")>; +def: InstRW<[SBWriteResGroup53], (instregex "ST_FP80m")>; + +def SBWriteResGroup54 : SchedWriteRes<[SBPort23]> { + let Latency = 7; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup54], (instregex "VBROADCASTSDYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VBROADCASTSSrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVAPDYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVAPSYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVDDUPYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVDQAYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVDQUYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVSHDUPYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVSLDUPYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVUPDYrm")>; +def: InstRW<[SBWriteResGroup54], (instregex "VMOVUPSYrm")>; + +def SBWriteResGroup55 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup55], (instregex "CVTPS2PDrm")>; +def: InstRW<[SBWriteResGroup55], (instregex "CVTSS2SDrm")>; +def: InstRW<[SBWriteResGroup55], (instregex "VCVTPS2PDYrm")>; +def: InstRW<[SBWriteResGroup55], (instregex "VCVTPS2PDrm")>; +def: InstRW<[SBWriteResGroup55], (instregex "VCVTSS2SDrm")>; +def: InstRW<[SBWriteResGroup55], (instregex "VTESTPDrm")>; +def: InstRW<[SBWriteResGroup55], (instregex "VTESTPSrm")>; + +def SBWriteResGroup56 : SchedWriteRes<[SBPort5,SBPort23]> { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup56], (instregex "ANDNPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "ANDNPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "ANDPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "ANDPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "INSERTPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "MOVHPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "MOVHPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "MOVLPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "MOVLPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "ORPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "ORPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "SHUFPDrmi")>; +def: InstRW<[SBWriteResGroup56], (instregex "SHUFPSrmi")>; +def: InstRW<[SBWriteResGroup56], (instregex "UNPCKHPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "UNPCKHPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "UNPCKLPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "UNPCKLPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VANDNPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VANDNPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VANDPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VANDPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VBROADCASTF128")>; +def: InstRW<[SBWriteResGroup56], (instregex "VINSERTPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VMOVHPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VMOVHPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VMOVLPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VMOVLPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VORPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VORPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VPERMILPDmi")>; +def: InstRW<[SBWriteResGroup56], (instregex "VPERMILPDri")>; +def: InstRW<[SBWriteResGroup56], (instregex "VPERMILPSmi")>; +def: InstRW<[SBWriteResGroup56], (instregex "VPERMILPSri")>; +def: InstRW<[SBWriteResGroup56], (instregex "VSHUFPDrmi")>; +def: InstRW<[SBWriteResGroup56], (instregex "VSHUFPSrmi")>; +def: InstRW<[SBWriteResGroup56], (instregex "VUNPCKHPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VUNPCKHPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VUNPCKLPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VUNPCKLPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VXORPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "VXORPSrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "XORPDrm")>; +def: InstRW<[SBWriteResGroup56], (instregex "XORPSrm")>; + +def SBWriteResGroup57 : SchedWriteRes<[SBPort5,SBPort015]> { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup57], (instregex "AESDECLASTrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "AESDECrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "AESENCLASTrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "AESENCrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "KANDQrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "VAESDECLASTrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "VAESDECrr")>; +def: InstRW<[SBWriteResGroup57], (instregex "VAESENCrr")>; + +def SBWriteResGroup58 : SchedWriteRes<[SBPort23,SBPort0]> { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup58], (instregex "BLENDPDrmi")>; +def: InstRW<[SBWriteResGroup58], (instregex "BLENDPSrmi")>; +def: InstRW<[SBWriteResGroup58], (instregex "VBLENDPDrmi")>; +def: InstRW<[SBWriteResGroup58], (instregex "VBLENDPSrmi")>; +def: InstRW<[SBWriteResGroup58], (instregex "VINSERTF128rm")>; + +def SBWriteResGroup59 : SchedWriteRes<[SBPort23,SBPort15]> { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup59], (instregex "MMX_PADDQirm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PABSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PABSDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PABSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PACKSSDWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PACKSSWBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PACKUSDWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PACKUSWBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDUSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDUSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PADDWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PALIGNRrmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "PAVGBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PAVGWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PBLENDWrmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPEQBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPEQDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPEQQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPEQWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPGTBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPGTDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PCMPGTWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PINSRBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PINSRDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PINSRQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PINSRWrmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMAXSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMAXSDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMAXSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMAXUBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMAXUDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMAXUWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMINSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMINSDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMINSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMINUBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMINUDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMINUWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVSXBDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVSXBQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVSXBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVSXDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVSXWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVSXWQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVZXBDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVZXBQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVZXBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVZXDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVZXWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PMOVZXWQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSHUFBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSHUFDmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSHUFHWmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSHUFLWmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSIGNBrm128")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSIGNDrm128")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSIGNWrm128")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBUSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBUSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PSUBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKHBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKHDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKHQDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKHWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKLBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKLDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKLQDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "PUNPCKLWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPABSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPABSDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPABSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPACKSSDWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPACKSSWBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPACKUSDWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPACKUSWBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDUSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDUSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPADDWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPALIGNRrmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPAVGBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPAVGWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPBLENDWrmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPEQBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPEQDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPEQQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPEQWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPGTBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPGTDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPCMPGTWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPINSRBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPINSRDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPINSRQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPINSRWrmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMAXSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMAXSDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMAXSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMAXUBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMAXUDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMAXUWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMINSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMINSDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMINSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMINUBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMINUDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMINUWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVSXBDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVSXBQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVSXBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVSXDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVSXWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVSXWQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVZXBDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVZXBQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVZXBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVZXDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVZXWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPMOVZXWQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSHUFBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSHUFDmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSHUFHWmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSHUFLWmi")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSIGNBrm128")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSIGNDrm128")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSIGNWrm128")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBUSBrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBUSWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPSUBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKHBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKHDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKHQDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKHWDrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKLBWrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKLDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKLQDQrm")>; +def: InstRW<[SBWriteResGroup59], (instregex "VPUNPCKLWDrm")>; + +def SBWriteResGroup60 : SchedWriteRes<[SBPort23,SBPort015]> { + let Latency = 7; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup60], (instregex "PANDNrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "PANDrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "PORrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "PXORrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "VPANDNrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "VPANDrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "VPORrm")>; +def: InstRW<[SBWriteResGroup60], (instregex "VPXORrm")>; + +def SBWriteResGroup61 : SchedWriteRes<[SBPort0,SBPort0]> { + let Latency = 7; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup61], (instregex "VRCPPSr")>; +def: InstRW<[SBWriteResGroup61], (instregex "VRSQRTPSYr")>; + +def SBWriteResGroup62 : SchedWriteRes<[SBPort5,SBPort23]> { + let Latency = 7; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup62], (instregex "VERRm")>; +def: InstRW<[SBWriteResGroup62], (instregex "VERWm")>; + +def SBWriteResGroup63 : SchedWriteRes<[SBPort23,SBPort015]> { + let Latency = 7; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup63], (instregex "LODSB")>; +def: InstRW<[SBWriteResGroup63], (instregex "LODSW")>; + +def SBWriteResGroup64 : SchedWriteRes<[SBPort5,SBPort01,SBPort23]> { + let Latency = 7; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup64], (instregex "FARJMP64")>; + +def SBWriteResGroup65 : SchedWriteRes<[SBPort23,SBPort0,SBPort015]> { + let Latency = 7; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup65], (instregex "ADC64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "ADC8rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVAE64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVB64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVE64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVG64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVGE64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVL64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVLE64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVNE64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVNO64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVNP64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVNS64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVO64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVP64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "CMOVS64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "SBB64rm")>; +def: InstRW<[SBWriteResGroup65], (instregex "SBB8rm")>; + +def SBWriteResGroup66 : SchedWriteRes<[SBPort0,SBPort4,SBPort23]> { + let Latency = 7; + let NumMicroOps = 4; + let ResourceCycles = [1,1,2]; +} +def: InstRW<[SBWriteResGroup66], (instregex "FNSTSWm")>; + +def SBWriteResGroup67 : SchedWriteRes<[SBPort1,SBPort5,SBPort015]> { + let Latency = 7; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup67], (instregex "SLDT32r")>; +def: InstRW<[SBWriteResGroup67], (instregex "STR32r")>; + +def SBWriteResGroup68 : SchedWriteRes<[SBPort4,SBPort5,SBPort23]> { + let Latency = 7; + let NumMicroOps = 4; + let ResourceCycles = [1,1,2]; +} +def: InstRW<[SBWriteResGroup68], (instregex "CALL64m")>; +def: InstRW<[SBWriteResGroup68], (instregex "FNSTCW16m")>; + +def SBWriteResGroup69 : SchedWriteRes<[SBPort4,SBPort23,SBPort0]> { + let Latency = 7; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup69], (instregex "BTC64mi8")>; +def: InstRW<[SBWriteResGroup69], (instregex "BTR64mi8")>; +def: InstRW<[SBWriteResGroup69], (instregex "BTS64mi8")>; +def: InstRW<[SBWriteResGroup69], (instregex "SAR64mi")>; +def: InstRW<[SBWriteResGroup69], (instregex "SAR8mi")>; +def: InstRW<[SBWriteResGroup69], (instregex "SHL64m1")>; +def: InstRW<[SBWriteResGroup69], (instregex "SHL64mi")>; +def: InstRW<[SBWriteResGroup69], (instregex "SHL8m1")>; +def: InstRW<[SBWriteResGroup69], (instregex "SHL8mi")>; +def: InstRW<[SBWriteResGroup69], (instregex "SHR64mi")>; +def: InstRW<[SBWriteResGroup69], (instregex "SHR8mi")>; + +def SBWriteResGroup70 : SchedWriteRes<[SBPort4,SBPort23,SBPort015]> { + let Latency = 7; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup70], (instregex "ADD64mi8")>; +def: InstRW<[SBWriteResGroup70], (instregex "ADD64mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "ADD8mi")>; +def: InstRW<[SBWriteResGroup70], (instregex "ADD8mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "AND64mi8")>; +def: InstRW<[SBWriteResGroup70], (instregex "AND64mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "AND8mi")>; +def: InstRW<[SBWriteResGroup70], (instregex "AND8mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "DEC64m")>; +def: InstRW<[SBWriteResGroup70], (instregex "DEC8m")>; +def: InstRW<[SBWriteResGroup70], (instregex "INC64m")>; +def: InstRW<[SBWriteResGroup70], (instregex "INC8m")>; +def: InstRW<[SBWriteResGroup70], (instregex "NEG64m")>; +def: InstRW<[SBWriteResGroup70], (instregex "NEG8m")>; +def: InstRW<[SBWriteResGroup70], (instregex "NOT64m")>; +def: InstRW<[SBWriteResGroup70], (instregex "NOT8m")>; +def: InstRW<[SBWriteResGroup70], (instregex "OR64mi8")>; +def: InstRW<[SBWriteResGroup70], (instregex "OR64mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "OR8mi")>; +def: InstRW<[SBWriteResGroup70], (instregex "OR8mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "SUB64mi8")>; +def: InstRW<[SBWriteResGroup70], (instregex "SUB64mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "SUB8mi")>; +def: InstRW<[SBWriteResGroup70], (instregex "SUB8mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "TEST64rm")>; +def: InstRW<[SBWriteResGroup70], (instregex "TEST8mi")>; +def: InstRW<[SBWriteResGroup70], (instregex "TEST8rm")>; +def: InstRW<[SBWriteResGroup70], (instregex "XOR64mi8")>; +def: InstRW<[SBWriteResGroup70], (instregex "XOR64mr")>; +def: InstRW<[SBWriteResGroup70], (instregex "XOR8mi")>; +def: InstRW<[SBWriteResGroup70], (instregex "XOR8mr")>; + +def SBWriteResGroup71 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 8; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup71], (instregex "MMX_PMADDUBSWrm64")>; +def: InstRW<[SBWriteResGroup71], (instregex "MMX_PMULHRSWrm64")>; +def: InstRW<[SBWriteResGroup71], (instregex "VTESTPDYrm")>; +def: InstRW<[SBWriteResGroup71], (instregex "VTESTPSYrm")>; + +def SBWriteResGroup72 : SchedWriteRes<[SBPort1,SBPort23]> { + let Latency = 8; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup72], (instregex "BSF64rm")>; +def: InstRW<[SBWriteResGroup72], (instregex "BSR64rm")>; +def: InstRW<[SBWriteResGroup72], (instregex "CRC32r32m16")>; +def: InstRW<[SBWriteResGroup72], (instregex "CRC32r32m8")>; +def: InstRW<[SBWriteResGroup72], (instregex "FCOM32m")>; +def: InstRW<[SBWriteResGroup72], (instregex "FCOM64m")>; +def: InstRW<[SBWriteResGroup72], (instregex "FCOMP32m")>; +def: InstRW<[SBWriteResGroup72], (instregex "FCOMP64m")>; +def: InstRW<[SBWriteResGroup72], (instregex "MUL8m")>; + +def SBWriteResGroup73 : SchedWriteRes<[SBPort5,SBPort23]> { + let Latency = 8; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup73], (instregex "VANDNPDYrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VANDNPSYrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VANDPDrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VANDPSrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VORPDYrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VORPSYrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VPERM2F128rm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VPERMILPDYri")>; +def: InstRW<[SBWriteResGroup73], (instregex "VPERMILPDmi")>; +def: InstRW<[SBWriteResGroup73], (instregex "VPERMILPSYri")>; +def: InstRW<[SBWriteResGroup73], (instregex "VPERMILPSmi")>; +def: InstRW<[SBWriteResGroup73], (instregex "VSHUFPDYrmi")>; +def: InstRW<[SBWriteResGroup73], (instregex "VSHUFPSYrmi")>; +def: InstRW<[SBWriteResGroup73], (instregex "VUNPCKHPDrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VUNPCKHPSrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VUNPCKLPDYrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VUNPCKLPSYrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VXORPDrm")>; +def: InstRW<[SBWriteResGroup73], (instregex "VXORPSrm")>; + +def SBWriteResGroup74 : SchedWriteRes<[SBPort23,SBPort0]> { + let Latency = 8; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup74], (instregex "VBLENDPDYrmi")>; +def: InstRW<[SBWriteResGroup74], (instregex "VBLENDPSYrmi")>; + +def SBWriteResGroup75 : SchedWriteRes<[SBPort23,SBPort0]> { + let Latency = 8; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup75], (instregex "BLENDVPDrm0")>; +def: InstRW<[SBWriteResGroup75], (instregex "BLENDVPSrm0")>; +def: InstRW<[SBWriteResGroup75], (instregex "VBLENDVPDrm")>; +def: InstRW<[SBWriteResGroup75], (instregex "VBLENDVPSrm")>; +def: InstRW<[SBWriteResGroup75], (instregex "VMASKMOVPDrm")>; +def: InstRW<[SBWriteResGroup75], (instregex "VMASKMOVPSrm")>; + +def SBWriteResGroup76 : SchedWriteRes<[SBPort23,SBPort15]> { + let Latency = 8; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup76], (instregex "PBLENDVBrr0")>; +def: InstRW<[SBWriteResGroup76], (instregex "VPBLENDVBrm")>; + +def SBWriteResGroup77 : SchedWriteRes<[SBPort0,SBPort1,SBPort23]> { + let Latency = 8; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup77], (instregex "COMISDrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "COMISSrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "UCOMISDrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "UCOMISSrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "VCOMISDrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "VCOMISSrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "VUCOMISDrm")>; +def: InstRW<[SBWriteResGroup77], (instregex "VUCOMISSrm")>; + +def SBWriteResGroup78 : SchedWriteRes<[SBPort0,SBPort5,SBPort23]> { + let Latency = 8; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup78], (instregex "PTESTrm")>; +def: InstRW<[SBWriteResGroup78], (instregex "VPTESTrm")>; + +def SBWriteResGroup79 : SchedWriteRes<[SBPort0,SBPort23,SBPort15]> { + let Latency = 8; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup79], (instregex "PSLLDrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSLLQrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSLLWrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSRADrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSRAWrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSRLDrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSRLQrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "PSRLWrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSLLDri")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSLLQri")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSLLWri")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSRADrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSRAWrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSRLDrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSRLQrm")>; +def: InstRW<[SBWriteResGroup79], (instregex "VPSRLWrm")>; + +def SBWriteResGroup80 : SchedWriteRes<[SBPort23,SBPort15]> { + let Latency = 8; + let NumMicroOps = 4; + let ResourceCycles = [1,3]; +} +def: InstRW<[SBWriteResGroup80], (instregex "MMX_PHADDSWrm64")>; +def: InstRW<[SBWriteResGroup80], (instregex "MMX_PHADDWrm64")>; +def: InstRW<[SBWriteResGroup80], (instregex "MMX_PHADDrm64")>; +def: InstRW<[SBWriteResGroup80], (instregex "MMX_PHSUBDrm64")>; +def: InstRW<[SBWriteResGroup80], (instregex "MMX_PHSUBSWrm64")>; +def: InstRW<[SBWriteResGroup80], (instregex "MMX_PHSUBWrm64")>; + +def SBWriteResGroup81 : SchedWriteRes<[SBPort23,SBPort015]> { + let Latency = 8; + let NumMicroOps = 4; + let ResourceCycles = [1,3]; +} +def: InstRW<[SBWriteResGroup81], (instregex "CMPXCHG64rm")>; +def: InstRW<[SBWriteResGroup81], (instregex "CMPXCHG8rm")>; + +def SBWriteResGroup82 : SchedWriteRes<[SBPort23,SBPort0,SBPort015]> { + let Latency = 8; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup82], (instregex "CMOVA64rm")>; +def: InstRW<[SBWriteResGroup82], (instregex "CMOVBE64rm")>; + +def SBWriteResGroup83 : SchedWriteRes<[SBPort23,SBPort015]> { + let Latency = 8; + let NumMicroOps = 5; + let ResourceCycles = [2,3]; +} +def: InstRW<[SBWriteResGroup83], (instregex "CMPSB")>; +def: InstRW<[SBWriteResGroup83], (instregex "CMPSL")>; +def: InstRW<[SBWriteResGroup83], (instregex "CMPSQ")>; +def: InstRW<[SBWriteResGroup83], (instregex "CMPSW")>; + +def SBWriteResGroup84 : SchedWriteRes<[SBPort4,SBPort5,SBPort23]> { + let Latency = 8; + let NumMicroOps = 5; + let ResourceCycles = [1,2,2]; +} +def: InstRW<[SBWriteResGroup84], (instregex "FLDCW16m")>; + +def SBWriteResGroup85 : SchedWriteRes<[SBPort4,SBPort23,SBPort0]> { + let Latency = 8; + let NumMicroOps = 5; + let ResourceCycles = [1,2,2]; +} +def: InstRW<[SBWriteResGroup85], (instregex "ROL64mi")>; +def: InstRW<[SBWriteResGroup85], (instregex "ROL8mi")>; +def: InstRW<[SBWriteResGroup85], (instregex "ROR64mi")>; +def: InstRW<[SBWriteResGroup85], (instregex "ROR8mi")>; + +def SBWriteResGroup86 : SchedWriteRes<[SBPort4,SBPort23,SBPort015]> { + let Latency = 8; + let NumMicroOps = 5; + let ResourceCycles = [1,2,2]; +} +def: InstRW<[SBWriteResGroup86], (instregex "MOVSB")>; +def: InstRW<[SBWriteResGroup86], (instregex "MOVSL")>; +def: InstRW<[SBWriteResGroup86], (instregex "MOVSQ")>; +def: InstRW<[SBWriteResGroup86], (instregex "MOVSW")>; +def: InstRW<[SBWriteResGroup86], (instregex "XADD64rm")>; +def: InstRW<[SBWriteResGroup86], (instregex "XADD8rm")>; + +def SBWriteResGroup87 : SchedWriteRes<[SBPort4,SBPort5,SBPort01,SBPort23]> { + let Latency = 8; + let NumMicroOps = 5; + let ResourceCycles = [1,1,1,2]; +} +def: InstRW<[SBWriteResGroup87], (instregex "FARCALL64")>; + +def SBWriteResGroup88 : SchedWriteRes<[SBPort4,SBPort23,SBPort0,SBPort015]> { + let Latency = 8; + let NumMicroOps = 5; + let ResourceCycles = [1,2,1,1]; +} +def: InstRW<[SBWriteResGroup88], (instregex "SHLD64mri8")>; +def: InstRW<[SBWriteResGroup88], (instregex "SHRD64mri8")>; + +def SBWriteResGroup89 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 9; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup89], (instregex "MMX_PMULUDQirm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMADDUBSWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMADDWDrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULDQrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULHRSWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULHUWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULHWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULLDrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULLWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PMULUDQrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "PSADBWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMADDUBSWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMADDWDrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULDQrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULHRSWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULHUWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULHWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULLDrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULLWrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPMULUDQrm")>; +def: InstRW<[SBWriteResGroup89], (instregex "VPSADBWrm")>; + +def SBWriteResGroup90 : SchedWriteRes<[SBPort1,SBPort23]> { + let Latency = 9; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup90], (instregex "ADDPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ADDPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ADDSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ADDSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ADDSUBPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ADDSUBPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "CMPPDrmi")>; +def: InstRW<[SBWriteResGroup90], (instregex "CMPPSrmi")>; +def: InstRW<[SBWriteResGroup90], (instregex "CMPSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "CVTDQ2PSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "CVTPS2DQrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "CVTSI2SD64rm")>; +def: InstRW<[SBWriteResGroup90], (instregex "CVTSI2SDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "CVTTPS2DQrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MAXPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MAXPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MAXSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MAXSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MINPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MINPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MINSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MINSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MMX_CVTPI2PSirm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MMX_CVTPS2PIirm")>; +def: InstRW<[SBWriteResGroup90], (instregex "MMX_CVTTPS2PIirm")>; +def: InstRW<[SBWriteResGroup90], (instregex "POPCNT64rm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ROUNDPDm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ROUNDPSm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ROUNDSDm")>; +def: InstRW<[SBWriteResGroup90], (instregex "ROUNDSSm")>; +def: InstRW<[SBWriteResGroup90], (instregex "SUBPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "SUBPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "SUBSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "SUBSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VADDPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VADDPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VADDSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VADDSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VADDSUBPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VADDSUBPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCMPPDrmi")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCMPPSrmi")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCMPSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCMPSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCVTDQ2PSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCVTPS2DQrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCVTSI2SD64rm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCVTSI2SDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VCVTTPS2DQrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMAXPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMAXPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMAXSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMAXSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMINPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMINPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMINSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VMINSSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VROUNDPDm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VROUNDPSm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VROUNDSDm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VROUNDSSm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VSUBPDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VSUBPSrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VSUBSDrm")>; +def: InstRW<[SBWriteResGroup90], (instregex "VSUBSSrm")>; + +def SBWriteResGroup91 : SchedWriteRes<[SBPort23,SBPort0]> { + let Latency = 9; + let NumMicroOps = 3; + let ResourceCycles = [1,2]; +} +def: InstRW<[SBWriteResGroup91], (instregex "VBLENDVPDYrm")>; +def: InstRW<[SBWriteResGroup91], (instregex "VBLENDVPSYrm")>; +def: InstRW<[SBWriteResGroup91], (instregex "VMASKMOVPDrm")>; +def: InstRW<[SBWriteResGroup91], (instregex "VMASKMOVPSrm")>; + +def SBWriteResGroup92 : SchedWriteRes<[SBPort0,SBPort1,SBPort5]> { + let Latency = 9; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup92], (instregex "DPPDrri")>; +def: InstRW<[SBWriteResGroup92], (instregex "VDPPDrri")>; + +def SBWriteResGroup93 : SchedWriteRes<[SBPort0,SBPort1,SBPort23]> { + let Latency = 9; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup93], (instregex "CVTSD2SI64rm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTSD2SIrm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTSS2SI64rm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTSS2SIrm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTTSD2SI64rm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTTSD2SIrm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTTSS2SI64rm")>; +def: InstRW<[SBWriteResGroup93], (instregex "CVTTSS2SIrm")>; +def: InstRW<[SBWriteResGroup93], (instregex "MUL64m")>; + +def SBWriteResGroup94 : SchedWriteRes<[SBPort0,SBPort5,SBPort23]> { + let Latency = 9; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup94], (instregex "VPTESTYrm")>; + +def SBWriteResGroup95 : SchedWriteRes<[SBPort5,SBPort01,SBPort23]> { + let Latency = 9; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup95], (instregex "LD_F32m")>; +def: InstRW<[SBWriteResGroup95], (instregex "LD_F64m")>; +def: InstRW<[SBWriteResGroup95], (instregex "LD_F80m")>; + +def SBWriteResGroup96 : SchedWriteRes<[SBPort23,SBPort15]> { + let Latency = 9; + let NumMicroOps = 4; + let ResourceCycles = [1,3]; +} +def: InstRW<[SBWriteResGroup96], (instregex "PHADDDrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "PHADDSWrm128")>; +def: InstRW<[SBWriteResGroup96], (instregex "PHADDWrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "PHSUBDrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "PHSUBSWrm128")>; +def: InstRW<[SBWriteResGroup96], (instregex "PHSUBWrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "VPHADDDrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "VPHADDSWrm128")>; +def: InstRW<[SBWriteResGroup96], (instregex "VPHADDWrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "VPHSUBDrm")>; +def: InstRW<[SBWriteResGroup96], (instregex "VPHSUBSWrm128")>; +def: InstRW<[SBWriteResGroup96], (instregex "VPHSUBWrm")>; + +def SBWriteResGroup97 : SchedWriteRes<[SBPort1,SBPort4,SBPort23]> { + let Latency = 9; + let NumMicroOps = 4; + let ResourceCycles = [1,1,2]; +} +def: InstRW<[SBWriteResGroup97], (instregex "IST_F16m")>; +def: InstRW<[SBWriteResGroup97], (instregex "IST_F32m")>; +def: InstRW<[SBWriteResGroup97], (instregex "IST_FP16m")>; +def: InstRW<[SBWriteResGroup97], (instregex "IST_FP32m")>; +def: InstRW<[SBWriteResGroup97], (instregex "IST_FP64m")>; +def: InstRW<[SBWriteResGroup97], (instregex "SHL64mCL")>; +def: InstRW<[SBWriteResGroup97], (instregex "SHL8mCL")>; + +def SBWriteResGroup98 : SchedWriteRes<[SBPort4,SBPort23,SBPort015]> { + let Latency = 9; + let NumMicroOps = 6; + let ResourceCycles = [1,2,3]; +} +def: InstRW<[SBWriteResGroup98], (instregex "ADC64mi8")>; +def: InstRW<[SBWriteResGroup98], (instregex "ADC8mi")>; +def: InstRW<[SBWriteResGroup98], (instregex "SBB64mi8")>; +def: InstRW<[SBWriteResGroup98], (instregex "SBB8mi")>; + +def SBWriteResGroup99 : SchedWriteRes<[SBPort4,SBPort23,SBPort0,SBPort015]> { + let Latency = 9; + let NumMicroOps = 6; + let ResourceCycles = [1,2,2,1]; +} +def: InstRW<[SBWriteResGroup99], (instregex "ADC64mr")>; +def: InstRW<[SBWriteResGroup99], (instregex "ADC8mr")>; +def: InstRW<[SBWriteResGroup99], (instregex "SBB64mr")>; +def: InstRW<[SBWriteResGroup99], (instregex "SBB8mr")>; + +def SBWriteResGroup100 : SchedWriteRes<[SBPort4,SBPort5,SBPort23,SBPort0,SBPort015]> { + let Latency = 9; + let NumMicroOps = 6; + let ResourceCycles = [1,1,2,1,1]; +} +def: InstRW<[SBWriteResGroup100], (instregex "BT64mr")>; +def: InstRW<[SBWriteResGroup100], (instregex "BTC64mr")>; +def: InstRW<[SBWriteResGroup100], (instregex "BTR64mr")>; +def: InstRW<[SBWriteResGroup100], (instregex "BTS64mr")>; + +def SBWriteResGroup101 : SchedWriteRes<[SBPort1,SBPort23]> { + let Latency = 10; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup101], (instregex "ADD_F32m")>; +def: InstRW<[SBWriteResGroup101], (instregex "ADD_F64m")>; +def: InstRW<[SBWriteResGroup101], (instregex "ILD_F16m")>; +def: InstRW<[SBWriteResGroup101], (instregex "ILD_F32m")>; +def: InstRW<[SBWriteResGroup101], (instregex "ILD_F64m")>; +def: InstRW<[SBWriteResGroup101], (instregex "SUBR_F32m")>; +def: InstRW<[SBWriteResGroup101], (instregex "SUBR_F64m")>; +def: InstRW<[SBWriteResGroup101], (instregex "SUB_F32m")>; +def: InstRW<[SBWriteResGroup101], (instregex "SUB_F64m")>; +def: InstRW<[SBWriteResGroup101], (instregex "VADDPDYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VADDPSYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VADDSUBPDYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VADDSUBPSYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VCMPPDYrmi")>; +def: InstRW<[SBWriteResGroup101], (instregex "VCMPPSYrmi")>; +def: InstRW<[SBWriteResGroup101], (instregex "VCVTDQ2PSYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VCVTPS2DQYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VCVTTPS2DQrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VMAXPDYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VMAXPSYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VMINPDrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VMINPSrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VROUNDPDm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VROUNDPSm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VSUBPDYrm")>; +def: InstRW<[SBWriteResGroup101], (instregex "VSUBPSYrm")>; + +def SBWriteResGroup102 : SchedWriteRes<[SBPort0,SBPort1,SBPort23]> { + let Latency = 10; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup102], (instregex "VCVTSD2SI64rm")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTSD2SI64rr")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTSS2SI64rm")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTSS2SIrm")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTTSD2SI64rm")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTTSD2SI64rr")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTTSS2SI64rm")>; +def: InstRW<[SBWriteResGroup102], (instregex "VCVTTSS2SIrm")>; + +def SBWriteResGroup103 : SchedWriteRes<[SBPort1,SBPort5,SBPort23]> { + let Latency = 10; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup103], (instregex "CVTDQ2PDrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "CVTPD2DQrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "CVTPD2PSrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "CVTSD2SSrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "CVTSI2SS64rm")>; +def: InstRW<[SBWriteResGroup103], (instregex "CVTSI2SSrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "CVTTPD2DQrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "MMX_CVTPD2PIirm")>; +def: InstRW<[SBWriteResGroup103], (instregex "MMX_CVTPI2PDirm")>; +def: InstRW<[SBWriteResGroup103], (instregex "MMX_CVTTPD2PIirm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTDQ2PDYrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTDQ2PDrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTPD2DQrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTPD2PSrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTSD2SSrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTSI2SS64rm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTSI2SSrm")>; +def: InstRW<[SBWriteResGroup103], (instregex "VCVTTPD2DQrm")>; + +def SBWriteResGroup104 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 11; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup104], (instregex "MULPDrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "MULPSrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "MULSDrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "MULSSrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "PCMPGTQrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "PHMINPOSUWrm128")>; +def: InstRW<[SBWriteResGroup104], (instregex "RCPPSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "RCPSSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "RSQRTPSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "RSQRTSSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VMULPDrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VMULPSrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VMULSDrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VMULSSrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VPCMPGTQrm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VPHMINPOSUWrm128")>; +def: InstRW<[SBWriteResGroup104], (instregex "VRCPPSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VRCPSSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VRSQRTPSm")>; +def: InstRW<[SBWriteResGroup104], (instregex "VRSQRTSSm")>; + +def SBWriteResGroup105 : SchedWriteRes<[SBPort0]> { + let Latency = 11; + let NumMicroOps = 3; + let ResourceCycles = [3]; +} +def: InstRW<[SBWriteResGroup105], (instregex "PCMPISTRIrr")>; +def: InstRW<[SBWriteResGroup105], (instregex "PCMPISTRM128rr")>; +def: InstRW<[SBWriteResGroup105], (instregex "VPCMPISTRIrr")>; +def: InstRW<[SBWriteResGroup105], (instregex "VPCMPISTRM128rr")>; + +def SBWriteResGroup106 : SchedWriteRes<[SBPort1,SBPort23]> { + let Latency = 11; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup106], (instregex "FICOM16m")>; +def: InstRW<[SBWriteResGroup106], (instregex "FICOM32m")>; +def: InstRW<[SBWriteResGroup106], (instregex "FICOMP16m")>; +def: InstRW<[SBWriteResGroup106], (instregex "FICOMP32m")>; + +def SBWriteResGroup107 : SchedWriteRes<[SBPort1,SBPort5,SBPort23]> { + let Latency = 11; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup107], (instregex "VCVTPD2DQYrm")>; +def: InstRW<[SBWriteResGroup107], (instregex "VCVTPD2PSYrm")>; +def: InstRW<[SBWriteResGroup107], (instregex "VCVTTPD2DQYrm")>; + +def SBWriteResGroup108 : SchedWriteRes<[SBPort0,SBPort23,SBPort15]> { + let Latency = 11; + let NumMicroOps = 4; + let ResourceCycles = [1,1,2]; +} +def: InstRW<[SBWriteResGroup108], (instregex "MPSADBWrmi")>; +def: InstRW<[SBWriteResGroup108], (instregex "VMPSADBWrmi")>; + +def SBWriteResGroup109 : SchedWriteRes<[SBPort1,SBPort5,SBPort23]> { + let Latency = 11; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup109], (instregex "HADDPDrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "HADDPSrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "HSUBPDrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "HSUBPSrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "VHADDPDrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "VHADDPSrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "VHSUBPDrm")>; +def: InstRW<[SBWriteResGroup109], (instregex "VHSUBPSrm")>; + +def SBWriteResGroup110 : SchedWriteRes<[SBPort5]> { + let Latency = 12; + let NumMicroOps = 2; + let ResourceCycles = [2]; +} +def: InstRW<[SBWriteResGroup110], (instregex "AESIMCrr")>; +def: InstRW<[SBWriteResGroup110], (instregex "VAESIMCrr")>; + +def SBWriteResGroup111 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 12; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup111], (instregex "MUL_F32m")>; +def: InstRW<[SBWriteResGroup111], (instregex "MUL_F64m")>; +def: InstRW<[SBWriteResGroup111], (instregex "VMULPDYrm")>; +def: InstRW<[SBWriteResGroup111], (instregex "VMULPSYrm")>; + +def SBWriteResGroup112 : SchedWriteRes<[SBPort0,SBPort1,SBPort5]> { + let Latency = 12; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup112], (instregex "DPPSrri")>; +def: InstRW<[SBWriteResGroup112], (instregex "VDPPSYrri")>; +def: InstRW<[SBWriteResGroup112], (instregex "VDPPSrri")>; + +def SBWriteResGroup113 : SchedWriteRes<[SBPort1,SBPort5,SBPort23]> { + let Latency = 12; + let NumMicroOps = 4; + let ResourceCycles = [1,2,1]; +} +def: InstRW<[SBWriteResGroup113], (instregex "VHADDPDrm")>; +def: InstRW<[SBWriteResGroup113], (instregex "VHADDPSYrm")>; +def: InstRW<[SBWriteResGroup113], (instregex "VHSUBPDYrm")>; +def: InstRW<[SBWriteResGroup113], (instregex "VHSUBPSYrm")>; + +def SBWriteResGroup114 : SchedWriteRes<[SBPort1,SBPort23]> { + let Latency = 13; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup114], (instregex "ADD_FI16m")>; +def: InstRW<[SBWriteResGroup114], (instregex "ADD_FI32m")>; +def: InstRW<[SBWriteResGroup114], (instregex "SUBR_FI16m")>; +def: InstRW<[SBWriteResGroup114], (instregex "SUBR_FI32m")>; +def: InstRW<[SBWriteResGroup114], (instregex "SUB_FI16m")>; +def: InstRW<[SBWriteResGroup114], (instregex "SUB_FI32m")>; + +def SBWriteResGroup115 : SchedWriteRes<[SBPort5,SBPort23,SBPort015]> { + let Latency = 13; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup115], (instregex "AESDECLASTrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "AESDECrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "AESENCLASTrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "AESENCrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "VAESDECLASTrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "VAESDECrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "VAESENCLASTrm")>; +def: InstRW<[SBWriteResGroup115], (instregex "VAESENCrm")>; + +def SBWriteResGroup116 : SchedWriteRes<[SBPort0]> { + let Latency = 14; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup116], (instregex "DIVPSrr")>; +def: InstRW<[SBWriteResGroup116], (instregex "DIVSSrr")>; +def: InstRW<[SBWriteResGroup116], (instregex "SQRTPSr")>; +def: InstRW<[SBWriteResGroup116], (instregex "SQRTSSr")>; +def: InstRW<[SBWriteResGroup116], (instregex "VDIVPSrr")>; +def: InstRW<[SBWriteResGroup116], (instregex "VDIVSSrr")>; +def: InstRW<[SBWriteResGroup116], (instregex "VSQRTPSr")>; + +def SBWriteResGroup117 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 14; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup117], (instregex "VSQRTSSm")>; + +def SBWriteResGroup118 : SchedWriteRes<[SBPort0,SBPort23,SBPort0]> { + let Latency = 14; + let NumMicroOps = 4; + let ResourceCycles = [2,1,1]; +} +def: InstRW<[SBWriteResGroup118], (instregex "VRCPPSm")>; +def: InstRW<[SBWriteResGroup118], (instregex "VRSQRTPSYm")>; + +def SBWriteResGroup119 : SchedWriteRes<[SBPort0,SBPort1,SBPort23]> { + let Latency = 15; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup119], (instregex "MUL_FI16m")>; +def: InstRW<[SBWriteResGroup119], (instregex "MUL_FI32m")>; + +def SBWriteResGroup120 : SchedWriteRes<[SBPort0,SBPort1,SBPort5,SBPort23]> { + let Latency = 15; + let NumMicroOps = 4; + let ResourceCycles = [1,1,1,1]; +} +def: InstRW<[SBWriteResGroup120], (instregex "DPPDrmi")>; +def: InstRW<[SBWriteResGroup120], (instregex "VDPPDrmi")>; + +def SBWriteResGroup121 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 17; + let NumMicroOps = 4; + let ResourceCycles = [3,1]; +} +def: InstRW<[SBWriteResGroup121], (instregex "PCMPISTRIrm")>; +def: InstRW<[SBWriteResGroup121], (instregex "PCMPISTRM128rm")>; +def: InstRW<[SBWriteResGroup121], (instregex "VPCMPISTRIrm")>; +def: InstRW<[SBWriteResGroup121], (instregex "VPCMPISTRM128rm")>; + +def SBWriteResGroup122 : SchedWriteRes<[SBPort5,SBPort23]> { + let Latency = 18; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup122], (instregex "AESIMCrm")>; +def: InstRW<[SBWriteResGroup122], (instregex "VAESIMCrm")>; + +def SBWriteResGroup123 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 20; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup123], (instregex "DIVPSrm")>; +def: InstRW<[SBWriteResGroup123], (instregex "DIVSSrm")>; +def: InstRW<[SBWriteResGroup123], (instregex "SQRTPSm")>; +def: InstRW<[SBWriteResGroup123], (instregex "SQRTSSm")>; +def: InstRW<[SBWriteResGroup123], (instregex "VDIVPSrm")>; +def: InstRW<[SBWriteResGroup123], (instregex "VDIVSSrm")>; +def: InstRW<[SBWriteResGroup123], (instregex "VSQRTPSm")>; + +def SBWriteResGroup124 : SchedWriteRes<[SBPort0]> { + let Latency = 21; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup124], (instregex "VSQRTSDr")>; + +def SBWriteResGroup125 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 21; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup125], (instregex "VSQRTSDm")>; + +def SBWriteResGroup126 : SchedWriteRes<[SBPort0]> { + let Latency = 22; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup126], (instregex "DIVPDrr")>; +def: InstRW<[SBWriteResGroup126], (instregex "DIVSDrr")>; +def: InstRW<[SBWriteResGroup126], (instregex "SQRTPDr")>; +def: InstRW<[SBWriteResGroup126], (instregex "SQRTSDr")>; +def: InstRW<[SBWriteResGroup126], (instregex "VDIVPDrr")>; +def: InstRW<[SBWriteResGroup126], (instregex "VDIVSDrr")>; +def: InstRW<[SBWriteResGroup126], (instregex "VSQRTPDr")>; + +def SBWriteResGroup127 : SchedWriteRes<[SBPort0]> { + let Latency = 24; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup127], (instregex "DIVR_FPrST0")>; +def: InstRW<[SBWriteResGroup127], (instregex "DIVR_FST0r")>; +def: InstRW<[SBWriteResGroup127], (instregex "DIVR_FrST0")>; +def: InstRW<[SBWriteResGroup127], (instregex "DIV_FPrST0")>; +def: InstRW<[SBWriteResGroup127], (instregex "DIV_FST0r")>; +def: InstRW<[SBWriteResGroup127], (instregex "DIV_FrST0")>; + +def SBWriteResGroup128 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 28; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup128], (instregex "DIVPDrm")>; +def: InstRW<[SBWriteResGroup128], (instregex "DIVSDrm")>; +def: InstRW<[SBWriteResGroup128], (instregex "SQRTPDm")>; +def: InstRW<[SBWriteResGroup128], (instregex "SQRTSDm")>; +def: InstRW<[SBWriteResGroup128], (instregex "VDIVPDrm")>; +def: InstRW<[SBWriteResGroup128], (instregex "VDIVSDrm")>; +def: InstRW<[SBWriteResGroup128], (instregex "VSQRTPDm")>; + +def SBWriteResGroup129 : SchedWriteRes<[SBPort0,SBPort0]> { + let Latency = 29; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup129], (instregex "VDIVPSYrr")>; +def: InstRW<[SBWriteResGroup129], (instregex "VSQRTPSYr")>; + +def SBWriteResGroup130 : SchedWriteRes<[SBPort0,SBPort23]> { + let Latency = 31; + let NumMicroOps = 2; + let ResourceCycles = [1,1]; +} +def: InstRW<[SBWriteResGroup130], (instregex "DIVR_F32m")>; +def: InstRW<[SBWriteResGroup130], (instregex "DIVR_F64m")>; +def: InstRW<[SBWriteResGroup130], (instregex "DIV_F32m")>; +def: InstRW<[SBWriteResGroup130], (instregex "DIV_F64m")>; + +def SBWriteResGroup131 : SchedWriteRes<[SBPort0,SBPort1,SBPort23]> { + let Latency = 34; + let NumMicroOps = 3; + let ResourceCycles = [1,1,1]; +} +def: InstRW<[SBWriteResGroup131], (instregex "DIVR_FI16m")>; +def: InstRW<[SBWriteResGroup131], (instregex "DIVR_FI32m")>; +def: InstRW<[SBWriteResGroup131], (instregex "DIV_FI16m")>; +def: InstRW<[SBWriteResGroup131], (instregex "DIV_FI32m")>; + +def SBWriteResGroup132 : SchedWriteRes<[SBPort0,SBPort23,SBPort0]> { + let Latency = 36; + let NumMicroOps = 4; + let ResourceCycles = [2,1,1]; +} +def: InstRW<[SBWriteResGroup132], (instregex "VDIVPSYrm")>; +def: InstRW<[SBWriteResGroup132], (instregex "VSQRTPSYm")>; + +def SBWriteResGroup133 : SchedWriteRes<[SBPort0,SBPort0]> { + let Latency = 45; + let NumMicroOps = 3; + let ResourceCycles = [2,1]; +} +def: InstRW<[SBWriteResGroup133], (instregex "VDIVPDYrr")>; +def: InstRW<[SBWriteResGroup133], (instregex "VSQRTPDYr")>; + +def SBWriteResGroup134 : SchedWriteRes<[SBPort0,SBPort23,SBPort0]> { + let Latency = 52; + let NumMicroOps = 4; + let ResourceCycles = [2,1,1]; +} +def: InstRW<[SBWriteResGroup134], (instregex "VDIVPDYrm")>; +def: InstRW<[SBWriteResGroup134], (instregex "VSQRTPDYm")>; + +def SBWriteResGroup135 : SchedWriteRes<[SBPort0]> { + let Latency = 114; + let NumMicroOps = 1; + let ResourceCycles = [1]; +} +def: InstRW<[SBWriteResGroup135], (instregex "VSQRTSSr")>; + } // SchedModel diff --git a/contrib/llvm/lib/Target/X86/X86ScheduleBtVer2.td b/contrib/llvm/lib/Target/X86/X86ScheduleBtVer2.td index 6cb2a3694d92..ed53893b779c 100644 --- a/contrib/llvm/lib/Target/X86/X86ScheduleBtVer2.td +++ b/contrib/llvm/lib/Target/X86/X86ScheduleBtVer2.td @@ -369,5 +369,82 @@ def : WriteRes { let Latency = 100; } def : WriteRes { let Latency = 100; } def : WriteRes; def : WriteRes; + +//////////////////////////////////////////////////////////////////////////////// +// AVX instructions. +//////////////////////////////////////////////////////////////////////////////// + +def WriteFAddY: SchedWriteRes<[JFPU0]> { + let Latency = 3; + let ResourceCycles = [2]; +} +def : InstRW<[WriteFAddY], (instregex "VADD(SUB)?P(S|D)Yrr", "VSUBP(S|D)Yrr")>; + +def WriteFAddYLd: SchedWriteRes<[JLAGU, JFPU0]> { + let Latency = 8; + let ResourceCycles = [1, 2]; +} +def : InstRW<[WriteFAddYLd, ReadAfterLd], (instregex "VADD(SUB)?P(S|D)Yrm", "VSUBP(S|D)Yrm")>; + +def WriteFDivY: SchedWriteRes<[JFPU1]> { + let Latency = 38; + let ResourceCycles = [38]; +} +def : InstRW<[WriteFDivY], (instregex "VDIVP(D|S)Yrr")>; + +def WriteFDivYLd: SchedWriteRes<[JLAGU, JFPU1]> { + let Latency = 43; + let ResourceCycles = [1, 38]; +} +def : InstRW<[WriteFDivYLd, ReadAfterLd], (instregex "VDIVP(S|D)Yrm")>; + +def WriteVMULYPD: SchedWriteRes<[JFPU1]> { + let Latency = 4; + let ResourceCycles = [4]; +} +def : InstRW<[WriteVMULYPD], (instregex "VMULPDYrr")>; + +def WriteVMULYPDLd: SchedWriteRes<[JLAGU, JFPU1]> { + let Latency = 9; + let ResourceCycles = [1, 4]; +} +def : InstRW<[WriteVMULYPDLd, ReadAfterLd], (instregex "VMULPDYrm")>; + +def WriteVMULYPS: SchedWriteRes<[JFPU1]> { + let Latency = 2; + let ResourceCycles = [2]; +} +def : InstRW<[WriteVMULYPS], (instregex "VMULPSYrr", "VRCPPSYr", "VRSQRTPSYr")>; + +def WriteVMULYPSLd: SchedWriteRes<[JLAGU, JFPU1]> { + let Latency = 7; + let ResourceCycles = [1, 2]; +} +def : InstRW<[WriteVMULYPSLd, ReadAfterLd], (instregex "VMULPSYrm", "VRCPPSYm", "VRSQRTPSYm")>; + +def WriteVSQRTYPD: SchedWriteRes<[JFPU1]> { + let Latency = 54; + let ResourceCycles = [54]; +} +def : InstRW<[WriteVSQRTYPD], (instregex "VSQRTPDYr")>; + +def WriteVSQRTYPDLd: SchedWriteRes<[JLAGU, JFPU1]> { + let Latency = 59; + let ResourceCycles = [1, 54]; +} +def : InstRW<[WriteVSQRTYPDLd], (instregex "VSQRTPDYm")>; + +def WriteVSQRTYPS: SchedWriteRes<[JFPU1]> { + let Latency = 42; + let ResourceCycles = [42]; +} +def : InstRW<[WriteVSQRTYPS], (instregex "VSQRTPSYr")>; + +def WriteVSQRTYPSLd: SchedWriteRes<[JLAGU, JFPU1]> { + let Latency = 47; + let ResourceCycles = [1, 42]; +} +def : InstRW<[WriteVSQRTYPSLd], (instregex "VSQRTPSYm")>; + } // SchedModel diff --git a/contrib/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/contrib/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 5ba8534d32d3..c9924f264939 100644 --- a/contrib/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/contrib/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -142,10 +142,15 @@ int X86TTIImpl::getArithmeticInstrCost( { ISD::FDIV, MVT::v2f64, 69 }, // divpd { ISD::FADD, MVT::v2f64, 2 }, // addpd { ISD::FSUB, MVT::v2f64, 2 }, // subpd - // v2i64/v4i64 mul is custom lowered as a series of long - // multiplies(3), shifts(3) and adds(2). - // slm muldq version throughput is 2 - { ISD::MUL, MVT::v2i64, 11 }, + // v2i64/v4i64 mul is custom lowered as a series of long: + // multiplies(3), shifts(3) and adds(2) + // slm muldq version throughput is 2 and addq throughput 4 + // thus: 3X2 (muldq throughput) + 3X1 (shift throuput) + + // 3X4 (addq throughput) = 17 + { ISD::MUL, MVT::v2i64, 17 }, + // slm addq\subq throughput is 4 + { ISD::ADD, MVT::v2i64, 4 }, + { ISD::SUB, MVT::v2i64, 4 }, }; if (ST->isSLM()) { diff --git a/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp index d8cf8d3f5da2..53223ab44316 100644 --- a/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/contrib/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -124,6 +124,10 @@ doPromotion(Function *F, SmallPtrSetImpl &ArgsToPromote, } else if (I->use_empty()) { // Dead argument (which are always marked as promotable) ++NumArgumentsDead; + + // There may be remaining metadata uses of the argument for things like + // llvm.dbg.value. Replace them with undef. + I->replaceAllUsesWith(UndefValue::get(I->getType())); } else { // Okay, this is being promoted. This means that the only uses are loads // or GEPs which are only used by loads diff --git a/contrib/llvm/lib/Transforms/IPO/FunctionImport.cpp b/contrib/llvm/lib/Transforms/IPO/FunctionImport.cpp index 6d34ab8b0d96..233a36d2bc54 100644 --- a/contrib/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/contrib/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -64,6 +64,12 @@ static cl::opt ImportHotMultiplier( "import-hot-multiplier", cl::init(3.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for hot callsites")); +static cl::opt ImportCriticalMultiplier( + "import-critical-multiplier", cl::init(100.0), cl::Hidden, + cl::value_desc("x"), + cl::desc( + "Multiply the `import-instr-limit` threshold for critical callsites")); + // FIXME: This multiplier was not really tuned up. static cl::opt ImportColdMultiplier( "import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), @@ -207,6 +213,8 @@ static void computeImportForFunction( return ImportHotMultiplier; if (Hotness == CalleeInfo::HotnessType::Cold) return ImportColdMultiplier; + if (Hotness == CalleeInfo::HotnessType::Critical) + return ImportCriticalMultiplier; return 1.0; }; @@ -537,8 +545,6 @@ void llvm::thinLTOResolveWeakForLinkerModule( }; auto updateLinkage = [&](GlobalValue &GV) { - if (!GlobalValue::isWeakForLinker(GV.getLinkage())) - return; // See if the global summary analysis computed a new resolved linkage. const auto &GS = DefinedGlobals.find(GV.getGUID()); if (GS == DefinedGlobals.end()) @@ -546,6 +552,21 @@ void llvm::thinLTOResolveWeakForLinkerModule( auto NewLinkage = GS->second->linkage(); if (NewLinkage == GV.getLinkage()) return; + + // Switch the linkage to weakany if asked for, e.g. we do this for + // linker redefined symbols (via --wrap or --defsym). + // We record that the visibility should be changed here in `addThinLTO` + // as we need access to the resolution vectors for each input file in + // order to find which symbols have been redefined. + // We may consider reorganizing this code and moving the linkage recording + // somewhere else, e.g. in thinLTOResolveWeakForLinkerInIndex. + if (NewLinkage == GlobalValue::WeakAnyLinkage) { + GV.setLinkage(NewLinkage); + return; + } + + if (!GlobalValue::isWeakForLinker(GV.getLinkage())) + return; // Check for a non-prevailing def that has interposable linkage // (e.g. non-odr weak or linkonce). In that case we can't simply // convert to available_externally, since it would lose the diff --git a/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp index f277a51ae659..3d57acf06e74 100644 --- a/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/contrib/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -837,7 +837,7 @@ OptimizeGlobalAddressOfMalloc(GlobalVariable *GV, CallInst *CI, Type *AllocTy, if (StoreInst *SI = dyn_cast(GV->user_back())) { // The global is initialized when the store to it occurs. new StoreInst(ConstantInt::getTrue(GV->getContext()), InitBool, false, 0, - SI->getOrdering(), SI->getSynchScope(), SI); + SI->getOrdering(), SI->getSyncScopeID(), SI); SI->eraseFromParent(); continue; } @@ -854,7 +854,7 @@ OptimizeGlobalAddressOfMalloc(GlobalVariable *GV, CallInst *CI, Type *AllocTy, // Replace the cmp X, 0 with a use of the bool value. // Sink the load to where the compare was, if atomic rules allow us to. Value *LV = new LoadInst(InitBool, InitBool->getName()+".val", false, 0, - LI->getOrdering(), LI->getSynchScope(), + LI->getOrdering(), LI->getSyncScopeID(), LI->isUnordered() ? (Instruction*)ICI : LI); InitBoolUsed = true; switch (ICI->getPredicate()) { @@ -1605,7 +1605,7 @@ static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) { assert(LI->getOperand(0) == GV && "Not a copy!"); // Insert a new load, to preserve the saved value. StoreVal = new LoadInst(NewGV, LI->getName()+".b", false, 0, - LI->getOrdering(), LI->getSynchScope(), LI); + LI->getOrdering(), LI->getSyncScopeID(), LI); } else { assert((isa(StoredVal) || isa(StoredVal)) && "This is not a form that we understand!"); @@ -1614,12 +1614,12 @@ static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) { } } new StoreInst(StoreVal, NewGV, false, 0, - SI->getOrdering(), SI->getSynchScope(), SI); + SI->getOrdering(), SI->getSyncScopeID(), SI); } else { // Change the load into a load of bool then a select. LoadInst *LI = cast(UI); LoadInst *NLI = new LoadInst(NewGV, LI->getName()+".b", false, 0, - LI->getOrdering(), LI->getSynchScope(), LI); + LI->getOrdering(), LI->getSyncScopeID(), LI); Value *NSI; if (IsOneZero) NSI = new ZExtInst(NLI, LI->getType(), "", LI); diff --git a/contrib/llvm/lib/Transforms/IPO/Inliner.cpp b/contrib/llvm/lib/Transforms/IPO/Inliner.cpp index ad89e40661c6..00ddb93df830 100644 --- a/contrib/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/contrib/llvm/lib/Transforms/IPO/Inliner.cpp @@ -989,5 +989,13 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, // And delete the actual function from the module. M.getFunctionList().erase(DeadF); } - return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + + if (!Changed) + return PreservedAnalyses::all(); + + // Even if we change the IR, we update the core CGSCC data structures and so + // can preserve the proxy to the function analysis manager. + PreservedAnalyses PA; + PA.preserve(); + return PA; } diff --git a/contrib/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/contrib/llvm/lib/Transforms/IPO/LowerTypeTests.cpp index b406c22c69d7..693df5e7ba92 100644 --- a/contrib/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/contrib/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -855,15 +855,20 @@ void LowerTypeTestsModule::importFunction(Function *F, bool isDefinition) { FDecl = Function::Create(F->getFunctionType(), GlobalValue::ExternalLinkage, Name + ".cfi_jt", &M); FDecl->setVisibility(GlobalValue::HiddenVisibility); - } else { - // Definition. - assert(isDefinition); + } else if (isDefinition) { F->setName(Name + ".cfi"); F->setLinkage(GlobalValue::ExternalLinkage); F->setVisibility(GlobalValue::HiddenVisibility); FDecl = Function::Create(F->getFunctionType(), GlobalValue::ExternalLinkage, Name, &M); FDecl->setVisibility(Visibility); + } else { + // Function definition without type metadata, where some other translation + // unit contained a declaration with type metadata. This normally happens + // during mixed CFI + non-CFI compilation. We do nothing with the function + // so that it is treated the same way as a function defined outside of the + // LTO unit. + return; } if (F->isWeakForLinker()) diff --git a/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp index 5b1b58b89c32..0b319f6a488b 100644 --- a/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/contrib/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -188,6 +188,13 @@ PassManagerBuilder::~PassManagerBuilder() { static ManagedStatic, 8> > GlobalExtensions; +/// Check if GlobalExtensions is constructed and not empty. +/// Since GlobalExtensions is a managed static, calling 'empty()' will trigger +/// the construction of the object. +static bool GlobalExtensionsNotEmpty() { + return GlobalExtensions.isConstructed() && !GlobalExtensions->empty(); +} + void PassManagerBuilder::addGlobalExtension( PassManagerBuilder::ExtensionPointTy Ty, PassManagerBuilder::ExtensionFn Fn) { @@ -200,9 +207,12 @@ void PassManagerBuilder::addExtension(ExtensionPointTy Ty, ExtensionFn Fn) { void PassManagerBuilder::addExtensionsToPM(ExtensionPointTy ETy, legacy::PassManagerBase &PM) const { - for (unsigned i = 0, e = GlobalExtensions->size(); i != e; ++i) - if ((*GlobalExtensions)[i].first == ETy) - (*GlobalExtensions)[i].second(*this, PM); + if (GlobalExtensionsNotEmpty()) { + for (auto &Ext : *GlobalExtensions) { + if (Ext.first == ETy) + Ext.second(*this, PM); + } + } for (unsigned i = 0, e = Extensions.size(); i != e; ++i) if (Extensions[i].first == ETy) Extensions[i].second(*this, PM); @@ -415,7 +425,7 @@ void PassManagerBuilder::populateModulePassManager( // builds. The function merging pass is if (MergeFunctions) MPM.add(createMergeFunctionsPass()); - else if (!GlobalExtensions->empty() || !Extensions.empty()) + else if (GlobalExtensionsNotEmpty() || !Extensions.empty()) MPM.add(createBarrierNoopPass()); addExtensionsToPM(EP_EnabledOnOptLevel0, MPM); diff --git a/contrib/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/contrib/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp index 8d494fe9cde2..8ef6bb652309 100644 --- a/contrib/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/contrib/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -271,7 +271,8 @@ void splitAndWriteThinLTOBitcode( if (!ArgT || ArgT->getBitWidth() > 64) return; } - if (computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) == MAK_ReadNone) + if (!F->isDeclaration() && + computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) == MAK_ReadNone) EligibleVirtualFns.insert(F); }); } diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index d5f0dd191415..809471cfd74f 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -164,7 +164,7 @@ namespace { /// class FAddCombine { public: - FAddCombine(InstCombiner::BuilderTy *B) : Builder(B), Instr(nullptr) {} + FAddCombine(InstCombiner::BuilderTy &B) : Builder(B), Instr(nullptr) {} Value *simplify(Instruction *FAdd); private: @@ -187,7 +187,7 @@ namespace { Value *createNaryFAdd(const AddendVect& Opnds, unsigned InstrQuota); void createInstPostProc(Instruction *NewInst, bool NoNumber = false); - InstCombiner::BuilderTy *Builder; + InstCombiner::BuilderTy &Builder; Instruction *Instr; // Debugging stuff are clustered here. @@ -735,7 +735,7 @@ Value *FAddCombine::createNaryFAdd } Value *FAddCombine::createFSub(Value *Opnd0, Value *Opnd1) { - Value *V = Builder->CreateFSub(Opnd0, Opnd1); + Value *V = Builder.CreateFSub(Opnd0, Opnd1); if (Instruction *I = dyn_cast(V)) createInstPostProc(I); return V; @@ -750,21 +750,21 @@ Value *FAddCombine::createFNeg(Value *V) { } Value *FAddCombine::createFAdd(Value *Opnd0, Value *Opnd1) { - Value *V = Builder->CreateFAdd(Opnd0, Opnd1); + Value *V = Builder.CreateFAdd(Opnd0, Opnd1); if (Instruction *I = dyn_cast(V)) createInstPostProc(I); return V; } Value *FAddCombine::createFMul(Value *Opnd0, Value *Opnd1) { - Value *V = Builder->CreateFMul(Opnd0, Opnd1); + Value *V = Builder.CreateFMul(Opnd0, Opnd1); if (Instruction *I = dyn_cast(V)) createInstPostProc(I); return V; } Value *FAddCombine::createFDiv(Value *Opnd0, Value *Opnd1) { - Value *V = Builder->CreateFDiv(Opnd0, Opnd1); + Value *V = Builder.CreateFDiv(Opnd0, Opnd1); if (Instruction *I = dyn_cast(V)) createInstPostProc(I); return V; @@ -895,7 +895,7 @@ bool InstCombiner::willNotOverflowUnsignedSub(const Value *LHS, // ADD(XOR(AND(Z, C), C), 1) == NEG(OR(Z, ~C)) // XOR(AND(Z, C), (C + 1)) == NEG(OR(Z, ~C)) if C is even static Value *checkForNegativeOperand(BinaryOperator &I, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); // This function creates 2 instructions to replace ADD, we need at least one @@ -919,13 +919,13 @@ static Value *checkForNegativeOperand(BinaryOperator &I, // X = XOR(Y, C1), Y = OR(Z, C2), C2 = NOT(C1) ==> X == NOT(AND(Z, C1)) // ADD(ADD(X, 1), RHS) == ADD(X, ADD(RHS, 1)) == SUB(RHS, AND(Z, C1)) if (match(Y, m_Or(m_Value(Z), m_APInt(C2))) && (*C2 == ~(*C1))) { - Value *NewAnd = Builder->CreateAnd(Z, *C1); - return Builder->CreateSub(RHS, NewAnd, "sub"); + Value *NewAnd = Builder.CreateAnd(Z, *C1); + return Builder.CreateSub(RHS, NewAnd, "sub"); } else if (match(Y, m_And(m_Value(Z), m_APInt(C2))) && (*C1 == *C2)) { // X = XOR(Y, C1), Y = AND(Z, C2), C2 == C1 ==> X == NOT(OR(Z, ~C1)) // ADD(ADD(X, 1), RHS) == ADD(X, ADD(RHS, 1)) == SUB(RHS, OR(Z, ~C1)) - Value *NewOr = Builder->CreateOr(Z, ~(*C1)); - return Builder->CreateSub(RHS, NewOr, "sub"); + Value *NewOr = Builder.CreateOr(Z, ~(*C1)); + return Builder.CreateSub(RHS, NewOr, "sub"); } } } @@ -944,8 +944,8 @@ static Value *checkForNegativeOperand(BinaryOperator &I, if (match(LHS, m_Xor(m_Value(Y), m_APInt(C1)))) if (C1->countTrailingZeros() == 0) if (match(Y, m_And(m_Value(Z), m_APInt(C2))) && *C1 == (*C2 + 1)) { - Value *NewOr = Builder->CreateOr(Z, ~(*C2)); - return Builder->CreateSub(RHS, NewOr, "sub"); + Value *NewOr = Builder.CreateOr(Z, ~(*C2)); + return Builder.CreateSub(RHS, NewOr, "sub"); } return nullptr; } @@ -1027,7 +1027,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { if (Value *V = SimplifyUsingDistributiveLaws(I)) return replaceInstUsesWith(I, V); - if (Instruction *X = foldAddWithConstant(I, *Builder)) + if (Instruction *X = foldAddWithConstant(I, Builder)) return X; // FIXME: This should be moved into the above helper function to allow these @@ -1060,7 +1060,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { if (ExtendAmt) { Constant *ShAmt = ConstantInt::get(I.getType(), ExtendAmt); - Value *NewShl = Builder->CreateShl(XorLHS, ShAmt, "sext"); + Value *NewShl = Builder.CreateShl(XorLHS, ShAmt, "sext"); return BinaryOperator::CreateAShr(NewShl, ShAmt); } @@ -1084,7 +1084,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { if (Instruction *NV = foldOpWithConstantIntoOperand(I)) return NV; - if (I.getType()->getScalarType()->isIntegerTy(1)) + if (I.getType()->isIntOrIntVectorTy(1)) return BinaryOperator::CreateXor(LHS, RHS); // X + X --> X << 1 @@ -1101,7 +1101,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { if (Value *LHSV = dyn_castNegVal(LHS)) { if (!isa(RHS)) if (Value *RHSV = dyn_castNegVal(RHS)) { - Value *NewAdd = Builder->CreateAdd(LHSV, RHSV, "sum"); + Value *NewAdd = Builder.CreateAdd(LHSV, RHSV, "sum"); return BinaryOperator::CreateNeg(NewAdd); } @@ -1148,7 +1148,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { if (AddRHSHighBits == AddRHSHighBitsAnd) { // Okay, the xform is safe. Insert the new add pronto. - Value *NewAdd = Builder->CreateAdd(X, CRHS, LHS->getName()); + Value *NewAdd = Builder.CreateAdd(X, CRHS, LHS->getName()); return BinaryOperator::CreateAnd(NewAdd, C2); } } @@ -1191,7 +1191,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { willNotOverflowSignedAdd(LHSConv->getOperand(0), CI, I)) { // Insert the new, smaller add. Value *NewAdd = - Builder->CreateNSWAdd(LHSConv->getOperand(0), CI, "addconv"); + Builder.CreateNSWAdd(LHSConv->getOperand(0), CI, "addconv"); return new SExtInst(NewAdd, I.getType()); } } @@ -1208,7 +1208,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { willNotOverflowSignedAdd(LHSConv->getOperand(0), RHSConv->getOperand(0), I)) { // Insert the new integer add. - Value *NewAdd = Builder->CreateNSWAdd(LHSConv->getOperand(0), + Value *NewAdd = Builder.CreateNSWAdd(LHSConv->getOperand(0), RHSConv->getOperand(0), "addconv"); return new SExtInst(NewAdd, I.getType()); } @@ -1227,7 +1227,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { willNotOverflowUnsignedAdd(LHSConv->getOperand(0), CI, I)) { // Insert the new, smaller add. Value *NewAdd = - Builder->CreateNUWAdd(LHSConv->getOperand(0), CI, "addconv"); + Builder.CreateNUWAdd(LHSConv->getOperand(0), CI, "addconv"); return new ZExtInst(NewAdd, I.getType()); } } @@ -1244,7 +1244,7 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) { willNotOverflowUnsignedAdd(LHSConv->getOperand(0), RHSConv->getOperand(0), I)) { // Insert the new integer add. - Value *NewAdd = Builder->CreateNUWAdd( + Value *NewAdd = Builder.CreateNUWAdd( LHSConv->getOperand(0), RHSConv->getOperand(0), "addconv"); return new ZExtInst(NewAdd, I.getType()); } @@ -1362,8 +1362,7 @@ Instruction *InstCombiner::visitFAdd(BinaryOperator &I) { ConstantExpr::getSIToFP(CI, I.getType()) == CFP && willNotOverflowSignedAdd(LHSIntVal, CI, I)) { // Insert the new integer add. - Value *NewAdd = Builder->CreateNSWAdd(LHSIntVal, - CI, "addconv"); + Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, CI, "addconv"); return new SIToFPInst(NewAdd, I.getType()); } } @@ -1381,8 +1380,7 @@ Instruction *InstCombiner::visitFAdd(BinaryOperator &I) { (LHSConv->hasOneUse() || RHSConv->hasOneUse()) && willNotOverflowSignedAdd(LHSIntVal, RHSIntVal, I)) { // Insert the new integer add. - Value *NewAdd = Builder->CreateNSWAdd(LHSIntVal, - RHSIntVal, "addconv"); + Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, RHSIntVal, "addconv"); return new SIToFPInst(NewAdd, I.getType()); } } @@ -1480,14 +1478,14 @@ Value *InstCombiner::OptimizePointerDifference(Value *LHS, Value *RHS, // pointer, subtract it from the offset we have. if (GEP2) { Value *Offset = EmitGEPOffset(GEP2); - Result = Builder->CreateSub(Result, Offset); + Result = Builder.CreateSub(Result, Offset); } // If we have p - gep(p, ...) then we have to negate the result. if (Swapped) - Result = Builder->CreateNeg(Result, "diff.neg"); + Result = Builder.CreateNeg(Result, "diff.neg"); - return Builder->CreateIntCast(Result, Ty, true); + return Builder.CreateIntCast(Result, Ty, true); } Instruction *InstCombiner::visitSub(BinaryOperator &I) { @@ -1522,7 +1520,7 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) { return Res; } - if (I.getType()->getScalarType()->isIntegerTy(1)) + if (I.getType()->isIntOrIntVectorTy(1)) return BinaryOperator::CreateXor(Op0, Op1); // Replace (-1 - A) with (~A). @@ -1552,12 +1550,12 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) { // Fold (sub 0, (zext bool to B)) --> (sext bool to B) if (C->isNullValue() && match(Op1, m_ZExt(m_Value(X)))) - if (X->getType()->getScalarType()->isIntegerTy(1)) + if (X->getType()->isIntOrIntVectorTy(1)) return CastInst::CreateSExtOrBitCast(X, Op1->getType()); // Fold (sub 0, (sext bool to B)) --> (zext bool to B) if (C->isNullValue() && match(Op1, m_SExt(m_Value(X)))) - if (X->getType()->getScalarType()->isIntegerTy(1)) + if (X->getType()->isIntOrIntVectorTy(1)) return CastInst::CreateZExtOrBitCast(X, Op1->getType()); } @@ -1615,7 +1613,7 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) { // ((X | Y) - X) --> (~X & Y) if (match(Op0, m_OneUse(m_c_Or(m_Value(Y), m_Specific(Op1))))) return BinaryOperator::CreateAnd( - Y, Builder->CreateNot(Op1, Op1->getName() + ".not")); + Y, Builder.CreateNot(Op1, Op1->getName() + ".not")); } if (Op1->hasOneUse()) { @@ -1625,13 +1623,13 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) { // (X - (Y - Z)) --> (X + (Z - Y)). if (match(Op1, m_Sub(m_Value(Y), m_Value(Z)))) return BinaryOperator::CreateAdd(Op0, - Builder->CreateSub(Z, Y, Op1->getName())); + Builder.CreateSub(Z, Y, Op1->getName())); // (X - (X & Y)) --> (X & ~Y) // if (match(Op1, m_c_And(m_Value(Y), m_Specific(Op0)))) return BinaryOperator::CreateAnd(Op0, - Builder->CreateNot(Y, Y->getName() + ".not")); + Builder.CreateNot(Y, Y->getName() + ".not")); // 0 - (X sdiv C) -> (X sdiv -C) provided the negation doesn't overflow. if (match(Op1, m_SDiv(m_Value(X), m_Constant(C))) && match(Op0, m_Zero()) && @@ -1648,7 +1646,7 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) { // 'nuw' is dropped in favor of the canonical form. if (match(Op1, m_SExt(m_Value(Y))) && Y->getType()->getScalarSizeInBits() == 1) { - Value *Zext = Builder->CreateZExt(Y, I.getType()); + Value *Zext = Builder.CreateZExt(Y, I.getType()); BinaryOperator *Add = BinaryOperator::CreateAdd(Op0, Zext); Add->setHasNoSignedWrap(I.hasNoSignedWrap()); return Add; @@ -1659,13 +1657,13 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) { Value *A, *B; Constant *CI; if (match(Op1, m_c_Mul(m_Value(A), m_Neg(m_Value(B))))) - return BinaryOperator::CreateAdd(Op0, Builder->CreateMul(A, B)); + return BinaryOperator::CreateAdd(Op0, Builder.CreateMul(A, B)); // X - A*CI -> X + A*-CI // No need to handle commuted multiply because multiply handling will // ensure constant will be move to the right hand side. if (match(Op1, m_Mul(m_Value(A), m_Constant(CI)))) { - Value *NewMul = Builder->CreateMul(A, ConstantExpr::getNeg(CI)); + Value *NewMul = Builder.CreateMul(A, ConstantExpr::getNeg(CI)); return BinaryOperator::CreateAdd(Op0, NewMul); } } @@ -1729,14 +1727,14 @@ Instruction *InstCombiner::visitFSub(BinaryOperator &I) { } if (FPTruncInst *FPTI = dyn_cast(Op1)) { if (Value *V = dyn_castFNegVal(FPTI->getOperand(0))) { - Value *NewTrunc = Builder->CreateFPTrunc(V, I.getType()); + Value *NewTrunc = Builder.CreateFPTrunc(V, I.getType()); Instruction *NewI = BinaryOperator::CreateFAdd(Op0, NewTrunc); NewI->copyFastMathFlags(&I); return NewI; } } else if (FPExtInst *FPEI = dyn_cast(Op1)) { if (Value *V = dyn_castFNegVal(FPEI->getOperand(0))) { - Value *NewExt = Builder->CreateFPExt(V, I.getType()); + Value *NewExt = Builder.CreateFPExt(V, I.getType()); Instruction *NewI = BinaryOperator::CreateFAdd(Op0, NewExt); NewI->copyFastMathFlags(&I); return NewI; diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index db98be2c98f5..773c86e23707 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -54,17 +54,17 @@ static unsigned getFCmpCode(FCmpInst::Predicate CC) { /// instruction. The sign is passed in to determine which kind of predicate to /// use in the new icmp instruction. static Value *getNewICmpValue(bool Sign, unsigned Code, Value *LHS, Value *RHS, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { ICmpInst::Predicate NewPred; if (Value *NewConstant = getICmpValue(Sign, Code, LHS, RHS, NewPred)) return NewConstant; - return Builder->CreateICmp(NewPred, LHS, RHS); + return Builder.CreateICmp(NewPred, LHS, RHS); } /// This is the complement of getFCmpCode, which turns an opcode and two /// operands into either a FCmp instruction, or a true/false constant. static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { const auto Pred = static_cast(Code); assert(FCmpInst::FCMP_FALSE <= Pred && Pred <= FCmpInst::FCMP_TRUE && "Unexpected FCmp predicate!"); @@ -72,53 +72,45 @@ static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS, return ConstantInt::get(CmpInst::makeCmpResultType(LHS->getType()), 0); if (Pred == FCmpInst::FCMP_TRUE) return ConstantInt::get(CmpInst::makeCmpResultType(LHS->getType()), 1); - return Builder->CreateFCmp(Pred, LHS, RHS); + return Builder.CreateFCmp(Pred, LHS, RHS); } -/// \brief Transform BITWISE_OP(BSWAP(A),BSWAP(B)) to BSWAP(BITWISE_OP(A, B)) +/// \brief Transform BITWISE_OP(BSWAP(A),BSWAP(B)) or +/// BITWISE_OP(BSWAP(A), Constant) to BSWAP(BITWISE_OP(A, B)) /// \param I Binary operator to transform. /// \return Pointer to node that must replace the original binary operator, or /// null pointer if no transformation was made. -Value *InstCombiner::SimplifyBSwap(BinaryOperator &I) { - IntegerType *ITy = dyn_cast(I.getType()); - - // Can't do vectors. - if (I.getType()->isVectorTy()) - return nullptr; - - // Can only do bitwise ops. - if (!I.isBitwiseLogicOp()) - return nullptr; +static Value *SimplifyBSwap(BinaryOperator &I, + InstCombiner::BuilderTy &Builder) { + assert(I.isBitwiseLogicOp() && "Unexpected opcode for bswap simplifying"); Value *OldLHS = I.getOperand(0); Value *OldRHS = I.getOperand(1); - ConstantInt *ConstLHS = dyn_cast(OldLHS); - ConstantInt *ConstRHS = dyn_cast(OldRHS); - IntrinsicInst *IntrLHS = dyn_cast(OldLHS); - IntrinsicInst *IntrRHS = dyn_cast(OldRHS); - bool IsBswapLHS = (IntrLHS && IntrLHS->getIntrinsicID() == Intrinsic::bswap); - bool IsBswapRHS = (IntrRHS && IntrRHS->getIntrinsicID() == Intrinsic::bswap); - if (!IsBswapLHS && !IsBswapRHS) + Value *NewLHS; + if (!match(OldLHS, m_BSwap(m_Value(NewLHS)))) return nullptr; - if (!IsBswapLHS && !ConstLHS) + Value *NewRHS; + const APInt *C; + + if (match(OldRHS, m_BSwap(m_Value(NewRHS)))) { + // OP( BSWAP(x), BSWAP(y) ) -> BSWAP( OP(x, y) ) + if (!OldLHS->hasOneUse() && !OldRHS->hasOneUse()) + return nullptr; + // NewRHS initialized by the matcher. + } else if (match(OldRHS, m_APInt(C))) { + // OP( BSWAP(x), CONSTANT ) -> BSWAP( OP(x, BSWAP(CONSTANT) ) ) + if (!OldLHS->hasOneUse()) + return nullptr; + NewRHS = ConstantInt::get(I.getType(), C->byteSwap()); + } else return nullptr; - if (!IsBswapRHS && !ConstRHS) - return nullptr; - - /// OP( BSWAP(x), BSWAP(y) ) -> BSWAP( OP(x, y) ) - /// OP( BSWAP(x), CONSTANT ) -> BSWAP( OP(x, BSWAP(CONSTANT) ) ) - Value *NewLHS = IsBswapLHS ? IntrLHS->getOperand(0) : - Builder->getInt(ConstLHS->getValue().byteSwap()); - - Value *NewRHS = IsBswapRHS ? IntrRHS->getOperand(0) : - Builder->getInt(ConstRHS->getValue().byteSwap()); - - Value *BinOp = Builder->CreateBinOp(I.getOpcode(), NewLHS, NewRHS); - Function *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::bswap, ITy); - return Builder->CreateCall(F, BinOp); + Value *BinOp = Builder.CreateBinOp(I.getOpcode(), NewLHS, NewRHS); + Function *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::bswap, + I.getType()); + return Builder.CreateCall(F, BinOp); } /// This handles expressions of the form ((val OP C1) & C2). Where @@ -137,7 +129,7 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op, case Instruction::Xor: if (Op->hasOneUse()) { // (X ^ C1) & C2 --> (X & C2) ^ (C1&C2) - Value *And = Builder->CreateAnd(X, AndRHS); + Value *And = Builder.CreateAnd(X, AndRHS); And->takeName(Op); return BinaryOperator::CreateXor(And, Together); } @@ -150,7 +142,7 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op, // NOTE: This reduces the number of bits set in the & mask, which // can expose opportunities for store narrowing. Together = ConstantExpr::getXor(AndRHS, Together); - Value *And = Builder->CreateAnd(X, Together); + Value *And = Builder.CreateAnd(X, Together); And->takeName(Op); return BinaryOperator::CreateOr(And, OpRHS); } @@ -182,7 +174,7 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op, return &TheAnd; } else { // Pull the XOR out of the AND. - Value *NewAnd = Builder->CreateAnd(X, AndRHS); + Value *NewAnd = Builder.CreateAnd(X, AndRHS); NewAnd->takeName(Op); return BinaryOperator::CreateXor(NewAnd, AndRHS); } @@ -198,7 +190,7 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op, uint32_t BitWidth = AndRHS->getType()->getBitWidth(); uint32_t OpRHSVal = OpRHS->getLimitedValue(BitWidth); APInt ShlMask(APInt::getHighBitsSet(BitWidth, BitWidth-OpRHSVal)); - ConstantInt *CI = Builder->getInt(AndRHS->getValue() & ShlMask); + ConstantInt *CI = Builder.getInt(AndRHS->getValue() & ShlMask); if (CI->getValue() == ShlMask) // Masking out bits that the shift already masks. @@ -218,7 +210,7 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op, uint32_t BitWidth = AndRHS->getType()->getBitWidth(); uint32_t OpRHSVal = OpRHS->getLimitedValue(BitWidth); APInt ShrMask(APInt::getLowBitsSet(BitWidth, BitWidth - OpRHSVal)); - ConstantInt *CI = Builder->getInt(AndRHS->getValue() & ShrMask); + ConstantInt *CI = Builder.getInt(AndRHS->getValue() & ShrMask); if (CI->getValue() == ShrMask) // Masking out bits that the shift already masks. @@ -238,12 +230,12 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op, uint32_t BitWidth = AndRHS->getType()->getBitWidth(); uint32_t OpRHSVal = OpRHS->getLimitedValue(BitWidth); APInt ShrMask(APInt::getLowBitsSet(BitWidth, BitWidth - OpRHSVal)); - Constant *C = Builder->getInt(AndRHS->getValue() & ShrMask); + Constant *C = Builder.getInt(AndRHS->getValue() & ShrMask); if (C == AndRHS) { // Masking out bits shifted in. // (Val ashr C1) & C2 -> (Val lshr C1) & C2 // Make the argument unsigned. Value *ShVal = Op->getOperand(0); - ShVal = Builder->CreateLShr(ShVal, OpRHS, Op->getName()); + ShVal = Builder.CreateLShr(ShVal, OpRHS, Op->getName()); return BinaryOperator::CreateAnd(ShVal, AndRHS, TheAnd.getName()); } } @@ -269,15 +261,15 @@ Value *InstCombiner::insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi, ICmpInst::Predicate Pred = Inside ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE; if (isSigned ? Lo.isMinSignedValue() : Lo.isMinValue()) { Pred = isSigned ? ICmpInst::getSignedPredicate(Pred) : Pred; - return Builder->CreateICmp(Pred, V, ConstantInt::get(Ty, Hi)); + return Builder.CreateICmp(Pred, V, ConstantInt::get(Ty, Hi)); } // V >= Lo && V < Hi --> V - Lo u< Hi - Lo // V < Lo || V >= Hi --> V - Lo u>= Hi - Lo Value *VMinusLo = - Builder->CreateSub(V, ConstantInt::get(Ty, Lo), V->getName() + ".off"); + Builder.CreateSub(V, ConstantInt::get(Ty, Lo), V->getName() + ".off"); Constant *HiMinusLo = ConstantInt::get(Ty, Hi - Lo); - return Builder->CreateICmp(Pred, VMinusLo, HiMinusLo); + return Builder.CreateICmp(Pred, VMinusLo, HiMinusLo); } /// Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns @@ -523,7 +515,7 @@ static unsigned getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C, /// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) /// into a single (icmp(A & X) ==/!= Y). static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, - llvm::InstCombiner::BuilderTy *Builder) { + llvm::InstCombiner::BuilderTy &Builder) { Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr; ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); unsigned Mask = @@ -556,27 +548,27 @@ static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, if (Mask & Mask_AllZeros) { // (icmp eq (A & B), 0) & (icmp eq (A & D), 0) // -> (icmp eq (A & (B|D)), 0) - Value *NewOr = Builder->CreateOr(B, D); - Value *NewAnd = Builder->CreateAnd(A, NewOr); + Value *NewOr = Builder.CreateOr(B, D); + Value *NewAnd = Builder.CreateAnd(A, NewOr); // We can't use C as zero because we might actually handle // (icmp ne (A & B), B) & (icmp ne (A & D), D) // with B and D, having a single bit set. Value *Zero = Constant::getNullValue(A->getType()); - return Builder->CreateICmp(NewCC, NewAnd, Zero); + return Builder.CreateICmp(NewCC, NewAnd, Zero); } if (Mask & BMask_AllOnes) { // (icmp eq (A & B), B) & (icmp eq (A & D), D) // -> (icmp eq (A & (B|D)), (B|D)) - Value *NewOr = Builder->CreateOr(B, D); - Value *NewAnd = Builder->CreateAnd(A, NewOr); - return Builder->CreateICmp(NewCC, NewAnd, NewOr); + Value *NewOr = Builder.CreateOr(B, D); + Value *NewAnd = Builder.CreateAnd(A, NewOr); + return Builder.CreateICmp(NewCC, NewAnd, NewOr); } if (Mask & AMask_AllOnes) { // (icmp eq (A & B), A) & (icmp eq (A & D), A) // -> (icmp eq (A & (B&D)), A) - Value *NewAnd1 = Builder->CreateAnd(B, D); - Value *NewAnd2 = Builder->CreateAnd(A, NewAnd1); - return Builder->CreateICmp(NewCC, NewAnd2, A); + Value *NewAnd1 = Builder.CreateAnd(B, D); + Value *NewAnd2 = Builder.CreateAnd(A, NewAnd1); + return Builder.CreateICmp(NewCC, NewAnd2, A); } // Remaining cases assume at least that B and D are constant, and depend on @@ -644,10 +636,10 @@ static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, (CCst->getValue() ^ ECst->getValue())).getBoolValue()) return ConstantInt::get(LHS->getType(), !IsAnd); - Value *NewOr1 = Builder->CreateOr(B, D); + Value *NewOr1 = Builder.CreateOr(B, D); Value *NewOr2 = ConstantExpr::getOr(CCst, ECst); - Value *NewAnd = Builder->CreateAnd(A, NewOr1); - return Builder->CreateICmp(NewCC, NewAnd, NewOr2); + Value *NewAnd = Builder.CreateAnd(A, NewOr1); + return Builder.CreateICmp(NewCC, NewAnd, NewOr2); } return nullptr; @@ -705,13 +697,13 @@ Value *InstCombiner::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1, if (Inverted) NewPred = ICmpInst::getInversePredicate(NewPred); - return Builder->CreateICmp(NewPred, Input, RangeEnd); + return Builder.CreateICmp(NewPred, Input, RangeEnd); } static Value * foldAndOrOfEqualityCmpsWithConstants(ICmpInst *LHS, ICmpInst *RHS, bool JoinedByAnd, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { Value *X = LHS->getOperand(0); if (X != RHS->getOperand(0)) return nullptr; @@ -742,8 +734,8 @@ foldAndOrOfEqualityCmpsWithConstants(ICmpInst *LHS, ICmpInst *RHS, // (X != C1 && X != C2) --> (X | (C1 ^ C2)) != C2 // We choose an 'or' with a Pow2 constant rather than the inverse mask with // 'and' because that may lead to smaller codegen from a smaller constant. - Value *Or = Builder->CreateOr(X, ConstantInt::get(X->getType(), Xor)); - return Builder->CreateICmp(Pred, Or, ConstantInt::get(X->getType(), *C2)); + Value *Or = Builder.CreateOr(X, ConstantInt::get(X->getType(), Xor)); + return Builder.CreateICmp(Pred, Or, ConstantInt::get(X->getType(), *C2)); } // Special case: get the ordering right when the values wrap around zero. @@ -755,9 +747,9 @@ foldAndOrOfEqualityCmpsWithConstants(ICmpInst *LHS, ICmpInst *RHS, // (X == 13 || X == 14) --> X - 13 <=u 1 // (X != 13 && X != 14) --> X - 13 >u 1 // An 'add' is the canonical IR form, so favor that over a 'sub'. - Value *Add = Builder->CreateAdd(X, ConstantInt::get(X->getType(), -(*C1))); + Value *Add = Builder.CreateAdd(X, ConstantInt::get(X->getType(), -(*C1))); auto NewPred = JoinedByAnd ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_ULE; - return Builder->CreateICmp(NewPred, Add, ConstantInt::get(X->getType(), 1)); + return Builder.CreateICmp(NewPred, Add, ConstantInt::get(X->getType(), 1)); } return nullptr; @@ -793,10 +785,10 @@ Value *InstCombiner::foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS, ICmpInst *RHS, if (A == C && isKnownToBeAPowerOfTwo(B, false, 0, &CxtI) && isKnownToBeAPowerOfTwo(D, false, 0, &CxtI)) { - Value *Mask = Builder->CreateOr(B, D); - Value *Masked = Builder->CreateAnd(A, Mask); + Value *Mask = Builder.CreateOr(B, D); + Value *Masked = Builder.CreateAnd(A, Mask); auto NewPred = JoinedByAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE; - return Builder->CreateICmp(NewPred, Masked, Mask); + return Builder.CreateICmp(NewPred, Masked, Mask); } } @@ -855,8 +847,8 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS, // (icmp eq A, 0) & (icmp eq B, 0) --> (icmp eq (A|B), 0) if ((PredL == ICmpInst::ICMP_ULT && LHSC->getValue().isPowerOf2()) || (PredL == ICmpInst::ICMP_EQ && LHSC->isZero())) { - Value *NewOr = Builder->CreateOr(LHS0, RHS0); - return Builder->CreateICmp(PredL, NewOr, LHSC); + Value *NewOr = Builder.CreateOr(LHS0, RHS0); + return Builder.CreateICmp(PredL, NewOr, LHSC); } } @@ -888,10 +880,10 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS, APInt Low = APInt::getLowBitsSet(BigBitSize, SmallBitSize); if ((Low & AndC->getValue()).isNullValue() && (Low & BigC->getValue()).isNullValue()) { - Value *NewAnd = Builder->CreateAnd(V, Low | AndC->getValue()); + Value *NewAnd = Builder.CreateAnd(V, Low | AndC->getValue()); APInt N = SmallC->getValue().zext(BigBitSize) | BigC->getValue(); Value *NewVal = ConstantInt::get(AndC->getType()->getContext(), N); - return Builder->CreateICmp(PredL, NewAnd, NewVal); + return Builder.CreateICmp(PredL, NewAnd, NewVal); } } } @@ -943,14 +935,14 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS, llvm_unreachable("Unknown integer condition code!"); case ICmpInst::ICMP_ULT: if (LHSC == SubOne(RHSC)) // (X != 13 & X u< 14) -> X < 13 - return Builder->CreateICmpULT(LHS0, LHSC); - if (LHSC->isNullValue()) // (X != 0 & X u< 14) -> X-1 u< 13 + return Builder.CreateICmpULT(LHS0, LHSC); + if (LHSC->isZero()) // (X != 0 & X u< 14) -> X-1 u< 13 return insertRangeTest(LHS0, LHSC->getValue() + 1, RHSC->getValue(), false, true); break; // (X != 13 & X u< 15) -> no change case ICmpInst::ICMP_SLT: if (LHSC == SubOne(RHSC)) // (X != 13 & X s< 14) -> X < 13 - return Builder->CreateICmpSLT(LHS0, LHSC); + return Builder.CreateICmpSLT(LHS0, LHSC); break; // (X != 13 & X s< 15) -> no change case ICmpInst::ICMP_NE: // Potential folds for this case should already be handled. @@ -963,7 +955,7 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS, llvm_unreachable("Unknown integer condition code!"); case ICmpInst::ICMP_NE: if (RHSC == AddOne(LHSC)) // (X u> 13 & X != 14) -> X u> 14 - return Builder->CreateICmp(PredL, LHS0, RHSC); + return Builder.CreateICmp(PredL, LHS0, RHSC); break; // (X u> 13 & X != 15) -> no change case ICmpInst::ICMP_ULT: // (X u> 13 & X u< 15) -> (X-14) getValue() + 1, RHSC->getValue(), @@ -976,7 +968,7 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS, llvm_unreachable("Unknown integer condition code!"); case ICmpInst::ICMP_NE: if (RHSC == AddOne(LHSC)) // (X s> 13 & X != 14) -> X s> 14 - return Builder->CreateICmp(PredL, LHS0, RHSC); + return Builder.CreateICmp(PredL, LHS0, RHSC); break; // (X s> 13 & X != 15) -> no change case ICmpInst::ICMP_SLT: // (X s> 13 & X s< 15) -> (X-14) s< 1 return insertRangeTest(LHS0, LHSC->getValue() + 1, RHSC->getValue(), true, @@ -1025,15 +1017,15 @@ Value *InstCombiner::foldAndOfFCmps(FCmpInst *LHS, FCmpInst *RHS) { // If either of the constants are nans, then the whole thing returns // false. if (LHSC->getValueAPF().isNaN() || RHSC->getValueAPF().isNaN()) - return Builder->getFalse(); - return Builder->CreateFCmpORD(LHS->getOperand(0), RHS->getOperand(0)); + return Builder.getFalse(); + return Builder.CreateFCmpORD(LHS->getOperand(0), RHS->getOperand(0)); } // Handle vector zeros. This occurs because the canonical form of // "fcmp ord x,x" is "fcmp ord x, 0". if (isa(LHS->getOperand(1)) && isa(RHS->getOperand(1))) - return Builder->CreateFCmpORD(LHS->getOperand(0), RHS->getOperand(0)); + return Builder.CreateFCmpORD(LHS->getOperand(0), RHS->getOperand(0)); return nullptr; } @@ -1088,7 +1080,7 @@ bool InstCombiner::shouldOptimizeCast(CastInst *CI) { /// Fold {and,or,xor} (cast X), C. static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { Constant *C; if (!match(Logic.getOperand(1), m_Constant(C))) return nullptr; @@ -1107,7 +1099,7 @@ static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast, Constant *ZextTruncC = ConstantExpr::getZExt(TruncC, DestTy); if (ZextTruncC == C) { // LogicOpc (zext X), C --> zext (LogicOpc X, C) - Value *NewOp = Builder->CreateBinOp(LogicOpc, X, TruncC); + Value *NewOp = Builder.CreateBinOp(LogicOpc, X, TruncC); return new ZExtInst(NewOp, DestTy); } } @@ -1150,7 +1142,7 @@ Instruction *InstCombiner::foldCastedBitwiseLogic(BinaryOperator &I) { // fold logic(cast(A), cast(B)) -> cast(logic(A, B)) if (shouldOptimizeCast(Cast0) && shouldOptimizeCast(Cast1)) { - Value *NewOp = Builder->CreateBinOp(LogicOpc, Cast0Src, Cast1Src, + Value *NewOp = Builder.CreateBinOp(LogicOpc, Cast0Src, Cast1Src, I.getName()); return CastInst::Create(CastOpcode, NewOp, DestTy); } @@ -1196,15 +1188,14 @@ static Instruction *foldBoolSextMaskToSelect(BinaryOperator &I) { // Fold (and (sext bool to A), B) --> (select bool, B, 0) Value *X = nullptr; - if (match(Op0, m_SExt(m_Value(X))) && - X->getType()->getScalarType()->isIntegerTy(1)) { + if (match(Op0, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) { Value *Zero = Constant::getNullValue(Op1->getType()); return SelectInst::Create(X, Op1, Zero); } // Fold (and ~(sext bool to A), B) --> (select bool, 0, B) if (match(Op0, m_Not(m_SExt(m_Value(X)))) && - X->getType()->getScalarType()->isIntegerTy(1)) { + X->getType()->isIntOrIntVectorTy(1)) { Value *Zero = Constant::getNullValue(Op0->getType()); return SelectInst::Create(X, Zero, Op1); } @@ -1283,14 +1274,14 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { return &I; // Do this before using distributive laws to catch simple and/or/not patterns. - if (Instruction *Xor = foldAndToXor(I, *Builder)) + if (Instruction *Xor = foldAndToXor(I, Builder)) return Xor; // (A|B)&(A|C) -> A|(B&C) etc if (Value *V = SimplifyUsingDistributiveLaws(I)) return replaceInstUsesWith(I, V); - if (Value *V = SimplifyBSwap(I)) + if (Value *V = SimplifyBSwap(I, Builder)) return replaceInstUsesWith(I, V); if (ConstantInt *AndRHS = dyn_cast(Op1)) { @@ -1310,15 +1301,15 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { APInt NotAndRHS(~AndRHSMask); if (MaskedValueIsZero(Op0LHS, NotAndRHS, 0, &I)) { // Not masking anything out for the LHS, move to RHS. - Value *NewRHS = Builder->CreateAnd(Op0RHS, AndRHS, - Op0RHS->getName()+".masked"); + Value *NewRHS = Builder.CreateAnd(Op0RHS, AndRHS, + Op0RHS->getName()+".masked"); return BinaryOperator::Create(Op0I->getOpcode(), Op0LHS, NewRHS); } if (!isa(Op0RHS) && MaskedValueIsZero(Op0RHS, NotAndRHS, 0, &I)) { // Not masking anything out for the RHS, move to LHS. - Value *NewLHS = Builder->CreateAnd(Op0LHS, AndRHS, - Op0LHS->getName()+".masked"); + Value *NewLHS = Builder.CreateAnd(Op0LHS, AndRHS, + Op0LHS->getName()+".masked"); return BinaryOperator::Create(Op0I->getOpcode(), NewLHS, Op0RHS); } @@ -1337,7 +1328,7 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { // (1 >> x) & 1 --> zext(x == 0) if (AndRHSMask.isOneValue() && Op0LHS == AndRHS) { Value *NewICmp = - Builder->CreateICmpEQ(Op0RHS, Constant::getNullValue(I.getType())); + Builder.CreateICmpEQ(Op0RHS, Constant::getNullValue(I.getType())); return new ZExtInst(NewICmp, I.getType()); } break; @@ -1360,11 +1351,11 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { auto *TruncC1 = ConstantExpr::getTrunc(C1, X->getType()); Value *BinOp; if (isa(Op0LHS)) - BinOp = Builder->CreateBinOp(Op0I->getOpcode(), X, TruncC1); + BinOp = Builder.CreateBinOp(Op0I->getOpcode(), X, TruncC1); else - BinOp = Builder->CreateBinOp(Op0I->getOpcode(), TruncC1, X); + BinOp = Builder.CreateBinOp(Op0I->getOpcode(), TruncC1, X); auto *TruncC2 = ConstantExpr::getTrunc(AndRHS, X->getType()); - auto *And = Builder->CreateAnd(BinOp, TruncC2); + auto *And = Builder.CreateAnd(BinOp, TruncC2); return new ZExtInst(And, I.getType()); } } @@ -1384,7 +1375,7 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { // into : and (trunc X to T), trunc(YC) & C2 // This will fold the two constants together, which may allow // other simplifications. - Value *NewCast = Builder->CreateTrunc(X, I.getType(), "and.shrunk"); + Value *NewCast = Builder.CreateTrunc(X, I.getType(), "and.shrunk"); Constant *C3 = ConstantExpr::getTrunc(YC, I.getType()); C3 = ConstantExpr::getAnd(C3, AndRHS); return BinaryOperator::CreateAnd(NewCast, C3); @@ -1396,7 +1387,7 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { if (Instruction *FoldedLogic = foldOpWithConstantIntoOperand(I)) return FoldedLogic; - if (Instruction *DeMorgan = matchDeMorgansLaws(I, *Builder)) + if (Instruction *DeMorgan = matchDeMorgansLaws(I, Builder)) return DeMorgan; { @@ -1422,7 +1413,7 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { // an endless loop. By checking that A is non-constant we ensure that // we will never get to the loop. if (A == tmpOp0 && !isa(A)) // A&(A^B) -> A & ~B - return BinaryOperator::CreateAnd(A, Builder->CreateNot(B)); + return BinaryOperator::CreateAnd(A, Builder.CreateNot(B)); } } @@ -1436,13 +1427,13 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) if (match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A)))) if (Op1->hasOneUse() || IsFreeToInvert(C, C->hasOneUse())) - return BinaryOperator::CreateAnd(Op0, Builder->CreateNot(C)); + return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(C)); // ((A ^ C) ^ B) & (B ^ A) -> (B ^ A) & ~C if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B)))) if (match(Op1, m_Xor(m_Specific(B), m_Specific(A)))) if (Op0->hasOneUse() || IsFreeToInvert(C, C->hasOneUse())) - return BinaryOperator::CreateAnd(Op1, Builder->CreateNot(C)); + return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(C)); // (A | B) & ((~A) ^ B) -> (A & B) // (A | B) & (B ^ (~A)) -> (A & B) @@ -1474,18 +1465,18 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) { if (LHS && match(Op1, m_OneUse(m_And(m_Value(X), m_Value(Y))))) { if (auto *Cmp = dyn_cast(X)) if (Value *Res = foldAndOfICmps(LHS, Cmp, I)) - return replaceInstUsesWith(I, Builder->CreateAnd(Res, Y)); + return replaceInstUsesWith(I, Builder.CreateAnd(Res, Y)); if (auto *Cmp = dyn_cast(Y)) if (Value *Res = foldAndOfICmps(LHS, Cmp, I)) - return replaceInstUsesWith(I, Builder->CreateAnd(Res, X)); + return replaceInstUsesWith(I, Builder.CreateAnd(Res, X)); } if (RHS && match(Op0, m_OneUse(m_And(m_Value(X), m_Value(Y))))) { if (auto *Cmp = dyn_cast(X)) if (Value *Res = foldAndOfICmps(Cmp, RHS, I)) - return replaceInstUsesWith(I, Builder->CreateAnd(Res, Y)); + return replaceInstUsesWith(I, Builder.CreateAnd(Res, Y)); if (auto *Cmp = dyn_cast(Y)) if (Value *Res = foldAndOfICmps(Cmp, RHS, I)) - return replaceInstUsesWith(I, Builder->CreateAnd(Res, X)); + return replaceInstUsesWith(I, Builder.CreateAnd(Res, X)); } } @@ -1567,14 +1558,14 @@ static Value *getSelectCondition(Value *A, Value *B, InstCombiner::BuilderTy &Builder) { // If these are scalars or vectors of i1, A can be used directly. Type *Ty = A->getType(); - if (match(A, m_Not(m_Specific(B))) && Ty->getScalarType()->isIntegerTy(1)) + if (match(A, m_Not(m_Specific(B))) && Ty->isIntOrIntVectorTy(1)) return A; // If A and B are sign-extended, look through the sexts to find the booleans. Value *Cond; Value *NotB; if (match(A, m_SExt(m_Value(Cond))) && - Cond->getType()->getScalarType()->isIntegerTy(1) && + Cond->getType()->isIntOrIntVectorTy(1) && match(B, m_OneUse(m_Not(m_Value(NotB))))) { NotB = peekThroughBitcast(NotB, true); if (match(NotB, m_SExt(m_Specific(Cond)))) @@ -1596,7 +1587,7 @@ static Value *getSelectCondition(Value *A, Value *B, // operand, see if the constants are inverse bitmasks. if (match(A, (m_Xor(m_SExt(m_Value(Cond)), m_Constant(AC)))) && match(B, (m_Xor(m_SExt(m_Specific(Cond)), m_Constant(BC)))) && - Cond->getType()->getScalarType()->isIntegerTy(1) && + Cond->getType()->isIntOrIntVectorTy(1) && areInverseVectorBitmasks(AC, BC)) { AC = ConstantExpr::getTrunc(AC, CmpInst::makeCmpResultType(Ty)); return Builder.CreateXor(Cond, AC); @@ -1687,9 +1678,9 @@ Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, RangeDiff.ugt(LHSC->getValue())) { Value *MaskC = ConstantInt::get(LAddC->getType(), ~DiffC); - Value *NewAnd = Builder->CreateAnd(LAddOpnd, MaskC); - Value *NewAdd = Builder->CreateAdd(NewAnd, MaxAddC); - return (Builder->CreateICmp(LHS->getPredicate(), NewAdd, LHSC)); + Value *NewAnd = Builder.CreateAnd(LAddOpnd, MaskC); + Value *NewAdd = Builder.CreateAdd(NewAnd, MaxAddC); + return Builder.CreateICmp(LHS->getPredicate(), NewAdd, LHSC); } } } @@ -1736,9 +1727,9 @@ Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, A = LHS->getOperand(1); } if (A && B) - return Builder->CreateICmp( + return Builder.CreateICmp( ICmpInst::ICMP_UGE, - Builder->CreateAdd(B, ConstantInt::getSigned(B->getType(), -1)), A); + Builder.CreateAdd(B, ConstantInt::getSigned(B->getType(), -1)), A); } // E.g. (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n @@ -1759,8 +1750,8 @@ Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, if (LHSC == RHSC && PredL == PredR) { // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0) if (PredL == ICmpInst::ICMP_NE && LHSC->isZero()) { - Value *NewOr = Builder->CreateOr(LHS0, RHS0); - return Builder->CreateICmp(PredL, NewOr, LHSC); + Value *NewOr = Builder.CreateOr(LHS0, RHS0); + return Builder.CreateICmp(PredL, NewOr, LHSC); } } @@ -1770,7 +1761,7 @@ Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, ConstantInt *AddC; if (match(LHS0, m_Add(m_Specific(RHS0), m_ConstantInt(AddC)))) if (RHSC->getValue() + AddC->getValue() == LHSC->getValue()) - return Builder->CreateICmpULE(LHS0, LHSC); + return Builder.CreateICmpULE(LHS0, LHSC); } // From here on, we only handle: @@ -1886,18 +1877,18 @@ Value *InstCombiner::foldOrOfFCmps(FCmpInst *LHS, FCmpInst *RHS) { // If either of the constants are nans, then the whole thing returns // true. if (LHSC->getValueAPF().isNaN() || RHSC->getValueAPF().isNaN()) - return Builder->getTrue(); + return Builder.getTrue(); // Otherwise, no need to compare the two constants, compare the // rest. - return Builder->CreateFCmpUNO(LHS->getOperand(0), RHS->getOperand(0)); + return Builder.CreateFCmpUNO(LHS->getOperand(0), RHS->getOperand(0)); } // Handle vector zeros. This occurs because the canonical form of // "fcmp uno x,x" is "fcmp uno x, 0". if (isa(LHS->getOperand(1)) && isa(RHS->getOperand(1))) - return Builder->CreateFCmpUNO(LHS->getOperand(0), RHS->getOperand(0)); + return Builder.CreateFCmpUNO(LHS->getOperand(0), RHS->getOperand(0)); return nullptr; } @@ -1916,7 +1907,7 @@ Value *InstCombiner::foldOrOfFCmps(FCmpInst *LHS, FCmpInst *RHS) { /// when the XOR of the two constants is "all ones" (-1). static Instruction *FoldOrWithConstants(BinaryOperator &I, Value *Op, Value *A, Value *B, Value *C, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { ConstantInt *CI1 = dyn_cast(C); if (!CI1) return nullptr; @@ -1928,7 +1919,7 @@ static Instruction *FoldOrWithConstants(BinaryOperator &I, Value *Op, if (!Xor.isAllOnesValue()) return nullptr; if (V1 == A || V1 == B) { - Value *NewOp = Builder->CreateAnd((V1 == A) ? B : A, CI1); + Value *NewOp = Builder.CreateAnd((V1 == A) ? B : A, CI1); return BinaryOperator::CreateOr(NewOp, V1); } @@ -1946,7 +1937,7 @@ static Instruction *FoldOrWithConstants(BinaryOperator &I, Value *Op, /// when the XOR of the two constants is "all ones" (-1). static Instruction *FoldXorWithConstants(BinaryOperator &I, Value *Op, Value *A, Value *B, Value *C, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { ConstantInt *CI1 = dyn_cast(C); if (!CI1) return nullptr; @@ -1961,7 +1952,7 @@ static Instruction *FoldXorWithConstants(BinaryOperator &I, Value *Op, return nullptr; if (V1 == A || V1 == B) { - Value *NewOp = Builder->CreateAnd(V1 == A ? B : A, CI1); + Value *NewOp = Builder.CreateAnd(V1 == A ? B : A, CI1); return BinaryOperator::CreateXor(NewOp, V1); } @@ -1987,14 +1978,14 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { return &I; // Do this before using distributive laws to catch simple and/or/not patterns. - if (Instruction *Xor = foldOrToXor(I, *Builder)) + if (Instruction *Xor = foldOrToXor(I, Builder)) return Xor; // (A&B)|(A&C) -> A&(B|C) etc if (Value *V = SimplifyUsingDistributiveLaws(I)) return replaceInstUsesWith(I, V); - if (Value *V = SimplifyBSwap(I)) + if (Value *V = SimplifyBSwap(I, Builder)) return replaceInstUsesWith(I, V); if (isa(Op1)) @@ -2011,7 +2002,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { // (X^C)|Y -> (X|Y)^C iff Y&C == 0 if (match(Op0, m_OneUse(m_Xor(m_Value(A), m_APInt(C)))) && MaskedValueIsZero(Op1, *C, 0, &I)) { - Value *NOr = Builder->CreateOr(A, Op1); + Value *NOr = Builder.CreateOr(A, Op1); NOr->takeName(Op0); return BinaryOperator::CreateXor(NOr, ConstantInt::get(NOr->getType(), *C)); @@ -2020,7 +2011,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { // Y|(X^C) -> (X|Y)^C iff Y&C == 0 if (match(Op1, m_OneUse(m_Xor(m_Value(A), m_APInt(C)))) && MaskedValueIsZero(Op0, *C, 0, &I)) { - Value *NOr = Builder->CreateOr(A, Op0); + Value *NOr = Builder.CreateOr(A, Op0); NOr->takeName(Op0); return BinaryOperator::CreateXor(NOr, ConstantInt::get(NOr->getType(), *C)); @@ -2058,7 +2049,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { (V2 == B && MaskedValueIsZero(V1, ~C1->getValue(), 0, &I)))) // (N|V) return BinaryOperator::CreateAnd(A, - Builder->getInt(C1->getValue()|C2->getValue())); + Builder.getInt(C1->getValue()|C2->getValue())); // Or commutes, try both ways. if (match(B, m_Or(m_Value(V1), m_Value(V2))) && ((V1 == A && @@ -2066,7 +2057,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { (V2 == A && MaskedValueIsZero(V1, ~C2->getValue(), 0, &I)))) // (N|V) return BinaryOperator::CreateAnd(B, - Builder->getInt(C1->getValue()|C2->getValue())); + Builder.getInt(C1->getValue()|C2->getValue())); // ((V|C3)&C1) | ((V|C4)&C2) --> (V|C3|C4)&(C1|C2) // iff (C1&C2) == 0 and (C3&~C1) == 0 and (C4&~C2) == 0. @@ -2075,9 +2066,9 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { (C3->getValue() & ~C1->getValue()).isNullValue() && match(B, m_Or(m_Specific(V1), m_ConstantInt(C4))) && (C4->getValue() & ~C2->getValue()).isNullValue()) { - V2 = Builder->CreateOr(V1, ConstantExpr::getOr(C3, C4), "bitfield"); + V2 = Builder.CreateOr(V1, ConstantExpr::getOr(C3, C4), "bitfield"); return BinaryOperator::CreateAnd(V2, - Builder->getInt(C1->getValue()|C2->getValue())); + Builder.getInt(C1->getValue()|C2->getValue())); } } } @@ -2087,21 +2078,21 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { // 'or' that it is replacing. if (Op0->hasOneUse() || Op1->hasOneUse()) { // (Cond & C) | (~Cond & D) -> Cond ? C : D, and commuted variants. - if (Value *V = matchSelectFromAndOr(A, C, B, D, *Builder)) + if (Value *V = matchSelectFromAndOr(A, C, B, D, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(A, C, D, B, *Builder)) + if (Value *V = matchSelectFromAndOr(A, C, D, B, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(C, A, B, D, *Builder)) + if (Value *V = matchSelectFromAndOr(C, A, B, D, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(C, A, D, B, *Builder)) + if (Value *V = matchSelectFromAndOr(C, A, D, B, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(B, D, A, C, *Builder)) + if (Value *V = matchSelectFromAndOr(B, D, A, C, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(B, D, C, A, *Builder)) + if (Value *V = matchSelectFromAndOr(B, D, C, A, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(D, B, A, C, *Builder)) + if (Value *V = matchSelectFromAndOr(D, B, A, C, Builder)) return replaceInstUsesWith(I, V); - if (Value *V = matchSelectFromAndOr(D, B, C, A, *Builder)) + if (Value *V = matchSelectFromAndOr(D, B, C, A, Builder)) return replaceInstUsesWith(I, V); } @@ -2139,9 +2130,9 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { // ((B | C) & A) | B -> B | (A & C) if (match(Op0, m_And(m_Or(m_Specific(Op1), m_Value(C)), m_Value(A)))) - return BinaryOperator::CreateOr(Op1, Builder->CreateAnd(A, C)); + return BinaryOperator::CreateOr(Op1, Builder.CreateAnd(A, C)); - if (Instruction *DeMorgan = matchDeMorgansLaws(I, *Builder)) + if (Instruction *DeMorgan = matchDeMorgansLaws(I, Builder)) return DeMorgan; // Canonicalize xor to the RHS. @@ -2163,11 +2154,11 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { return BinaryOperator::CreateOr(A, B); if (Op1->hasOneUse() && match(A, m_Not(m_Specific(Op0)))) { - Value *Not = Builder->CreateNot(B, B->getName()+".not"); + Value *Not = Builder.CreateNot(B, B->getName() + ".not"); return BinaryOperator::CreateOr(Not, Op0); } if (Op1->hasOneUse() && match(B, m_Not(m_Specific(Op0)))) { - Value *Not = Builder->CreateNot(A, A->getName()+".not"); + Value *Not = Builder.CreateNot(A, A->getName() + ".not"); return BinaryOperator::CreateOr(Not, Op0); } } @@ -2181,7 +2172,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { B->getOpcode() == Instruction::Xor)) { Value *NotOp = Op0 == B->getOperand(0) ? B->getOperand(1) : B->getOperand(0); - Value *Not = Builder->CreateNot(NotOp, NotOp->getName()+".not"); + Value *Not = Builder.CreateNot(NotOp, NotOp->getName() + ".not"); return BinaryOperator::CreateOr(Not, Op0); } @@ -2194,7 +2185,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { // xor was canonicalized to Op1 above. if (match(Op1, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) && match(Op0, m_c_And(m_Specific(A), m_Specific(B)))) - return BinaryOperator::CreateXor(Builder->CreateNot(A), B); + return BinaryOperator::CreateXor(Builder.CreateNot(A), B); if (SwappedForXor) std::swap(Op0, Op1); @@ -2212,18 +2203,18 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { if (LHS && match(Op1, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) { if (auto *Cmp = dyn_cast(X)) if (Value *Res = foldOrOfICmps(LHS, Cmp, I)) - return replaceInstUsesWith(I, Builder->CreateOr(Res, Y)); + return replaceInstUsesWith(I, Builder.CreateOr(Res, Y)); if (auto *Cmp = dyn_cast(Y)) if (Value *Res = foldOrOfICmps(LHS, Cmp, I)) - return replaceInstUsesWith(I, Builder->CreateOr(Res, X)); + return replaceInstUsesWith(I, Builder.CreateOr(Res, X)); } if (RHS && match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) { if (auto *Cmp = dyn_cast(X)) if (Value *Res = foldOrOfICmps(Cmp, RHS, I)) - return replaceInstUsesWith(I, Builder->CreateOr(Res, Y)); + return replaceInstUsesWith(I, Builder.CreateOr(Res, Y)); if (auto *Cmp = dyn_cast(Y)) if (Value *Res = foldOrOfICmps(Cmp, RHS, I)) - return replaceInstUsesWith(I, Builder->CreateOr(Res, X)); + return replaceInstUsesWith(I, Builder.CreateOr(Res, X)); } } @@ -2238,10 +2229,10 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { // or(sext(A), B) / or(B, sext(A)) --> A ? -1 : B, where A is i1 or . if (match(Op0, m_OneUse(m_SExt(m_Value(A)))) && - A->getType()->getScalarType()->isIntegerTy(1)) + A->getType()->isIntOrIntVectorTy(1)) return SelectInst::Create(A, ConstantInt::getSigned(I.getType(), -1), Op1); if (match(Op1, m_OneUse(m_SExt(m_Value(A)))) && - A->getType()->getScalarType()->isIntegerTy(1)) + A->getType()->isIntOrIntVectorTy(1)) return SelectInst::Create(A, ConstantInt::getSigned(I.getType(), -1), Op0); // Note: If we've gotten to the point of visiting the outer OR, then the @@ -2252,7 +2243,7 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { ConstantInt *C1; if (Op0->hasOneUse() && !isa(Op1) && match(Op0, m_Or(m_Value(A), m_ConstantInt(C1)))) { - Value *Inner = Builder->CreateOr(A, Op1); + Value *Inner = Builder.CreateOr(A, Op1); Inner->takeName(Op0); return BinaryOperator::CreateOr(Inner, C1); } @@ -2265,8 +2256,8 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { if (Op0->hasOneUse() && Op1->hasOneUse() && match(Op0, m_Select(m_Value(X), m_Value(A), m_Value(B))) && match(Op1, m_Select(m_Value(Y), m_Value(C), m_Value(D))) && X == Y) { - Value *orTrue = Builder->CreateOr(A, C); - Value *orFalse = Builder->CreateOr(B, D); + Value *orTrue = Builder.CreateOr(A, C); + Value *orFalse = Builder.CreateOr(B, D); return SelectInst::Create(X, orTrue, orFalse); } } @@ -2276,7 +2267,8 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) { /// A ^ B can be specified using other logic ops in a variety of patterns. We /// can fold these early and efficiently by morphing an existing instruction. -static Instruction *foldXorToXor(BinaryOperator &I) { +static Instruction *foldXorToXor(BinaryOperator &I, + InstCombiner::BuilderTy &Builder) { assert(I.getOpcode() == Instruction::Xor); Value *Op0 = I.getOperand(0); Value *Op1 = I.getOperand(1); @@ -2323,6 +2315,21 @@ static Instruction *foldXorToXor(BinaryOperator &I) { return &I; } + // For the remaining cases we need to get rid of one of the operands. + if (!Op0->hasOneUse() && !Op1->hasOneUse()) + return nullptr; + + // (A | B) ^ ~(A & B) -> ~(A ^ B) + // (A | B) ^ ~(B & A) -> ~(A ^ B) + // (A & B) ^ ~(A | B) -> ~(A ^ B) + // (A & B) ^ ~(B | A) -> ~(A ^ B) + // Complexity sorting ensures the not will be on the right side. + if ((match(Op0, m_Or(m_Value(A), m_Value(B))) && + match(Op1, m_Not(m_c_And(m_Specific(A), m_Specific(B))))) || + (match(Op0, m_And(m_Value(A), m_Value(B))) && + match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))) + return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); + return nullptr; } @@ -2355,12 +2362,12 @@ Value *InstCombiner::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS) { if (OrICmp == LHS && AndICmp == RHS && RHS->hasOneUse()) { // (LHS | RHS) & !(LHS & RHS) --> LHS & !RHS RHS->setPredicate(RHS->getInversePredicate()); - return Builder->CreateAnd(LHS, RHS); + return Builder.CreateAnd(LHS, RHS); } if (OrICmp == RHS && AndICmp == LHS && LHS->hasOneUse()) { // !(LHS & RHS) & (LHS | RHS) --> !LHS & RHS LHS->setPredicate(LHS->getInversePredicate()); - return Builder->CreateAnd(LHS, RHS); + return Builder.CreateAnd(LHS, RHS); } } } @@ -2381,7 +2388,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { if (Value *V = SimplifyXorInst(Op0, Op1, SQ.getWithInstruction(&I))) return replaceInstUsesWith(I, V); - if (Instruction *NewXor = foldXorToXor(I)) + if (Instruction *NewXor = foldXorToXor(I, Builder)) return NewXor; // (A&B)^(A&C) -> A&(B^C) etc @@ -2393,7 +2400,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { if (SimplifyDemandedInstructionBits(I)) return &I; - if (Value *V = SimplifyBSwap(I)) + if (Value *V = SimplifyBSwap(I, Builder)) return replaceInstUsesWith(I, V); // Apply DeMorgan's Law for 'nand' / 'nor' logic with an inverted operand. @@ -2404,13 +2411,13 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { // ~(~X & Y) --> (X | ~Y) // ~(Y & ~X) --> (X | ~Y) if (match(&I, m_Not(m_OneUse(m_c_And(m_Not(m_Value(X)), m_Value(Y)))))) { - Value *NotY = Builder->CreateNot(Y, Y->getName() + ".not"); + Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); return BinaryOperator::CreateOr(X, NotY); } // ~(~X | Y) --> (X & ~Y) // ~(Y | ~X) --> (X & ~Y) if (match(&I, m_Not(m_OneUse(m_c_Or(m_Not(m_Value(X)), m_Value(Y)))))) { - Value *NotY = Builder->CreateNot(Y, Y->getName() + ".not"); + Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); return BinaryOperator::CreateAnd(X, NotY); } @@ -2426,8 +2433,8 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { NotVal->getOperand(0)->hasOneUse()) && IsFreeToInvert(NotVal->getOperand(1), NotVal->getOperand(1)->hasOneUse())) { - Value *NotX = Builder->CreateNot(NotVal->getOperand(0), "notlhs"); - Value *NotY = Builder->CreateNot(NotVal->getOperand(1), "notrhs"); + Value *NotX = Builder.CreateNot(NotVal->getOperand(0), "notlhs"); + Value *NotY = Builder.CreateNot(NotVal->getOperand(1), "notrhs"); if (NotVal->getOpcode() == Instruction::And) return BinaryOperator::CreateOr(NotX, NotY); return BinaryOperator::CreateAnd(NotX, NotY); @@ -2457,7 +2464,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { } // not (cmp A, B) = !cmp A, B - ICmpInst::Predicate Pred; + CmpInst::Predicate Pred; if (match(&I, m_Not(m_OneUse(m_Cmp(Pred, m_Value(), m_Value()))))) { cast(Op0)->setPredicate(CmpInst::getInversePredicate(Pred)); return replaceInstUsesWith(I, Op0); @@ -2470,8 +2477,8 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { if (CI->hasOneUse() && Op0C->hasOneUse()) { Instruction::CastOps Opcode = Op0C->getOpcode(); if ((Opcode == Instruction::ZExt || Opcode == Instruction::SExt) && - (RHSC == ConstantExpr::getCast(Opcode, Builder->getTrue(), - Op0C->getDestTy()))) { + (RHSC == ConstantExpr::getCast(Opcode, Builder.getTrue(), + Op0C->getDestTy()))) { CI->setPredicate(CI->getInversePredicate()); return CastInst::Create(Opcode, CI, Op0C->getType()); } @@ -2481,7 +2488,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { if (BinaryOperator *Op0I = dyn_cast(Op0)) { // ~(c-X) == X-c-1 == X+(-c-1) - if (Op0I->getOpcode() == Instruction::Sub && RHSC->isAllOnesValue()) + if (Op0I->getOpcode() == Instruction::Sub && RHSC->isMinusOne()) if (Constant *Op0I0C = dyn_cast(Op0I->getOperand(0))) { Constant *NegOp0I0C = ConstantExpr::getNeg(Op0I0C); return BinaryOperator::CreateAdd(Op0I->getOperand(1), @@ -2491,13 +2498,13 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { if (ConstantInt *Op0CI = dyn_cast(Op0I->getOperand(1))) { if (Op0I->getOpcode() == Instruction::Add) { // ~(X-c) --> (-c-1)-X - if (RHSC->isAllOnesValue()) { + if (RHSC->isMinusOne()) { Constant *NegOp0CI = ConstantExpr::getNeg(Op0CI); return BinaryOperator::CreateSub(SubOne(NegOp0CI), Op0I->getOperand(0)); } else if (RHSC->getValue().isSignMask()) { // (X + C) ^ signmask -> (X + C + signmask) - Constant *C = Builder->getInt(RHSC->getValue() + Op0CI->getValue()); + Constant *C = Builder.getInt(RHSC->getValue() + Op0CI->getValue()); return BinaryOperator::CreateAdd(Op0I->getOperand(0), C); } @@ -2530,7 +2537,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { APInt FoldConst = C1->getValue().lshr(C2->getValue()); FoldConst ^= C3->getValue(); // Prepare the two operands. - Value *Opnd0 = Builder->CreateLShr(E1->getOperand(0), C2); + Value *Opnd0 = Builder.CreateLShr(E1->getOperand(0), C2); Opnd0->takeName(Op0I); cast(Opnd0)->setDebugLoc(I.getDebugLoc()); Value *FoldVal = ConstantInt::get(Opnd0->getType(), FoldConst); @@ -2575,14 +2582,14 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { if (A == Op1) // (B|A)^B == (A|B)^B std::swap(A, B); if (B == Op1) // (A|B)^B == A & ~B - return BinaryOperator::CreateAnd(A, Builder->CreateNot(Op1)); + return BinaryOperator::CreateAnd(A, Builder.CreateNot(Op1)); } else if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B))))) { if (A == Op1) // (A&B)^A -> (B&A)^A std::swap(A, B); const APInt *C; if (B == Op1 && // (B&A)^A == ~B & A !match(Op1, m_APInt(C))) { // Canonical form is (B&C)^C - return BinaryOperator::CreateAnd(Builder->CreateNot(A), Op1); + return BinaryOperator::CreateAnd(Builder.CreateNot(A), Op1); } } } @@ -2594,20 +2601,20 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { match(Op1, m_Or(m_Value(A), m_Value(B)))) { if (D == A) return BinaryOperator::CreateXor( - Builder->CreateAnd(Builder->CreateNot(A), B), C); + Builder.CreateAnd(Builder.CreateNot(A), B), C); if (D == B) return BinaryOperator::CreateXor( - Builder->CreateAnd(Builder->CreateNot(B), A), C); + Builder.CreateAnd(Builder.CreateNot(B), A), C); } // (A | B)^(A ^ C) -> ((~A) & B) ^ C if (match(Op0, m_Or(m_Value(A), m_Value(B))) && match(Op1, m_Xor(m_Value(D), m_Value(C)))) { if (D == A) return BinaryOperator::CreateXor( - Builder->CreateAnd(Builder->CreateNot(A), B), C); + Builder.CreateAnd(Builder.CreateNot(A), B), C); if (D == B) return BinaryOperator::CreateXor( - Builder->CreateAnd(Builder->CreateNot(B), A), C); + Builder.CreateAnd(Builder.CreateNot(B), A), C); } // (A & B) ^ (A ^ B) -> (A | B) if (match(Op0, m_And(m_Value(A), m_Value(B))) && @@ -2624,7 +2631,7 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) { Value *A, *B; if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) && match(Op1, m_Not(m_Specific(A)))) - return BinaryOperator::CreateNot(Builder->CreateAnd(A, B)); + return BinaryOperator::CreateNot(Builder.CreateAnd(A, B)); if (auto *LHS = dyn_cast(I.getOperand(0))) if (auto *RHS = dyn_cast(I.getOperand(1))) diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 3770021de100..391c430dab75 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -128,23 +128,23 @@ Instruction *InstCombiner::SimplifyElementUnorderedAtomicMemCpy( Type::getIntNPtrTy(AMI->getContext(), ElementSizeInBits, Src->getType()->getPointerAddressSpace()); - Value *SrcCasted = Builder->CreatePointerCast(Src, ElementPointerType, - "memcpy_unfold.src_casted"); - Value *DstCasted = Builder->CreatePointerCast(Dst, ElementPointerType, - "memcpy_unfold.dst_casted"); + Value *SrcCasted = Builder.CreatePointerCast(Src, ElementPointerType, + "memcpy_unfold.src_casted"); + Value *DstCasted = Builder.CreatePointerCast(Dst, ElementPointerType, + "memcpy_unfold.dst_casted"); for (uint64_t i = 0; i < NumElements; ++i) { // Get current element addresses ConstantInt *ElementIdxCI = ConstantInt::get(AMI->getContext(), APInt(64, i)); Value *SrcElementAddr = - Builder->CreateGEP(SrcCasted, ElementIdxCI, "memcpy_unfold.src_addr"); + Builder.CreateGEP(SrcCasted, ElementIdxCI, "memcpy_unfold.src_addr"); Value *DstElementAddr = - Builder->CreateGEP(DstCasted, ElementIdxCI, "memcpy_unfold.dst_addr"); + Builder.CreateGEP(DstCasted, ElementIdxCI, "memcpy_unfold.dst_addr"); // Load from the source. Transfer alignment information and mark load as // unordered atomic. - LoadInst *Load = Builder->CreateLoad(SrcElementAddr, "memcpy_unfold.val"); + LoadInst *Load = Builder.CreateLoad(SrcElementAddr, "memcpy_unfold.val"); Load->setOrdering(AtomicOrdering::Unordered); // We know alignment of the first element. It is also guaranteed by the // verifier that element size is less or equal than first element @@ -157,7 +157,7 @@ Instruction *InstCombiner::SimplifyElementUnorderedAtomicMemCpy( Load->setDebugLoc(AMI->getDebugLoc()); // Store loaded value via unordered atomic store. - StoreInst *Store = Builder->CreateStore(Load, DstElementAddr); + StoreInst *Store = Builder.CreateStore(Load, DstElementAddr); Store->setOrdering(AtomicOrdering::Unordered); Store->setAlignment(i == 0 ? AMI->getParamAlignment(0) : ElementSizeInBytes); @@ -213,7 +213,7 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) { if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) { if (M->getNumOperands() == 3 && M->getOperand(0) && mdconst::hasa(M->getOperand(0)) && - mdconst::extract(M->getOperand(0))->isNullValue() && + mdconst::extract(M->getOperand(0))->isZero() && M->getOperand(1) && mdconst::hasa(M->getOperand(1)) && mdconst::extract(M->getOperand(1))->getValue() == @@ -227,9 +227,9 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) { SrcAlign = std::max(SrcAlign, CopyAlign); DstAlign = std::max(DstAlign, CopyAlign); - Value *Src = Builder->CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy); - Value *Dest = Builder->CreateBitCast(MI->getArgOperand(0), NewDstPtrTy); - LoadInst *L = Builder->CreateLoad(Src, MI->isVolatile()); + Value *Src = Builder.CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy); + Value *Dest = Builder.CreateBitCast(MI->getArgOperand(0), NewDstPtrTy); + LoadInst *L = Builder.CreateLoad(Src, MI->isVolatile()); L->setAlignment(SrcAlign); if (CopyMD) L->setMetadata(LLVMContext::MD_tbaa, CopyMD); @@ -238,7 +238,7 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) { if (LoopMemParallelMD) L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD); - StoreInst *S = Builder->CreateStore(L, Dest, MI->isVolatile()); + StoreInst *S = Builder.CreateStore(L, Dest, MI->isVolatile()); S->setAlignment(DstAlign); if (CopyMD) S->setMetadata(LLVMContext::MD_tbaa, CopyMD); @@ -274,15 +274,15 @@ Instruction *InstCombiner::SimplifyMemSet(MemSetInst *MI) { Value *Dest = MI->getDest(); unsigned DstAddrSp = cast(Dest->getType())->getAddressSpace(); Type *NewDstPtrTy = PointerType::get(ITy, DstAddrSp); - Dest = Builder->CreateBitCast(Dest, NewDstPtrTy); + Dest = Builder.CreateBitCast(Dest, NewDstPtrTy); // Alignment 0 is identity for alignment 1 for memset, but not store. if (Alignment == 0) Alignment = 1; // Extract the fill value and store. uint64_t Fill = FillC->getZExtValue()*0x0101010101010101ULL; - StoreInst *S = Builder->CreateStore(ConstantInt::get(ITy, Fill), Dest, - MI->isVolatile()); + StoreInst *S = Builder.CreateStore(ConstantInt::get(ITy, Fill), Dest, + MI->isVolatile()); S->setAlignment(Alignment); // Set the size of the copy to 0, it will be deleted on the next iteration. @@ -600,8 +600,7 @@ static Value *simplifyX86muldq(const IntrinsicInst &II, return Builder.CreateMul(LHS, RHS); } -static Value *simplifyX86pack(IntrinsicInst &II, InstCombiner &IC, - InstCombiner::BuilderTy &Builder, bool IsSigned) { +static Value *simplifyX86pack(IntrinsicInst &II, bool IsSigned) { Value *Arg0 = II.getArgOperand(0); Value *Arg1 = II.getArgOperand(1); Type *ResTy = II.getType(); @@ -676,8 +675,7 @@ static Value *simplifyX86pack(IntrinsicInst &II, InstCombiner &IC, return ConstantVector::get(Vals); } -static Value *simplifyX86movmsk(const IntrinsicInst &II, - InstCombiner::BuilderTy &Builder) { +static Value *simplifyX86movmsk(const IntrinsicInst &II) { Value *Arg = II.getArgOperand(0); Type *ResTy = II.getType(); Type *ArgTy = Arg->getType(); @@ -860,7 +858,7 @@ static Value *simplifyX86extrq(IntrinsicInst &II, Value *Op0, } // Constant Fold - extraction from zero is always {zero, undef}. - if (CI0 && CI0->equalsInt(0)) + if (CI0 && CI0->isZero()) return LowConstantHighUndef(0); return nullptr; @@ -1404,7 +1402,7 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombiner &IC) { isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II, &IC.getDominatorTree())) { if (!match(II.getArgOperand(1), m_One())) { - II.setOperand(1, IC.Builder->getTrue()); + II.setOperand(1, IC.Builder.getTrue()); return &II; } } @@ -1477,7 +1475,7 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) { // the LLVM intrinsic definition for the pointer argument. unsigned AddrSpace = cast(Ptr->getType())->getAddressSpace(); PointerType *VecPtrTy = PointerType::get(II.getType(), AddrSpace); - Value *PtrCast = IC.Builder->CreateBitCast(Ptr, VecPtrTy, "castvec"); + Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec"); // Second, convert the x86 XMM integer vector mask to a vector of bools based // on each element's most significant bit (the sign bit). @@ -1485,7 +1483,7 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) { // The pass-through vector for an x86 masked load is a zero vector. CallInst *NewMaskedLoad = - IC.Builder->CreateMaskedLoad(PtrCast, 1, BoolMask, ZeroVec); + IC.Builder.CreateMaskedLoad(PtrCast, 1, BoolMask, ZeroVec); return IC.replaceInstUsesWith(II, NewMaskedLoad); } @@ -1520,13 +1518,13 @@ static bool simplifyX86MaskedStore(IntrinsicInst &II, InstCombiner &IC) { // the LLVM intrinsic definition for the pointer argument. unsigned AddrSpace = cast(Ptr->getType())->getAddressSpace(); PointerType *VecPtrTy = PointerType::get(Vec->getType(), AddrSpace); - Value *PtrCast = IC.Builder->CreateBitCast(Ptr, VecPtrTy, "castvec"); + Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec"); // Second, convert the x86 XMM integer vector mask to a vector of bools based // on each element's most significant bit (the sign bit). Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask); - IC.Builder->CreateMaskedStore(Vec, PtrCast, 1, BoolMask); + IC.Builder.CreateMaskedStore(Vec, PtrCast, 1, BoolMask); // 'Replace uses' doesn't work for stores. Erase the original masked store. IC.eraseInstFromFunction(II); @@ -1764,7 +1762,7 @@ static Instruction *SimplifyNVVMIntrinsic(IntrinsicInst *II, InstCombiner &IC) { // The remainder of cases are NVVM intrinsics that map to LLVM idioms, but // need special handling. // - // We seem to be mising intrinsics for rcp.approx.{ftz.}f32, which is just + // We seem to be missing intrinsics for rcp.approx.{ftz.}f32, which is just // as well. case Intrinsic::nvvm_rcp_rn_d: return {SPC_Reciprocal, FTZ_Any}; @@ -1965,16 +1963,17 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { Value *IIOperand = II->getArgOperand(0); Value *X = nullptr; + // TODO should this be in InstSimplify? // bswap(bswap(x)) -> x if (match(IIOperand, m_BSwap(m_Value(X)))) - return replaceInstUsesWith(CI, X); + return replaceInstUsesWith(CI, X); // bswap(trunc(bswap(x))) -> trunc(lshr(x, c)) if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) { unsigned C = X->getType()->getPrimitiveSizeInBits() - IIOperand->getType()->getPrimitiveSizeInBits(); Value *CV = ConstantInt::get(X->getType(), C); - Value *V = Builder->CreateLShr(X, CV); + Value *V = Builder.CreateLShr(X, CV); return new TruncInst(V, IIOperand->getType()); } break; @@ -1984,6 +1983,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { Value *IIOperand = II->getArgOperand(0); Value *X = nullptr; + // TODO should this be in InstSimplify? // bitreverse(bitreverse(x)) -> x if (match(IIOperand, m_BitReverse(m_Value(X)))) return replaceInstUsesWith(CI, X); @@ -1991,7 +1991,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { } case Intrinsic::masked_load: - if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II, *Builder)) + if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II, Builder)) return replaceInstUsesWith(CI, SimplifiedMaskedOp); break; case Intrinsic::masked_store: @@ -2010,7 +2010,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { if (Power->isOne()) return replaceInstUsesWith(CI, II->getArgOperand(0)); // powi(x, -1) -> 1/x - if (Power->isAllOnesValue()) + if (Power->isMinusOne()) return BinaryOperator::CreateFDiv(ConstantFP::get(CI.getType(), 1.0), II->getArgOperand(0)); } @@ -2073,11 +2073,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::fmuladd: { // Canonicalize fast fmuladd to the separate fmul + fadd. if (II->hasUnsafeAlgebra()) { - BuilderTy::FastMathFlagGuard Guard(*Builder); - Builder->setFastMathFlags(II->getFastMathFlags()); - Value *Mul = Builder->CreateFMul(II->getArgOperand(0), - II->getArgOperand(1)); - Value *Add = Builder->CreateFAdd(Mul, II->getArgOperand(2)); + BuilderTy::FastMathFlagGuard Guard(Builder); + Builder.setFastMathFlags(II->getFastMathFlags()); + Value *Mul = Builder.CreateFMul(II->getArgOperand(0), + II->getArgOperand(1)); + Value *Add = Builder.CreateFAdd(Mul, II->getArgOperand(2)); Add->takeName(II); return replaceInstUsesWith(*II, Add); } @@ -2128,8 +2128,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { Constant *LHS, *RHS; if (match(II->getArgOperand(0), m_Select(m_Value(Cond), m_Constant(LHS), m_Constant(RHS)))) { - CallInst *Call0 = Builder->CreateCall(II->getCalledFunction(), {LHS}); - CallInst *Call1 = Builder->CreateCall(II->getCalledFunction(), {RHS}); + CallInst *Call0 = Builder.CreateCall(II->getCalledFunction(), {LHS}); + CallInst *Call1 = Builder.CreateCall(II->getCalledFunction(), {RHS}); return SelectInst::Create(Cond, Call0, Call1); } @@ -2147,7 +2147,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // fabs (fpext x) -> fpext (fabs x) Value *F = Intrinsic::getDeclaration(II->getModule(), II->getIntrinsicID(), { ExtSrc->getType() }); - CallInst *NewFabs = Builder->CreateCall(F, ExtSrc); + CallInst *NewFabs = Builder.CreateCall(F, ExtSrc); NewFabs->copyFastMathFlags(II); NewFabs->takeName(II); return new FPExtInst(NewFabs, II->getType()); @@ -2174,7 +2174,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // Turn PPC lvx -> load if the pointer is known aligned. if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, &AC, &DT) >= 16) { - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0), PointerType::getUnqual(II->getType())); return new LoadInst(Ptr); } @@ -2182,8 +2182,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::ppc_vsx_lxvw4x: case Intrinsic::ppc_vsx_lxvd2x: { // Turn PPC VSX loads into normal loads. - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), - PointerType::getUnqual(II->getType())); + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0), + PointerType::getUnqual(II->getType())); return new LoadInst(Ptr, Twine(""), false, 1); } case Intrinsic::ppc_altivec_stvx: @@ -2193,7 +2193,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { &DT) >= 16) { Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType()); - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy); return new StoreInst(II->getArgOperand(0), Ptr); } break; @@ -2201,18 +2201,18 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::ppc_vsx_stxvd2x: { // Turn PPC VSX stores into normal stores. Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType()); - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy); return new StoreInst(II->getArgOperand(0), Ptr, false, 1); } case Intrinsic::ppc_qpx_qvlfs: // Turn PPC QPX qvlfs -> load if the pointer is known aligned. if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, &AC, &DT) >= 16) { - Type *VTy = VectorType::get(Builder->getFloatTy(), + Type *VTy = VectorType::get(Builder.getFloatTy(), II->getType()->getVectorNumElements()); - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0), PointerType::getUnqual(VTy)); - Value *Load = Builder->CreateLoad(Ptr); + Value *Load = Builder.CreateLoad(Ptr); return new FPExtInst(Load, II->getType()); } break; @@ -2220,7 +2220,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // Turn PPC QPX qvlfd -> load if the pointer is known aligned. if (getOrEnforceKnownAlignment(II->getArgOperand(0), 32, DL, II, &AC, &DT) >= 32) { - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0), PointerType::getUnqual(II->getType())); return new LoadInst(Ptr); } @@ -2229,11 +2229,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // Turn PPC QPX qvstfs -> store if the pointer is known aligned. if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, &AC, &DT) >= 16) { - Type *VTy = VectorType::get(Builder->getFloatTy(), + Type *VTy = VectorType::get(Builder.getFloatTy(), II->getArgOperand(0)->getType()->getVectorNumElements()); - Value *TOp = Builder->CreateFPTrunc(II->getArgOperand(0), VTy); + Value *TOp = Builder.CreateFPTrunc(II->getArgOperand(0), VTy); Type *OpPtrTy = PointerType::getUnqual(VTy); - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy); return new StoreInst(TOp, Ptr); } break; @@ -2243,7 +2243,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { &DT) >= 32) { Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType()); - Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); + Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy); return new StoreInst(II->getArgOperand(0), Ptr); } break; @@ -2272,15 +2272,15 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { SmallVector SubVecMask; for (unsigned i = 0; i != RetWidth; ++i) SubVecMask.push_back((int)i); - VectorHalfAsShorts = Builder->CreateShuffleVector( + VectorHalfAsShorts = Builder.CreateShuffleVector( Arg, UndefValue::get(ArgType), SubVecMask); } auto VectorHalfType = VectorType::get(Type::getHalfTy(II->getContext()), RetWidth); auto VectorHalfs = - Builder->CreateBitCast(VectorHalfAsShorts, VectorHalfType); - auto VectorFloats = Builder->CreateFPExt(VectorHalfs, RetType); + Builder.CreateBitCast(VectorHalfAsShorts, VectorHalfType); + auto VectorFloats = Builder.CreateFPExt(VectorHalfs, RetType); return replaceInstUsesWith(*II, VectorFloats); } @@ -2334,7 +2334,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx_movmsk_pd_256: case Intrinsic::x86_avx_movmsk_ps_256: case Intrinsic::x86_avx2_pmovmskb: { - if (Value *V = simplifyX86movmsk(*II, *Builder)) + if (Value *V = simplifyX86movmsk(*II)) return replaceInstUsesWith(*II, V); break; } @@ -2437,25 +2437,25 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { default: llvm_unreachable("Case stmts out of sync!"); case Intrinsic::x86_avx512_mask_add_ps_512: case Intrinsic::x86_avx512_mask_add_pd_512: - V = Builder->CreateFAdd(Arg0, Arg1); + V = Builder.CreateFAdd(Arg0, Arg1); break; case Intrinsic::x86_avx512_mask_sub_ps_512: case Intrinsic::x86_avx512_mask_sub_pd_512: - V = Builder->CreateFSub(Arg0, Arg1); + V = Builder.CreateFSub(Arg0, Arg1); break; case Intrinsic::x86_avx512_mask_mul_ps_512: case Intrinsic::x86_avx512_mask_mul_pd_512: - V = Builder->CreateFMul(Arg0, Arg1); + V = Builder.CreateFMul(Arg0, Arg1); break; case Intrinsic::x86_avx512_mask_div_ps_512: case Intrinsic::x86_avx512_mask_div_pd_512: - V = Builder->CreateFDiv(Arg0, Arg1); + V = Builder.CreateFDiv(Arg0, Arg1); break; } // Create a select for the masking. V = emitX86MaskSelect(II->getArgOperand(3), V, II->getArgOperand(2), - *Builder); + Builder); return replaceInstUsesWith(*II, V); } } @@ -2476,27 +2476,27 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // Extract the element as scalars. Value *Arg0 = II->getArgOperand(0); Value *Arg1 = II->getArgOperand(1); - Value *LHS = Builder->CreateExtractElement(Arg0, (uint64_t)0); - Value *RHS = Builder->CreateExtractElement(Arg1, (uint64_t)0); + Value *LHS = Builder.CreateExtractElement(Arg0, (uint64_t)0); + Value *RHS = Builder.CreateExtractElement(Arg1, (uint64_t)0); Value *V; switch (II->getIntrinsicID()) { default: llvm_unreachable("Case stmts out of sync!"); case Intrinsic::x86_avx512_mask_add_ss_round: case Intrinsic::x86_avx512_mask_add_sd_round: - V = Builder->CreateFAdd(LHS, RHS); + V = Builder.CreateFAdd(LHS, RHS); break; case Intrinsic::x86_avx512_mask_sub_ss_round: case Intrinsic::x86_avx512_mask_sub_sd_round: - V = Builder->CreateFSub(LHS, RHS); + V = Builder.CreateFSub(LHS, RHS); break; case Intrinsic::x86_avx512_mask_mul_ss_round: case Intrinsic::x86_avx512_mask_mul_sd_round: - V = Builder->CreateFMul(LHS, RHS); + V = Builder.CreateFMul(LHS, RHS); break; case Intrinsic::x86_avx512_mask_div_ss_round: case Intrinsic::x86_avx512_mask_div_sd_round: - V = Builder->CreateFDiv(LHS, RHS); + V = Builder.CreateFDiv(LHS, RHS); break; } @@ -2506,18 +2506,18 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // We don't need a select if we know the mask bit is a 1. if (!C || !C->getValue()[0]) { // Cast the mask to an i1 vector and then extract the lowest element. - auto *MaskTy = VectorType::get(Builder->getInt1Ty(), + auto *MaskTy = VectorType::get(Builder.getInt1Ty(), cast(Mask->getType())->getBitWidth()); - Mask = Builder->CreateBitCast(Mask, MaskTy); - Mask = Builder->CreateExtractElement(Mask, (uint64_t)0); + Mask = Builder.CreateBitCast(Mask, MaskTy); + Mask = Builder.CreateExtractElement(Mask, (uint64_t)0); // Extract the lowest element from the passthru operand. - Value *Passthru = Builder->CreateExtractElement(II->getArgOperand(2), + Value *Passthru = Builder.CreateExtractElement(II->getArgOperand(2), (uint64_t)0); - V = Builder->CreateSelect(Mask, V, Passthru); + V = Builder.CreateSelect(Mask, V, Passthru); } // Insert the result back into the original argument 0. - V = Builder->CreateInsertElement(Arg0, V, (uint64_t)0); + V = Builder.CreateInsertElement(Arg0, V, (uint64_t)0); return replaceInstUsesWith(*II, V); } @@ -2598,7 +2598,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx512_pslli_d_512: case Intrinsic::x86_avx512_pslli_q_512: case Intrinsic::x86_avx512_pslli_w_512: - if (Value *V = simplifyX86immShift(*II, *Builder)) + if (Value *V = simplifyX86immShift(*II, Builder)) return replaceInstUsesWith(*II, V); break; @@ -2629,7 +2629,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx512_psll_d_512: case Intrinsic::x86_avx512_psll_q_512: case Intrinsic::x86_avx512_psll_w_512: { - if (Value *V = simplifyX86immShift(*II, *Builder)) + if (Value *V = simplifyX86immShift(*II, Builder)) return replaceInstUsesWith(*II, V); // SSE2/AVX2 uses only the first 64-bits of the 128-bit vector @@ -2673,7 +2673,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx512_psrlv_w_128: case Intrinsic::x86_avx512_psrlv_w_256: case Intrinsic::x86_avx512_psrlv_w_512: - if (Value *V = simplifyX86varShift(*II, *Builder)) + if (Value *V = simplifyX86varShift(*II, Builder)) return replaceInstUsesWith(*II, V); break; @@ -2683,7 +2683,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx2_pmulu_dq: case Intrinsic::x86_avx512_pmul_dq_512: case Intrinsic::x86_avx512_pmulu_dq_512: { - if (Value *V = simplifyX86muldq(*II, *Builder)) + if (Value *V = simplifyX86muldq(*II, Builder)) return replaceInstUsesWith(*II, V); unsigned VWidth = II->getType()->getVectorNumElements(); @@ -2703,7 +2703,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx2_packsswb: case Intrinsic::x86_avx512_packssdw_512: case Intrinsic::x86_avx512_packsswb_512: - if (Value *V = simplifyX86pack(*II, *this, *Builder, true)) + if (Value *V = simplifyX86pack(*II, true)) return replaceInstUsesWith(*II, V); break; @@ -2713,7 +2713,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx2_packuswb: case Intrinsic::x86_avx512_packusdw_512: case Intrinsic::x86_avx512_packuswb_512: - if (Value *V = simplifyX86pack(*II, *this, *Builder, false)) + if (Value *V = simplifyX86pack(*II, false)) return replaceInstUsesWith(*II, V); break; @@ -2756,7 +2756,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { } case Intrinsic::x86_sse41_insertps: - if (Value *V = simplifyX86insertps(*II, *Builder)) + if (Value *V = simplifyX86insertps(*II, Builder)) return replaceInstUsesWith(*II, V); break; @@ -2779,7 +2779,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { : nullptr; // Attempt to simplify to a constant, shuffle vector or EXTRQI call. - if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, *Builder)) + if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, Builder)) return replaceInstUsesWith(*II, V); // EXTRQ only uses the lowest 64-bits of the first 128-bit vector @@ -2811,7 +2811,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { ConstantInt *CIIndex = dyn_cast(II->getArgOperand(2)); // Attempt to simplify to a constant or shuffle vector. - if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, *Builder)) + if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, Builder)) return replaceInstUsesWith(*II, V); // EXTRQI only uses the lowest 64-bits of the first 128-bit vector @@ -2843,7 +2843,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { const APInt &V11 = CI11->getValue(); APInt Len = V11.zextOrTrunc(6); APInt Idx = V11.lshr(8).zextOrTrunc(6); - if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, *Builder)) + if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, Builder)) return replaceInstUsesWith(*II, V); } @@ -2876,7 +2876,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { if (CILength && CIIndex) { APInt Len = CILength->getValue().zextOrTrunc(6); APInt Idx = CIIndex->getValue().zextOrTrunc(6); - if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, *Builder)) + if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, Builder)) return replaceInstUsesWith(*II, V); } @@ -2930,7 +2930,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_ssse3_pshuf_b_128: case Intrinsic::x86_avx2_pshuf_b: case Intrinsic::x86_avx512_pshuf_b_512: - if (Value *V = simplifyX86pshufb(*II, *Builder)) + if (Value *V = simplifyX86pshufb(*II, Builder)) return replaceInstUsesWith(*II, V); break; @@ -2940,13 +2940,13 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx_vpermilvar_pd: case Intrinsic::x86_avx_vpermilvar_pd_256: case Intrinsic::x86_avx512_vpermilvar_pd_512: - if (Value *V = simplifyX86vpermilvar(*II, *Builder)) + if (Value *V = simplifyX86vpermilvar(*II, Builder)) return replaceInstUsesWith(*II, V); break; case Intrinsic::x86_avx2_permd: case Intrinsic::x86_avx2_permps: - if (Value *V = simplifyX86vpermv(*II, *Builder)) + if (Value *V = simplifyX86vpermv(*II, Builder)) return replaceInstUsesWith(*II, V); break; @@ -2964,10 +2964,10 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx512_mask_permvar_sf_512: case Intrinsic::x86_avx512_mask_permvar_si_256: case Intrinsic::x86_avx512_mask_permvar_si_512: - if (Value *V = simplifyX86vpermv(*II, *Builder)) { + if (Value *V = simplifyX86vpermv(*II, Builder)) { // We simplified the permuting, now create a select for the masking. V = emitX86MaskSelect(II->getArgOperand(3), V, II->getArgOperand(2), - *Builder); + Builder); return replaceInstUsesWith(*II, V); } break; @@ -2976,7 +2976,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_avx_vperm2f128_ps_256: case Intrinsic::x86_avx_vperm2f128_si_256: case Intrinsic::x86_avx2_vperm2i128: - if (Value *V = simplifyX86vperm2(*II, *Builder)) + if (Value *V = simplifyX86vperm2(*II, Builder)) return replaceInstUsesWith(*II, V); break; @@ -3009,7 +3009,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_xop_vpcomd: case Intrinsic::x86_xop_vpcomq: case Intrinsic::x86_xop_vpcomw: - if (Value *V = simplifyX86vpcom(*II, *Builder, true)) + if (Value *V = simplifyX86vpcom(*II, Builder, true)) return replaceInstUsesWith(*II, V); break; @@ -3017,7 +3017,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { case Intrinsic::x86_xop_vpcomud: case Intrinsic::x86_xop_vpcomuq: case Intrinsic::x86_xop_vpcomuw: - if (Value *V = simplifyX86vpcom(*II, *Builder, false)) + if (Value *V = simplifyX86vpcom(*II, Builder, false)) return replaceInstUsesWith(*II, V); break; @@ -3044,10 +3044,10 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { if (AllEltsOk) { // Cast the input vectors to byte vectors. - Value *Op0 = Builder->CreateBitCast(II->getArgOperand(0), - Mask->getType()); - Value *Op1 = Builder->CreateBitCast(II->getArgOperand(1), - Mask->getType()); + Value *Op0 = Builder.CreateBitCast(II->getArgOperand(0), + Mask->getType()); + Value *Op1 = Builder.CreateBitCast(II->getArgOperand(1), + Mask->getType()); Value *Result = UndefValue::get(Op0->getType()); // Only extract each element once. @@ -3067,13 +3067,13 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0; Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1; ExtractedElts[Idx] = - Builder->CreateExtractElement(Idx < 16 ? Op0ToUse : Op1ToUse, - Builder->getInt32(Idx&15)); + Builder.CreateExtractElement(Idx < 16 ? Op0ToUse : Op1ToUse, + Builder.getInt32(Idx&15)); } // Insert this value into the result vector. - Result = Builder->CreateInsertElement(Result, ExtractedElts[Idx], - Builder->getInt32(i)); + Result = Builder.CreateInsertElement(Result, ExtractedElts[Idx], + Builder.getInt32(i)); } return CastInst::Create(Instruction::BitCast, Result, CI.getType()); } @@ -3238,7 +3238,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { if (Mask == (S_NAN | Q_NAN)) { // Equivalent of isnan. Replace with standard fcmp. - Value *FCmp = Builder->CreateFCmpUNO(Src0, Src0); + Value *FCmp = Builder.CreateFCmpUNO(Src0, Src0); FCmp->takeName(II); return replaceInstUsesWith(*II, FCmp); } @@ -3250,7 +3250,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // Clamp mask to used bits if ((Mask & FullMask) != Mask) { - CallInst *NewCall = Builder->CreateCall(II->getCalledFunction(), + CallInst *NewCall = Builder.CreateCall(II->getCalledFunction(), { Src0, ConstantInt::get(Src1->getType(), Mask & FullMask) } ); @@ -3343,13 +3343,12 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // TODO: Also emit sub if only width is constant. if (!CWidth && COffset && Offset == 0) { Constant *KSize = ConstantInt::get(COffset->getType(), IntSize); - Value *ShiftVal = Builder->CreateSub(KSize, II->getArgOperand(2)); - ShiftVal = Builder->CreateZExt(ShiftVal, II->getType()); + Value *ShiftVal = Builder.CreateSub(KSize, II->getArgOperand(2)); + ShiftVal = Builder.CreateZExt(ShiftVal, II->getType()); - Value *Shl = Builder->CreateShl(Src, ShiftVal); - Value *RightShift = Signed ? - Builder->CreateAShr(Shl, ShiftVal) : - Builder->CreateLShr(Shl, ShiftVal); + Value *Shl = Builder.CreateShl(Src, ShiftVal); + Value *RightShift = Signed ? Builder.CreateAShr(Shl, ShiftVal) + : Builder.CreateLShr(Shl, ShiftVal); RightShift->takeName(II); return replaceInstUsesWith(*II, RightShift); } @@ -3360,17 +3359,15 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // TODO: This allows folding to undef when the hardware has specific // behavior? if (Offset + Width < IntSize) { - Value *Shl = Builder->CreateShl(Src, IntSize - Offset - Width); - Value *RightShift = Signed ? - Builder->CreateAShr(Shl, IntSize - Width) : - Builder->CreateLShr(Shl, IntSize - Width); + Value *Shl = Builder.CreateShl(Src, IntSize - Offset - Width); + Value *RightShift = Signed ? Builder.CreateAShr(Shl, IntSize - Width) + : Builder.CreateLShr(Shl, IntSize - Width); RightShift->takeName(II); return replaceInstUsesWith(*II, RightShift); } - Value *RightShift = Signed ? - Builder->CreateAShr(Src, Offset) : - Builder->CreateLShr(Src, Offset); + Value *RightShift = Signed ? Builder.CreateAShr(Src, Offset) + : Builder.CreateLShr(Src, Offset); RightShift->takeName(II); return replaceInstUsesWith(*II, RightShift); @@ -3439,7 +3436,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { } if (match(Src2, m_NaN()) || isa(Src2)) { - CallInst *NewCall = Builder->CreateMinNum(Src0, Src1); + CallInst *NewCall = Builder.CreateMinNum(Src0, Src1); NewCall->copyFastMathFlags(II); NewCall->takeName(II); return replaceInstUsesWith(*II, NewCall); @@ -3451,7 +3448,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(), C2->getValueAPF()); return replaceInstUsesWith(*II, - ConstantFP::get(Builder->getContext(), Result)); + ConstantFP::get(Builder.getContext(), Result)); } } } @@ -3494,7 +3491,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { Metadata *MDArgs[] = {MDString::get(II->getContext(), "exec")}; MDNode *MD = MDNode::get(II->getContext(), MDArgs); Value *Args[] = {MetadataAsValue::get(II->getContext(), MD)}; - CallInst *NewCall = Builder->CreateCall(NewF, Args); + CallInst *NewCall = Builder.CreateCall(NewF, Args); NewCall->addAttribute(AttributeList::FunctionIndex, Attribute::Convergent); NewCall->takeName(II); @@ -3556,7 +3553,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { SrcLHS->getType()); Value *Args[] = { SrcLHS, SrcRHS, ConstantInt::get(CC->getType(), SrcPred) }; - CallInst *NewCall = Builder->CreateCall(NewF, Args); + CallInst *NewCall = Builder.CreateCall(NewF, Args); NewCall->takeName(II); return replaceInstUsesWith(*II, NewCall); } @@ -3633,16 +3630,14 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { // the InstCombineIRInserter object. Value *AssumeIntrinsic = II->getCalledValue(), *A, *B; if (match(IIOperand, m_And(m_Value(A), m_Value(B)))) { - Builder->CreateCall(AssumeIntrinsic, A, II->getName()); - Builder->CreateCall(AssumeIntrinsic, B, II->getName()); + Builder.CreateCall(AssumeIntrinsic, A, II->getName()); + Builder.CreateCall(AssumeIntrinsic, B, II->getName()); return eraseInstFromFunction(*II); } // assume(!(a || b)) -> assume(!a); assume(!b); if (match(IIOperand, m_Not(m_Or(m_Value(A), m_Value(B))))) { - Builder->CreateCall(AssumeIntrinsic, Builder->CreateNot(A), - II->getName()); - Builder->CreateCall(AssumeIntrinsic, Builder->CreateNot(B), - II->getName()); + Builder.CreateCall(AssumeIntrinsic, Builder.CreateNot(A), II->getName()); + Builder.CreateCall(AssumeIntrinsic, Builder.CreateNot(B), II->getName()); return eraseInstFromFunction(*II); } @@ -3726,7 +3721,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { return eraseInstFromFunction(*NextInst); // Otherwise canonicalize guard(a); guard(b) -> guard(a & b). - II->setArgOperand(0, Builder->CreateAnd(CurrCond, NextCond)); + II->setArgOperand(0, Builder.CreateAnd(CurrCond, NextCond)); return eraseInstFromFunction(*NextInst); } break; @@ -4163,7 +4158,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) { Value *NewArg = *AI; if ((*AI)->getType() != ParamTy) - NewArg = Builder->CreateBitOrPointerCast(*AI, ParamTy); + NewArg = Builder.CreateBitOrPointerCast(*AI, ParamTy); Args.push_back(NewArg); // Add any parameter attributes. @@ -4189,7 +4184,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) { // Must promote to pass through va_arg area! Instruction::CastOps opcode = CastInst::getCastOpcode(*AI, false, PTy, false); - NewArg = Builder->CreateCast(opcode, *AI, PTy); + NewArg = Builder.CreateCast(opcode, *AI, PTy); } Args.push_back(NewArg); @@ -4215,10 +4210,10 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) { CallSite NewCS; if (InvokeInst *II = dyn_cast(Caller)) { - NewCS = Builder->CreateInvoke(Callee, II->getNormalDest(), - II->getUnwindDest(), Args, OpBundles); + NewCS = Builder.CreateInvoke(Callee, II->getNormalDest(), + II->getUnwindDest(), Args, OpBundles); } else { - NewCS = Builder->CreateCall(Callee, Args, OpBundles); + NewCS = Builder.CreateCall(Callee, Args, OpBundles); cast(NewCS.getInstruction()) ->setTailCallKind(cast(Caller)->getTailCallKind()); } @@ -4328,7 +4323,7 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS, // Add the chain argument and attributes. Value *NestVal = Tramp->getArgOperand(2); if (NestVal->getType() != NestTy) - NestVal = Builder->CreateBitCast(NestVal, NestTy, "nest"); + NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest"); NewArgs.push_back(NestVal); NewArgAttrs.push_back(NestAttr); } diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index d3049389dfb9..dfdfd3e9da84 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -84,7 +84,7 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI, AllocaInst &AI) { PointerType *PTy = cast(CI.getType()); - BuilderTy AllocaBuilder(*Builder); + BuilderTy AllocaBuilder(Builder); AllocaBuilder.SetInsertPoint(&AI); // Get the type really allocated and the type casted to. @@ -406,8 +406,7 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombiner &IC, /// trunc (lshr (bitcast <4 x i32> %X to i128), 32) to i32 /// ---> /// extractelement <4 x i32> %X, 1 -static Instruction *foldVecTruncToExtElt(TruncInst &Trunc, InstCombiner &IC, - const DataLayout &DL) { +static Instruction *foldVecTruncToExtElt(TruncInst &Trunc, InstCombiner &IC) { Value *TruncOp = Trunc.getOperand(0); Type *DestType = Trunc.getType(); if (!TruncOp->hasOneUse() || !isa(DestType)) @@ -434,14 +433,14 @@ static Instruction *foldVecTruncToExtElt(TruncInst &Trunc, InstCombiner &IC, unsigned NumVecElts = VecWidth / DestWidth; if (VecType->getElementType() != DestType) { VecType = VectorType::get(DestType, NumVecElts); - VecInput = IC.Builder->CreateBitCast(VecInput, VecType, "bc"); + VecInput = IC.Builder.CreateBitCast(VecInput, VecType, "bc"); } unsigned Elt = ShiftAmount / DestWidth; - if (DL.isBigEndian()) + if (IC.getDataLayout().isBigEndian()) Elt = NumVecElts - 1 - Elt; - return ExtractElementInst::Create(VecInput, IC.Builder->getInt32(Elt)); + return ExtractElementInst::Create(VecInput, IC.Builder.getInt32(Elt)); } /// Try to narrow the width of bitwise logic instructions with constants. @@ -460,7 +459,7 @@ Instruction *InstCombiner::shrinkBitwiseLogic(TruncInst &Trunc) { // trunc (logic X, C) --> logic (trunc X, C') Constant *NarrowC = ConstantExpr::getTrunc(C, DestTy); - Value *NarrowOp0 = Builder->CreateTrunc(LogicOp->getOperand(0), DestTy); + Value *NarrowOp0 = Builder.CreateTrunc(LogicOp->getOperand(0), DestTy); return BinaryOperator::Create(LogicOp->getOpcode(), NarrowOp0, NarrowC); } @@ -554,7 +553,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) { // Canonicalize trunc x to i1 -> (icmp ne (and x, 1), 0), likewise for vector. if (DestTy->getScalarSizeInBits() == 1) { Constant *One = ConstantInt::get(SrcTy, 1); - Src = Builder->CreateAnd(Src, One); + Src = Builder.CreateAnd(Src, One); Value *Zero = Constant::getNullValue(Src->getType()); return new ICmpInst(ICmpInst::ICMP_NE, Src, Zero); } @@ -580,7 +579,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) { // Since we're doing an lshr and a zero extend, and know that the shift // amount is smaller than ASize, it is always safe to do the shift in A's // type, then zero extend or truncate to the result. - Value *Shift = Builder->CreateLShr(A, Cst->getZExtValue()); + Value *Shift = Builder.CreateLShr(A, Cst->getZExtValue()); Shift->takeName(Src); return CastInst::CreateIntegerCast(Shift, DestTy, false); } @@ -610,7 +609,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) { return BinaryOperator::CreateAShr(A, ConstantInt::get(CI.getType(), std::min(ShiftAmt, ASize - 1))); if (SExt->hasOneUse()) { - Value *Shift = Builder->CreateAShr(A, std::min(ShiftAmt, ASize-1)); + Value *Shift = Builder.CreateAShr(A, std::min(ShiftAmt, ASize - 1)); Shift->takeName(Src); return CastInst::CreateIntegerCast(Shift, CI.getType(), true); } @@ -620,10 +619,10 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) { if (Instruction *I = shrinkBitwiseLogic(CI)) return I; - if (Instruction *I = shrinkSplatShuffle(CI, *Builder)) + if (Instruction *I = shrinkSplatShuffle(CI, Builder)) return I; - if (Instruction *I = shrinkInsertElt(CI, *Builder)) + if (Instruction *I = shrinkInsertElt(CI, Builder)) return I; if (Src->hasOneUse() && isa(SrcTy) && @@ -636,7 +635,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) { // FoldShiftByConstant and is the extend in reg pattern. const unsigned DestSize = DestTy->getScalarSizeInBits(); if (Cst->getValue().ult(DestSize)) { - Value *NewTrunc = Builder->CreateTrunc(A, DestTy, A->getName() + ".tr"); + Value *NewTrunc = Builder.CreateTrunc(A, DestTy, A->getName() + ".tr"); return BinaryOperator::Create( Instruction::Shl, NewTrunc, @@ -645,7 +644,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) { } } - if (Instruction *I = foldVecTruncToExtElt(CI, *this, DL)) + if (Instruction *I = foldVecTruncToExtElt(CI, *this)) return I; return nullptr; @@ -668,13 +667,13 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, ZExtInst &CI, Value *In = ICI->getOperand(0); Value *Sh = ConstantInt::get(In->getType(), In->getType()->getScalarSizeInBits() - 1); - In = Builder->CreateLShr(In, Sh, In->getName() + ".lobit"); + In = Builder.CreateLShr(In, Sh, In->getName() + ".lobit"); if (In->getType() != CI.getType()) - In = Builder->CreateIntCast(In, CI.getType(), false/*ZExt*/); + In = Builder.CreateIntCast(In, CI.getType(), false /*ZExt*/); if (ICI->getPredicate() == ICmpInst::ICMP_SGT) { Constant *One = ConstantInt::get(In->getType(), 1); - In = Builder->CreateXor(In, One, In->getName() + ".not"); + In = Builder.CreateXor(In, One, In->getName() + ".not"); } return replaceInstUsesWith(CI, In); @@ -713,19 +712,19 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, ZExtInst &CI, if (ShAmt) { // Perform a logical shr by shiftamt. // Insert the shift to put the result in the low bit. - In = Builder->CreateLShr(In, ConstantInt::get(In->getType(), ShAmt), - In->getName() + ".lobit"); + In = Builder.CreateLShr(In, ConstantInt::get(In->getType(), ShAmt), + In->getName() + ".lobit"); } if (!Op1CV.isNullValue() == isNE) { // Toggle the low bit. Constant *One = ConstantInt::get(In->getType(), 1); - In = Builder->CreateXor(In, One); + In = Builder.CreateXor(In, One); } if (CI.getType() == In->getType()) return replaceInstUsesWith(CI, In); - Value *IntCast = Builder->CreateIntCast(In, CI.getType(), false); + Value *IntCast = Builder.CreateIntCast(In, CI.getType(), false); return replaceInstUsesWith(CI, IntCast); } } @@ -748,19 +747,19 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *ICI, ZExtInst &CI, if (UnknownBit.countPopulation() == 1) { if (!DoTransform) return ICI; - Value *Result = Builder->CreateXor(LHS, RHS); + Value *Result = Builder.CreateXor(LHS, RHS); // Mask off any bits that are set and won't be shifted away. if (KnownLHS.One.uge(UnknownBit)) - Result = Builder->CreateAnd(Result, + Result = Builder.CreateAnd(Result, ConstantInt::get(ITy, UnknownBit)); // Shift the bit we're testing down to the lsb. - Result = Builder->CreateLShr( + Result = Builder.CreateLShr( Result, ConstantInt::get(ITy, UnknownBit.countTrailingZeros())); if (ICI->getPredicate() == ICmpInst::ICMP_EQ) - Result = Builder->CreateXor(Result, ConstantInt::get(ITy, 1)); + Result = Builder.CreateXor(Result, ConstantInt::get(ITy, 1)); Result->takeName(ICI); return replaceInstUsesWith(CI, Result); } @@ -960,7 +959,7 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) { if (SrcSize < DstSize) { APInt AndValue(APInt::getLowBitsSet(SrcSize, MidSize)); Constant *AndConst = ConstantInt::get(A->getType(), AndValue); - Value *And = Builder->CreateAnd(A, AndConst, CSrc->getName()+".mask"); + Value *And = Builder.CreateAnd(A, AndConst, CSrc->getName() + ".mask"); return new ZExtInst(And, CI.getType()); } @@ -970,7 +969,7 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) { AndValue)); } if (SrcSize > DstSize) { - Value *Trunc = Builder->CreateTrunc(A, CI.getType()); + Value *Trunc = Builder.CreateTrunc(A, CI.getType()); APInt AndValue(APInt::getLowBitsSet(DstSize, MidSize)); return BinaryOperator::CreateAnd(Trunc, ConstantInt::get(Trunc->getType(), @@ -992,8 +991,8 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) { (transformZExtICmp(LHS, CI, false) || transformZExtICmp(RHS, CI, false))) { // zext (or icmp, icmp) -> or (zext icmp), (zext icmp) - Value *LCast = Builder->CreateZExt(LHS, CI.getType(), LHS->getName()); - Value *RCast = Builder->CreateZExt(RHS, CI.getType(), RHS->getName()); + Value *LCast = Builder.CreateZExt(LHS, CI.getType(), LHS->getName()); + Value *RCast = Builder.CreateZExt(RHS, CI.getType(), RHS->getName()); BinaryOperator *Or = BinaryOperator::Create(Instruction::Or, LCast, RCast); // Perform the elimination. @@ -1020,7 +1019,7 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) { match(And, m_OneUse(m_And(m_Trunc(m_Value(X)), m_Specific(C)))) && X->getType() == CI.getType()) { Constant *ZC = ConstantExpr::getZExt(C, CI.getType()); - return BinaryOperator::CreateXor(Builder->CreateAnd(X, ZC), ZC); + return BinaryOperator::CreateXor(Builder.CreateAnd(X, ZC), ZC); } return nullptr; @@ -1043,12 +1042,12 @@ Instruction *InstCombiner::transformSExtICmp(ICmpInst *ICI, Instruction &CI) { Value *Sh = ConstantInt::get(Op0->getType(), Op0->getType()->getScalarSizeInBits()-1); - Value *In = Builder->CreateAShr(Op0, Sh, Op0->getName()+".lobit"); + Value *In = Builder.CreateAShr(Op0, Sh, Op0->getName() + ".lobit"); if (In->getType() != CI.getType()) - In = Builder->CreateIntCast(In, CI.getType(), true/*SExt*/); + In = Builder.CreateIntCast(In, CI.getType(), true /*SExt*/); if (Pred == ICmpInst::ICMP_SGT) - In = Builder->CreateNot(In, In->getName()+".not"); + In = Builder.CreateNot(In, In->getName() + ".not"); return replaceInstUsesWith(CI, In); } } @@ -1079,26 +1078,26 @@ Instruction *InstCombiner::transformSExtICmp(ICmpInst *ICI, Instruction &CI) { unsigned ShiftAmt = KnownZeroMask.countTrailingZeros(); // Perform a right shift to place the desired bit in the LSB. if (ShiftAmt) - In = Builder->CreateLShr(In, - ConstantInt::get(In->getType(), ShiftAmt)); + In = Builder.CreateLShr(In, + ConstantInt::get(In->getType(), ShiftAmt)); // At this point "In" is either 1 or 0. Subtract 1 to turn // {1, 0} -> {0, -1}. - In = Builder->CreateAdd(In, - ConstantInt::getAllOnesValue(In->getType()), - "sext"); + In = Builder.CreateAdd(In, + ConstantInt::getAllOnesValue(In->getType()), + "sext"); } else { // sext ((x & 2^n) != 0) -> (x << bitwidth-n) a>> bitwidth-1 // sext ((x & 2^n) == 2^n) -> (x << bitwidth-n) a>> bitwidth-1 unsigned ShiftAmt = KnownZeroMask.countLeadingZeros(); // Perform a left shift to place the desired bit in the MSB. if (ShiftAmt) - In = Builder->CreateShl(In, - ConstantInt::get(In->getType(), ShiftAmt)); + In = Builder.CreateShl(In, + ConstantInt::get(In->getType(), ShiftAmt)); // Distribute the bit over the whole bit width. - In = Builder->CreateAShr(In, ConstantInt::get(In->getType(), - KnownZeroMask.getBitWidth() - 1), "sext"); + In = Builder.CreateAShr(In, ConstantInt::get(In->getType(), + KnownZeroMask.getBitWidth() - 1), "sext"); } if (CI.getType() == In->getType()) @@ -1191,7 +1190,7 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) { // instead. KnownBits Known = computeKnownBits(Src, 0, &CI); if (Known.isNonNegative()) { - Value *ZExt = Builder->CreateZExt(Src, DestTy); + Value *ZExt = Builder.CreateZExt(Src, DestTy); return replaceInstUsesWith(CI, ZExt); } @@ -1217,7 +1216,7 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) { // We need to emit a shl + ashr to do the sign extend. Value *ShAmt = ConstantInt::get(DestTy, DestBitSize-SrcBitSize); - return BinaryOperator::CreateAShr(Builder->CreateShl(Res, ShAmt, "sext"), + return BinaryOperator::CreateAShr(Builder.CreateShl(Res, ShAmt, "sext"), ShAmt); } @@ -1229,7 +1228,7 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) { unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); unsigned DestBitSize = DestTy->getScalarSizeInBits(); Constant *ShAmt = ConstantInt::get(DestTy, DestBitSize - SrcBitSize); - return BinaryOperator::CreateAShr(Builder->CreateShl(X, ShAmt), ShAmt); + return BinaryOperator::CreateAShr(Builder.CreateShl(X, ShAmt), ShAmt); } if (ICmpInst *ICI = dyn_cast(Src)) @@ -1258,7 +1257,7 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) { unsigned SrcDstSize = CI.getType()->getScalarSizeInBits(); unsigned ShAmt = CA->getZExtValue()+SrcDstSize-MidSize; Constant *ShAmtV = ConstantInt::get(CI.getType(), ShAmt); - A = Builder->CreateShl(A, ShAmtV, CI.getName()); + A = Builder.CreateShl(A, ShAmtV, CI.getName()); return BinaryOperator::CreateAShr(A, ShAmtV); } @@ -1347,9 +1346,9 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { // case of interest here is (float)((double)float + float)). if (OpWidth >= 2*DstWidth+1 && DstWidth >= SrcWidth) { if (LHSOrig->getType() != CI.getType()) - LHSOrig = Builder->CreateFPExt(LHSOrig, CI.getType()); + LHSOrig = Builder.CreateFPExt(LHSOrig, CI.getType()); if (RHSOrig->getType() != CI.getType()) - RHSOrig = Builder->CreateFPExt(RHSOrig, CI.getType()); + RHSOrig = Builder.CreateFPExt(RHSOrig, CI.getType()); Instruction *RI = BinaryOperator::Create(OpI->getOpcode(), LHSOrig, RHSOrig); RI->copyFastMathFlags(OpI); @@ -1364,9 +1363,9 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { // in the destination format if it can represent both sources. if (OpWidth >= LHSWidth + RHSWidth && DstWidth >= SrcWidth) { if (LHSOrig->getType() != CI.getType()) - LHSOrig = Builder->CreateFPExt(LHSOrig, CI.getType()); + LHSOrig = Builder.CreateFPExt(LHSOrig, CI.getType()); if (RHSOrig->getType() != CI.getType()) - RHSOrig = Builder->CreateFPExt(RHSOrig, CI.getType()); + RHSOrig = Builder.CreateFPExt(RHSOrig, CI.getType()); Instruction *RI = BinaryOperator::CreateFMul(LHSOrig, RHSOrig); RI->copyFastMathFlags(OpI); @@ -1382,9 +1381,9 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { // TODO: Tighten bound via rigorous analysis of the unbalanced case. if (OpWidth >= 2*DstWidth && DstWidth >= SrcWidth) { if (LHSOrig->getType() != CI.getType()) - LHSOrig = Builder->CreateFPExt(LHSOrig, CI.getType()); + LHSOrig = Builder.CreateFPExt(LHSOrig, CI.getType()); if (RHSOrig->getType() != CI.getType()) - RHSOrig = Builder->CreateFPExt(RHSOrig, CI.getType()); + RHSOrig = Builder.CreateFPExt(RHSOrig, CI.getType()); Instruction *RI = BinaryOperator::CreateFDiv(LHSOrig, RHSOrig); RI->copyFastMathFlags(OpI); @@ -1399,11 +1398,11 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { if (SrcWidth == OpWidth) break; if (LHSWidth < SrcWidth) - LHSOrig = Builder->CreateFPExt(LHSOrig, RHSOrig->getType()); + LHSOrig = Builder.CreateFPExt(LHSOrig, RHSOrig->getType()); else if (RHSWidth <= SrcWidth) - RHSOrig = Builder->CreateFPExt(RHSOrig, LHSOrig->getType()); + RHSOrig = Builder.CreateFPExt(RHSOrig, LHSOrig->getType()); if (LHSOrig != OpI->getOperand(0) || RHSOrig != OpI->getOperand(1)) { - Value *ExactResult = Builder->CreateFRem(LHSOrig, RHSOrig); + Value *ExactResult = Builder.CreateFRem(LHSOrig, RHSOrig); if (Instruction *RI = dyn_cast(ExactResult)) RI->copyFastMathFlags(OpI); return CastInst::CreateFPCast(ExactResult, CI.getType()); @@ -1412,8 +1411,8 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { // (fptrunc (fneg x)) -> (fneg (fptrunc x)) if (BinaryOperator::isFNeg(OpI)) { - Value *InnerTrunc = Builder->CreateFPTrunc(OpI->getOperand(1), - CI.getType()); + Value *InnerTrunc = Builder.CreateFPTrunc(OpI->getOperand(1), + CI.getType()); Instruction *RI = BinaryOperator::CreateFNeg(InnerTrunc); RI->copyFastMathFlags(OpI); return RI; @@ -1432,10 +1431,8 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { (isa(SI->getOperand(1)) || isa(SI->getOperand(2))) && matchSelectPattern(SI, LHS, RHS).Flavor == SPF_UNKNOWN) { - Value *LHSTrunc = Builder->CreateFPTrunc(SI->getOperand(1), - CI.getType()); - Value *RHSTrunc = Builder->CreateFPTrunc(SI->getOperand(2), - CI.getType()); + Value *LHSTrunc = Builder.CreateFPTrunc(SI->getOperand(1), CI.getType()); + Value *RHSTrunc = Builder.CreateFPTrunc(SI->getOperand(2), CI.getType()); return SelectInst::Create(SI->getOperand(0), LHSTrunc, RHSTrunc); } @@ -1465,7 +1462,7 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { // Do unary FP operation on smaller type. // (fptrunc (fabs x)) -> (fabs (fptrunc x)) - Value *InnerTrunc = Builder->CreateFPTrunc(Src, CI.getType()); + Value *InnerTrunc = Builder.CreateFPTrunc(Src, CI.getType()); Type *IntrinsicType[] = { CI.getType() }; Function *Overload = Intrinsic::getDeclaration( CI.getModule(), II->getIntrinsicID(), IntrinsicType); @@ -1482,7 +1479,7 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &CI) { } } - if (Instruction *I = shrinkInsertElt(CI, *Builder)) + if (Instruction *I = shrinkInsertElt(CI, Builder)) return I; return nullptr; @@ -1577,7 +1574,7 @@ Instruction *InstCombiner::visitIntToPtr(IntToPtrInst &CI) { if (CI.getType()->isVectorTy()) // Handle vectors of pointers. Ty = VectorType::get(Ty, CI.getType()->getVectorNumElements()); - Value *P = Builder->CreateZExtOrTrunc(CI.getOperand(0), Ty); + Value *P = Builder.CreateZExtOrTrunc(CI.getOperand(0), Ty); return new IntToPtrInst(P, CI.getType()); } @@ -1627,7 +1624,7 @@ Instruction *InstCombiner::visitPtrToInt(PtrToIntInst &CI) { if (Ty->isVectorTy()) // Handle vectors of pointers. PtrTy = VectorType::get(PtrTy, Ty->getVectorNumElements()); - Value *P = Builder->CreatePtrToInt(CI.getOperand(0), PtrTy); + Value *P = Builder.CreatePtrToInt(CI.getOperand(0), PtrTy); return CastInst::CreateIntegerCast(P, Ty, /*isSigned=*/false); } @@ -1653,7 +1650,7 @@ static Instruction *optimizeVectorResize(Value *InVal, VectorType *DestTy, return nullptr; SrcTy = VectorType::get(DestTy->getElementType(), SrcTy->getNumElements()); - InVal = IC.Builder->CreateBitCast(InVal, SrcTy); + InVal = IC.Builder.CreateBitCast(InVal, SrcTy); } // Now that the element types match, get the shuffle mask and RHS of the @@ -1833,8 +1830,8 @@ static Value *optimizeIntegerToVectorInsertions(BitCastInst &CI, for (unsigned i = 0, e = Elements.size(); i != e; ++i) { if (!Elements[i]) continue; // Unset element. - Result = IC.Builder->CreateInsertElement(Result, Elements[i], - IC.Builder->getInt32(i)); + Result = IC.Builder.CreateInsertElement(Result, Elements[i], + IC.Builder.getInt32(i)); } return Result; @@ -1845,8 +1842,7 @@ static Value *optimizeIntegerToVectorInsertions(BitCastInst &CI, /// vectors better than bitcasts of scalars because vector registers are /// usually not type-specific like scalar integer or scalar floating-point. static Instruction *canonicalizeBitCastExtElt(BitCastInst &BitCast, - InstCombiner &IC, - const DataLayout &DL) { + InstCombiner &IC) { // TODO: Create and use a pattern matcher for ExtractElementInst. auto *ExtElt = dyn_cast(BitCast.getOperand(0)); if (!ExtElt || !ExtElt->hasOneUse()) @@ -1860,8 +1856,8 @@ static Instruction *canonicalizeBitCastExtElt(BitCastInst &BitCast, unsigned NumElts = ExtElt->getVectorOperandType()->getNumElements(); auto *NewVecType = VectorType::get(DestType, NumElts); - auto *NewBC = IC.Builder->CreateBitCast(ExtElt->getVectorOperand(), - NewVecType, "bc"); + auto *NewBC = IC.Builder.CreateBitCast(ExtElt->getVectorOperand(), + NewVecType, "bc"); return ExtractElementInst::Create(NewBC, ExtElt->getIndexOperand()); } @@ -1870,7 +1866,7 @@ static Instruction *foldBitCastBitwiseLogic(BitCastInst &BitCast, InstCombiner::BuilderTy &Builder) { Type *DestTy = BitCast.getType(); BinaryOperator *BO; - if (!DestTy->getScalarType()->isIntegerTy() || + if (!DestTy->isIntOrIntVectorTy() || !match(BitCast.getOperand(0), m_OneUse(m_BinOp(BO))) || !BO->isBitwiseLogicOp()) return nullptr; @@ -2033,8 +2029,8 @@ Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) { // For each old PHI node, create a corresponding new PHI node with a type A. SmallDenseMap NewPNodes; for (auto *OldPN : OldPhiNodes) { - Builder->SetInsertPoint(OldPN); - PHINode *NewPN = Builder->CreatePHI(DestTy, OldPN->getNumOperands()); + Builder.SetInsertPoint(OldPN); + PHINode *NewPN = Builder.CreatePHI(DestTy, OldPN->getNumOperands()); NewPNodes[OldPN] = NewPN; } @@ -2047,8 +2043,8 @@ Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) { if (auto *C = dyn_cast(V)) { NewV = ConstantExpr::getBitCast(C, DestTy); } else if (auto *LI = dyn_cast(V)) { - Builder->SetInsertPoint(LI->getNextNode()); - NewV = Builder->CreateBitCast(LI, DestTy); + Builder.SetInsertPoint(LI->getNextNode()); + NewV = Builder.CreateBitCast(LI, DestTy); Worklist.Add(LI); } else if (auto *BCI = dyn_cast(V)) { NewV = BCI->getOperand(0); @@ -2064,9 +2060,9 @@ Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) { for (User *U : PN->users()) { auto *SI = dyn_cast(U); if (SI && SI->isSimple() && SI->getOperand(0) == PN) { - Builder->SetInsertPoint(SI); + Builder.SetInsertPoint(SI); auto *NewBC = - cast(Builder->CreateBitCast(NewPNodes[PN], SrcTy)); + cast(Builder.CreateBitCast(NewPNodes[PN], SrcTy)); SI->setOperand(0, NewBC); Worklist.Add(SI); assert(hasStoreUsersOnly(*NewBC)); @@ -2121,14 +2117,14 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) { // If we found a path from the src to dest, create the getelementptr now. if (SrcElTy == DstElTy) { - SmallVector Idxs(NumZeros + 1, Builder->getInt32(0)); + SmallVector Idxs(NumZeros + 1, Builder.getInt32(0)); return GetElementPtrInst::CreateInBounds(Src, Idxs); } } if (VectorType *DestVTy = dyn_cast(DestTy)) { if (DestVTy->getNumElements() == 1 && !SrcTy->isVectorTy()) { - Value *Elem = Builder->CreateBitCast(Src, DestVTy->getElementType()); + Value *Elem = Builder.CreateBitCast(Src, DestVTy->getElementType()); return InsertElementInst::Create(UndefValue::get(DestTy), Elem, Constant::getNullValue(Type::getInt32Ty(CI.getContext()))); // FIXME: Canonicalize bitcast(insertelement) -> insertelement(bitcast) @@ -2161,7 +2157,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) { // scalar-scalar cast. if (!DestTy->isVectorTy()) { Value *Elem = - Builder->CreateExtractElement(Src, + Builder.CreateExtractElement(Src, Constant::getNullValue(Type::getInt32Ty(CI.getContext()))); return CastInst::Create(Instruction::BitCast, Elem, DestTy); } @@ -2190,8 +2186,8 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) { Tmp->getOperand(0)->getType() == DestTy) || ((Tmp = dyn_cast(SVI->getOperand(1))) && Tmp->getOperand(0)->getType() == DestTy)) { - Value *LHS = Builder->CreateBitCast(SVI->getOperand(0), DestTy); - Value *RHS = Builder->CreateBitCast(SVI->getOperand(1), DestTy); + Value *LHS = Builder.CreateBitCast(SVI->getOperand(0), DestTy); + Value *RHS = Builder.CreateBitCast(SVI->getOperand(1), DestTy); // Return a new shuffle vector. Use the same element ID's, as we // know the vector types match #elts. return new ShuffleVectorInst(LHS, RHS, SVI->getOperand(2)); @@ -2204,13 +2200,13 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) { if (Instruction *I = optimizeBitCastFromPhi(CI, PN)) return I; - if (Instruction *I = canonicalizeBitCastExtElt(CI, *this, DL)) + if (Instruction *I = canonicalizeBitCastExtElt(CI, *this)) return I; - if (Instruction *I = foldBitCastBitwiseLogic(CI, *Builder)) + if (Instruction *I = foldBitCastBitwiseLogic(CI, Builder)) return I; - if (Instruction *I = foldBitCastSelect(CI, *Builder)) + if (Instruction *I = foldBitCastSelect(CI, Builder)) return I; if (SrcTy->isPointerTy()) @@ -2234,7 +2230,7 @@ Instruction *InstCombiner::visitAddrSpaceCast(AddrSpaceCastInst &CI) { MidTy = VectorType::get(MidTy, VT->getNumElements()); } - Value *NewBitCast = Builder->CreateBitCast(Src, MidTy); + Value *NewBitCast = Builder.CreateBitCast(Src, MidTy); return new AddrSpaceCastInst(NewBitCast, CI.getType()); } diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 58b8b2f52629..60d1cde971dd 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -392,7 +392,7 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); if (Idx->getType()->getPrimitiveSizeInBits() > PtrSize) - Idx = Builder->CreateTrunc(Idx, IntPtrTy); + Idx = Builder.CreateTrunc(Idx, IntPtrTy); } // If the comparison is only true for one or two elements, emit direct @@ -400,7 +400,7 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, if (SecondTrueElement != Overdefined) { // None true -> false. if (FirstTrueElement == Undefined) - return replaceInstUsesWith(ICI, Builder->getFalse()); + return replaceInstUsesWith(ICI, Builder.getFalse()); Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); @@ -409,9 +409,9 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); // True for two elements -> 'i == 47 | i == 72'. - Value *C1 = Builder->CreateICmpEQ(Idx, FirstTrueIdx); + Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx); Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); - Value *C2 = Builder->CreateICmpEQ(Idx, SecondTrueIdx); + Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx); return BinaryOperator::CreateOr(C1, C2); } @@ -420,7 +420,7 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, if (SecondFalseElement != Overdefined) { // None false -> true. if (FirstFalseElement == Undefined) - return replaceInstUsesWith(ICI, Builder->getTrue()); + return replaceInstUsesWith(ICI, Builder.getTrue()); Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); @@ -429,9 +429,9 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); // False for two elements -> 'i != 47 & i != 72'. - Value *C1 = Builder->CreateICmpNE(Idx, FirstFalseIdx); + Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx); Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); - Value *C2 = Builder->CreateICmpNE(Idx, SecondFalseIdx); + Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx); return BinaryOperator::CreateAnd(C1, C2); } @@ -443,7 +443,7 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, // Generate (i-FirstTrue) getType(), -FirstTrueElement); - Idx = Builder->CreateAdd(Idx, Offs); + Idx = Builder.CreateAdd(Idx, Offs); } Value *End = ConstantInt::get(Idx->getType(), @@ -457,7 +457,7 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). if (FirstFalseElement) { Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); - Idx = Builder->CreateAdd(Idx, Offs); + Idx = Builder.CreateAdd(Idx, Offs); } Value *End = ConstantInt::get(Idx->getType(), @@ -481,9 +481,9 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); if (Ty) { - Value *V = Builder->CreateIntCast(Idx, Ty, false); - V = Builder->CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); - V = Builder->CreateAnd(ConstantInt::get(Ty, 1), V); + Value *V = Builder.CreateIntCast(Idx, Ty, false); + V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); + V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V); return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); } } @@ -566,7 +566,7 @@ static Value *evaluateGEPOffsetExpression(User *GEP, InstCombiner &IC, // we don't need to bother extending: the extension won't affect where the // computation crosses zero. if (VariableIdx->getType()->getPrimitiveSizeInBits() > IntPtrWidth) { - VariableIdx = IC.Builder->CreateTrunc(VariableIdx, IntPtrTy); + VariableIdx = IC.Builder.CreateTrunc(VariableIdx, IntPtrTy); } return VariableIdx; } @@ -588,10 +588,10 @@ static Value *evaluateGEPOffsetExpression(User *GEP, InstCombiner &IC, // Okay, we can do this evaluation. Start by converting the index to intptr. if (VariableIdx->getType() != IntPtrTy) - VariableIdx = IC.Builder->CreateIntCast(VariableIdx, IntPtrTy, + VariableIdx = IC.Builder.CreateIntCast(VariableIdx, IntPtrTy, true /*Signed*/); Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); - return IC.Builder->CreateAdd(VariableIdx, OffsetVal, "offset"); + return IC.Builder.CreateAdd(VariableIdx, OffsetVal, "offset"); } /// Returns true if we can rewrite Start as a GEP with pointer Base @@ -981,13 +981,13 @@ Instruction *InstCombiner::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, if (LHSIndexTy != RHSIndexTy) { if (LHSIndexTy->getPrimitiveSizeInBits() < RHSIndexTy->getPrimitiveSizeInBits()) { - ROffset = Builder->CreateTrunc(ROffset, LHSIndexTy); + ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy); } else - LOffset = Builder->CreateTrunc(LOffset, RHSIndexTy); + LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy); } - Value *Cmp = Builder->CreateICmp(ICmpInst::getSignedPredicate(Cond), - LOffset, ROffset); + Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond), + LOffset, ROffset); return replaceInstUsesWith(I, Cmp); } @@ -1026,7 +1026,7 @@ Instruction *InstCombiner::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, if (NumDifferences == 0) // SAME GEP? return replaceInstUsesWith(I, // No comparison is needed here. - Builder->getInt1(ICmpInst::isTrueWhenEqual(Cond))); + Builder.getInt1(ICmpInst::isTrueWhenEqual(Cond))); else if (NumDifferences == 1 && GEPsInBounds) { Value *LHSV = GEPLHS->getOperand(DiffOperand); @@ -1174,7 +1174,7 @@ Instruction *InstCombiner::foldICmpAddOpConst(Instruction &ICI, // (X+ -1) >s X --> X X == -128 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE); - Constant *C = Builder->getInt(CI->getValue()-1); + Constant *C = Builder.getInt(CI->getValue() - 1); return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantExpr::getSub(SMax, C)); } @@ -1347,17 +1347,17 @@ static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, Value *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::sadd_with_overflow, NewType); - InstCombiner::BuilderTy *Builder = IC.Builder; + InstCombiner::BuilderTy &Builder = IC.Builder; // Put the new code above the original add, in case there are any uses of the // add between the add and the compare. - Builder->SetInsertPoint(OrigAdd); + Builder.SetInsertPoint(OrigAdd); - Value *TruncA = Builder->CreateTrunc(A, NewType, A->getName() + ".trunc"); - Value *TruncB = Builder->CreateTrunc(B, NewType, B->getName() + ".trunc"); - CallInst *Call = Builder->CreateCall(F, {TruncA, TruncB}, "sadd"); - Value *Add = Builder->CreateExtractValue(Call, 0, "sadd.result"); - Value *ZExt = Builder->CreateZExt(Add, OrigAdd->getType()); + Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc"); + Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc"); + CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd"); + Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result"); + Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType()); // The inner add was the result of the narrow add, zero extended to the // wider type. Replace it with the result computed by the intrinsic. @@ -1434,9 +1434,9 @@ Instruction *InstCombiner::foldICmpWithConstant(ICmpInst &Cmp) { ConstantRange Intersection = DominatingCR.intersectWith(CR); ConstantRange Difference = DominatingCR.difference(CR); if (Intersection.isEmptySet()) - return replaceInstUsesWith(Cmp, Builder->getFalse()); + return replaceInstUsesWith(Cmp, Builder.getFalse()); if (Difference.isEmptySet()) - return replaceInstUsesWith(Cmp, Builder->getTrue()); + return replaceInstUsesWith(Cmp, Builder.getTrue()); // If this is a normal comparison, it demands all bits. If it is a sign // bit comparison, it only demands the sign bit. @@ -1452,9 +1452,9 @@ Instruction *InstCombiner::foldICmpWithConstant(ICmpInst &Cmp) { return nullptr; if (auto *AI = Intersection.getSingleElement()) - return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder->getInt(*AI)); + return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*AI)); if (auto *AD = Difference.getSingleElement()) - return new ICmpInst(ICmpInst::ICMP_NE, X, Builder->getInt(*AD)); + return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*AD)); } return nullptr; @@ -1628,11 +1628,11 @@ Instruction *InstCombiner::foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, !Shift->isArithmeticShift() && !isa(Shift->getOperand(0))) { // Compute C2 << Y. Value *NewShift = - IsShl ? Builder->CreateLShr(And->getOperand(1), Shift->getOperand(1)) - : Builder->CreateShl(And->getOperand(1), Shift->getOperand(1)); + IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) + : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); // Compute X & (C2 << Y). - Value *NewAnd = Builder->CreateAnd(Shift->getOperand(0), NewShift); + Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); Cmp.setOperand(0, NewAnd); return &Cmp; } @@ -1670,7 +1670,7 @@ Instruction *InstCombiner::foldICmpAndConstConst(ICmpInst &Cmp, unsigned WideScalarBits = WideType->getScalarSizeInBits(); Constant *ZextC1 = ConstantInt::get(WideType, C1->zext(WideScalarBits)); Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); - Value *NewAnd = Builder->CreateAnd(W, ZextC2, And->getName()); + Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName()); return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); } } @@ -1704,12 +1704,12 @@ Instruction *InstCombiner::foldICmpAndConstConst(ICmpInst &Cmp, NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); } else { if (UsesRemoved >= 3) - NewOr = Builder->CreateOr(Builder->CreateShl(One, B, LShr->getName(), - /*HasNUW=*/true), - One, Or->getName()); + NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(), + /*HasNUW=*/true), + One, Or->getName()); } if (NewOr) { - Value *NewAnd = Builder->CreateAnd(A, NewOr, And->getName()); + Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName()); Cmp.setOperand(0, NewAnd); return &Cmp; } @@ -1772,7 +1772,7 @@ Instruction *InstCombiner::foldICmpAndConstant(ICmpInst &Cmp, Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); if (And->getType()->isVectorTy()) NTy = VectorType::get(NTy, And->getType()->getVectorNumElements()); - Value *Trunc = Builder->CreateTrunc(X, NTy); + Value *Trunc = Builder.CreateTrunc(X, NTy); auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE : CmpInst::ICMP_SLT; return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); @@ -1811,9 +1811,9 @@ Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 // -> and (icmp eq P, null), (icmp eq Q, null). Value *CmpP = - Builder->CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); + Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); Value *CmpQ = - Builder->CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); + Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); auto LogicOpc = Pred == ICmpInst::Predicate::ICMP_EQ ? Instruction::And : Instruction::Or; return BinaryOperator::Create(LogicOpc, CmpP, CmpQ); @@ -1993,7 +1993,7 @@ Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp, Constant *Mask = ConstantInt::get( ShType, APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); - Value *And = Builder->CreateAnd(X, Mask, Shl->getName() + ".mask"); + Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); Constant *LShrC = ConstantInt::get(ShType, C->lshr(*ShiftAmt)); return new ICmpInst(Pred, And, LShrC); } @@ -2005,7 +2005,7 @@ Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp, Constant *Mask = ConstantInt::get( ShType, APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); - Value *And = Builder->CreateAnd(X, Mask, Shl->getName() + ".mask"); + Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, And, Constant::getNullValue(ShType)); } @@ -2024,7 +2024,7 @@ Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp, TruncTy = VectorType::get(TruncTy, ShType->getVectorNumElements()); Constant *NewC = ConstantInt::get(TruncTy, C->ashr(*ShiftAmt).trunc(TypeBits - Amt)); - return new ICmpInst(Pred, Builder->CreateTrunc(X, TruncTy), NewC); + return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC); } return nullptr; @@ -2076,8 +2076,8 @@ Instruction *InstCombiner::foldICmpShrConstant(ICmpInst &Cmp, Constant *DivCst = ConstantInt::get( Shr->getType(), APInt::getOneBitSet(TypeBits, ShAmtVal)); - Value *Tmp = IsAShr ? Builder->CreateSDiv(X, DivCst, "", Shr->isExact()) - : Builder->CreateUDiv(X, DivCst, "", Shr->isExact()); + Value *Tmp = IsAShr ? Builder.CreateSDiv(X, DivCst, "", Shr->isExact()) + : Builder.CreateUDiv(X, DivCst, "", Shr->isExact()); Cmp.setOperand(0, Tmp); @@ -2115,7 +2115,7 @@ Instruction *InstCombiner::foldICmpShrConstant(ICmpInst &Cmp, // Otherwise strength reduce the shift into an 'and'. APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); Constant *Mask = ConstantInt::get(Shr->getType(), Val); - Value *And = Builder->CreateAnd(X, Mask, Shr->getName() + ".mask"); + Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask"); return new ICmpInst(Pred, And, ShiftedCmpRHS); } @@ -2279,7 +2279,7 @@ Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp, default: llvm_unreachable("Unhandled icmp opcode!"); case ICmpInst::ICMP_EQ: if (LoOverflow && HiOverflow) - return replaceInstUsesWith(Cmp, Builder->getFalse()); + return replaceInstUsesWith(Cmp, Builder.getFalse()); if (HiOverflow) return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, X, LoBound); @@ -2291,7 +2291,7 @@ Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp, HiBound->getUniqueInteger(), DivIsSigned, true)); case ICmpInst::ICMP_NE: if (LoOverflow && HiOverflow) - return replaceInstUsesWith(Cmp, Builder->getTrue()); + return replaceInstUsesWith(Cmp, Builder.getTrue()); if (HiOverflow) return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT, X, LoBound); @@ -2305,16 +2305,16 @@ Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp, case ICmpInst::ICMP_ULT: case ICmpInst::ICMP_SLT: if (LoOverflow == +1) // Low bound is greater than input range. - return replaceInstUsesWith(Cmp, Builder->getTrue()); + return replaceInstUsesWith(Cmp, Builder.getTrue()); if (LoOverflow == -1) // Low bound is less than input range. - return replaceInstUsesWith(Cmp, Builder->getFalse()); + return replaceInstUsesWith(Cmp, Builder.getFalse()); return new ICmpInst(Pred, X, LoBound); case ICmpInst::ICMP_UGT: case ICmpInst::ICMP_SGT: if (HiOverflow == +1) // High bound greater than input range. - return replaceInstUsesWith(Cmp, Builder->getFalse()); + return replaceInstUsesWith(Cmp, Builder.getFalse()); if (HiOverflow == -1) // High bound less than input range. - return replaceInstUsesWith(Cmp, Builder->getTrue()); + return replaceInstUsesWith(Cmp, Builder.getTrue()); if (Pred == ICmpInst::ICMP_UGT) return new ICmpInst(ICmpInst::ICMP_UGE, X, HiBound); return new ICmpInst(ICmpInst::ICMP_SGE, X, HiBound); @@ -2361,12 +2361,12 @@ Instruction *InstCombiner::foldICmpSubConstant(ICmpInst &Cmp, // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 if (Pred == ICmpInst::ICMP_ULT && C->isPowerOf2() && (*C2 & (*C - 1)) == (*C - 1)) - return new ICmpInst(ICmpInst::ICMP_EQ, Builder->CreateOr(Y, *C - 1), X); + return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, *C - 1), X); // C2 - Y >u C -> (Y | C) != C2 // iff C2 & C == C and C + 1 is a power of 2 if (Pred == ICmpInst::ICMP_UGT && (*C + 1).isPowerOf2() && (*C2 & *C) == *C) - return new ICmpInst(ICmpInst::ICMP_NE, Builder->CreateOr(Y, *C), X); + return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, *C), X); return nullptr; } @@ -2422,14 +2422,14 @@ Instruction *InstCombiner::foldICmpAddConstant(ICmpInst &Cmp, // iff C & (C2-1) == 0 // C2 is a power of 2 if (Pred == ICmpInst::ICMP_ULT && C->isPowerOf2() && (*C2 & (*C - 1)) == 0) - return new ICmpInst(ICmpInst::ICMP_EQ, Builder->CreateAnd(X, -(*C)), + return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -(*C)), ConstantExpr::getNeg(cast(Y))); // X+C >u C2 -> (X & ~C2) != C // iff C & C2 == 0 // C2+1 is a power of 2 if (Pred == ICmpInst::ICMP_UGT && (*C + 1).isPowerOf2() && (*C2 & *C) == 0) - return new ICmpInst(ICmpInst::ICMP_NE, Builder->CreateAnd(X, ~(*C)), + return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~(*C)), ConstantExpr::getNeg(cast(Y))); return nullptr; @@ -2493,13 +2493,13 @@ Instruction *InstCombiner::foldICmpSelectConstant(ICmpInst &Cmp, // When none of the three constants satisfy the predicate for the RHS (C), // the entire original Cmp can be simplified to a false. - Value *Cond = Builder->getFalse(); + Value *Cond = Builder.getFalse(); if (TrueWhenLessThan) - Cond = Builder->CreateOr(Cond, Builder->CreateICmp(ICmpInst::ICMP_SLT, OrigLHS, OrigRHS)); + Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, OrigLHS, OrigRHS)); if (TrueWhenEqual) - Cond = Builder->CreateOr(Cond, Builder->CreateICmp(ICmpInst::ICMP_EQ, OrigLHS, OrigRHS)); + Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, OrigLHS, OrigRHS)); if (TrueWhenGreaterThan) - Cond = Builder->CreateOr(Cond, Builder->CreateICmp(ICmpInst::ICMP_SGT, OrigLHS, OrigRHS)); + Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, OrigLHS, OrigRHS)); return replaceInstUsesWith(Cmp, Cond); } @@ -2615,7 +2615,7 @@ Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, if (C->isNullValue() && BO->hasOneUse()) { const APInt *BOC; if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { - Value *NewRem = Builder->CreateURem(BOp0, BOp1, BO->getName()); + Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); return new ICmpInst(Pred, NewRem, Constant::getNullValue(BO->getType())); } @@ -2637,7 +2637,7 @@ Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, if (Value *NegVal = dyn_castNegVal(BOp0)) return new ICmpInst(Pred, NegVal, BOp1); if (BO->hasOneUse()) { - Value *Neg = Builder->CreateNeg(BOp1); + Value *Neg = Builder.CreateNeg(BOp1); Neg->takeName(BO); return new ICmpInst(Pred, BOp0, Neg); } @@ -2676,7 +2676,7 @@ Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, // Replace (X | C) == -1 with (X & ~C) == ~C. // This removes the -1 constant. Constant *NotBOC = ConstantExpr::getNot(cast(BOp1)); - Value *And = Builder->CreateAnd(BOp0, NotBOC); + Value *And = Builder.CreateAnd(BOp0, NotBOC); return new ICmpInst(Pred, And, NotBOC); } break; @@ -2740,23 +2740,26 @@ Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, if (!II || !Cmp.isEquality()) return nullptr; - // Handle icmp {eq|ne} , intcst. + // Handle icmp {eq|ne} , Constant. + Type *Ty = II->getType(); switch (II->getIntrinsicID()) { case Intrinsic::bswap: Worklist.Add(II); Cmp.setOperand(0, II->getArgOperand(0)); - Cmp.setOperand(1, Builder->getInt(C->byteSwap())); + Cmp.setOperand(1, ConstantInt::get(Ty, C->byteSwap())); return &Cmp; + case Intrinsic::ctlz: case Intrinsic::cttz: // ctz(A) == bitwidth(A) -> A == 0 and likewise for != if (*C == C->getBitWidth()) { Worklist.Add(II); Cmp.setOperand(0, II->getArgOperand(0)); - Cmp.setOperand(1, ConstantInt::getNullValue(II->getType())); + Cmp.setOperand(1, ConstantInt::getNullValue(Ty)); return &Cmp; } break; + case Intrinsic::ctpop: { // popcount(A) == 0 -> A == 0 and likewise for != // popcount(A) == bitwidth(A) -> A == -1 and likewise for != @@ -2764,8 +2767,8 @@ Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, if (IsZero || *C == C->getBitWidth()) { Worklist.Add(II); Cmp.setOperand(0, II->getArgOperand(0)); - auto *NewOp = IsZero ? Constant::getNullValue(II->getType()) - : Constant::getAllOnesValue(II->getType()); + auto *NewOp = + IsZero ? Constant::getNullValue(Ty) : Constant::getAllOnesValue(Ty); Cmp.setOperand(1, NewOp); return &Cmp; } @@ -2774,6 +2777,7 @@ Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, default: break; } + return nullptr; } @@ -2841,11 +2845,11 @@ Instruction *InstCombiner::foldICmpInstWithConstantNotInt(ICmpInst &I) { } if (Transform) { if (!Op1) - Op1 = Builder->CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, - I.getName()); + Op1 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, + I.getName()); if (!Op2) - Op2 = Builder->CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, - I.getName()); + Op2 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, + I.getName()); return SelectInst::Create(LHSI->getOperand(0), Op1, Op2); } break; @@ -3029,12 +3033,12 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) { APInt AP1Abs = C1->getValue().abs(); APInt AP2Abs = C2->getValue().abs(); if (AP1Abs.uge(AP2Abs)) { - ConstantInt *C3 = Builder->getInt(AP1 - AP2); - Value *NewAdd = Builder->CreateNSWAdd(A, C3); + ConstantInt *C3 = Builder.getInt(AP1 - AP2); + Value *NewAdd = Builder.CreateNSWAdd(A, C3); return new ICmpInst(Pred, NewAdd, C); } else { - ConstantInt *C3 = Builder->getInt(AP2 - AP1); - Value *NewAdd = Builder->CreateNSWAdd(C, C3); + ConstantInt *C3 = Builder.getInt(AP2 - AP1); + Value *NewAdd = Builder.CreateNSWAdd(C, C3); return new ICmpInst(Pred, A, NewAdd); } } @@ -3157,8 +3161,8 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) { Constant *Mask = ConstantInt::get( BO0->getType(), APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); - Value *And1 = Builder->CreateAnd(BO0->getOperand(0), Mask); - Value *And2 = Builder->CreateAnd(BO1->getOperand(0), Mask); + Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); + Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); return new ICmpInst(Pred, And1, And2); } // If there are no trailing zeros in the multiplier, just eliminate @@ -3315,8 +3319,8 @@ Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { ConstantInt *C1, *C2; if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && Op1->hasOneUse()) { - Constant *NC = Builder->getInt(C1->getValue() ^ C2->getValue()); - Value *Xor = Builder->CreateXor(C, NC); + Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue()); + Value *Xor = Builder.CreateXor(C, NC); return new ICmpInst(Pred, A, Xor); } @@ -3362,8 +3366,8 @@ Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { } if (X) { // Build (X^Y) & Z - Op1 = Builder->CreateXor(X, Y); - Op1 = Builder->CreateAnd(Op1, Z); + Op1 = Builder.CreateXor(X, Y); + Op1 = Builder.CreateAnd(Op1, Z); I.setOperand(0, Op1); I.setOperand(1, Constant::getNullValue(Op1->getType())); return &I; @@ -3380,7 +3384,7 @@ Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { APInt Pow2 = Cst1->getValue() + 1; if (Pow2.isPowerOf2() && isa(A->getType()) && Pow2.logBase2() == cast(A->getType())->getBitWidth()) - return new ICmpInst(Pred, A, Builder->CreateTrunc(B, A->getType())); + return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType())); } // (A >> C) == (B >> C) --> (A^B) u< (1 << C) @@ -3394,9 +3398,9 @@ Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { if (ShAmt < TypeBits && ShAmt != 0) { ICmpInst::Predicate NewPred = Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; - Value *Xor = Builder->CreateXor(A, B, I.getName() + ".unshifted"); + Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); - return new ICmpInst(NewPred, Xor, Builder->getInt(CmpVal)); + return new ICmpInst(NewPred, Xor, Builder.getInt(CmpVal)); } } @@ -3406,9 +3410,9 @@ Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { unsigned TypeBits = Cst1->getBitWidth(); unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); if (ShAmt < TypeBits && ShAmt != 0) { - Value *Xor = Builder->CreateXor(A, B, I.getName() + ".unshifted"); + Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); - Value *And = Builder->CreateAnd(Xor, Builder->getInt(AndVal), + Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal), I.getName() + ".mask"); return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType())); } @@ -3433,11 +3437,20 @@ Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { APInt CmpV = Cst1->getValue().zext(ASize); CmpV <<= ShAmt; - Value *Mask = Builder->CreateAnd(A, Builder->getInt(MaskV)); - return new ICmpInst(Pred, Mask, Builder->getInt(CmpV)); + Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV)); + return new ICmpInst(Pred, Mask, Builder.getInt(CmpV)); } } + // If both operands are byte-swapped or bit-reversed, just compare the + // original values. + // TODO: Move this to a function similar to foldICmpIntrinsicWithConstant() + // and handle more intrinsics. + if ((match(Op0, m_BSwap(m_Value(A))) && match(Op1, m_BSwap(m_Value(B)))) || + (match(Op0, m_BitReverse(m_Value(A))) && + match(Op1, m_BitReverse(m_Value(B))))) + return new ICmpInst(Pred, A, B); + return nullptr; } @@ -3462,7 +3475,7 @@ Instruction *InstCombiner::foldICmpWithCastAndCast(ICmpInst &ICmp) { RHSOp = RHSC->getOperand(0); // If the pointer types don't match, insert a bitcast. if (LHSCIOp->getType() != RHSOp->getType()) - RHSOp = Builder->CreateBitCast(RHSOp, LHSCIOp->getType()); + RHSOp = Builder.CreateBitCast(RHSOp, LHSCIOp->getType()); } } else if (auto *RHSC = dyn_cast(ICmp.getOperand(1))) { RHSOp = ConstantExpr::getIntToPtr(RHSC, SrcTy); @@ -3546,7 +3559,7 @@ Instruction *InstCombiner::foldICmpWithCastAndCast(ICmpInst &ICmp) { // We're performing an unsigned comp with a sign extended value. // This is true if the input is >= 0. [aka >s -1] Constant *NegOne = Constant::getAllOnesValue(SrcTy); - Value *Result = Builder->CreateICmpSGT(LHSCIOp, NegOne, ICmp.getName()); + Value *Result = Builder.CreateICmpSGT(LHSCIOp, NegOne, ICmp.getName()); // Finally, return the value computed. if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) @@ -3574,7 +3587,7 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS, // may be pointing to the compare. We want to insert the new instructions // before the add in case there are uses of the add between the add and the // compare. - Builder->SetInsertPoint(&OrigI); + Builder.SetInsertPoint(&OrigI); switch (OCF) { case OCF_INVALID: @@ -3583,11 +3596,11 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS, case OCF_UNSIGNED_ADD: { OverflowResult OR = computeOverflowForUnsignedAdd(LHS, RHS, &OrigI); if (OR == OverflowResult::NeverOverflows) - return SetResult(Builder->CreateNUWAdd(LHS, RHS), Builder->getFalse(), + return SetResult(Builder.CreateNUWAdd(LHS, RHS), Builder.getFalse(), true); if (OR == OverflowResult::AlwaysOverflows) - return SetResult(Builder->CreateAdd(LHS, RHS), Builder->getTrue(), true); + return SetResult(Builder.CreateAdd(LHS, RHS), Builder.getTrue(), true); // Fall through uadd into sadd LLVM_FALLTHROUGH; @@ -3595,13 +3608,13 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS, case OCF_SIGNED_ADD: { // X + 0 -> {X, false} if (match(RHS, m_Zero())) - return SetResult(LHS, Builder->getFalse(), false); + return SetResult(LHS, Builder.getFalse(), false); // We can strength reduce this signed add into a regular add if we can prove // that it will never overflow. if (OCF == OCF_SIGNED_ADD) if (willNotOverflowSignedAdd(LHS, RHS, OrigI)) - return SetResult(Builder->CreateNSWAdd(LHS, RHS), Builder->getFalse(), + return SetResult(Builder.CreateNSWAdd(LHS, RHS), Builder.getFalse(), true); break; } @@ -3610,15 +3623,15 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS, case OCF_SIGNED_SUB: { // X - 0 -> {X, false} if (match(RHS, m_Zero())) - return SetResult(LHS, Builder->getFalse(), false); + return SetResult(LHS, Builder.getFalse(), false); if (OCF == OCF_SIGNED_SUB) { if (willNotOverflowSignedSub(LHS, RHS, OrigI)) - return SetResult(Builder->CreateNSWSub(LHS, RHS), Builder->getFalse(), + return SetResult(Builder.CreateNSWSub(LHS, RHS), Builder.getFalse(), true); } else { if (willNotOverflowUnsignedSub(LHS, RHS, OrigI)) - return SetResult(Builder->CreateNUWSub(LHS, RHS), Builder->getFalse(), + return SetResult(Builder.CreateNUWSub(LHS, RHS), Builder.getFalse(), true); } break; @@ -3627,28 +3640,28 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS, case OCF_UNSIGNED_MUL: { OverflowResult OR = computeOverflowForUnsignedMul(LHS, RHS, &OrigI); if (OR == OverflowResult::NeverOverflows) - return SetResult(Builder->CreateNUWMul(LHS, RHS), Builder->getFalse(), + return SetResult(Builder.CreateNUWMul(LHS, RHS), Builder.getFalse(), true); if (OR == OverflowResult::AlwaysOverflows) - return SetResult(Builder->CreateMul(LHS, RHS), Builder->getTrue(), true); + return SetResult(Builder.CreateMul(LHS, RHS), Builder.getTrue(), true); LLVM_FALLTHROUGH; } case OCF_SIGNED_MUL: // X * undef -> undef if (isa(RHS)) - return SetResult(RHS, UndefValue::get(Builder->getInt1Ty()), false); + return SetResult(RHS, UndefValue::get(Builder.getInt1Ty()), false); // X * 0 -> {0, false} if (match(RHS, m_Zero())) - return SetResult(RHS, Builder->getFalse(), false); + return SetResult(RHS, Builder.getFalse(), false); // X * 1 -> {X, false} if (match(RHS, m_One())) - return SetResult(LHS, Builder->getFalse(), false); + return SetResult(LHS, Builder.getFalse(), false); if (OCF == OCF_SIGNED_MUL) if (willNotOverflowSignedMul(LHS, RHS, OrigI)) - return SetResult(Builder->CreateNSWMul(LHS, RHS), Builder->getFalse(), + return SetResult(Builder.CreateNSWMul(LHS, RHS), Builder.getFalse(), true); break; } @@ -3813,25 +3826,25 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, return nullptr; } - InstCombiner::BuilderTy *Builder = IC.Builder; - Builder->SetInsertPoint(MulInstr); + InstCombiner::BuilderTy &Builder = IC.Builder; + Builder.SetInsertPoint(MulInstr); // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) Value *MulA = A, *MulB = B; if (WidthA < MulWidth) - MulA = Builder->CreateZExt(A, MulType); + MulA = Builder.CreateZExt(A, MulType); if (WidthB < MulWidth) - MulB = Builder->CreateZExt(B, MulType); + MulB = Builder.CreateZExt(B, MulType); Value *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::umul_with_overflow, MulType); - CallInst *Call = Builder->CreateCall(F, {MulA, MulB}, "umul"); + CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul"); IC.Worklist.Add(MulInstr); // If there are uses of mul result other than the comparison, we know that // they are truncation or binary AND. Change them to use result of // mul.with.overflow and adjust properly mask/size. if (MulVal->hasNUsesOrMore(2)) { - Value *Mul = Builder->CreateExtractValue(Call, 0, "umul.value"); + Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value"); for (User *U : MulVal->users()) { if (U == &I || U == OtherVal) continue; @@ -3843,17 +3856,18 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, } else if (BinaryOperator *BO = dyn_cast(U)) { assert(BO->getOpcode() == Instruction::And); // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) - ConstantInt *CI = cast(BO->getOperand(1)); - APInt ShortMask = CI->getValue().trunc(MulWidth); - Value *ShortAnd = Builder->CreateAnd(Mul, ShortMask); - Instruction *Zext = - cast(Builder->CreateZExt(ShortAnd, BO->getType())); - IC.Worklist.Add(Zext); + Value *ShortMask = + Builder.CreateTrunc(BO->getOperand(1), Builder.getIntNTy(MulWidth)); + Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask); + Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType()); + if (auto *ZextI = dyn_cast(Zext)) + IC.Worklist.Add(ZextI); IC.replaceInstUsesWith(*BO, Zext); } else { llvm_unreachable("Unexpected Binary operation"); } - IC.Worklist.Add(cast(U)); + if (auto *UI = dyn_cast(U)) + IC.Worklist.Add(UI); } } if (isa(OtherVal)) @@ -3884,7 +3898,7 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, llvm_unreachable("Unexpected predicate"); } if (Inverse) { - Value *Res = Builder->CreateExtractValue(Call, 1); + Value *Res = Builder.CreateExtractValue(Call, 1); return BinaryOperator::CreateNot(Res); } @@ -4239,7 +4253,7 @@ Instruction *InstCombiner::foldICmpUsingKnownBits(ICmpInst &I) { if (ConstantInt *CI = dyn_cast(Op1)) { if (Op1Max == Op0Min + 1) // A A == C-1 if min(A)+1 == C return new ICmpInst(ICmpInst::ICMP_EQ, Op0, - Builder->getInt(CI->getValue() - 1)); + Builder.getInt(CI->getValue() - 1)); } break; case ICmpInst::ICMP_SGT: @@ -4253,7 +4267,7 @@ Instruction *InstCombiner::foldICmpUsingKnownBits(ICmpInst &I) { if (ConstantInt *CI = dyn_cast(Op1)) { if (Op1Min == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C return new ICmpInst(ICmpInst::ICMP_EQ, Op0, - Builder->getInt(CI->getValue() + 1)); + Builder.getInt(CI->getValue() + 1)); } break; case ICmpInst::ICMP_SGE: @@ -4358,7 +4372,7 @@ static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { static Instruction *canonicalizeICmpBool(ICmpInst &I, InstCombiner::BuilderTy &Builder) { Value *A = I.getOperand(0), *B = I.getOperand(1); - assert(A->getType()->getScalarType()->isIntegerTy(1) && "Bools only"); + assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only"); // A boolean compared to true/false can be simplified to Op0/true/false in // 14 out of the 20 (10 predicates * 2 constants) possible combinations. @@ -4465,8 +4479,8 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { } } - if (Op0->getType()->getScalarType()->isIntegerTy(1)) - if (Instruction *Res = canonicalizeICmpBool(I, *Builder)) + if (Op0->getType()->isIntOrIntVectorTy(1)) + if (Instruction *Res = canonicalizeICmpBool(I, Builder)) return Res; if (ICmpInst *NewICmp = canonicalizeCmpWithConstant(I)) @@ -4559,7 +4573,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { Op1 = ConstantExpr::getBitCast(Op1C, Op0->getType()); } else { // Otherwise, cast the RHS right before the icmp - Op1 = Builder->CreateBitCast(Op1, Op0->getType()); + Op1 = Builder.CreateBitCast(Op1, Op0->getType()); } } return new ICmpInst(I.getPredicate(), Op0, Op1); @@ -4592,8 +4606,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality()) - return new ICmpInst(I.getInversePredicate(), - Builder->CreateAnd(A, B), + return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B), Op1); // ~X < ~Y --> Y < X @@ -4693,10 +4706,10 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); if (RHS.compare(RHSRoundInt) != APFloat::cmpEqual) { if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getFalse()); assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE); - return replaceInstUsesWith(I, Builder->getTrue()); + return replaceInstUsesWith(I, Builder.getTrue()); } } @@ -4762,9 +4775,9 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Pred = ICmpInst::ICMP_NE; break; case FCmpInst::FCMP_ORD: - return replaceInstUsesWith(I, Builder->getTrue()); + return replaceInstUsesWith(I, Builder.getTrue()); case FCmpInst::FCMP_UNO: - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getFalse()); } // Now we know that the APFloat is a normal number, zero or inf. @@ -4782,8 +4795,8 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, if (SMax.compare(RHS) == APFloat::cmpLessThan) { // smax < 13123.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) - return replaceInstUsesWith(I, Builder->getTrue()); - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, Builder.getFalse()); } } else { // If the RHS value is > UnsignedMax, fold the comparison. This handles @@ -4794,8 +4807,8 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, if (UMax.compare(RHS) == APFloat::cmpLessThan) { // umax < 13123.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) - return replaceInstUsesWith(I, Builder->getTrue()); - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, Builder.getFalse()); } } @@ -4807,8 +4820,8 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, if (SMin.compare(RHS) == APFloat::cmpGreaterThan) { // smin > 12312.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) - return replaceInstUsesWith(I, Builder->getTrue()); - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, Builder.getFalse()); } } else { // See if the RHS value is < UnsignedMin. @@ -4818,8 +4831,8 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, if (SMin.compare(RHS) == APFloat::cmpGreaterThan) { // umin > 12312.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) - return replaceInstUsesWith(I, Builder->getTrue()); - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, Builder.getFalse()); } } @@ -4841,14 +4854,14 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, switch (Pred) { default: llvm_unreachable("Unexpected integer comparison!"); case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true - return replaceInstUsesWith(I, Builder->getTrue()); + return replaceInstUsesWith(I, Builder.getTrue()); case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getFalse()); case ICmpInst::ICMP_ULE: // (float)int <= 4.4 --> int <= 4 // (float)int <= -4.4 --> false if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getFalse()); break; case ICmpInst::ICMP_SLE: // (float)int <= 4.4 --> int <= 4 @@ -4860,7 +4873,7 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, // (float)int < -4.4 --> false // (float)int < 4.4 --> int <= 4 if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder->getFalse()); + return replaceInstUsesWith(I, Builder.getFalse()); Pred = ICmpInst::ICMP_ULE; break; case ICmpInst::ICMP_SLT: @@ -4873,7 +4886,7 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, // (float)int > 4.4 --> int > 4 // (float)int > -4.4 --> true if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder->getTrue()); + return replaceInstUsesWith(I, Builder.getTrue()); break; case ICmpInst::ICMP_SGT: // (float)int > 4.4 --> int > 4 @@ -4885,7 +4898,7 @@ Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, // (float)int >= -4.4 --> true // (float)int >= 4.4 --> int > 4 if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder->getTrue()); + return replaceInstUsesWith(I, Builder.getTrue()); Pred = ICmpInst::ICMP_UGT; break; case ICmpInst::ICMP_SGE: diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/contrib/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index 87f11467b95e..c38a4981bf1d 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -21,8 +21,6 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetFolder.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/IR/DIBuilder.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" @@ -212,7 +210,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner /// \brief An IRBuilder that automatically inserts new instructions into the /// worklist. typedef IRBuilder BuilderTy; - BuilderTy *Builder; + BuilderTy &Builder; private: // Mode in which we are running the combiner. @@ -235,7 +233,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner bool MadeIRChange; public: - InstCombiner(InstCombineWorklist &Worklist, BuilderTy *Builder, + InstCombiner(InstCombineWorklist &Worklist, BuilderTy &Builder, bool MinimizeSize, bool ExpensiveCombines, AliasAnalysis *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, DominatorTree &DT, const DataLayout &DL, LoopInfo *LI) @@ -598,9 +596,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner /// This tries to simplify binary operations by factorizing out common terms /// (e. g. "(A*B)+(A*C)" -> "A*(B+C)"). - Value *tryFactorization(InstCombiner::BuilderTy *, BinaryOperator &, - Instruction::BinaryOps, Value *, Value *, Value *, - Value *); + Value *tryFactorization(BinaryOperator &, Instruction::BinaryOps, Value *, + Value *, Value *, Value *); /// Match a select chain which produces one of three values based on whether /// the LHS is less than, equal to, or greater than RHS respectively. @@ -639,7 +636,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner APInt &UndefElts, unsigned Depth = 0); Value *SimplifyVectorOp(BinaryOperator &Inst); - Value *SimplifyBSwap(BinaryOperator &Inst); /// Given a binary operator, cast instruction, or select which has a PHI node diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index 26bee204e5a4..c59e1ce69ac2 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -189,7 +189,7 @@ static Instruction *simplifyAllocaArraySize(InstCombiner &IC, AllocaInst &AI) { return nullptr; // Canonicalize it. - Value *V = IC.Builder->getInt32(1); + Value *V = IC.Builder.getInt32(1); AI.setOperand(0, V); return &AI; } @@ -197,7 +197,7 @@ static Instruction *simplifyAllocaArraySize(InstCombiner &IC, AllocaInst &AI) { // Convert: alloca Ty, C - where C is a constant != 1 into: alloca [C x Ty], 1 if (const ConstantInt *C = dyn_cast(AI.getArraySize())) { Type *NewTy = ArrayType::get(AI.getAllocatedType(), C->getZExtValue()); - AllocaInst *New = IC.Builder->CreateAlloca(NewTy, nullptr, AI.getName()); + AllocaInst *New = IC.Builder.CreateAlloca(NewTy, nullptr, AI.getName()); New->setAlignment(AI.getAlignment()); // Scan to the end of the allocation instructions, to skip over a block of @@ -229,7 +229,7 @@ static Instruction *simplifyAllocaArraySize(InstCombiner &IC, AllocaInst &AI) { // any casting is exposed early. Type *IntPtrTy = IC.getDataLayout().getIntPtrType(AI.getType()); if (AI.getArraySize()->getType() != IntPtrTy) { - Value *V = IC.Builder->CreateIntCast(AI.getArraySize(), IntPtrTy, false); + Value *V = IC.Builder.CreateIntCast(AI.getArraySize(), IntPtrTy, false); AI.setOperand(0, V); return &AI; } @@ -458,10 +458,10 @@ static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewT SmallVector, 8> MD; LI.getAllMetadata(MD); - LoadInst *NewLoad = IC.Builder->CreateAlignedLoad( - IC.Builder->CreateBitCast(Ptr, NewTy->getPointerTo(AS)), + LoadInst *NewLoad = IC.Builder.CreateAlignedLoad( + IC.Builder.CreateBitCast(Ptr, NewTy->getPointerTo(AS)), LI.getAlignment(), LI.isVolatile(), LI.getName() + Suffix); - NewLoad->setAtomic(LI.getOrdering(), LI.getSynchScope()); + NewLoad->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); MDBuilder MDB(NewLoad->getContext()); for (const auto &MDPair : MD) { unsigned ID = MDPair.first; @@ -518,10 +518,10 @@ static StoreInst *combineStoreToNewValue(InstCombiner &IC, StoreInst &SI, Value SmallVector, 8> MD; SI.getAllMetadata(MD); - StoreInst *NewStore = IC.Builder->CreateAlignedStore( - V, IC.Builder->CreateBitCast(Ptr, V->getType()->getPointerTo(AS)), + StoreInst *NewStore = IC.Builder.CreateAlignedStore( + V, IC.Builder.CreateBitCast(Ptr, V->getType()->getPointerTo(AS)), SI.getAlignment(), SI.isVolatile()); - NewStore->setAtomic(SI.getOrdering(), SI.getSynchScope()); + NewStore->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); for (const auto &MDPair : MD) { unsigned ID = MDPair.first; MDNode *N = MDPair.second; @@ -613,7 +613,7 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) { // Replace all the stores with stores of the newly loaded value. for (auto UI = LI.user_begin(), UE = LI.user_end(); UI != UE;) { auto *SI = cast(*UI++); - IC.Builder->SetInsertPoint(SI); + IC.Builder.SetInsertPoint(SI); combineStoreToNewValue(IC, *SI, NewLoad); IC.eraseInstFromFunction(*SI); } @@ -664,7 +664,7 @@ static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) { AAMDNodes AAMD; LI.getAAMetadata(AAMD); NewLoad->setAAMetadata(AAMD); - return IC.replaceInstUsesWith(LI, IC.Builder->CreateInsertValue( + return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue( UndefValue::get(T), NewLoad, 0, Name)); } @@ -689,15 +689,15 @@ static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) { Zero, ConstantInt::get(IdxType, i), }; - auto *Ptr = IC.Builder->CreateInBoundsGEP(ST, Addr, makeArrayRef(Indices), - Name + ".elt"); + auto *Ptr = IC.Builder.CreateInBoundsGEP(ST, Addr, makeArrayRef(Indices), + Name + ".elt"); auto EltAlign = MinAlign(Align, SL->getElementOffset(i)); - auto *L = IC.Builder->CreateAlignedLoad(Ptr, EltAlign, Name + ".unpack"); + auto *L = IC.Builder.CreateAlignedLoad(Ptr, EltAlign, Name + ".unpack"); // Propagate AA metadata. It'll still be valid on the narrowed load. AAMDNodes AAMD; LI.getAAMetadata(AAMD); L->setAAMetadata(AAMD); - V = IC.Builder->CreateInsertValue(V, L, i); + V = IC.Builder.CreateInsertValue(V, L, i); } V->setName(Name); @@ -712,7 +712,7 @@ static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) { AAMDNodes AAMD; LI.getAAMetadata(AAMD); NewLoad->setAAMetadata(AAMD); - return IC.replaceInstUsesWith(LI, IC.Builder->CreateInsertValue( + return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue( UndefValue::get(T), NewLoad, 0, Name)); } @@ -740,14 +740,14 @@ static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) { Zero, ConstantInt::get(IdxType, i), }; - auto *Ptr = IC.Builder->CreateInBoundsGEP(AT, Addr, makeArrayRef(Indices), - Name + ".elt"); - auto *L = IC.Builder->CreateAlignedLoad(Ptr, MinAlign(Align, Offset), - Name + ".unpack"); + auto *Ptr = IC.Builder.CreateInBoundsGEP(AT, Addr, makeArrayRef(Indices), + Name + ".elt"); + auto *L = IC.Builder.CreateAlignedLoad(Ptr, MinAlign(Align, Offset), + Name + ".unpack"); AAMDNodes AAMD; LI.getAAMetadata(AAMD); L->setAAMetadata(AAMD); - V = IC.Builder->CreateInsertValue(V, L, i); + V = IC.Builder.CreateInsertValue(V, L, i); Offset += EltSize; } @@ -982,8 +982,8 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { combineMetadataForCSE(cast(AvailableVal), &LI); return replaceInstUsesWith( - LI, Builder->CreateBitOrPointerCast(AvailableVal, LI.getType(), - LI.getName() + ".cast")); + LI, Builder.CreateBitOrPointerCast(AvailableVal, LI.getType(), + LI.getName() + ".cast")); } // None of the following transforms are legal for volatile/ordered atomic @@ -1019,15 +1019,15 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { unsigned Align = LI.getAlignment(); if (isSafeToLoadUnconditionally(SI->getOperand(1), Align, DL, SI) && isSafeToLoadUnconditionally(SI->getOperand(2), Align, DL, SI)) { - LoadInst *V1 = Builder->CreateLoad(SI->getOperand(1), - SI->getOperand(1)->getName()+".val"); - LoadInst *V2 = Builder->CreateLoad(SI->getOperand(2), - SI->getOperand(2)->getName()+".val"); + LoadInst *V1 = Builder.CreateLoad(SI->getOperand(1), + SI->getOperand(1)->getName()+".val"); + LoadInst *V2 = Builder.CreateLoad(SI->getOperand(2), + SI->getOperand(2)->getName()+".val"); assert(LI.isUnordered() && "implied by above"); V1->setAlignment(Align); - V1->setAtomic(LI.getOrdering(), LI.getSynchScope()); + V1->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); V2->setAlignment(Align); - V2->setAtomic(LI.getOrdering(), LI.getSynchScope()); + V2->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); return SelectInst::Create(SI->getCondition(), V1, V2); } @@ -1172,7 +1172,7 @@ static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) { // If the struct only have one element, we unpack. unsigned Count = ST->getNumElements(); if (Count == 1) { - V = IC.Builder->CreateExtractValue(V, 0); + V = IC.Builder.CreateExtractValue(V, 0); combineStoreToNewValue(IC, SI, V); return true; } @@ -1201,12 +1201,11 @@ static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) { Zero, ConstantInt::get(IdxType, i), }; - auto *Ptr = IC.Builder->CreateInBoundsGEP(ST, Addr, makeArrayRef(Indices), - AddrName); - auto *Val = IC.Builder->CreateExtractValue(V, i, EltName); + auto *Ptr = IC.Builder.CreateInBoundsGEP(ST, Addr, makeArrayRef(Indices), + AddrName); + auto *Val = IC.Builder.CreateExtractValue(V, i, EltName); auto EltAlign = MinAlign(Align, SL->getElementOffset(i)); - llvm::Instruction *NS = - IC.Builder->CreateAlignedStore(Val, Ptr, EltAlign); + llvm::Instruction *NS = IC.Builder.CreateAlignedStore(Val, Ptr, EltAlign); AAMDNodes AAMD; SI.getAAMetadata(AAMD); NS->setAAMetadata(AAMD); @@ -1219,7 +1218,7 @@ static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) { // If the array only have one element, we unpack. auto NumElements = AT->getNumElements(); if (NumElements == 1) { - V = IC.Builder->CreateExtractValue(V, 0); + V = IC.Builder.CreateExtractValue(V, 0); combineStoreToNewValue(IC, SI, V); return true; } @@ -1252,11 +1251,11 @@ static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) { Zero, ConstantInt::get(IdxType, i), }; - auto *Ptr = IC.Builder->CreateInBoundsGEP(AT, Addr, makeArrayRef(Indices), - AddrName); - auto *Val = IC.Builder->CreateExtractValue(V, i, EltName); + auto *Ptr = IC.Builder.CreateInBoundsGEP(AT, Addr, makeArrayRef(Indices), + AddrName); + auto *Val = IC.Builder.CreateExtractValue(V, i, EltName); auto EltAlign = MinAlign(Align, Offset); - Instruction *NS = IC.Builder->CreateAlignedStore(Val, Ptr, EltAlign); + Instruction *NS = IC.Builder.CreateAlignedStore(Val, Ptr, EltAlign); AAMDNodes AAMD; SI.getAAMetadata(AAMD); NS->setAAMetadata(AAMD); @@ -1541,7 +1540,7 @@ bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) { SI.isVolatile(), SI.getAlignment(), SI.getOrdering(), - SI.getSynchScope()); + SI.getSyncScopeID()); InsertNewInstBefore(NewSI, *BBI); // The debug locations of the original instructions might differ; merge them. NewSI->setDebugLoc(DILocation::getMergedLocation(SI.getDebugLoc(), diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 579639a6194e..e3a50220f94e 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -39,8 +39,8 @@ static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC, Value *A = nullptr, *B = nullptr, *One = nullptr; if (match(V, m_LShr(m_OneUse(m_Shl(m_Value(One), m_Value(A))), m_Value(B))) && match(One, m_One())) { - A = IC.Builder->CreateSub(A, B); - return IC.Builder->CreateShl(One, A); + A = IC.Builder.CreateSub(A, B); + return IC.Builder.CreateShl(One, A); } // (PowerOfTwo >>u B) --> isExact since shifting out the result would make it @@ -250,9 +250,9 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { ConstantInt *C1; Value *Sub = nullptr; if (match(Op0, m_Sub(m_Value(Y), m_Value(X)))) - Sub = Builder->CreateSub(X, Y, "suba"); + Sub = Builder.CreateSub(X, Y, "suba"); else if (match(Op0, m_Add(m_Value(Y), m_ConstantInt(C1)))) - Sub = Builder->CreateSub(Builder->CreateNeg(C1), Y, "subc"); + Sub = Builder.CreateSub(Builder.CreateNeg(C1), Y, "subc"); if (Sub) return BinaryOperator::CreateMul(Sub, @@ -272,11 +272,11 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { Value *X; Constant *C1; if (match(Op0, m_OneUse(m_Add(m_Value(X), m_Constant(C1))))) { - Value *Mul = Builder->CreateMul(C1, Op1); + Value *Mul = Builder.CreateMul(C1, Op1); // Only go forward with the transform if C1*CI simplifies to a tidier // constant. if (!match(Mul, m_Mul(m_Value(), m_Value()))) - return BinaryOperator::CreateAdd(Builder->CreateMul(X, Op1), Mul); + return BinaryOperator::CreateAdd(Builder.CreateMul(X, Op1), Mul); } } } @@ -318,7 +318,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { auto RemOpc = Div->getOpcode() == Instruction::UDiv ? Instruction::URem : Instruction::SRem; - Value *Rem = Builder->CreateBinOp(RemOpc, X, DivOp1); + Value *Rem = Builder.CreateBinOp(RemOpc, X, DivOp1); if (DivOp1 == Y) return BinaryOperator::CreateSub(X, Rem); return BinaryOperator::CreateSub(Rem, X); @@ -326,7 +326,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { } /// i1 mul -> i1 and. - if (I.getType()->getScalarType()->isIntegerTy(1)) + if (I.getType()->isIntOrIntVectorTy(1)) return BinaryOperator::CreateAnd(Op0, Op1); // X*(1 << Y) --> X << Y @@ -368,7 +368,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { } if (BoolCast) { - Value *V = Builder->CreateSub(Constant::getNullValue(I.getType()), + Value *V = Builder.CreateSub(Constant::getNullValue(I.getType()), BoolCast); return BinaryOperator::CreateAnd(V, OtherOp); } @@ -386,7 +386,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { willNotOverflowSignedMul(Op0Conv->getOperand(0), CI, I)) { // Insert the new, smaller mul. Value *NewMul = - Builder->CreateNSWMul(Op0Conv->getOperand(0), CI, "mulconv"); + Builder.CreateNSWMul(Op0Conv->getOperand(0), CI, "mulconv"); return new SExtInst(NewMul, I.getType()); } } @@ -403,7 +403,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { willNotOverflowSignedMul(Op0Conv->getOperand(0), Op1Conv->getOperand(0), I)) { // Insert the new integer mul. - Value *NewMul = Builder->CreateNSWMul( + Value *NewMul = Builder.CreateNSWMul( Op0Conv->getOperand(0), Op1Conv->getOperand(0), "mulconv"); return new SExtInst(NewMul, I.getType()); } @@ -422,7 +422,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { willNotOverflowUnsignedMul(Op0Conv->getOperand(0), CI, I)) { // Insert the new, smaller mul. Value *NewMul = - Builder->CreateNUWMul(Op0Conv->getOperand(0), CI, "mulconv"); + Builder.CreateNUWMul(Op0Conv->getOperand(0), CI, "mulconv"); return new ZExtInst(NewMul, I.getType()); } } @@ -439,7 +439,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) { willNotOverflowUnsignedMul(Op0Conv->getOperand(0), Op1Conv->getOperand(0), I)) { // Insert the new integer mul. - Value *NewMul = Builder->CreateNUWMul( + Value *NewMul = Builder.CreateNUWMul( Op0Conv->getOperand(0), Op1Conv->getOperand(0), "mulconv"); return new ZExtInst(NewMul, I.getType()); } @@ -698,11 +698,11 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { } // if pattern detected emit alternate sequence if (OpX && OpY) { - BuilderTy::FastMathFlagGuard Guard(*Builder); - Builder->setFastMathFlags(Log2->getFastMathFlags()); + BuilderTy::FastMathFlagGuard Guard(Builder); + Builder.setFastMathFlags(Log2->getFastMathFlags()); Log2->setArgOperand(0, OpY); - Value *FMulVal = Builder->CreateFMul(OpX, Log2); - Value *FSub = Builder->CreateFSub(FMulVal, OpX); + Value *FMulVal = Builder.CreateFMul(OpX, Log2); + Value *FSub = Builder.CreateFSub(FMulVal, OpX); FSub->takeName(&I); return replaceInstUsesWith(I, FSub); } @@ -714,23 +714,23 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { for (int i = 0; i < 2; i++) { bool IgnoreZeroSign = I.hasNoSignedZeros(); if (BinaryOperator::isFNeg(Opnd0, IgnoreZeroSign)) { - BuilderTy::FastMathFlagGuard Guard(*Builder); - Builder->setFastMathFlags(I.getFastMathFlags()); + BuilderTy::FastMathFlagGuard Guard(Builder); + Builder.setFastMathFlags(I.getFastMathFlags()); Value *N0 = dyn_castFNegVal(Opnd0, IgnoreZeroSign); Value *N1 = dyn_castFNegVal(Opnd1, IgnoreZeroSign); // -X * -Y => X*Y if (N1) { - Value *FMul = Builder->CreateFMul(N0, N1); + Value *FMul = Builder.CreateFMul(N0, N1); FMul->takeName(&I); return replaceInstUsesWith(I, FMul); } if (Opnd0->hasOneUse()) { // -X * Y => -(X*Y) (Promote negation as high as possible) - Value *T = Builder->CreateFMul(N0, Opnd1); - Value *Neg = Builder->CreateFNeg(T); + Value *T = Builder.CreateFMul(N0, Opnd1); + Value *Neg = Builder.CreateFNeg(T); Neg->takeName(&I); return replaceInstUsesWith(I, Neg); } @@ -755,10 +755,10 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { Y = Opnd0_0; if (Y) { - BuilderTy::FastMathFlagGuard Guard(*Builder); - Builder->setFastMathFlags(I.getFastMathFlags()); - Value *T = Builder->CreateFMul(Opnd1, Opnd1); - Value *R = Builder->CreateFMul(T, Y); + BuilderTy::FastMathFlagGuard Guard(Builder); + Builder.setFastMathFlags(I.getFastMathFlags()); + Value *T = Builder.CreateFMul(Opnd1, Opnd1); + Value *R = Builder.CreateFMul(T, Y); R->takeName(&I); return replaceInstUsesWith(I, R); } @@ -824,7 +824,7 @@ bool InstCombiner::SimplifyDivRemOfSelect(BinaryOperator &I) { *I = SI->getOperand(NonNullOperand); Worklist.Add(&*BBI); } else if (*I == SelectCond) { - *I = Builder->getInt1(NonNullOperand == 1); + *I = Builder.getInt1(NonNullOperand == 1); Worklist.Add(&*BBI); } } @@ -938,20 +938,18 @@ Instruction *InstCombiner::commonIDivTransforms(BinaryOperator &I) { } if (match(Op0, m_One())) { - assert(!I.getType()->getScalarType()->isIntegerTy(1) && - "i1 divide not removed?"); + assert(!I.getType()->isIntOrIntVectorTy(1) && "i1 divide not removed?"); if (I.getOpcode() == Instruction::SDiv) { // If Op1 is 0 then it's undefined behaviour, if Op1 is 1 then the // result is one, if Op1 is -1 then the result is minus one, otherwise // it's zero. - Value *Inc = Builder->CreateAdd(Op1, Op0); - Value *Cmp = Builder->CreateICmpULT( - Inc, ConstantInt::get(I.getType(), 3)); + Value *Inc = Builder.CreateAdd(Op1, Op0); + Value *Cmp = Builder.CreateICmpULT(Inc, ConstantInt::get(I.getType(), 3)); return SelectInst::Create(Cmp, Op1, ConstantInt::get(I.getType(), 0)); } else { // If Op1 is 0 then it's undefined behaviour. If Op1 is 1 then the // result is one, otherwise it's zero. - return new ZExtInst(Builder->CreateICmpEQ(Op1, Op0), I.getType()); + return new ZExtInst(Builder.CreateICmpEQ(Op1, Op0), I.getType()); } } @@ -1026,7 +1024,7 @@ static Instruction *foldUDivPow2Cst(Value *Op0, Value *Op1, // X udiv C, where C >= signbit static Instruction *foldUDivNegCst(Value *Op0, Value *Op1, const BinaryOperator &I, InstCombiner &IC) { - Value *ICI = IC.Builder->CreateICmpULT(Op0, cast(Op1)); + Value *ICI = IC.Builder.CreateICmpULT(Op0, cast(Op1)); return SelectInst::Create(ICI, Constant::getNullValue(I.getType()), ConstantInt::get(I.getType(), 1)); @@ -1045,10 +1043,9 @@ static Instruction *foldUDivShl(Value *Op0, Value *Op1, const BinaryOperator &I, if (!match(ShiftLeft, m_Shl(m_APInt(CI), m_Value(N)))) llvm_unreachable("match should never fail here!"); if (*CI != 1) - N = IC.Builder->CreateAdd(N, - ConstantInt::get(N->getType(), CI->logBase2())); + N = IC.Builder.CreateAdd(N, ConstantInt::get(N->getType(), CI->logBase2())); if (Op1 != ShiftLeft) - N = IC.Builder->CreateZExt(N, Op1->getType()); + N = IC.Builder.CreateZExt(N, Op1->getType()); BinaryOperator *LShr = BinaryOperator::CreateLShr(Op0, N); if (I.isExact()) LShr->setIsExact(); @@ -1134,7 +1131,7 @@ Instruction *InstCombiner::visitUDiv(BinaryOperator &I) { if (ZExtInst *ZOp0 = dyn_cast(Op0)) if (Value *ZOp1 = dyn_castZExtVal(Op1, ZOp0->getSrcTy())) return new ZExtInst( - Builder->CreateUDiv(ZOp0->getOperand(0), ZOp1, "div", I.isExact()), + Builder.CreateUDiv(ZOp0->getOperand(0), ZOp1, "div", I.isExact()), I.getType()); // (LHS udiv (select (select (...)))) -> (LHS >> (select (select (...)))) @@ -1209,7 +1206,7 @@ Instruction *InstCombiner::visitSDiv(BinaryOperator &I) { Constant *NarrowDivisor = ConstantExpr::getTrunc(cast(Op1), Op0Src->getType()); - Value *NarrowOp = Builder->CreateSDiv(Op0Src, NarrowDivisor); + Value *NarrowOp = Builder.CreateSDiv(Op0Src, NarrowDivisor); return new SExtInst(NarrowOp, Op0->getType()); } } @@ -1217,7 +1214,7 @@ Instruction *InstCombiner::visitSDiv(BinaryOperator &I) { if (Constant *RHS = dyn_cast(Op1)) { // X/INT_MIN -> X == INT_MIN if (RHS->isMinSignedValue()) - return new ZExtInst(Builder->CreateICmpEQ(Op0, Op1), I.getType()); + return new ZExtInst(Builder.CreateICmpEQ(Op0, Op1), I.getType()); // -X/C --> X/-C provided the negation doesn't overflow. Value *X; @@ -1380,7 +1377,7 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) { // (X/Y) / Z => X / (Y*Z) // if (!isa(Y) || !isa(Op1)) { - NewInst = Builder->CreateFMul(Y, Op1); + NewInst = Builder.CreateFMul(Y, Op1); if (Instruction *RI = dyn_cast(NewInst)) { FastMathFlags Flags = I.getFastMathFlags(); Flags &= cast(Op0)->getFastMathFlags(); @@ -1392,7 +1389,7 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) { // Z / (X/Y) => Z*Y / X // if (!isa(Y) || !isa(Op0)) { - NewInst = Builder->CreateFMul(Op0, Y); + NewInst = Builder.CreateFMul(Op0, Y); if (Instruction *RI = dyn_cast(NewInst)) { FastMathFlags Flags = I.getFastMathFlags(); Flags &= cast(Op1)->getFastMathFlags(); @@ -1483,28 +1480,28 @@ Instruction *InstCombiner::visitURem(BinaryOperator &I) { // (zext A) urem (zext B) --> zext (A urem B) if (ZExtInst *ZOp0 = dyn_cast(Op0)) if (Value *ZOp1 = dyn_castZExtVal(Op1, ZOp0->getSrcTy())) - return new ZExtInst(Builder->CreateURem(ZOp0->getOperand(0), ZOp1), + return new ZExtInst(Builder.CreateURem(ZOp0->getOperand(0), ZOp1), I.getType()); // X urem Y -> X and Y-1, where Y is a power of 2, if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) { Constant *N1 = Constant::getAllOnesValue(I.getType()); - Value *Add = Builder->CreateAdd(Op1, N1); + Value *Add = Builder.CreateAdd(Op1, N1); return BinaryOperator::CreateAnd(Op0, Add); } // 1 urem X -> zext(X != 1) if (match(Op0, m_One())) { - Value *Cmp = Builder->CreateICmpNE(Op1, Op0); - Value *Ext = Builder->CreateZExt(Cmp, I.getType()); + Value *Cmp = Builder.CreateICmpNE(Op1, Op0); + Value *Ext = Builder.CreateZExt(Cmp, I.getType()); return replaceInstUsesWith(I, Ext); } // X urem C -> X < C ? X : X - C, where C >= signbit. const APInt *DivisorC; if (match(Op1, m_APInt(DivisorC)) && DivisorC->isNegative()) { - Value *Cmp = Builder->CreateICmpULT(Op0, Op1); - Value *Sub = Builder->CreateSub(Op0, Op1); + Value *Cmp = Builder.CreateICmpULT(Op0, Op1); + Value *Sub = Builder.CreateSub(Op0, Op1); return SelectInst::Create(Cmp, Op0, Sub); } diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 5dbf1e85b05b..0011412c2bf4 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -636,10 +636,10 @@ static bool PHIsEqualValue(PHINode *PN, Value *NonPhiInVal, /// Return an existing non-zero constant if this phi node has one, otherwise /// return constant 1. static ConstantInt *GetAnyNonZeroConstInt(PHINode &PN) { - assert(isa(PN.getType()) && "Expect only intger type phi"); + assert(isa(PN.getType()) && "Expect only integer type phi"); for (Value *V : PN.operands()) if (auto *ConstVA = dyn_cast(V)) - if (!ConstVA->isZeroValue()) + if (!ConstVA->isZero()) return ConstVA; return ConstantInt::get(cast(PN.getType()), 1); } @@ -836,12 +836,12 @@ Instruction *InstCombiner::SliceUpIllegalIntegerPHI(PHINode &FirstPhi) { } // Otherwise, do an extract in the predecessor. - Builder->SetInsertPoint(Pred->getTerminator()); + Builder.SetInsertPoint(Pred->getTerminator()); Value *Res = InVal; if (Offset) - Res = Builder->CreateLShr(Res, ConstantInt::get(InVal->getType(), + Res = Builder.CreateLShr(Res, ConstantInt::get(InVal->getType(), Offset), "extract"); - Res = Builder->CreateTrunc(Res, Ty, "extract.t"); + Res = Builder.CreateTrunc(Res, Ty, "extract.t"); PredVal = Res; EltPHI->addIncoming(Res, Pred); diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 80c6595904e1..4eebe8255998 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -61,12 +61,12 @@ static CmpInst::Predicate getCmpPredicateForMinMax(SelectPatternFlavor SPF, } } -static Value *generateMinMaxSelectPattern(InstCombiner::BuilderTy *Builder, +static Value *generateMinMaxSelectPattern(InstCombiner::BuilderTy &Builder, SelectPatternFlavor SPF, Value *A, Value *B) { CmpInst::Predicate Pred = getCmpPredicateForMinMax(SPF); assert(CmpInst::isIntPredicate(Pred)); - return Builder->CreateSelect(Builder->CreateICmp(Pred, A, B), A, B); + return Builder.CreateSelect(Builder.CreateICmp(Pred, A, B), A, B); } /// We want to turn code that looks like this: @@ -167,8 +167,8 @@ Instruction *InstCombiner::foldSelectOpOp(SelectInst &SI, Instruction *TI, // Fold this by inserting a select from the input values. Value *NewSI = - Builder->CreateSelect(SI.getCondition(), TI->getOperand(0), - FI->getOperand(0), SI.getName() + ".v", &SI); + Builder.CreateSelect(SI.getCondition(), TI->getOperand(0), + FI->getOperand(0), SI.getName() + ".v", &SI); return CastInst::Create(Instruction::CastOps(TI->getOpcode()), NewSI, TI->getType()); } @@ -211,8 +211,8 @@ Instruction *InstCombiner::foldSelectOpOp(SelectInst &SI, Instruction *TI, } // If we reach here, they do have operations in common. - Value *NewSI = Builder->CreateSelect(SI.getCondition(), OtherOpT, OtherOpF, - SI.getName() + ".v", &SI); + Value *NewSI = Builder.CreateSelect(SI.getCondition(), OtherOpT, OtherOpF, + SI.getName() + ".v", &SI); Value *Op0 = MatchIsOpZero ? MatchOp : NewSI; Value *Op1 = MatchIsOpZero ? NewSI : MatchOp; return BinaryOperator::Create(BO->getOpcode(), Op0, Op1); @@ -227,8 +227,8 @@ static bool isSelect01(Constant *C1, Constant *C2) { return false; if (!C1I->isZero() && !C2I->isZero()) // One side must be zero. return false; - return C1I->isOne() || C1I->isAllOnesValue() || - C2I->isOne() || C2I->isAllOnesValue(); + return C1I->isOne() || C1I->isMinusOne() || + C2I->isOne() || C2I->isMinusOne(); } /// Try to fold the select into one of the operands to allow further @@ -254,7 +254,7 @@ Instruction *InstCombiner::foldSelectIntoOp(SelectInst &SI, Value *TrueVal, // Avoid creating select between 2 constants unless it's selecting // between 0, 1 and -1. if (!isa(OOp) || isSelect01(C, cast(OOp))) { - Value *NewSel = Builder->CreateSelect(SI.getCondition(), OOp, C); + Value *NewSel = Builder.CreateSelect(SI.getCondition(), OOp, C); NewSel->takeName(TVI); BinaryOperator *TVI_BO = cast(TVI); BinaryOperator *BO = BinaryOperator::Create(TVI_BO->getOpcode(), @@ -284,7 +284,7 @@ Instruction *InstCombiner::foldSelectIntoOp(SelectInst &SI, Value *TrueVal, // Avoid creating select between 2 constants unless it's selecting // between 0, 1 and -1. if (!isa(OOp) || isSelect01(C, cast(OOp))) { - Value *NewSel = Builder->CreateSelect(SI.getCondition(), C, OOp); + Value *NewSel = Builder.CreateSelect(SI.getCondition(), C, OOp); NewSel->takeName(FVI); BinaryOperator *FVI_BO = cast(FVI); BinaryOperator *BO = BinaryOperator::Create(FVI_BO->getOpcode(), @@ -315,7 +315,7 @@ Instruction *InstCombiner::foldSelectIntoOp(SelectInst &SI, Value *TrueVal, /// 3. The magnitude of C2 and C1 are flipped static Value *foldSelectICmpAndOr(const SelectInst &SI, Value *TrueVal, Value *FalseVal, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { const ICmpInst *IC = dyn_cast(SI.getCondition()); if (!IC || !SI.getType()->isIntegerTy()) return nullptr; @@ -383,22 +383,22 @@ static Value *foldSelectICmpAndOr(const SelectInst &SI, Value *TrueVal, if (NeedAnd) { // Insert the AND instruction on the input to the truncate. APInt C1 = APInt::getOneBitSet(V->getType()->getScalarSizeInBits(), C1Log); - V = Builder->CreateAnd(V, ConstantInt::get(V->getType(), C1)); + V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), C1)); } if (C2Log > C1Log) { - V = Builder->CreateZExtOrTrunc(V, Y->getType()); - V = Builder->CreateShl(V, C2Log - C1Log); + V = Builder.CreateZExtOrTrunc(V, Y->getType()); + V = Builder.CreateShl(V, C2Log - C1Log); } else if (C1Log > C2Log) { - V = Builder->CreateLShr(V, C1Log - C2Log); - V = Builder->CreateZExtOrTrunc(V, Y->getType()); + V = Builder.CreateLShr(V, C1Log - C2Log); + V = Builder.CreateZExtOrTrunc(V, Y->getType()); } else - V = Builder->CreateZExtOrTrunc(V, Y->getType()); + V = Builder.CreateZExtOrTrunc(V, Y->getType()); if (NeedXor) - V = Builder->CreateXor(V, *C2); + V = Builder.CreateXor(V, *C2); - return Builder->CreateOr(V, Y); + return Builder.CreateOr(V, Y); } /// Attempt to fold a cttz/ctlz followed by a icmp plus select into a single @@ -414,7 +414,7 @@ static Value *foldSelectICmpAndOr(const SelectInst &SI, Value *TrueVal, /// into: /// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 false) static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal, - InstCombiner::BuilderTy *Builder) { + InstCombiner::BuilderTy &Builder) { ICmpInst::Predicate Pred = ICI->getPredicate(); Value *CmpLHS = ICI->getOperand(0); Value *CmpRHS = ICI->getOperand(1); @@ -449,8 +449,8 @@ static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal, IntrinsicInst *NewI = cast(II->clone()); Type *Ty = NewI->getArgOperand(1)->getType(); NewI->setArgOperand(1, Constant::getNullValue(Ty)); - Builder->Insert(NewI); - return Builder->CreateZExtOrTrunc(NewI, ValueOnZero->getType()); + Builder.Insert(NewI); + return Builder.CreateZExtOrTrunc(NewI, ValueOnZero->getType()); } return nullptr; @@ -597,7 +597,7 @@ canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp, /// Visit a SelectInst that has an ICmpInst as its first operand. Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI, ICmpInst *ICI) { - if (Instruction *NewSel = canonicalizeMinMaxWithConstant(SI, *ICI, *Builder)) + if (Instruction *NewSel = canonicalizeMinMaxWithConstant(SI, *ICI, Builder)) return NewSel; bool Changed = adjustMinMax(SI, *ICI); @@ -617,23 +617,23 @@ Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI, if (TrueVal->getType() == Ty) { if (ConstantInt *Cmp = dyn_cast(CmpRHS)) { ConstantInt *C1 = nullptr, *C2 = nullptr; - if (Pred == ICmpInst::ICMP_SGT && Cmp->isAllOnesValue()) { + if (Pred == ICmpInst::ICMP_SGT && Cmp->isMinusOne()) { C1 = dyn_cast(TrueVal); C2 = dyn_cast(FalseVal); - } else if (Pred == ICmpInst::ICMP_SLT && Cmp->isNullValue()) { + } else if (Pred == ICmpInst::ICMP_SLT && Cmp->isZero()) { C1 = dyn_cast(FalseVal); C2 = dyn_cast(TrueVal); } if (C1 && C2) { // This shift results in either -1 or 0. - Value *AShr = Builder->CreateAShr(CmpLHS, Ty->getBitWidth()-1); + Value *AShr = Builder.CreateAShr(CmpLHS, Ty->getBitWidth() - 1); // Check if we can express the operation with a single or. - if (C2->isAllOnesValue()) - return replaceInstUsesWith(SI, Builder->CreateOr(AShr, C1)); + if (C2->isMinusOne()) + return replaceInstUsesWith(SI, Builder.CreateOr(AShr, C1)); - Value *And = Builder->CreateAnd(AShr, C2->getValue()-C1->getValue()); - return replaceInstUsesWith(SI, Builder->CreateAdd(And, C1)); + Value *And = Builder.CreateAnd(AShr, C2->getValue() - C1->getValue()); + return replaceInstUsesWith(SI, Builder.CreateAdd(And, C1)); } } } @@ -684,19 +684,19 @@ Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI, // (X & Y) == 0 ? X : X ^ Y --> X & ~Y if (TrueWhenUnset && TrueVal == X && match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) - V = Builder->CreateAnd(X, ~(*Y)); + V = Builder.CreateAnd(X, ~(*Y)); // (X & Y) != 0 ? X ^ Y : X --> X & ~Y else if (!TrueWhenUnset && FalseVal == X && match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) - V = Builder->CreateAnd(X, ~(*Y)); + V = Builder.CreateAnd(X, ~(*Y)); // (X & Y) == 0 ? X ^ Y : X --> X | Y else if (TrueWhenUnset && FalseVal == X && match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) - V = Builder->CreateOr(X, *Y); + V = Builder.CreateOr(X, *Y); // (X & Y) != 0 ? X : X ^ Y --> X | Y else if (!TrueWhenUnset && TrueVal == X && match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) - V = Builder->CreateOr(X, *Y); + V = Builder.CreateOr(X, *Y); if (V) return replaceInstUsesWith(SI, V); @@ -809,8 +809,8 @@ Instruction *InstCombiner::foldSPFofSPF(Instruction *Inner, (SPF1 == SPF_NABS && SPF2 == SPF_ABS)) { SelectInst *SI = cast(Inner); Value *NewSI = - Builder->CreateSelect(SI->getCondition(), SI->getFalseValue(), - SI->getTrueValue(), SI->getName(), SI); + Builder.CreateSelect(SI->getCondition(), SI->getFalseValue(), + SI->getTrueValue(), SI->getName(), SI); return replaceInstUsesWith(Outer, NewSI); } @@ -848,15 +848,15 @@ Instruction *InstCombiner::foldSPFofSPF(Instruction *Inner, IsFreeOrProfitableToInvert(B, NotB, ElidesXor) && IsFreeOrProfitableToInvert(C, NotC, ElidesXor) && ElidesXor) { if (!NotA) - NotA = Builder->CreateNot(A); + NotA = Builder.CreateNot(A); if (!NotB) - NotB = Builder->CreateNot(B); + NotB = Builder.CreateNot(B); if (!NotC) - NotC = Builder->CreateNot(C); + NotC = Builder.CreateNot(C); Value *NewInner = generateMinMaxSelectPattern( Builder, getInverseMinMaxSelectPattern(SPF1), NotA, NotB); - Value *NewOuter = Builder->CreateNot(generateMinMaxSelectPattern( + Value *NewOuter = Builder.CreateNot(generateMinMaxSelectPattern( Builder, getInverseMinMaxSelectPattern(SPF2), NewInner, NotC)); return replaceInstUsesWith(Outer, NewOuter); } @@ -868,9 +868,9 @@ Instruction *InstCombiner::foldSPFofSPF(Instruction *Inner, /// icmp instruction with zero, and we have an 'and' with the non-constant value /// and a power of two we can turn the select into a shift on the result of the /// 'and'. -static Value *foldSelectICmpAnd(const SelectInst &SI, ConstantInt *TrueVal, - ConstantInt *FalseVal, - InstCombiner::BuilderTy *Builder) { +static Value *foldSelectICmpAnd(const SelectInst &SI, APInt TrueVal, + APInt FalseVal, + InstCombiner::BuilderTy &Builder) { const ICmpInst *IC = dyn_cast(SI.getCondition()); if (!IC || !IC->isEquality() || !SI.getType()->isIntegerTy()) return nullptr; @@ -886,56 +886,53 @@ static Value *foldSelectICmpAnd(const SelectInst &SI, ConstantInt *TrueVal, // If both select arms are non-zero see if we have a select of the form // 'x ? 2^n + C : C'. Then we can offset both arms by C, use the logic // for 'x ? 2^n : 0' and fix the thing up at the end. - ConstantInt *Offset = nullptr; - if (!TrueVal->isZero() && !FalseVal->isZero()) { - if ((TrueVal->getValue() - FalseVal->getValue()).isPowerOf2()) + APInt Offset(TrueVal.getBitWidth(), 0); + if (!TrueVal.isNullValue() && !FalseVal.isNullValue()) { + if ((TrueVal - FalseVal).isPowerOf2()) Offset = FalseVal; - else if ((FalseVal->getValue() - TrueVal->getValue()).isPowerOf2()) + else if ((FalseVal - TrueVal).isPowerOf2()) Offset = TrueVal; else return nullptr; // Adjust TrueVal and FalseVal to the offset. - TrueVal = ConstantInt::get(Builder->getContext(), - TrueVal->getValue() - Offset->getValue()); - FalseVal = ConstantInt::get(Builder->getContext(), - FalseVal->getValue() - Offset->getValue()); + TrueVal -= Offset; + FalseVal -= Offset; } // Make sure the mask in the 'and' and one of the select arms is a power of 2. if (!AndRHS->getValue().isPowerOf2() || - (!TrueVal->getValue().isPowerOf2() && - !FalseVal->getValue().isPowerOf2())) + (!TrueVal.isPowerOf2() && !FalseVal.isPowerOf2())) return nullptr; // Determine which shift is needed to transform result of the 'and' into the // desired result. - ConstantInt *ValC = !TrueVal->isZero() ? TrueVal : FalseVal; - unsigned ValZeros = ValC->getValue().logBase2(); + const APInt &ValC = !TrueVal.isNullValue() ? TrueVal : FalseVal; + unsigned ValZeros = ValC.logBase2(); unsigned AndZeros = AndRHS->getValue().logBase2(); // If types don't match we can still convert the select by introducing a zext // or a trunc of the 'and'. The trunc case requires that all of the truncated // bits are zero, we can figure that out by looking at the 'and' mask. - if (AndZeros >= ValC->getBitWidth()) + if (AndZeros >= ValC.getBitWidth()) return nullptr; - Value *V = Builder->CreateZExtOrTrunc(LHS, SI.getType()); + Value *V = Builder.CreateZExtOrTrunc(LHS, SI.getType()); if (ValZeros > AndZeros) - V = Builder->CreateShl(V, ValZeros - AndZeros); + V = Builder.CreateShl(V, ValZeros - AndZeros); else if (ValZeros < AndZeros) - V = Builder->CreateLShr(V, AndZeros - ValZeros); + V = Builder.CreateLShr(V, AndZeros - ValZeros); // Okay, now we know that everything is set up, we just don't know whether we // have a icmp_ne or icmp_eq and whether the true or false val is the zero. - bool ShouldNotVal = !TrueVal->isZero(); + bool ShouldNotVal = !TrueVal.isNullValue(); ShouldNotVal ^= IC->getPredicate() == ICmpInst::ICMP_NE; if (ShouldNotVal) - V = Builder->CreateXor(V, ValC); + V = Builder.CreateXor(V, ValC); // Apply an offset if needed. - if (Offset) - V = Builder->CreateAdd(V, Offset); + if (!Offset.isNullValue()) + V = Builder.CreateAdd(V, ConstantInt::get(V->getType(), Offset)); return V; } @@ -1024,7 +1021,7 @@ Instruction *InstCombiner::foldSelectExtConst(SelectInst &Sel) { // TODO: Handle larger types? That requires adjusting FoldOpIntoSelect too. Value *X = ExtInst->getOperand(0); Type *SmallType = X->getType(); - if (!SmallType->getScalarType()->isIntegerTy(1)) + if (!SmallType->isIntOrIntVectorTy(1)) return nullptr; Constant *C; @@ -1045,7 +1042,7 @@ Instruction *InstCombiner::foldSelectExtConst(SelectInst &Sel) { // select Cond, (ext X), C --> ext(select Cond, X, C') // select Cond, C, (ext X) --> ext(select Cond, C', X) - Value *NewSel = Builder->CreateSelect(Cond, X, TruncCVal, "narrow", &Sel); + Value *NewSel = Builder.CreateSelect(Cond, X, TruncCVal, "narrow", &Sel); return CastInst::Create(Instruction::CastOps(ExtOpcode), NewSel, SelType); } @@ -1184,7 +1181,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { return &SI; } - if (SelType->getScalarType()->isIntegerTy(1) && + if (SelType->isIntOrIntVectorTy(1) && TrueVal->getType() == CondVal->getType()) { if (match(TrueVal, m_One())) { // Change: A = select B, true, C --> A = or B, C @@ -1192,7 +1189,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { } if (match(TrueVal, m_Zero())) { // Change: A = select B, false, C --> A = and !B, C - Value *NotCond = Builder->CreateNot(CondVal, "not." + CondVal->getName()); + Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); return BinaryOperator::CreateAnd(NotCond, FalseVal); } if (match(FalseVal, m_Zero())) { @@ -1201,7 +1198,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { } if (match(FalseVal, m_One())) { // Change: A = select B, C, true --> A = or !B, C - Value *NotCond = Builder->CreateNot(CondVal, "not." + CondVal->getName()); + Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); return BinaryOperator::CreateOr(NotCond, TrueVal); } @@ -1226,7 +1223,8 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { // select i1 %c, <2 x i8> <1, 1>, <2 x i8> <0, 0> // because that may need 3 instructions to splat the condition value: // extend, insertelement, shufflevector. - if (CondVal->getType()->isVectorTy() == SelType->isVectorTy()) { + if (SelType->isIntOrIntVectorTy() && + CondVal->getType()->isVectorTy() == SelType->isVectorTy()) { // select C, 1, 0 -> zext C to int if (match(TrueVal, m_One()) && match(FalseVal, m_Zero())) return new ZExtInst(CondVal, SelType); @@ -1237,20 +1235,21 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { // select C, 0, 1 -> zext !C to int if (match(TrueVal, m_Zero()) && match(FalseVal, m_One())) { - Value *NotCond = Builder->CreateNot(CondVal, "not." + CondVal->getName()); + Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); return new ZExtInst(NotCond, SelType); } // select C, 0, -1 -> sext !C to int if (match(TrueVal, m_Zero()) && match(FalseVal, m_AllOnes())) { - Value *NotCond = Builder->CreateNot(CondVal, "not." + CondVal->getName()); + Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); return new SExtInst(NotCond, SelType); } } if (ConstantInt *TrueValC = dyn_cast(TrueVal)) if (ConstantInt *FalseValC = dyn_cast(FalseVal)) - if (Value *V = foldSelectICmpAnd(SI, TrueValC, FalseValC, Builder)) + if (Value *V = foldSelectICmpAnd(SI, TrueValC->getValue(), + FalseValC->getValue(), Builder)) return replaceInstUsesWith(SI, V); // See if we are selecting two values based on a comparison of the two values. @@ -1288,10 +1287,10 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { // (X ugt Y) ? X : Y -> (X ole Y) ? Y : X if (FCI->hasOneUse() && FCmpInst::isUnordered(FCI->getPredicate())) { FCmpInst::Predicate InvPred = FCI->getInversePredicate(); - IRBuilder<>::FastMathFlagGuard FMFG(*Builder); - Builder->setFastMathFlags(FCI->getFastMathFlags()); - Value *NewCond = Builder->CreateFCmp(InvPred, TrueVal, FalseVal, - FCI->getName() + ".inv"); + IRBuilder<>::FastMathFlagGuard FMFG(Builder); + Builder.setFastMathFlags(FCI->getFastMathFlags()); + Value *NewCond = Builder.CreateFCmp(InvPred, TrueVal, FalseVal, + FCI->getName() + ".inv"); return SelectInst::Create(NewCond, FalseVal, TrueVal, SI.getName() + ".p"); @@ -1331,10 +1330,10 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { // (X ugt Y) ? X : Y -> (X ole Y) ? X : Y if (FCI->hasOneUse() && FCmpInst::isUnordered(FCI->getPredicate())) { FCmpInst::Predicate InvPred = FCI->getInversePredicate(); - IRBuilder<>::FastMathFlagGuard FMFG(*Builder); - Builder->setFastMathFlags(FCI->getFastMathFlags()); - Value *NewCond = Builder->CreateFCmp(InvPred, FalseVal, TrueVal, - FCI->getName() + ".inv"); + IRBuilder<>::FastMathFlagGuard FMFG(Builder); + Builder.setFastMathFlags(FCI->getFastMathFlags()); + Value *NewCond = Builder.CreateFCmp(InvPred, FalseVal, TrueVal, + FCI->getName() + ".inv"); return SelectInst::Create(NewCond, FalseVal, TrueVal, SI.getName() + ".p"); @@ -1350,7 +1349,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { if (Instruction *Result = foldSelectInstWithICmp(SI, ICI)) return Result; - if (Instruction *Add = foldAddSubSelect(SI, *Builder)) + if (Instruction *Add = foldAddSubSelect(SI, Builder)) return Add; // Turn (select C, (op X, Y), (op X, Z)) -> (op X, (select C, Y, Z)) @@ -1381,16 +1380,16 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { Value *Cmp; if (CmpInst::isIntPredicate(Pred)) { - Cmp = Builder->CreateICmp(Pred, LHS, RHS); + Cmp = Builder.CreateICmp(Pred, LHS, RHS); } else { - IRBuilder<>::FastMathFlagGuard FMFG(*Builder); + IRBuilder<>::FastMathFlagGuard FMFG(Builder); auto FMF = cast(SI.getCondition())->getFastMathFlags(); - Builder->setFastMathFlags(FMF); - Cmp = Builder->CreateFCmp(Pred, LHS, RHS); + Builder.setFastMathFlags(FMF); + Cmp = Builder.CreateFCmp(Pred, LHS, RHS); } - Value *NewSI = Builder->CreateCast( - CastOp, Builder->CreateSelect(Cmp, LHS, RHS, SI.getName(), &SI), + Value *NewSI = Builder.CreateCast( + CastOp, Builder.CreateSelect(Cmp, LHS, RHS, SI.getName(), &SI), SelType); return replaceInstUsesWith(SI, NewSI); } @@ -1425,13 +1424,12 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { (SI.hasOneUse() && match(*SI.user_begin(), m_Not(m_Value()))); if (NumberOfNots >= 2) { - Value *NewLHS = Builder->CreateNot(LHS); - Value *NewRHS = Builder->CreateNot(RHS); - Value *NewCmp = SPF == SPF_SMAX - ? Builder->CreateICmpSLT(NewLHS, NewRHS) - : Builder->CreateICmpULT(NewLHS, NewRHS); + Value *NewLHS = Builder.CreateNot(LHS); + Value *NewRHS = Builder.CreateNot(RHS); + Value *NewCmp = SPF == SPF_SMAX ? Builder.CreateICmpSLT(NewLHS, NewRHS) + : Builder.CreateICmpULT(NewLHS, NewRHS); Value *NewSI = - Builder->CreateNot(Builder->CreateSelect(NewCmp, NewLHS, NewRHS)); + Builder.CreateNot(Builder.CreateSelect(NewCmp, NewLHS, NewRHS)); return replaceInstUsesWith(SI, NewSI); } } @@ -1461,7 +1459,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { // We choose this as normal form to enable folding on the And and shortening // paths for the values (this helps GetUnderlyingObjects() for example). if (TrueSI->getFalseValue() == FalseVal && TrueSI->hasOneUse()) { - Value *And = Builder->CreateAnd(CondVal, TrueSI->getCondition()); + Value *And = Builder.CreateAnd(CondVal, TrueSI->getCondition()); SI.setOperand(0, And); SI.setOperand(1, TrueSI->getTrueValue()); return &SI; @@ -1479,7 +1477,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { } // select(C0, a, select(C1, a, b)) -> select(C0|C1, a, b) if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) { - Value *Or = Builder->CreateOr(CondVal, FalseSI->getCondition()); + Value *Or = Builder.CreateOr(CondVal, FalseSI->getCondition()); SI.setOperand(0, Or); SI.setOperand(2, FalseSI->getFalseValue()); return &SI; @@ -1541,7 +1539,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) { return replaceInstUsesWith(SI, FalseVal); } - if (Instruction *BitCastSel = foldSelectCmpBitcasts(SI, *Builder)) + if (Instruction *BitCastSel = foldSelectCmpBitcasts(SI, Builder)) return BitCastSel; return nullptr; diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp index 1bb1a85367d1..7ed141c7fd79 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp @@ -47,7 +47,7 @@ Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) { if (isKnownNonNegative(A, DL, 0, &AC, &I, &DT) && isKnownNonNegative(C, DL, 0, &AC, &I, &DT)) return BinaryOperator::Create( - I.getOpcode(), Builder->CreateBinOp(I.getOpcode(), Op0, C), A); + I.getOpcode(), Builder.CreateBinOp(I.getOpcode(), Op0, C), A); // X shift (A srem B) -> X shift (A and B-1) iff B is a power of 2. // Because shifts by negative values (which could occur if A were negative) @@ -56,8 +56,8 @@ Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) { if (Op1->hasOneUse() && match(Op1, m_SRem(m_Value(A), m_Power2(B)))) { // FIXME: Should this get moved into SimplifyDemandedBits by saying we don't // demand the sign bit (and many others) here?? - Value *Rem = Builder->CreateAnd(A, ConstantInt::get(I.getType(), *B-1), - Op1->getName()); + Value *Rem = Builder.CreateAnd(A, ConstantInt::get(I.getType(), *B - 1), + Op1->getName()); I.setOperand(1, Rem); return &I; } @@ -260,9 +260,9 @@ static Value *getShiftedValue(Value *V, unsigned NumBits, bool isLeftShift, // We can always evaluate constants shifted. if (Constant *C = dyn_cast(V)) { if (isLeftShift) - V = IC.Builder->CreateShl(C, NumBits); + V = IC.Builder.CreateShl(C, NumBits); else - V = IC.Builder->CreateLShr(C, NumBits); + V = IC.Builder.CreateLShr(C, NumBits); // If we got a constantexpr back, try to simplify it with TD info. if (auto *C = dyn_cast(V)) if (auto *FoldedC = @@ -289,7 +289,7 @@ static Value *getShiftedValue(Value *V, unsigned NumBits, bool isLeftShift, case Instruction::Shl: case Instruction::LShr: return foldShiftedShift(cast(I), NumBits, isLeftShift, - *(IC.Builder)); + IC.Builder); case Instruction::Select: I->setOperand( @@ -353,7 +353,7 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, Constant *ShAmt = ConstantExpr::getZExt(cast(Op1), TrOp->getType()); // (shift2 (shift1 & 0x00FF), c2) - Value *NSh = Builder->CreateBinOp(I.getOpcode(), TrOp, ShAmt,I.getName()); + Value *NSh = Builder.CreateBinOp(I.getOpcode(), TrOp, ShAmt, I.getName()); // For logical shifts, the truncation has the effect of making the high // part of the register be zeros. Emulate this by inserting an AND to @@ -375,9 +375,9 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, } // shift1 & 0x00FF - Value *And = Builder->CreateAnd(NSh, - ConstantInt::get(I.getContext(), MaskV), - TI->getName()); + Value *And = Builder.CreateAnd(NSh, + ConstantInt::get(I.getContext(), MaskV), + TI->getName()); // Return the value truncated to the interesting size. return new TruncInst(And, I.getType()); @@ -401,10 +401,10 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, match(Op0BO->getOperand(1), m_Shr(m_Value(V1), m_Specific(Op1)))) { Value *YS = // (Y << C) - Builder->CreateShl(Op0BO->getOperand(0), Op1, Op0BO->getName()); + Builder.CreateShl(Op0BO->getOperand(0), Op1, Op0BO->getName()); // (X + (Y << C)) - Value *X = Builder->CreateBinOp(Op0BO->getOpcode(), YS, V1, - Op0BO->getOperand(1)->getName()); + Value *X = Builder.CreateBinOp(Op0BO->getOpcode(), YS, V1, + Op0BO->getOperand(1)->getName()); unsigned Op1Val = Op1C->getLimitedValue(TypeBits); APInt Bits = APInt::getHighBitsSet(TypeBits, TypeBits - Op1Val); @@ -421,11 +421,10 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, m_And(m_OneUse(m_Shr(m_Value(V1), m_Specific(Op1))), m_ConstantInt(CC)))) { Value *YS = // (Y << C) - Builder->CreateShl(Op0BO->getOperand(0), Op1, - Op0BO->getName()); + Builder.CreateShl(Op0BO->getOperand(0), Op1, Op0BO->getName()); // X & (CC << C) - Value *XM = Builder->CreateAnd(V1, ConstantExpr::getShl(CC, Op1), - V1->getName()+".mask"); + Value *XM = Builder.CreateAnd(V1, ConstantExpr::getShl(CC, Op1), + V1->getName()+".mask"); return BinaryOperator::Create(Op0BO->getOpcode(), YS, XM); } LLVM_FALLTHROUGH; @@ -437,10 +436,10 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, match(Op0BO->getOperand(0), m_Shr(m_Value(V1), m_Specific(Op1)))) { Value *YS = // (Y << C) - Builder->CreateShl(Op0BO->getOperand(1), Op1, Op0BO->getName()); + Builder.CreateShl(Op0BO->getOperand(1), Op1, Op0BO->getName()); // (X + (Y << C)) - Value *X = Builder->CreateBinOp(Op0BO->getOpcode(), V1, YS, - Op0BO->getOperand(0)->getName()); + Value *X = Builder.CreateBinOp(Op0BO->getOpcode(), V1, YS, + Op0BO->getOperand(0)->getName()); unsigned Op1Val = Op1C->getLimitedValue(TypeBits); APInt Bits = APInt::getHighBitsSet(TypeBits, TypeBits - Op1Val); @@ -456,10 +455,10 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, m_And(m_OneUse(m_Shr(m_Value(V1), m_Value(V2))), m_ConstantInt(CC))) && V2 == Op1) { Value *YS = // (Y << C) - Builder->CreateShl(Op0BO->getOperand(1), Op1, Op0BO->getName()); + Builder.CreateShl(Op0BO->getOperand(1), Op1, Op0BO->getName()); // X & (CC << C) - Value *XM = Builder->CreateAnd(V1, ConstantExpr::getShl(CC, Op1), - V1->getName()+".mask"); + Value *XM = Builder.CreateAnd(V1, ConstantExpr::getShl(CC, Op1), + V1->getName()+".mask"); return BinaryOperator::Create(Op0BO->getOpcode(), XM, YS); } @@ -502,7 +501,7 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1, Constant *NewRHS = ConstantExpr::get(I.getOpcode(), Op0C, Op1); Value *NewShift = - Builder->CreateBinOp(I.getOpcode(), Op0BO->getOperand(0), Op1); + Builder.CreateBinOp(I.getOpcode(), Op0BO->getOperand(0), Op1); NewShift->takeName(Op0BO); return BinaryOperator::Create(Op0BO->getOpcode(), NewShift, @@ -541,7 +540,7 @@ Instruction *InstCombiner::visitShl(BinaryOperator &I) { unsigned SrcWidth = X->getType()->getScalarSizeInBits(); if (ShAmt < SrcWidth && MaskedValueIsZero(X, APInt::getHighBitsSet(SrcWidth, ShAmt), 0, &I)) - return new ZExtInst(Builder->CreateShl(X, ShAmt), Ty); + return new ZExtInst(Builder.CreateShl(X, ShAmt), Ty); } // (X >>u C) << C --> X & (-1 << C) @@ -641,7 +640,7 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) { // ctpop.i32(x)>>5 --> zext(x == -1) bool IsPop = II->getIntrinsicID() == Intrinsic::ctpop; Constant *RHS = ConstantInt::getSigned(Ty, IsPop ? -1 : 0); - Value *Cmp = Builder->CreateICmpEQ(II->getArgOperand(0), RHS); + Value *Cmp = Builder.CreateICmpEQ(II->getArgOperand(0), RHS); return new ZExtInst(Cmp, Ty); } @@ -658,7 +657,7 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) { return NewLShr; } // (X << C1) >>u C2 --> (X >>u (C2 - C1)) & (-1 >> C2) - Value *NewLShr = Builder->CreateLShr(X, ShiftDiff, "", I.isExact()); + Value *NewLShr = Builder.CreateLShr(X, ShiftDiff, "", I.isExact()); APInt Mask(APInt::getLowBitsSet(BitWidth, BitWidth - ShAmt)); return BinaryOperator::CreateAnd(NewLShr, ConstantInt::get(Ty, Mask)); } @@ -671,7 +670,7 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) { return NewShl; } // (X << C1) >>u C2 --> X << (C1 - C2) & (-1 >> C2) - Value *NewShl = Builder->CreateShl(X, ShiftDiff); + Value *NewShl = Builder.CreateShl(X, ShiftDiff); APInt Mask(APInt::getLowBitsSet(BitWidth, BitWidth - ShAmt)); return BinaryOperator::CreateAnd(NewShl, ConstantInt::get(Ty, Mask)); } @@ -692,7 +691,7 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) { // lshr (sext iM X to iN), N-1 --> zext (lshr X, M-1) to iN if (Op0->hasOneUse()) { - Value *NewLShr = Builder->CreateLShr(X, SrcTyBitWidth - 1); + Value *NewLShr = Builder.CreateLShr(X, SrcTyBitWidth - 1); return new ZExtInst(NewLShr, Ty); } } @@ -701,7 +700,7 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) { if (ShAmt == BitWidth - SrcTyBitWidth && Op0->hasOneUse()) { // The new shift amount can't be more than the narrow source type. unsigned NewShAmt = std::min(ShAmt, SrcTyBitWidth - 1); - Value *AShr = Builder->CreateAShr(X, NewShAmt); + Value *AShr = Builder.CreateAShr(X, NewShAmt); return new ZExtInst(AShr, Ty); } } diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 03841164b58d..5689c0604239 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -548,7 +548,7 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, if (ConstantInt *Rem = dyn_cast(I->getOperand(1))) { // X % -1 demands all the bits because we don't want to introduce // INT_MIN % -1 (== undef) by accident. - if (Rem->isAllOnesValue()) + if (Rem->isMinusOne()) break; APInt RA = Rem->getValue().abs(); if (RA.isPowerOf2()) { @@ -1627,10 +1627,10 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, for (unsigned I = 0, E = II->getNumArgOperands(); I != E; ++I) Args.push_back(II->getArgOperand(I)); - IRBuilderBase::InsertPointGuard Guard(*Builder); - Builder->SetInsertPoint(II); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(II); - CallInst *NewCall = Builder->CreateCall(NewIntrin, Args); + CallInst *NewCall = Builder.CreateCall(NewIntrin, Args); NewCall->takeName(II); NewCall->copyMetadata(*II); @@ -1657,15 +1657,15 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, if (NewNumElts == 1) { - return Builder->CreateInsertElement(UndefValue::get(V->getType()), - NewCall, static_cast(0)); + return Builder.CreateInsertElement(UndefValue::get(V->getType()), + NewCall, static_cast(0)); } SmallVector EltMask; for (unsigned I = 0; I < VWidth; ++I) EltMask.push_back(I); - Value *Shuffle = Builder->CreateShuffleVector( + Value *Shuffle = Builder.CreateShuffleVector( NewCall, UndefValue::get(NewTy), EltMask); MadeChange = true; diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index 926e46655eb8..dd71a31b644b 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -204,11 +204,11 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) { if (I->hasOneUse() && cheapToScalarize(BO, isa(EI.getOperand(1)))) { Value *newEI0 = - Builder->CreateExtractElement(BO->getOperand(0), EI.getOperand(1), - EI.getName()+".lhs"); + Builder.CreateExtractElement(BO->getOperand(0), EI.getOperand(1), + EI.getName()+".lhs"); Value *newEI1 = - Builder->CreateExtractElement(BO->getOperand(1), EI.getOperand(1), - EI.getName()+".rhs"); + Builder.CreateExtractElement(BO->getOperand(1), EI.getOperand(1), + EI.getName()+".rhs"); return BinaryOperator::CreateWithCopiedFlags(BO->getOpcode(), newEI0, newEI1, BO); } @@ -250,8 +250,8 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) { // Bitcasts can change the number of vector elements, and they cost // nothing. if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast)) { - Value *EE = Builder->CreateExtractElement(CI->getOperand(0), - EI.getIndexOperand()); + Value *EE = Builder.CreateExtractElement(CI->getOperand(0), + EI.getIndexOperand()); Worklist.AddValue(EE); return CastInst::Create(CI->getOpcode(), EE, EI.getType()); } @@ -269,20 +269,20 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) { Value *Cond = SI->getCondition(); if (Cond->getType()->isVectorTy()) { - Cond = Builder->CreateExtractElement(Cond, - EI.getIndexOperand(), - Cond->getName() + ".elt"); + Cond = Builder.CreateExtractElement(Cond, + EI.getIndexOperand(), + Cond->getName() + ".elt"); } Value *V1Elem - = Builder->CreateExtractElement(TrueVal, - EI.getIndexOperand(), - TrueVal->getName() + ".elt"); + = Builder.CreateExtractElement(TrueVal, + EI.getIndexOperand(), + TrueVal->getName() + ".elt"); Value *V2Elem - = Builder->CreateExtractElement(FalseVal, - EI.getIndexOperand(), - FalseVal->getName() + ".elt"); + = Builder.CreateExtractElement(FalseVal, + EI.getIndexOperand(), + FalseVal->getName() + ".elt"); return SelectInst::Create(Cond, V1Elem, V2Elem, @@ -837,7 +837,7 @@ Instruction *InstCombiner::visitInsertElementInst(InsertElementInst &IE) { if (Instruction *Shuf = foldConstantInsEltIntoShuffle(IE)) return Shuf; - if (Instruction *NewInsElt = hoistInsEltConst(IE, *Builder)) + if (Instruction *NewInsElt = hoistInsEltConst(IE, Builder)) return NewInsElt; // Turn a sequence of inserts that broadcasts a scalar into a single @@ -1020,9 +1020,9 @@ InstCombiner::EvaluateInDifferentElementOrder(Value *V, ArrayRef Mask) { SmallVector MaskValues; for (int i = 0, e = Mask.size(); i != e; ++i) { if (Mask[i] == -1) - MaskValues.push_back(UndefValue::get(Builder->getInt32Ty())); + MaskValues.push_back(UndefValue::get(Builder.getInt32Ty())); else - MaskValues.push_back(Builder->getInt32(Mask[i])); + MaskValues.push_back(Builder.getInt32(Mask[i])); } return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()), ConstantVector::get(MaskValues)); @@ -1095,7 +1095,7 @@ InstCombiner::EvaluateInDifferentElementOrder(Value *V, ArrayRef Mask) { Value *V = EvaluateInDifferentElementOrder(I->getOperand(0), Mask); return InsertElementInst::Create(V, I->getOperand(1), - Builder->getInt32(Index), "", I); + Builder.getInt32(Index), "", I); } } llvm_unreachable("failed to reorder elements of vector instruction!"); @@ -1275,9 +1275,9 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) { UndefValue::get(Int32Ty)); for (unsigned I = 0, E = MaskElems, Idx = BegIdx; I != E; ++Idx, ++I) ShuffleMask[I] = ConstantInt::get(Int32Ty, Idx); - V = Builder->CreateShuffleVector(V, UndefValue::get(V->getType()), - ConstantVector::get(ShuffleMask), - SVI.getName() + ".extract"); + V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()), + ConstantVector::get(ShuffleMask), + SVI.getName() + ".extract"); BegIdx = 0; } unsigned SrcElemsPerTgtElem = TgtElemBitWidth / SrcElemBitWidth; @@ -1287,10 +1287,10 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) { auto *NewBC = BCAlreadyExists ? NewBCs[CastSrcTy] - : Builder->CreateBitCast(V, CastSrcTy, SVI.getName() + ".bc"); + : Builder.CreateBitCast(V, CastSrcTy, SVI.getName() + ".bc"); if (!BCAlreadyExists) NewBCs[CastSrcTy] = NewBC; - auto *Ext = Builder->CreateExtractElement( + auto *Ext = Builder.CreateExtractElement( NewBC, ConstantInt::get(Int32Ty, BegIdx), SVI.getName() + ".extract"); // The shufflevector isn't being replaced: the bitcast that used it // is. InstCombine will visit the newly-created instructions. diff --git a/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 723414635d6f..90e232399155 100644 --- a/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/contrib/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -88,7 +88,7 @@ MaxArraySize("instcombine-maxarray-size", cl::init(1024), cl::desc("Maximum array size considered when doing a combine")); Value *InstCombiner::EmitGEPOffset(User *GEP) { - return llvm::EmitGEPOffset(Builder, DL, GEP); + return llvm::EmitGEPOffset(&Builder, DL, GEP); } /// Return true if it is desirable to convert an integer computation from a @@ -498,8 +498,7 @@ getBinOpsForFactorization(Instruction::BinaryOps TopLevelOpcode, /// This tries to simplify binary operations by factorizing out common terms /// (e. g. "(A*B)+(A*C)" -> "A*(B+C)"). -Value *InstCombiner::tryFactorization(InstCombiner::BuilderTy *Builder, - BinaryOperator &I, +Value *InstCombiner::tryFactorization(BinaryOperator &I, Instruction::BinaryOps InnerOpcode, Value *A, Value *B, Value *C, Value *D) { assert(A && B && C && D && "All values must be provided"); @@ -525,9 +524,9 @@ Value *InstCombiner::tryFactorization(InstCombiner::BuilderTy *Builder, // If "B op D" doesn't simplify then only go on if both of the existing // operations "A op' B" and "C op' D" will be zapped as no longer used. if (!V && LHS->hasOneUse() && RHS->hasOneUse()) - V = Builder->CreateBinOp(TopLevelOpcode, B, D, RHS->getName()); + V = Builder.CreateBinOp(TopLevelOpcode, B, D, RHS->getName()); if (V) { - SimplifiedInst = Builder->CreateBinOp(InnerOpcode, A, V); + SimplifiedInst = Builder.CreateBinOp(InnerOpcode, A, V); } } @@ -545,9 +544,9 @@ Value *InstCombiner::tryFactorization(InstCombiner::BuilderTy *Builder, // If "A op C" doesn't simplify then only go on if both of the existing // operations "A op' B" and "C op' D" will be zapped as no longer used. if (!V && LHS->hasOneUse() && RHS->hasOneUse()) - V = Builder->CreateBinOp(TopLevelOpcode, A, C, LHS->getName()); + V = Builder.CreateBinOp(TopLevelOpcode, A, C, LHS->getName()); if (V) { - SimplifiedInst = Builder->CreateBinOp(InnerOpcode, V, B); + SimplifiedInst = Builder.CreateBinOp(InnerOpcode, V, B); } } @@ -610,7 +609,7 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) { // The instruction has the form "(A op' B) op (C op' D)". Try to factorize // a common term. if (Op0 && Op1 && LHSOpcode == RHSOpcode) - if (Value *V = tryFactorization(Builder, I, LHSOpcode, A, B, C, D)) + if (Value *V = tryFactorization(I, LHSOpcode, A, B, C, D)) return V; // The instruction has the form "(A op' B) op (C)". Try to factorize common @@ -618,7 +617,7 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) { if (Op0) if (Value *Ident = getIdentityValue(LHSOpcode, RHS)) if (Value *V = - tryFactorization(Builder, I, LHSOpcode, A, B, RHS, Ident)) + tryFactorization(I, LHSOpcode, A, B, RHS, Ident)) return V; // The instruction has the form "(B) op (C op' D)". Try to factorize common @@ -626,7 +625,7 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) { if (Op1) if (Value *Ident = getIdentityValue(RHSOpcode, LHS)) if (Value *V = - tryFactorization(Builder, I, RHSOpcode, LHS, Ident, C, D)) + tryFactorization(I, RHSOpcode, LHS, Ident, C, D)) return V; } @@ -644,7 +643,7 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) { SimplifyBinOp(TopLevelOpcode, B, C, SQ.getWithInstruction(&I))) { // They do! Return "L op' R". ++NumExpand; - C = Builder->CreateBinOp(InnerOpcode, L, R); + C = Builder.CreateBinOp(InnerOpcode, L, R); C->takeName(&I); return C; } @@ -663,7 +662,7 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) { SimplifyBinOp(TopLevelOpcode, A, C, SQ.getWithInstruction(&I))) { // They do! Return "L op' R". ++NumExpand; - A = Builder->CreateBinOp(InnerOpcode, L, R); + A = Builder.CreateBinOp(InnerOpcode, L, R); A->takeName(&I); return A; } @@ -678,18 +677,18 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) { if (Value *V = SimplifyBinOp(TopLevelOpcode, SI0->getFalseValue(), SI1->getFalseValue(), SQ.getWithInstruction(&I))) - SI = Builder->CreateSelect(SI0->getCondition(), - Builder->CreateBinOp(TopLevelOpcode, - SI0->getTrueValue(), - SI1->getTrueValue()), - V); + SI = Builder.CreateSelect(SI0->getCondition(), + Builder.CreateBinOp(TopLevelOpcode, + SI0->getTrueValue(), + SI1->getTrueValue()), + V); if (Value *V = SimplifyBinOp(TopLevelOpcode, SI0->getTrueValue(), SI1->getTrueValue(), SQ.getWithInstruction(&I))) - SI = Builder->CreateSelect( + SI = Builder.CreateSelect( SI0->getCondition(), V, - Builder->CreateBinOp(TopLevelOpcode, SI0->getFalseValue(), - SI1->getFalseValue())); + Builder.CreateBinOp(TopLevelOpcode, SI0->getFalseValue(), + SI1->getFalseValue())); if (SI) { SI->takeName(&I); return SI; @@ -751,9 +750,9 @@ Value *InstCombiner::dyn_castFNegVal(Value *V, bool IgnoreZeroSign) const { } static Value *foldOperationIntoSelectOperand(Instruction &I, Value *SO, - InstCombiner *IC) { + InstCombiner::BuilderTy &Builder) { if (auto *Cast = dyn_cast(&I)) - return IC->Builder->CreateCast(Cast->getOpcode(), SO, I.getType()); + return Builder.CreateCast(Cast->getOpcode(), SO, I.getType()); assert(I.isBinaryOp() && "Unexpected opcode for select folding"); @@ -772,8 +771,8 @@ static Value *foldOperationIntoSelectOperand(Instruction &I, Value *SO, std::swap(Op0, Op1); auto *BO = cast(&I); - Value *RI = IC->Builder->CreateBinOp(BO->getOpcode(), Op0, Op1, - SO->getName() + ".op"); + Value *RI = Builder.CreateBinOp(BO->getOpcode(), Op0, Op1, + SO->getName() + ".op"); auto *FPInst = dyn_cast(RI); if (FPInst && isa(FPInst)) FPInst->copyFastMathFlags(BO); @@ -791,7 +790,7 @@ Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) { return nullptr; // Bool selects with constant operands can be folded to logical ops. - if (SI->getType()->getScalarType()->isIntegerTy(1)) + if (SI->getType()->isIntOrIntVectorTy(1)) return nullptr; // If it's a bitcast involving vectors, make sure it has the same number of @@ -825,13 +824,13 @@ Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) { } } - Value *NewTV = foldOperationIntoSelectOperand(Op, TV, this); - Value *NewFV = foldOperationIntoSelectOperand(Op, FV, this); + Value *NewTV = foldOperationIntoSelectOperand(Op, TV, Builder); + Value *NewFV = foldOperationIntoSelectOperand(Op, FV, Builder); return SelectInst::Create(SI->getCondition(), NewTV, NewFV, "", nullptr, SI); } static Value *foldOperationIntoPhiValue(BinaryOperator *I, Value *InV, - InstCombiner *IC) { + InstCombiner::BuilderTy &Builder) { bool ConstIsRHS = isa(I->getOperand(1)); Constant *C = cast(I->getOperand(ConstIsRHS)); @@ -845,7 +844,7 @@ static Value *foldOperationIntoPhiValue(BinaryOperator *I, Value *InV, if (!ConstIsRHS) std::swap(Op0, Op1); - Value *RI = IC->Builder->CreateBinOp(I->getOpcode(), Op0, Op1, "phitmp"); + Value *RI = Builder.CreateBinOp(I->getOpcode(), Op0, Op1, "phitmp"); auto *FPInst = dyn_cast(RI); if (FPInst && isa(FPInst)) FPInst->copyFastMathFlags(I); @@ -916,7 +915,7 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) { // If we are going to have to insert a new computation, do so right before the // predecessor's terminator. if (NonConstBB) - Builder->SetInsertPoint(NonConstBB->getTerminator()); + Builder.SetInsertPoint(NonConstBB->getTerminator()); // Next, add all of the operands to the PHI. if (SelectInst *SI = dyn_cast(&I)) { @@ -948,9 +947,9 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) { // folded to TrueVInPred or FalseVInPred as done for ConstantInt. For // non-vector phis, this transformation was always profitable because // the select would be generated exactly once in the NonConstBB. - Builder->SetInsertPoint(ThisBB->getTerminator()); - InV = Builder->CreateSelect(PN->getIncomingValue(i), - TrueVInPred, FalseVInPred, "phitmp"); + Builder.SetInsertPoint(ThisBB->getTerminator()); + InV = Builder.CreateSelect(PN->getIncomingValue(i), TrueVInPred, + FalseVInPred, "phitmp"); } NewPN->addIncoming(InV, ThisBB); } @@ -961,16 +960,17 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) { if (Constant *InC = dyn_cast(PN->getIncomingValue(i))) InV = ConstantExpr::getCompare(CI->getPredicate(), InC, C); else if (isa(CI)) - InV = Builder->CreateICmp(CI->getPredicate(), PN->getIncomingValue(i), - C, "phitmp"); + InV = Builder.CreateICmp(CI->getPredicate(), PN->getIncomingValue(i), + C, "phitmp"); else - InV = Builder->CreateFCmp(CI->getPredicate(), PN->getIncomingValue(i), - C, "phitmp"); + InV = Builder.CreateFCmp(CI->getPredicate(), PN->getIncomingValue(i), + C, "phitmp"); NewPN->addIncoming(InV, PN->getIncomingBlock(i)); } } else if (auto *BO = dyn_cast(&I)) { for (unsigned i = 0; i != NumPHIValues; ++i) { - Value *InV = foldOperationIntoPhiValue(BO, PN->getIncomingValue(i), this); + Value *InV = foldOperationIntoPhiValue(BO, PN->getIncomingValue(i), + Builder); NewPN->addIncoming(InV, PN->getIncomingBlock(i)); } } else { @@ -981,8 +981,8 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) { if (Constant *InC = dyn_cast(PN->getIncomingValue(i))) InV = ConstantExpr::getCast(CI->getOpcode(), InC, RetTy); else - InV = Builder->CreateCast(CI->getOpcode(), - PN->getIncomingValue(i), I.getType(), "phitmp"); + InV = Builder.CreateCast(CI->getOpcode(), PN->getIncomingValue(i), + I.getType(), "phitmp"); NewPN->addIncoming(InV, PN->getIncomingBlock(i)); } } @@ -1328,8 +1328,8 @@ Value *InstCombiner::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) { /// \brief Creates node of binary operation with the same attributes as the /// specified one but with other operands. static Value *CreateBinOpAsGiven(BinaryOperator &Inst, Value *LHS, Value *RHS, - InstCombiner::BuilderTy *B) { - Value *BO = B->CreateBinOp(Inst.getOpcode(), LHS, RHS); + InstCombiner::BuilderTy &B) { + Value *BO = B.CreateBinOp(Inst.getOpcode(), LHS, RHS); // If LHS and RHS are constant, BO won't be a binary operator. if (BinaryOperator *NewBO = dyn_cast(BO)) NewBO->copyIRFlags(&Inst); @@ -1365,7 +1365,7 @@ Value *InstCombiner::SimplifyVectorOp(BinaryOperator &Inst) { LShuf->getOperand(0)->getType() == RShuf->getOperand(0)->getType()) { Value *NewBO = CreateBinOpAsGiven(Inst, LShuf->getOperand(0), RShuf->getOperand(0), Builder); - return Builder->CreateShuffleVector( + return Builder.CreateShuffleVector( NewBO, UndefValue::get(NewBO->getType()), LShuf->getMask()); } @@ -1404,7 +1404,7 @@ Value *InstCombiner::SimplifyVectorOp(BinaryOperator &Inst) { Value *NewLHS = isa(LHS) ? C2 : Shuffle->getOperand(0); Value *NewRHS = isa(LHS) ? Shuffle->getOperand(0) : C2; Value *NewBO = CreateBinOpAsGiven(Inst, NewLHS, NewRHS, Builder); - return Builder->CreateShuffleVector(NewBO, + return Builder.CreateShuffleVector(NewBO, UndefValue::get(Inst.getType()), Shuffle->getMask()); } } @@ -1452,7 +1452,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { // If we are using a wider index than needed for this platform, shrink // it to what we need. If narrower, sign-extend it to what we need. // This explicit cast can make subsequent optimizations more obvious. - *I = Builder->CreateIntCast(*I, NewIndexType, true); + *I = Builder.CreateIntCast(*I, NewIndexType, true); MadeChange = true; } } @@ -1546,10 +1546,10 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { // set that index. PHINode *NewPN; { - IRBuilderBase::InsertPointGuard Guard(*Builder); - Builder->SetInsertPoint(PN); - NewPN = Builder->CreatePHI(Op1->getOperand(DI)->getType(), - PN->getNumOperands()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(PN); + NewPN = Builder.CreatePHI(Op1->getOperand(DI)->getType(), + PN->getNumOperands()); } for (auto &I : PN->operands()) @@ -1669,8 +1669,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { // pointer arithmetic. if (match(V, m_Neg(m_PtrToInt(m_Value())))) { Operator *Index = cast(V); - Value *PtrToInt = Builder->CreatePtrToInt(PtrOp, Index->getType()); - Value *NewSub = Builder->CreateSub(PtrToInt, Index->getOperand(1)); + Value *PtrToInt = Builder.CreatePtrToInt(PtrOp, Index->getType()); + Value *NewSub = Builder.CreateSub(PtrToInt, Index->getOperand(1)); return CastInst::Create(Instruction::IntToPtr, NewSub, GEP.getType()); } // Canonicalize (gep i8* X, (ptrtoint Y)-(ptrtoint X)) @@ -1723,7 +1723,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { // -> // %0 = GEP i8 addrspace(1)* X, ... // addrspacecast i8 addrspace(1)* %0 to i8* - return new AddrSpaceCastInst(Builder->Insert(Res), GEP.getType()); + return new AddrSpaceCastInst(Builder.Insert(Res), GEP.getType()); } if (ArrayType *XATy = @@ -1751,10 +1751,10 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { // addrspacecast i8 addrspace(1)* %0 to i8* SmallVector Idx(GEP.idx_begin(), GEP.idx_end()); Value *NewGEP = GEP.isInBounds() - ? Builder->CreateInBoundsGEP( + ? Builder.CreateInBoundsGEP( nullptr, StrippedPtr, Idx, GEP.getName()) - : Builder->CreateGEP(nullptr, StrippedPtr, Idx, - GEP.getName()); + : Builder.CreateGEP(nullptr, StrippedPtr, Idx, + GEP.getName()); return new AddrSpaceCastInst(NewGEP, GEP.getType()); } } @@ -1772,9 +1772,9 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { Value *Idx[2] = { Constant::getNullValue(IdxType), GEP.getOperand(1) }; Value *NewGEP = GEP.isInBounds() - ? Builder->CreateInBoundsGEP(nullptr, StrippedPtr, Idx, - GEP.getName()) - : Builder->CreateGEP(nullptr, StrippedPtr, Idx, GEP.getName()); + ? Builder.CreateInBoundsGEP(nullptr, StrippedPtr, Idx, + GEP.getName()) + : Builder.CreateGEP(nullptr, StrippedPtr, Idx, GEP.getName()); // V and GEP are both pointer types --> BitCast return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, @@ -1807,10 +1807,10 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { // GEP may not be "inbounds". Value *NewGEP = GEP.isInBounds() && NSW - ? Builder->CreateInBoundsGEP(nullptr, StrippedPtr, NewIdx, - GEP.getName()) - : Builder->CreateGEP(nullptr, StrippedPtr, NewIdx, - GEP.getName()); + ? Builder.CreateInBoundsGEP(nullptr, StrippedPtr, NewIdx, + GEP.getName()) + : Builder.CreateGEP(nullptr, StrippedPtr, NewIdx, + GEP.getName()); // The NewGEP must be pointer typed, so must the old one -> BitCast return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, @@ -1849,10 +1849,10 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { NewIdx}; Value *NewGEP = GEP.isInBounds() && NSW - ? Builder->CreateInBoundsGEP( + ? Builder.CreateInBoundsGEP( SrcElTy, StrippedPtr, Off, GEP.getName()) - : Builder->CreateGEP(SrcElTy, StrippedPtr, Off, - GEP.getName()); + : Builder.CreateGEP(SrcElTy, StrippedPtr, Off, + GEP.getName()); // The NewGEP must be pointer typed, so must the old one -> BitCast return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, GEP.getType()); @@ -1916,8 +1916,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) { if (FindElementAtOffset(OpType, Offset.getSExtValue(), NewIndices)) { Value *NGEP = GEP.isInBounds() - ? Builder->CreateInBoundsGEP(nullptr, Operand, NewIndices) - : Builder->CreateGEP(nullptr, Operand, NewIndices); + ? Builder.CreateInBoundsGEP(nullptr, Operand, NewIndices) + : Builder.CreateGEP(nullptr, Operand, NewIndices); if (NGEP->getType() == GEP.getType()) return replaceInstUsesWith(GEP, NGEP); @@ -2166,8 +2166,8 @@ Instruction *InstCombiner::visitFree(CallInst &FI) { // free undef -> unreachable. if (isa(Op)) { // Insert a new store to null because we cannot modify the CFG here. - Builder->CreateStore(ConstantInt::getTrue(FI.getContext()), - UndefValue::get(Type::getInt1PtrTy(FI.getContext()))); + Builder.CreateStore(ConstantInt::getTrue(FI.getContext()), + UndefValue::get(Type::getInt1PtrTy(FI.getContext()))); return eraseInstFromFunction(FI); } @@ -2281,8 +2281,8 @@ Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) { // the backend should extend back to a legal type for the target. if (NewWidth > 0 && NewWidth < Known.getBitWidth()) { IntegerType *Ty = IntegerType::get(SI.getContext(), NewWidth); - Builder->SetInsertPoint(&SI); - Value *NewCond = Builder->CreateTrunc(Cond, Ty, "trunc"); + Builder.SetInsertPoint(&SI); + Value *NewCond = Builder.CreateTrunc(Cond, Ty, "trunc"); SI.setCondition(NewCond); for (auto Case : SI.cases()) { @@ -2339,8 +2339,8 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) { // %E = insertvalue { i32 } %X, i32 42, 0 // by switching the order of the insert and extract (though the // insertvalue should be left in, since it may have other uses). - Value *NewEV = Builder->CreateExtractValue(IV->getAggregateOperand(), - EV.getIndices()); + Value *NewEV = Builder.CreateExtractValue(IV->getAggregateOperand(), + EV.getIndices()); return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(), makeArrayRef(insi, inse)); } @@ -2415,17 +2415,17 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) { // extractvalue has integer indices, getelementptr has Value*s. Convert. SmallVector Indices; // Prefix an i32 0 since we need the first element. - Indices.push_back(Builder->getInt32(0)); + Indices.push_back(Builder.getInt32(0)); for (ExtractValueInst::idx_iterator I = EV.idx_begin(), E = EV.idx_end(); I != E; ++I) - Indices.push_back(Builder->getInt32(*I)); + Indices.push_back(Builder.getInt32(*I)); // We need to insert these at the location of the old load, not at that of // the extractvalue. - Builder->SetInsertPoint(L); - Value *GEP = Builder->CreateInBoundsGEP(L->getType(), - L->getPointerOperand(), Indices); - Instruction *NL = Builder->CreateLoad(GEP); + Builder.SetInsertPoint(L); + Value *GEP = Builder.CreateInBoundsGEP(L->getType(), + L->getPointerOperand(), Indices); + Instruction *NL = Builder.CreateLoad(GEP); // Whatever aliasing information we had for the orignal load must also // hold for the smaller load, so propagate the annotations. AAMDNodes Nodes; @@ -2922,8 +2922,8 @@ bool InstCombiner::run() { } // Now that we have an instruction, try combining it to simplify it. - Builder->SetInsertPoint(I); - Builder->SetCurrentDebugLocation(I->getDebugLoc()); + Builder.SetInsertPoint(I); + Builder.SetCurrentDebugLocation(I->getDebugLoc()); #ifndef NDEBUG std::string OrigI; @@ -3160,7 +3160,7 @@ combineInstructionsOverFunction(Function &F, InstCombineWorklist &Worklist, MadeIRChange |= prepareICWorklistFromFunction(F, DL, &TLI, Worklist); - InstCombiner IC(Worklist, &Builder, F.optForMinSize(), ExpensiveCombines, + InstCombiner IC(Worklist, Builder, F.optForMinSize(), ExpensiveCombines, AA, AC, TLI, DT, DL, LI); IC.MaxArraySizeForCombine = MaxArraySize; diff --git a/contrib/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/contrib/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index 7eea44d6aca0..184940b7ea58 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/contrib/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -1230,7 +1230,7 @@ static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, if (auto *Vector = dyn_cast(Mask)) { // dyn_cast as we might get UndefValue if (auto *Masked = dyn_cast(Vector->getOperand(Idx))) { - if (Masked->isNullValue()) + if (Masked->isZero()) // Mask is constant false, so no instrumentation needed. continue; // If we have a true or undef value, fall through to doInstrumentAddress diff --git a/contrib/llvm/lib/Transforms/Instrumentation/CFGMST.h b/contrib/llvm/lib/Transforms/Instrumentation/CFGMST.h index 3802f9fbf7db..16e2e6b4e730 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/CFGMST.h +++ b/contrib/llvm/lib/Transforms/Instrumentation/CFGMST.h @@ -12,6 +12,9 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_LIB_TRANSFORMS_INSTRUMENTATION_CFGMST_H +#define LLVM_LIB_TRANSFORMS_INSTRUMENTATION_CFGMST_H + #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/BlockFrequencyInfo.h" @@ -24,10 +27,10 @@ #include #include -namespace llvm { - #define DEBUG_TYPE "cfgmst" +namespace llvm { + /// \brief An union-find based Minimum Spanning Tree for CFG /// /// Implements a Union-find algorithm to compute Minimum Spanning Tree @@ -220,5 +223,8 @@ template class CFGMST { } }; -#undef DEBUG_TYPE // "cfgmst" } // end namespace llvm + +#undef DEBUG_TYPE // "cfgmst" + +#endif // LLVM_LIB_TRANSFORMS_INSTRUMENTATION_CFGMST_H diff --git a/contrib/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/contrib/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 9c14b0149fdc..db8fa8977947 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/contrib/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -112,7 +112,7 @@ cl::opt DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore, cl::desc("Do counter register promotion"), cl::init(false)); cl::opt MaxNumOfPromotionsPerLoop( - cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(10), + cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20), cl::desc("Max number counter promotions per loop to avoid" " increasing register pressure too much")); @@ -121,10 +121,21 @@ cl::opt MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1), cl::desc("Max number of allowed counter promotions")); -cl::opt SpeculativeCounterPromotion( - cl::ZeroOrMore, "speculative-counter-promotion", cl::init(false), - cl::desc("Allow counter promotion for loops with multiple exiting blocks " - " or top-tested loops. ")); +cl::opt SpeculativeCounterPromotionMaxExiting( + cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3), + cl::desc("The max number of exiting blocks of a loop to allow " + " speculative counter promotion")); + +cl::opt SpeculativeCounterPromotionToLoop( + cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false), + cl::desc("When the option is false, if the target block is in a loop, " + "the promotion will be disallowed unless the promoted counter " + " update can be further/iteratively promoted into an acyclic " + " region.")); + +cl::opt IterativeCounterPromotion( + cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true), + cl::desc("Allow counter promotion across the whole loop nest.")); class InstrProfilingLegacyPass : public ModulePass { InstrProfiling InstrProf; @@ -150,6 +161,7 @@ class InstrProfilingLegacyPass : public ModulePass { } }; +/// /// A helper class to promote one counter RMW operation in the loop /// into register update. /// @@ -158,16 +170,19 @@ class InstrProfilingLegacyPass : public ModulePass { /// class PGOCounterPromoterHelper : public LoadAndStorePromoter { public: - PGOCounterPromoterHelper(Instruction *L, Instruction *S, SSAUpdater &SSA, - Value *Init, BasicBlock *PH, - ArrayRef ExitBlocks, - ArrayRef InsertPts) + PGOCounterPromoterHelper( + Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init, + BasicBlock *PH, ArrayRef ExitBlocks, + ArrayRef InsertPts, + DenseMap> &LoopToCands, + LoopInfo &LI) : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks), - InsertPts(InsertPts) { + InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) { assert(isa(L)); assert(isa(S)); SSA.AddAvailableValue(PH, Init); } + void doExtraRewritesBeforeFinalDeletion() const override { for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { BasicBlock *ExitBlock = ExitBlocks[i]; @@ -179,12 +194,21 @@ class PGOCounterPromoterHelper : public LoadAndStorePromoter { Value *Addr = cast(Store)->getPointerOperand(); IRBuilder<> Builder(InsertPos); if (AtomicCounterUpdatePromoted) + // automic update currently can only be promoted across the current + // loop, not the whole loop nest. Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue, AtomicOrdering::SequentiallyConsistent); else { LoadInst *OldVal = Builder.CreateLoad(Addr, "pgocount.promoted"); auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue); - Builder.CreateStore(NewVal, Addr); + auto *NewStore = Builder.CreateStore(NewVal, Addr); + + // Now update the parent loop's candidate list: + if (IterativeCounterPromotion) { + auto *TargetLoop = LI.getLoopFor(ExitBlock); + if (TargetLoop) + LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore); + } } } } @@ -193,6 +217,8 @@ class PGOCounterPromoterHelper : public LoadAndStorePromoter { Instruction *Store; ArrayRef ExitBlocks; ArrayRef InsertPts; + DenseMap> &LoopToCandidates; + LoopInfo &LI; }; /// A helper class to do register promotion for all profile counter @@ -200,12 +226,15 @@ class PGOCounterPromoterHelper : public LoadAndStorePromoter { /// class PGOCounterPromoter { public: - PGOCounterPromoter(ArrayRef Cands, Loop &Loop) - : Candidates(Cands), ExitBlocks(), InsertPts(), ParentLoop(Loop) { + PGOCounterPromoter( + DenseMap> &LoopToCands, + Loop &CurLoop, LoopInfo &LI) + : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop), + LI(LI) { SmallVector LoopExitBlocks; SmallPtrSet BlockSet; - ParentLoop.getExitBlocks(LoopExitBlocks); + L.getExitBlocks(LoopExitBlocks); for (BasicBlock *ExitBlock : LoopExitBlocks) { if (BlockSet.insert(ExitBlock).second) { @@ -216,55 +245,97 @@ class PGOCounterPromoter { } bool run(int64_t *NumPromoted) { - // We can't insert into a catchswitch. - bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) { - return isa(Exit->getTerminator()); - }); - - if (HasCatchSwitch) - return false; - - if (!ParentLoop.hasDedicatedExits()) - return false; - - BasicBlock *PH = ParentLoop.getLoopPreheader(); - if (!PH) - return false; - - BasicBlock *H = ParentLoop.getHeader(); - bool TopTested = - ((ParentLoop.getBlocks().size() > 1) && ParentLoop.isLoopExiting(H)); - if (!SpeculativeCounterPromotion && - (TopTested || ParentLoop.getExitingBlock() == nullptr)) + unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L); + if (MaxProm == 0) return false; unsigned Promoted = 0; - for (auto &Cand : Candidates) { + for (auto &Cand : LoopToCandidates[&L]) { SmallVector NewPHIs; SSAUpdater SSA(&NewPHIs); Value *InitVal = ConstantInt::get(Cand.first->getType(), 0); + PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, - PH, ExitBlocks, InsertPts); + L.getLoopPreheader(), ExitBlocks, + InsertPts, LoopToCandidates, LI); Promoter.run(SmallVector({Cand.first, Cand.second})); Promoted++; - if (Promoted >= MaxNumOfPromotionsPerLoop) + if (Promoted >= MaxProm) break; + (*NumPromoted)++; if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions) break; } DEBUG(dbgs() << Promoted << " counters promoted for loop (depth=" - << ParentLoop.getLoopDepth() << ")\n"); + << L.getLoopDepth() << ")\n"); return Promoted != 0; } private: - ArrayRef Candidates; + bool allowSpeculativeCounterPromotion(Loop *LP) { + SmallVector ExitingBlocks; + L.getExitingBlocks(ExitingBlocks); + // Not considierered speculative. + if (ExitingBlocks.size() == 1) + return true; + if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) + return false; + return true; + } + + // Returns the max number of Counter Promotions for LP. + unsigned getMaxNumOfPromotionsInLoop(Loop *LP) { + // We can't insert into a catchswitch. + SmallVector LoopExitBlocks; + LP->getExitBlocks(LoopExitBlocks); + if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) { + return isa(Exit->getTerminator()); + })) + return 0; + + if (!LP->hasDedicatedExits()) + return 0; + + BasicBlock *PH = LP->getLoopPreheader(); + if (!PH) + return 0; + + SmallVector ExitingBlocks; + LP->getExitingBlocks(ExitingBlocks); + // Not considierered speculative. + if (ExitingBlocks.size() == 1) + return MaxNumOfPromotionsPerLoop; + + if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) + return 0; + + // Whether the target block is in a loop does not matter: + if (SpeculativeCounterPromotionToLoop) + return MaxNumOfPromotionsPerLoop; + + // Now check the target block: + unsigned MaxProm = MaxNumOfPromotionsPerLoop; + for (auto *TargetBlock : LoopExitBlocks) { + auto *TargetLoop = LI.getLoopFor(TargetBlock); + if (!TargetLoop) + continue; + unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop); + unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size(); + MaxProm = + std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) - + PendingCandsInTarget); + } + return MaxProm; + } + + DenseMap> &LoopToCandidates; SmallVector ExitBlocks; SmallVector InsertPts; - Loop &ParentLoop; + Loop &L; + LoopInfo &LI; }; } // end anonymous namespace @@ -349,8 +420,10 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) { SmallVector Loops = LI.getLoopsInPreorder(); - for (auto *Loop : Loops) { - PGOCounterPromoter Promoter(LoopPromotionCandidates[Loop], *Loop); + // Do a post-order traversal of the loops so that counter updates can be + // iteratively hoisted outside the loop nest. + for (auto *Loop : llvm::reverse(Loops)) { + PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI); Promoter.run(&TotalCountersPromoted); } } diff --git a/contrib/llvm/lib/Transforms/Instrumentation/MaximumSpanningTree.h b/contrib/llvm/lib/Transforms/Instrumentation/MaximumSpanningTree.h index 363539b2886f..4eb758c69c58 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/MaximumSpanningTree.h +++ b/contrib/llvm/lib/Transforms/Instrumentation/MaximumSpanningTree.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_ANALYSIS_MAXIMUMSPANNINGTREE_H -#define LLVM_ANALYSIS_MAXIMUMSPANNINGTREE_H +#ifndef LLVM_LIB_TRANSFORMS_INSTRUMENTATION_MAXIMUMSPANNINGTREE_H +#define LLVM_LIB_TRANSFORMS_INSTRUMENTATION_MAXIMUMSPANNINGTREE_H #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/IR/BasicBlock.h" @@ -108,4 +108,4 @@ namespace llvm { } // End llvm namespace -#endif +#endif // LLVM_LIB_TRANSFORMS_INSTRUMENTATION_MAXIMUMSPANNINGTREE_H diff --git a/contrib/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/contrib/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index df4ee9969c02..1348e0ed0ed0 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/contrib/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2918,8 +2918,11 @@ struct MemorySanitizerVisitor : public InstVisitor { if (ClDumpStrictInstructions) dumpInst(I); DEBUG(dbgs() << "DEFAULT: " << I << "\n"); - for (size_t i = 0, n = I.getNumOperands(); i < n; i++) - insertShadowCheck(I.getOperand(i), &I); + for (size_t i = 0, n = I.getNumOperands(); i < n; i++) { + Value *Operand = I.getOperand(i); + if (Operand->getType()->isSized()) + insertShadowCheck(Operand, &I); + } setShadow(&I, getCleanShadow(&I)); setOrigin(&I, getCleanOrigin()); } diff --git a/contrib/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/contrib/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 0e7d11c55397..8e4bfc0b91bc 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/contrib/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -224,7 +224,7 @@ std::string getBranchCondString(Instruction *TI) { OS << "_Zero"; else if (CV->isOne()) OS << "_One"; - else if (CV->isAllOnesValue()) + else if (CV->isMinusOne()) OS << "_MinusOne"; else OS << "_Const"; diff --git a/contrib/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/contrib/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp index a991792bf5a3..ec6904486e10 100644 --- a/contrib/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp +++ b/contrib/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp @@ -379,10 +379,11 @@ void ThreadSanitizer::chooseInstructionsToInstrument( } static bool isAtomic(Instruction *I) { + // TODO: Ask TTI whether synchronization scope is between threads. if (LoadInst *LI = dyn_cast(I)) - return LI->isAtomic() && LI->getSynchScope() == CrossThread; + return LI->isAtomic() && LI->getSyncScopeID() != SyncScope::SingleThread; if (StoreInst *SI = dyn_cast(I)) - return SI->isAtomic() && SI->getSynchScope() == CrossThread; + return SI->isAtomic() && SI->getSyncScopeID() != SyncScope::SingleThread; if (isa(I)) return true; if (isa(I)) @@ -676,7 +677,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { I->eraseFromParent(); } else if (FenceInst *FI = dyn_cast(I)) { Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; - Function *F = FI->getSynchScope() == SingleThread ? + Function *F = FI->getSyncScopeID() == SyncScope::SingleThread ? TsanAtomicSignalFence : TsanAtomicThreadFence; CallInst *C = CallInst::Create(F, Args); ReplaceInstWithInst(I, C); diff --git a/contrib/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/contrib/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp index a49c9b68c97d..122c9314e022 100644 --- a/contrib/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp @@ -44,6 +44,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Utils/Local.h" #include using namespace llvm; @@ -55,7 +56,7 @@ STATISTIC(NumConstantsHoisted, "Number of constants hoisted"); STATISTIC(NumConstantsRebased, "Number of constants rebased"); static cl::opt ConstHoistWithBlockFrequency( - "consthoist-with-block-frequency", cl::init(false), cl::Hidden, + "consthoist-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to reduce the " "chance to execute const materialization more frequently than " "without hoisting.")); @@ -231,7 +232,8 @@ static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI, // Return the optimal insert points in BBs. if (Node == Entry) { BBs.clear(); - if (InsertPtsFreq > BFI.getBlockFreq(Node)) + if (InsertPtsFreq > BFI.getBlockFreq(Node) || + (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1)) BBs.insert(Entry); else BBs.insert(InsertPts.begin(), InsertPts.end()); @@ -244,7 +246,15 @@ static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI, SmallPtrSet &ParentInsertPts = InsertPtsMap[Parent].first; BlockFrequency &ParentPtsFreq = InsertPtsMap[Parent].second; // Choose to insert in Node or in subtree of Node. - if (InsertPtsFreq > BFI.getBlockFreq(Node) || NodeInBBs) { + // Don't hoist to EHPad because we may not find a proper place to insert + // in EHPad. + // If the total frequency of InsertPts is the same as the frequency of the + // target Node, and InsertPts contains more than one nodes, choose hoisting + // to reduce code size. + if (NodeInBBs || + (!Node->isEHPad() && + (InsertPtsFreq > BFI.getBlockFreq(Node) || + (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1)))) { ParentInsertPts.insert(Node); ParentPtsFreq += BFI.getBlockFreq(Node); } else { @@ -392,42 +402,15 @@ void ConstantHoistingPass::collectConstantCandidates( if (Inst->isCast()) return; - // Can't handle inline asm. Skip it. - if (auto Call = dyn_cast(Inst)) - if (isa(Call->getCalledValue())) - return; - - // Switch cases must remain constant, and if the value being tested is - // constant the entire thing should disappear. - if (isa(Inst)) - return; - - // Static allocas (constant size in the entry block) are handled by - // prologue/epilogue insertion so they're free anyway. We definitely don't - // want to make them non-constant. - auto AI = dyn_cast(Inst); - if (AI && AI->isStaticAlloca()) - return; - - // Constants in GEPs that index into a struct type should not be hoisted. - if (isa(Inst)) { - gep_type_iterator GTI = gep_type_begin(Inst); - - // Collect constant for first operand. - collectConstantCandidates(ConstCandMap, Inst, 0); - // Scan rest operands. - for (unsigned Idx = 1, E = Inst->getNumOperands(); Idx != E; ++Idx, ++GTI) { - // Only collect constants that index into a non struct type. - if (!GTI.isStruct()) { - collectConstantCandidates(ConstCandMap, Inst, Idx); - } - } - return; - } - // Scan all operands. for (unsigned Idx = 0, E = Inst->getNumOperands(); Idx != E; ++Idx) { - collectConstantCandidates(ConstCandMap, Inst, Idx); + // The cost of materializing the constants (defined in + // `TargetTransformInfo::getIntImmCost`) for instructions which only take + // constant variables is lower than `TargetTransformInfo::TCC_Basic`. So + // it's safe for us to collect constant candidates from all IntrinsicInsts. + if (canReplaceOperandWithVariable(Inst, Idx) || isa(Inst)) { + collectConstantCandidates(ConstCandMap, Inst, Idx); + } } // end of for all operands } diff --git a/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp index 0f92760a874b..7fd77a082b82 100644 --- a/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/EarlyCSE.cpp @@ -670,7 +670,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) { if (auto *KnownCond = AvailableValues.lookup(CondI)) { // Is the condition known to be true? if (isa(KnownCond) && - cast(KnownCond)->isOneValue()) { + cast(KnownCond)->isOne()) { DEBUG(dbgs() << "EarlyCSE removing guard: " << *Inst << '\n'); removeMSSA(Inst); Inst->eraseFromParent(); diff --git a/contrib/llvm/lib/Transforms/Scalar/GVN.cpp b/contrib/llvm/lib/Transforms/Scalar/GVN.cpp index c0f628eb61e6..0fe72f3f7331 100644 --- a/contrib/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/GVN.cpp @@ -80,10 +80,9 @@ MaxRecurseDepth("max-recurse-depth", cl::Hidden, cl::init(1000), cl::ZeroOrMore, struct llvm::GVN::Expression { uint32_t opcode; Type *type; - bool commutative; SmallVector varargs; - Expression(uint32_t o = ~2U) : opcode(o), commutative(false) {} + Expression(uint32_t o = ~2U) : opcode(o) {} bool operator==(const Expression &other) const { if (opcode != other.opcode) @@ -247,7 +246,6 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) { assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!"); if (e.varargs[0] > e.varargs[1]) std::swap(e.varargs[0], e.varargs[1]); - e.commutative = true; } if (CmpInst *C = dyn_cast(I)) { @@ -258,7 +256,6 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) { Predicate = CmpInst::getSwappedPredicate(Predicate); } e.opcode = (C->getOpcode() << 8) | Predicate; - e.commutative = true; } else if (InsertValueInst *E = dyn_cast(I)) { for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end(); II != IE; ++II) @@ -284,7 +281,6 @@ GVN::Expression GVN::ValueTable::createCmpExpr(unsigned Opcode, Predicate = CmpInst::getSwappedPredicate(Predicate); } e.opcode = (Opcode << 8) | Predicate; - e.commutative = true; return e; } @@ -352,25 +348,25 @@ GVN::ValueTable::~ValueTable() = default; /// add - Insert a value into the table with a specified value number. void GVN::ValueTable::add(Value *V, uint32_t num) { valueNumbering.insert(std::make_pair(V, num)); - if (PHINode *PN = dyn_cast(V)) - NumberingPhi[num] = PN; } uint32_t GVN::ValueTable::lookupOrAddCall(CallInst *C) { if (AA->doesNotAccessMemory(C)) { Expression exp = createExpr(C); - uint32_t e = assignExpNewValueNum(exp).first; + uint32_t &e = expressionNumbering[exp]; + if (!e) e = nextValueNumber++; valueNumbering[C] = e; return e; } else if (AA->onlyReadsMemory(C)) { Expression exp = createExpr(C); - auto ValNum = assignExpNewValueNum(exp); - if (ValNum.second) { - valueNumbering[C] = ValNum.first; - return ValNum.first; + uint32_t &e = expressionNumbering[exp]; + if (!e) { + e = nextValueNumber++; + valueNumbering[C] = e; + return e; } if (!MD) { - uint32_t e = assignExpNewValueNum(exp).first; + e = nextValueNumber++; valueNumbering[C] = e; return e; } @@ -526,29 +522,23 @@ uint32_t GVN::ValueTable::lookupOrAdd(Value *V) { case Instruction::ExtractValue: exp = createExtractvalueExpr(cast(I)); break; - case Instruction::PHI: - valueNumbering[V] = nextValueNumber; - NumberingPhi[nextValueNumber] = cast(V); - return nextValueNumber++; default: valueNumbering[V] = nextValueNumber; return nextValueNumber++; } - uint32_t e = assignExpNewValueNum(exp).first; + uint32_t& e = expressionNumbering[exp]; + if (!e) e = nextValueNumber++; valueNumbering[V] = e; return e; } /// Returns the value number of the specified value. Fails if /// the value has not yet been numbered. -uint32_t GVN::ValueTable::lookup(Value *V, bool Verify) const { +uint32_t GVN::ValueTable::lookup(Value *V) const { DenseMap::const_iterator VI = valueNumbering.find(V); - if (Verify) { - assert(VI != valueNumbering.end() && "Value not numbered?"); - return VI->second; - } - return (VI != valueNumbering.end()) ? VI->second : 0; + assert(VI != valueNumbering.end() && "Value not numbered?"); + return VI->second; } /// Returns the value number of the given comparison, @@ -559,28 +549,21 @@ uint32_t GVN::ValueTable::lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Predicate, Value *LHS, Value *RHS) { Expression exp = createCmpExpr(Opcode, Predicate, LHS, RHS); - return assignExpNewValueNum(exp).first; + uint32_t& e = expressionNumbering[exp]; + if (!e) e = nextValueNumber++; + return e; } /// Remove all entries from the ValueTable. void GVN::ValueTable::clear() { valueNumbering.clear(); expressionNumbering.clear(); - NumberingPhi.clear(); - PhiTranslateTable.clear(); nextValueNumber = 1; - Expressions.clear(); - ExprIdx.clear(); - nextExprNumber = 0; } /// Remove a value from the value numbering. void GVN::ValueTable::erase(Value *V) { - uint32_t Num = valueNumbering.lookup(V); valueNumbering.erase(V); - // If V is PHINode, V <--> value number is an one-to-one mapping. - if (isa(V)) - NumberingPhi.erase(Num); } /// verifyRemoved - Verify that the value is removed from all internal data @@ -1183,7 +1166,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock, auto *NewLoad = new LoadInst(LoadPtr, LI->getName()+".pre", LI->isVolatile(), LI->getAlignment(), - LI->getOrdering(), LI->getSynchScope(), + LI->getOrdering(), LI->getSyncScopeID(), UnavailablePred->getTerminator()); // Transfer the old load's AA tags to the new load. @@ -1219,7 +1202,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock, V->takeName(LI); if (Instruction *I = dyn_cast(V)) I->setDebugLoc(LI->getDebugLoc()); - if (V->getType()->getScalarType()->isPointerTy()) + if (V->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(V); markInstructionForDeletion(LI); ORE->emit(OptimizationRemark(DEBUG_TYPE, "LoadPRE", LI) @@ -1306,7 +1289,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) { // to propagate LI's DebugLoc because LI may not post-dominate I. if (LI->getDebugLoc() && LI->getParent() == I->getParent()) I->setDebugLoc(LI->getDebugLoc()); - if (V->getType()->getScalarType()->isPointerTy()) + if (V->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(V); markInstructionForDeletion(LI); ++NumGVNLoad; @@ -1460,7 +1443,7 @@ bool GVN::processLoad(LoadInst *L) { reportLoadElim(L, AvailableValue, ORE); // Tell MDA to rexamine the reused pointer since we might have more // information after forwarding it. - if (MD && AvailableValue->getType()->getScalarType()->isPointerTy()) + if (MD && AvailableValue->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(AvailableValue); return true; } @@ -1468,95 +1451,6 @@ bool GVN::processLoad(LoadInst *L) { return false; } -/// Return a pair the first field showing the value number of \p Exp and the -/// second field showing whether it is a value number newly created. -std::pair -GVN::ValueTable::assignExpNewValueNum(Expression &Exp) { - uint32_t &e = expressionNumbering[Exp]; - bool CreateNewValNum = !e; - if (CreateNewValNum) { - Expressions.push_back(Exp); - if (ExprIdx.size() < nextValueNumber + 1) - ExprIdx.resize(nextValueNumber * 2); - e = nextValueNumber; - ExprIdx[nextValueNumber++] = nextExprNumber++; - } - return {e, CreateNewValNum}; -} - -/// Return whether all the values related with the same \p num are -/// defined in \p BB. -bool GVN::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB, - GVN &Gvn) { - LeaderTableEntry *Vals = &Gvn.LeaderTable[Num]; - while (Vals && Vals->BB == BB) - Vals = Vals->Next; - return !Vals; -} - -/// Wrap phiTranslateImpl to provide caching functionality. -uint32_t GVN::ValueTable::phiTranslate(const BasicBlock *Pred, - const BasicBlock *PhiBlock, uint32_t Num, - GVN &Gvn) { - auto FindRes = PhiTranslateTable.find({Num, Pred}); - if (FindRes != PhiTranslateTable.end()) - return FindRes->second; - uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, Gvn); - PhiTranslateTable.insert({{Num, Pred}, NewNum}); - return NewNum; -} - -/// Translate value number \p Num using phis, so that it has the values of -/// the phis in BB. -uint32_t GVN::ValueTable::phiTranslateImpl(const BasicBlock *Pred, - const BasicBlock *PhiBlock, - uint32_t Num, GVN &Gvn) { - if (PHINode *PN = NumberingPhi[Num]) { - for (unsigned i = 0; i != PN->getNumIncomingValues(); ++i) { - if (PN->getParent() == PhiBlock && PN->getIncomingBlock(i) == Pred) - if (uint32_t TransVal = lookup(PN->getIncomingValue(i), false)) - return TransVal; - } - return Num; - } - - // If there is any value related with Num is defined in a BB other than - // PhiBlock, it cannot depend on a phi in PhiBlock without going through - // a backedge. We can do an early exit in that case to save compile time. - if (!areAllValsInBB(Num, PhiBlock, Gvn)) - return Num; - - if (Num >= ExprIdx.size() || ExprIdx[Num] == 0) - return Num; - Expression Exp = Expressions[ExprIdx[Num]]; - - for (unsigned i = 0; i < Exp.varargs.size(); i++) { - // For InsertValue and ExtractValue, some varargs are index numbers - // instead of value numbers. Those index numbers should not be - // translated. - if ((i > 1 && Exp.opcode == Instruction::InsertValue) || - (i > 0 && Exp.opcode == Instruction::ExtractValue)) - continue; - Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn); - } - - if (Exp.commutative) { - assert(Exp.varargs.size() == 2 && "Unsupported commutative expression!"); - if (Exp.varargs[0] > Exp.varargs[1]) { - std::swap(Exp.varargs[0], Exp.varargs[1]); - uint32_t Opcode = Exp.opcode >> 8; - if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) - Exp.opcode = (Opcode << 8) | - CmpInst::getSwappedPredicate( - static_cast(Exp.opcode & 255)); - } - } - - if (uint32_t NewNum = expressionNumbering[Exp]) - return NewNum; - return Num; -} - // In order to find a leader for a given value number at a // specific basic block, we first obtain the list of all Values for that number, // and then scan the list to find one whose block dominates the block in @@ -1601,15 +1495,6 @@ static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E, return Pred != nullptr; } - -void GVN::assignBlockRPONumber(Function &F) { - uint32_t NextBlockNumber = 1; - ReversePostOrderTraversal RPOT(&F); - for (BasicBlock *BB : RPOT) - BlockRPONumber[BB] = NextBlockNumber++; -} - - // Tries to replace instruction with const, using information from // ReplaceWithConstMap. bool GVN::replaceOperandsWithConsts(Instruction *Instr) const { @@ -1713,7 +1598,7 @@ bool GVN::propagateEquality(Value *LHS, Value *RHS, const BasicBlockEdge &Root, // RHS neither 'true' nor 'false' - bail out. continue; // Whether RHS equals 'true'. Otherwise it equals 'false'. - bool isKnownTrue = CI->isAllOnesValue(); + bool isKnownTrue = CI->isMinusOne(); bool isKnownFalse = !isKnownTrue; // If "A && B" is known true then both A and B are known true. If "A || B" @@ -1813,7 +1698,7 @@ bool GVN::processInstruction(Instruction *I) { Changed = true; } if (Changed) { - if (MD && V->getType()->getScalarType()->isPointerTy()) + if (MD && V->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(V); ++NumGVNSimpl; return true; @@ -1924,7 +1809,7 @@ bool GVN::processInstruction(Instruction *I) { // Remove it! patchAndReplaceAllUsesWith(I, Repl); - if (MD && Repl->getType()->getScalarType()->isPointerTy()) + if (MD && Repl->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(Repl); markInstructionForDeletion(I); return true; @@ -1971,7 +1856,6 @@ bool GVN::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, // Fabricate val-num for dead-code in order to suppress assertion in // performPRE(). assignValNumForDeadCode(); - assignBlockRPONumber(F); bool PREChanged = true; while (PREChanged) { PREChanged = performPRE(F); @@ -2043,7 +1927,7 @@ bool GVN::processBlock(BasicBlock *BB) { // Instantiate an expression in a predecessor that lacked it. bool GVN::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, - BasicBlock *Curr, unsigned int ValNo) { + unsigned int ValNo) { // Because we are going top-down through the block, all value numbers // will be available in the predecessor by the time we need them. Any // that weren't originally present will have been instantiated earlier @@ -2061,9 +1945,7 @@ bool GVN::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, success = false; break; } - uint32_t TValNo = - VN.phiTranslate(Pred, Curr, VN.lookup(Op), *this); - if (Value *V = findLeader(Pred, TValNo)) { + if (Value *V = findLeader(Pred, VN.lookup(Op))) { Instr->setOperand(i, V); } else { success = false; @@ -2080,12 +1962,10 @@ bool GVN::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, Instr->insertBefore(Pred->getTerminator()); Instr->setName(Instr->getName() + ".pre"); Instr->setDebugLoc(Instr->getDebugLoc()); - - unsigned Num = VN.lookupOrAdd(Instr); - VN.add(Instr, Num); + VN.add(Instr, ValNo); // Update the availability map to include the new instruction. - addToLeaderTable(Num, Instr, Pred); + addToLeaderTable(ValNo, Instr, Pred); return true; } @@ -2123,27 +2003,18 @@ bool GVN::performScalarPRE(Instruction *CurInst) { SmallVector, 8> predMap; for (BasicBlock *P : predecessors(CurrentBlock)) { - // We're not interested in PRE where blocks with predecessors that are - // not reachable. - if (!DT->isReachableFromEntry(P)) { + // We're not interested in PRE where the block is its + // own predecessor, or in blocks with predecessors + // that are not reachable. + if (P == CurrentBlock) { NumWithout = 2; break; - } - // It is not safe to do PRE when P->CurrentBlock is a loop backedge, and - // when CurInst has operand defined in CurrentBlock (so it may be defined - // by phi in the loop header). - if (BlockRPONumber[P] >= BlockRPONumber[CurrentBlock] && - any_of(CurInst->operands(), [&](const Use &U) { - if (auto *Inst = dyn_cast(U.get())) - return Inst->getParent() == CurrentBlock; - return false; - })) { + } else if (!DT->isReachableFromEntry(P)) { NumWithout = 2; break; } - uint32_t TValNo = VN.phiTranslate(P, CurrentBlock, ValNo, *this); - Value *predV = findLeader(P, TValNo); + Value *predV = findLeader(P, ValNo); if (!predV) { predMap.push_back(std::make_pair(static_cast(nullptr), P)); PREPred = P; @@ -2183,7 +2054,7 @@ bool GVN::performScalarPRE(Instruction *CurInst) { } // We need to insert somewhere, so let's give it a shot PREInstr = CurInst->clone(); - if (!performScalarPREInsertion(PREInstr, PREPred, CurrentBlock, ValNo)) { + if (!performScalarPREInsertion(PREInstr, PREPred, ValNo)) { // If we failed insertion, make sure we remove the instruction. DEBUG(verifyRemoved(PREInstr)); PREInstr->deleteValue(); @@ -2212,7 +2083,7 @@ bool GVN::performScalarPRE(Instruction *CurInst) { addToLeaderTable(ValNo, Phi, CurrentBlock); Phi->setDebugLoc(CurInst->getDebugLoc()); CurInst->replaceAllUsesWith(Phi); - if (MD && Phi->getType()->getScalarType()->isPointerTy()) + if (MD && Phi->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(Phi); VN.erase(CurInst); removeFromLeaderTable(ValNo, CurInst, CurrentBlock); @@ -2297,7 +2168,6 @@ bool GVN::iterateOnFunction(Function &F) { void GVN::cleanupGlobalSets() { VN.clear(); LeaderTable.clear(); - BlockRPONumber.clear(); TableAllocator.Reset(); } diff --git a/contrib/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/contrib/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp index 3c8fbd35bf8c..89b28f0aeee6 100644 --- a/contrib/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp @@ -232,7 +232,7 @@ bool InferAddressSpaces::rewriteIntrinsicOperands(IntrinsicInst *II, case Intrinsic::amdgcn_atomic_inc: case Intrinsic::amdgcn_atomic_dec:{ const ConstantInt *IsVolatile = dyn_cast(II->getArgOperand(4)); - if (!IsVolatile || !IsVolatile->isNullValue()) + if (!IsVolatile || !IsVolatile->isZero()) return false; LLVM_FALLTHROUGH; @@ -358,7 +358,8 @@ InferAddressSpaces::collectFlatAddressExpressions(Function &F) const { // If the operands of the expression on the top are already explored, // adds that expression to the resultant postorder. if (PostorderStack.back().second) { - Postorder.push_back(TopVal); + if (TopVal->getType()->getPointerAddressSpace() == FlatAddrSpace) + Postorder.push_back(TopVal); PostorderStack.pop_back(); continue; } diff --git a/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp index 05293eb0079f..ee3de51b1360 100644 --- a/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -1212,7 +1212,7 @@ bool JumpThreadingPass::SimplifyPartiallyRedundantLoad(LoadInst *LI) { LoadInst *NewVal = new LoadInst( LoadedPtr->DoPHITranslation(LoadBB, UnavailablePred), LI->getName() + ".pr", false, LI->getAlignment(), LI->getOrdering(), - LI->getSynchScope(), UnavailablePred->getTerminator()); + LI->getSyncScopeID(), UnavailablePred->getTerminator()); NewVal->setDebugLoc(LI->getDebugLoc()); if (AATags) NewVal->setAAMetadata(AATags); diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopDeletion.cpp index c41cc42db5e2..ac4dd44a0e90 100644 --- a/contrib/llvm/lib/Transforms/Scalar/LoopDeletion.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/LoopDeletion.cpp @@ -148,25 +148,27 @@ static bool deleteLoopIfDead(Loop *L, DominatorTree &DT, ScalarEvolution &SE, LoopInfo &LI, LPMUpdater *Updater = nullptr) { assert(L->isLCSSAForm(DT) && "Expected LCSSA!"); - // We can only remove the loop if there is a preheader that we can - // branch from after removing it. + // We can only remove the loop if there is a preheader that we can branch from + // after removing it. Also, if LoopSimplify form is not available, stay out + // of trouble. BasicBlock *Preheader = L->getLoopPreheader(); - if (!Preheader) + if (!Preheader || !L->hasDedicatedExits()) { + DEBUG(dbgs() + << "Deletion requires Loop with preheader and dedicated exits.\n"); return false; - - // If LoopSimplify form is not available, stay out of trouble. - if (!L->hasDedicatedExits()) - return false; - + } // We can't remove loops that contain subloops. If the subloops were dead, // they would already have been removed in earlier executions of this pass. - if (L->begin() != L->end()) + if (L->begin() != L->end()) { + DEBUG(dbgs() << "Loop contains subloops.\n"); return false; + } BasicBlock *ExitBlock = L->getUniqueExitBlock(); if (ExitBlock && isLoopNeverExecuted(L)) { + DEBUG(dbgs() << "Loop is proven to never execute, delete it!"); // Set incoming value to undef for phi nodes in the exit block. BasicBlock::iterator BI = ExitBlock->begin(); while (PHINode *P = dyn_cast(BI)) { @@ -188,20 +190,26 @@ static bool deleteLoopIfDead(Loop *L, DominatorTree &DT, ScalarEvolution &SE, // be in the situation of needing to be able to solve statically which exit // block will be branched to, or trying to preserve the branching logic in // a loop invariant manner. - if (!ExitBlock) + if (!ExitBlock) { + DEBUG(dbgs() << "Deletion requires single exit block\n"); return false; - + } // Finally, we have to check that the loop really is dead. bool Changed = false; - if (!isLoopDead(L, SE, ExitingBlocks, ExitBlock, Changed, Preheader)) + if (!isLoopDead(L, SE, ExitingBlocks, ExitBlock, Changed, Preheader)) { + DEBUG(dbgs() << "Loop is not invariant, cannot delete.\n"); return Changed; + } // Don't remove loops for which we can't solve the trip count. // They could be infinite, in which case we'd be changing program behavior. const SCEV *S = SE.getMaxBackedgeTakenCount(L); - if (isa(S)) + if (isa(S)) { + DEBUG(dbgs() << "Could not compute SCEV MaxBackedgeTakenCount.\n"); return Changed; + } + DEBUG(dbgs() << "Loop is invariant, delete it!"); deleteDeadLoop(L, DT, SE, LI, Updater); ++NumDeleted; @@ -311,6 +319,9 @@ static void deleteDeadLoop(Loop *L, DominatorTree &DT, ScalarEvolution &SE, PreservedAnalyses LoopDeletionPass::run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &Updater) { + + DEBUG(dbgs() << "Analyzing Loop for deletion: "); + DEBUG(L.dump()); if (!deleteLoopIfDead(&L, AR.DT, AR.SE, AR.LI, &Updater)) return PreservedAnalyses::all(); @@ -350,5 +361,7 @@ bool LoopDeletionLegacyPass::runOnLoop(Loop *L, LPPassManager &) { ScalarEvolution &SE = getAnalysis().getSE(); LoopInfo &LI = getAnalysis().getLoopInfo(); + DEBUG(dbgs() << "Analyzing Loop for deletion: "); + DEBUG(L->dump()); return deleteLoopIfDead(L, DT, SE, LI); } diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp index 8b435050ac76..4a6a35c0ab1b 100644 --- a/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp @@ -1160,7 +1160,7 @@ static bool detectPopcountIdiom(Loop *CurLoop, BasicBlock *PreCondBB, if (!Dec || !((SubInst->getOpcode() == Instruction::Sub && Dec->isOne()) || (SubInst->getOpcode() == Instruction::Add && - Dec->isAllOnesValue()))) { + Dec->isMinusOne()))) { return false; } } diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index 9f3875a3027f..606136dc31a4 100644 --- a/contrib/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -757,8 +757,11 @@ bool LoopInterchangeLegality::currentLimitations() { PHINode *InnerInductionVar; SmallVector Inductions; SmallVector Reductions; - if (!findInductionAndReductions(InnerLoop, Inductions, Reductions)) + if (!findInductionAndReductions(InnerLoop, Inductions, Reductions)) { + DEBUG(dbgs() << "Only inner loops with induction or reduction PHI nodes " + << "are supported currently.\n"); return true; + } // TODO: Currently we handle only loops with 1 induction variable. if (Inductions.size() != 1) { @@ -771,16 +774,25 @@ bool LoopInterchangeLegality::currentLimitations() { InnerInductionVar = Inductions.pop_back_val(); Reductions.clear(); - if (!findInductionAndReductions(OuterLoop, Inductions, Reductions)) + if (!findInductionAndReductions(OuterLoop, Inductions, Reductions)) { + DEBUG(dbgs() << "Only outer loops with induction or reduction PHI nodes " + << "are supported currently.\n"); return true; + } // Outer loop cannot have reduction because then loops will not be tightly // nested. - if (!Reductions.empty()) + if (!Reductions.empty()) { + DEBUG(dbgs() << "Outer loops with reductions are not supported " + << "currently.\n"); return true; + } // TODO: Currently we handle only loops with 1 induction variable. - if (Inductions.size() != 1) + if (Inductions.size() != 1) { + DEBUG(dbgs() << "Loops with more than 1 induction variables are not " + << "supported currently.\n"); return true; + } // TODO: Triangular loops are not handled for now. if (!isLoopStructureUnderstood(InnerInductionVar)) { @@ -791,12 +803,16 @@ bool LoopInterchangeLegality::currentLimitations() { // TODO: We only handle LCSSA PHI's corresponding to reduction for now. BasicBlock *LoopExitBlock = getLoopLatchExitBlock(OuterLoopLatch, OuterLoopHeader); - if (!LoopExitBlock || !containsSafePHI(LoopExitBlock, true)) + if (!LoopExitBlock || !containsSafePHI(LoopExitBlock, true)) { + DEBUG(dbgs() << "Can only handle LCSSA PHIs in outer loops currently.\n"); return true; + } LoopExitBlock = getLoopLatchExitBlock(InnerLoopLatch, InnerLoopHeader); - if (!LoopExitBlock || !containsSafePHI(LoopExitBlock, false)) + if (!LoopExitBlock || !containsSafePHI(LoopExitBlock, false)) { + DEBUG(dbgs() << "Can only handle LCSSA PHIs in inner loops currently.\n"); return true; + } // TODO: Current limitation: Since we split the inner loop latch at the point // were induction variable is incremented (induction.next); We cannot have @@ -816,8 +832,11 @@ bool LoopInterchangeLegality::currentLimitations() { InnerIndexVarInc = dyn_cast(InnerInductionVar->getIncomingValue(0)); - if (!InnerIndexVarInc) + if (!InnerIndexVarInc) { + DEBUG(dbgs() << "Did not find an instruction to increment the induction " + << "variable.\n"); return true; + } // Since we split the inner loop latch on this induction variable. Make sure // we do not have any instruction between the induction variable and branch @@ -827,19 +846,24 @@ bool LoopInterchangeLegality::currentLimitations() { for (const Instruction &I : reverse(*InnerLoopLatch)) { if (isa(I) || isa(I) || isa(I)) continue; + // We found an instruction. If this is not induction variable then it is not // safe to split this loop latch. - if (!I.isIdenticalTo(InnerIndexVarInc)) + if (!I.isIdenticalTo(InnerIndexVarInc)) { + DEBUG(dbgs() << "Found unsupported instructions between induction " + << "variable increment and branch.\n"); return true; + } FoundInduction = true; break; } // The loop latch ended and we didn't find the induction variable return as // current limitation. - if (!FoundInduction) + if (!FoundInduction) { + DEBUG(dbgs() << "Did not find the induction variable.\n"); return true; - + } return false; } diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopRotation.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopRotation.cpp index 7312d97f8efe..3506ac343d59 100644 --- a/contrib/llvm/lib/Transforms/Scalar/LoopRotation.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/LoopRotation.cpp @@ -485,10 +485,22 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { DomTreeNode *Node = HeaderChildren[I]; BasicBlock *BB = Node->getBlock(); - pred_iterator PI = pred_begin(BB); - BasicBlock *NearestDom = *PI; - for (pred_iterator PE = pred_end(BB); PI != PE; ++PI) - NearestDom = DT->findNearestCommonDominator(NearestDom, *PI); + BasicBlock *NearestDom = nullptr; + for (BasicBlock *Pred : predecessors(BB)) { + // Consider only reachable basic blocks. + if (!DT->getNode(Pred)) + continue; + + if (!NearestDom) { + NearestDom = Pred; + continue; + } + + NearestDom = DT->findNearestCommonDominator(NearestDom, Pred); + assert(NearestDom && "No NearestCommonDominator found"); + } + + assert(NearestDom && "Nearest dominator not found"); // Remember if this changes the DomTree. if (Node->getIDom()->getBlock() != NearestDom) { diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 73436f13c94e..3638da118cb7 100644 --- a/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -140,6 +140,13 @@ static cl::opt LSRExpNarrow( cl::desc("Narrow LSR complex solution using" " expectation of registers number")); +// Flag to narrow search space by filtering non-optimal formulae with +// the same ScaledReg and Scale. +static cl::opt FilterSameScaledReg( + "lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), + cl::desc("Narrow LSR search space by filtering non-optimal formulae" + " with the same ScaledReg and Scale")); + #ifndef NDEBUG // Stress test IV chain generation. static cl::opt StressIVChain( @@ -1902,6 +1909,7 @@ class LSRInstance { void NarrowSearchSpaceByDetectingSupersets(); void NarrowSearchSpaceByCollapsingUnrolledCode(); void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); + void NarrowSearchSpaceByFilterFormulaWithSameScaledReg(); void NarrowSearchSpaceByDeletingCostlyFormulas(); void NarrowSearchSpaceByPickingWinnerRegs(); void NarrowSearchSpaceUsingHeuristics(); @@ -2318,7 +2326,7 @@ LSRInstance::OptimizeLoopTermCond() { dyn_cast_or_null(getExactSDiv(B, A, SE))) { const ConstantInt *C = D->getValue(); // Stride of one or negative one can have reuse with non-addresses. - if (C->isOne() || C->isAllOnesValue()) + if (C->isOne() || C->isMinusOne()) goto decline_post_inc; // Avoid weird situations. if (C->getValue().getMinSignedBits() >= 64 || @@ -4306,6 +4314,104 @@ void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){ } } +/// If a LSRUse has multiple formulae with the same ScaledReg and Scale. +/// Pick the best one and delete the others. +/// This narrowing heuristic is to keep as many formulae with different +/// Scale and ScaledReg pair as possible while narrowing the search space. +/// The benefit is that it is more likely to find out a better solution +/// from a formulae set with more Scale and ScaledReg variations than +/// a formulae set with the same Scale and ScaledReg. The picking winner +/// reg heurstic will often keep the formulae with the same Scale and +/// ScaledReg and filter others, and we want to avoid that if possible. +void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() { + if (EstimateSearchSpaceComplexity() < ComplexityLimit) + return; + + DEBUG(dbgs() << "The search space is too complex.\n" + "Narrowing the search space by choosing the best Formula " + "from the Formulae with the same Scale and ScaledReg.\n"); + + // Map the "Scale * ScaledReg" pair to the best formula of current LSRUse. + typedef DenseMap, size_t> BestFormulaeTy; + BestFormulaeTy BestFormulae; +#ifndef NDEBUG + bool ChangedFormulae = false; +#endif + DenseSet VisitedRegs; + SmallPtrSet Regs; + + for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { + LSRUse &LU = Uses[LUIdx]; + DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n'); + + // Return true if Formula FA is better than Formula FB. + auto IsBetterThan = [&](Formula &FA, Formula &FB) { + // First we will try to choose the Formula with fewer new registers. + // For a register used by current Formula, the more the register is + // shared among LSRUses, the less we increase the register number + // counter of the formula. + size_t FARegNum = 0; + for (const SCEV *Reg : FA.BaseRegs) { + const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg); + FARegNum += (NumUses - UsedByIndices.count() + 1); + } + size_t FBRegNum = 0; + for (const SCEV *Reg : FB.BaseRegs) { + const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg); + FBRegNum += (NumUses - UsedByIndices.count() + 1); + } + if (FARegNum != FBRegNum) + return FARegNum < FBRegNum; + + // If the new register numbers are the same, choose the Formula with + // less Cost. + Cost CostFA, CostFB; + Regs.clear(); + CostFA.RateFormula(TTI, FA, Regs, VisitedRegs, L, SE, DT, LU); + Regs.clear(); + CostFB.RateFormula(TTI, FB, Regs, VisitedRegs, L, SE, DT, LU); + return CostFA.isLess(CostFB, TTI); + }; + + bool Any = false; + for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms; + ++FIdx) { + Formula &F = LU.Formulae[FIdx]; + if (!F.ScaledReg) + continue; + auto P = BestFormulae.insert({{F.ScaledReg, F.Scale}, FIdx}); + if (P.second) + continue; + + Formula &Best = LU.Formulae[P.first->second]; + if (IsBetterThan(F, Best)) + std::swap(F, Best); + DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs()); + dbgs() << "\n" + " in favor of formula "; + Best.print(dbgs()); dbgs() << '\n'); +#ifndef NDEBUG + ChangedFormulae = true; +#endif + LU.DeleteFormula(F); + --FIdx; + --NumForms; + Any = true; + } + if (Any) + LU.RecomputeRegs(LUIdx, RegUses); + + // Reset this to prepare for the next use. + BestFormulae.clear(); + } + + DEBUG(if (ChangedFormulae) { + dbgs() << "\n" + "After filtering out undesirable candidates:\n"; + print_uses(dbgs()); + }); +} + /// The function delete formulas with high registers number expectation. /// Assuming we don't know the value of each formula (already delete /// all inefficient), generate probability of not selecting for each @@ -4516,6 +4622,8 @@ void LSRInstance::NarrowSearchSpaceUsingHeuristics() { NarrowSearchSpaceByDetectingSupersets(); NarrowSearchSpaceByCollapsingUnrolledCode(); NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); + if (FilterSameScaledReg) + NarrowSearchSpaceByFilterFormulaWithSameScaledReg(); if (LSRExpNarrow) NarrowSearchSpaceByDeletingCostlyFormulas(); else diff --git a/contrib/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp b/contrib/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp index acd3ef6791be..6727cf0179c1 100644 --- a/contrib/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp @@ -238,7 +238,7 @@ PHINode *MergedLoadStoreMotion::getPHIOperand(BasicBlock *BB, StoreInst *S0, &BB->front()); NewPN->addIncoming(Opd1, S0->getParent()); NewPN->addIncoming(Opd2, S1->getParent()); - if (MD && NewPN->getType()->getScalarType()->isPointerTy()) + if (MD && NewPN->getType()->isPtrOrPtrVectorTy()) MD->invalidateCachedPointerInfo(NewPN); return NewPN; } diff --git a/contrib/llvm/lib/Transforms/Scalar/NewGVN.cpp b/contrib/llvm/lib/Transforms/Scalar/NewGVN.cpp index 9cf01c6582b5..9d018563618e 100644 --- a/contrib/llvm/lib/Transforms/Scalar/NewGVN.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/NewGVN.cpp @@ -866,9 +866,7 @@ PHIExpression *NewGVN::createPHIExpression(Instruction *I, bool &HasBackedge, // Things in TOPClass are equivalent to everything. if (ValueToClass.lookup(*U) == TOPClass) return false; - if (lookupOperandLeader(*U) == PN) - return false; - return true; + return lookupOperandLeader(*U) != PN; }); std::transform(Filtered.begin(), Filtered.end(), op_inserter(E), [&](const Use *U) -> Value * { @@ -2063,9 +2061,10 @@ Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const { // // The invariants of this function are: // -// I must be moving to NewClass from OldClass The StoreCount of OldClass and -// NewClass is expected to have been updated for I already if it is is a store. -// The OldClass memory leader has not been updated yet if I was the leader. +// - I must be moving to NewClass from OldClass +// - The StoreCount of OldClass and NewClass is expected to have been updated +// for I already if it is is a store. +// - The OldClass memory leader has not been updated yet if I was the leader. void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I, MemoryAccess *InstMA, CongruenceClass *OldClass, @@ -2074,7 +2073,8 @@ void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I, // be the MemoryAccess of OldClass. assert((!InstMA || !OldClass->getMemoryLeader() || OldClass->getLeader() != I || - OldClass->getMemoryLeader() == InstMA) && + MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) == + MemoryAccessToClass.lookup(InstMA)) && "Representative MemoryAccess mismatch"); // First, see what happens to the new class if (!NewClass->getMemoryLeader()) { @@ -2136,7 +2136,7 @@ void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E, << NewClass->getID() << " from " << *NewClass->getLeader() << " to " << *SI << " because store joined class\n"); // If we changed the leader, we have to mark it changed because we don't - // know what it will do to symbolic evlauation. + // know what it will do to symbolic evaluation. NewClass->setLeader(SI); } // We rely on the code below handling the MemoryAccess change. diff --git a/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp b/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp index cdba0062953f..29d1ba406ae4 100644 --- a/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/Reassociate.cpp @@ -2148,7 +2148,7 @@ void ReassociatePass::ReassociateExpression(BinaryOperator *I) { if (I->getOpcode() == Instruction::Mul && cast(I->user_back())->getOpcode() == Instruction::Add && isa(Ops.back().Op) && - cast(Ops.back().Op)->isAllOnesValue()) { + cast(Ops.back().Op)->isMinusOne()) { ValueEntry Tmp = Ops.pop_back_val(); Ops.insert(Ops.begin(), Tmp); } else if (I->getOpcode() == Instruction::FMul && diff --git a/contrib/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/contrib/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index a73e9aec0617..f19d45329d23 100644 --- a/contrib/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -1994,7 +1994,7 @@ static void rematerializeLiveValues(CallSite CS, Instruction *LastClonedValue = nullptr; Instruction *LastValue = nullptr; for (Instruction *Instr: ChainToBase) { - // Only GEP's and casts are suported as we need to be careful to not + // Only GEP's and casts are supported as we need to be careful to not // introduce any new uses of pointers not in the liveset. // Note that it's fine to introduce new uses of pointers which were // otherwise not used after this statepoint. diff --git a/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp b/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp index 7a6fa1711411..a738ebb4607e 100644 --- a/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -963,7 +963,7 @@ void SCCPSolver::visitBinaryOperator(Instruction &I) { } else { // X or -1 = -1 if (ConstantInt *CI = NonOverdefVal->getConstantInt()) - if (CI->isAllOnesValue()) + if (CI->isMinusOne()) return markConstant(IV, &I, NonOverdefVal->getConstant()); } } diff --git a/contrib/llvm/lib/Transforms/Scalar/SROA.cpp b/contrib/llvm/lib/Transforms/Scalar/SROA.cpp index 4729f4ef5956..b9cee5b2ba95 100644 --- a/contrib/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/SROA.cpp @@ -1673,8 +1673,7 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V, // See if we need inttoptr for this type pair. A cast involving both scalars // and vectors requires and additional bitcast. - if (OldTy->getScalarType()->isIntegerTy() && - NewTy->getScalarType()->isPointerTy()) { + if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) { // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8* if (OldTy->isVectorTy() && !NewTy->isVectorTy()) return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), @@ -1690,8 +1689,7 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V, // See if we need ptrtoint for this type pair. A cast involving both scalars // and vectors requires and additional bitcast. - if (OldTy->getScalarType()->isPointerTy() && - NewTy->getScalarType()->isIntegerTy()) { + if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) { // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128 if (OldTy->isVectorTy() && !NewTy->isVectorTy()) return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), @@ -2400,7 +2398,7 @@ class llvm::sroa::AllocaSliceRewriter LoadInst *NewLI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), LI.isVolatile(), LI.getName()); if (LI.isVolatile()) - NewLI->setAtomic(LI.getOrdering(), LI.getSynchScope()); + NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); // Any !nonnull metadata or !range metadata on the old load is also valid // on the new load. This is even true in some cases even when the loads @@ -2435,7 +2433,7 @@ class llvm::sroa::AllocaSliceRewriter getSliceAlign(TargetTy), LI.isVolatile(), LI.getName()); if (LI.isVolatile()) - NewLI->setAtomic(LI.getOrdering(), LI.getSynchScope()); + NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); V = NewLI; IsPtrAdjusted = true; @@ -2578,7 +2576,7 @@ class llvm::sroa::AllocaSliceRewriter } NewSI->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access); if (SI.isVolatile()) - NewSI->setAtomic(SI.getOrdering(), SI.getSynchScope()); + NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); Pass.DeadInsts.insert(&SI); deleteIfTriviallyDead(OldOp); diff --git a/contrib/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/contrib/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp index 486f3e5a43d4..0cccb415efdb 100644 --- a/contrib/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp +++ b/contrib/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp @@ -329,7 +329,7 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) { Loops[Exit] = N->getEntry(); } else { - // Test for sucessors as back edge + // Test for successors as back edge BasicBlock *BB = N->getNodeAs(); BranchInst *Term = cast(BB->getTerminator()); diff --git a/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp b/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp index 314c990293cc..7e75e8847785 100644 --- a/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/contrib/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -46,13 +46,21 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, if (BB->hasName()) NewBB->setName(BB->getName()+NameSuffix); bool hasCalls = false, hasDynamicAllocas = false, hasStaticAllocas = false; - + Module *TheModule = F ? F->getParent() : nullptr; + // Loop over all instructions, and copy them over. for (BasicBlock::const_iterator II = BB->begin(), IE = BB->end(); II != IE; ++II) { - if (DIFinder && F->getParent() && II->getDebugLoc()) - DIFinder->processLocation(*F->getParent(), II->getDebugLoc().get()); + if (DIFinder && TheModule) { + if (auto *DDI = dyn_cast(II)) + DIFinder->processDeclare(*TheModule, DDI); + else if (auto *DVI = dyn_cast(II)) + DIFinder->processValue(*TheModule, DVI); + + if (auto DbgLoc = II->getDebugLoc()) + DIFinder->processLocation(*TheModule, DbgLoc.get()); + } Instruction *NewInst = II->clone(); if (II->hasName()) @@ -153,6 +161,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, // When we remap instructions, we want to avoid duplicating inlined // DISubprograms, so record all subprograms we find as we duplicate // instructions and then freeze them in the MD map. + // We also record information about dbg.value and dbg.declare to avoid + // duplicating the types. DebugInfoFinder DIFinder; // Loop over all of the basic blocks in the function, cloning them as @@ -193,6 +203,10 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, } } + for (auto *Type : DIFinder.types()) { + VMap.MD()[Type].reset(Type); + } + // Loop over all of the instructions in the function, fixing up operand // references as we go. This uses VMap to do all the hard work. for (Function::iterator BB = diff --git a/contrib/llvm/lib/Transforms/Utils/CmpInstAnalysis.cpp b/contrib/llvm/lib/Transforms/Utils/CmpInstAnalysis.cpp index 9f4d9c7e3981..d9294c499309 100644 --- a/contrib/llvm/lib/Transforms/Utils/CmpInstAnalysis.cpp +++ b/contrib/llvm/lib/Transforms/Utils/CmpInstAnalysis.cpp @@ -81,7 +81,7 @@ bool llvm::decomposeBitTestICmp(const ICmpInst *I, CmpInst::Predicate &Pred, break; case ICmpInst::ICMP_SGT: // X > -1 is equivalent to (X & SignMask) == 0. - if (!C->isAllOnesValue()) + if (!C->isMinusOne()) return false; Y = ConstantInt::get(I->getContext(), APInt::getSignMask(C->getBitWidth())); Pred = ICmpInst::ICMP_EQ; diff --git a/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 30d8856cfbef..1189714dfab1 100644 --- a/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/contrib/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -1116,12 +1116,6 @@ Function *CodeExtractor::extractCodeRegion() { } } - //cerr << "NEW FUNCTION: " << *newFunction; - // verifyFunction(*newFunction); - - // cerr << "OLD FUNCTION: " << *oldFunction; - // verifyFunction(*oldFunction); - DEBUG(if (verifyFunction(*newFunction)) report_fatal_error("verifyFunction failed!")); return newFunction; diff --git a/contrib/llvm/lib/Transforms/Utils/Evaluator.cpp b/contrib/llvm/lib/Transforms/Utils/Evaluator.cpp index c97e544e620a..1328f2f3ec01 100644 --- a/contrib/llvm/lib/Transforms/Utils/Evaluator.cpp +++ b/contrib/llvm/lib/Transforms/Utils/Evaluator.cpp @@ -402,7 +402,7 @@ bool Evaluator::EvaluateBlock(BasicBlock::iterator CurInst, Value *Ptr = PtrArg->stripPointerCasts(); if (GlobalVariable *GV = dyn_cast(Ptr)) { Type *ElemTy = GV->getValueType(); - if (!Size->isAllOnesValue() && + if (!Size->isMinusOne() && Size->getValue().getLimitedValue() >= DL.getTypeStoreSize(ElemTy)) { Invariants.insert(GV); diff --git a/contrib/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/contrib/llvm/lib/Transforms/Utils/FunctionComparator.cpp index 0457294361b5..4a2be3a53176 100644 --- a/contrib/llvm/lib/Transforms/Utils/FunctionComparator.cpp +++ b/contrib/llvm/lib/Transforms/Utils/FunctionComparator.cpp @@ -513,8 +513,8 @@ int FunctionComparator::cmpOperations(const Instruction *L, if (int Res = cmpOrderings(LI->getOrdering(), cast(R)->getOrdering())) return Res; - if (int Res = - cmpNumbers(LI->getSynchScope(), cast(R)->getSynchScope())) + if (int Res = cmpNumbers(LI->getSyncScopeID(), + cast(R)->getSyncScopeID())) return Res; return cmpRangeMetadata(LI->getMetadata(LLVMContext::MD_range), cast(R)->getMetadata(LLVMContext::MD_range)); @@ -529,7 +529,8 @@ int FunctionComparator::cmpOperations(const Instruction *L, if (int Res = cmpOrderings(SI->getOrdering(), cast(R)->getOrdering())) return Res; - return cmpNumbers(SI->getSynchScope(), cast(R)->getSynchScope()); + return cmpNumbers(SI->getSyncScopeID(), + cast(R)->getSyncScopeID()); } if (const CmpInst *CI = dyn_cast(L)) return cmpNumbers(CI->getPredicate(), cast(R)->getPredicate()); @@ -584,7 +585,8 @@ int FunctionComparator::cmpOperations(const Instruction *L, if (int Res = cmpOrderings(FI->getOrdering(), cast(R)->getOrdering())) return Res; - return cmpNumbers(FI->getSynchScope(), cast(R)->getSynchScope()); + return cmpNumbers(FI->getSyncScopeID(), + cast(R)->getSyncScopeID()); } if (const AtomicCmpXchgInst *CXI = dyn_cast(L)) { if (int Res = cmpNumbers(CXI->isVolatile(), @@ -601,8 +603,8 @@ int FunctionComparator::cmpOperations(const Instruction *L, cmpOrderings(CXI->getFailureOrdering(), cast(R)->getFailureOrdering())) return Res; - return cmpNumbers(CXI->getSynchScope(), - cast(R)->getSynchScope()); + return cmpNumbers(CXI->getSyncScopeID(), + cast(R)->getSyncScopeID()); } if (const AtomicRMWInst *RMWI = dyn_cast(L)) { if (int Res = cmpNumbers(RMWI->getOperation(), @@ -614,8 +616,8 @@ int FunctionComparator::cmpOperations(const Instruction *L, if (int Res = cmpOrderings(RMWI->getOrdering(), cast(R)->getOrdering())) return Res; - return cmpNumbers(RMWI->getSynchScope(), - cast(R)->getSynchScope()); + return cmpNumbers(RMWI->getSyncScopeID(), + cast(R)->getSyncScopeID()); } if (const PHINode *PNL = dyn_cast(L)) { const PHINode *PNR = cast(R); diff --git a/contrib/llvm/lib/Transforms/Utils/Local.cpp b/contrib/llvm/lib/Transforms/Utils/Local.cpp index 5127eba3f9ae..74610613001c 100644 --- a/contrib/llvm/lib/Transforms/Utils/Local.cpp +++ b/contrib/llvm/lib/Transforms/Utils/Local.cpp @@ -1662,9 +1662,10 @@ void llvm::removeUnwindEdge(BasicBlock *BB) { TI->eraseFromParent(); } -/// removeUnreachableBlocksFromFn - Remove blocks that are not reachable, even +/// removeUnreachableBlocks - Remove blocks that are not reachable, even /// if they are in a dead cycle. Return true if a change was made, false -/// otherwise. +/// otherwise. If `LVI` is passed, this function preserves LazyValueInfo +/// after modifying the CFG. bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI) { SmallPtrSet Reachable; bool Changed = markAliveBlocks(F, Reachable); @@ -2168,6 +2169,9 @@ bool llvm::canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx) { return true; case Instruction::Call: case Instruction::Invoke: + // Can't handle inline asm. Skip it. + if (isa(ImmutableCallSite(I).getCalledValue())) + return false; // Many arithmetic intrinsics have no issue taking a // variable, however it's hard to distingish these from // specials such as @llvm.frameaddress that require a constant. @@ -2182,12 +2186,18 @@ bool llvm::canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx) { case Instruction::ShuffleVector: // Shufflevector masks are constant. return OpIdx != 2; + case Instruction::Switch: case Instruction::ExtractValue: - case Instruction::InsertValue: // All operands apart from the first are constant. return OpIdx == 0; + case Instruction::InsertValue: + // All operands apart from the first and the second are constant. + return OpIdx < 2; case Instruction::Alloca: - return false; + // Static allocas (constant size in the entry block) are handled by + // prologue/epilogue insertion so they're free anyway. We definitely don't + // want to make them non-constant. + return !dyn_cast(I)->isStaticAlloca(); case Instruction::GetElementPtr: if (OpIdx == 0) return true; diff --git a/contrib/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/contrib/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp index 9ad2b707e6b2..5170c68e2915 100644 --- a/contrib/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/contrib/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -65,9 +65,11 @@ static cl::opt UnrollRuntimeMultiExit( /// than the unroll factor. /// static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, - BasicBlock *PrologExit, BasicBlock *PreHeader, - BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, - DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA) { + BasicBlock *PrologExit, + BasicBlock *OriginalLoopLatchExit, + BasicBlock *PreHeader, BasicBlock *NewPreHeader, + ValueToValueMapTy &VMap, DominatorTree *DT, + LoopInfo *LI, bool PreserveLCSSA) { BasicBlock *Latch = L->getLoopLatch(); assert(Latch && "Loop must have a latch"); BasicBlock *PrologLatch = cast(VMap[Latch]); @@ -142,17 +144,15 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, // then (BECount + 1) cannot unsigned-overflow. Value *BrLoopExit = B.CreateICmpULT(BECount, ConstantInt::get(BECount->getType(), Count - 1)); - BasicBlock *Exit = L->getUniqueExitBlock(); - assert(Exit && "Loop must have a single exit block only"); // Split the exit to maintain loop canonicalization guarantees - SmallVector Preds(predecessors(Exit)); - SplitBlockPredecessors(Exit, Preds, ".unr-lcssa", DT, LI, + SmallVector Preds(predecessors(OriginalLoopLatchExit)); + SplitBlockPredecessors(OriginalLoopLatchExit, Preds, ".unr-lcssa", DT, LI, PreserveLCSSA); // Add the branch to the exit block (around the unrolled loop) - B.CreateCondBr(BrLoopExit, Exit, NewPreHeader); + B.CreateCondBr(BrLoopExit, OriginalLoopLatchExit, NewPreHeader); InsertPt->eraseFromParent(); if (DT) - DT->changeImmediateDominator(Exit, PrologExit); + DT->changeImmediateDominator(OriginalLoopLatchExit, PrologExit); } /// Connect the unrolling epilog code to the original loop. @@ -427,6 +427,50 @@ CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop, return nullptr; } +/// Returns true if we can safely unroll a multi-exit/exiting loop. OtherExits +/// is populated with all the loop exit blocks other than the LatchExit block. +static bool +canSafelyUnrollMultiExitLoop(Loop *L, SmallVectorImpl &OtherExits, + BasicBlock *LatchExit, bool PreserveLCSSA, + bool UseEpilogRemainder) { + + // Support runtime unrolling for multiple exit blocks and multiple exiting + // blocks. + if (!UnrollRuntimeMultiExit) + return false; + // Even if runtime multi exit is enabled, we currently have some correctness + // constrains in unrolling a multi-exit loop. + // We rely on LCSSA form being preserved when the exit blocks are transformed. + if (!PreserveLCSSA) + return false; + SmallVector Exits; + L->getUniqueExitBlocks(Exits); + for (auto *BB : Exits) + if (BB != LatchExit) + OtherExits.push_back(BB); + + // TODO: Support multiple exiting blocks jumping to the `LatchExit` when + // UnrollRuntimeMultiExit is true. This will need updating the logic in + // connectEpilog/connectProlog. + if (!LatchExit->getSinglePredecessor()) { + DEBUG(dbgs() << "Bailout for multi-exit handling when latch exit has >1 " + "predecessor.\n"); + return false; + } + // FIXME: We bail out of multi-exit unrolling when epilog loop is generated + // and L is an inner loop. This is because in presence of multiple exits, the + // outer loop is incorrect: we do not add the EpilogPreheader and exit to the + // outer loop. This is automatically handled in the prolog case, so we do not + // have that bug in prolog generation. + if (UseEpilogRemainder && L->getParentLoop()) + return false; + + // All constraints have been satisfied. + return true; +} + + + /// Insert code in the prolog/epilog code when unrolling a loop with a /// run-time trip-count. /// @@ -470,53 +514,40 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool UseEpilogRemainder, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, bool PreserveLCSSA) { - // for now, only unroll loops that contain a single exit - if (!UnrollRuntimeMultiExit && !L->getExitingBlock()) - return false; + DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n"); + DEBUG(L->dump()); // Make sure the loop is in canonical form. - if (!L->isLoopSimplifyForm()) + if (!L->isLoopSimplifyForm()) { + DEBUG(dbgs() << "Not in simplify form!\n"); return false; + } // Guaranteed by LoopSimplifyForm. BasicBlock *Latch = L->getLoopLatch(); BasicBlock *Header = L->getHeader(); - BasicBlock *LatchExit = L->getUniqueExitBlock(); // successor out of loop - if (!LatchExit && !UnrollRuntimeMultiExit) - return false; - // These are exit blocks other than the target of the latch exiting block. - SmallVector OtherExits; BranchInst *LatchBR = cast(Latch->getTerminator()); - unsigned int ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0; + unsigned ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0; + BasicBlock *LatchExit = LatchBR->getSuccessor(ExitIndex); // Cloning the loop basic blocks (`CloneLoopBlocks`) requires that one of the // targets of the Latch be an exit block out of the loop. This needs // to be guaranteed by the callers of UnrollRuntimeLoopRemainder. - assert(!L->contains(LatchBR->getSuccessor(ExitIndex)) && + assert(!L->contains(LatchExit) && "one of the loop latch successors should be the exit block!"); - // Support runtime unrolling for multiple exit blocks and multiple exiting - // blocks. - if (!LatchExit) { - assert(UseEpilogRemainder && "Multi exit unrolling is currently supported " - "unrolling with epilog remainder only!"); - LatchExit = LatchBR->getSuccessor(ExitIndex); - // We rely on LCSSA form being preserved when the exit blocks are - // transformed. - if (!PreserveLCSSA) - return false; - // TODO: Support multiple exiting blocks jumping to the `LatchExit`. This - // will need updating the logic in connectEpilog. - if (!LatchExit->getSinglePredecessor()) - return false; - SmallVector Exits; - L->getUniqueExitBlocks(Exits); - for (auto *BB : Exits) - if (BB != LatchExit) - OtherExits.push_back(BB); + // These are exit blocks other than the target of the latch exiting block. + SmallVector OtherExits; + bool isMultiExitUnrollingEnabled = canSafelyUnrollMultiExitLoop( + L, OtherExits, LatchExit, PreserveLCSSA, UseEpilogRemainder); + // Support only single exit and exiting block unless multi-exit loop unrolling is enabled. + if (!isMultiExitUnrollingEnabled && + (!L->getExitingBlock() || OtherExits.size())) { + DEBUG( + dbgs() + << "Multiple exit/exiting blocks in loop and multi-exit unrolling not " + "enabled!\n"); + return false; } - - assert(LatchExit && "Latch Exit should exist!"); - // Use Scalar Evolution to compute the trip count. This allows more loops to // be unrolled than relying on induction var simplification. if (!SE) @@ -530,29 +561,38 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, // exiting blocks). const SCEV *BECountSC = SE->getExitCount(L, Latch); if (isa(BECountSC) || - !BECountSC->getType()->isIntegerTy()) + !BECountSC->getType()->isIntegerTy()) { + DEBUG(dbgs() << "Could not compute exit block SCEV\n"); return false; + } unsigned BEWidth = cast(BECountSC->getType())->getBitWidth(); // Add 1 since the backedge count doesn't include the first loop iteration. const SCEV *TripCountSC = SE->getAddExpr(BECountSC, SE->getConstant(BECountSC->getType(), 1)); - if (isa(TripCountSC)) + if (isa(TripCountSC)) { + DEBUG(dbgs() << "Could not compute trip count SCEV.\n"); return false; + } BasicBlock *PreHeader = L->getLoopPreheader(); BranchInst *PreHeaderBR = cast(PreHeader->getTerminator()); const DataLayout &DL = Header->getModule()->getDataLayout(); SCEVExpander Expander(*SE, DL, "loop-unroll"); if (!AllowExpensiveTripCount && - Expander.isHighCostExpansion(TripCountSC, L, PreHeaderBR)) + Expander.isHighCostExpansion(TripCountSC, L, PreHeaderBR)) { + DEBUG(dbgs() << "High cost for expanding trip count scev!\n"); return false; + } // This constraint lets us deal with an overflowing trip count easily; see the // comment on ModVal below. - if (Log2_32(Count) > BEWidth) + if (Log2_32(Count) > BEWidth) { + DEBUG(dbgs() + << "Count failed constraint on overflow trip count calculation.\n"); return false; + } // Loop structure is the following: // @@ -711,11 +751,10 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, // node. for (unsigned i =0; i < oldNumOperands; i++){ Value *newVal = VMap[Phi->getIncomingValue(i)]; - if (!newVal) { - assert(isa(Phi->getIncomingValue(i)) && - "VMap should exist for all values except constants!"); + // newVal can be a constant or derived from values outside the loop, and + // hence need not have a VMap value. + if (!newVal) newVal = Phi->getIncomingValue(i); - } Phi->addIncoming(newVal, cast(VMap[Phi->getIncomingBlock(i)])); } @@ -781,8 +820,8 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, } else { // Connect the prolog code to the original loop and update the // PHI functions. - ConnectProlog(L, BECount, Count, PrologExit, PreHeader, NewPreHeader, - VMap, DT, LI, PreserveLCSSA); + ConnectProlog(L, BECount, Count, PrologExit, LatchExit, PreHeader, + NewPreHeader, VMap, DT, LI, PreserveLCSSA); } // If this loop is nested, then the loop unroller changes the code in the diff --git a/contrib/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/contrib/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp index 1c2a60a6b8b2..900450b40061 100644 --- a/contrib/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp +++ b/contrib/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp @@ -8,12 +8,256 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" using namespace llvm; +static unsigned getLoopOperandSizeInBytes(Type *Type) { + if (VectorType *VTy = dyn_cast(Type)) { + return VTy->getBitWidth() / 8; + } + + return Type->getPrimitiveSizeInBits() / 8; +} + +void llvm::createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr, + Value *DstAddr, ConstantInt *CopyLen, + unsigned SrcAlign, unsigned DestAlign, + bool SrcIsVolatile, bool DstIsVolatile, + const TargetTransformInfo &TTI) { + // No need to expand zero length copies. + if (CopyLen->isZero()) + return; + + BasicBlock *PreLoopBB = InsertBefore->getParent(); + BasicBlock *PostLoopBB = nullptr; + Function *ParentFunc = PreLoopBB->getParent(); + LLVMContext &Ctx = PreLoopBB->getContext(); + + Type *TypeOfCopyLen = CopyLen->getType(); + Type *LoopOpType = + TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAlign, DestAlign); + + unsigned LoopOpSize = getLoopOperandSizeInBytes(LoopOpType); + uint64_t LoopEndCount = CopyLen->getZExtValue() / LoopOpSize; + + unsigned SrcAS = cast(SrcAddr->getType())->getAddressSpace(); + unsigned DstAS = cast(DstAddr->getType())->getAddressSpace(); + + if (LoopEndCount != 0) { + // Split + PostLoopBB = PreLoopBB->splitBasicBlock(InsertBefore, "memcpy-split"); + BasicBlock *LoopBB = + BasicBlock::Create(Ctx, "load-store-loop", ParentFunc, PostLoopBB); + PreLoopBB->getTerminator()->setSuccessor(0, LoopBB); + + IRBuilder<> PLBuilder(PreLoopBB->getTerminator()); + + // Cast the Src and Dst pointers to pointers to the loop operand type (if + // needed). + PointerType *SrcOpType = PointerType::get(LoopOpType, SrcAS); + PointerType *DstOpType = PointerType::get(LoopOpType, DstAS); + if (SrcAddr->getType() != SrcOpType) { + SrcAddr = PLBuilder.CreateBitCast(SrcAddr, SrcOpType); + } + if (DstAddr->getType() != DstOpType) { + DstAddr = PLBuilder.CreateBitCast(DstAddr, DstOpType); + } + + IRBuilder<> LoopBuilder(LoopBB); + PHINode *LoopIndex = LoopBuilder.CreatePHI(TypeOfCopyLen, 2, "loop-index"); + LoopIndex->addIncoming(ConstantInt::get(TypeOfCopyLen, 0U), PreLoopBB); + // Loop Body + Value *SrcGEP = + LoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, LoopIndex); + Value *Load = LoopBuilder.CreateLoad(SrcGEP, SrcIsVolatile); + Value *DstGEP = + LoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, LoopIndex); + LoopBuilder.CreateStore(Load, DstGEP, DstIsVolatile); + + Value *NewIndex = + LoopBuilder.CreateAdd(LoopIndex, ConstantInt::get(TypeOfCopyLen, 1U)); + LoopIndex->addIncoming(NewIndex, LoopBB); + + // Create the loop branch condition. + Constant *LoopEndCI = ConstantInt::get(TypeOfCopyLen, LoopEndCount); + LoopBuilder.CreateCondBr(LoopBuilder.CreateICmpULT(NewIndex, LoopEndCI), + LoopBB, PostLoopBB); + } + + uint64_t BytesCopied = LoopEndCount * LoopOpSize; + uint64_t RemainingBytes = CopyLen->getZExtValue() - BytesCopied; + if (RemainingBytes) { + IRBuilder<> RBuilder(PostLoopBB ? PostLoopBB->getFirstNonPHI() + : InsertBefore); + + // Update the alignment based on the copy size used in the loop body. + SrcAlign = std::min(SrcAlign, LoopOpSize); + DestAlign = std::min(DestAlign, LoopOpSize); + + SmallVector RemainingOps; + TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes, + SrcAlign, DestAlign); + + for (auto OpTy : RemainingOps) { + // Calaculate the new index + unsigned OperandSize = getLoopOperandSizeInBytes(OpTy); + uint64_t GepIndex = BytesCopied / OperandSize; + assert(GepIndex * OperandSize == BytesCopied && + "Division should have no Remainder!"); + // Cast source to operand type and load + PointerType *SrcPtrType = PointerType::get(OpTy, SrcAS); + Value *CastedSrc = SrcAddr->getType() == SrcPtrType + ? SrcAddr + : RBuilder.CreateBitCast(SrcAddr, SrcPtrType); + Value *SrcGEP = RBuilder.CreateInBoundsGEP( + OpTy, CastedSrc, ConstantInt::get(TypeOfCopyLen, GepIndex)); + Value *Load = RBuilder.CreateLoad(SrcGEP, SrcIsVolatile); + + // Cast destination to operand type and store. + PointerType *DstPtrType = PointerType::get(OpTy, DstAS); + Value *CastedDst = DstAddr->getType() == DstPtrType + ? DstAddr + : RBuilder.CreateBitCast(DstAddr, DstPtrType); + Value *DstGEP = RBuilder.CreateInBoundsGEP( + OpTy, CastedDst, ConstantInt::get(TypeOfCopyLen, GepIndex)); + RBuilder.CreateStore(Load, DstGEP, DstIsVolatile); + + BytesCopied += OperandSize; + } + } + assert(BytesCopied == CopyLen->getZExtValue() && + "Bytes copied should match size in the call!"); +} + +void llvm::createMemCpyLoopUnknownSize(Instruction *InsertBefore, + Value *SrcAddr, Value *DstAddr, + Value *CopyLen, unsigned SrcAlign, + unsigned DestAlign, bool SrcIsVolatile, + bool DstIsVolatile, + const TargetTransformInfo &TTI) { + BasicBlock *PreLoopBB = InsertBefore->getParent(); + BasicBlock *PostLoopBB = + PreLoopBB->splitBasicBlock(InsertBefore, "post-loop-memcpy-expansion"); + + Function *ParentFunc = PreLoopBB->getParent(); + LLVMContext &Ctx = PreLoopBB->getContext(); + + Type *LoopOpType = + TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAlign, DestAlign); + unsigned LoopOpSize = getLoopOperandSizeInBytes(LoopOpType); + + IRBuilder<> PLBuilder(PreLoopBB->getTerminator()); + + unsigned SrcAS = cast(SrcAddr->getType())->getAddressSpace(); + unsigned DstAS = cast(DstAddr->getType())->getAddressSpace(); + PointerType *SrcOpType = PointerType::get(LoopOpType, SrcAS); + PointerType *DstOpType = PointerType::get(LoopOpType, DstAS); + if (SrcAddr->getType() != SrcOpType) { + SrcAddr = PLBuilder.CreateBitCast(SrcAddr, SrcOpType); + } + if (DstAddr->getType() != DstOpType) { + DstAddr = PLBuilder.CreateBitCast(DstAddr, DstOpType); + } + + // Calculate the loop trip count, and remaining bytes to copy after the loop. + Type *CopyLenType = CopyLen->getType(); + IntegerType *ILengthType = dyn_cast(CopyLenType); + assert(ILengthType && + "expected size argument to memcpy to be an integer type!"); + ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize); + Value *RuntimeLoopCount = PLBuilder.CreateUDiv(CopyLen, CILoopOpSize); + Value *RuntimeResidual = PLBuilder.CreateURem(CopyLen, CILoopOpSize); + Value *RuntimeBytesCopied = PLBuilder.CreateSub(CopyLen, RuntimeResidual); + + BasicBlock *LoopBB = + BasicBlock::Create(Ctx, "loop-memcpy-expansion", ParentFunc, nullptr); + IRBuilder<> LoopBuilder(LoopBB); + + PHINode *LoopIndex = LoopBuilder.CreatePHI(CopyLenType, 2, "loop-index"); + LoopIndex->addIncoming(ConstantInt::get(CopyLenType, 0U), PreLoopBB); + + Value *SrcGEP = LoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, LoopIndex); + Value *Load = LoopBuilder.CreateLoad(SrcGEP, SrcIsVolatile); + Value *DstGEP = LoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, LoopIndex); + LoopBuilder.CreateStore(Load, DstGEP, DstIsVolatile); + + Value *NewIndex = + LoopBuilder.CreateAdd(LoopIndex, ConstantInt::get(CopyLenType, 1U)); + LoopIndex->addIncoming(NewIndex, LoopBB); + + Type *Int8Type = Type::getInt8Ty(Ctx); + if (LoopOpType != Int8Type) { + // Loop body for the residual copy. + BasicBlock *ResLoopBB = BasicBlock::Create(Ctx, "loop-memcpy-residual", + PreLoopBB->getParent(), nullptr); + // Residual loop header. + BasicBlock *ResHeaderBB = BasicBlock::Create( + Ctx, "loop-memcpy-residual-header", PreLoopBB->getParent(), nullptr); + + // Need to update the pre-loop basic block to branch to the correct place. + // branch to the main loop if the count is non-zero, branch to the residual + // loop if the copy size is smaller then 1 iteration of the main loop but + // non-zero and finally branch to after the residual loop if the memcpy + // size is zero. + ConstantInt *Zero = ConstantInt::get(ILengthType, 0U); + PLBuilder.CreateCondBr(PLBuilder.CreateICmpNE(RuntimeLoopCount, Zero), + LoopBB, ResHeaderBB); + PreLoopBB->getTerminator()->eraseFromParent(); + + LoopBuilder.CreateCondBr( + LoopBuilder.CreateICmpULT(NewIndex, RuntimeLoopCount), LoopBB, + ResHeaderBB); + + // Determine if we need to branch to the residual loop or bypass it. + IRBuilder<> RHBuilder(ResHeaderBB); + RHBuilder.CreateCondBr(RHBuilder.CreateICmpNE(RuntimeResidual, Zero), + ResLoopBB, PostLoopBB); + + // Copy the residual with single byte load/store loop. + IRBuilder<> ResBuilder(ResLoopBB); + PHINode *ResidualIndex = + ResBuilder.CreatePHI(CopyLenType, 2, "residual-loop-index"); + ResidualIndex->addIncoming(Zero, ResHeaderBB); + + Value *SrcAsInt8 = + ResBuilder.CreateBitCast(SrcAddr, PointerType::get(Int8Type, SrcAS)); + Value *DstAsInt8 = + ResBuilder.CreateBitCast(DstAddr, PointerType::get(Int8Type, DstAS)); + Value *FullOffset = ResBuilder.CreateAdd(RuntimeBytesCopied, ResidualIndex); + Value *SrcGEP = + ResBuilder.CreateInBoundsGEP(Int8Type, SrcAsInt8, FullOffset); + Value *Load = ResBuilder.CreateLoad(SrcGEP, SrcIsVolatile); + Value *DstGEP = + ResBuilder.CreateInBoundsGEP(Int8Type, DstAsInt8, FullOffset); + ResBuilder.CreateStore(Load, DstGEP, DstIsVolatile); + + Value *ResNewIndex = + ResBuilder.CreateAdd(ResidualIndex, ConstantInt::get(CopyLenType, 1U)); + ResidualIndex->addIncoming(ResNewIndex, ResLoopBB); + + // Create the loop branch condition. + ResBuilder.CreateCondBr( + ResBuilder.CreateICmpULT(ResNewIndex, RuntimeResidual), ResLoopBB, + PostLoopBB); + } else { + // In this case the loop operand type was a byte, and there is no need for a + // residual loop to copy the remaining memory after the main loop. + // We do however need to patch up the control flow by creating the + // terminators for the preloop block and the memcpy loop. + ConstantInt *Zero = ConstantInt::get(ILengthType, 0U); + PLBuilder.CreateCondBr(PLBuilder.CreateICmpNE(RuntimeLoopCount, Zero), + LoopBB, PostLoopBB); + PreLoopBB->getTerminator()->eraseFromParent(); + LoopBuilder.CreateCondBr( + LoopBuilder.CreateICmpULT(NewIndex, RuntimeLoopCount), LoopBB, + PostLoopBB); + } +} + void llvm::createMemCpyLoop(Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, Value *CopyLen, unsigned SrcAlign, unsigned DestAlign, @@ -208,15 +452,41 @@ static void createMemSetLoop(Instruction *InsertBefore, NewBB); } -void llvm::expandMemCpyAsLoop(MemCpyInst *Memcpy) { - createMemCpyLoop(/* InsertBefore */ Memcpy, - /* SrcAddr */ Memcpy->getRawSource(), - /* DstAddr */ Memcpy->getRawDest(), - /* CopyLen */ Memcpy->getLength(), - /* SrcAlign */ Memcpy->getAlignment(), - /* DestAlign */ Memcpy->getAlignment(), - /* SrcIsVolatile */ Memcpy->isVolatile(), - /* DstIsVolatile */ Memcpy->isVolatile()); +void llvm::expandMemCpyAsLoop(MemCpyInst *Memcpy, + const TargetTransformInfo &TTI) { + // Original implementation + if (!TTI.useWideIRMemcpyLoopLowering()) { + createMemCpyLoop(/* InsertBefore */ Memcpy, + /* SrcAddr */ Memcpy->getRawSource(), + /* DstAddr */ Memcpy->getRawDest(), + /* CopyLen */ Memcpy->getLength(), + /* SrcAlign */ Memcpy->getAlignment(), + /* DestAlign */ Memcpy->getAlignment(), + /* SrcIsVolatile */ Memcpy->isVolatile(), + /* DstIsVolatile */ Memcpy->isVolatile()); + } else { + if (ConstantInt *CI = dyn_cast(Memcpy->getLength())) { + createMemCpyLoopKnownSize(/* InsertBefore */ Memcpy, + /* SrcAddr */ Memcpy->getRawSource(), + /* DstAddr */ Memcpy->getRawDest(), + /* CopyLen */ CI, + /* SrcAlign */ Memcpy->getAlignment(), + /* DestAlign */ Memcpy->getAlignment(), + /* SrcIsVolatile */ Memcpy->isVolatile(), + /* DstIsVolatile */ Memcpy->isVolatile(), + /* TargetTransformInfo */ TTI); + } else { + createMemCpyLoopUnknownSize(/* InsertBefore */ Memcpy, + /* SrcAddr */ Memcpy->getRawSource(), + /* DstAddr */ Memcpy->getRawDest(), + /* CopyLen */ Memcpy->getLength(), + /* SrcAlign */ Memcpy->getAlignment(), + /* DestAlign */ Memcpy->getAlignment(), + /* SrcIsVolatile */ Memcpy->isVolatile(), + /* DstIsVolatile */ Memcpy->isVolatile(), + /* TargetTransfomrInfo */ TTI); + } + } } void llvm::expandMemMoveAsLoop(MemMoveInst *Memmove) { diff --git a/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index e724b0a28c32..dee658f98393 100644 --- a/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/contrib/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -5754,8 +5754,8 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) { if (BasicBlock *Dom = BB->getSinglePredecessor()) { auto *PBI = dyn_cast_or_null(Dom->getTerminator()); if (PBI && PBI->isConditional() && - PBI->getSuccessor(0) != PBI->getSuccessor(1) && - (PBI->getSuccessor(0) == BB || PBI->getSuccessor(1) == BB)) { + PBI->getSuccessor(0) != PBI->getSuccessor(1)) { + assert(PBI->getSuccessor(0) == BB || PBI->getSuccessor(1) == BB); bool CondIsFalse = PBI->getSuccessor(1) == BB; Optional Implication = isImpliedCondition( PBI->getCondition(), BI->getCondition(), DL, CondIsFalse); diff --git a/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index ec8b0d426265..6d90e6b48358 100644 --- a/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/contrib/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -25,6 +25,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -80,6 +81,7 @@ namespace { bool IsSigned); bool eliminateSDiv(BinaryOperator *SDiv); bool strengthenOverflowingOperation(BinaryOperator *OBO, Value *IVOperand); + bool strengthenRightShift(BinaryOperator *BO, Value *IVOperand); }; } @@ -154,6 +156,7 @@ Value *SimplifyIndvar::foldIVUser(Instruction *UseInst, Instruction *IVOperand) void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) { unsigned IVOperIdx = 0; ICmpInst::Predicate Pred = ICmp->getPredicate(); + ICmpInst::Predicate OriginalPred = Pred; if (IVOperand != ICmp->getOperand(0)) { // Swapped assert(IVOperand == ICmp->getOperand(1) && "Can't find IVOperand"); @@ -262,6 +265,16 @@ void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) { ICmp->setPredicate(InvariantPredicate); ICmp->setOperand(0, NewLHS); ICmp->setOperand(1, NewRHS); + } else if (ICmpInst::isSigned(OriginalPred) && + SE->isKnownNonNegative(S) && SE->isKnownNonNegative(X)) { + // If we were unable to make anything above, all we can is to canonicalize + // the comparison hoping that it will open the doors for other + // optimizations. If we find out that we compare two non-negative values, + // we turn the instruction's predicate to its unsigned version. Note that + // we cannot rely on Pred here unless we check if we have swapped it. + assert(ICmp->getPredicate() == OriginalPred && "Predicate changed?"); + DEBUG(dbgs() << "INDVARS: Turn to unsigned comparison: " << *ICmp << '\n'); + ICmp->setPredicate(ICmpInst::getUnsignedPredicate(OriginalPred)); } else return; @@ -583,6 +596,35 @@ bool SimplifyIndvar::strengthenOverflowingOperation(BinaryOperator *BO, return Changed; } +/// Annotate the Shr in (X << IVOperand) >> C as exact using the +/// information from the IV's range. Returns true if anything changed, false +/// otherwise. +bool SimplifyIndvar::strengthenRightShift(BinaryOperator *BO, + Value *IVOperand) { + using namespace llvm::PatternMatch; + + if (BO->getOpcode() == Instruction::Shl) { + bool Changed = false; + ConstantRange IVRange = SE->getUnsignedRange(SE->getSCEV(IVOperand)); + for (auto *U : BO->users()) { + const APInt *C; + if (match(U, + m_AShr(m_Shl(m_Value(), m_Specific(IVOperand)), m_APInt(C))) || + match(U, + m_LShr(m_Shl(m_Value(), m_Specific(IVOperand)), m_APInt(C)))) { + BinaryOperator *Shr = cast(U); + if (!Shr->isExact() && IVRange.getUnsignedMin().uge(*C)) { + Shr->setIsExact(true); + Changed = true; + } + } + } + return Changed; + } + + return false; +} + /// Add all uses of Def to the current IV's worklist. static void pushIVUsers( Instruction *Def, @@ -675,8 +717,9 @@ void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) { } if (BinaryOperator *BO = dyn_cast(UseOper.first)) { - if (isa(BO) && - strengthenOverflowingOperation(BO, IVOperand)) { + if ((isa(BO) && + strengthenOverflowingOperation(BO, IVOperand)) || + (isa(BO) && strengthenRightShift(BO, IVOperand))) { // re-queue uses of the now modified binary operator and fall // through to the checks that remain. pushIVUsers(IVOperand, Simplified, SimpleIVUsers); diff --git a/contrib/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/contrib/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp index b723b65f35e5..77c0a41929ac 100644 --- a/contrib/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp +++ b/contrib/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp @@ -656,7 +656,7 @@ Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilder<> &B) { ConstantInt *LenC = dyn_cast(CI->getArgOperand(2)); // memchr(x, y, 0) -> null - if (LenC && LenC->isNullValue()) + if (LenC && LenC->isZero()) return Constant::getNullValue(CI->getType()); // From now on we need at least constant length and string. @@ -2280,7 +2280,7 @@ bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(CallInst *CI, return true; if (ConstantInt *ObjSizeCI = dyn_cast(CI->getArgOperand(ObjSizeOp))) { - if (ObjSizeCI->isAllOnesValue()) + if (ObjSizeCI->isMinusOne()) return true; // If the object size wasn't -1 (unknown), bail out if we were asked to. if (OnlyLowerUnknownSize) diff --git a/contrib/llvm/lib/Transforms/Utils/VNCoercion.cpp b/contrib/llvm/lib/Transforms/Utils/VNCoercion.cpp index 60d9ede2c487..c3feea6a0a41 100644 --- a/contrib/llvm/lib/Transforms/Utils/VNCoercion.cpp +++ b/contrib/llvm/lib/Transforms/Utils/VNCoercion.cpp @@ -51,25 +51,24 @@ static T *coerceAvailableValueToLoadTypeHelper(T *StoredVal, Type *LoadedTy, // If the store and reload are the same size, we can always reuse it. if (StoredValSize == LoadedValSize) { // Pointer to Pointer -> use bitcast. - if (StoredValTy->getScalarType()->isPointerTy() && - LoadedTy->getScalarType()->isPointerTy()) { + if (StoredValTy->isPtrOrPtrVectorTy() && LoadedTy->isPtrOrPtrVectorTy()) { StoredVal = Helper.CreateBitCast(StoredVal, LoadedTy); } else { // Convert source pointers to integers, which can be bitcast. - if (StoredValTy->getScalarType()->isPointerTy()) { + if (StoredValTy->isPtrOrPtrVectorTy()) { StoredValTy = DL.getIntPtrType(StoredValTy); StoredVal = Helper.CreatePtrToInt(StoredVal, StoredValTy); } Type *TypeToCastTo = LoadedTy; - if (TypeToCastTo->getScalarType()->isPointerTy()) + if (TypeToCastTo->isPtrOrPtrVectorTy()) TypeToCastTo = DL.getIntPtrType(TypeToCastTo); if (StoredValTy != TypeToCastTo) StoredVal = Helper.CreateBitCast(StoredVal, TypeToCastTo); // Cast to pointer if the load needs a pointer type. - if (LoadedTy->getScalarType()->isPointerTy()) + if (LoadedTy->isPtrOrPtrVectorTy()) StoredVal = Helper.CreateIntToPtr(StoredVal, LoadedTy); } @@ -86,7 +85,7 @@ static T *coerceAvailableValueToLoadTypeHelper(T *StoredVal, Type *LoadedTy, "canCoerceMustAliasedValueToLoad fail"); // Convert source pointers to integers, which can be manipulated. - if (StoredValTy->getScalarType()->isPointerTy()) { + if (StoredValTy->isPtrOrPtrVectorTy()) { StoredValTy = DL.getIntPtrType(StoredValTy); StoredVal = Helper.CreatePtrToInt(StoredVal, StoredValTy); } @@ -112,7 +111,7 @@ static T *coerceAvailableValueToLoadTypeHelper(T *StoredVal, Type *LoadedTy, if (LoadedTy != NewIntTy) { // If the result is a pointer, inttoptr. - if (LoadedTy->getScalarType()->isPointerTy()) + if (LoadedTy->isPtrOrPtrVectorTy()) StoredVal = Helper.CreateIntToPtr(StoredVal, LoadedTy); else // Otherwise, bitcast. @@ -316,7 +315,7 @@ static T *getStoreValueForLoadHelper(T *SrcVal, unsigned Offset, Type *LoadTy, uint64_t LoadSize = (DL.getTypeSizeInBits(LoadTy) + 7) / 8; // Compute which bits of the stored value are being used by the load. Convert // to an integer type to start with. - if (SrcVal->getType()->getScalarType()->isPointerTy()) + if (SrcVal->getType()->isPtrOrPtrVectorTy()) SrcVal = Helper.CreatePtrToInt(SrcVal, DL.getIntPtrType(SrcVal->getType())); if (!SrcVal->getType()->isIntegerTy()) SrcVal = Helper.CreateBitCast(SrcVal, IntegerType::get(Ctx, StoreSize * 8)); diff --git a/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 193cc4d13787..eb82ee283d44 100644 --- a/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -5315,8 +5315,13 @@ void LoopVectorizationLegality::addInductionPhi( // Both the PHI node itself, and the "post-increment" value feeding // back into the PHI node may have external users. - AllowedExit.insert(Phi); - AllowedExit.insert(Phi->getIncomingValueForBlock(TheLoop->getLoopLatch())); + // We can allow those uses, except if the SCEVs we have for them rely + // on predicates that only hold within the loop, since allowing the exit + // currently means re-using this SCEV outside the loop. + if (PSE.getUnionPredicate().isAlwaysTrue()) { + AllowedExit.insert(Phi); + AllowedExit.insert(Phi->getIncomingValueForBlock(TheLoop->getLoopLatch())); + } DEBUG(dbgs() << "LV: Found an induction variable.\n"); return; diff --git a/contrib/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/contrib/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index b494526369d6..4425043ad39a 100644 --- a/contrib/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/contrib/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -860,7 +860,7 @@ class BoUpSLP { bool tryScheduleBundle(ArrayRef VL, BoUpSLP *SLP); /// Un-bundles a group of instructions. - void cancelScheduling(ArrayRef VL); + void cancelScheduling(ArrayRef VL, Value *OpValue); /// Extends the scheduling region so that V is inside the region. /// \returns true if the region size is within the limit. @@ -1258,7 +1258,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, cast(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i))); if (Term) { DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"); - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); return; } @@ -1284,7 +1284,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (Reuse) { DEBUG(dbgs() << "SLP: Reusing extract sequence.\n"); } else { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); } newTreeEntry(VL, Reuse, UserTreeIdx); return; @@ -1301,7 +1301,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (DL->getTypeSizeInBits(ScalarTy) != DL->getTypeAllocSizeInBits(ScalarTy)) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n"); return; @@ -1312,7 +1312,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) { LoadInst *L = cast(VL[i]); if (!L->isSimple()) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n"); return; @@ -1349,7 +1349,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, break; } - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); if (ReverseConsecutive) { @@ -1376,7 +1376,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, for (unsigned i = 0; i < VL.size(); ++i) { Type *Ty = cast(VL[i])->getOperand(0)->getType(); if (Ty != SrcTy || !isValidElementType(Ty)) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n"); return; @@ -1404,7 +1404,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, CmpInst *Cmp = cast(VL[i]); if (Cmp->getPredicate() != P0 || Cmp->getOperand(0)->getType() != ComparedTy) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n"); return; @@ -1471,7 +1471,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, for (unsigned j = 0; j < VL.size(); ++j) { if (cast(VL[j])->getNumOperands() != 2) { DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"); - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); return; } @@ -1484,7 +1484,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, Type *CurTy = cast(VL[j])->getOperand(0)->getType(); if (Ty0 != CurTy) { DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n"); - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); return; } @@ -1496,7 +1496,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (!isa(Op)) { DEBUG( dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"); - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); return; } @@ -1518,7 +1518,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // Check if the stores are consecutive or of we need to swizzle them. for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Non-consecutive store.\n"); return; @@ -1541,7 +1541,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // represented by an intrinsic call Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); if (!isTriviallyVectorizable(ID)) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Non-vectorizable call.\n"); return; @@ -1555,7 +1555,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (!CI2 || CI2->getCalledFunction() != Int || getVectorIntrinsicIDForCall(CI2, TLI) != ID || !CI->hasIdenticalOperandBundleSchema(*CI2)) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i] << "\n"); @@ -1566,7 +1566,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (hasVectorInstrinsicScalarOpd(ID, 1)) { Value *A1J = CI2->getArgOperand(1); if (A1I != A1J) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI << " argument "<< A1I<<"!=" << A1J @@ -1579,7 +1579,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(), CI->op_begin() + CI->getBundleOperandsEndIndex(), CI2->op_begin() + CI2->getBundleOperandsStartIndex())) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!=" << *VL[i] << '\n'); @@ -1603,7 +1603,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // If this is not an alternate sequence of opcode like add-sub // then do not vectorize this instruction. if (!isAltShuffle) { - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n"); return; @@ -1631,7 +1631,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, return; } default: - BS.cancelScheduling(VL); + BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx); DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n"); return; @@ -3177,17 +3177,18 @@ bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef VL, } } if (!Bundle->isReady()) { - cancelScheduling(VL); + cancelScheduling(VL, VL[0]); return false; } return true; } -void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef VL) { - if (isa(VL[0])) +void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef VL, + Value *OpValue) { + if (isa(OpValue)) return; - ScheduleData *Bundle = getScheduleData(VL[0]); + ScheduleData *Bundle = getScheduleData(OpValue); DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n"); assert(!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"); diff --git a/contrib/llvm/tools/clang/include/clang-c/Index.h b/contrib/llvm/tools/clang/include/clang-c/Index.h index f404e6d72ec9..09f4403556c8 100644 --- a/contrib/llvm/tools/clang/include/clang-c/Index.h +++ b/contrib/llvm/tools/clang/include/clang-c/Index.h @@ -4416,6 +4416,11 @@ CINDEX_LINKAGE unsigned clang_CXXMethod_isStatic(CXCursor C); */ CINDEX_LINKAGE unsigned clang_CXXMethod_isVirtual(CXCursor C); +/** + * \brief Determine if an enum declaration refers to a scoped enum. + */ +CINDEX_LINKAGE unsigned clang_EnumDecl_isScoped(CXCursor C); + /** * \brief Determine if a C++ member function or member function template is * declared 'const'. diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h b/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h index 0f1f481ae49b..041f0fd484d4 100644 --- a/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h +++ b/contrib/llvm/tools/clang/include/clang/AST/DeclBase.h @@ -749,7 +749,7 @@ class LLVM_ALIGNAS(/*alignof(uint64_t)*/ 8) Decl { /// Set that this declaration is globally visible, even if it came from a /// module that is not visible. void setVisibleDespiteOwningModule() { - if (hasOwningModule()) + if (isHidden()) setModuleOwnershipKind(ModuleOwnershipKind::Visible); } diff --git a/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h b/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h index 6965e8143ff6..9d64f0244ec3 100644 --- a/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h +++ b/contrib/llvm/tools/clang/include/clang/AST/DeclCXX.h @@ -1886,6 +1886,19 @@ class CXXMethodDecl : public FunctionDecl { return (CD->begin_overridden_methods() != CD->end_overridden_methods()); } + /// If it's possible to devirtualize a call to this method, return the called + /// function. Otherwise, return null. + + /// \param Base The object on which this virtual function is called. + /// \param IsAppleKext True if we are compiling for Apple kext. + CXXMethodDecl *getDevirtualizedMethod(const Expr *Base, bool IsAppleKext); + + const CXXMethodDecl *getDevirtualizedMethod(const Expr *Base, + bool IsAppleKext) const { + return const_cast(this)->getDevirtualizedMethod( + Base, IsAppleKext); + } + /// \brief Determine whether this is a usual deallocation function /// (C++ [basic.stc.dynamic.deallocation]p2), which is an overloaded /// delete or delete[] operator with a particular signature. diff --git a/contrib/llvm/tools/clang/include/clang/AST/ExternalASTMerger.h b/contrib/llvm/tools/clang/include/clang/AST/ExternalASTMerger.h index 92d7b39c48d2..51d0c30ad23b 100644 --- a/contrib/llvm/tools/clang/include/clang/AST/ExternalASTMerger.h +++ b/contrib/llvm/tools/clang/include/clang/AST/ExternalASTMerger.h @@ -44,10 +44,6 @@ class ExternalASTMerger : public ExternalASTSource { FindExternalLexicalDecls(const DeclContext *DC, llvm::function_ref IsKindWeWant, SmallVectorImpl &Result) override; - - using ExternalASTSource::CompleteType; - - void CompleteType(TagDecl *Tag) override; }; } // end namespace clang diff --git a/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h b/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h index 152e05bca740..917b240428e7 100644 --- a/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/contrib/llvm/tools/clang/include/clang/AST/RecursiveASTVisitor.h @@ -1799,6 +1799,7 @@ DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); }) if (TypeSourceInfo *TSI = D->getTypeAsWritten()) \ TRY_TO(TraverseTypeLoc(TSI->getTypeLoc())); \ \ + TRY_TO(TraverseNestedNameSpecifierLoc(D->getQualifierLoc())); \ if (!getDerived().shouldVisitTemplateInstantiations() && \ D->getTemplateSpecializationKind() != TSK_ExplicitSpecialization) \ /* Returning from here skips traversing the \ diff --git a/contrib/llvm/tools/clang/include/clang/ASTMatchers/ASTMatchers.h b/contrib/llvm/tools/clang/include/clang/ASTMatchers/ASTMatchers.h index cba4c99be959..c9b496df33f7 100644 --- a/contrib/llvm/tools/clang/include/clang/ASTMatchers/ASTMatchers.h +++ b/contrib/llvm/tools/clang/include/clang/ASTMatchers/ASTMatchers.h @@ -3821,6 +3821,14 @@ AST_MATCHER_P(CompoundStmt, statementCountIs, unsigned, N) { /// integerLiteral(equals(42)) /// matches 42 /// +/// Note that you cannot directly match a negative numeric literal because the +/// minus sign is not part of the literal: It is a unary operator whose operand +/// is the positive numeric literal. Instead, you must use a unaryOperator() +/// matcher to match the minus sign: +/// +/// unaryOperator(hasOperatorName("-"), +/// hasUnaryOperand(integerLiteral(equals(13)))) +/// /// Usable as: Matcher, Matcher, /// Matcher, Matcher template diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h b/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h index f6a47d646d1d..ec7549d4535c 100644 --- a/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h +++ b/contrib/llvm/tools/clang/include/clang/Analysis/AnalysisContext.h @@ -426,6 +426,7 @@ class AnalysisDeclContextManager { bool addImplicitDtors = false, bool addInitializers = false, bool addTemporaryDtors = false, + bool addLifetime = false, bool synthesizeBodies = false, bool addStaticInitBranches = false, bool addCXXNewAllocator = true, diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h b/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h index d23ed77ded13..97639bbfade2 100644 --- a/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h +++ b/contrib/llvm/tools/clang/include/clang/Analysis/CFG.h @@ -58,6 +58,7 @@ class CFGElement { Statement, Initializer, NewAllocator, + LifetimeEnds, // dtor kind AutomaticObjectDtor, DeleteDtor, @@ -167,6 +168,28 @@ class CFGNewAllocator : public CFGElement { } }; +/// Represents the point where the lifetime of an automatic object ends +class CFGLifetimeEnds : public CFGElement { +public: + explicit CFGLifetimeEnds(const VarDecl *var, const Stmt *stmt) + : CFGElement(LifetimeEnds, var, stmt) {} + + const VarDecl *getVarDecl() const { + return static_cast(Data1.getPointer()); + } + + const Stmt *getTriggerStmt() const { + return static_cast(Data2.getPointer()); + } + +private: + friend class CFGElement; + CFGLifetimeEnds() {} + static bool isKind(const CFGElement &elem) { + return elem.getKind() == LifetimeEnds; + } +}; + /// CFGImplicitDtor - Represents C++ object destructor implicitly generated /// by compiler on various occasions. class CFGImplicitDtor : public CFGElement { @@ -701,6 +724,10 @@ class CFGBlock { Elements.push_back(CFGAutomaticObjDtor(VD, S), C); } + void appendLifetimeEnds(VarDecl *VD, Stmt *S, BumpVectorContext &C) { + Elements.push_back(CFGLifetimeEnds(VD, S), C); + } + void appendDeleteDtor(CXXRecordDecl *RD, CXXDeleteExpr *DE, BumpVectorContext &C) { Elements.push_back(CFGDeleteDtor(RD, DE), C); } @@ -717,6 +744,19 @@ class CFGBlock { *I = CFGAutomaticObjDtor(VD, S); return ++I; } + + // Scope leaving must be performed in reversed order. So insertion is in two + // steps. First we prepare space for some number of elements, then we insert + // the elements beginning at the last position in prepared space. + iterator beginLifetimeEndsInsert(iterator I, size_t Cnt, + BumpVectorContext &C) { + return iterator( + Elements.insert(I.base(), Cnt, CFGLifetimeEnds(nullptr, nullptr), C)); + } + iterator insertLifetimeEnds(iterator I, VarDecl *VD, Stmt *S) { + *I = CFGLifetimeEnds(VD, S); + return ++I; + } }; /// \brief CFGCallback defines methods that should be called when a logical @@ -753,6 +793,7 @@ class CFG { bool AddEHEdges; bool AddInitializers; bool AddImplicitDtors; + bool AddLifetime; bool AddTemporaryDtors; bool AddStaticInitBranches; bool AddCXXNewAllocator; @@ -774,8 +815,10 @@ class CFG { BuildOptions() : forcedBlkExprs(nullptr), Observer(nullptr), - PruneTriviallyFalseEdges(true), AddEHEdges(false), + PruneTriviallyFalseEdges(true), + AddEHEdges(false), AddInitializers(false), AddImplicitDtors(false), + AddLifetime(false), AddTemporaryDtors(false), AddStaticInitBranches(false), AddCXXNewAllocator(false), AddCXXDefaultInitExprInCtors(false) {} }; diff --git a/contrib/llvm/tools/clang/include/clang/Analysis/CloneDetection.h b/contrib/llvm/tools/clang/include/clang/Analysis/CloneDetection.h index 1ca3514e69b0..6339deef41bd 100644 --- a/contrib/llvm/tools/clang/include/clang/Analysis/CloneDetection.h +++ b/contrib/llvm/tools/clang/include/clang/Analysis/CloneDetection.h @@ -15,6 +15,8 @@ #ifndef LLVM_CLANG_AST_CLONEDETECTION_H #define LLVM_CLANG_AST_CLONEDETECTION_H +#include "clang/AST/DeclTemplate.h" +#include "clang/AST/StmtVisitor.h" #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -29,6 +31,192 @@ class VarDecl; class ASTContext; class CompoundStmt; +namespace clone_detection { + +/// Returns a string that represents all macro expansions that expanded into the +/// given SourceLocation. +/// +/// If 'getMacroStack(A) == getMacroStack(B)' is true, then the SourceLocations +/// A and B are expanded from the same macros in the same order. +std::string getMacroStack(SourceLocation Loc, ASTContext &Context); + +/// Collects the data of a single Stmt. +/// +/// This class defines what a code clone is: If it collects for two statements +/// the same data, then those two statements are considered to be clones of each +/// other. +/// +/// All collected data is forwarded to the given data consumer of the type T. +/// The data consumer class needs to provide a member method with the signature: +/// update(StringRef Str) +template +class StmtDataCollector : public ConstStmtVisitor> { + + ASTContext &Context; + /// The data sink to which all data is forwarded. + T &DataConsumer; + +public: + /// Collects data of the given Stmt. + /// \param S The given statement. + /// \param Context The ASTContext of S. + /// \param DataConsumer The data sink to which all data is forwarded. + StmtDataCollector(const Stmt *S, ASTContext &Context, T &DataConsumer) + : Context(Context), DataConsumer(DataConsumer) { + this->Visit(S); + } + + typedef unsigned DataPiece; + + // Below are utility methods for appending different data to the vector. + + void addData(DataPiece Integer) { + DataConsumer.update( + StringRef(reinterpret_cast(&Integer), sizeof(Integer))); + } + + void addData(llvm::StringRef Str) { DataConsumer.update(Str); } + + void addData(const QualType &QT) { addData(QT.getAsString()); } + +// The functions below collect the class specific data of each Stmt subclass. + +// Utility macro for defining a visit method for a given class. This method +// calls back to the ConstStmtVisitor to visit all parent classes. +#define DEF_ADD_DATA(CLASS, CODE) \ + void Visit##CLASS(const CLASS *S) { \ + CODE; \ + ConstStmtVisitor::Visit##CLASS(S); \ + } + + DEF_ADD_DATA(Stmt, { + addData(S->getStmtClass()); + // This ensures that macro generated code isn't identical to macro-generated + // code. + addData(getMacroStack(S->getLocStart(), Context)); + addData(getMacroStack(S->getLocEnd(), Context)); + }) + DEF_ADD_DATA(Expr, { addData(S->getType()); }) + + //--- Builtin functionality ----------------------------------------------// + DEF_ADD_DATA(ArrayTypeTraitExpr, { addData(S->getTrait()); }) + DEF_ADD_DATA(ExpressionTraitExpr, { addData(S->getTrait()); }) + DEF_ADD_DATA(PredefinedExpr, { addData(S->getIdentType()); }) + DEF_ADD_DATA(TypeTraitExpr, { + addData(S->getTrait()); + for (unsigned i = 0; i < S->getNumArgs(); ++i) + addData(S->getArg(i)->getType()); + }) + + //--- Calls --------------------------------------------------------------// + DEF_ADD_DATA(CallExpr, { + // Function pointers don't have a callee and we just skip hashing it. + if (const FunctionDecl *D = S->getDirectCallee()) { + // If the function is a template specialization, we also need to handle + // the template arguments as they are not included in the qualified name. + if (auto Args = D->getTemplateSpecializationArgs()) { + std::string ArgString; + + // Print all template arguments into ArgString + llvm::raw_string_ostream OS(ArgString); + for (unsigned i = 0; i < Args->size(); ++i) { + Args->get(i).print(Context.getLangOpts(), OS); + // Add a padding character so that 'foo()' != 'foo()'. + OS << '\n'; + } + OS.flush(); + + addData(ArgString); + } + addData(D->getQualifiedNameAsString()); + } + }) + + //--- Exceptions ---------------------------------------------------------// + DEF_ADD_DATA(CXXCatchStmt, { addData(S->getCaughtType()); }) + + //--- C++ OOP Stmts ------------------------------------------------------// + DEF_ADD_DATA(CXXDeleteExpr, { + addData(S->isArrayFormAsWritten()); + addData(S->isGlobalDelete()); + }) + + //--- Casts --------------------------------------------------------------// + DEF_ADD_DATA(ObjCBridgedCastExpr, { addData(S->getBridgeKind()); }) + + //--- Miscellaneous Exprs ------------------------------------------------// + DEF_ADD_DATA(BinaryOperator, { addData(S->getOpcode()); }) + DEF_ADD_DATA(UnaryOperator, { addData(S->getOpcode()); }) + + //--- Control flow -------------------------------------------------------// + DEF_ADD_DATA(GotoStmt, { addData(S->getLabel()->getName()); }) + DEF_ADD_DATA(IndirectGotoStmt, { + if (S->getConstantTarget()) + addData(S->getConstantTarget()->getName()); + }) + DEF_ADD_DATA(LabelStmt, { addData(S->getDecl()->getName()); }) + DEF_ADD_DATA(MSDependentExistsStmt, { addData(S->isIfExists()); }) + DEF_ADD_DATA(AddrLabelExpr, { addData(S->getLabel()->getName()); }) + + //--- Objective-C --------------------------------------------------------// + DEF_ADD_DATA(ObjCIndirectCopyRestoreExpr, { addData(S->shouldCopy()); }) + DEF_ADD_DATA(ObjCPropertyRefExpr, { + addData(S->isSuperReceiver()); + addData(S->isImplicitProperty()); + }) + DEF_ADD_DATA(ObjCAtCatchStmt, { addData(S->hasEllipsis()); }) + + //--- Miscellaneous Stmts ------------------------------------------------// + DEF_ADD_DATA(CXXFoldExpr, { + addData(S->isRightFold()); + addData(S->getOperator()); + }) + DEF_ADD_DATA(GenericSelectionExpr, { + for (unsigned i = 0; i < S->getNumAssocs(); ++i) { + addData(S->getAssocType(i)); + } + }) + DEF_ADD_DATA(LambdaExpr, { + for (const LambdaCapture &C : S->captures()) { + addData(C.isPackExpansion()); + addData(C.getCaptureKind()); + if (C.capturesVariable()) + addData(C.getCapturedVar()->getType()); + } + addData(S->isGenericLambda()); + addData(S->isMutable()); + }) + DEF_ADD_DATA(DeclStmt, { + auto numDecls = std::distance(S->decl_begin(), S->decl_end()); + addData(static_cast(numDecls)); + for (const Decl *D : S->decls()) { + if (const VarDecl *VD = dyn_cast(D)) { + addData(VD->getType()); + } + } + }) + DEF_ADD_DATA(AsmStmt, { + addData(S->isSimple()); + addData(S->isVolatile()); + addData(S->generateAsmString(Context)); + for (unsigned i = 0; i < S->getNumInputs(); ++i) { + addData(S->getInputConstraint(i)); + } + for (unsigned i = 0; i < S->getNumOutputs(); ++i) { + addData(S->getOutputConstraint(i)); + } + for (unsigned i = 0; i < S->getNumClobbers(); ++i) { + addData(S->getClobber(i)); + } + }) + DEF_ADD_DATA(AttributedStmt, { + for (const Attr *A : S->getAttrs()) { + addData(std::string(A->getSpelling())); + } + }) +}; +} // namespace clone_detection + /// Identifies a list of statements. /// /// Can either identify a single arbitrary Stmt object, a continuous sequence of diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td index 42e1e5edaf9e..a28d63182749 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -178,6 +178,8 @@ def warn_drv_optimization_value : Warning<"optimization level '%0' is not suppor InGroup; def warn_ignored_gcc_optimization : Warning<"optimization flag '%0' is not supported">, InGroup; +def warn_ignored_clang_option : Warning<"the flag '%0' has been deprecated and will be ignored">, + InGroup; def warn_drv_unsupported_opt_for_target : Warning< "optimization flag '%0' is not supported for target '%1'">, InGroup; diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 8b4cb47e545d..57c24e9be73a 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -179,6 +179,8 @@ def warn_incompatible_analyzer_plugin_api : Warning< def note_incompatible_analyzer_plugin_api : Note< "current API version is '%0', but plugin was compiled with version '%1'">; +def err_module_build_requires_fmodules : Error< + "module compilation requires '-fmodules'">; def err_module_interface_requires_modules_ts : Error< "module interface compilation requires '-fmodules-ts'">; def warn_module_config_mismatch : Warning< diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td index 136e48ab5e54..5a8750e4dab6 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -1029,6 +1029,8 @@ def warn_auto_synthesizing_protocol_property :Warning< "auto property synthesis will not synthesize property %0" " declared in protocol %1">, InGroup>; +def note_add_synthesize_directive : Note< + "add a '@synthesize' directive">; def warn_no_autosynthesis_shared_ivar_property : Warning < "auto property synthesis will not synthesize property " "%0 because it cannot share an ivar with another synthesized property">, @@ -2878,7 +2880,7 @@ def warn_partial_availability : Warning<"%0 is only available conditionally">, def warn_partial_availability_new : Warning, InGroup; def note_partial_availability_silence : Note< - "explicitly redeclare %0 to silence this warning">; + "annotate %select{%1|anonymous %1}0 with an availability attribute to silence">; def note_unguarded_available_silence : Note< "enclose %0 in %select{an @available|a __builtin_available}1 check to silence" " this warning">; @@ -3516,6 +3518,8 @@ def note_ovl_candidate_substitution_failure : Note< "candidate template ignored: substitution failure%0%1">; def note_ovl_candidate_disabled_by_enable_if : Note< "candidate template ignored: disabled by %0%1">; +def note_ovl_candidate_disabled_by_requirement : Note< + "candidate template ignored: requirement '%0' was not satisfied%1">; def note_ovl_candidate_has_pass_object_size_params: Note< "candidate address cannot be taken because parameter %0 has " "pass_object_size attribute">; @@ -4429,6 +4433,9 @@ def err_typename_nested_not_found : Error<"no type named %0 in %1">; def err_typename_nested_not_found_enable_if : Error< "no type named 'type' in %0; 'enable_if' cannot be used to disable " "this declaration">; +def err_typename_nested_not_found_requirement : Error< + "failed requirement '%0'; 'enable_if' cannot be used to disable this " + "declaration">; def err_typename_nested_not_type : Error< "typename specifier refers to non-type member %0 in %1">; def note_typename_refers_here : Note< @@ -6353,15 +6360,13 @@ def err_exceptions_disabled : Error< "cannot use '%0' with exceptions disabled">; def err_objc_exceptions_disabled : Error< "cannot use '%0' with Objective-C exceptions disabled">; -def warn_throw_in_noexcept_func - : Warning<"%0 has a non-throwing exception specification but can still " - "throw, resulting in unexpected program termination">, - InGroup; -def note_throw_in_dtor - : Note<"destructor or deallocator has a (possibly implicit) non-throwing " - "excepton specification">; -def note_throw_in_function - : Note<"non-throwing function declare here">; +def warn_throw_in_noexcept_func : Warning< + "%0 has a non-throwing exception specification but can still throw">, + InGroup; +def note_throw_in_dtor : Note< + "%select{destructor|deallocator}0 has a %select{non-throwing|implicit " + "non-throwing}1 exception specification">; +def note_throw_in_function : Note<"function declared non-throwing here">; def err_seh_try_outside_functions : Error< "cannot use SEH '__try' in blocks, captured regions, or Obj-C method decls">; def err_mixing_cxx_try_seh_try : Error< @@ -8455,8 +8460,6 @@ def err_opencl_builtin_to_addr_invalid_arg : Error< // OpenCL v2.0 s6.13.17 Enqueue kernel restrictions. def err_opencl_enqueue_kernel_incorrect_args : Error< "illegal call to enqueue_kernel, incorrect argument types">; -def err_opencl_enqueue_kernel_expected_type : Error< - "illegal call to enqueue_kernel, expected %0 argument type">; def err_opencl_enqueue_kernel_local_size_args : Error< "mismatch in number of block parameters and local size arguments passed">; def err_opencl_enqueue_kernel_invalid_local_size_type : Error< @@ -8466,6 +8469,9 @@ def err_opencl_enqueue_kernel_blocks_non_local_void_args : Error< def err_opencl_enqueue_kernel_blocks_no_args : Error< "blocks with parameters are not accepted in this prototype of enqueue_kernel call">; +def err_opencl_builtin_expected_type : Error< + "illegal call to %0, expected %1 argument type">; + // OpenCL v2.2 s2.1.2.3 - Vector Component Access def ext_opencl_ext_vector_type_rgba_selector: ExtWarn< "vector component name '%0' is an OpenCL version 2.2 feature">, @@ -8854,6 +8860,10 @@ def warn_omp_nesting_simd : Warning< def err_omp_orphaned_device_directive : Error< "orphaned 'omp %0' directives are prohibited" "; perhaps you forget to enclose the directive into a %select{|||target |teams }1region?">; +def err_omp_reduction_non_addressable_expression : Error< + "expected addressable reduction item for the task-based directives">; +def err_omp_reduction_with_nogroup : Error< + "'reduction' clause cannot be used with 'nogroup' clause">; } // end of OpenMP category let CategoryName = "Related Result Type Issue" in { diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSerializationKinds.td index 3c64ebb9c7f4..0fc54848581c 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSerializationKinds.td +++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSerializationKinds.td @@ -127,11 +127,11 @@ def err_module_odr_violation_mismatch_decl : Error< "%select{definition in module '%2'|defined here}1 found " "%select{end of class|public access specifier|private access specifier|" "protected access specifier|static assert|field|method|type alias|typedef|" - "data member}3">; + "data member|friend declaration}3">; def note_module_odr_violation_mismatch_decl : Note<"but in '%0' found " "%select{end of class|public access specifier|private access specifier|" "protected access specifier|static assert|field|method|type alias|typedef|" - "data member}1">; + "data member|friend declaration}1">; def err_module_odr_violation_mismatch_decl_diff : Error< "%q0 has different definitions in different modules; first difference is " @@ -166,6 +166,9 @@ def err_module_odr_violation_mismatch_decl_diff : Error< "data member %4 with%select{out|}5 an initializer|" "data member %4 with an initializer|" "data member %4 %select{is constexpr|is not constexpr}5|" + "friend %select{class|function}4|" + "friend %4|" + "friend function %4|" "}3">; def note_module_odr_violation_mismatch_decl_diff : Note<"but in '%0' found " @@ -199,18 +202,21 @@ def note_module_odr_violation_mismatch_decl_diff : Note<"but in '%0' found " "data member %2 with%select{out|}3 an initializer|" "data member %2 with a different initializer|" "data member %2 %select{is constexpr|is not constexpr}3|" + "friend %select{class|function}2|" + "friend %2|" + "friend function %2|" "}1">; def err_module_odr_violation_mismatch_decl_unknown : Error< "%q0 %select{with definition in module '%2'|defined here}1 has different " "definitions in different modules; first difference is this " "%select{||||static assert|field|method|type alias|typedef|data member|" - "unexpected decl}3">; + "friend declaration|unexpected decl}3">; def note_module_odr_violation_mismatch_decl_unknown : Note< "but in '%0' found " "%select{||||different static assert|different field|different method|" "different type alias|different typedef|different data member|" - "another unexpected decl}1">; + "different friend declaration|another unexpected decl}1">; def warn_duplicate_module_file_extension : Warning< "duplicate module file extension block name '%0'">, diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Linkage.h b/contrib/llvm/tools/clang/include/clang/Basic/Linkage.h index e96fb568c009..6ec8763f2491 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/Linkage.h +++ b/contrib/llvm/tools/clang/include/clang/Basic/Linkage.h @@ -45,6 +45,17 @@ enum Linkage : unsigned char { /// translation units because of types defined in a inline function. VisibleNoLinkage, + /// \brief Internal linkage according to the Modules TS, but can be referred + /// to from other translation units indirectly through inline functions and + /// templates in the module interface. + ModuleInternalLinkage, + + /// \brief Module linkage, which indicates that the entity can be referred + /// to from other translation units within the same module, and indirectly + /// from arbitrary other translation units through inline functions and + /// templates in the module interface. + ModuleLinkage, + /// \brief External linkage, which indicates that the entity can /// be referred to from other translation units. ExternalLinkage @@ -74,15 +85,20 @@ inline bool isDiscardableGVALinkage(GVALinkage L) { } inline bool isExternallyVisible(Linkage L) { - return L == ExternalLinkage || L == VisibleNoLinkage; + return L >= VisibleNoLinkage; } inline Linkage getFormalLinkage(Linkage L) { - if (L == UniqueExternalLinkage) + switch (L) { + case UniqueExternalLinkage: return ExternalLinkage; - if (L == VisibleNoLinkage) + case VisibleNoLinkage: return NoLinkage; - return L; + case ModuleInternalLinkage: + return InternalLinkage; + default: + return L; + } } inline bool isExternalFormalLinkage(Linkage L) { diff --git a/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h index 53ce95cab1b0..b2f14afe5695 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h +++ b/contrib/llvm/tools/clang/include/clang/Basic/PartialDiagnostic.h @@ -329,6 +329,15 @@ class PartialDiagnostic { bool hasStorage() const { return DiagStorage != nullptr; } + /// Retrieve the string argument at the given index. + StringRef getStringArg(unsigned I) { + assert(DiagStorage && "No diagnostic storage?"); + assert(I < DiagStorage->NumDiagArgs && "Not enough diagnostic args"); + assert(DiagStorage->DiagArgumentsKind[I] + == DiagnosticsEngine::ak_std_string && "Not a string arg"); + return DiagStorage->DiagArgumentsStr[I]; + } + friend const PartialDiagnostic &operator<<(const PartialDiagnostic &PD, unsigned I) { PD.AddTaggedVal(I, DiagnosticsEngine::ak_uint); diff --git a/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h b/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h index 9bdb288eef4f..5885532b91db 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h +++ b/contrib/llvm/tools/clang/include/clang/Basic/TargetInfo.h @@ -23,6 +23,7 @@ #include "clang/Basic/VersionTuple.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" @@ -954,6 +955,14 @@ class TargetInfo : public RefCountedBase { return *AddrSpaceMap; } + /// \brief Return an AST address space which can be used opportunistically + /// for constant global memory. It must be possible to convert pointers into + /// this address space to LangAS::Default. If no such address space exists, + /// this may return None, and such optimizations will be disabled. + virtual llvm::Optional getConstantAddressSpace() const { + return LangAS::Default; + } + /// \brief Retrieve the name of the platform as it is used in the /// availability attribute. StringRef getPlatformName() const { return PlatformName; } diff --git a/contrib/llvm/tools/clang/include/clang/Basic/arm_neon.td b/contrib/llvm/tools/clang/include/clang/Basic/arm_neon.td index d5c16a91a34f..ad8d679a1664 100644 --- a/contrib/llvm/tools/clang/include/clang/Basic/arm_neon.td +++ b/contrib/llvm/tools/clang/include/clang/Basic/arm_neon.td @@ -227,7 +227,6 @@ def OP_UNAVAILABLE : Operation { // u: unsigned integer (int/float args) // f: float (int args) // F: double (int args) -// H: half (int args) // d: default // g: default, ignore 'Q' size modifier. // j: default, force 'Q' size modifier. @@ -346,7 +345,6 @@ def OP_MLSLHi : Op<(call "vmlsl", $p0, (call "vget_high", $p1), (call "vget_high", $p2))>; def OP_MLSLHi_N : Op<(call "vmlsl_n", $p0, (call "vget_high", $p1), $p2)>; def OP_MUL_N : Op<(op "*", $p0, (dup $p1))>; -def OP_MULX_N : Op<(call "vmulx", $p0, (dup $p1))>; def OP_MLA_N : Op<(op "+", $p0, (op "*", $p1, (dup $p2)))>; def OP_MLS_N : Op<(op "-", $p0, (op "*", $p1, (dup $p2)))>; def OP_FMLA_N : Op<(call "vfma", $p0, $p1, (dup $p2))>; @@ -1663,186 +1661,3 @@ def SCALAR_SQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "sssji", "SsSi", OP_SCALAR def SCALAR_VDUP_LANE : IInst<"vdup_lane", "sdi", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">; def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "sji", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">; } - -// ARMv8.2-A FP16 intrinsics. -let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in { - - // ARMv8.2-A FP16 one-operand vector intrinsics. - - // Comparison - def CMEQH : SInst<"vceqz", "ud", "hQh">; - def CMGEH : SInst<"vcgez", "ud", "hQh">; - def CMGTH : SInst<"vcgtz", "ud", "hQh">; - def CMLEH : SInst<"vclez", "ud", "hQh">; - def CMLTH : SInst<"vcltz", "ud", "hQh">; - - // Vector conversion - def VCVT_F16 : SInst<"vcvt_f16", "Hd", "sUsQsQUs">; - def VCVT_S16 : SInst<"vcvt_s16", "xd", "hQh">; - def VCVT_U16 : SInst<"vcvt_u16", "ud", "hQh">; - def VCVTA_S16 : SInst<"vcvta_s16", "xd", "hQh">; - def VCVTA_U16 : SInst<"vcvta_u16", "ud", "hQh">; - def VCVTM_S16 : SInst<"vcvtm_s16", "xd", "hQh">; - def VCVTM_U16 : SInst<"vcvtm_u16", "ud", "hQh">; - def VCVTN_S16 : SInst<"vcvtn_s16", "xd", "hQh">; - def VCVTN_U16 : SInst<"vcvtn_u16", "ud", "hQh">; - def VCVTP_S16 : SInst<"vcvtp_s16", "xd", "hQh">; - def VCVTP_U16 : SInst<"vcvtp_u16", "ud", "hQh">; - - // Vector rounding - def FRINTZH : SInst<"vrnd", "dd", "hQh">; - def FRINTNH : SInst<"vrndn", "dd", "hQh">; - def FRINTAH : SInst<"vrnda", "dd", "hQh">; - def FRINTPH : SInst<"vrndp", "dd", "hQh">; - def FRINTMH : SInst<"vrndm", "dd", "hQh">; - def FRINTXH : SInst<"vrndx", "dd", "hQh">; - def FRINTIH : SInst<"vrndi", "dd", "hQh">; - - // Misc. - def VABSH : SInst<"vabs", "dd", "hQh">; - def VNEGH : SOpInst<"vneg", "dd", "hQh", OP_NEG>; - def VRECPEH : SInst<"vrecpe", "dd", "hQh">; - def FRSQRTEH : SInst<"vrsqrte", "dd", "hQh">; - def FSQRTH : SInst<"vsqrt", "dd", "hQh">; - - // ARMv8.2-A FP16 two-operands vector intrinsics. - - // Misc. - def VADDH : SOpInst<"vadd", "ddd", "hQh", OP_ADD>; - def VABDH : SInst<"vabd", "ddd", "hQh">; - def VSUBH : SOpInst<"vsub", "ddd", "hQh", OP_SUB>; - - // Comparison - let InstName = "vacge" in { - def VCAGEH : SInst<"vcage", "udd", "hQh">; - def VCALEH : SInst<"vcale", "udd", "hQh">; - } - let InstName = "vacgt" in { - def VCAGTH : SInst<"vcagt", "udd", "hQh">; - def VCALTH : SInst<"vcalt", "udd", "hQh">; - } - def VCEQH : SOpInst<"vceq", "udd", "hQh", OP_EQ>; - def VCGEH : SOpInst<"vcge", "udd", "hQh", OP_GE>; - def VCGTH : SOpInst<"vcgt", "udd", "hQh", OP_GT>; - let InstName = "vcge" in - def VCLEH : SOpInst<"vcle", "udd", "hQh", OP_LE>; - let InstName = "vcgt" in - def VCLTH : SOpInst<"vclt", "udd", "hQh", OP_LT>; - - // Vector conversion - let isVCVT_N = 1 in { - def VCVT_N_F16 : SInst<"vcvt_n_f16", "Hdi", "sUsQsQUs">; - def VCVT_N_S16 : SInst<"vcvt_n_s16", "xdi", "hQh">; - def VCVT_N_U16 : SInst<"vcvt_n_u16", "udi", "hQh">; - } - - // Max/Min - def VMAXH : SInst<"vmax", "ddd", "hQh">; - def VMINH : SInst<"vmin", "ddd", "hQh">; - def FMAXNMH : SInst<"vmaxnm", "ddd", "hQh">; - def FMINNMH : SInst<"vminnm", "ddd", "hQh">; - - // Multiplication/Division - def VMULH : SOpInst<"vmul", "ddd", "hQh", OP_MUL>; - def MULXH : SInst<"vmulx", "ddd", "hQh">; - def FDIVH : IOpInst<"vdiv", "ddd", "hQh", OP_DIV>; - - // Pairwise addition - def VPADDH : SInst<"vpadd", "ddd", "hQh">; - - // Pairwise Max/Min - def VPMAXH : SInst<"vpmax", "ddd", "hQh">; - def VPMINH : SInst<"vpmin", "ddd", "hQh">; - // Pairwise MaxNum/MinNum - def FMAXNMPH : SInst<"vpmaxnm", "ddd", "hQh">; - def FMINNMPH : SInst<"vpminnm", "ddd", "hQh">; - - // Reciprocal/Sqrt - def VRECPSH : SInst<"vrecps", "ddd", "hQh">; - def VRSQRTSH : SInst<"vrsqrts", "ddd", "hQh">; - - // ARMv8.2-A FP16 three-operands vector intrinsics. - - // Vector fused multiply-add operations - def VFMAH : SInst<"vfma", "dddd", "hQh">; - def VFMSH : SOpInst<"vfms", "dddd", "hQh", OP_FMLS>; - - // ARMv8.2-A FP16 lane vector intrinsics. - - // FMA lane - def VFMA_LANEH : IInst<"vfma_lane", "dddgi", "hQh">; - def VFMA_LANEQH : IInst<"vfma_laneq", "dddji", "hQh">; - - // FMA lane with scalar argument - def FMLA_NH : SOpInst<"vfma_n", "ddds", "hQh", OP_FMLA_N>; - // Scalar floating point fused multiply-add (scalar, by element) - def SCALAR_FMLA_LANEH : IInst<"vfma_lane", "sssdi", "Sh">; - def SCALAR_FMLA_LANEQH : IInst<"vfma_laneq", "sssji", "Sh">; - - // FMS lane - def VFMS_LANEH : IOpInst<"vfms_lane", "dddgi", "hQh", OP_FMS_LN>; - def VFMS_LANEQH : IOpInst<"vfms_laneq", "dddji", "hQh", OP_FMS_LNQ>; - // FMS lane with scalar argument - def FMLS_NH : SOpInst<"vfms_n", "ddds", "hQh", OP_FMLS_N>; - // Scalar floating foint fused multiply-subtract (scalar, by element) - def SCALAR_FMLS_LANEH : IOpInst<"vfms_lane", "sssdi", "Sh", OP_FMS_LN>; - def SCALAR_FMLS_LANEQH : IOpInst<"vfms_laneq", "sssji", "Sh", OP_FMS_LNQ>; - - // Mul lane - def VMUL_LANEH : IOpInst<"vmul_lane", "ddgi", "hQh", OP_MUL_LN>; - def VMUL_LANEQH : IOpInst<"vmul_laneq", "ddji", "hQh", OP_MUL_LN>; - def VMUL_NH : IOpInst<"vmul_n", "dds", "hQh", OP_MUL_N>; - // Scalar floating point multiply (scalar, by element) - def SCALAR_FMUL_LANEH : IOpInst<"vmul_lane", "ssdi", "Sh", OP_SCALAR_MUL_LN>; - def SCALAR_FMUL_LANEQH : IOpInst<"vmul_laneq", "ssji", "Sh", OP_SCALAR_MUL_LN>; - - // Mulx lane - def VMULX_LANEH : IOpInst<"vmulx_lane", "ddgi", "hQh", OP_MULX_LN>; - def VMULX_LANEQH : IOpInst<"vmulx_laneq", "ddji", "hQh", OP_MULX_LN>; - def VMULX_NH : IOpInst<"vmulx_n", "dds", "hQh", OP_MULX_N>; - // TODO: Scalar floating point multiply extended (scalar, by element) - // Below ones are commented out because they need vmulx_f16(float16_t, float16_t) - // which will be implemented later with fp16 scalar intrinsic (arm_fp16.h) - //def SCALAR_FMULX_LANEH : IOpInst<"vmulx_lane", "ssdi", "Sh", OP_SCALAR_MUL_LN>; - //def SCALAR_FMULX_LANEQH : IOpInst<"vmulx_laneq", "ssji", "Sh", OP_SCALAR_MUL_LN>; - - // ARMv8.2-A FP16 reduction vector intrinsics. - def VMAXVH : SInst<"vmaxv", "sd", "hQh">; - def VMINVH : SInst<"vminv", "sd", "hQh">; - def FMAXNMVH : SInst<"vmaxnmv", "sd", "hQh">; - def FMINNMVH : SInst<"vminnmv", "sd", "hQh">; - - // Data processing intrinsics - section 5 - - // Logical operations - let isHiddenLInst = 1 in - def VBSLH : SInst<"vbsl", "dudd", "hQh">; - - // Transposition operations - def VZIPH : WInst<"vzip", "2dd", "hQh">; - def VUZPH : WInst<"vuzp", "2dd", "hQh">; - def VTRNH : WInst<"vtrn", "2dd", "hQh">; - - // Set all lanes to same value. - /* Already implemented prior to ARMv8.2-A. - def VMOV_NH : WOpInst<"vmov_n", "ds", "hQh", OP_DUP>; - def VDUP_NH : WOpInst<"vdup_n", "ds", "hQh", OP_DUP>; - def VDUP_LANE1H : WOpInst<"vdup_lane", "dgi", "hQh", OP_DUP_LN>;*/ - - // Vector Extract - def VEXTH : WInst<"vext", "dddi", "hQh">; - - // Reverse vector elements - def VREV64H : WOpInst<"vrev64", "dd", "hQh", OP_REV64>; - - // Permutation - def VTRN1H : SOpInst<"vtrn1", "ddd", "hQh", OP_TRN1>; - def VZIP1H : SOpInst<"vzip1", "ddd", "hQh", OP_ZIP1>; - def VUZP1H : SOpInst<"vuzp1", "ddd", "hQh", OP_UZP1>; - def VTRN2H : SOpInst<"vtrn2", "ddd", "hQh", OP_TRN2>; - def VZIP2H : SOpInst<"vzip2", "ddd", "hQh", OP_ZIP2>; - def VUZP2H : SOpInst<"vuzp2", "ddd", "hQh", OP_UZP2>; - - def SCALAR_VDUP_LANEH : IInst<"vdup_lane", "sdi", "Sh">; - def SCALAR_VDUP_LANEQH : IInst<"vdup_laneq", "sji", "Sh">; -} diff --git a/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenABITypes.h b/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenABITypes.h index 8ba769dfc3af..615e55c8b69f 100644 --- a/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenABITypes.h +++ b/contrib/llvm/tools/clang/include/clang/CodeGen/CodeGenABITypes.h @@ -31,6 +31,8 @@ namespace llvm { class DataLayout; class Module; + class FunctionType; + class Type; } namespace clang { @@ -70,6 +72,12 @@ const CGFunctionInfo &arrangeFreeFunctionCall(CodeGenModule &CGM, FunctionType::ExtInfo info, RequiredArgs args); +// Returns null if the function type is incomplete and can't be lowered. +llvm::FunctionType *convertFreeFunctionType(CodeGenModule &CGM, + const FunctionDecl *FD); + +llvm::Type *convertTypeForMemory(CodeGenModule &CGM, QualType T); + } // end namespace CodeGen } // end namespace clang diff --git a/contrib/llvm/tools/clang/include/clang/Driver/Options.td b/contrib/llvm/tools/clang/include/clang/Driver/Options.td index b65b984731f6..861dfbf1916e 100644 --- a/contrib/llvm/tools/clang/include/clang/Driver/Options.td +++ b/contrib/llvm/tools/clang/include/clang/Driver/Options.td @@ -194,6 +194,16 @@ def clang_ignored_f_Group : OptionGroup<"">, def clang_ignored_m_Group : OptionGroup<"">, Group, Flags<[Ignored]>; +// Group for clang options in the process of deprecation. +// Please include the version that deprecated the flag as comment to allow +// easier garbage collection. +def clang_ignored_legacy_options_Group : OptionGroup<"">, + Group, Flags<[Ignored]>; + +// Retired with clang-5.0 +def : Flag<["-"], "fslp-vectorize-aggressive">, Group; +def : Flag<["-"], "fno-slp-vectorize-aggressive">, Group; + // Group that ignores all gcc optimizations that won't be implemented def clang_ignored_gcc_optimization_f_Group : OptionGroup< "">, Group, Flags<[Ignored]>; @@ -2017,6 +2027,10 @@ def mmsa : Flag<["-"], "mmsa">, Group, HelpText<"Enable MSA ASE (MIPS only)">; def mno_msa : Flag<["-"], "mno-msa">, Group, HelpText<"Disable MSA ASE (MIPS only)">; +def mmt : Flag<["-"], "mmt">, Group, + HelpText<"Enable MT ASE (MIPS only)">; +def mno_mt : Flag<["-"], "mno-mt">, Group, + HelpText<"Disable MT ASE (MIPS only)">; def mfp64 : Flag<["-"], "mfp64">, Group, HelpText<"Use 64-bit floating point registers (MIPS only)">; def mfp32 : Flag<["-"], "mfp32">, Group, diff --git a/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h b/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h index 105d0f338ac6..eb42f1260d92 100644 --- a/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h +++ b/contrib/llvm/tools/clang/include/clang/Driver/ToolChain.h @@ -411,7 +411,8 @@ class ToolChain { /// \brief Add options that need to be passed to cc1 for this target. virtual void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const; /// \brief Add warning options that need to be passed to cc1 for this target. virtual void addClangWarningOptions(llvm::opt::ArgStringList &CC1Args) const; diff --git a/contrib/llvm/tools/clang/include/clang/Format/Format.h b/contrib/llvm/tools/clang/include/clang/Format/Format.h index ee24c55fef61..99d54e55e828 100644 --- a/contrib/llvm/tools/clang/include/clang/Format/Format.h +++ b/contrib/llvm/tools/clang/include/clang/Format/Format.h @@ -1120,7 +1120,10 @@ struct FormatStyle { /// (https://developers.google.com/protocol-buffers/). LK_Proto, /// Should be used for TableGen code. - LK_TableGen + LK_TableGen, + /// Should be used for Protocol Buffer messages in text format + /// (https://developers.google.com/protocol-buffers/). + LK_TextProto }; bool isCpp() const { return Language == LK_Cpp || Language == LK_ObjC; } @@ -1750,6 +1753,8 @@ inline StringRef getLanguageName(FormatStyle::LanguageKind Language) { return "JavaScript"; case FormatStyle::LK_Proto: return "Proto"; + case FormatStyle::LK_TextProto: + return "TextProto"; default: return "Unknown"; } diff --git a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h index 84db293c46f3..c45aeaa208c8 100644 --- a/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h +++ b/contrib/llvm/tools/clang/include/clang/Frontend/FrontendActions.h @@ -111,6 +111,8 @@ class GenerateModuleAction : public ASTFrontendAction { class GenerateModuleFromModuleMapAction : public GenerateModuleAction { private: + bool BeginSourceFileAction(CompilerInstance &CI) override; + std::unique_ptr CreateOutputFile(CompilerInstance &CI, StringRef InFile) override; }; diff --git a/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h b/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h index 712e1ab9fbf5..62090d6496ed 100644 --- a/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h +++ b/contrib/llvm/tools/clang/include/clang/Lex/Preprocessor.h @@ -1048,6 +1048,10 @@ class Preprocessor { /// which implicitly adds the builtin defines etc. void EnterMainSourceFile(); + /// \brief After parser warm-up, initialize the conditional stack from + /// the preamble. + void replayPreambleConditionalStack(); + /// \brief Inform the preprocessor callbacks that processing is complete. void EndSourceFile(); @@ -1733,11 +1737,6 @@ class Preprocessor { /// \brief Return true if we're in the top-level file, not in a \#include. bool isInPrimaryFile() const; - /// \brief Return true if we're in the main file (specifically, if we are 0 - /// (zero) levels deep \#include. This is used by the lexer to determine if - /// it needs to generate errors about unterminated \#if directives. - bool isInMainFile() const; - /// \brief Handle cases where the \#include name is expanded /// from a macro as multiple tokens, which need to be glued together. /// diff --git a/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h b/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h index b73ec0868f52..d65dbf0cd34e 100644 --- a/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h +++ b/contrib/llvm/tools/clang/include/clang/Sema/DelayedDiagnostic.h @@ -124,7 +124,8 @@ class DelayedDiagnostic { static DelayedDiagnostic makeAvailability(AvailabilityResult AR, SourceLocation Loc, - const NamedDecl *D, + const NamedDecl *ReferringDecl, + const NamedDecl *OffendingDecl, const ObjCInterfaceDecl *UnknownObjCClass, const ObjCPropertyDecl *ObjCProperty, StringRef Msg, @@ -164,9 +165,13 @@ class DelayedDiagnostic { return *reinterpret_cast(AccessData); } - const NamedDecl *getAvailabilityDecl() const { + const NamedDecl *getAvailabilityReferringDecl() const { assert(Kind == Availability && "Not an availability diagnostic."); - return AvailabilityData.Decl; + return AvailabilityData.ReferringDecl; + } + + const NamedDecl *getAvailabilityOffendingDecl() const { + return AvailabilityData.OffendingDecl; } StringRef getAvailabilityMessage() const { @@ -213,7 +218,8 @@ class DelayedDiagnostic { private: struct AD { - const NamedDecl *Decl; + const NamedDecl *ReferringDecl; + const NamedDecl *OffendingDecl; const ObjCInterfaceDecl *UnknownObjCClass; const ObjCPropertyDecl *ObjCProperty; const char *Message; diff --git a/contrib/llvm/tools/clang/include/clang/Sema/Sema.h b/contrib/llvm/tools/clang/include/clang/Sema/Sema.h index 95134d52f873..95629a2591cf 100644 --- a/contrib/llvm/tools/clang/include/clang/Sema/Sema.h +++ b/contrib/llvm/tools/clang/include/clang/Sema/Sema.h @@ -1266,6 +1266,7 @@ class Sema { void emitAndClearUnusedLocalTypedefWarnings(); + void ActOnStartOfTranslationUnit(); void ActOnEndOfTranslationUnit(); void CheckDelegatingCtorCycles(); @@ -1541,6 +1542,7 @@ class Sema { llvm::SmallVectorImpl *Modules); bool hasVisibleMergedDefinition(NamedDecl *Def); + bool hasMergedDefinitionInCurrentModule(NamedDecl *Def); /// Determine if \p D and \p Suggested have a structurally compatible /// layout as described in C11 6.2.7/1. @@ -3358,9 +3360,10 @@ class Sema { /// DefaultSynthesizeProperties - This routine default synthesizes all /// properties which must be synthesized in the class's \@implementation. - void DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl, - ObjCInterfaceDecl *IDecl); - void DefaultSynthesizeProperties(Scope *S, Decl *D); + void DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl, + ObjCInterfaceDecl *IDecl, + SourceLocation AtEnd); + void DefaultSynthesizeProperties(Scope *S, Decl *D, SourceLocation AtEnd); /// IvarBacksCurrentMethodAccessor - This routine returns 'true' if 'IV' is /// an ivar synthesized for 'Method' and 'Method' is a property accessor @@ -3878,7 +3881,9 @@ class Sema { void redelayDiagnostics(sema::DelayedDiagnosticPool &pool); - void EmitAvailabilityWarning(AvailabilityResult AR, NamedDecl *D, + void EmitAvailabilityWarning(AvailabilityResult AR, + const NamedDecl *ReferringDecl, + const NamedDecl *OffendingDecl, StringRef Message, SourceLocation Loc, const ObjCInterfaceDecl *UnknownObjCClass, const ObjCPropertyDecl *ObjCProperty, @@ -3895,8 +3900,9 @@ class Sema { bool CanUseDecl(NamedDecl *D, bool TreatUnavailableAsInvalid); bool DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc, - const ObjCInterfaceDecl *UnknownObjCClass=nullptr, - bool ObjCPropertyAccess=false); + const ObjCInterfaceDecl *UnknownObjCClass = nullptr, + bool ObjCPropertyAccess = false, + bool AvoidPartialAvailabilityChecks = false); void NoteDeletedFunction(FunctionDecl *FD); void NoteDeletedInheritingConstructor(CXXConstructorDecl *CD); std::string getDeletedOrUnavailableSuffix(const FunctionDecl *FD); @@ -3938,7 +3944,7 @@ class Sema { void MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func, bool MightBeOdrUse = true); void MarkVariableReferenced(SourceLocation Loc, VarDecl *Var); - void MarkDeclRefReferenced(DeclRefExpr *E); + void MarkDeclRefReferenced(DeclRefExpr *E, const Expr *Base = nullptr); void MarkMemberReferenced(MemberExpr *E); void UpdateMarkingForLValueToRValue(Expr *E); @@ -10410,16 +10416,14 @@ class Sema { return OriginalLexicalContext ? OriginalLexicalContext : CurContext; } - /// \brief The diagnostic we should emit for \c D, or \c AR_Available. - /// - /// \param D The declaration to check. Note that this may be altered to point - /// to another declaration that \c D gets it's availability from. i.e., we - /// walk the list of typedefs to find an availability attribute. + /// The diagnostic we should emit for \c D, and the declaration that + /// originated it, or \c AR_Available. /// + /// \param D The declaration to check. /// \param Message If non-null, this will be populated with the message from /// the availability attribute that is selected. - AvailabilityResult ShouldDiagnoseAvailabilityOfDecl(NamedDecl *&D, - std::string *Message); + std::pair + ShouldDiagnoseAvailabilityOfDecl(const NamedDecl *D, std::string *Message); const DeclContext *getCurObjCLexicalContext() const { const DeclContext *DC = getCurLexicalContext(); diff --git a/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h b/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h index d92cbab4fbcf..cd9ed6abfaf9 100644 --- a/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h +++ b/contrib/llvm/tools/clang/include/clang/Sema/TemplateDeduction.h @@ -88,6 +88,12 @@ class TemplateDeductionInfo { HasSFINAEDiagnostic = false; } + /// Peek at the SFINAE diagnostic. + const PartialDiagnosticAt &peekSFINAEDiagnostic() const { + assert(HasSFINAEDiagnostic); + return SuppressedDiagnostics.front(); + } + /// \brief Provide a new template argument list that contains the /// results of template argument deduction. void reset(TemplateArgumentList *NewDeduced) { diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h index 4fb50deb0f6b..5dd6bdf38496 100644 --- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h +++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h @@ -205,9 +205,15 @@ class AnalyzerOptions : public RefCountedBase { /// Controls which C++ member functions will be considered for inlining. CXXInlineableMemberKind CXXMemberInliningMode; + /// \sa includeImplicitDtorsInCFG + Optional IncludeImplicitDtorsInCFG; + /// \sa includeTemporaryDtorsInCFG Optional IncludeTemporaryDtorsInCFG; - + + /// \sa IncludeLifetimeInCFG + Optional IncludeLifetimeInCFG; + /// \sa mayInlineCXXStandardLibrary Optional InlineCXXStandardLibrary; @@ -395,6 +401,20 @@ class AnalyzerOptions : public RefCountedBase { /// accepts the values "true" and "false". bool includeTemporaryDtorsInCFG(); + /// Returns whether or not implicit destructors for C++ objects should + /// be included in the CFG. + /// + /// This is controlled by the 'cfg-implicit-dtors' config option, which + /// accepts the values "true" and "false". + bool includeImplicitDtorsInCFG(); + + /// Returns whether or not end-of-lifetime information should be included in + /// the CFG. + /// + /// This is controlled by the 'cfg-lifetime' config option, which accepts + /// the values "true" and "false". + bool includeLifetimeInCFG(); + /// Returns whether or not C++ standard library functions may be considered /// for inlining. /// diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h index 52ed260346bf..88cb08a4b647 100644 --- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h +++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h @@ -286,7 +286,7 @@ class CheckerManager { void runCheckersForEndAnalysis(ExplodedGraph &G, BugReporter &BR, ExprEngine &Eng); - /// \brief Run checkers on begining of function. + /// \brief Run checkers on beginning of function. void runCheckersForBeginFunction(ExplodedNodeSet &Dst, const BlockEdge &L, ExplodedNode *Pred, diff --git a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h index b8ec2aa6ae8d..4aa87443e4c2 100644 --- a/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h +++ b/contrib/llvm/tools/clang/include/clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h @@ -124,7 +124,7 @@ class BasicValueFactory { /// Returns the type of the APSInt used to store values of the given QualType. APSIntType getAPSIntType(QualType T) const { assert(T->isIntegralOrEnumerationType() || Loc::isLocType(T)); - return APSIntType(Ctx.getTypeSize(T), + return APSIntType(Ctx.getIntWidth(T), !T->isSignedIntegerOrEnumerationType()); } diff --git a/contrib/llvm/tools/clang/include/clang/Tooling/ArgumentsAdjusters.h b/contrib/llvm/tools/clang/include/clang/Tooling/ArgumentsAdjusters.h index 1fd7be688761..4eb02251a775 100644 --- a/contrib/llvm/tools/clang/include/clang/Tooling/ArgumentsAdjusters.h +++ b/contrib/llvm/tools/clang/include/clang/Tooling/ArgumentsAdjusters.h @@ -44,6 +44,10 @@ ArgumentsAdjuster getClangSyntaxOnlyAdjuster(); /// arguments. ArgumentsAdjuster getClangStripOutputAdjuster(); +/// \brief Gets an argument adjuster which removes dependency-file +/// related command line arguments. +ArgumentsAdjuster getClangStripDependencyFileAdjuster(); + enum class ArgumentInsertPosition { BEGIN, END }; /// \brief Gets an argument adjuster which inserts \p Extra arguments in the diff --git a/contrib/llvm/tools/clang/include/clang/Tooling/Core/Diagnostic.h b/contrib/llvm/tools/clang/include/clang/Tooling/Core/Diagnostic.h index d657f16df183..b4920d4fe456 100644 --- a/contrib/llvm/tools/clang/include/clang/Tooling/Core/Diagnostic.h +++ b/contrib/llvm/tools/clang/include/clang/Tooling/Core/Diagnostic.h @@ -58,9 +58,9 @@ struct Diagnostic { Diagnostic(llvm::StringRef DiagnosticName, Level DiagLevel, StringRef BuildDirectory); - Diagnostic(llvm::StringRef DiagnosticName, DiagnosticMessage &Message, - llvm::StringMap &Fix, - SmallVector &Notes, Level DiagLevel, + Diagnostic(llvm::StringRef DiagnosticName, const DiagnosticMessage &Message, + const llvm::StringMap &Fix, + const SmallVector &Notes, Level DiagLevel, llvm::StringRef BuildDirectory); /// \brief Name identifying the Diagnostic. diff --git a/contrib/llvm/tools/clang/include/clang/Tooling/Tooling.h b/contrib/llvm/tools/clang/include/clang/Tooling/Tooling.h index 1c974f998852..6f9bc9e1a150 100644 --- a/contrib/llvm/tools/clang/include/clang/Tooling/Tooling.h +++ b/contrib/llvm/tools/clang/include/clang/Tooling/Tooling.h @@ -202,12 +202,15 @@ buildASTFromCode(const Twine &Code, const Twine &FileName = "input.cc", /// \param PCHContainerOps The PCHContainerOperations for loading and creating /// clang modules. /// +/// \param Adjuster A function to filter the command line arguments as specified. +/// /// \return The resulting AST or null if an error occurred. std::unique_ptr buildASTFromCodeWithArgs( const Twine &Code, const std::vector &Args, const Twine &FileName = "input.cc", const Twine &ToolName = "clang-tool", std::shared_ptr PCHContainerOps = - std::make_shared()); + std::make_shared(), + ArgumentsAdjuster Adjuster = getClangStripDependencyFileAdjuster()); /// \brief Utility to run a FrontendAction in a single clang invocation. class ToolInvocation { diff --git a/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp b/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp index a2ff176df11f..fd9723298fca 100644 --- a/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp +++ b/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp @@ -8523,7 +8523,10 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context, RequiresICE = false; // Read the prefixed modifiers first. - bool Done = false, IsSpecialLong = false; + bool Done = false; + #ifndef NDEBUG + bool IsSpecialLong = false; + #endif while (!Done) { switch (*Str++) { default: Done = true; --Str; break; @@ -8549,7 +8552,9 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context, // 'N' behaves like 'L' for all non LP64 targets and 'int' otherwise. assert(!IsSpecialLong && "Can't use two 'N' or 'W' modifiers!"); assert(HowLong == 0 && "Can't use both 'L' and 'N' modifiers!"); + #ifndef NDEBUG IsSpecialLong = true; + #endif if (Context.getTargetInfo().getLongWidth() == 32) ++HowLong; break; @@ -8558,7 +8563,9 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context, // This modifier represents int64 type. assert(!IsSpecialLong && "Can't use two 'N' or 'W' modifiers!"); assert(HowLong == 0 && "Can't use both 'L' and 'W' modifiers!"); + #ifndef NDEBUG IsSpecialLong = true; + #endif switch (Context.getTargetInfo().getInt64Type()) { default: llvm_unreachable("Unexpected integer type"); diff --git a/contrib/llvm/tools/clang/lib/AST/Decl.cpp b/contrib/llvm/tools/clang/lib/AST/Decl.cpp index 267c6992af89..573a98efe980 100644 --- a/contrib/llvm/tools/clang/lib/AST/Decl.cpp +++ b/contrib/llvm/tools/clang/lib/AST/Decl.cpp @@ -573,6 +573,44 @@ static bool isSingleLineLanguageLinkage(const Decl &D) { return false; } +static bool isExportedFromModuleIntefaceUnit(const NamedDecl *D) { + switch (D->getModuleOwnershipKind()) { + case Decl::ModuleOwnershipKind::Unowned: + case Decl::ModuleOwnershipKind::ModulePrivate: + return false; + case Decl::ModuleOwnershipKind::Visible: + case Decl::ModuleOwnershipKind::VisibleWhenImported: + if (auto *M = D->getOwningModule()) + return M->Kind == Module::ModuleInterfaceUnit; + } + llvm_unreachable("unexpected module ownership kind"); +} + +static LinkageInfo getInternalLinkageFor(const NamedDecl *D) { + // Internal linkage declarations within a module interface unit are modeled + // as "module-internal linkage", which means that they have internal linkage + // formally but can be indirectly accessed from outside the module via inline + // functions and templates defined within the module. + if (auto *M = D->getOwningModule()) + if (M->Kind == Module::ModuleInterfaceUnit) + return LinkageInfo(ModuleInternalLinkage, DefaultVisibility, false); + + return LinkageInfo::internal(); +} + +static LinkageInfo getExternalLinkageFor(const NamedDecl *D) { + // C++ Modules TS [basic.link]/6.8: + // - A name declared at namespace scope that does not have internal linkage + // by the previous rules and that is introduced by a non-exported + // declaration has module linkage. + if (auto *M = D->getOwningModule()) + if (M->Kind == Module::ModuleInterfaceUnit) + if (!isExportedFromModuleIntefaceUnit(D)) + return LinkageInfo(ModuleLinkage, DefaultVisibility, false); + + return LinkageInfo::external(); +} + static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, LVComputationKind computation) { assert(D->getDeclContext()->getRedeclContext()->isFileContext() && @@ -588,16 +626,18 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, if (const auto *Var = dyn_cast(D)) { // Explicitly declared static. if (Var->getStorageClass() == SC_Static) - return LinkageInfo::internal(); + return getInternalLinkageFor(Var); // - a non-inline, non-volatile object or reference that is explicitly // declared const or constexpr and neither explicitly declared extern // nor previously declared to have external linkage; or (there is no // equivalent in C99) + // The C++ modules TS adds "non-exported" to this list. if (Context.getLangOpts().CPlusPlus && Var->getType().isConstQualified() && !Var->getType().isVolatileQualified() && - !Var->isInline()) { + !Var->isInline() && + !isExportedFromModuleIntefaceUnit(Var)) { const VarDecl *PrevVar = Var->getPreviousDecl(); if (PrevVar) return getLVForDecl(PrevVar, computation); @@ -605,7 +645,7 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, if (Var->getStorageClass() != SC_Extern && Var->getStorageClass() != SC_PrivateExtern && !isSingleLineLanguageLinkage(*Var)) - return LinkageInfo::internal(); + return getInternalLinkageFor(Var); } for (const VarDecl *PrevVar = Var->getPreviousDecl(); PrevVar; @@ -615,7 +655,7 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, return PrevVar->getLinkageAndVisibility(); // Explicitly declared static. if (PrevVar->getStorageClass() == SC_Static) - return LinkageInfo::internal(); + return getInternalLinkageFor(Var); } } else if (const FunctionDecl *Function = D->getAsFunction()) { // C++ [temp]p4: @@ -624,7 +664,7 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, // Explicitly declared static. if (Function->getCanonicalDecl()->getStorageClass() == SC_Static) - return LinkageInfo(InternalLinkage, DefaultVisibility, false); + return getInternalLinkageFor(Function); } else if (const auto *IFD = dyn_cast(D)) { // - a data member of an anonymous union. const VarDecl *VD = IFD->getVarDecl(); @@ -637,7 +677,12 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, const auto *Var = dyn_cast(D); const auto *Func = dyn_cast(D); // FIXME: In C++11 onwards, anonymous namespaces should give decls - // within them internal linkage, not unique external linkage. + // within them (including those inside extern "C" contexts) internal + // linkage, not unique external linkage: + // + // C++11 [basic.link]p4: + // An unnamed namespace or a namespace declared directly or indirectly + // within an unnamed namespace has internal linkage. if ((!Var || !isFirstInExternCContext(Var)) && (!Func || !isFirstInExternCContext(Func))) return LinkageInfo::uniqueExternal(); @@ -718,7 +763,8 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, // because of this, but unique-external linkage suits us. if (Context.getLangOpts().CPlusPlus && !isFirstInExternCContext(Var)) { LinkageInfo TypeLV = getLVForType(*Var->getType(), computation); - if (TypeLV.getLinkage() != ExternalLinkage) + if (TypeLV.getLinkage() != ExternalLinkage && + TypeLV.getLinkage() != ModuleLinkage) return LinkageInfo::uniqueExternal(); if (!LV.isVisibilityExplicit()) LV.mergeVisibility(TypeLV); @@ -816,7 +862,9 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, // - a namespace (7.3), unless it is declared within an unnamed // namespace. - } else if (isa(D) && !D->isInAnonymousNamespace()) { + // + // We handled names in anonymous namespaces above. + } else if (isa(D)) { return LV; // By extension, we assign external linkage to Objective-C @@ -1125,6 +1173,8 @@ static LinkageInfo getLVForClosure(const DeclContext *DC, Decl *ContextDecl, if (const auto *ND = dyn_cast(DC)) return getLVForDecl(ND, computation); + // FIXME: We have a closure at TU scope with no context declaration. This + // should probably have no linkage. return LinkageInfo::external(); } @@ -1137,7 +1187,7 @@ static LinkageInfo getLVForLocalDecl(const NamedDecl *D, // This is a "void f();" which got merged with a file static. if (Function->getCanonicalDecl()->getStorageClass() == SC_Static) - return LinkageInfo::internal(); + return getInternalLinkageFor(Function); LinkageInfo LV; if (!hasExplicitVisibilityAlready(computation)) { @@ -1226,7 +1276,7 @@ static LinkageInfo computeLVForDecl(const NamedDecl *D, LVComputationKind computation) { // Internal_linkage attribute overrides other considerations. if (D->hasAttr()) - return LinkageInfo::internal(); + return getInternalLinkageFor(D); // Objective-C: treat all Objective-C declarations as having external // linkage. @@ -1259,8 +1309,7 @@ static LinkageInfo computeLVForDecl(const NamedDecl *D, case Decl::TypeAlias: // A typedef declaration has linkage if it gives a type a name for // linkage purposes. - if (!D->getASTContext().getLangOpts().CPlusPlus || - !cast(D) + if (!cast(D) ->getAnonDeclWithTypedefName(/*AnyRedecl*/true)) return LinkageInfo::none(); break; @@ -1276,14 +1325,14 @@ static LinkageInfo computeLVForDecl(const NamedDecl *D, case Decl::ObjCProperty: case Decl::ObjCPropertyImpl: case Decl::ObjCProtocol: - return LinkageInfo::external(); + return getExternalLinkageFor(D); case Decl::CXXRecord: { const auto *Record = cast(D); if (Record->isLambda()) { if (!Record->getLambdaManglingNumber()) { // This lambda has no mangling number, so it's internal. - return LinkageInfo::internal(); + return getInternalLinkageFor(D); } // This lambda has its linkage/visibility determined: @@ -1299,7 +1348,7 @@ static LinkageInfo computeLVForDecl(const NamedDecl *D, const CXXRecordDecl *OuterMostLambda = getOutermostEnclosingLambda(Record); if (!OuterMostLambda->getLambdaManglingNumber()) - return LinkageInfo::internal(); + return getInternalLinkageFor(D); return getLVForClosure( OuterMostLambda->getDeclContext()->getRedeclContext(), @@ -1350,7 +1399,7 @@ class LinkageComputer { LVComputationKind computation) { // Internal_linkage attribute overrides other considerations. if (D->hasAttr()) - return LinkageInfo::internal(); + return getInternalLinkageFor(D); if (computation == LVForLinkageOnly && D->hasCachedLinkage()) return LinkageInfo(D->getCachedLinkage(), DefaultVisibility, false); diff --git a/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp b/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp index a0594a020362..cd2c83a02f59 100644 --- a/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp +++ b/contrib/llvm/tools/clang/lib/AST/DeclBase.cpp @@ -283,8 +283,10 @@ void Decl::setLexicalDeclContext(DeclContext *DC) { setLocalOwningModule(cast(DC)->getOwningModule()); } - assert((!hasOwningModule() || getOwningModule()) && - "hidden declaration has no owning module"); + assert( + (getModuleOwnershipKind() != ModuleOwnershipKind::VisibleWhenImported || + getOwningModule()) && + "hidden declaration has no owning module"); } void Decl::setDeclContextsImpl(DeclContext *SemaDC, DeclContext *LexicalDC, diff --git a/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp b/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp index 07d128ba555b..5cab48882251 100644 --- a/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp +++ b/contrib/llvm/tools/clang/lib/AST/DeclCXX.cpp @@ -1605,6 +1605,84 @@ CXXMethodDecl *CXXMethodDecl::CreateDeserialized(ASTContext &C, unsigned ID) { SC_None, false, false, SourceLocation()); } +CXXMethodDecl *CXXMethodDecl::getDevirtualizedMethod(const Expr *Base, + bool IsAppleKext) { + assert(isVirtual() && "this method is expected to be virtual"); + + // When building with -fapple-kext, all calls must go through the vtable since + // the kernel linker can do runtime patching of vtables. + if (IsAppleKext) + return nullptr; + + // If the member function is marked 'final', we know that it can't be + // overridden and can therefore devirtualize it unless it's pure virtual. + if (hasAttr()) + return isPure() ? nullptr : this; + + // If Base is unknown, we cannot devirtualize. + if (!Base) + return nullptr; + + // If the base expression (after skipping derived-to-base conversions) is a + // class prvalue, then we can devirtualize. + Base = Base->getBestDynamicClassTypeExpr(); + if (Base->isRValue() && Base->getType()->isRecordType()) + return this; + + // If we don't even know what we would call, we can't devirtualize. + const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType(); + if (!BestDynamicDecl) + return nullptr; + + // There may be a method corresponding to MD in a derived class. + CXXMethodDecl *DevirtualizedMethod = + getCorrespondingMethodInClass(BestDynamicDecl); + + // If that method is pure virtual, we can't devirtualize. If this code is + // reached, the result would be UB, not a direct call to the derived class + // function, and we can't assume the derived class function is defined. + if (DevirtualizedMethod->isPure()) + return nullptr; + + // If that method is marked final, we can devirtualize it. + if (DevirtualizedMethod->hasAttr()) + return DevirtualizedMethod; + + // Similarly, if the class itself is marked 'final' it can't be overridden + // and we can therefore devirtualize the member function call. + if (BestDynamicDecl->hasAttr()) + return DevirtualizedMethod; + + if (const DeclRefExpr *DRE = dyn_cast(Base)) { + if (const VarDecl *VD = dyn_cast(DRE->getDecl())) + if (VD->getType()->isRecordType()) + // This is a record decl. We know the type and can devirtualize it. + return DevirtualizedMethod; + + return nullptr; + } + + // We can devirtualize calls on an object accessed by a class member access + // expression, since by C++11 [basic.life]p6 we know that it can't refer to + // a derived class object constructed in the same location. + if (const MemberExpr *ME = dyn_cast(Base)) + if (const ValueDecl *VD = dyn_cast(ME->getMemberDecl())) + return VD->getType()->isRecordType() ? DevirtualizedMethod : nullptr; + + // Likewise for calls on an object accessed by a (non-reference) pointer to + // member access. + if (auto *BO = dyn_cast(Base)) { + if (BO->isPtrMemOp()) { + auto *MPT = BO->getRHS()->getType()->castAs(); + if (MPT->getPointeeType()->isRecordType()) + return DevirtualizedMethod; + } + } + + // We can't devirtualize the call. + return nullptr; +} + bool CXXMethodDecl::isUsualDeallocationFunction() const { if (getOverloadedOperator() != OO_Delete && getOverloadedOperator() != OO_Array_Delete) diff --git a/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp b/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp index e836135cf2f9..0c0c861e5d56 100644 --- a/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp +++ b/contrib/llvm/tools/clang/lib/AST/ExprConstant.cpp @@ -1665,6 +1665,19 @@ static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc, return true; } +/// Member pointers are constant expressions unless they point to a +/// non-virtual dllimport member function. +static bool CheckMemberPointerConstantExpression(EvalInfo &Info, + SourceLocation Loc, + QualType Type, + const APValue &Value) { + const ValueDecl *Member = Value.getMemberPointerDecl(); + const auto *FD = dyn_cast_or_null(Member); + if (!FD) + return true; + return FD->isVirtual() || !FD->hasAttr(); +} + /// Check that this core constant expression is of literal type, and if not, /// produce an appropriate diagnostic. static bool CheckLiteralType(EvalInfo &Info, const Expr *E, @@ -1757,6 +1770,9 @@ static bool CheckConstantExpression(EvalInfo &Info, SourceLocation DiagLoc, return CheckLValueConstantExpression(Info, DiagLoc, Type, LVal); } + if (Value.isMemberPointer()) + return CheckMemberPointerConstantExpression(Info, DiagLoc, Type, Value); + // Everything else is fine. return true; } @@ -9508,7 +9524,7 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { case BO_Mul: if (Result.isComplexFloat()) { // This is an implementation of complex multiplication according to the - // constraints laid out in C11 Annex G. The implemantion uses the + // constraints laid out in C11 Annex G. The implemention uses the // following naming scheme: // (a + ib) * (c + id) ComplexValue LHS = Result; @@ -9589,7 +9605,7 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { case BO_Div: if (Result.isComplexFloat()) { // This is an implementation of complex division according to the - // constraints laid out in C11 Annex G. The implemantion uses the + // constraints laid out in C11 Annex G. The implemention uses the // following naming scheme: // (a + ib) / (c + id) ComplexValue LHS = Result; diff --git a/contrib/llvm/tools/clang/lib/AST/ExternalASTMerger.cpp b/contrib/llvm/tools/clang/lib/AST/ExternalASTMerger.cpp index b746edaf6439..4f4a99794c5b 100644 --- a/contrib/llvm/tools/clang/lib/AST/ExternalASTMerger.cpp +++ b/contrib/llvm/tools/clang/lib/AST/ExternalASTMerger.cpp @@ -180,8 +180,3 @@ void ExternalASTMerger::FindExternalLexicalDecls( }); } -void ExternalASTMerger::CompleteType(TagDecl *Tag) { - SmallVector Result; - FindExternalLexicalDecls(Tag, [](Decl::Kind) { return true; }, Result); - Tag->setHasExternalLexicalStorage(false); -} diff --git a/contrib/llvm/tools/clang/lib/AST/ODRHash.cpp b/contrib/llvm/tools/clang/lib/AST/ODRHash.cpp index 3f66e58eb868..66b9940b8b08 100644 --- a/contrib/llvm/tools/clang/lib/AST/ODRHash.cpp +++ b/contrib/llvm/tools/clang/lib/AST/ODRHash.cpp @@ -228,6 +228,13 @@ class ODRDeclVisitor : public ConstDeclVisitor { Hash.AddQualType(T); } + void AddDecl(const Decl *D) { + Hash.AddBoolean(D); + if (D) { + Hash.AddDecl(D); + } + } + void Visit(const Decl *D) { ID.AddInteger(D->getKind()); Inherited::Visit(D); @@ -321,6 +328,16 @@ class ODRDeclVisitor : public ConstDeclVisitor { void VisitTypeAliasDecl(const TypeAliasDecl *D) { Inherited::VisitTypeAliasDecl(D); } + + void VisitFriendDecl(const FriendDecl *D) { + TypeSourceInfo *TSI = D->getFriendType(); + Hash.AddBoolean(TSI); + if (TSI) { + AddQualType(TSI->getType()); + } else { + AddDecl(D->getFriendDecl()); + } + } }; // Only allow a small portion of Decl's to be processed. Remove this once @@ -335,6 +352,7 @@ bool ODRHash::isWhitelistedDecl(const Decl *D, const CXXRecordDecl *Parent) { case Decl::AccessSpec: case Decl::CXXMethod: case Decl::Field: + case Decl::Friend: case Decl::StaticAssert: case Decl::TypeAlias: case Decl::Typedef: diff --git a/contrib/llvm/tools/clang/lib/Analysis/AnalysisDeclContext.cpp b/contrib/llvm/tools/clang/lib/Analysis/AnalysisDeclContext.cpp index 7c0f5543da04..ec15f34fb231 100644 --- a/contrib/llvm/tools/clang/lib/Analysis/AnalysisDeclContext.cpp +++ b/contrib/llvm/tools/clang/lib/Analysis/AnalysisDeclContext.cpp @@ -67,6 +67,7 @@ AnalysisDeclContextManager::AnalysisDeclContextManager(bool useUnoptimizedCFG, bool addImplicitDtors, bool addInitializers, bool addTemporaryDtors, + bool addLifetime, bool synthesizeBodies, bool addStaticInitBranch, bool addCXXNewAllocator, @@ -77,6 +78,7 @@ AnalysisDeclContextManager::AnalysisDeclContextManager(bool useUnoptimizedCFG, cfgBuildOptions.AddImplicitDtors = addImplicitDtors; cfgBuildOptions.AddInitializers = addInitializers; cfgBuildOptions.AddTemporaryDtors = addTemporaryDtors; + cfgBuildOptions.AddLifetime = addLifetime; cfgBuildOptions.AddStaticInitBranches = addStaticInitBranch; cfgBuildOptions.AddCXXNewAllocator = addCXXNewAllocator; } diff --git a/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp b/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp index 2a2b3d73b5ca..6a77455edeef 100644 --- a/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp +++ b/contrib/llvm/tools/clang/lib/Analysis/CFG.cpp @@ -233,6 +233,7 @@ class LocalScope { } int distance(const_iterator L); + const_iterator shared_parent(const_iterator L); }; friend class const_iterator; @@ -275,6 +276,30 @@ int LocalScope::const_iterator::distance(LocalScope::const_iterator L) { return D; } +/// Calculates the closest parent of this iterator +/// that is in a scope reachable through the parents of L. +/// I.e. when using 'goto' from this to L, the lifetime of all variables +/// between this and shared_parent(L) end. +LocalScope::const_iterator +LocalScope::const_iterator::shared_parent(LocalScope::const_iterator L) { + llvm::SmallPtrSet ScopesOfL; + while (true) { + ScopesOfL.insert(L.Scope); + if (L == const_iterator()) + break; + L = L.Scope->Prev; + } + + const_iterator F = *this; + while (true) { + if (ScopesOfL.count(F.Scope)) + return F; + assert(F != const_iterator() && + "L iterator is not reachable from F iterator."); + F = F.Scope->Prev; + } +} + /// Structure for specifying position in CFG during its build process. It /// consists of CFGBlock that specifies position in CFG and /// LocalScope::const_iterator that specifies position in LocalScope graph. @@ -579,6 +604,10 @@ class CFGBuilder { CFGBlock *addInitializer(CXXCtorInitializer *I); void addAutomaticObjDtors(LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt *S); + void addLifetimeEnds(LocalScope::const_iterator B, + LocalScope::const_iterator E, Stmt *S); + void addAutomaticObjHandling(LocalScope::const_iterator B, + LocalScope::const_iterator E, Stmt *S); void addImplicitDtorsForDestructor(const CXXDestructorDecl *DD); // Local scopes creation. @@ -619,6 +648,10 @@ class CFGBuilder { B->appendAutomaticObjDtor(VD, S, cfg->getBumpVectorContext()); } + void appendLifetimeEnds(CFGBlock *B, VarDecl *VD, Stmt *S) { + B->appendLifetimeEnds(VD, S, cfg->getBumpVectorContext()); + } + void appendDeleteDtor(CFGBlock *B, CXXRecordDecl *RD, CXXDeleteExpr *DE) { B->appendDeleteDtor(RD, DE, cfg->getBumpVectorContext()); } @@ -626,6 +659,10 @@ class CFGBuilder { void prependAutomaticObjDtorsWithTerminator(CFGBlock *Blk, LocalScope::const_iterator B, LocalScope::const_iterator E); + void prependAutomaticObjLifetimeWithTerminator(CFGBlock *Blk, + LocalScope::const_iterator B, + LocalScope::const_iterator E); + void addSuccessor(CFGBlock *B, CFGBlock *S, bool IsReachable = true) { B->addSuccessor(CFGBlock::AdjacentBlock(S, IsReachable), cfg->getBumpVectorContext()); @@ -957,7 +994,8 @@ class CFGBuilder { return TryResult(); } - + + bool hasTrivialDestructor(VarDecl *VD); }; inline bool AddStmtChoice::alwaysAdd(CFGBuilder &builder, @@ -1031,6 +1069,9 @@ std::unique_ptr CFGBuilder::buildCFG(const Decl *D, Stmt *Statement) { assert(Succ == &cfg->getExit()); Block = nullptr; // the EXIT block is empty. Create all other blocks lazily. + assert(!(BuildOpts.AddImplicitDtors && BuildOpts.AddLifetime) && + "AddImplicitDtors and AddLifetime cannot be used at the same time"); + if (BuildOpts.AddImplicitDtors) if (const CXXDestructorDecl *DD = dyn_cast_or_null(D)) addImplicitDtorsForDestructor(DD); @@ -1067,6 +1108,8 @@ std::unique_ptr CFGBuilder::buildCFG(const Decl *D, Stmt *Statement) { if (LI == LabelMap.end()) continue; JumpTarget JT = LI->second; + prependAutomaticObjLifetimeWithTerminator(B, I->scopePosition, + JT.scopePosition); prependAutomaticObjDtorsWithTerminator(B, I->scopePosition, JT.scopePosition); addSuccessor(B, JT.block); @@ -1209,7 +1252,61 @@ static QualType getReferenceInitTemporaryType(ASTContext &Context, return Init->getType(); } - + +void CFGBuilder::addAutomaticObjHandling(LocalScope::const_iterator B, + LocalScope::const_iterator E, + Stmt *S) { + if (BuildOpts.AddImplicitDtors) + addAutomaticObjDtors(B, E, S); + if (BuildOpts.AddLifetime) + addLifetimeEnds(B, E, S); +} + +/// Add to current block automatic objects that leave the scope. +void CFGBuilder::addLifetimeEnds(LocalScope::const_iterator B, + LocalScope::const_iterator E, Stmt *S) { + if (!BuildOpts.AddLifetime) + return; + + if (B == E) + return; + + // To go from B to E, one first goes up the scopes from B to P + // then sideways in one scope from P to P' and then down + // the scopes from P' to E. + // The lifetime of all objects between B and P end. + LocalScope::const_iterator P = B.shared_parent(E); + int dist = B.distance(P); + if (dist <= 0) + return; + + // We need to perform the scope leaving in reverse order + SmallVector DeclsTrivial; + SmallVector DeclsNonTrivial; + DeclsTrivial.reserve(dist); + DeclsNonTrivial.reserve(dist); + + for (LocalScope::const_iterator I = B; I != P; ++I) + if (hasTrivialDestructor(*I)) + DeclsTrivial.push_back(*I); + else + DeclsNonTrivial.push_back(*I); + + autoCreateBlock(); + // object with trivial destructor end their lifetime last (when storage + // duration ends) + for (SmallVectorImpl::reverse_iterator I = DeclsTrivial.rbegin(), + E = DeclsTrivial.rend(); + I != E; ++I) + appendLifetimeEnds(Block, *I, S); + + for (SmallVectorImpl::reverse_iterator + I = DeclsNonTrivial.rbegin(), + E = DeclsNonTrivial.rend(); + I != E; ++I) + appendLifetimeEnds(Block, *I, S); +} + /// addAutomaticObjDtors - Add to current block automatic objects destructors /// for objects in range of local scope positions. Use S as trigger statement /// for destructors. @@ -1309,7 +1406,7 @@ LocalScope* CFGBuilder::createOrReuseLocalScope(LocalScope* Scope) { /// addLocalScopeForStmt - Add LocalScope to local scopes tree for statement /// that should create implicit scope (e.g. if/else substatements). void CFGBuilder::addLocalScopeForStmt(Stmt *S) { - if (!BuildOpts.AddImplicitDtors) + if (!BuildOpts.AddImplicitDtors && !BuildOpts.AddLifetime) return; LocalScope *Scope = nullptr; @@ -1334,7 +1431,7 @@ void CFGBuilder::addLocalScopeForStmt(Stmt *S) { /// reuse Scope if not NULL. LocalScope* CFGBuilder::addLocalScopeForDeclStmt(DeclStmt *DS, LocalScope* Scope) { - if (!BuildOpts.AddImplicitDtors) + if (!BuildOpts.AddImplicitDtors && !BuildOpts.AddLifetime) return Scope; for (auto *DI : DS->decls()) @@ -1343,12 +1440,50 @@ LocalScope* CFGBuilder::addLocalScopeForDeclStmt(DeclStmt *DS, return Scope; } +bool CFGBuilder::hasTrivialDestructor(VarDecl *VD) { + // Check for const references bound to temporary. Set type to pointee. + QualType QT = VD->getType(); + if (QT.getTypePtr()->isReferenceType()) { + // Attempt to determine whether this declaration lifetime-extends a + // temporary. + // + // FIXME: This is incorrect. Non-reference declarations can lifetime-extend + // temporaries, and a single declaration can extend multiple temporaries. + // We should look at the storage duration on each nested + // MaterializeTemporaryExpr instead. + + const Expr *Init = VD->getInit(); + if (!Init) + return true; + + // Lifetime-extending a temporary. + bool FoundMTE = false; + QT = getReferenceInitTemporaryType(*Context, Init, &FoundMTE); + if (!FoundMTE) + return true; + } + + // Check for constant size array. Set type to array element type. + while (const ConstantArrayType *AT = Context->getAsConstantArrayType(QT)) { + if (AT->getSize() == 0) + return true; + QT = AT->getElementType(); + } + + // Check if type is a C++ class with non-trivial destructor. + if (const CXXRecordDecl *CD = QT->getAsCXXRecordDecl()) + return !CD->hasDefinition() || CD->hasTrivialDestructor(); + return true; +} + /// addLocalScopeForVarDecl - Add LocalScope for variable declaration. It will /// create add scope for automatic objects and temporary objects bound to /// const reference. Will reuse Scope if not NULL. LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl *VD, LocalScope* Scope) { - if (!BuildOpts.AddImplicitDtors) + assert(!(BuildOpts.AddImplicitDtors && BuildOpts.AddLifetime) && + "AddImplicitDtors and AddLifetime cannot be used at the same time"); + if (!BuildOpts.AddImplicitDtors && !BuildOpts.AddLifetime) return Scope; // Check if variable is local. @@ -1360,54 +1495,30 @@ LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl *VD, default: return Scope; } - // Check for const references bound to temporary. Set type to pointee. - QualType QT = VD->getType(); - if (QT.getTypePtr()->isReferenceType()) { - // Attempt to determine whether this declaration lifetime-extends a - // temporary. - // - // FIXME: This is incorrect. Non-reference declarations can lifetime-extend - // temporaries, and a single declaration can extend multiple temporaries. - // We should look at the storage duration on each nested - // MaterializeTemporaryExpr instead. - const Expr *Init = VD->getInit(); - if (!Init) - return Scope; - - // Lifetime-extending a temporary. - bool FoundMTE = false; - QT = getReferenceInitTemporaryType(*Context, Init, &FoundMTE); - if (!FoundMTE) - return Scope; - } - - // Check for constant size array. Set type to array element type. - while (const ConstantArrayType *AT = Context->getAsConstantArrayType(QT)) { - if (AT->getSize() == 0) - return Scope; - QT = AT->getElementType(); - } - - // Check if type is a C++ class with non-trivial destructor. - if (const CXXRecordDecl *CD = QT->getAsCXXRecordDecl()) - if (CD->hasDefinition() && !CD->hasTrivialDestructor()) { + if (BuildOpts.AddImplicitDtors) { + if (!hasTrivialDestructor(VD)) { // Add the variable to scope Scope = createOrReuseLocalScope(Scope); Scope->addVar(VD); ScopePos = Scope->begin(); } + return Scope; + } + + assert(BuildOpts.AddLifetime); + // Add the variable to scope + Scope = createOrReuseLocalScope(Scope); + Scope->addVar(VD); + ScopePos = Scope->begin(); return Scope; } /// addLocalScopeAndDtors - For given statement add local scope for it and /// add destructors that will cleanup the scope. Will reuse Scope if not NULL. void CFGBuilder::addLocalScopeAndDtors(Stmt *S) { - if (!BuildOpts.AddImplicitDtors) - return; - LocalScope::const_iterator scopeBeginPos = ScopePos; addLocalScopeForStmt(S); - addAutomaticObjDtors(ScopePos, scopeBeginPos, S); + addAutomaticObjHandling(ScopePos, scopeBeginPos, S); } /// prependAutomaticObjDtorsWithTerminator - Prepend destructor CFGElements for @@ -1419,6 +1530,8 @@ void CFGBuilder::addLocalScopeAndDtors(Stmt *S) { /// no-return destructors properly. void CFGBuilder::prependAutomaticObjDtorsWithTerminator(CFGBlock *Blk, LocalScope::const_iterator B, LocalScope::const_iterator E) { + if (!BuildOpts.AddImplicitDtors) + return; BumpVectorContext &C = cfg->getBumpVectorContext(); CFGBlock::iterator InsertPos = Blk->beginAutomaticObjDtorsInsert(Blk->end(), B.distance(E), C); @@ -1427,6 +1540,21 @@ void CFGBuilder::prependAutomaticObjDtorsWithTerminator(CFGBlock *Blk, Blk->getTerminator()); } +/// prependAutomaticObjLifetimeWithTerminator - Prepend lifetime CFGElements for +/// variables with automatic storage duration to CFGBlock's elements vector. +/// Elements will be prepended to physical beginning of the vector which +/// happens to be logical end. Use blocks terminator as statement that specifies +/// where lifetime ends. +void CFGBuilder::prependAutomaticObjLifetimeWithTerminator( + CFGBlock *Blk, LocalScope::const_iterator B, LocalScope::const_iterator E) { + if (!BuildOpts.AddLifetime) + return; + BumpVectorContext &C = cfg->getBumpVectorContext(); + CFGBlock::iterator InsertPos = + Blk->beginLifetimeEndsInsert(Blk->end(), B.distance(E), C); + for (LocalScope::const_iterator I = B; I != E; ++I) + InsertPos = Blk->insertLifetimeEnds(InsertPos, *I, Blk->getTerminator()); +} /// Visit - Walk the subtree of a statement and add extra /// blocks for ternary operators, &&, and ||. We also process "," and /// DeclStmts (which may contain nested control-flow). @@ -1815,7 +1943,7 @@ CFGBlock *CFGBuilder::VisitBreakStmt(BreakStmt *B) { // If there is no target for the break, then we are looking at an incomplete // AST. This means that the CFG cannot be constructed. if (BreakJumpTarget.block) { - addAutomaticObjDtors(ScopePos, BreakJumpTarget.scopePosition, B); + addAutomaticObjHandling(ScopePos, BreakJumpTarget.scopePosition, B); addSuccessor(Block, BreakJumpTarget.block); } else badCFG = true; @@ -1947,13 +2075,12 @@ CFGBlock *CFGBuilder::VisitChooseExpr(ChooseExpr *C, CFGBlock *CFGBuilder::VisitCompoundStmt(CompoundStmt *C) { LocalScope::const_iterator scopeBeginPos = ScopePos; - if (BuildOpts.AddImplicitDtors) { - addLocalScopeForStmt(C); - } + addLocalScopeForStmt(C); + if (!C->body_empty() && !isa(*C->body_rbegin())) { // If the body ends with a ReturnStmt, the dtors will be added in // VisitReturnStmt. - addAutomaticObjDtors(ScopePos, scopeBeginPos, C); + addAutomaticObjHandling(ScopePos, scopeBeginPos, C); } CFGBlock *LastBlock = Block; @@ -2183,7 +2310,7 @@ CFGBlock *CFGBuilder::VisitIfStmt(IfStmt *I) { if (VarDecl *VD = I->getConditionVariable()) addLocalScopeForVarDecl(VD); - addAutomaticObjDtors(ScopePos, save_scope_pos.get(), I); + addAutomaticObjHandling(ScopePos, save_scope_pos.get(), I); // The block we were processing is now finished. Make it the successor // block. @@ -2308,7 +2435,7 @@ CFGBlock *CFGBuilder::VisitReturnStmt(ReturnStmt *R) { // Create the new block. Block = createBlock(false); - addAutomaticObjDtors(ScopePos, LocalScope::const_iterator(), R); + addAutomaticObjHandling(ScopePos, LocalScope::const_iterator(), R); // If the one of the destructors does not return, we already have the Exit // block as a successor. @@ -2389,7 +2516,7 @@ CFGBlock *CFGBuilder::VisitGotoStmt(GotoStmt *G) { BackpatchBlocks.push_back(JumpSource(Block, ScopePos)); else { JumpTarget JT = I->second; - addAutomaticObjDtors(ScopePos, JT.scopePosition, G); + addAutomaticObjHandling(ScopePos, JT.scopePosition, G); addSuccessor(Block, JT.block); } @@ -2414,7 +2541,7 @@ CFGBlock *CFGBuilder::VisitForStmt(ForStmt *F) { addLocalScopeForVarDecl(VD); LocalScope::const_iterator ContinueScopePos = ScopePos; - addAutomaticObjDtors(ScopePos, save_scope_pos.get(), F); + addAutomaticObjHandling(ScopePos, save_scope_pos.get(), F); // "for" is a control-flow statement. Thus we stop processing the current // block. @@ -2466,7 +2593,7 @@ CFGBlock *CFGBuilder::VisitForStmt(ForStmt *F) { ContinueJumpTarget.block->setLoopTarget(F); // Loop body should end with destructor of Condition variable (if any). - addAutomaticObjDtors(ScopePos, LoopBeginScopePos, F); + addAutomaticObjHandling(ScopePos, LoopBeginScopePos, F); // If body is not a compound statement create implicit scope // and add destructors. @@ -2753,7 +2880,7 @@ CFGBlock *CFGBuilder::VisitWhileStmt(WhileStmt *W) { LocalScope::const_iterator LoopBeginScopePos = ScopePos; if (VarDecl *VD = W->getConditionVariable()) { addLocalScopeForVarDecl(VD); - addAutomaticObjDtors(ScopePos, LoopBeginScopePos, W); + addAutomaticObjHandling(ScopePos, LoopBeginScopePos, W); } // "while" is a control-flow statement. Thus we stop processing the current @@ -2788,7 +2915,7 @@ CFGBlock *CFGBuilder::VisitWhileStmt(WhileStmt *W) { BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos); // Loop body should end with destructor of Condition variable (if any). - addAutomaticObjDtors(ScopePos, LoopBeginScopePos, W); + addAutomaticObjHandling(ScopePos, LoopBeginScopePos, W); // If body is not a compound statement create implicit scope // and add destructors. @@ -3030,7 +3157,7 @@ CFGBlock *CFGBuilder::VisitContinueStmt(ContinueStmt *C) { // If there is no target for the continue, then we are looking at an // incomplete AST. This means the CFG cannot be constructed. if (ContinueJumpTarget.block) { - addAutomaticObjDtors(ScopePos, ContinueJumpTarget.scopePosition, C); + addAutomaticObjHandling(ScopePos, ContinueJumpTarget.scopePosition, C); addSuccessor(Block, ContinueJumpTarget.block); } else badCFG = true; @@ -3085,7 +3212,7 @@ CFGBlock *CFGBuilder::VisitSwitchStmt(SwitchStmt *Terminator) { if (VarDecl *VD = Terminator->getConditionVariable()) addLocalScopeForVarDecl(VD); - addAutomaticObjDtors(ScopePos, save_scope_pos.get(), Terminator); + addAutomaticObjHandling(ScopePos, save_scope_pos.get(), Terminator); if (Block) { if (badCFG) @@ -3373,7 +3500,7 @@ CFGBlock *CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt *CS) { if (VarDecl *VD = CS->getExceptionDecl()) { LocalScope::const_iterator BeginScopePos = ScopePos; addLocalScopeForVarDecl(VD); - addAutomaticObjDtors(ScopePos, BeginScopePos, CS); + addAutomaticObjHandling(ScopePos, BeginScopePos, CS); } if (CS->getHandlerBlock()) @@ -3427,7 +3554,7 @@ CFGBlock *CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt *S) { addLocalScopeForStmt(Begin); if (Stmt *End = S->getEndStmt()) addLocalScopeForStmt(End); - addAutomaticObjDtors(ScopePos, save_scope_pos.get(), S); + addAutomaticObjHandling(ScopePos, save_scope_pos.get(), S); LocalScope::const_iterator ContinueScopePos = ScopePos; @@ -3898,6 +4025,7 @@ CFGImplicitDtor::getDestructorDecl(ASTContext &astContext) const { case CFGElement::Statement: case CFGElement::Initializer: case CFGElement::NewAllocator: + case CFGElement::LifetimeEnds: llvm_unreachable("getDestructorDecl should only be used with " "ImplicitDtors"); case CFGElement::AutomaticObjectDtor: { @@ -4308,6 +4436,12 @@ static void print_elem(raw_ostream &OS, StmtPrinterHelper &Helper, OS << ".~" << T->getAsCXXRecordDecl()->getName().str() << "()"; OS << " (Implicit destructor)\n"; + } else if (Optional DE = E.getAs()) { + const VarDecl *VD = DE->getVarDecl(); + Helper.handleDecl(VD, OS); + + OS << " (Lifetime ends)\n"; + } else if (Optional NE = E.getAs()) { OS << "CFGNewAllocator("; if (const CXXNewExpr *AllocExpr = NE->getAllocatorExpr()) diff --git a/contrib/llvm/tools/clang/lib/Analysis/CloneDetection.cpp b/contrib/llvm/tools/clang/lib/Analysis/CloneDetection.cpp index ee848ac711d6..5ea74989a7ec 100644 --- a/contrib/llvm/tools/clang/lib/Analysis/CloneDetection.cpp +++ b/contrib/llvm/tools/clang/lib/Analysis/CloneDetection.cpp @@ -16,13 +16,13 @@ #include "clang/AST/ASTContext.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/AST/Stmt.h" -#include "clang/AST/StmtVisitor.h" #include "clang/Lex/Lexer.h" #include "llvm/Support/MD5.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/Path.h" using namespace clang; +using namespace clang::clone_detection; StmtSequence::StmtSequence(const CompoundStmt *Stmt, const Decl *D, unsigned StartIndex, unsigned EndIndex) @@ -103,12 +103,8 @@ static void printMacroName(llvm::raw_string_ostream &MacroStack, MacroStack << " "; } -/// Returns a string that represents all macro expansions that expanded into the -/// given SourceLocation. -/// -/// If 'getMacroStack(A) == getMacroStack(B)' is true, then the SourceLocations -/// A and B are expanded from the same macros in the same order. -static std::string getMacroStack(SourceLocation Loc, ASTContext &Context) { +std::string clone_detection::getMacroStack(SourceLocation Loc, + ASTContext &Context) { std::string MacroStack; llvm::raw_string_ostream MacroStackStream(MacroStack); SourceManager &SM = Context.getSourceManager(); @@ -123,184 +119,6 @@ static std::string getMacroStack(SourceLocation Loc, ASTContext &Context) { return MacroStack; } -namespace { -typedef unsigned DataPiece; - -/// Collects the data of a single Stmt. -/// -/// This class defines what a code clone is: If it collects for two statements -/// the same data, then those two statements are considered to be clones of each -/// other. -/// -/// All collected data is forwarded to the given data consumer of the type T. -/// The data consumer class needs to provide a member method with the signature: -/// update(StringRef Str) -template -class StmtDataCollector : public ConstStmtVisitor> { - - ASTContext &Context; - /// The data sink to which all data is forwarded. - T &DataConsumer; - -public: - /// Collects data of the given Stmt. - /// \param S The given statement. - /// \param Context The ASTContext of S. - /// \param DataConsumer The data sink to which all data is forwarded. - StmtDataCollector(const Stmt *S, ASTContext &Context, T &DataConsumer) - : Context(Context), DataConsumer(DataConsumer) { - this->Visit(S); - } - - // Below are utility methods for appending different data to the vector. - - void addData(DataPiece Integer) { - DataConsumer.update( - StringRef(reinterpret_cast(&Integer), sizeof(Integer))); - } - - void addData(llvm::StringRef Str) { DataConsumer.update(Str); } - - void addData(const QualType &QT) { addData(QT.getAsString()); } - -// The functions below collect the class specific data of each Stmt subclass. - -// Utility macro for defining a visit method for a given class. This method -// calls back to the ConstStmtVisitor to visit all parent classes. -#define DEF_ADD_DATA(CLASS, CODE) \ - void Visit##CLASS(const CLASS *S) { \ - CODE; \ - ConstStmtVisitor::Visit##CLASS(S); \ - } - - DEF_ADD_DATA(Stmt, { - addData(S->getStmtClass()); - // This ensures that macro generated code isn't identical to macro-generated - // code. - addData(getMacroStack(S->getLocStart(), Context)); - addData(getMacroStack(S->getLocEnd(), Context)); - }) - DEF_ADD_DATA(Expr, { addData(S->getType()); }) - - //--- Builtin functionality ----------------------------------------------// - DEF_ADD_DATA(ArrayTypeTraitExpr, { addData(S->getTrait()); }) - DEF_ADD_DATA(ExpressionTraitExpr, { addData(S->getTrait()); }) - DEF_ADD_DATA(PredefinedExpr, { addData(S->getIdentType()); }) - DEF_ADD_DATA(TypeTraitExpr, { - addData(S->getTrait()); - for (unsigned i = 0; i < S->getNumArgs(); ++i) - addData(S->getArg(i)->getType()); - }) - - //--- Calls --------------------------------------------------------------// - DEF_ADD_DATA(CallExpr, { - // Function pointers don't have a callee and we just skip hashing it. - if (const FunctionDecl *D = S->getDirectCallee()) { - // If the function is a template specialization, we also need to handle - // the template arguments as they are not included in the qualified name. - if (auto Args = D->getTemplateSpecializationArgs()) { - std::string ArgString; - - // Print all template arguments into ArgString - llvm::raw_string_ostream OS(ArgString); - for (unsigned i = 0; i < Args->size(); ++i) { - Args->get(i).print(Context.getLangOpts(), OS); - // Add a padding character so that 'foo()' != 'foo()'. - OS << '\n'; - } - OS.flush(); - - addData(ArgString); - } - addData(D->getQualifiedNameAsString()); - } - }) - - //--- Exceptions ---------------------------------------------------------// - DEF_ADD_DATA(CXXCatchStmt, { addData(S->getCaughtType()); }) - - //--- C++ OOP Stmts ------------------------------------------------------// - DEF_ADD_DATA(CXXDeleteExpr, { - addData(S->isArrayFormAsWritten()); - addData(S->isGlobalDelete()); - }) - - //--- Casts --------------------------------------------------------------// - DEF_ADD_DATA(ObjCBridgedCastExpr, { addData(S->getBridgeKind()); }) - - //--- Miscellaneous Exprs ------------------------------------------------// - DEF_ADD_DATA(BinaryOperator, { addData(S->getOpcode()); }) - DEF_ADD_DATA(UnaryOperator, { addData(S->getOpcode()); }) - - //--- Control flow -------------------------------------------------------// - DEF_ADD_DATA(GotoStmt, { addData(S->getLabel()->getName()); }) - DEF_ADD_DATA(IndirectGotoStmt, { - if (S->getConstantTarget()) - addData(S->getConstantTarget()->getName()); - }) - DEF_ADD_DATA(LabelStmt, { addData(S->getDecl()->getName()); }) - DEF_ADD_DATA(MSDependentExistsStmt, { addData(S->isIfExists()); }) - DEF_ADD_DATA(AddrLabelExpr, { addData(S->getLabel()->getName()); }) - - //--- Objective-C --------------------------------------------------------// - DEF_ADD_DATA(ObjCIndirectCopyRestoreExpr, { addData(S->shouldCopy()); }) - DEF_ADD_DATA(ObjCPropertyRefExpr, { - addData(S->isSuperReceiver()); - addData(S->isImplicitProperty()); - }) - DEF_ADD_DATA(ObjCAtCatchStmt, { addData(S->hasEllipsis()); }) - - //--- Miscellaneous Stmts ------------------------------------------------// - DEF_ADD_DATA(CXXFoldExpr, { - addData(S->isRightFold()); - addData(S->getOperator()); - }) - DEF_ADD_DATA(GenericSelectionExpr, { - for (unsigned i = 0; i < S->getNumAssocs(); ++i) { - addData(S->getAssocType(i)); - } - }) - DEF_ADD_DATA(LambdaExpr, { - for (const LambdaCapture &C : S->captures()) { - addData(C.isPackExpansion()); - addData(C.getCaptureKind()); - if (C.capturesVariable()) - addData(C.getCapturedVar()->getType()); - } - addData(S->isGenericLambda()); - addData(S->isMutable()); - }) - DEF_ADD_DATA(DeclStmt, { - auto numDecls = std::distance(S->decl_begin(), S->decl_end()); - addData(static_cast(numDecls)); - for (const Decl *D : S->decls()) { - if (const VarDecl *VD = dyn_cast(D)) { - addData(VD->getType()); - } - } - }) - DEF_ADD_DATA(AsmStmt, { - addData(S->isSimple()); - addData(S->isVolatile()); - addData(S->generateAsmString(Context)); - for (unsigned i = 0; i < S->getNumInputs(); ++i) { - addData(S->getInputConstraint(i)); - } - for (unsigned i = 0; i < S->getNumOutputs(); ++i) { - addData(S->getOutputConstraint(i)); - } - for (unsigned i = 0; i < S->getNumClobbers(); ++i) { - addData(S->getClobber(i)); - } - }) - DEF_ADD_DATA(AttributedStmt, { - for (const Attr *A : S->getAttrs()) { - addData(std::string(A->getSpelling())); - } - }) -}; -} // end anonymous namespace - void CloneDetector::analyzeCodeBody(const Decl *D) { assert(D); assert(D->hasBody()); @@ -421,16 +239,27 @@ size_t RecursiveCloneTypeIIConstraint::saveHash( } if (CS) { - for (unsigned Length = 2; Length <= CS->size(); ++Length) { - for (unsigned Pos = 0; Pos <= CS->size() - Length; ++Pos) { - llvm::MD5 Hash; - for (unsigned i = Pos; i < Pos + Length; ++i) { - size_t ChildHash = ChildHashes[i]; - Hash.update(StringRef(reinterpret_cast(&ChildHash), - sizeof(ChildHash))); + // If we're in a CompoundStmt, we hash all possible combinations of child + // statements to find clones in those subsequences. + // We first go through every possible starting position of a subsequence. + for (unsigned Pos = 0; Pos < CS->size(); ++Pos) { + // Then we try all possible lengths this subsequence could have and + // reuse the same hash object to make sure we only hash every child + // hash exactly once. + llvm::MD5 Hash; + for (unsigned Length = 1; Length <= CS->size() - Pos; ++Length) { + // Grab the current child hash and put it into our hash. We do + // -1 on the index because we start counting the length at 1. + size_t ChildHash = ChildHashes[Pos + Length - 1]; + Hash.update( + StringRef(reinterpret_cast(&ChildHash), sizeof(ChildHash))); + // If we have at least two elements in our subsequence, we can start + // saving it. + if (Length > 1) { + llvm::MD5 SubHash = Hash; + StmtsByHash.push_back(std::make_pair( + createHash(SubHash), StmtSequence(CS, D, Pos, Pos + Length))); } - StmtsByHash.push_back(std::make_pair( - createHash(Hash), StmtSequence(CS, D, Pos, Pos + Length))); } } } diff --git a/contrib/llvm/tools/clang/lib/Basic/Targets.cpp b/contrib/llvm/tools/clang/lib/Basic/Targets.cpp index 4f04489a4a10..50b4fc34ad3a 100644 --- a/contrib/llvm/tools/clang/lib/Basic/Targets.cpp +++ b/contrib/llvm/tools/clang/lib/Basic/Targets.cpp @@ -499,6 +499,10 @@ class LinuxTargetInfo : public OSTargetInfo { switch (Triple.getArch()) { default: break; + case llvm::Triple::mips: + case llvm::Triple::mipsel: + case llvm::Triple::mips64: + case llvm::Triple::mips64el: case llvm::Triple::ppc: case llvm::Triple::ppc64: case llvm::Triple::ppc64le: @@ -2049,7 +2053,7 @@ ArrayRef NVPTXTargetInfo::getGCCRegNames() const { return llvm::makeArrayRef(GCCRegNames); } -static const LangAS::Map AMDGPUNonOpenCLPrivateIsZeroMap = { +static const LangAS::Map AMDGPUPrivIsZeroDefIsGenMap = { 4, // Default 1, // opencl_global 3, // opencl_local @@ -2059,7 +2063,7 @@ static const LangAS::Map AMDGPUNonOpenCLPrivateIsZeroMap = { 2, // cuda_constant 3 // cuda_shared }; -static const LangAS::Map AMDGPUNonOpenCLGenericIsZeroMap = { +static const LangAS::Map AMDGPUGenIsZeroDefIsGenMap = { 0, // Default 1, // opencl_global 3, // opencl_local @@ -2069,7 +2073,7 @@ static const LangAS::Map AMDGPUNonOpenCLGenericIsZeroMap = { 2, // cuda_constant 3 // cuda_shared }; -static const LangAS::Map AMDGPUOpenCLPrivateIsZeroMap = { +static const LangAS::Map AMDGPUPrivIsZeroDefIsPrivMap = { 0, // Default 1, // opencl_global 3, // opencl_local @@ -2079,7 +2083,7 @@ static const LangAS::Map AMDGPUOpenCLPrivateIsZeroMap = { 2, // cuda_constant 3 // cuda_shared }; -static const LangAS::Map AMDGPUOpenCLGenericIsZeroMap = { +static const LangAS::Map AMDGPUGenIsZeroDefIsPrivMap = { 5, // Default 1, // opencl_global 3, // opencl_local @@ -2184,18 +2188,35 @@ class AMDGPUTargetInfo final : public TargetInfo { : DataLayoutStringR600); assert(DataLayout->getAllocaAddrSpace() == AS.Private); + setAddressSpaceMap(Triple.getOS() == llvm::Triple::Mesa3D || + Triple.getEnvironment() == llvm::Triple::OpenCL || + Triple.getEnvironmentName() == "amdgizcl" || + !isAMDGCN(Triple)); UseAddrSpaceMapMangling = true; + + // Set pointer width and alignment for target address space 0. + PointerWidth = PointerAlign = DataLayout->getPointerSizeInBits(); + if (getMaxPointerWidth() == 64) { + LongWidth = LongAlign = 64; + SizeType = UnsignedLong; + PtrDiffType = SignedLong; + IntPtrType = SignedLong; + } + } + + void setAddressSpaceMap(bool DefaultIsPrivate) { + if (isGenericZero(getTriple())) { + AddrSpaceMap = DefaultIsPrivate ? &AMDGPUGenIsZeroDefIsPrivMap + : &AMDGPUGenIsZeroDefIsGenMap; + } else { + AddrSpaceMap = DefaultIsPrivate ? &AMDGPUPrivIsZeroDefIsPrivMap + : &AMDGPUPrivIsZeroDefIsGenMap; + } } void adjust(LangOptions &Opts) override { TargetInfo::adjust(Opts); - if (isGenericZero(getTriple())) { - AddrSpaceMap = Opts.OpenCL ? &AMDGPUOpenCLGenericIsZeroMap - : &AMDGPUNonOpenCLGenericIsZeroMap; - } else { - AddrSpaceMap = Opts.OpenCL ? &AMDGPUOpenCLPrivateIsZeroMap - : &AMDGPUNonOpenCLPrivateIsZeroMap; - } + setAddressSpaceMap(Opts.OpenCL || !isAMDGCN(getTriple())); } uint64_t getPointerWidthV(unsigned AddrSpace) const override { @@ -2208,6 +2229,10 @@ class AMDGPUTargetInfo final : public TargetInfo { return 64; } + uint64_t getPointerAlignV(unsigned AddrSpace) const override { + return getPointerWidthV(AddrSpace); + } + uint64_t getMaxPointerWidth() const override { return getTriple().getArch() == llvm::Triple::amdgcn ? 64 : 32; } @@ -2383,14 +2408,13 @@ class AMDGPUTargetInfo final : public TargetInfo { return LangAS::opencl_constant; } - /// \returns Target specific vtbl ptr address space. - unsigned getVtblPtrAddressSpace() const override { - // \todo: We currently have address spaces defined in AMDGPU Backend. It - // would be nice if we could use it here instead of using bare numbers (same - // applies to getDWARFAddressSpace). - return 2; // constant. + llvm::Optional getConstantAddressSpace() const override { + return LangAS::FirstTargetAddressSpace + AS.Constant; } + /// \returns Target specific vtbl ptr address space. + unsigned getVtblPtrAddressSpace() const override { return AS.Constant; } + /// \returns If a target requires an address within a target specific address /// space \p AddressSpace to be converted in order to be used, then return the /// corresponding target specific DWARF address space. @@ -3874,7 +3898,7 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts, case CK_PentiumMMX: Builder.defineMacro("__pentium_mmx__"); Builder.defineMacro("__tune_pentium_mmx__"); - // Fallthrough + LLVM_FALLTHROUGH; case CK_i586: case CK_Pentium: defineCPUMacros(Builder, "i586"); @@ -3884,15 +3908,15 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts, case CK_Pentium3M: case CK_PentiumM: Builder.defineMacro("__tune_pentium3__"); - // Fallthrough + LLVM_FALLTHROUGH; case CK_Pentium2: case CK_C3_2: Builder.defineMacro("__tune_pentium2__"); - // Fallthrough + LLVM_FALLTHROUGH; case CK_PentiumPro: Builder.defineMacro("__tune_i686__"); Builder.defineMacro("__tune_pentiumpro__"); - // Fallthrough + LLVM_FALLTHROUGH; case CK_i686: Builder.defineMacro("__i686"); Builder.defineMacro("__i686__"); @@ -3948,7 +3972,7 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts, case CK_K6_2: Builder.defineMacro("__k6_2__"); Builder.defineMacro("__tune_k6_2__"); - // Fallthrough + LLVM_FALLTHROUGH; case CK_K6_3: if (CPU != CK_K6_2) { // In case of fallthrough // FIXME: GCC may be enabling these in cases where some other k6 @@ -3957,7 +3981,7 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__k6_3__"); Builder.defineMacro("__tune_k6_3__"); } - // Fallthrough + LLVM_FALLTHROUGH; case CK_K6: defineCPUMacros(Builder, "k6"); break; @@ -6310,9 +6334,6 @@ class AArch64TargetInfo : public TargetInfo { MacroBuilder &Builder) const { // Also include the ARMv8.1 defines getTargetDefinesARMV81A(Opts, Builder); - - if (FPU == NeonMode && HasFullFP16) - Builder.defineMacro("__ARM_FEATURE_FP16_VECTOR_ARITHMETIC", "1"); } void getTargetDefines(const LangOptions &Opts, @@ -7408,13 +7429,14 @@ class SystemZTargetInfo : public TargetInfo { static const Builtin::Info BuiltinInfo[]; static const char *const GCCRegNames[]; std::string CPU; + int ISARevision; bool HasTransactionalExecution; bool HasVector; public: SystemZTargetInfo(const llvm::Triple &Triple, const TargetOptions &) - : TargetInfo(Triple), CPU("z10"), HasTransactionalExecution(false), - HasVector(false) { + : TargetInfo(Triple), CPU("z10"), ISARevision(8), + HasTransactionalExecution(false), HasVector(false) { IntMaxType = SignedLong; Int64Type = SignedLong; TLSSupported = true; @@ -7436,14 +7458,7 @@ class SystemZTargetInfo : public TargetInfo { Builder.defineMacro("__zarch__"); Builder.defineMacro("__LONG_DOUBLE_128__"); - const std::string ISARev = llvm::StringSwitch(CPU) - .Cases("arch8", "z10", "8") - .Cases("arch9", "z196", "9") - .Cases("arch10", "zEC12", "10") - .Cases("arch11", "z13", "11") - .Default(""); - if (!ISARev.empty()) - Builder.defineMacro("__ARCH__", ISARev); + Builder.defineMacro("__ARCH__", Twine(ISARevision)); Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1"); Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2"); @@ -7476,37 +7491,35 @@ class SystemZTargetInfo : public TargetInfo { BuiltinVaListKind getBuiltinVaListKind() const override { return TargetInfo::SystemZBuiltinVaList; } + int getISARevision(const StringRef &Name) const { + return llvm::StringSwitch(Name) + .Cases("arch8", "z10", 8) + .Cases("arch9", "z196", 9) + .Cases("arch10", "zEC12", 10) + .Cases("arch11", "z13", 11) + .Default(-1); + } bool setCPU(const std::string &Name) override { CPU = Name; - bool CPUKnown = llvm::StringSwitch(Name) - .Case("z10", true) - .Case("arch8", true) - .Case("z196", true) - .Case("arch9", true) - .Case("zEC12", true) - .Case("arch10", true) - .Case("z13", true) - .Case("arch11", true) - .Default(false); - - return CPUKnown; + ISARevision = getISARevision(CPU); + return ISARevision != -1; } bool initFeatureMap(llvm::StringMap &Features, DiagnosticsEngine &Diags, StringRef CPU, const std::vector &FeaturesVec) const override { - if (CPU == "zEC12" || CPU == "arch10") - Features["transactional-execution"] = true; - if (CPU == "z13" || CPU == "arch11") { + int ISARevision = getISARevision(CPU); + if (ISARevision >= 10) Features["transactional-execution"] = true; + if (ISARevision >= 11) Features["vector"] = true; - } return TargetInfo::initFeatureMap(Features, Diags, CPU, FeaturesVec); } bool handleTargetFeatures(std::vector &Features, DiagnosticsEngine &Diags) override { HasTransactionalExecution = false; + HasVector = false; for (const auto &Feature : Features) { if (Feature == "+transactional-execution") HasTransactionalExecution = true; @@ -7525,6 +7538,10 @@ class SystemZTargetInfo : public TargetInfo { bool hasFeature(StringRef Feature) const override { return llvm::StringSwitch(Feature) .Case("systemz", true) + .Case("arch8", ISARevision >= 8) + .Case("arch9", ISARevision >= 9) + .Case("arch10", ISARevision >= 10) + .Case("arch11", ISARevision >= 11) .Case("htm", HasTransactionalExecution) .Case("vx", HasVector) .Default(false); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp index 9b3850abcce7..513896d98634 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp @@ -998,7 +998,7 @@ static void runThinLTOBackend(ModuleSummaryIndex *CombinedIndex, Module *M, std::unique_ptr OS, std::string SampleProfile, BackendAction Action) { - StringMap> + StringMap> ModuleToDefinedGVSummaries; CombinedIndex->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp index 528a2b33acf8..181048957879 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp @@ -736,9 +736,9 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { llvm::Constant *isa = (!CGM.getContext().getLangOpts().OpenCL) ? CGM.getNSConcreteStackBlock() - : CGM.getNullPointer(cast( - CGM.getNSConcreteStackBlock()->getType()), - QualType(getContext().VoidPtrTy)); + : CGM.getNullPointer(VoidPtrPtrTy, + CGM.getContext().getPointerType( + QualType(CGM.getContext().VoidPtrTy))); isa = llvm::ConstantExpr::getBitCast(isa, VoidPtrTy); // Build the block descriptor. @@ -1141,12 +1141,11 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, auto fields = builder.beginStruct(); // isa - fields.add( - (!CGM.getContext().getLangOpts().OpenCL) - ? CGM.getNSConcreteGlobalBlock() - : CGM.getNullPointer(cast( - CGM.getNSConcreteGlobalBlock()->getType()), - QualType(CGM.getContext().VoidPtrTy))); + fields.add((!CGM.getContext().getLangOpts().OpenCL) + ? CGM.getNSConcreteGlobalBlock() + : CGM.getNullPointer(CGM.VoidPtrPtrTy, + CGM.getContext().getPointerType(QualType( + CGM.getContext().VoidPtrTy)))); // __flags BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE; @@ -1255,7 +1254,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, // For OpenCL passed block pointer can be private AS local variable or // global AS program scope variable (for the case with and without captures). - // Generic AS is used therefore to be able to accomodate both private and + // Generic AS is used therefore to be able to accommodate both private and // generic AS in one implementation. if (getLangOpts().OpenCL) selfTy = getContext().getPointerType(getContext().getAddrSpaceQualType( diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp index 2a6e92e7f3ce..bc902507c46e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp @@ -1810,12 +1810,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, case Builtin::BI__atomic_signal_fence: case Builtin::BI__c11_atomic_thread_fence: case Builtin::BI__c11_atomic_signal_fence: { - llvm::SynchronizationScope Scope; + llvm::SyncScope::ID SSID; if (BuiltinID == Builtin::BI__atomic_signal_fence || BuiltinID == Builtin::BI__c11_atomic_signal_fence) - Scope = llvm::SingleThread; + SSID = llvm::SyncScope::SingleThread; else - Scope = llvm::CrossThread; + SSID = llvm::SyncScope::System; Value *Order = EmitScalarExpr(E->getArg(0)); if (isa(Order)) { int ord = cast(Order)->getZExtValue(); @@ -1825,17 +1825,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, break; case 1: // memory_order_consume case 2: // memory_order_acquire - Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); + Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID); break; case 3: // memory_order_release - Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); + Builder.CreateFence(llvm::AtomicOrdering::Release, SSID); break; case 4: // memory_order_acq_rel - Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); + Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID); break; case 5: // memory_order_seq_cst - Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, - Scope); + Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID); break; } return RValue::get(nullptr); @@ -1852,23 +1851,23 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); Builder.SetInsertPoint(AcquireBB); - Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); + Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(1), AcquireBB); SI->addCase(Builder.getInt32(2), AcquireBB); Builder.SetInsertPoint(ReleaseBB); - Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); + Builder.CreateFence(llvm::AtomicOrdering::Release, SSID); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(3), ReleaseBB); Builder.SetInsertPoint(AcqRelBB); - Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); + Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(4), AcqRelBB); Builder.SetInsertPoint(SeqCstBB); - Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope); + Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(5), SeqCstBB); @@ -2956,9 +2955,8 @@ static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); case NeonTypeFlags::Int16: case NeonTypeFlags::Poly16: - return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); case NeonTypeFlags::Float16: - return llvm::VectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad)); + return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); case NeonTypeFlags::Int32: return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); case NeonTypeFlags::Int64: @@ -2981,8 +2979,6 @@ static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags) { int IsQuad = IntTypeFlags.isQuad(); switch (IntTypeFlags.getEltType()) { - case NeonTypeFlags::Int16: - return llvm::VectorType::get(CGF->HalfTy, (4 << IsQuad)); case NeonTypeFlags::Int32: return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad)); case NeonTypeFlags::Int64: @@ -3130,80 +3126,55 @@ static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0), NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), NEONMAP0(vcvt_f32_v), - NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), - NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0), NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), - NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0), NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), - NEONMAP0(vcvt_s16_v), NEONMAP0(vcvt_s32_v), NEONMAP0(vcvt_s64_v), - NEONMAP0(vcvt_u16_v), NEONMAP0(vcvt_u32_v), NEONMAP0(vcvt_u64_v), - NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0), NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), - NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0), NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), - NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0), NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), - NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0), NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), - NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0), NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), - NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0), NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), - NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0), NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), - NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0), NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), - NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0), NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), - NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0), NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), - NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0), NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), - NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0), NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), - NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0), NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), - NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0), NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), - NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0), NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), NEONMAP0(vcvtq_f32_v), - NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), - NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0), NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), - NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0), NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), - NEONMAP0(vcvtq_s16_v), NEONMAP0(vcvtq_s32_v), NEONMAP0(vcvtq_s64_v), - NEONMAP0(vcvtq_u16_v), NEONMAP0(vcvtq_u32_v), NEONMAP0(vcvtq_u64_v), NEONMAP0(vext_v), @@ -3366,27 +3337,19 @@ static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { NEONMAP1(vcnt_v, ctpop, Add1ArgType), NEONMAP1(vcntq_v, ctpop, Add1ArgType), NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0), - NEONMAP0(vcvt_f16_v), NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), NEONMAP0(vcvt_f32_v), - NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), - NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0), NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), - NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0), NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), - NEONMAP0(vcvtq_f16_v), NEONMAP0(vcvtq_f32_v), - NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), - NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0), NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), - NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0), NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), @@ -3855,20 +3818,9 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vcageq_v: case NEON::BI__builtin_neon_vcagt_v: case NEON::BI__builtin_neon_vcagtq_v: { - llvm::Type *Ty; - switch (VTy->getScalarSizeInBits()) { - default: llvm_unreachable("unexpected type"); - case 32: - Ty = FloatTy; - break; - case 64: - Ty = DoubleTy; - break; - case 16: - Ty = HalfTy; - break; - } - llvm::Type *VecFlt = llvm::VectorType::get(Ty, VTy->getNumElements()); + llvm::Type *VecFlt = llvm::VectorType::get( + VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, + VTy->getNumElements()); llvm::Type *Tys[] = { VTy, VecFlt }; Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); return EmitNeonCall(F, Ops, NameHint); @@ -3885,16 +3837,8 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); - case NEON::BI__builtin_neon_vcvt_f16_v: - case NEON::BI__builtin_neon_vcvtq_f16_v: - Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad)); - return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") - : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); - case NEON::BI__builtin_neon_vcvt_n_f16_v: case NEON::BI__builtin_neon_vcvt_n_f32_v: case NEON::BI__builtin_neon_vcvt_n_f64_v: - case NEON::BI__builtin_neon_vcvtq_n_f16_v: case NEON::BI__builtin_neon_vcvtq_n_f32_v: case NEON::BI__builtin_neon_vcvtq_n_f64_v: { llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty }; @@ -3902,15 +3846,11 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Function *F = CGM.getIntrinsic(Int, Tys); return EmitNeonCall(F, Ops, "vcvt_n"); } - case NEON::BI__builtin_neon_vcvt_n_s16_v: case NEON::BI__builtin_neon_vcvt_n_s32_v: - case NEON::BI__builtin_neon_vcvt_n_u16_v: case NEON::BI__builtin_neon_vcvt_n_u32_v: case NEON::BI__builtin_neon_vcvt_n_s64_v: case NEON::BI__builtin_neon_vcvt_n_u64_v: - case NEON::BI__builtin_neon_vcvtq_n_s16_v: case NEON::BI__builtin_neon_vcvtq_n_s32_v: - case NEON::BI__builtin_neon_vcvtq_n_u16_v: case NEON::BI__builtin_neon_vcvtq_n_u32_v: case NEON::BI__builtin_neon_vcvtq_n_s64_v: case NEON::BI__builtin_neon_vcvtq_n_u64_v: { @@ -3922,63 +3862,44 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vcvt_u32_v: case NEON::BI__builtin_neon_vcvt_s64_v: case NEON::BI__builtin_neon_vcvt_u64_v: - case NEON::BI__builtin_neon_vcvt_s16_v: - case NEON::BI__builtin_neon_vcvt_u16_v: case NEON::BI__builtin_neon_vcvtq_s32_v: case NEON::BI__builtin_neon_vcvtq_u32_v: case NEON::BI__builtin_neon_vcvtq_s64_v: - case NEON::BI__builtin_neon_vcvtq_u64_v: - case NEON::BI__builtin_neon_vcvtq_s16_v: - case NEON::BI__builtin_neon_vcvtq_u16_v: { + case NEON::BI__builtin_neon_vcvtq_u64_v: { Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); } - case NEON::BI__builtin_neon_vcvta_s16_v: case NEON::BI__builtin_neon_vcvta_s32_v: case NEON::BI__builtin_neon_vcvta_s64_v: case NEON::BI__builtin_neon_vcvta_u32_v: case NEON::BI__builtin_neon_vcvta_u64_v: - case NEON::BI__builtin_neon_vcvtaq_s16_v: case NEON::BI__builtin_neon_vcvtaq_s32_v: case NEON::BI__builtin_neon_vcvtaq_s64_v: - case NEON::BI__builtin_neon_vcvtaq_u16_v: case NEON::BI__builtin_neon_vcvtaq_u32_v: case NEON::BI__builtin_neon_vcvtaq_u64_v: - case NEON::BI__builtin_neon_vcvtn_s16_v: case NEON::BI__builtin_neon_vcvtn_s32_v: case NEON::BI__builtin_neon_vcvtn_s64_v: - case NEON::BI__builtin_neon_vcvtn_u16_v: case NEON::BI__builtin_neon_vcvtn_u32_v: case NEON::BI__builtin_neon_vcvtn_u64_v: - case NEON::BI__builtin_neon_vcvtnq_s16_v: case NEON::BI__builtin_neon_vcvtnq_s32_v: case NEON::BI__builtin_neon_vcvtnq_s64_v: - case NEON::BI__builtin_neon_vcvtnq_u16_v: case NEON::BI__builtin_neon_vcvtnq_u32_v: case NEON::BI__builtin_neon_vcvtnq_u64_v: - case NEON::BI__builtin_neon_vcvtp_s16_v: case NEON::BI__builtin_neon_vcvtp_s32_v: case NEON::BI__builtin_neon_vcvtp_s64_v: - case NEON::BI__builtin_neon_vcvtp_u16_v: case NEON::BI__builtin_neon_vcvtp_u32_v: case NEON::BI__builtin_neon_vcvtp_u64_v: - case NEON::BI__builtin_neon_vcvtpq_s16_v: case NEON::BI__builtin_neon_vcvtpq_s32_v: case NEON::BI__builtin_neon_vcvtpq_s64_v: - case NEON::BI__builtin_neon_vcvtpq_u16_v: case NEON::BI__builtin_neon_vcvtpq_u32_v: case NEON::BI__builtin_neon_vcvtpq_u64_v: - case NEON::BI__builtin_neon_vcvtm_s16_v: case NEON::BI__builtin_neon_vcvtm_s32_v: case NEON::BI__builtin_neon_vcvtm_s64_v: - case NEON::BI__builtin_neon_vcvtm_u16_v: case NEON::BI__builtin_neon_vcvtm_u32_v: case NEON::BI__builtin_neon_vcvtm_u64_v: - case NEON::BI__builtin_neon_vcvtmq_s16_v: case NEON::BI__builtin_neon_vcvtmq_s32_v: case NEON::BI__builtin_neon_vcvtmq_s64_v: - case NEON::BI__builtin_neon_vcvtmq_u16_v: case NEON::BI__builtin_neon_vcvtmq_u32_v: case NEON::BI__builtin_neon_vcvtmq_u64_v: { llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; @@ -6188,9 +6109,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[2] = EmitNeonSplat(Ops[2], cast(Ops[3])); return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); } - case NEON::BI__builtin_neon_vfmah_lane_f16: case NEON::BI__builtin_neon_vfmas_lane_f32: - case NEON::BI__builtin_neon_vfmah_laneq_f16: case NEON::BI__builtin_neon_vfmas_laneq_f32: case NEON::BI__builtin_neon_vfmad_lane_f64: case NEON::BI__builtin_neon_vfmad_laneq_f64: { @@ -6365,25 +6284,18 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vcvt_u32_v: case NEON::BI__builtin_neon_vcvt_s64_v: case NEON::BI__builtin_neon_vcvt_u64_v: - case NEON::BI__builtin_neon_vcvt_s16_v: - case NEON::BI__builtin_neon_vcvt_u16_v: case NEON::BI__builtin_neon_vcvtq_s32_v: case NEON::BI__builtin_neon_vcvtq_u32_v: case NEON::BI__builtin_neon_vcvtq_s64_v: - case NEON::BI__builtin_neon_vcvtq_u64_v: - case NEON::BI__builtin_neon_vcvtq_s16_v: - case NEON::BI__builtin_neon_vcvtq_u16_v: { + case NEON::BI__builtin_neon_vcvtq_u64_v: { Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); if (usgn) return Builder.CreateFPToUI(Ops[0], Ty); return Builder.CreateFPToSI(Ops[0], Ty); } - case NEON::BI__builtin_neon_vcvta_s16_v: case NEON::BI__builtin_neon_vcvta_s32_v: - case NEON::BI__builtin_neon_vcvtaq_s16_v: case NEON::BI__builtin_neon_vcvtaq_s32_v: case NEON::BI__builtin_neon_vcvta_u32_v: - case NEON::BI__builtin_neon_vcvtaq_u16_v: case NEON::BI__builtin_neon_vcvtaq_u32_v: case NEON::BI__builtin_neon_vcvta_s64_v: case NEON::BI__builtin_neon_vcvtaq_s64_v: @@ -6393,13 +6305,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); } - case NEON::BI__builtin_neon_vcvtm_s16_v: case NEON::BI__builtin_neon_vcvtm_s32_v: - case NEON::BI__builtin_neon_vcvtmq_s16_v: case NEON::BI__builtin_neon_vcvtmq_s32_v: - case NEON::BI__builtin_neon_vcvtm_u16_v: case NEON::BI__builtin_neon_vcvtm_u32_v: - case NEON::BI__builtin_neon_vcvtmq_u16_v: case NEON::BI__builtin_neon_vcvtmq_u32_v: case NEON::BI__builtin_neon_vcvtm_s64_v: case NEON::BI__builtin_neon_vcvtmq_s64_v: @@ -6409,13 +6317,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); } - case NEON::BI__builtin_neon_vcvtn_s16_v: case NEON::BI__builtin_neon_vcvtn_s32_v: - case NEON::BI__builtin_neon_vcvtnq_s16_v: case NEON::BI__builtin_neon_vcvtnq_s32_v: - case NEON::BI__builtin_neon_vcvtn_u16_v: case NEON::BI__builtin_neon_vcvtn_u32_v: - case NEON::BI__builtin_neon_vcvtnq_u16_v: case NEON::BI__builtin_neon_vcvtnq_u32_v: case NEON::BI__builtin_neon_vcvtn_s64_v: case NEON::BI__builtin_neon_vcvtnq_s64_v: @@ -6425,13 +6329,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); } - case NEON::BI__builtin_neon_vcvtp_s16_v: case NEON::BI__builtin_neon_vcvtp_s32_v: - case NEON::BI__builtin_neon_vcvtpq_s16_v: case NEON::BI__builtin_neon_vcvtpq_s32_v: - case NEON::BI__builtin_neon_vcvtp_u16_v: case NEON::BI__builtin_neon_vcvtp_u32_v: - case NEON::BI__builtin_neon_vcvtpq_u16_v: case NEON::BI__builtin_neon_vcvtpq_u32_v: case NEON::BI__builtin_neon_vcvtp_s64_v: case NEON::BI__builtin_neon_vcvtpq_s64_v: @@ -6604,24 +6504,6 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); return Builder.CreateTrunc(Ops[0], Int16Ty); } - case NEON::BI__builtin_neon_vmaxv_f16: { - Int = Intrinsic::aarch64_neon_fmaxv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 4); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } - case NEON::BI__builtin_neon_vmaxvq_f16: { - Int = Intrinsic::aarch64_neon_fmaxv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 8); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } case NEON::BI__builtin_neon_vminv_u8: { Int = Intrinsic::aarch64_neon_uminv; Ty = Int32Ty; @@ -6694,60 +6576,6 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); return Builder.CreateTrunc(Ops[0], Int16Ty); } - case NEON::BI__builtin_neon_vminv_f16: { - Int = Intrinsic::aarch64_neon_fminv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 4); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } - case NEON::BI__builtin_neon_vminvq_f16: { - Int = Intrinsic::aarch64_neon_fminv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 8); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } - case NEON::BI__builtin_neon_vmaxnmv_f16: { - Int = Intrinsic::aarch64_neon_fmaxnmv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 4); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } - case NEON::BI__builtin_neon_vmaxnmvq_f16: { - Int = Intrinsic::aarch64_neon_fmaxnmv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 8); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } - case NEON::BI__builtin_neon_vminnmv_f16: { - Int = Intrinsic::aarch64_neon_fminnmv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 4); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } - case NEON::BI__builtin_neon_vminnmvq_f16: { - Int = Intrinsic::aarch64_neon_fminnmv; - Ty = HalfTy; - VTy = llvm::VectorType::get(HalfTy, 8); - llvm::Type *Tys[2] = { Ty, VTy }; - Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); - return Builder.CreateTrunc(Ops[0], HalfTy); - } case NEON::BI__builtin_neon_vmul_n_f64: { Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); @@ -7506,6 +7334,8 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, AVX512PF, AVX512VBMI, AVX512IFMA, + AVX5124VNNIW, // TODO implement this fully + AVX5124FMAPS, // TODO implement this fully AVX512VPOPCNTDQ, MAX }; @@ -8208,13 +8038,13 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, case X86::BI__faststorefence: { return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, - llvm::CrossThread); + llvm::SyncScope::System); } case X86::BI_ReadWriteBarrier: case X86::BI_ReadBarrier: case X86::BI_WriteBarrier: { return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, - llvm::SingleThread); + llvm::SyncScope::SingleThread); } case X86::BI_BitScanForward: case X86::BI_BitScanForward64: diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp index 13a156c7bbd7..cee656a62fe7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp @@ -129,7 +129,7 @@ static void addExtParameterInfosForCall( paramInfos.resize(totalArgs); } -/// Adds the formal paramaters in FPT to the given prefix. If any parameter in +/// Adds the formal parameters in FPT to the given prefix. If any parameter in /// FPT has pass_object_size attrs, then we'll add parameters for those, too. static void appendParameterTypes(const CodeGenTypes &CGT, SmallVectorImpl &prefix, diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp index 127d7df348ee..50d702c62268 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp @@ -2716,88 +2716,6 @@ llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad( cast(VTable->getType())->getElementType()); } -bool -CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, - const CXXMethodDecl *MD) { - // When building with -fapple-kext, all calls must go through the vtable since - // the kernel linker can do runtime patching of vtables. - if (getLangOpts().AppleKext) - return false; - - // If the member function is marked 'final', we know that it can't be - // overridden and can therefore devirtualize it unless it's pure virtual. - if (MD->hasAttr()) - return !MD->isPure(); - - // If the base expression (after skipping derived-to-base conversions) is a - // class prvalue, then we can devirtualize. - Base = Base->getBestDynamicClassTypeExpr(); - if (Base->isRValue() && Base->getType()->isRecordType()) - return true; - - // If we don't even know what we would call, we can't devirtualize. - const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType(); - if (!BestDynamicDecl) - return false; - - // There may be a method corresponding to MD in a derived class. - const CXXMethodDecl *DevirtualizedMethod = - MD->getCorrespondingMethodInClass(BestDynamicDecl); - - // If that method is pure virtual, we can't devirtualize. If this code is - // reached, the result would be UB, not a direct call to the derived class - // function, and we can't assume the derived class function is defined. - if (DevirtualizedMethod->isPure()) - return false; - - // If that method is marked final, we can devirtualize it. - if (DevirtualizedMethod->hasAttr()) - return true; - - // Similarly, if the class itself is marked 'final' it can't be overridden - // and we can therefore devirtualize the member function call. - if (BestDynamicDecl->hasAttr()) - return true; - - if (const DeclRefExpr *DRE = dyn_cast(Base)) { - if (const VarDecl *VD = dyn_cast(DRE->getDecl())) { - // This is a record decl. We know the type and can devirtualize it. - return VD->getType()->isRecordType(); - } - - return false; - } - - // We can devirtualize calls on an object accessed by a class member access - // expression, since by C++11 [basic.life]p6 we know that it can't refer to - // a derived class object constructed in the same location. However, we avoid - // devirtualizing a call to a template function that we could instantiate - // implicitly, but have not decided to do so. This is needed because if this - // function does not get instantiated, the devirtualization will create a - // direct call to a function whose body may not exist. In contrast, calls to - // template functions that are not defined in this TU are allowed to be - // devirtualized under assumption that it is user responsibility to - // instantiate them in some other TU. - if (const MemberExpr *ME = dyn_cast(Base)) - if (const ValueDecl *VD = dyn_cast(ME->getMemberDecl())) - return VD->getType()->isRecordType() && - (MD->instantiationIsPending() || MD->isDefined() || - !MD->isImplicitlyInstantiable()); - - // Likewise for calls on an object accessed by a (non-reference) pointer to - // member access. - if (auto *BO = dyn_cast(Base)) { - if (BO->isPtrMemOp()) { - auto *MPT = BO->getRHS()->getType()->castAs(); - if (MPT->getPointeeType()->isRecordType()) - return true; - } - } - - // We can't devirtualize the call. - return false; -} - void CodeGenFunction::EmitForwardingCallToLambda( const CXXMethodDecl *callOperator, CallArgList &callArgs) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp index 4b656ea4a879..23517867437c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp @@ -221,8 +221,8 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl( Name = getStaticDeclName(*this, D); llvm::Type *LTy = getTypes().ConvertTypeForMem(Ty); - unsigned AddrSpace = - GetGlobalVarAddressSpace(&D, getContext().getTargetAddressSpace(Ty)); + unsigned AS = GetGlobalVarAddressSpace(&D); + unsigned TargetAS = getContext().getTargetAddressSpace(AS); // Local address space cannot have an initializer. llvm::Constant *Init = nullptr; @@ -231,12 +231,9 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl( else Init = llvm::UndefValue::get(LTy); - llvm::GlobalVariable *GV = - new llvm::GlobalVariable(getModule(), LTy, - Ty.isConstant(getContext()), Linkage, - Init, Name, nullptr, - llvm::GlobalVariable::NotThreadLocal, - AddrSpace); + llvm::GlobalVariable *GV = new llvm::GlobalVariable( + getModule(), LTy, Ty.isConstant(getContext()), Linkage, Init, Name, + nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS); GV->setAlignment(getContext().getDeclAlign(&D).getQuantity()); setGlobalVisibility(GV, &D); @@ -254,11 +251,12 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl( } // Make sure the result is of the correct type. - unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(Ty); + unsigned ExpectedAS = Ty.getAddressSpace(); llvm::Constant *Addr = GV; - if (AddrSpace != ExpectedAddrSpace) { - llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace); - Addr = llvm::ConstantExpr::getAddrSpaceCast(GV, PTy); + if (AS != ExpectedAS) { + Addr = getTargetCodeGenInfo().performAddrSpaceCast( + *this, GV, AS, ExpectedAS, + LTy->getPointerTo(getContext().getTargetAddressSpace(ExpectedAS))); } setStaticLocalDeclAddress(&D, Addr); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp index 2ee1c96a6619..9f40ee5a00a3 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp @@ -338,9 +338,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } } -static Address -createReferenceTemporary(CodeGenFunction &CGF, - const MaterializeTemporaryExpr *M, const Expr *Inner) { +static Address createReferenceTemporary(CodeGenFunction &CGF, + const MaterializeTemporaryExpr *M, + const Expr *Inner) { + auto &TCG = CGF.getTargetHooks(); switch (M->getStorageDuration()) { case SD_FullExpression: case SD_Automatic: { @@ -353,13 +354,24 @@ createReferenceTemporary(CodeGenFunction &CGF, (Ty->isArrayType() || Ty->isRecordType()) && CGF.CGM.isTypeConstant(Ty, true)) if (llvm::Constant *Init = CGF.CGM.EmitConstantExpr(Inner, Ty, &CGF)) { - auto *GV = new llvm::GlobalVariable( - CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, - llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp"); - CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); - GV->setAlignment(alignment.getQuantity()); - // FIXME: Should we put the new global into a COMDAT? - return Address(GV, alignment); + if (auto AddrSpace = CGF.getTarget().getConstantAddressSpace()) { + auto AS = AddrSpace.getValue(); + auto *GV = new llvm::GlobalVariable( + CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr, + llvm::GlobalValue::NotThreadLocal, + CGF.getContext().getTargetAddressSpace(AS)); + CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); + GV->setAlignment(alignment.getQuantity()); + llvm::Constant *C = GV; + if (AS != LangAS::Default) + C = TCG.performAddrSpaceCast( + CGF.CGM, GV, AS, LangAS::Default, + GV->getValueType()->getPointerTo( + CGF.getContext().getTargetAddressSpace(LangAS::Default))); + // FIXME: Should we put the new global into a COMDAT? + return Address(C, alignment); + } } return CGF.CreateMemTemp(Ty, "ref.tmp"); } @@ -440,9 +452,11 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // Create and initialize the reference temporary. Address Object = createReferenceTemporary(*this, M, E); - if (auto *Var = dyn_cast(Object.getPointer())) { + if (auto *Var = dyn_cast( + Object.getPointer()->stripPointerCasts())) { Object = Address(llvm::ConstantExpr::getBitCast( - Var, ConvertTypeForMem(E->getType())->getPointerTo()), + cast(Object.getPointer()), + ConvertTypeForMem(E->getType())->getPointerTo()), Object.getAlignment()); // If the temporary is a global and has a constant initializer or is a // constant temporary that we promoted to a global, we may have already diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp index a9865f3703d4..ab170245284c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp @@ -199,7 +199,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( bool CanUseVirtualCall = MD->isVirtual() && !HasQualifier; const CXXMethodDecl *DevirtualizedMethod = nullptr; - if (CanUseVirtualCall && CanDevirtualizeMemberFunctionCall(Base, MD)) { + if (CanUseVirtualCall && + MD->getDevirtualizedMethod(Base, getLangOpts().AppleKext)) { const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType(); DevirtualizedMethod = MD->getCorrespondingMethodInClass(BestDynamicDecl); assert(DevirtualizedMethod); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 3df95a4e9b2a..a2ea0dec3e9d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -4157,9 +4157,15 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, /*IsInitializer=*/true); enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 }; llvm::Value *TaskArgs[] = { - UpLoc, ThreadID, Result.NewTask, IfVal, LBLVal.getPointer(), - UBLVal.getPointer(), CGF.EmitLoadOfScalar(StLVal, SourceLocation()), - llvm::ConstantInt::getSigned(CGF.IntTy, Data.Nogroup ? 1 : 0), + UpLoc, + ThreadID, + Result.NewTask, + IfVal, + LBLVal.getPointer(), + UBLVal.getPointer(), + CGF.EmitLoadOfScalar(StLVal, SourceLocation()), + llvm::ConstantInt::getNullValue( + CGF.IntTy), // Always 0 because taskgroup emitted by the compiler llvm::ConstantInt::getSigned( CGF.IntTy, Data.Schedule.getPointer() ? Data.Schedule.getInt() ? NumTasks : Grainsize @@ -4168,10 +4174,9 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, /*isSigned=*/false) : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), - Result.TaskDupFn - ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Result.TaskDupFn, - CGF.VoidPtrTy) - : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; + Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + Result.TaskDupFn, CGF.VoidPtrTy) + : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp index 493cd627e418..71797e2e6fbe 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -4363,7 +4363,18 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) { CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, CodeGen); }; - EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); + if (Data.Nogroup) + EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); + else { + CGM.getOpenMPRuntime().emitTaskgroupRegion( + *this, + [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, + PrePostActionTy &Action) { + Action.Enter(CGF); + CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); + }, + S.getLocStart()); + } } void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp index 166f44f816f3..0735a9c3dfbc 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp @@ -64,3 +64,19 @@ CodeGen::arrangeFreeFunctionCall(CodeGenModule &CGM, returnType, /*IsInstanceMethod=*/false, /*IsChainCall=*/false, argTypes, info, {}, args); } + +llvm::FunctionType * +CodeGen::convertFreeFunctionType(CodeGenModule &CGM, const FunctionDecl *FD) { + assert(FD != nullptr && "Expected a non-null function declaration!"); + llvm::Type *T = CGM.getTypes().ConvertFunctionType(FD->getType(), FD); + + if (auto FT = dyn_cast(T)) + return FT; + + return nullptr; +} + +llvm::Type * +CodeGen::convertTypeForMemory(CodeGenModule &CGM, QualType T) { + return CGM.getTypes().ConvertTypeForMem(T); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h index 6785111bd052..5933e029be8d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h @@ -1479,6 +1479,9 @@ class CodeGenFunction : public CodeGenTypeCache { const TargetInfo &getTarget() const { return Target; } llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); } + const TargetCodeGenInfo &getTargetHooks() const { + return CGM.getTargetCodeGenInfo(); + } //===--------------------------------------------------------------------===// // Cleanups @@ -1749,11 +1752,6 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset); - /// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given - /// expr can be devirtualized. - bool CanDevirtualizeMemberFunctionCall(const Expr *Base, - const CXXMethodDecl *MD); - /// EnterDtorCleanups - Enter the cleanups necessary to complete the /// given phase of destruction for a destructor. The end result /// should call destructors on members and base classes in reverse @@ -3820,10 +3818,6 @@ class CodeGenFunction : public CodeGenTypeCache { private: QualType getVarArgType(const Expr *Arg); - const TargetCodeGenInfo &getTargetHooks() const { - return CGM.getTargetCodeGenInfo(); - } - void EmitDeclMetadata(); BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType, diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp index 5319ccec163f..4b15b8ac4c71 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp @@ -98,7 +98,6 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, Int16Ty = llvm::Type::getInt16Ty(LLVMContext); Int32Ty = llvm::Type::getInt32Ty(LLVMContext); Int64Ty = llvm::Type::getInt64Ty(LLVMContext); - HalfTy = llvm::Type::getHalfTy(LLVMContext); FloatTy = llvm::Type::getFloatTy(LLVMContext); DoubleTy = llvm::Type::getDoubleTy(LLVMContext); PointerWidthInBits = C.getTargetInfo().getPointerWidth(0); @@ -1099,7 +1098,7 @@ static void setLinkageAndVisibilityForGV(llvm::GlobalValue *GV, const NamedDecl *ND) { // Set linkage and visibility in case we never see a definition. LinkageInfo LV = ND->getLinkageAndVisibility(); - if (LV.getLinkage() != ExternalLinkage) { + if (!isExternallyVisible(LV.getLinkage())) { // Don't set internal linkage on declarations. } else { if (ND->hasAttr()) { @@ -2368,11 +2367,13 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, return llvm::ConstantExpr::getBitCast(Entry, Ty); } - unsigned AddrSpace = GetGlobalVarAddressSpace(D, Ty->getAddressSpace()); + auto AddrSpace = GetGlobalVarAddressSpace(D); + auto TargetAddrSpace = getContext().getTargetAddressSpace(AddrSpace); + auto *GV = new llvm::GlobalVariable( getModule(), Ty->getElementType(), false, llvm::GlobalValue::ExternalLinkage, nullptr, MangledName, nullptr, - llvm::GlobalVariable::NotThreadLocal, AddrSpace); + llvm::GlobalVariable::NotThreadLocal, TargetAddrSpace); // If we already created a global with the same mangled name (but different // type) before, take its name and remove it from its parent. @@ -2429,8 +2430,15 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, GV->setSection(".cp.rodata"); } - if (AddrSpace != Ty->getAddressSpace()) - return llvm::ConstantExpr::getAddrSpaceCast(GV, Ty); + auto ExpectedAS = + D ? D->getType().getAddressSpace() + : static_cast(LangOpts.OpenCL ? LangAS::opencl_global + : LangAS::Default); + assert(getContext().getTargetAddressSpace(ExpectedAS) == + Ty->getPointerAddressSpace()); + if (AddrSpace != ExpectedAS) + return getTargetCodeGenInfo().performAddrSpaceCast(*this, GV, AddrSpace, + ExpectedAS, Ty); return GV; } @@ -2564,18 +2572,28 @@ CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { getDataLayout().getTypeStoreSizeInBits(Ty)); } -unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D, - unsigned AddrSpace) { - if (D && LangOpts.CUDA && LangOpts.CUDAIsDevice) { - if (D->hasAttr()) - AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_constant); - else if (D->hasAttr()) - AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_shared); - else - AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_device); +unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) { + unsigned AddrSpace; + if (LangOpts.OpenCL) { + AddrSpace = D ? D->getType().getAddressSpace() + : static_cast(LangAS::opencl_global); + assert(AddrSpace == LangAS::opencl_global || + AddrSpace == LangAS::opencl_constant || + AddrSpace == LangAS::opencl_local || + AddrSpace >= LangAS::FirstTargetAddressSpace); + return AddrSpace; } - return AddrSpace; + if (LangOpts.CUDA && LangOpts.CUDAIsDevice) { + if (D && D->hasAttr()) + return LangAS::cuda_constant; + else if (D && D->hasAttr()) + return LangAS::cuda_shared; + else + return LangAS::cuda_device; + } + + return getTargetCodeGenInfo().getGlobalVarAddressSpace(*this, D); } template @@ -2728,10 +2746,9 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, // "extern int x[];") and then a definition of a different type (e.g. // "int x[10];"). This also happens when an initializer has a different type // from the type of the global (this happens with unions). - if (!GV || - GV->getType()->getElementType() != InitType || + if (!GV || GV->getType()->getElementType() != InitType || GV->getType()->getAddressSpace() != - GetGlobalVarAddressSpace(D, getContext().getTargetAddressSpace(ASTTy))) { + getContext().getTargetAddressSpace(GetGlobalVarAddressSpace(D))) { // Move the old entry aside so that we'll create a new one. Entry->setName(StringRef()); @@ -3740,20 +3757,26 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( Linkage = llvm::GlobalVariable::InternalLinkage; } } - unsigned AddrSpace = GetGlobalVarAddressSpace( - VD, getContext().getTargetAddressSpace(MaterializedType)); + unsigned AddrSpace = + VD ? GetGlobalVarAddressSpace(VD) : MaterializedType.getAddressSpace(); + auto TargetAS = getContext().getTargetAddressSpace(AddrSpace); auto *GV = new llvm::GlobalVariable( getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(), - /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, - AddrSpace); + /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS); setGlobalVisibility(GV, VD); GV->setAlignment(Align.getQuantity()); if (supportsCOMDAT() && GV->isWeakForLinker()) GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); if (VD->getTLSKind()) setTLSMode(GV, *VD); - MaterializedGlobalTemporaryMap[E] = GV; - return ConstantAddress(GV, Align); + llvm::Constant *CV = GV; + if (AddrSpace != LangAS::Default) + CV = getTargetCodeGenInfo().performAddrSpaceCast( + *this, GV, AddrSpace, LangAS::Default, + Type->getPointerTo( + getContext().getTargetAddressSpace(LangAS::Default))); + MaterializedGlobalTemporaryMap[E] = CV; + return ConstantAddress(CV, Align); } /// EmitObjCPropertyImplementations - Emit information for synthesized diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h index c5f1a2b409ee..b162e72d1992 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h @@ -710,11 +710,15 @@ class CodeGenModule : public CodeGenTypeCache { SourceLocation Loc = SourceLocation(), bool TLS = false); - /// Return the address space of the underlying global variable for D, as + /// Return the AST address space of the underlying global variable for D, as /// determined by its declaration. Normally this is the same as the address /// space of D's type, but in CUDA, address spaces are associated with - /// declarations, not types. - unsigned GetGlobalVarAddressSpace(const VarDecl *D, unsigned AddrSpace); + /// declarations, not types. If D is nullptr, return the default address + /// space for global variable. + /// + /// For languages without explicit address spaces, if D has default address + /// space, target-specific global or constant address space may be returned. + unsigned GetGlobalVarAddressSpace(const VarDecl *D); /// Return the llvm::Constant for the address of the given global variable. /// If Ty is non-null and if the global doesn't exist, then it will be created diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h index 6910d36733dc..450eab48a3b4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h @@ -36,7 +36,7 @@ struct CodeGenTypeCache { /// i8, i16, i32, and i64 llvm::IntegerType *Int8Ty, *Int16Ty, *Int32Ty, *Int64Ty; /// float, double - llvm::Type *HalfTy, *FloatTy, *DoubleTy; + llvm::Type *FloatTy, *DoubleTy; /// int llvm::IntegerType *IntTy; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp index 39efb9f43921..c82b9677eacf 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -2732,7 +2732,9 @@ static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM, // function. bool IsDLLImport = RD->hasAttr(); if (CGM.getVTables().isVTableExternal(RD)) - return IsDLLImport ? false : true; + return IsDLLImport && !CGM.getTriple().isWindowsItaniumEnvironment() + ? false + : true; if (IsDLLImport) return true; @@ -2957,6 +2959,8 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM, return llvm::GlobalValue::InternalLinkage; case VisibleNoLinkage: + case ModuleInternalLinkage: + case ModuleLinkage: case ExternalLinkage: // RTTI is not enabled, which means that this type info struct is going // to be used for exception handling. Give it linkonce_odr linkage. @@ -2968,7 +2972,8 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM, if (RD->hasAttr()) return llvm::GlobalValue::WeakODRLinkage; if (CGM.getTriple().isWindowsItaniumEnvironment()) - if (RD->hasAttr()) + if (RD->hasAttr() && + ShouldUseExternalRTTIDescriptor(CGM, Ty)) return llvm::GlobalValue::ExternalLinkage; if (RD->isDynamicClass()) { llvm::GlobalValue::LinkageTypes LT = CGM.getVTableLinkage(RD); @@ -3181,7 +3186,8 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force, if (DLLExport || (RD && RD->hasAttr())) { TypeName->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); - } else if (CGM.getLangOpts().RTTI && RD && RD->hasAttr()) { + } else if (RD && RD->hasAttr() && + ShouldUseExternalRTTIDescriptor(CGM, Ty)) { TypeName->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp index e68a16e0bd51..78b510bb4665 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -3425,6 +3425,8 @@ static llvm::GlobalValue::LinkageTypes getLinkageForRTTI(QualType Ty) { return llvm::GlobalValue::InternalLinkage; case VisibleNoLinkage: + case ModuleInternalLinkage: + case ModuleLinkage: case ExternalLinkage: return llvm::GlobalValue::LinkOnceODRLinkage; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp index 8d00e055306d..eeebd60a2d20 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp @@ -416,14 +416,34 @@ llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule & return llvm::ConstantPointerNull::get(T); } +unsigned TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM, + const VarDecl *D) const { + assert(!CGM.getLangOpts().OpenCL && + !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) && + "Address space agnostic languages only"); + return D ? D->getType().getAddressSpace() + : static_cast(LangAS::Default); +} + llvm::Value *TargetCodeGenInfo::performAddrSpaceCast( CodeGen::CodeGenFunction &CGF, llvm::Value *Src, unsigned SrcAddr, unsigned DestAddr, llvm::Type *DestTy, bool isNonNull) const { // Since target may map different address spaces in AST to the same address // space, an address space conversion may end up as a bitcast. + if (auto *C = dyn_cast(Src)) + return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy); return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DestTy); } +llvm::Constant * +TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src, + unsigned SrcAddr, unsigned DestAddr, + llvm::Type *DestTy) const { + // Since target may map different address spaces in AST to the same address + // space, an address space conversion may end up as a bitcast. + return llvm::ConstantExpr::getPointerCast(Src, DestTy); +} + static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays); /// isEmptyField - Return true iff a the field is "empty", that is it @@ -7325,6 +7345,8 @@ class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo { return LangAS::FirstTargetAddressSpace + getABIInfo().getDataLayout().getAllocaAddrSpace(); } + unsigned getGlobalVarAddressSpace(CodeGenModule &CGM, + const VarDecl *D) const override; }; } @@ -7408,6 +7430,31 @@ llvm::Constant *AMDGPUTargetCodeGenInfo::getNullPointer( llvm::ConstantPointerNull::get(NPT), PT); } +unsigned +AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM, + const VarDecl *D) const { + assert(!CGM.getLangOpts().OpenCL && + !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) && + "Address space agnostic languages only"); + unsigned DefaultGlobalAS = + LangAS::FirstTargetAddressSpace + + CGM.getContext().getTargetAddressSpace(LangAS::opencl_global); + if (!D) + return DefaultGlobalAS; + + unsigned AddrSpace = D->getType().getAddressSpace(); + assert(AddrSpace == LangAS::Default || + AddrSpace >= LangAS::FirstTargetAddressSpace); + if (AddrSpace != LangAS::Default) + return AddrSpace; + + if (CGM.isTypeConstant(D->getType(), false)) { + if (auto ConstAS = CGM.getTarget().getConstantAddressSpace()) + return ConstAS.getValue(); + } + return DefaultGlobalAS; +} + //===----------------------------------------------------------------------===// // SPARC v8 ABI Implementation. // Based on the SPARC Compliance Definition version 2.4.1. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h index 247d01dcb086..952ef96c4aef 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h @@ -229,6 +229,13 @@ class TargetCodeGenInfo { virtual llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM, llvm::PointerType *T, QualType QT) const; + /// Get target favored AST address space of a global variable for languages + /// other than OpenCL and CUDA. + /// If \p D is nullptr, returns the default target favored address space + /// for global variable. + virtual unsigned getGlobalVarAddressSpace(CodeGenModule &CGM, + const VarDecl *D) const; + /// Get the AST address space for alloca. virtual unsigned getASTAllocaAddressSpace() const { return LangAS::Default; } @@ -243,6 +250,15 @@ class TargetCodeGenInfo { unsigned DestAddr, llvm::Type *DestTy, bool IsNonNull = false) const; + + /// Perform address space cast of a constant expression of pointer type. + /// \param V is the LLVM constant to be casted to another address space. + /// \param SrcAddr is the language address space of \p V. + /// \param DestAddr is the targeted language address space. + /// \param DestTy is the destination LLVM pointer type. + virtual llvm::Constant * + performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *V, unsigned SrcAddr, + unsigned DestAddr, llvm::Type *DestTy) const; }; } // namespace CodeGen diff --git a/contrib/llvm/tools/clang/lib/Driver/Driver.cpp b/contrib/llvm/tools/clang/lib/Driver/Driver.cpp index faced0697ed9..42478013ccec 100644 --- a/contrib/llvm/tools/clang/lib/Driver/Driver.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/Driver.cpp @@ -572,8 +572,22 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C, if (TT.getArch() == llvm::Triple::UnknownArch) Diag(clang::diag::err_drv_invalid_omp_target) << Val; else { - const ToolChain &TC = getToolChain(C.getInputArgs(), TT); - C.addOffloadDeviceToolChain(&TC, Action::OFK_OpenMP); + const ToolChain *TC; + // CUDA toolchains have to be selected differently. They pair host + // and device in their implementation. + if (TT.isNVPTX()) { + const ToolChain *HostTC = + C.getSingleOffloadToolChain(); + assert(HostTC && "Host toolchain should be always defined."); + auto &CudaTC = + ToolChains[TT.str() + "/" + HostTC->getTriple().str()]; + if (!CudaTC) + CudaTC = llvm::make_unique( + *this, TT, *HostTC, C.getInputArgs()); + TC = CudaTC.get(); + } else + TC = &getToolChain(C.getInputArgs(), TT); + C.addOffloadDeviceToolChain(TC, Action::OFK_OpenMP); } } } else @@ -1247,11 +1261,20 @@ bool Driver::HandleImmediateArgs(const Compilation &C) { StringRef PassedFlags = A->getValue(); std::vector SuggestedCompletions; + unsigned short DisableFlags = options::NoDriverOption | options::Unsupported | options::Ignored; + // We want to show cc1-only options only when clang is invoked as "clang -cc1". + // When clang is invoked as "clang -cc1", we add "#" to the beginning of an --autocomplete + // option so that the clang driver can distinguish whether it is requested to show cc1-only options or not. + if (PassedFlags[0] == '#') { + DisableFlags &= ~options::NoDriverOption; + PassedFlags = PassedFlags.substr(1); + } + if (PassedFlags.find(',') == StringRef::npos) { // If the flag is in the form of "--autocomplete=-foo", // we were requested to print out all option names that start with "-foo". // For example, "--autocomplete=-fsyn" is expanded to "-fsyntax-only". - SuggestedCompletions = Opts->findByPrefix(PassedFlags); + SuggestedCompletions = Opts->findByPrefix(PassedFlags, DisableFlags); } else { // If the flag is in the form of "--autocomplete=foo,bar", we were // requested to print out all option values for "-foo" that start with diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp index 4f82503276f4..2be7f0f69004 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChain.cpp @@ -544,9 +544,9 @@ void ToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, // Each toolchain should provide the appropriate include flags. } -void ToolChain::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { -} +void ToolChain::addClangTargetOptions( + const ArgList &DriverArgs, ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const {} void ToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {} diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/ARM.cpp index 8cafd3c74bfb..95b86f784f91 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/ARM.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/ARM.cpp @@ -179,6 +179,18 @@ arm::FloatABI arm::getARMFloatABI(const ToolChain &TC, const ArgList &Args) { ABI = FloatABI::Hard; break; + case llvm::Triple::NetBSD: + switch (Triple.getEnvironment()) { + case llvm::Triple::EABIHF: + case llvm::Triple::GNUEABIHF: + ABI = FloatABI::Hard; + break; + default: + ABI = FloatABI::Soft; + break; + } + break; + case llvm::Triple::FreeBSD: switch (Triple.getEnvironment()) { case llvm::Triple::GNUEABIHF: diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/Mips.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/Mips.cpp index 33eff32b9731..b45dcd6db678 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/Mips.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Arch/Mips.cpp @@ -227,12 +227,11 @@ void mips::getMIPSTargetFeatures(const Driver &D, const llvm::Triple &Triple, O.matches(options::OPT_fno_PIE) || O.matches(options::OPT_fno_pie)); } - if (IsN64 && NonPIC) { + if (IsN64 && NonPIC) Features.push_back("+noabicalls"); - } else { + else AddTargetFeature(Args, Features, options::OPT_mno_abicalls, options::OPT_mabicalls, "noabicalls"); - } mips::FloatABI FloatABI = mips::getMipsFloatABI(D, Args); if (FloatABI == mips::FloatABI::Soft) { @@ -298,13 +297,11 @@ void mips::getMIPSTargetFeatures(const Driver &D, const llvm::Triple &Triple, AddTargetFeature(Args, Features, options::OPT_mno_odd_spreg, options::OPT_modd_spreg, "nooddspreg"); - - if (Arg *A = Args.getLastArg(options::OPT_mmadd4, options::OPT_mno_madd4)) { - if (A->getOption().matches(options::OPT_mmadd4)) - Features.push_back("-nomadd4"); - else - Features.push_back("+nomadd4"); - } + AddTargetFeature(Args, Features, options::OPT_mno_madd4, options::OPT_mmadd4, + "nomadd4"); + AddTargetFeature(Args, Features, options::OPT_mlong_calls, + options::OPT_mno_long_calls, "long-calls"); + AddTargetFeature(Args, Features, options::OPT_mmt, options::OPT_mno_mt,"mt"); } mips::NanEncoding mips::getSupportedNanEncoding(StringRef &CPU) { diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.cpp index 66246f6d71cd..5dc6dfad927b 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.cpp @@ -98,7 +98,8 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs, } void BareMetal::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { + ArgStringList &CC1Args, + Action::OffloadKind) const { CC1Args.push_back("-nostdsysteminc"); } diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.h index 064c1199735b..4b74899fa53e 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/BareMetal.h @@ -54,7 +54,8 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain { void AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args) const override; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; std::string findLibCxxIncludePath(ToolChain::CXXStdlibType LibType) const; void AddClangCXXStdlibIncludeArgs( const llvm::opt::ArgList &DriverArgs, diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Clang.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Clang.cpp index 3731aa83ef06..e759e3ec619a 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Clang.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Clang.cpp @@ -2544,7 +2544,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, AsynchronousUnwindTables)) CmdArgs.push_back("-munwind-tables"); - getToolChain().addClangTargetOptions(Args, CmdArgs); + getToolChain().addClangTargetOptions(Args, CmdArgs, + JA.getOffloadingDeviceKind()); if (Arg *A = Args.getLastArg(options::OPT_flimited_precision_EQ)) { CmdArgs.push_back("-mlimit-float-precision"); @@ -2974,6 +2975,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, A->claim(); } + for (const Arg *A : + Args.filtered(options::OPT_clang_ignored_legacy_options_Group)) { + D.Diag(diag::warn_ignored_clang_option) << A->getAsString(Args); + A->claim(); + } + claimNoWarnArgs(Args); Args.AddAllArgs(CmdArgs, options::OPT_R_Group); diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/CommonArgs.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/CommonArgs.cpp index e8bb703054de..00bd60bc24bb 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -524,6 +524,7 @@ void tools::linkSanitizerRuntimeDeps(const ToolChain &TC, CmdArgs.push_back("-lm"); // There's no libdl on FreeBSD or RTEMS. if (TC.getTriple().getOS() != llvm::Triple::FreeBSD && + TC.getTriple().getOS() != llvm::Triple::NetBSD && TC.getTriple().getOS() != llvm::Triple::RTEMS) CmdArgs.push_back("-ldl"); } diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.cpp index 42bf164f1b3f..935a5a37ada5 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.cpp @@ -338,24 +338,31 @@ CudaToolChain::CudaToolChain(const Driver &D, const llvm::Triple &Triple, void CudaToolChain::addClangTargetOptions( const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const { - HostTC.addClangTargetOptions(DriverArgs, CC1Args); - - CC1Args.push_back("-fcuda-is-device"); - - if (DriverArgs.hasFlag(options::OPT_fcuda_flush_denormals_to_zero, - options::OPT_fno_cuda_flush_denormals_to_zero, false)) - CC1Args.push_back("-fcuda-flush-denormals-to-zero"); - - if (DriverArgs.hasFlag(options::OPT_fcuda_approx_transcendentals, - options::OPT_fno_cuda_approx_transcendentals, false)) - CC1Args.push_back("-fcuda-approx-transcendentals"); - - if (DriverArgs.hasArg(options::OPT_nocudalib)) - return; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadingKind) const { + HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind); StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ); assert(!GpuArch.empty() && "Must have an explicit GPU arch."); + assert((DeviceOffloadingKind == Action::OFK_OpenMP || + DeviceOffloadingKind == Action::OFK_Cuda) && + "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs."); + + if (DeviceOffloadingKind == Action::OFK_Cuda) { + CC1Args.push_back("-fcuda-is-device"); + + if (DriverArgs.hasFlag(options::OPT_fcuda_flush_denormals_to_zero, + options::OPT_fno_cuda_flush_denormals_to_zero, false)) + CC1Args.push_back("-fcuda-flush-denormals-to-zero"); + + if (DriverArgs.hasFlag(options::OPT_fcuda_approx_transcendentals, + options::OPT_fno_cuda_approx_transcendentals, false)) + CC1Args.push_back("-fcuda-approx-transcendentals"); + + if (DriverArgs.hasArg(options::OPT_nocudalib)) + return; + } + std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch); if (LibDeviceFile.empty()) { @@ -396,6 +403,24 @@ CudaToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args, const OptTable &Opts = getDriver().getOpts(); + // For OpenMP device offloading, append derived arguments. Make sure + // flags are not duplicated. + // TODO: Append the compute capability. + if (DeviceOffloadKind == Action::OFK_OpenMP) { + for (Arg *A : Args){ + bool IsDuplicate = false; + for (Arg *DALArg : *DAL){ + if (A == DALArg) { + IsDuplicate = true; + break; + } + } + if (!IsDuplicate) + DAL->append(A); + } + return DAL; + } + for (Arg *A : Args) { if (A->getOption().matches(options::OPT_Xarch__)) { // Skip this argument unless the architecture matches BoundArch diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.h index acdb4c4efd6d..e66fc23d82f3 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Cuda.h @@ -130,7 +130,8 @@ class LLVM_LIBRARY_VISIBILITY CudaToolChain : public ToolChain { TranslateArgs(const llvm::opt::DerivedArgList &Args, StringRef BoundArch, Action::OffloadKind DeviceOffloadKind) const override; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; // Never try to use the integrated assembler with CUDA; always fork out to // ptxas. diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.cpp index 7b61095c3ba9..ba1a5ee95594 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.cpp @@ -1118,6 +1118,27 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args, } } +/// Returns the most appropriate macOS target version for the current process. +/// +/// If the macOS SDK version is the same or earlier than the system version, +/// then the SDK version is returned. Otherwise the system version is returned. +static std::string getSystemOrSDKMacOSVersion(StringRef MacOSSDKVersion) { + unsigned Major, Minor, Micro; + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + if (!SystemTriple.isMacOSX()) + return MacOSSDKVersion; + SystemTriple.getMacOSXVersion(Major, Minor, Micro); + VersionTuple SystemVersion(Major, Minor, Micro); + bool HadExtra; + if (!Driver::GetReleaseVersion(MacOSSDKVersion, Major, Minor, Micro, + HadExtra)) + return MacOSSDKVersion; + VersionTuple SDKVersion(Major, Minor, Micro); + if (SDKVersion > SystemVersion) + return SystemVersion.getAsString(); + return MacOSSDKVersion; +} + void Darwin::AddDeploymentTarget(DerivedArgList &Args) const { const OptTable &Opts = getDriver().getOpts(); @@ -1229,7 +1250,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const { SDK.startswith("iPhoneSimulator")) iOSTarget = Version; else if (SDK.startswith("MacOSX")) - OSXTarget = Version; + OSXTarget = getSystemOrSDKMacOSVersion(Version); else if (SDK.startswith("WatchOS") || SDK.startswith("WatchSimulator")) WatchOSTarget = Version; @@ -1708,7 +1729,8 @@ bool Darwin::isAlignedAllocationUnavailable() const { } void Darwin::addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const { + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const { if (isAlignedAllocationUnavailable()) CC1Args.push_back("-faligned-alloc-unavailable"); } diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.h index ffcdf9a71a46..6cb1d04b78c0 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Darwin.h @@ -390,7 +390,8 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO { bool isAlignedAllocationUnavailable() const; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; StringRef getPlatformFamily() const; static StringRef getSDKName(StringRef isysroot); diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.cpp index b8757cf4aa73..d8b8fe8f0bfe 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.cpp @@ -131,16 +131,44 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA, /// Fuchsia - Fuchsia tool chain which can call as(1) and ld(1) directly. -Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple, - const ArgList &Args) - : Generic_ELF(D, Triple, Args) { - - getFilePaths().push_back(D.SysRoot + "/lib"); - getFilePaths().push_back(D.ResourceDir + "/lib/fuchsia"); +static std::string normalizeTriple(llvm::Triple Triple) { + SmallString<64> T; + T += Triple.getArchName(); + T += "-"; + T += Triple.getOSName(); + return T.str(); } -Tool *Fuchsia::buildAssembler() const { - return new tools::gnutools::Assembler(*this); +static std::string getTargetDir(const Driver &D, + llvm::Triple Triple) { + SmallString<128> P(llvm::sys::path::parent_path(D.Dir)); + llvm::sys::path::append(P, "lib", normalizeTriple(Triple)); + return P.str(); +} + +Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple, + const ArgList &Args) + : ToolChain(D, Triple, Args) { + getProgramPaths().push_back(getDriver().getInstalledDir()); + if (getDriver().getInstalledDir() != D.Dir) + getProgramPaths().push_back(D.Dir); + + SmallString<128> P(getTargetDir(D, getTriple())); + llvm::sys::path::append(P, "lib"); + getFilePaths().push_back(P.str()); + + if (!D.SysRoot.empty()) { + SmallString<128> P(D.SysRoot); + llvm::sys::path::append(P, "lib"); + getFilePaths().push_back(P.str()); + } +} + +std::string Fuchsia::ComputeEffectiveClangTriple(const ArgList &Args, + types::ID InputType) const { + llvm::Triple Triple(ComputeLLVMTriple(Args, InputType)); + Triple.setTriple(normalizeTriple(Triple)); + return Triple.getTriple(); } Tool *Fuchsia::buildLinker() const { @@ -172,7 +200,8 @@ Fuchsia::GetCXXStdlibType(const ArgList &Args) const { } void Fuchsia::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { + ArgStringList &CC1Args, + Action::OffloadKind) const { if (DriverArgs.hasFlag(options::OPT_fuse_init_array, options::OPT_fno_use_init_array, true)) CC1Args.push_back("-fuse-init-array"); @@ -207,19 +236,44 @@ void Fuchsia::AddClangSystemIncludeArgs(const ArgList &DriverArgs, return; } - addExternCSystemInclude(DriverArgs, CC1Args, D.SysRoot + "/include"); + if (!D.SysRoot.empty()) { + SmallString<128> P(D.SysRoot); + llvm::sys::path::append(P, "include"); + addExternCSystemInclude(DriverArgs, CC1Args, P.str()); + } } -std::string Fuchsia::findLibCxxIncludePath() const { - return getDriver().SysRoot + "/include/c++/v1"; +void Fuchsia::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs, + ArgStringList &CC1Args) const { + if (DriverArgs.hasArg(options::OPT_nostdlibinc) || + DriverArgs.hasArg(options::OPT_nostdincxx)) + return; + + switch (GetCXXStdlibType(DriverArgs)) { + case ToolChain::CST_Libcxx: { + SmallString<128> P(getTargetDir(getDriver(), getTriple())); + llvm::sys::path::append(P, "include", "c++", "v1"); + addSystemInclude(DriverArgs, CC1Args, P.str()); + break; + } + + default: + llvm_unreachable("invalid stdlib name"); + } } void Fuchsia::AddCXXStdlibLibArgs(const ArgList &Args, ArgStringList &CmdArgs) const { - (void) GetCXXStdlibType(Args); - CmdArgs.push_back("-lc++"); - CmdArgs.push_back("-lc++abi"); - CmdArgs.push_back("-lunwind"); + switch (GetCXXStdlibType(Args)) { + case ToolChain::CST_Libcxx: + CmdArgs.push_back("-lc++"); + CmdArgs.push_back("-lc++abi"); + CmdArgs.push_back("-lunwind"); + break; + + case ToolChain::CST_Libstdcxx: + llvm_unreachable("invalid stdlib name"); + } } SanitizerMask Fuchsia::getSupportedSanitizers() const { diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.h index 1a8c9903fe4d..a723a99dfa3b 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Fuchsia.h @@ -35,18 +35,29 @@ class LLVM_LIBRARY_VISIBILITY Linker : public GnuTool { namespace toolchains { -class LLVM_LIBRARY_VISIBILITY Fuchsia : public Generic_ELF { +class LLVM_LIBRARY_VISIBILITY Fuchsia : public ToolChain { public: Fuchsia(const Driver &D, const llvm::Triple &Triple, const llvm::opt::ArgList &Args); - bool isPIEDefault() const override { return true; } bool HasNativeLLVMSupport() const override { return true; } bool IsIntegratedAssemblerDefault() const override { return true; } + RuntimeLibType GetDefaultRuntimeLibType() const override { + return ToolChain::RLT_CompilerRT; + } + CXXStdlibType GetDefaultCXXStdlibType() const override { + return ToolChain::CST_Libcxx; + } + bool isPICDefault() const override { return false; } + bool isPIEDefault() const override { return true; } + bool isPICDefaultForced() const override { return false; } llvm::DebuggerKind getDefaultDebuggerTuning() const override { return llvm::DebuggerKind::GDB; } + std::string ComputeEffectiveClangTriple(const llvm::opt::ArgList &Args, + types::ID InputType) const override; + SanitizerMask getSupportedSanitizers() const override; RuntimeLibType @@ -55,11 +66,14 @@ class LLVM_LIBRARY_VISIBILITY Fuchsia : public Generic_ELF { GetCXXStdlibType(const llvm::opt::ArgList &Args) const override; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; void AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args) const override; - std::string findLibCxxIncludePath() const override; + void + AddClangCXXStdlibIncludeArgs(const llvm::opt::ArgList &DriverArgs, + llvm::opt::ArgStringList &CC1Args) const override; void AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs) const override; @@ -68,7 +82,6 @@ class LLVM_LIBRARY_VISIBILITY Fuchsia : public Generic_ELF { } protected: - Tool *buildAssembler() const override; Tool *buildLinker() const override; }; diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.cpp index bca5d3a3f28b..ad5f7df50d2e 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.cpp @@ -2461,7 +2461,8 @@ Generic_GCC::TranslateArgs(const llvm::opt::DerivedArgList &Args, StringRef, void Generic_ELF::anchor() {} void Generic_ELF::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { + ArgStringList &CC1Args, + Action::OffloadKind) const { const Generic_GCC::GCCVersion &V = GCCInstallation.getVersion(); bool UseInitArrayDefault = getTriple().getArch() == llvm::Triple::aarch64 || diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.h index 1dc1ad49e305..cdf610054401 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Gnu.h @@ -341,7 +341,8 @@ class LLVM_LIBRARY_VISIBILITY Generic_ELF : public Generic_GCC { : Generic_GCC(D, Triple, Args) {} void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; }; } // end namespace toolchains diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.cpp index 1d7bcf8e4df0..9bf1590e6a37 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.cpp @@ -428,7 +428,8 @@ unsigned HexagonToolChain::getOptimizationLevel( } void HexagonToolChain::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { + ArgStringList &CC1Args, + Action::OffloadKind) const { if (DriverArgs.hasArg(options::OPT_ffp_contract)) return; unsigned OptLevel = getOptimizationLevel(DriverArgs); diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.h index 78f97a3d59fd..7f9a6b2f34b9 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/Hexagon.h @@ -69,7 +69,8 @@ class LLVM_LIBRARY_VISIBILITY HexagonToolChain : public Linux { ~HexagonToolChain() override; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; void AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args) const override; diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.cpp index d7d3ad61df42..a1a3108cb28d 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.cpp @@ -15,6 +15,7 @@ #include "clang/Driver/Compilation.h" #include "clang/Driver/Driver.h" #include "clang/Driver/Options.h" +#include "clang/Driver/SanitizerArgs.h" #include "llvm/Option/ArgList.h" using namespace clang::driver; @@ -246,6 +247,7 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag); Args.AddAllArgs(CmdArgs, options::OPT_r); + bool NeedsSanitizerDeps = addSanitizerRuntimes(getToolChain(), Args, CmdArgs); AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA); unsigned Major, Minor, Micro; @@ -279,6 +281,8 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs); CmdArgs.push_back("-lm"); } + if (NeedsSanitizerDeps) + linkSanitizerRuntimeDeps(getToolChain(), CmdArgs); if (Args.hasArg(options::OPT_pthread)) CmdArgs.push_back("-lpthread"); CmdArgs.push_back("-lc"); @@ -410,3 +414,12 @@ void NetBSD::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs, addLibStdCXXIncludePaths(getDriver().SysRoot, "/usr/include/g++", "", "", "", "", DriverArgs, CC1Args); } + +SanitizerMask NetBSD::getSupportedSanitizers() const { + const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64; + SanitizerMask Res = ToolChain::getSupportedSanitizers(); + if (IsX86_64) { + Res |= SanitizerKind::Address; + } + return Res; +} diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.h index d53aa6867872..412d0815e81a 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/NetBSD.h @@ -66,6 +66,7 @@ class LLVM_LIBRARY_VISIBILITY NetBSD : public Generic_ELF { llvm::opt::ArgStringList &CC1Args) const override; bool IsUnwindTablesDefault() const override { return true; } + SanitizerMask getSupportedSanitizers() const override; protected: Tool *buildAssembler() const override; diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.cpp index fcb6418b2517..058bc42323e2 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.cpp @@ -134,7 +134,8 @@ bool WebAssembly::SupportsProfiling() const { return false; } bool WebAssembly::HasNativeLLVMSupport() const { return true; } void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { + ArgStringList &CC1Args, + Action::OffloadKind) const { if (DriverArgs.hasFlag(clang::driver::options::OPT_fuse_init_array, options::OPT_fno_use_init_array, true)) CC1Args.push_back("-fuse-init-array"); diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.h index ca42fc651a6d..2999db477f79 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/WebAssembly.h @@ -53,7 +53,8 @@ class LLVM_LIBRARY_VISIBILITY WebAssembly final : public ToolChain { bool SupportsProfiling() const override; bool HasNativeLLVMSupport() const override; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; RuntimeLibType GetDefaultRuntimeLibType() const override; CXXStdlibType GetCXXStdlibType(const llvm::opt::ArgList &Args) const override; void AddClangSystemIncludeArgs( diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.cpp b/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.cpp index c3ae9582124f..43175ad7d632 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.cpp +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.cpp @@ -124,7 +124,8 @@ void XCoreToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, } void XCoreToolChain::addClangTargetOptions(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { + ArgStringList &CC1Args, + Action::OffloadKind) const { CC1Args.push_back("-nostdsysteminc"); } diff --git a/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.h b/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.h index 4084b1cdec13..00c89bd15f78 100644 --- a/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.h +++ b/contrib/llvm/tools/clang/lib/Driver/ToolChains/XCore.h @@ -67,7 +67,8 @@ class LLVM_LIBRARY_VISIBILITY XCoreToolChain : public ToolChain { AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args) const override; void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args) const override; + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; void AddClangCXXStdlibIncludeArgs( const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args) const override; diff --git a/contrib/llvm/tools/clang/lib/Format/ContinuationIndenter.cpp b/contrib/llvm/tools/clang/lib/Format/ContinuationIndenter.cpp index 4197587a74c0..3bf1cd8f7c13 100644 --- a/contrib/llvm/tools/clang/lib/Format/ContinuationIndenter.cpp +++ b/contrib/llvm/tools/clang/lib/Format/ContinuationIndenter.cpp @@ -66,6 +66,16 @@ static bool startsNextParameter(const FormatToken &Current, !Style.BreakBeforeInheritanceComma)); } +static bool opensProtoMessageField(const FormatToken &LessTok, + const FormatStyle &Style) { + if (LessTok.isNot(tok::less)) + return false; + return Style.Language == FormatStyle::LK_TextProto || + (Style.Language == FormatStyle::LK_Proto && + (LessTok.NestingLevel > 0 || + (LessTok.Previous && LessTok.Previous->is(tok::equal)))); +} + ContinuationIndenter::ContinuationIndenter(const FormatStyle &Style, const AdditionalKeywords &Keywords, const SourceManager &SourceMgr, @@ -94,6 +104,13 @@ LineState ContinuationIndenter::getInitialState(unsigned FirstIndent, State.LowestLevelOnLine = 0; State.IgnoreStackForComparison = false; + if (Style.Language == FormatStyle::LK_TextProto) { + // We need this in order to deal with the bin packing of text fields at + // global scope. + State.Stack.back().AvoidBinPacking = true; + State.Stack.back().BreakBeforeParameter = true; + } + // The first token has already been indented and thus consumed. moveStateToNextToken(State, DryRun, /*Newline=*/false); return State; @@ -176,7 +193,8 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { return true; if (((Previous.is(TT_DictLiteral) && Previous.is(tok::l_brace)) || (Previous.is(TT_ArrayInitializerLSquare) && - Previous.ParameterCount > 1)) && + Previous.ParameterCount > 1) || + opensProtoMessageField(Previous, Style)) && Style.ColumnLimit > 0 && getLengthToMatchingParen(Previous) + State.Column - 1 > getColumnLimit(State)) @@ -501,13 +519,6 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun, } } -static bool lessOpensProtoMessageField(const FormatToken &LessTok, - const LineState &State) { - assert(LessTok.is(tok::less)); - return LessTok.NestingLevel > 0 || - (LessTok.Previous && LessTok.Previous->is(tok::equal)); -} - unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State, bool DryRun) { FormatToken &Current = *State.NextToken; @@ -650,9 +661,7 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State, // before the corresponding } or ]. if (PreviousNonComment && (PreviousNonComment->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) || - (Style.Language == FormatStyle::LK_Proto && - PreviousNonComment->is(tok::less) && - lessOpensProtoMessageField(*PreviousNonComment, State)) || + opensProtoMessageField(*PreviousNonComment, Style) || (PreviousNonComment->is(TT_TemplateString) && PreviousNonComment->opensScope()))) State.Stack.back().BreakBeforeClosingBrace = true; @@ -695,7 +704,9 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { return Current.NestingLevel == 0 ? State.FirstIndent : State.Stack.back().Indent; if ((Current.isOneOf(tok::r_brace, tok::r_square) || - (Current.is(tok::greater) && Style.Language == FormatStyle::LK_Proto)) && + (Current.is(tok::greater) && + (Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto))) && State.Stack.size() > 1) { if (Current.closesBlockOrBlockTypeList(Style)) return State.Stack[State.Stack.size() - 2].NestedBlockIndent; @@ -1050,8 +1061,7 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State, unsigned NestedBlockIndent = std::max(State.Stack.back().StartOfFunctionCall, State.Stack.back().NestedBlockIndent); if (Current.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) || - (Style.Language == FormatStyle::LK_Proto && Current.is(tok::less) && - lessOpensProtoMessageField(Current, State))) { + opensProtoMessageField(Current, Style)) { if (Current.opensBlockOrBlockTypeList(Style)) { NewIndent = Style.IndentWidth + std::min(State.Column, State.Stack.back().NestedBlockIndent); @@ -1064,7 +1074,9 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State, Current.MatchingParen->Previous->is(tok::comma); AvoidBinPacking = EndsInComma || Current.is(TT_DictLiteral) || - Style.Language == FormatStyle::LK_Proto || !Style.BinPackArguments || + Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto || + !Style.BinPackArguments || (NextNoComment && NextNoComment->isOneOf(TT_DesignatedInitializerPeriod, TT_DesignatedInitializerLSquare)); diff --git a/contrib/llvm/tools/clang/lib/Format/Format.cpp b/contrib/llvm/tools/clang/lib/Format/Format.cpp index bb6781d79517..aa4ed8c42a70 100644 --- a/contrib/llvm/tools/clang/lib/Format/Format.cpp +++ b/contrib/llvm/tools/clang/lib/Format/Format.cpp @@ -56,6 +56,7 @@ template <> struct ScalarEnumerationTraits { IO.enumCase(Value, "ObjC", FormatStyle::LK_ObjC); IO.enumCase(Value, "Proto", FormatStyle::LK_Proto); IO.enumCase(Value, "TableGen", FormatStyle::LK_TableGen); + IO.enumCase(Value, "TextProto", FormatStyle::LK_TextProto); } }; @@ -631,6 +632,12 @@ FormatStyle getLLVMStyle() { } FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) { + if (Language == FormatStyle::LK_TextProto) { + FormatStyle GoogleStyle = getGoogleStyle(FormatStyle::LK_Proto); + GoogleStyle.Language = FormatStyle::LK_TextProto; + return GoogleStyle; + } + FormatStyle GoogleStyle = getLLVMStyle(); GoogleStyle.Language = Language; diff --git a/contrib/llvm/tools/clang/lib/Format/FormatToken.h b/contrib/llvm/tools/clang/lib/Format/FormatToken.h index 0fe91adcd472..a60361a8e5fa 100644 --- a/contrib/llvm/tools/clang/lib/Format/FormatToken.h +++ b/contrib/llvm/tools/clang/lib/Format/FormatToken.h @@ -21,6 +21,7 @@ #include "clang/Format/Format.h" #include "clang/Lex/Lexer.h" #include +#include namespace clang { namespace format { @@ -466,7 +467,8 @@ struct FormatToken { (is(tok::l_brace) && (BlockKind == BK_Block || is(TT_DictLiteral) || (!Style.Cpp11BracedListStyle && NestingLevel == 0))) || - (is(tok::less) && Style.Language == FormatStyle::LK_Proto); + (is(tok::less) && (Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto)); } /// \brief Same as opensBlockOrBlockTypeList, but for the closing token. @@ -639,6 +641,7 @@ struct AdditionalKeywords { kw_is = &IdentTable.get("is"); kw_let = &IdentTable.get("let"); kw_module = &IdentTable.get("module"); + kw_readonly = &IdentTable.get("readonly"); kw_set = &IdentTable.get("set"); kw_type = &IdentTable.get("type"); kw_var = &IdentTable.get("var"); @@ -671,6 +674,15 @@ struct AdditionalKeywords { kw_qsignals = &IdentTable.get("Q_SIGNALS"); kw_slots = &IdentTable.get("slots"); kw_qslots = &IdentTable.get("Q_SLOTS"); + + // Keep this at the end of the constructor to make sure everything here is + // already initialized. + JsExtraKeywords = std::unordered_set( + {kw_as, kw_async, kw_await, kw_declare, kw_finally, kw_from, + kw_function, kw_get, kw_import, kw_is, kw_let, kw_module, kw_readonly, + kw_set, kw_type, kw_var, kw_yield, + // Keywords from the Java section. + kw_abstract, kw_extends, kw_implements, kw_instanceof, kw_interface}); } // Context sensitive keywords. @@ -699,6 +711,7 @@ struct AdditionalKeywords { IdentifierInfo *kw_is; IdentifierInfo *kw_let; IdentifierInfo *kw_module; + IdentifierInfo *kw_readonly; IdentifierInfo *kw_set; IdentifierInfo *kw_type; IdentifierInfo *kw_var; @@ -732,6 +745,18 @@ struct AdditionalKeywords { IdentifierInfo *kw_qsignals; IdentifierInfo *kw_slots; IdentifierInfo *kw_qslots; + + /// \brief Returns \c true if \p Tok is a true JavaScript identifier, returns + /// \c false if it is a keyword or a pseudo keyword. + bool IsJavaScriptIdentifier(const FormatToken &Tok) const { + return Tok.is(tok::identifier) && + JsExtraKeywords.find(Tok.Tok.getIdentifierInfo()) == + JsExtraKeywords.end(); + } + +private: + /// \brief The JavaScript keywords beyond the C++ keyword set. + std::unordered_set JsExtraKeywords; }; } // namespace format diff --git a/contrib/llvm/tools/clang/lib/Format/TokenAnnotator.cpp b/contrib/llvm/tools/clang/lib/Format/TokenAnnotator.cpp index d78a37532fe8..c6e90a9175e1 100644 --- a/contrib/llvm/tools/clang/lib/Format/TokenAnnotator.cpp +++ b/contrib/llvm/tools/clang/lib/Format/TokenAnnotator.cpp @@ -90,7 +90,8 @@ class AnnotatingParser { } if (CurrentToken->isOneOf(tok::r_paren, tok::r_square, tok::r_brace) || (CurrentToken->isOneOf(tok::colon, tok::question) && InExprContext && - Style.Language != FormatStyle::LK_Proto)) + Style.Language != FormatStyle::LK_Proto && + Style.Language != FormatStyle::LK_TextProto)) return false; // If a && or || is found and interpreted as a binary operator, this set // of angles is likely part of something like "a < b && c > d". If the @@ -453,7 +454,8 @@ class AnnotatingParser { FormatToken *Previous = CurrentToken->getPreviousNonComment(); if (((CurrentToken->is(tok::colon) && (!Contexts.back().ColonIsDictLiteral || !Style.isCpp())) || - Style.Language == FormatStyle::LK_Proto) && + Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto) && (Previous->Tok.getIdentifierInfo() || Previous->is(tok::string_literal))) Previous->Type = TT_SelectorName; @@ -536,8 +538,13 @@ class AnnotatingParser { } } if (Contexts.back().ColonIsDictLiteral || - Style.Language == FormatStyle::LK_Proto) { + Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto) { Tok->Type = TT_DictLiteral; + if (Style.Language == FormatStyle::LK_TextProto) { + if (FormatToken *Previous = Tok->getPreviousNonComment()) + Previous->Type = TT_SelectorName; + } } else if (Contexts.back().ColonIsObjCMethodExpr || Line.startsWith(TT_ObjCMethodSpecifier)) { Tok->Type = TT_ObjCMethodExpr; @@ -635,12 +642,22 @@ class AnnotatingParser { return false; break; case tok::l_brace: + if (Style.Language == FormatStyle::LK_TextProto) { + FormatToken *Previous =Tok->getPreviousNonComment(); + if (Previous && Previous->Type != TT_DictLiteral) + Previous->Type = TT_SelectorName; + } if (!parseBrace()) return false; break; case tok::less: if (parseAngle()) { Tok->Type = TT_TemplateOpener; + if (Style.Language == FormatStyle::LK_TextProto) { + FormatToken *Previous = Tok->getPreviousNonComment(); + if (Previous && Previous->Type != TT_DictLiteral) + Previous->Type = TT_SelectorName; + } } else { Tok->Type = TT_BinaryOperator; NonTemplateLess.insert(Tok); @@ -1572,7 +1589,8 @@ class ExpressionParser { return prec::Conditional; if (NextNonComment && Current->is(TT_SelectorName) && (NextNonComment->is(TT_DictLiteral) || - (Style.Language == FormatStyle::LK_Proto && + ((Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto) && NextNonComment->is(tok::less)))) return prec::Assignment; if (Current->is(TT_JsComputedPropertyName)) @@ -1676,17 +1694,26 @@ void TokenAnnotator::setCommentLineLevels( for (SmallVectorImpl::reverse_iterator I = Lines.rbegin(), E = Lines.rend(); I != E; ++I) { - bool CommentLine = (*I)->First; + bool CommentLine = true; for (const FormatToken *Tok = (*I)->First; Tok; Tok = Tok->Next) { if (!Tok->is(tok::comment)) { CommentLine = false; break; } } - if (NextNonCommentLine && CommentLine) - (*I)->Level = NextNonCommentLine->Level; - else + + if (NextNonCommentLine && CommentLine) { + // If the comment is currently aligned with the line immediately following + // it, that's probably intentional and we should keep it. + bool AlignedWithNextLine = + NextNonCommentLine->First->NewlinesBefore <= 1 && + NextNonCommentLine->First->OriginalColumn == + (*I)->First->OriginalColumn; + if (AlignedWithNextLine) + (*I)->Level = NextNonCommentLine->Level; + } else { NextNonCommentLine = (*I)->First->isNot(tok::r_brace) ? (*I) : nullptr; + } setCommentLineLevels((*I)->Children); } @@ -2274,7 +2301,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if (Style.isCpp()) { if (Left.is(tok::kw_operator)) return Right.is(tok::coloncolon); - } else if (Style.Language == FormatStyle::LK_Proto) { + } else if (Style.Language == FormatStyle::LK_Proto || + Style.Language == FormatStyle::LK_TextProto) { if (Right.is(tok::period) && Left.isOneOf(Keywords.kw_optional, Keywords.kw_required, Keywords.kw_repeated, Keywords.kw_extend)) @@ -2282,6 +2310,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if (Right.is(tok::l_paren) && Left.isOneOf(Keywords.kw_returns, Keywords.kw_option)) return true; + if (Right.isOneOf(tok::l_brace, tok::less) && Left.is(TT_SelectorName)) + return true; } else if (Style.Language == FormatStyle::LK_JavaScript) { if (Left.is(TT_JsFatArrow)) return true; @@ -2300,7 +2330,11 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) || (Right.is(TT_TemplateString) && Right.TokenText.startswith("}"))) return false; - if (Left.is(tok::identifier) && Right.is(TT_TemplateString)) + // In tagged template literals ("html`bar baz`"), there is no space between + // the tag identifier and the template string. getIdentifierInfo makes sure + // that the identifier is not a pseudo keyword like `yield`, either. + if (Left.is(tok::identifier) && Keywords.IsJavaScriptIdentifier(Left) && + Right.is(TT_TemplateString)) return false; if (Right.is(tok::star) && Left.isOneOf(Keywords.kw_function, Keywords.kw_yield)) @@ -2604,11 +2638,12 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, } else if (Style.Language == FormatStyle::LK_JavaScript) { const FormatToken *NonComment = Right.getPreviousNonComment(); if (NonComment && - NonComment->isOneOf( - tok::kw_return, tok::kw_continue, tok::kw_break, tok::kw_throw, - Keywords.kw_interface, Keywords.kw_type, tok::kw_static, - tok::kw_public, tok::kw_private, tok::kw_protected, - Keywords.kw_abstract, Keywords.kw_get, Keywords.kw_set)) + NonComment->isOneOf(tok::kw_return, tok::kw_continue, tok::kw_break, + tok::kw_throw, Keywords.kw_interface, + Keywords.kw_type, tok::kw_static, tok::kw_public, + tok::kw_private, tok::kw_protected, + Keywords.kw_readonly, Keywords.kw_abstract, + Keywords.kw_get, Keywords.kw_set)) return false; // Otherwise automatic semicolon insertion would trigger. if (Left.is(TT_JsFatArrow) && Right.is(tok::l_brace)) return false; diff --git a/contrib/llvm/tools/clang/lib/Format/UnwrappedLineParser.cpp b/contrib/llvm/tools/clang/lib/Format/UnwrappedLineParser.cpp index ba3a4c17ee12..4b57919d1929 100644 --- a/contrib/llvm/tools/clang/lib/Format/UnwrappedLineParser.cpp +++ b/contrib/llvm/tools/clang/lib/Format/UnwrappedLineParser.cpp @@ -286,7 +286,10 @@ void UnwrappedLineParser::parseFile() { !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript; ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack, MustBeDeclaration); - parseLevel(/*HasOpeningBrace=*/false); + if (Style.Language == FormatStyle::LK_TextProto) + parseBracedList(); + else + parseLevel(/*HasOpeningBrace=*/false); // Make sure to format the remaining tokens. flushComments(true); addUnwrappedLine(); @@ -832,6 +835,7 @@ void UnwrappedLineParser::parseStructuralElement() { case tok::at: nextToken(); if (FormatTok->Tok.is(tok::l_brace)) { + nextToken(); parseBracedList(); break; } @@ -996,8 +1000,10 @@ void UnwrappedLineParser::parseStructuralElement() { switch (FormatTok->Tok.getKind()) { case tok::at: nextToken(); - if (FormatTok->Tok.is(tok::l_brace)) + if (FormatTok->Tok.is(tok::l_brace)) { + nextToken(); parseBracedList(); + } break; case tok::kw_enum: // Ignore if this is part of "template Tok.is(tok::l_brace)) + if (FormatTok->Tok.is(tok::l_brace)) { + nextToken(); parseBracedList(); - else if (Style.Language == FormatStyle::LK_Proto && - FormatTok->Tok.is(tok::less)) + } else if (Style.Language == FormatStyle::LK_Proto && + FormatTok->Tok.is(tok::less)) { + nextToken(); parseBracedList(/*ContinueOnSemicolons=*/false, /*ClosingBraceKind=*/tok::greater); + } break; case tok::l_square: parseSquare(); @@ -1345,6 +1354,7 @@ bool UnwrappedLineParser::tryToParseBracedList() { assert(FormatTok->BlockKind != BK_Unknown); if (FormatTok->BlockKind == BK_Block) return false; + nextToken(); parseBracedList(); return true; } @@ -1352,7 +1362,6 @@ bool UnwrappedLineParser::tryToParseBracedList() { bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, tok::TokenKind ClosingBraceKind) { bool HasError = false; - nextToken(); // FIXME: Once we have an expression parser in the UnwrappedLineParser, // replace this by using parseAssigmentExpression() inside. @@ -1407,6 +1416,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, // Assume there are no blocks inside a braced init list apart // from the ones we explicitly parse out (like lambdas). FormatTok->BlockKind = BK_BracedInit; + nextToken(); parseBracedList(); break; case tok::semi: @@ -1459,8 +1469,10 @@ void UnwrappedLineParser::parseParens() { break; case tok::at: nextToken(); - if (FormatTok->Tok.is(tok::l_brace)) + if (FormatTok->Tok.is(tok::l_brace)) { + nextToken(); parseBracedList(); + } break; case tok::kw_class: if (Style.Language == FormatStyle::LK_JavaScript) @@ -1508,8 +1520,10 @@ void UnwrappedLineParser::parseSquare() { } case tok::at: nextToken(); - if (FormatTok->Tok.is(tok::l_brace)) + if (FormatTok->Tok.is(tok::l_brace)) { + nextToken(); parseBracedList(); + } break; default: nextToken(); @@ -1836,6 +1850,7 @@ bool UnwrappedLineParser::parseEnum() { } // Parse enum body. + nextToken(); bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true); if (HasError) { if (FormatTok->is(tok::semi)) @@ -1870,6 +1885,7 @@ void UnwrappedLineParser::parseJavaEnumBody() { FormatTok = Tokens->setPosition(StoredPosition); if (IsSimple) { + nextToken(); parseBracedList(); addUnwrappedLine(); return; @@ -2081,6 +2097,7 @@ void UnwrappedLineParser::parseJavaScriptEs6ImportExport() { } if (FormatTok->is(tok::l_brace)) { FormatTok->BlockKind = BK_Block; + nextToken(); parseBracedList(); } else { nextToken(); diff --git a/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp b/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp index 0fbcc1c7399e..d42400183a43 100644 --- a/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp +++ b/contrib/llvm/tools/clang/lib/Frontend/FrontendActions.cpp @@ -163,6 +163,16 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, return llvm::make_unique(std::move(Consumers)); } +bool GenerateModuleFromModuleMapAction::BeginSourceFileAction( + CompilerInstance &CI) { + if (!CI.getLangOpts().Modules) { + CI.getDiagnostics().Report(diag::err_module_build_requires_fmodules); + return false; + } + + return GenerateModuleAction::BeginSourceFileAction(CI); +} + std::unique_ptr GenerateModuleFromModuleMapAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) { diff --git a/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp b/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp index 83290a6fbc28..38be684cec86 100644 --- a/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp +++ b/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp @@ -146,7 +146,7 @@ namespace { llvm::DenseMap RewrittenBlockExprs; llvm::DenseMap > ReferencedIvars; + llvm::SmallSetVector > ReferencedIvars; // ivar bitfield grouping containers llvm::DenseSet ObjCInterefaceHasBitfieldGroups; @@ -1013,7 +1013,7 @@ void RewriteModernObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID, Setr = "\nextern \"C\" __declspec(dllimport) " "void objc_setProperty (id, SEL, long, id, bool, bool);\n"; } - + RewriteObjCMethodDecl(OID->getContainingInterface(), PD->getSetterMethodDecl(), Setr); Setr += "{ "; @@ -3965,10 +3965,11 @@ void RewriteModernObjC::RewriteIvarOffsetSymbols(ObjCInterfaceDecl *CDecl, std::string &Result) { // write out ivar offset symbols which have been referenced in an ivar // access expression. - llvm::SmallPtrSet Ivars = ReferencedIvars[CDecl]; + llvm::SmallSetVector Ivars = ReferencedIvars[CDecl]; + if (Ivars.empty()) return; - + llvm::DenseSet > GroupSymbolOutput; for (ObjCIvarDecl *IvarDecl : Ivars) { const ObjCInterfaceDecl *IDecl = IvarDecl->getContainingInterface(); @@ -6068,7 +6069,7 @@ void RewriteModernObjC::Initialize(ASTContext &context) { Preamble += "\n#define __OFFSETOFIVAR__(TYPE, MEMBER) ((long long) &((TYPE *)0)->MEMBER)\n"; } -/// RewriteIvarOffsetComputation - This rutine synthesizes computation of +/// RewriteIvarOffsetComputation - This routine synthesizes computation of /// ivar offset. void RewriteModernObjC::RewriteIvarOffsetComputation(ObjCIvarDecl *ivar, std::string &Result) { diff --git a/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteObjC.cpp b/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteObjC.cpp index 7d809c610c86..5a1e001d65b8 100644 --- a/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteObjC.cpp +++ b/contrib/llvm/tools/clang/lib/Frontend/Rewrite/RewriteObjC.cpp @@ -5052,7 +5052,7 @@ void RewriteObjCFragileABI::Initialize(ASTContext &context) { Preamble += "\n#define __OFFSETOFIVAR__(TYPE, MEMBER) ((long long) &((TYPE *)0)->MEMBER)\n"; } -/// RewriteIvarOffsetComputation - This rutine synthesizes computation of +/// RewriteIvarOffsetComputation - This routine synthesizes computation of /// ivar offset. void RewriteObjCFragileABI::RewriteIvarOffsetComputation(ObjCIvarDecl *ivar, std::string &Result) { diff --git a/contrib/llvm/tools/clang/lib/Frontend/SerializedDiagnosticReader.cpp b/contrib/llvm/tools/clang/lib/Frontend/SerializedDiagnosticReader.cpp index 8a8161488f44..08b7087fbad6 100644 --- a/contrib/llvm/tools/clang/lib/Frontend/SerializedDiagnosticReader.cpp +++ b/contrib/llvm/tools/clang/lib/Frontend/SerializedDiagnosticReader.cpp @@ -27,6 +27,9 @@ std::error_code SerializedDiagnosticReader::readDiagnostics(StringRef File) { llvm::BitstreamCursor Stream(**Buffer); Optional BlockInfo; + if (Stream.AtEndOfStream()) + return SDError::InvalidSignature; + // Sniff for the signature. if (Stream.Read(8) != 'D' || Stream.Read(8) != 'I' || diff --git a/contrib/llvm/tools/clang/lib/Frontend/TextDiagnostic.cpp b/contrib/llvm/tools/clang/lib/Frontend/TextDiagnostic.cpp index 1e12ea5e597a..6a72b00c602b 100644 --- a/contrib/llvm/tools/clang/lib/Frontend/TextDiagnostic.cpp +++ b/contrib/llvm/tools/clang/lib/Frontend/TextDiagnostic.cpp @@ -1052,7 +1052,8 @@ static void highlightRange(const CharSourceRange &R, std::fill(CaretLine.begin()+StartColNo,CaretLine.begin()+EndColNo,'~'); } -static std::string buildFixItInsertionLine(unsigned LineNo, +static std::string buildFixItInsertionLine(FileID FID, + unsigned LineNo, const SourceColumnMap &map, ArrayRef Hints, const SourceManager &SM, @@ -1069,7 +1070,8 @@ static std::string buildFixItInsertionLine(unsigned LineNo, // code contains no newlines and is on the same line as the caret. std::pair HintLocInfo = SM.getDecomposedExpansionLoc(I->RemoveRange.getBegin()); - if (LineNo == SM.getLineNumber(HintLocInfo.first, HintLocInfo.second) && + if (FID == HintLocInfo.first && + LineNo == SM.getLineNumber(HintLocInfo.first, HintLocInfo.second) && StringRef(I->CodeToInsert).find_first_of("\n\r") == StringRef::npos) { // Insert the new code into the line just below the code // that the user wrote. @@ -1105,9 +1107,6 @@ static std::string buildFixItInsertionLine(unsigned LineNo, PrevHintEndCol = HintCol + llvm::sys::locale::columnWidth(I->CodeToInsert); - } else { - FixItInsertionLine.clear(); - break; } } } @@ -1222,7 +1221,7 @@ void TextDiagnostic::emitSnippetAndCaret( } std::string FixItInsertionLine = buildFixItInsertionLine( - LineNo, sourceColMap, Hints, SM, DiagOpts.get()); + FID, LineNo, sourceColMap, Hints, SM, DiagOpts.get()); // If the source line is too long for our terminal, select only the // "interesting" source region within that line. diff --git a/contrib/llvm/tools/clang/lib/Headers/bmiintrin.h b/contrib/llvm/tools/clang/lib/Headers/bmiintrin.h index 361e5f720ea1..e812a1632b91 100644 --- a/contrib/llvm/tools/clang/lib/Headers/bmiintrin.h +++ b/contrib/llvm/tools/clang/lib/Headers/bmiintrin.h @@ -318,7 +318,7 @@ __blsi_u64(unsigned long long __X) /// /// \param __X /// An unsigned 64-bit integer used to create the mask. -/// \returns A unsigned 64-bit integer containing the newly created mask. +/// \returns An unsigned 64-bit integer containing the newly created mask. static __inline__ unsigned long long __DEFAULT_FN_ATTRS __blsmsk_u64(unsigned long long __X) { diff --git a/contrib/llvm/tools/clang/lib/Headers/cpuid.h b/contrib/llvm/tools/clang/lib/Headers/cpuid.h index 400dcfacd552..2dd0add236b8 100644 --- a/contrib/llvm/tools/clang/lib/Headers/cpuid.h +++ b/contrib/llvm/tools/clang/lib/Headers/cpuid.h @@ -79,7 +79,7 @@ #define signature_VORTEX_edx 0x36387865 #define signature_VORTEX_ecx 0x436f5320 -/* Features in %ecx for level 1 */ +/* Features in %ecx for leaf 1 */ #define bit_SSE3 0x00000001 #define bit_PCLMULQDQ 0x00000002 #define bit_PCLMUL bit_PCLMULQDQ /* for gcc compat */ @@ -114,7 +114,7 @@ #define bit_F16C 0x20000000 #define bit_RDRND 0x40000000 -/* Features in %edx for level 1 */ +/* Features in %edx for leaf 1 */ #define bit_FPU 0x00000001 #define bit_VME 0x00000002 #define bit_DE 0x00000004 @@ -147,44 +147,95 @@ #define bit_TM 0x20000000 #define bit_PBE 0x80000000 -/* Features in %ebx for level 7 sub-leaf 0 */ +/* Features in %ebx for leaf 7 sub-leaf 0 */ #define bit_FSGSBASE 0x00000001 +#define bit_SGX 0x00000004 +#define bit_BMI 0x00000008 +#define bit_HLE 0x00000010 +#define bit_AVX2 0x00000020 #define bit_SMEP 0x00000080 +#define bit_BMI2 0x00000100 #define bit_ENH_MOVSB 0x00000200 +#define bit_RTM 0x00000800 +#define bit_MPX 0x00004000 +#define bit_AVX512F 0x00010000 +#define bit_AVX512DQ 0x00020000 +#define bit_RDSEED 0x00040000 +#define bit_ADX 0x00080000 +#define bit_AVX512IFMA 0x00200000 +#define bit_CLFLUSHOPT 0x00800000 +#define bit_CLWB 0x01000000 +#define bit_AVX512PF 0x04000000 +#define bit_AVX51SER 0x08000000 +#define bit_AVX512CD 0x10000000 +#define bit_SHA 0x20000000 +#define bit_AVX512BW 0x40000000 +#define bit_AVX512VL 0x80000000 + +/* Features in %ecx for leaf 7 sub-leaf 0 */ +#define bit_PREFTCHWT1 0x00000001 +#define bit_AVX512VBMI 0x00000002 +#define bit_PKU 0x00000004 +#define bit_OSPKE 0x00000010 +#define bit_AVX512VPOPCNTDQ 0x00004000 +#define bit_RDPID 0x00400000 + +/* Features in %edx for leaf 7 sub-leaf 0 */ +#define bit_AVX5124VNNIW 0x00000004 +#define bit_AVX5124FMAPS 0x00000008 + +/* Features in %eax for leaf 13 sub-leaf 1 */ +#define bit_XSAVEOPT 0x00000001 +#define bit_XSAVEC 0x00000002 +#define bit_XSAVES 0x00000008 + +/* Features in %ecx for leaf 0x80000001 */ +#define bit_LAHF_LM 0x00000001 +#define bit_ABM 0x00000020 +#define bit_SSE4a 0x00000040 +#define bit_PRFCHW 0x00000100 +#define bit_XOP 0x00000800 +#define bit_LWP 0x00008000 +#define bit_FMA4 0x00010000 +#define bit_TBM 0x00200000 +#define bit_MWAITX 0x20000000 + +/* Features in %edx for leaf 0x80000001 */ +#define bit_MMXEXT 0x00400000 +#define bit_LM 0x20000000 +#define bit_3DNOWP 0x40000000 +#define bit_3DNOW 0x80000000 + +/* Features in %ebx for leaf 0x80000001 */ +#define bit_CLZERO 0x00000001 + #if __i386__ -#define __cpuid(__level, __eax, __ebx, __ecx, __edx) \ +#define __cpuid(__leaf, __eax, __ebx, __ecx, __edx) \ __asm("cpuid" : "=a"(__eax), "=b" (__ebx), "=c"(__ecx), "=d"(__edx) \ - : "0"(__level)) + : "0"(__leaf)) -#define __cpuid_count(__level, __count, __eax, __ebx, __ecx, __edx) \ +#define __cpuid_count(__leaf, __count, __eax, __ebx, __ecx, __edx) \ __asm("cpuid" : "=a"(__eax), "=b" (__ebx), "=c"(__ecx), "=d"(__edx) \ - : "0"(__level), "2"(__count)) + : "0"(__leaf), "2"(__count)) #else /* x86-64 uses %rbx as the base register, so preserve it. */ -#define __cpuid(__level, __eax, __ebx, __ecx, __edx) \ +#define __cpuid(__leaf, __eax, __ebx, __ecx, __edx) \ __asm(" xchgq %%rbx,%q1\n" \ " cpuid\n" \ " xchgq %%rbx,%q1" \ : "=a"(__eax), "=r" (__ebx), "=c"(__ecx), "=d"(__edx) \ - : "0"(__level)) + : "0"(__leaf)) -#define __cpuid_count(__level, __count, __eax, __ebx, __ecx, __edx) \ +#define __cpuid_count(__leaf, __count, __eax, __ebx, __ecx, __edx) \ __asm(" xchgq %%rbx,%q1\n" \ " cpuid\n" \ " xchgq %%rbx,%q1" \ : "=a"(__eax), "=r" (__ebx), "=c"(__ecx), "=d"(__edx) \ - : "0"(__level), "2"(__count)) + : "0"(__leaf), "2"(__count)) #endif -static __inline int __get_cpuid (unsigned int __level, unsigned int *__eax, - unsigned int *__ebx, unsigned int *__ecx, - unsigned int *__edx) { - __cpuid(__level, *__eax, *__ebx, *__ecx, *__edx); - return 1; -} - -static __inline int __get_cpuid_max (unsigned int __level, unsigned int *__sig) +static __inline int __get_cpuid_max (unsigned int __leaf, unsigned int *__sig) { unsigned int __eax, __ebx, __ecx, __edx; #if __i386__ @@ -208,8 +259,35 @@ static __inline int __get_cpuid_max (unsigned int __level, unsigned int *__sig) return 0; #endif - __cpuid(__level, __eax, __ebx, __ecx, __edx); + __cpuid(__leaf, __eax, __ebx, __ecx, __edx); if (__sig) *__sig = __ebx; return __eax; } + +static __inline int __get_cpuid (unsigned int __leaf, unsigned int *__eax, + unsigned int *__ebx, unsigned int *__ecx, + unsigned int *__edx) +{ + unsigned int __max_leaf = __get_cpuid_max(__leaf & 0x80000000, 0); + + if (__max_leaf == 0 || __max_leaf < __leaf) + return 0; + + __cpuid(__leaf, *__eax, *__ebx, *__ecx, *__edx); + return 1; +} + +static __inline int __get_cpuid_count (unsigned int __leaf, + unsigned int __subleaf, + unsigned int *__eax, unsigned int *__ebx, + unsigned int *__ecx, unsigned int *__edx) +{ + unsigned int __max_leaf = __get_cpuid_max(__leaf & 0x80000000, 0); + + if (__max_leaf == 0 || __max_leaf < __leaf) + return 0; + + __cpuid_count(__leaf, __subleaf, *__eax, *__ebx, *__ecx, *__edx); + return 1; +} diff --git a/contrib/llvm/tools/clang/lib/Headers/immintrin.h b/contrib/llvm/tools/clang/lib/Headers/immintrin.h index e22dd231427d..c5f25bfcb5c1 100644 --- a/contrib/llvm/tools/clang/lib/Headers/immintrin.h +++ b/contrib/llvm/tools/clang/lib/Headers/immintrin.h @@ -212,6 +212,15 @@ _rdrand32_step(unsigned int *__p) return __builtin_ia32_rdrand32_step(__p); } +#ifdef __x86_64__ +static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) +_rdrand64_step(unsigned long long *__p) +{ + return __builtin_ia32_rdrand64_step(__p); +} +#endif +#endif /* __RDRND__ */ + /* __bit_scan_forward */ static __inline__ int __attribute__((__always_inline__, __nodebug__)) _bit_scan_forward(int __A) { @@ -224,15 +233,6 @@ _bit_scan_reverse(int __A) { return 31 - __builtin_clz(__A); } -#ifdef __x86_64__ -static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) -_rdrand64_step(unsigned long long *__p) -{ - return __builtin_ia32_rdrand64_step(__p); -} -#endif -#endif /* __RDRND__ */ - #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__) #ifdef __x86_64__ static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) diff --git a/contrib/llvm/tools/clang/lib/Headers/mmintrin.h b/contrib/llvm/tools/clang/lib/Headers/mmintrin.h index 5a7968bec842..4b38d51713d8 100644 --- a/contrib/llvm/tools/clang/lib/Headers/mmintrin.h +++ b/contrib/llvm/tools/clang/lib/Headers/mmintrin.h @@ -1289,7 +1289,7 @@ _mm_cmpgt_pi32(__m64 __m1, __m64 __m2) /// /// \headerfile /// -/// This intrinsic corresponds to the the VXORPS / XORPS instruction. +/// This intrinsic corresponds to the VXORPS / XORPS instruction. /// /// \returns An initialized 64-bit integer vector with all elements set to zero. static __inline__ __m64 __DEFAULT_FN_ATTRS diff --git a/contrib/llvm/tools/clang/lib/Index/IndexBody.cpp b/contrib/llvm/tools/clang/lib/Index/IndexBody.cpp index d3632b8b9b15..6bbd38102509 100644 --- a/contrib/llvm/tools/clang/lib/Index/IndexBody.cpp +++ b/contrib/llvm/tools/clang/lib/Index/IndexBody.cpp @@ -230,7 +230,31 @@ class BodyIndexer : public RecursiveASTVisitor { SmallVector Relations; addCallRole(Roles, Relations); Stmt *Containing = getParentStmt(); - if (E->isImplicit() || (Containing && isa(Containing))) + + auto IsImplicitProperty = [](const PseudoObjectExpr *POE) -> bool { + const auto *E = POE->getSyntacticForm(); + if (const auto *BinOp = dyn_cast(E)) + E = BinOp->getLHS(); + const auto *PRE = dyn_cast(E); + if (!PRE) + return false; + if (PRE->isExplicitProperty()) + return false; + if (const ObjCMethodDecl *Getter = PRE->getImplicitPropertyGetter()) { + // Class properties that are explicitly defined using @property + // declarations are represented implicitly as there is no ivar for + // class properties. + if (Getter->isClassMethod() && + Getter->getCanonicalDecl()->findPropertyDecl()) + return false; + } + return true; + }; + bool IsPropCall = Containing && isa(Containing); + // Implicit property message sends are not 'implicit'. + if ((E->isImplicit() || IsPropCall) && + !(IsPropCall && + IsImplicitProperty(cast(Containing)))) Roles |= (unsigned)SymbolRole::Implicit; if (isDynamic(E)) { diff --git a/contrib/llvm/tools/clang/lib/Index/IndexDecl.cpp b/contrib/llvm/tools/clang/lib/Index/IndexDecl.cpp index d1127722c8ca..c5230c0f9acf 100644 --- a/contrib/llvm/tools/clang/lib/Index/IndexDecl.cpp +++ b/contrib/llvm/tools/clang/lib/Index/IndexDecl.cpp @@ -618,6 +618,8 @@ class IndexingDeclVisitor : public ConstDeclVisitor { Template.is() ? (Decl *)Template.get() : Template.get(); + if (!D->isThisDeclarationADefinition()) + IndexCtx.indexNestedNameSpecifierLoc(D->getQualifierLoc(), D); IndexCtx.indexTagDecl( D, SymbolRelation(SymbolRoleSet(SymbolRole::RelationSpecializationOf), SpecializationOf)); diff --git a/contrib/llvm/tools/clang/lib/Index/IndexSymbol.cpp b/contrib/llvm/tools/clang/lib/Index/IndexSymbol.cpp index bf358a372149..0dc3720208ca 100644 --- a/contrib/llvm/tools/clang/lib/Index/IndexSymbol.cpp +++ b/contrib/llvm/tools/clang/lib/Index/IndexSymbol.cpp @@ -69,11 +69,13 @@ bool index::isFunctionLocalSymbol(const Decl *D) { if (const NamedDecl *ND = dyn_cast(D)) { switch (ND->getFormalLinkage()) { case NoLinkage: - case VisibleNoLinkage: case InternalLinkage: return true; + case VisibleNoLinkage: case UniqueExternalLinkage: + case ModuleInternalLinkage: llvm_unreachable("Not a sema linkage"); + case ModuleLinkage: case ExternalLinkage: return false; } diff --git a/contrib/llvm/tools/clang/lib/Index/IndexingContext.cpp b/contrib/llvm/tools/clang/lib/Index/IndexingContext.cpp index 754bc84ff4b2..c4aa51d62f02 100644 --- a/contrib/llvm/tools/clang/lib/Index/IndexingContext.cpp +++ b/contrib/llvm/tools/clang/lib/Index/IndexingContext.cpp @@ -229,6 +229,12 @@ static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC, AST return false; } +/// Whether the given NamedDecl should be skipped because it has no name. +static bool shouldSkipNamelessDecl(const NamedDecl *ND) { + return ND->getDeclName().isEmpty() && !isa(ND) && + !isa(ND); +} + static const Decl *adjustParent(const Decl *Parent) { if (!Parent) return nullptr; @@ -243,8 +249,8 @@ static const Decl *adjustParent(const Decl *Parent) { } else if (auto RD = dyn_cast(Parent)) { if (RD->isAnonymousStructOrUnion()) continue; - } else if (auto FD = dyn_cast(Parent)) { - if (FD->getDeclName().isEmpty()) + } else if (auto ND = dyn_cast(Parent)) { + if (shouldSkipNamelessDecl(ND)) continue; } return Parent; @@ -315,9 +321,7 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc, const DeclContext *ContainerDC) { if (D->isImplicit() && !isa(D)) return true; - if (!isa(D) || - (cast(D)->getDeclName().isEmpty() && - !isa(D) && !isa(D))) + if (!isa(D) || shouldSkipNamelessDecl(cast(D))) return true; SourceManager &SM = Ctx->getSourceManager(); diff --git a/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp b/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp index 012189aa6f9f..61bcef8cb760 100644 --- a/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp +++ b/contrib/llvm/tools/clang/lib/Lex/Lexer.cpp @@ -2550,7 +2550,7 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { return true; } - if (PP->isRecordingPreamble() && !PP->isInMainFile()) { + if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) { PP->setRecordedPreambleConditionalStack(ConditionalStack); ConditionalStack.clear(); } diff --git a/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp b/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp index 1c0cd5636835..5a589d6a17b3 100644 --- a/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp +++ b/contrib/llvm/tools/clang/lib/Lex/PPLexerChange.cpp @@ -46,12 +46,6 @@ bool Preprocessor::isInPrimaryFile() const { }); } -bool Preprocessor::isInMainFile() const { - if (IsFileLexer()) - return IncludeMacroStack.size() == 0; - return true; -} - /// getCurrentLexer - Return the current file lexer being lexed from. Note /// that this ignores any potentially active macro expansions and _Pragma /// expansions going on at the time. diff --git a/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp b/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp index f9a399cd7fd7..63f39524d12a 100644 --- a/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp +++ b/contrib/llvm/tools/clang/lib/Lex/Preprocessor.cpp @@ -535,7 +535,9 @@ void Preprocessor::EnterMainSourceFile() { // Start parsing the predefines. EnterSourceFile(FID, nullptr, SourceLocation()); +} +void Preprocessor::replayPreambleConditionalStack() { // Restore the conditional stack from the preamble, if there is one. if (PreambleConditionalStack.isReplaying()) { CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp index cc1e8850a523..b68559485a5e 100644 --- a/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/contrib/llvm/tools/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -889,7 +889,7 @@ bool Parser::ConsumeAndStoreFunctionPrologue(CachedTokens &Toks) { // If the opening brace is not preceded by one of these tokens, we are // missing the mem-initializer-id. In order to recover better, we need // to use heuristics to determine if this '{' is most likely the - // begining of a brace-init-list or the function body. + // beginning of a brace-init-list or the function body. // Check the token after the corresponding '}'. TentativeParsingAction PA(*this); if (SkipUntil(tok::r_brace) && diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp index 07054546f42f..a4610698c46d 100644 --- a/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp +++ b/contrib/llvm/tools/clang/lib/Parse/ParseDecl.cpp @@ -6650,7 +6650,7 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) { return; } - // If we get here, the operand to the typeof was an expresion. + // If we get here, the operand to the typeof was an expression. if (Operand.isInvalid()) { DS.SetTypeSpecError(); return; diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp index aacb00e8be64..44b87af01abd 100644 --- a/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp +++ b/contrib/llvm/tools/clang/lib/Parse/ParseExpr.cpp @@ -1866,7 +1866,7 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok, } } - // If we get here, the operand to the typeof/sizeof/alignof was an expresion. + // If we get here, the operand to the typeof/sizeof/alignof was an expression. isCastExpr = false; return Operand; } @@ -1972,7 +1972,7 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() { if (OpTok.isOneOf(tok::kw_alignof, tok::kw__Alignof)) Diag(OpTok, diag::ext_alignof_expr) << OpTok.getIdentifierInfo(); - // If we get here, the operand to the sizeof/alignof was an expresion. + // If we get here, the operand to the sizeof/alignof was an expression. if (!Operand.isInvalid()) Operand = Actions.ActOnUnaryExprOrTypeTraitExpr(OpTok.getLocation(), ExprKind, diff --git a/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp b/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp index caa6323d3209..f7410b8a092a 100644 --- a/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp +++ b/contrib/llvm/tools/clang/lib/Parse/ParseObjc.cpp @@ -2255,7 +2255,7 @@ Parser::ObjCImplParsingDataRAII::~ObjCImplParsingDataRAII() { void Parser::ObjCImplParsingDataRAII::finish(SourceRange AtEnd) { assert(!Finished); - P.Actions.DefaultSynthesizeProperties(P.getCurScope(), Dcl); + P.Actions.DefaultSynthesizeProperties(P.getCurScope(), Dcl, AtEnd.getBegin()); for (size_t i = 0; i < LateParsedObjCMethods.size(); ++i) P.ParseLexedObjCMethodDefs(*LateParsedObjCMethods[i], true/*Methods*/); diff --git a/contrib/llvm/tools/clang/lib/Parse/Parser.cpp b/contrib/llvm/tools/clang/lib/Parse/Parser.cpp index af29b5e9c673..4aa9a5971929 100644 --- a/contrib/llvm/tools/clang/lib/Parse/Parser.cpp +++ b/contrib/llvm/tools/clang/lib/Parse/Parser.cpp @@ -516,6 +516,8 @@ void Parser::Initialize() { // Prime the lexer look-ahead. ConsumeToken(); + + PP.replayPreambleConditionalStack(); } void Parser::LateTemplateParserCleanupCallback(void *P) { @@ -526,6 +528,8 @@ void Parser::LateTemplateParserCleanupCallback(void *P) { } bool Parser::ParseFirstTopLevelDecl(DeclGroupPtrTy &Result) { + Actions.ActOnStartOfTranslationUnit(); + // C11 6.9p1 says translation units must have at least one top-level // declaration. C++ doesn't have this restriction. We also don't want to // complain if we have a precompiled header, although technically if the PCH diff --git a/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp b/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp index fd2d07957c2b..f83baa790b49 100644 --- a/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/AnalysisBasedWarnings.cpp @@ -394,15 +394,21 @@ static bool hasThrowOutNonThrowingFunc(SourceLocation &OpLoc, CFG *BodyCFG) { static void EmitDiagForCXXThrowInNonThrowingFunc(Sema &S, SourceLocation OpLoc, const FunctionDecl *FD) { - if (!S.getSourceManager().isInSystemHeader(OpLoc)) { + if (!S.getSourceManager().isInSystemHeader(OpLoc) && + FD->getTypeSourceInfo()) { S.Diag(OpLoc, diag::warn_throw_in_noexcept_func) << FD; if (S.getLangOpts().CPlusPlus11 && (isa(FD) || FD->getDeclName().getCXXOverloadedOperator() == OO_Delete || - FD->getDeclName().getCXXOverloadedOperator() == OO_Array_Delete)) - S.Diag(FD->getLocation(), diag::note_throw_in_dtor); - else - S.Diag(FD->getLocation(), diag::note_throw_in_function); + FD->getDeclName().getCXXOverloadedOperator() == OO_Array_Delete)) { + if (const auto *Ty = FD->getTypeSourceInfo()->getType()-> + getAs()) + S.Diag(FD->getLocation(), diag::note_throw_in_dtor) + << !isa(FD) << !Ty->hasExceptionSpec() + << FD->getExceptionSpecSourceRange(); + } else + S.Diag(FD->getLocation(), diag::note_throw_in_function) + << FD->getExceptionSpecSourceRange(); } } @@ -420,8 +426,7 @@ static void checkThrowInNonThrowingFunc(Sema &S, const FunctionDecl *FD, static bool isNoexcept(const FunctionDecl *FD) { const auto *FPT = FD->getType()->castAs(); - if (FPT->getExceptionSpecType() != EST_None && - FPT->isNothrow(FD->getASTContext())) + if (FPT->isNothrow(FD->getASTContext())) return true; return false; } diff --git a/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp b/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp index 2fa5718d4e9b..3d321d561e60 100644 --- a/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/DelayedDiagnostic.cpp @@ -22,7 +22,8 @@ using namespace sema; DelayedDiagnostic DelayedDiagnostic::makeAvailability(AvailabilityResult AR, SourceLocation Loc, - const NamedDecl *D, + const NamedDecl *ReferringDecl, + const NamedDecl *OffendingDecl, const ObjCInterfaceDecl *UnknownObjCClass, const ObjCPropertyDecl *ObjCProperty, StringRef Msg, @@ -31,7 +32,8 @@ DelayedDiagnostic::makeAvailability(AvailabilityResult AR, DD.Kind = Availability; DD.Triggered = false; DD.Loc = Loc; - DD.AvailabilityData.Decl = D; + DD.AvailabilityData.ReferringDecl = ReferringDecl; + DD.AvailabilityData.OffendingDecl = OffendingDecl; DD.AvailabilityData.UnknownObjCClass = UnknownObjCClass; DD.AvailabilityData.ObjCProperty = ObjCProperty; char *MessageData = nullptr; diff --git a/contrib/llvm/tools/clang/lib/Sema/Sema.cpp b/contrib/llvm/tools/clang/lib/Sema/Sema.cpp index 34f5e26be810..dc9f977d41ac 100644 --- a/contrib/llvm/tools/clang/lib/Sema/Sema.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/Sema.cpp @@ -705,6 +705,18 @@ void Sema::emitAndClearUnusedLocalTypedefWarnings() { UnusedLocalTypedefNameCandidates.clear(); } +/// This is called before the very first declaration in the translation unit +/// is parsed. Note that the ASTContext may have already injected some +/// declarations. +void Sema::ActOnStartOfTranslationUnit() { + if (getLangOpts().ModulesTS) { + // We start in the global module; all those declarations are implicitly + // module-private (though they do not have module linkage). + Context.getTranslationUnitDecl()->setModuleOwnershipKind( + Decl::ModuleOwnershipKind::ModulePrivate); + } +} + /// ActOnEndOfTranslationUnit - This is called at the very end of the /// translation unit when EOF is reached and all but the top-level scope is /// popped. diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp index 7d534263f468..ba2049d8a606 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaCast.cpp @@ -143,6 +143,9 @@ namespace { }; } +static void DiagnoseCastQual(Sema &Self, const ExprResult &SrcExpr, + QualType DestType); + // The Try functions attempt a specific way of casting. If they succeed, they // return TC_Success. If their way of casting is not appropriate for the given // arguments, they return TC_NotApplicable and *may* set diag to a diagnostic @@ -427,6 +430,10 @@ static void diagnoseBadCast(Sema &S, unsigned msg, CastType castType, /// the same kind of pointer (plain or to-member). Unlike the Sema function, /// this one doesn't care if the two pointers-to-member don't point into the /// same class. This is because CastsAwayConstness doesn't care. +/// And additionally, it handles C++ references. If both the types are +/// references, then their pointee types are returned, +/// else if only one of them is reference, it's pointee type is returned, +/// and the other type is returned as-is. static bool UnwrapDissimilarPointerTypes(QualType& T1, QualType& T2) { const PointerType *T1PtrType = T1->getAs(), *T2PtrType = T2->getAs(); @@ -475,6 +482,26 @@ static bool UnwrapDissimilarPointerTypes(QualType& T1, QualType& T2) { return true; } + const LValueReferenceType *T1RefType = T1->getAs(), + *T2RefType = T2->getAs(); + if (T1RefType && T2RefType) { + T1 = T1RefType->getPointeeType(); + T2 = T2RefType->getPointeeType(); + return true; + } + + if (T1RefType) { + T1 = T1RefType->getPointeeType(); + // T2 = T2; + return true; + } + + if (T2RefType) { + // T1 = T1; + T2 = T2RefType->getPointeeType(); + return true; + } + return false; } @@ -503,11 +530,13 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType, // the rules are non-trivial. So first we construct Tcv *...cv* as described // in C++ 5.2.11p8. assert((SrcType->isAnyPointerType() || SrcType->isMemberPointerType() || - SrcType->isBlockPointerType()) && + SrcType->isBlockPointerType() || + DestType->isLValueReferenceType()) && "Source type is not pointer or pointer to member."); assert((DestType->isAnyPointerType() || DestType->isMemberPointerType() || - DestType->isBlockPointerType()) && - "Destination type is not pointer or pointer to member."); + DestType->isBlockPointerType() || + DestType->isLValueReferenceType()) && + "Destination type is not pointer or pointer to member, or reference."); QualType UnwrappedSrcType = Self.Context.getCanonicalType(SrcType), UnwrappedDestType = Self.Context.getCanonicalType(DestType); @@ -2177,6 +2206,8 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr, void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle, bool ListInitialization) { + assert(Self.getLangOpts().CPlusPlus); + // Handle placeholders. if (isPlaceholder()) { // C-style casts can resolve __unknown_any types. @@ -2580,30 +2611,42 @@ void CastOperation::CheckCStyleCast() { if (Kind == CK_BitCast) checkCastAlign(); +} + +/// DiagnoseCastQual - Warn whenever casts discards a qualifiers, be it either +/// const, volatile or both. +static void DiagnoseCastQual(Sema &Self, const ExprResult &SrcExpr, + QualType DestType) { + if (SrcExpr.isInvalid()) + return; + + QualType SrcType = SrcExpr.get()->getType(); + if (!((SrcType->isAnyPointerType() && DestType->isAnyPointerType()) || + DestType->isLValueReferenceType())) + return; - // -Wcast-qual QualType TheOffendingSrcType, TheOffendingDestType; Qualifiers CastAwayQualifiers; - if (SrcType->isAnyPointerType() && DestType->isAnyPointerType() && - CastsAwayConstness(Self, SrcType, DestType, true, false, - &TheOffendingSrcType, &TheOffendingDestType, - &CastAwayQualifiers)) { - int qualifiers = -1; - if (CastAwayQualifiers.hasConst() && CastAwayQualifiers.hasVolatile()) { - qualifiers = 0; - } else if (CastAwayQualifiers.hasConst()) { - qualifiers = 1; - } else if (CastAwayQualifiers.hasVolatile()) { - qualifiers = 2; - } - // This is a variant of int **x; const int **y = (const int **)x; - if (qualifiers == -1) - Self.Diag(SrcExpr.get()->getLocStart(), diag::warn_cast_qual2) << - SrcType << DestType; - else - Self.Diag(SrcExpr.get()->getLocStart(), diag::warn_cast_qual) << - TheOffendingSrcType << TheOffendingDestType << qualifiers; + if (!CastsAwayConstness(Self, SrcType, DestType, true, false, + &TheOffendingSrcType, &TheOffendingDestType, + &CastAwayQualifiers)) + return; + + int qualifiers = -1; + if (CastAwayQualifiers.hasConst() && CastAwayQualifiers.hasVolatile()) { + qualifiers = 0; + } else if (CastAwayQualifiers.hasConst()) { + qualifiers = 1; + } else if (CastAwayQualifiers.hasVolatile()) { + qualifiers = 2; } + // This is a variant of int **x; const int **y = (const int **)x; + if (qualifiers == -1) + Self.Diag(SrcExpr.get()->getLocStart(), diag::warn_cast_qual2) + << SrcType << DestType; + else + Self.Diag(SrcExpr.get()->getLocStart(), diag::warn_cast_qual) + << TheOffendingSrcType << TheOffendingDestType << qualifiers; } ExprResult Sema::BuildCStyleCastExpr(SourceLocation LPLoc, @@ -2624,6 +2667,9 @@ ExprResult Sema::BuildCStyleCastExpr(SourceLocation LPLoc, if (Op.SrcExpr.isInvalid()) return ExprError(); + // -Wcast-qual + DiagnoseCastQual(Op.Self, Op.SrcExpr, Op.DestType); + return Op.complete(CStyleCastExpr::Create(Context, Op.ResultType, Op.ValueKind, Op.Kind, Op.SrcExpr.get(), &Op.BasePath, CastTypeInfo, LPLoc, RPLoc)); diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp index 845c4bf61b7a..8446601334ee 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp @@ -309,7 +309,8 @@ static bool SemaOpenCLBuiltinKernelWorkGroupSize(Sema &S, CallExpr *TheCall) { Expr *BlockArg = TheCall->getArg(0); if (!isBlockPointer(BlockArg)) { S.Diag(BlockArg->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) << "block"; + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << "block"; return true; } return checkOpenCLBlockArgs(S, BlockArg); @@ -394,24 +395,24 @@ static bool SemaOpenCLBuiltinEnqueueKernel(Sema &S, CallExpr *TheCall) { // First argument always needs to be a queue_t type. if (!Arg0->getType()->isQueueT()) { S.Diag(TheCall->getArg(0)->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) - << S.Context.OCLQueueTy; + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << S.Context.OCLQueueTy; return true; } // Second argument always needs to be a kernel_enqueue_flags_t enum value. if (!Arg1->getType()->isIntegerType()) { S.Diag(TheCall->getArg(1)->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) - << "'kernel_enqueue_flags_t' (i.e. uint)"; + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << "'kernel_enqueue_flags_t' (i.e. uint)"; return true; } // Third argument is always an ndrange_t type. if (Arg2->getType().getUnqualifiedType().getAsString() != "ndrange_t") { S.Diag(TheCall->getArg(2)->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) - << "'ndrange_t'"; + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << "'ndrange_t'"; return true; } @@ -420,8 +421,8 @@ static bool SemaOpenCLBuiltinEnqueueKernel(Sema &S, CallExpr *TheCall) { if (NumArgs == 4) { // check that the last argument is the right block type. if (!isBlockPointer(Arg3)) { - S.Diag(Arg3->getLocStart(), diag::err_opencl_enqueue_kernel_expected_type) - << "block"; + S.Diag(Arg3->getLocStart(), diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << "block"; return true; } // we have a block type, check the prototype @@ -443,8 +444,8 @@ static bool SemaOpenCLBuiltinEnqueueKernel(Sema &S, CallExpr *TheCall) { // check common block argument. Expr *Arg6 = TheCall->getArg(6); if (!isBlockPointer(Arg6)) { - S.Diag(Arg6->getLocStart(), diag::err_opencl_enqueue_kernel_expected_type) - << "block"; + S.Diag(Arg6->getLocStart(), diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << "block"; return true; } if (checkOpenCLBlockArgs(S, Arg6)) @@ -453,8 +454,8 @@ static bool SemaOpenCLBuiltinEnqueueKernel(Sema &S, CallExpr *TheCall) { // Forth argument has to be any integer type. if (!Arg3->getType()->isIntegerType()) { S.Diag(TheCall->getArg(3)->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) - << "integer"; + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << "integer"; return true; } // check remaining common arguments. @@ -466,7 +467,8 @@ static bool SemaOpenCLBuiltinEnqueueKernel(Sema &S, CallExpr *TheCall) { Expr::NPC_ValueDependentIsNotNull) && !Arg4->getType()->getPointeeOrArrayElementType()->isClkEventT()) { S.Diag(TheCall->getArg(4)->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << S.Context.getPointerType(S.Context.OCLClkEventTy); return true; } @@ -477,7 +479,8 @@ static bool SemaOpenCLBuiltinEnqueueKernel(Sema &S, CallExpr *TheCall) { !(Arg5->getType()->isPointerType() && Arg5->getType()->getPointeeType()->isClkEventT())) { S.Diag(TheCall->getArg(5)->getLocStart(), - diag::err_opencl_enqueue_kernel_expected_type) + diag::err_opencl_builtin_expected_type) + << TheCall->getDirectCallee() << S.Context.getPointerType(S.Context.OCLClkEventTy); return true; } @@ -12094,6 +12097,8 @@ void Sema::RefersToMemberWithReducedAlignment( if (ME->isArrow()) BaseType = BaseType->getPointeeType(); RecordDecl *RD = BaseType->getAs()->getDecl(); + if (RD->isInvalidDecl()) + return; ValueDecl *MD = ME->getMemberDecl(); auto *FD = dyn_cast(MD); diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaCoroutine.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaCoroutine.cpp index b05c0998d3dd..dc7d8e4e9cec 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaCoroutine.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaCoroutine.cpp @@ -43,9 +43,10 @@ static bool lookupMember(Sema &S, const char *Name, CXXRecordDecl *RD, /// Look up the std::coroutine_traits<...>::promise_type for the given /// function type. -static QualType lookupPromiseType(Sema &S, const FunctionProtoType *FnType, - SourceLocation KwLoc, - SourceLocation FuncLoc) { +static QualType lookupPromiseType(Sema &S, const FunctionDecl *FD, + SourceLocation KwLoc) { + const FunctionProtoType *FnType = FD->getType()->castAs(); + const SourceLocation FuncLoc = FD->getLocation(); // FIXME: Cache std::coroutine_traits once we've found it. NamespaceDecl *StdExp = S.lookupStdExperimentalNamespace(); if (!StdExp) { @@ -71,16 +72,35 @@ static QualType lookupPromiseType(Sema &S, const FunctionProtoType *FnType, return QualType(); } - // Form template argument list for coroutine_traits. + // Form template argument list for coroutine_traits according + // to [dcl.fct.def.coroutine]3 TemplateArgumentListInfo Args(KwLoc, KwLoc); - Args.addArgument(TemplateArgumentLoc( - TemplateArgument(FnType->getReturnType()), - S.Context.getTrivialTypeSourceInfo(FnType->getReturnType(), KwLoc))); - // FIXME: If the function is a non-static member function, add the type - // of the implicit object parameter before the formal parameters. - for (QualType T : FnType->getParamTypes()) + auto AddArg = [&](QualType T) { Args.addArgument(TemplateArgumentLoc( TemplateArgument(T), S.Context.getTrivialTypeSourceInfo(T, KwLoc))); + }; + AddArg(FnType->getReturnType()); + // If the function is a non-static member function, add the type + // of the implicit object parameter before the formal parameters. + if (auto *MD = dyn_cast(FD)) { + if (MD->isInstance()) { + // [over.match.funcs]4 + // For non-static member functions, the type of the implicit object + // parameter is + // -- "lvalue reference to cv X" for functions declared without a + // ref-qualifier or with the & ref-qualifier + // -- "rvalue reference to cv X" for functions declared with the && + // ref-qualifier + QualType T = + MD->getThisType(S.Context)->getAs()->getPointeeType(); + T = FnType->getRefQualifier() == RQ_RValue + ? S.Context.getRValueReferenceType(T) + : S.Context.getLValueReferenceType(T, /*SpelledAsLValue*/ true); + AddArg(T); + } + } + for (QualType T : FnType->getParamTypes()) + AddArg(T); // Build the template-id. QualType CoroTrait = @@ -424,12 +444,16 @@ static ExprResult buildPromiseCall(Sema &S, VarDecl *Promise, VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) { assert(isa(CurContext) && "not in a function scope"); auto *FD = cast(CurContext); + bool IsThisDependentType = [&] { + if (auto *MD = dyn_cast_or_null(FD)) + return MD->isInstance() && MD->getThisType(Context)->isDependentType(); + else + return false; + }(); - QualType T = - FD->getType()->isDependentType() - ? Context.DependentTy - : lookupPromiseType(*this, FD->getType()->castAs(), - Loc, FD->getLocation()); + QualType T = FD->getType()->isDependentType() || IsThisDependentType + ? Context.DependentTy + : lookupPromiseType(*this, FD, Loc); if (T.isNull()) return nullptr; @@ -721,8 +745,6 @@ static Expr *buildStdNoThrowDeclRef(Sema &S, SourceLocation Loc) { return nullptr; } - // FIXME: Mark the variable as ODR used. This currently does not work - // likely due to the scope at in which this function is called. auto *VD = Result.getAsSingle(); if (!VD) { Result.suppressDiagnostics(); diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp index ef8a408f90de..31b24f91c1d9 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp @@ -1998,8 +1998,7 @@ static void filterNonConflictingPreviousTypedefDecls(Sema &S, // If both declarations give a tag declaration a typedef name for linkage // purposes, then they declare the same entity. - if (S.getLangOpts().CPlusPlus && - OldTD->getAnonDeclWithTypedefName(/*AnyRedecl*/true) && + if (OldTD->getAnonDeclWithTypedefName(/*AnyRedecl*/true) && Decl->getAnonDeclWithTypedefName()) continue; } @@ -2117,7 +2116,7 @@ void Sema::MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New, auto *OldTag = OldTD->getAnonDeclWithTypedefName(/*AnyRedecl*/true); auto *NewTag = New->getAnonDeclWithTypedefName(); NamedDecl *Hidden = nullptr; - if (getLangOpts().CPlusPlus && OldTag && NewTag && + if (OldTag && NewTag && OldTag->getCanonicalDecl() != NewTag->getCanonicalDecl() && !hasVisibleDefinition(OldTag, &Hidden)) { // There is a definition of this tag, but it is not visible. Use it @@ -16055,8 +16054,6 @@ Sema::DeclGroupPtrTy Sema::ActOnModuleDecl(SourceLocation StartLoc, return nullptr; } - // FIXME: Create a ModuleDecl and return it. - // FIXME: Most of this work should be done by the preprocessor rather than // here, in order to support macro import. @@ -16070,6 +16067,8 @@ Sema::DeclGroupPtrTy Sema::ActOnModuleDecl(SourceLocation StartLoc, ModuleName += Piece.first->getName(); } + // FIXME: If we've already seen a module-declaration, report an error. + // If a module name was explicitly specified on the command line, it must be // correct. if (!getLangOpts().CurrentModule.empty() && @@ -16082,6 +16081,7 @@ Sema::DeclGroupPtrTy Sema::ActOnModuleDecl(SourceLocation StartLoc, const_cast(getLangOpts()).CurrentModule = ModuleName; auto &Map = PP.getHeaderSearchInfo().getModuleMap(); + Module *Mod; switch (MDK) { case ModuleDeclKind::Module: { @@ -16100,12 +16100,9 @@ Sema::DeclGroupPtrTy Sema::ActOnModuleDecl(SourceLocation StartLoc, } // Create a Module for the module that we're defining. - Module *Mod = Map.createModuleForInterfaceUnit(ModuleLoc, ModuleName); + Mod = Map.createModuleForInterfaceUnit(ModuleLoc, ModuleName); assert(Mod && "module creation should not fail"); - - // Enter the semantic scope of the module. - ActOnModuleBegin(ModuleLoc, Mod); - return nullptr; + break; } case ModuleDeclKind::Partition: @@ -16115,14 +16112,26 @@ Sema::DeclGroupPtrTy Sema::ActOnModuleDecl(SourceLocation StartLoc, case ModuleDeclKind::Implementation: std::pair ModuleNameLoc( PP.getIdentifierInfo(ModuleName), Path[0].second); - - DeclResult Import = ActOnModuleImport(ModuleLoc, ModuleLoc, ModuleNameLoc); - if (Import.isInvalid()) + Mod = getModuleLoader().loadModule(ModuleLoc, Path, Module::AllVisible, + /*IsIncludeDirective=*/false); + if (!Mod) return nullptr; - return ConvertDeclToDeclGroup(Import.get()); + break; } - llvm_unreachable("unexpected module decl kind"); + // Enter the semantic scope of the module. + ModuleScopes.push_back({}); + ModuleScopes.back().Module = Mod; + ModuleScopes.back().OuterVisibleModules = std::move(VisibleModules); + VisibleModules.setVisible(Mod, ModuleLoc); + + // From now on, we have an owning module for all declarations we see. + // However, those declarations are module-private unless explicitly + // exported. + Context.getTranslationUnitDecl()->setLocalOwningModule(Mod); + + // FIXME: Create a ModuleDecl. + return nullptr; } DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc, @@ -16311,6 +16320,7 @@ Decl *Sema::ActOnStartExportDecl(Scope *S, SourceLocation ExportLoc, CurContext->addDecl(D); PushDeclContext(S, D); + D->setModuleOwnershipKind(Decl::ModuleOwnershipKind::VisibleWhenImported); return D; } diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp index 1929bc539188..5fb79a6bf630 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaDeclAttr.cpp @@ -6929,8 +6929,34 @@ shouldDiagnoseAvailabilityByDefault(const ASTContext &Context, DeclVersion >= ForceAvailabilityFromVersion; } +static NamedDecl *findEnclosingDeclToAnnotate(Decl *OrigCtx) { + for (Decl *Ctx = OrigCtx; Ctx; + Ctx = cast_or_null(Ctx->getDeclContext())) { + if (isa(Ctx) || isa(Ctx) || isa(Ctx)) + return cast(Ctx); + if (auto *CD = dyn_cast(Ctx)) { + if (auto *Imp = dyn_cast(Ctx)) + return Imp->getClassInterface(); + return CD; + } + } + + return dyn_cast(OrigCtx); +} + +/// Actually emit an availability diagnostic for a reference to an unavailable +/// decl. +/// +/// \param Ctx The context that the reference occurred in +/// \param ReferringDecl The exact declaration that was referenced. +/// \param OffendingDecl A related decl to \c ReferringDecl that has an +/// availability attribute corrisponding to \c K attached to it. Note that this +/// may not be the same as ReferringDecl, i.e. if an EnumDecl is annotated and +/// we refer to a member EnumConstantDecl, ReferringDecl is the EnumConstantDecl +/// and OffendingDecl is the EnumDecl. static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, - Decl *Ctx, const NamedDecl *D, + Decl *Ctx, const NamedDecl *ReferringDecl, + const NamedDecl *OffendingDecl, StringRef Message, SourceLocation Loc, const ObjCInterfaceDecl *UnknownObjCClass, const ObjCPropertyDecl *ObjCProperty, @@ -6938,7 +6964,7 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, // Diagnostics for deprecated or unavailable. unsigned diag, diag_message, diag_fwdclass_message; unsigned diag_available_here = diag::note_availability_specified_here; - SourceLocation NoteLocation = D->getLocation(); + SourceLocation NoteLocation = OffendingDecl->getLocation(); // Matches 'diag::note_property_attribute' options. unsigned property_note_select; @@ -6947,7 +6973,7 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, unsigned available_here_select_kind; VersionTuple DeclVersion; - if (const AvailabilityAttr *AA = getAttrForPlatform(S.Context, D)) + if (const AvailabilityAttr *AA = getAttrForPlatform(S.Context, OffendingDecl)) DeclVersion = AA->getIntroduced(); if (!ShouldDiagnoseAvailabilityInContext(S, K, DeclVersion, Ctx)) @@ -6961,7 +6987,7 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, diag_fwdclass_message = diag::warn_deprecated_fwdclass_message; property_note_select = /* deprecated */ 0; available_here_select_kind = /* deprecated */ 2; - if (const auto *attr = D->getAttr()) + if (const auto *attr = OffendingDecl->getAttr()) NoteLocation = attr->getLocation(); break; @@ -6973,13 +6999,14 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, property_note_select = /* unavailable */ 1; available_here_select_kind = /* unavailable */ 0; - if (auto attr = D->getAttr()) { + if (auto attr = OffendingDecl->getAttr()) { if (attr->isImplicit() && attr->getImplicitReason()) { // Most of these failures are due to extra restrictions in ARC; // reflect that in the primary diagnostic when applicable. auto flagARCError = [&] { if (S.getLangOpts().ObjCAutoRefCount && - S.getSourceManager().isInSystemHeader(D->getLocation())) + S.getSourceManager().isInSystemHeader( + OffendingDecl->getLocation())) diag = diag::err_unavailable_in_arc; }; @@ -7022,7 +7049,8 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, // not specified for deployment targets >= to iOS 11 or equivalent or // for declarations that were introduced in iOS 11 (macOS 10.13, ...) or // later. - const AvailabilityAttr *AA = getAttrForPlatform(S.getASTContext(), D); + const AvailabilityAttr *AA = + getAttrForPlatform(S.getASTContext(), OffendingDecl); VersionTuple Introduced = AA->getIntroduced(); bool NewWarning = shouldDiagnoseAvailabilityByDefault( S.Context, S.Context.getTargetInfo().getPlatformMinVersion(), @@ -7045,9 +7073,9 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, CharSourceRange UseRange; StringRef Replacement; if (K == AR_Deprecated) { - if (auto attr = D->getAttr()) + if (auto attr = OffendingDecl->getAttr()) Replacement = attr->getReplacement(); - if (auto attr = getAttrForPlatform(S.Context, D)) + if (auto attr = getAttrForPlatform(S.Context, OffendingDecl)) Replacement = attr->getReplacement(); if (!Replacement.empty()) @@ -7056,21 +7084,21 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, } if (!Message.empty()) { - S.Diag(Loc, diag_message) << D << Message + S.Diag(Loc, diag_message) << ReferringDecl << Message << (UseRange.isValid() ? FixItHint::CreateReplacement(UseRange, Replacement) : FixItHint()); if (ObjCProperty) S.Diag(ObjCProperty->getLocation(), diag::note_property_attribute) << ObjCProperty->getDeclName() << property_note_select; } else if (!UnknownObjCClass) { - S.Diag(Loc, diag) << D + S.Diag(Loc, diag) << ReferringDecl << (UseRange.isValid() ? FixItHint::CreateReplacement(UseRange, Replacement) : FixItHint()); if (ObjCProperty) S.Diag(ObjCProperty->getLocation(), diag::note_property_attribute) << ObjCProperty->getDeclName() << property_note_select; } else { - S.Diag(Loc, diag_fwdclass_message) << D + S.Diag(Loc, diag_fwdclass_message) << ReferringDecl << (UseRange.isValid() ? FixItHint::CreateReplacement(UseRange, Replacement) : FixItHint()); S.Diag(UnknownObjCClass->getLocation(), diag::note_forward_class); @@ -7078,16 +7106,16 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, // The declaration can have multiple availability attributes, we are looking // at one of them. - const AvailabilityAttr *A = getAttrForPlatform(S.Context, D); + const AvailabilityAttr *A = getAttrForPlatform(S.Context, OffendingDecl); if (A && A->isInherited()) { - for (const Decl *Redecl = D->getMostRecentDecl(); Redecl; + for (const Decl *Redecl = OffendingDecl->getMostRecentDecl(); Redecl; Redecl = Redecl->getPreviousDecl()) { const AvailabilityAttr *AForRedecl = getAttrForPlatform(S.Context, Redecl); if (AForRedecl && !AForRedecl->isInherited()) { // If D is a declaration with inherited attributes, the note should // point to the declaration with actual attributes. - S.Diag(Redecl->getLocation(), diag_available_here) << D + S.Diag(Redecl->getLocation(), diag_available_here) << OffendingDecl << available_here_select_kind; break; } @@ -7095,10 +7123,19 @@ static void DoEmitAvailabilityWarning(Sema &S, AvailabilityResult K, } else S.Diag(NoteLocation, diag_available_here) - << D << available_here_select_kind; + << OffendingDecl << available_here_select_kind; if (K == AR_NotYetIntroduced) - S.Diag(Loc, diag::note_partial_availability_silence) << D; + if (const auto *Enclosing = findEnclosingDeclToAnnotate(Ctx)) { + if (auto *TD = dyn_cast(Enclosing)) + if (TD->getDeclName().isEmpty()) { + S.Diag(TD->getLocation(), diag::note_partial_availability_silence) + << /*Anonymous*/1 << TD->getKindName(); + return; + } + S.Diag(Enclosing->getLocation(), diag::note_partial_availability_silence) + << /*Named*/0 << Enclosing; + } } static void handleDelayedAvailabilityCheck(Sema &S, DelayedDiagnostic &DD, @@ -7108,9 +7145,9 @@ static void handleDelayedAvailabilityCheck(Sema &S, DelayedDiagnostic &DD, DD.Triggered = true; DoEmitAvailabilityWarning( - S, DD.getAvailabilityResult(), Ctx, DD.getAvailabilityDecl(), - DD.getAvailabilityMessage(), DD.Loc, DD.getUnknownObjCClass(), - DD.getObjCProperty(), false); + S, DD.getAvailabilityResult(), Ctx, DD.getAvailabilityReferringDecl(), + DD.getAvailabilityOffendingDecl(), DD.getAvailabilityMessage(), DD.Loc, + DD.getUnknownObjCClass(), DD.getObjCProperty(), false); } void Sema::PopParsingDeclaration(ParsingDeclState state, Decl *decl) { @@ -7169,22 +7206,25 @@ void Sema::redelayDiagnostics(DelayedDiagnosticPool &pool) { } void Sema::EmitAvailabilityWarning(AvailabilityResult AR, - NamedDecl *D, StringRef Message, - SourceLocation Loc, + const NamedDecl *ReferringDecl, + const NamedDecl *OffendingDecl, + StringRef Message, SourceLocation Loc, const ObjCInterfaceDecl *UnknownObjCClass, - const ObjCPropertyDecl *ObjCProperty, + const ObjCPropertyDecl *ObjCProperty, bool ObjCPropertyAccess) { // Delay if we're currently parsing a declaration. if (DelayedDiagnostics.shouldDelayDiagnostics()) { - DelayedDiagnostics.add(DelayedDiagnostic::makeAvailability( - AR, Loc, D, UnknownObjCClass, ObjCProperty, Message, - ObjCPropertyAccess)); + DelayedDiagnostics.add( + DelayedDiagnostic::makeAvailability( + AR, Loc, ReferringDecl, OffendingDecl, UnknownObjCClass, + ObjCProperty, Message, ObjCPropertyAccess)); return; } Decl *Ctx = cast(getCurLexicalContext()); - DoEmitAvailabilityWarning(*this, AR, Ctx, D, Message, Loc, UnknownObjCClass, - ObjCProperty, ObjCPropertyAccess); + DoEmitAvailabilityWarning(*this, AR, Ctx, ReferringDecl, OffendingDecl, + Message, Loc, UnknownObjCClass, ObjCProperty, + ObjCPropertyAccess); } namespace { @@ -7336,19 +7376,21 @@ class DiagnoseUnguardedAvailability void DiagnoseUnguardedAvailability::DiagnoseDeclAvailability( NamedDecl *D, SourceRange Range) { - - VersionTuple ContextVersion = AvailabilityStack.back(); - if (AvailabilityResult Result = - SemaRef.ShouldDiagnoseAvailabilityOfDecl(D, nullptr)) { + AvailabilityResult Result; + const NamedDecl *OffendingDecl; + std::tie(Result, OffendingDecl) = + SemaRef.ShouldDiagnoseAvailabilityOfDecl(D, nullptr); + if (Result != AR_Available) { // All other diagnostic kinds have already been handled in // DiagnoseAvailabilityOfDecl. if (Result != AR_NotYetIntroduced) return; - const AvailabilityAttr *AA = getAttrForPlatform(SemaRef.getASTContext(), D); + const AvailabilityAttr *AA = + getAttrForPlatform(SemaRef.getASTContext(), OffendingDecl); VersionTuple Introduced = AA->getIntroduced(); - if (ContextVersion >= Introduced) + if (AvailabilityStack.back() >= Introduced) return; // If the context of this function is less available than D, we should not @@ -7373,8 +7415,9 @@ void DiagnoseUnguardedAvailability::DiagnoseDeclAvailability( SemaRef.getASTContext().getTargetInfo().getPlatformName()) << Introduced.getAsString(); - SemaRef.Diag(D->getLocation(), diag::note_availability_specified_here) - << D << /* partial */ 3; + SemaRef.Diag(OffendingDecl->getLocation(), + diag::note_availability_specified_here) + << OffendingDecl << /* partial */ 3; auto FixitDiag = SemaRef.Diag(Range.getBegin(), diag::note_unguarded_available_silence) diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp index 2c8080dbf02b..778b8062f68c 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaDeclObjC.cpp @@ -458,7 +458,10 @@ static void diagnoseUseOfProtocols(Sema &TheSema, // Diagnose availability in the context of the ObjC container. Sema::ContextRAII SavedContext(TheSema, CD); for (unsigned i = 0; i < NumProtoRefs; ++i) { - (void)TheSema.DiagnoseUseOfDecl(ProtoRefs[i], ProtoLocs[i]); + (void)TheSema.DiagnoseUseOfDecl(ProtoRefs[i], ProtoLocs[i], + /*UnknownObjCClass=*/nullptr, + /*ObjCPropertyAccess=*/false, + /*AvoidPartialAvailabilityChecks=*/true); } } diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp index f49df6b3216d..8016bf99889f 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaExpr.cpp @@ -87,24 +87,9 @@ static void DiagnoseUnusedOfDecl(Sema &S, NamedDecl *D, SourceLocation Loc) { } } -static bool HasRedeclarationWithoutAvailabilityInCategory(const Decl *D) { - const auto *OMD = dyn_cast(D); - if (!OMD) - return false; - const ObjCInterfaceDecl *OID = OMD->getClassInterface(); - if (!OID) - return false; - - for (const ObjCCategoryDecl *Cat : OID->visible_categories()) - if (ObjCMethodDecl *CatMeth = - Cat->getMethod(OMD->getSelector(), OMD->isInstanceMethod())) - if (!CatMeth->hasAttr()) - return true; - return false; -} - -AvailabilityResult -Sema::ShouldDiagnoseAvailabilityOfDecl(NamedDecl *&D, std::string *Message) { +std::pair +Sema::ShouldDiagnoseAvailabilityOfDecl(const NamedDecl *D, + std::string *Message) { AvailabilityResult Result = D->getAvailability(Message); // For typedefs, if the typedef declaration appears available look @@ -121,78 +106,61 @@ Sema::ShouldDiagnoseAvailabilityOfDecl(NamedDecl *&D, std::string *Message) { } // Forward class declarations get their attributes from their definition. - if (ObjCInterfaceDecl *IDecl = dyn_cast(D)) { + if (const ObjCInterfaceDecl *IDecl = dyn_cast(D)) { if (IDecl->getDefinition()) { D = IDecl->getDefinition(); Result = D->getAvailability(Message); } } - if (const EnumConstantDecl *ECD = dyn_cast(D)) + if (const auto *ECD = dyn_cast(D)) if (Result == AR_Available) { const DeclContext *DC = ECD->getDeclContext(); - if (const EnumDecl *TheEnumDecl = dyn_cast(DC)) + if (const auto *TheEnumDecl = dyn_cast(DC)) { Result = TheEnumDecl->getAvailability(Message); + D = TheEnumDecl; + } } - if (Result == AR_NotYetIntroduced) { - // Don't do this for enums, they can't be redeclared. - if (isa(D) || isa(D)) - return AR_Available; - - bool Warn = !D->getAttr()->isInherited(); - // Objective-C method declarations in categories are not modelled as - // redeclarations, so manually look for a redeclaration in a category - // if necessary. - if (Warn && HasRedeclarationWithoutAvailabilityInCategory(D)) - Warn = false; - // In general, D will point to the most recent redeclaration. However, - // for `@class A;` decls, this isn't true -- manually go through the - // redecl chain in that case. - if (Warn && isa(D)) - for (Decl *Redecl = D->getMostRecentDecl(); Redecl && Warn; - Redecl = Redecl->getPreviousDecl()) - if (!Redecl->hasAttr() || - Redecl->getAttr()->isInherited()) - Warn = false; - - return Warn ? AR_NotYetIntroduced : AR_Available; - } - - return Result; + return {Result, D}; } static void DiagnoseAvailabilityOfDecl(Sema &S, NamedDecl *D, SourceLocation Loc, const ObjCInterfaceDecl *UnknownObjCClass, - bool ObjCPropertyAccess) { + bool ObjCPropertyAccess, + bool AvoidPartialAvailabilityChecks = false) { std::string Message; + AvailabilityResult Result; + const NamedDecl* OffendingDecl; // See if this declaration is unavailable, deprecated, or partial. - if (AvailabilityResult Result = - S.ShouldDiagnoseAvailabilityOfDecl(D, &Message)) { + std::tie(Result, OffendingDecl) = S.ShouldDiagnoseAvailabilityOfDecl(D, &Message); + if (Result == AR_Available) + return; - if (Result == AR_NotYetIntroduced) { - if (S.getCurFunctionOrMethodDecl()) { - S.getEnclosingFunction()->HasPotentialAvailabilityViolations = true; - return; - } else if (S.getCurBlock() || S.getCurLambda()) { - S.getCurFunction()->HasPotentialAvailabilityViolations = true; - return; - } + if (Result == AR_NotYetIntroduced) { + if (AvoidPartialAvailabilityChecks) + return; + if (S.getCurFunctionOrMethodDecl()) { + S.getEnclosingFunction()->HasPotentialAvailabilityViolations = true; + return; + } else if (S.getCurBlock() || S.getCurLambda()) { + S.getCurFunction()->HasPotentialAvailabilityViolations = true; + return; } - - const ObjCPropertyDecl *ObjCPDecl = nullptr; - if (const ObjCMethodDecl *MD = dyn_cast(D)) { - if (const ObjCPropertyDecl *PD = MD->findPropertyDecl()) { - AvailabilityResult PDeclResult = PD->getAvailability(nullptr); - if (PDeclResult == Result) - ObjCPDecl = PD; - } - } - - S.EmitAvailabilityWarning(Result, D, Message, Loc, UnknownObjCClass, - ObjCPDecl, ObjCPropertyAccess); } + + const ObjCPropertyDecl *ObjCPDecl = nullptr; + if (const ObjCMethodDecl *MD = dyn_cast(D)) { + if (const ObjCPropertyDecl *PD = MD->findPropertyDecl()) { + AvailabilityResult PDeclResult = PD->getAvailability(nullptr); + if (PDeclResult == Result) + ObjCPDecl = PD; + } + } + + S.EmitAvailabilityWarning(Result, D, OffendingDecl, Message, Loc, + UnknownObjCClass, ObjCPDecl, ObjCPropertyAccess); } /// \brief Emit a note explaining that this function is deleted. @@ -310,7 +278,8 @@ void Sema::MaybeSuggestAddingStaticToDecl(const FunctionDecl *Cur) { /// bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc, const ObjCInterfaceDecl *UnknownObjCClass, - bool ObjCPropertyAccess) { + bool ObjCPropertyAccess, + bool AvoidPartialAvailabilityChecks) { if (getLangOpts().CPlusPlus && isa(D)) { // If there were any diagnostics suppressed by template argument deduction, // emit them now. @@ -395,7 +364,8 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc, } DiagnoseAvailabilityOfDecl(*this, D, Loc, UnknownObjCClass, - ObjCPropertyAccess); + ObjCPropertyAccess, + AvoidPartialAvailabilityChecks); DiagnoseUnusedOfDecl(*this, D, Loc); @@ -14695,24 +14665,24 @@ static void MarkExprReferenced(Sema &SemaRef, SourceLocation Loc, ME->performsVirtualDispatch(SemaRef.getLangOpts()); if (!IsVirtualCall) return; - const Expr *Base = ME->getBase(); - const CXXRecordDecl *MostDerivedClassDecl = Base->getBestDynamicClassType(); - if (!MostDerivedClassDecl) - return; - CXXMethodDecl *DM = MD->getCorrespondingMethodInClass(MostDerivedClassDecl); - if (!DM || DM->isPure()) - return; - SemaRef.MarkAnyDeclReferenced(Loc, DM, MightBeOdrUse); + + // If it's possible to devirtualize the call, mark the called function + // referenced. + CXXMethodDecl *DM = MD->getDevirtualizedMethod( + ME->getBase(), SemaRef.getLangOpts().AppleKext); + if (DM) + SemaRef.MarkAnyDeclReferenced(Loc, DM, MightBeOdrUse); } /// \brief Perform reference-marking and odr-use handling for a DeclRefExpr. -void Sema::MarkDeclRefReferenced(DeclRefExpr *E) { +void Sema::MarkDeclRefReferenced(DeclRefExpr *E, const Expr *Base) { // TODO: update this with DR# once a defect report is filed. // C++11 defect. The address of a pure member should not be an ODR use, even // if it's a qualified reference. bool OdrUse = true; - if (CXXMethodDecl *Method = dyn_cast(E->getDecl())) - if (Method->isVirtual()) + if (const CXXMethodDecl *Method = dyn_cast(E->getDecl())) + if (Method->isVirtual() && + !Method->getDevirtualizedMethod(Base, getLangOpts().AppleKext)) OdrUse = false; MarkExprReferenced(*this, E->getLocation(), E->getDecl(), E, OdrUse); } diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp index b18de7e94686..c3d0e2db76b6 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaExprMember.cpp @@ -1842,10 +1842,6 @@ Sema::BuildFieldReferenceExpr(Expr *BaseExpr, bool IsArrow, FoundDecl, Field); if (Base.isInvalid()) return ExprError(); - MemberExpr *ME = - BuildMemberExpr(*this, Context, Base.get(), IsArrow, OpLoc, SS, - /*TemplateKWLoc=*/SourceLocation(), Field, FoundDecl, - MemberNameInfo, MemberType, VK, OK); // Build a reference to a private copy for non-static data members in // non-static member functions, privatized by OpenMP constructs. @@ -1855,7 +1851,10 @@ Sema::BuildFieldReferenceExpr(Expr *BaseExpr, bool IsArrow, if (auto *PrivateCopy = IsOpenMPCapturedDecl(Field)) return getOpenMPCapturedExpr(PrivateCopy, VK, OK, OpLoc); } - return ME; + + return BuildMemberExpr(*this, Context, Base.get(), IsArrow, OpLoc, SS, + /*TemplateKWLoc=*/SourceLocation(), Field, FoundDecl, + MemberNameInfo, MemberType, VK, OK); } /// Builds an implicit member access expression. The current context diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaLambda.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaLambda.cpp index d6b70610d461..46f2ba376006 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaLambda.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaLambda.cpp @@ -1595,7 +1595,7 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc, ContainsUnexpandedParameterPack); // If the lambda expression's call operator is not explicitly marked constexpr // and we are not in a dependent context, analyze the call operator to infer - // its constexpr-ness, supressing diagnostics while doing so. + // its constexpr-ness, suppressing diagnostics while doing so. if (getLangOpts().CPlusPlus1z && !CallOperator->isInvalidDecl() && !CallOperator->isConstexpr() && !isa(CallOperator->getBody()) && diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp index 2e7fb875a276..85596ed52e9d 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaLookup.cpp @@ -1395,6 +1395,20 @@ bool Sema::hasVisibleMergedDefinition(NamedDecl *Def) { return false; } +bool Sema::hasMergedDefinitionInCurrentModule(NamedDecl *Def) { + // FIXME: When not in local visibility mode, we can't tell the difference + // between a declaration being visible because we merged a local copy of + // the same declaration into it, and it being visible because its owning + // module is visible. + if (Def->getModuleOwnershipKind() == Decl::ModuleOwnershipKind::Visible && + getLangOpts().ModulesLocalVisibility) + return true; + for (Module *Merged : Context.getModulesWithMergedDefinition(Def)) + if (Merged->getTopLevelModuleName() == getLangOpts().CurrentModule) + return true; + return false; +} + template static bool hasVisibleDefaultArgument(Sema &S, const ParmDecl *D, @@ -1495,23 +1509,40 @@ bool LookupResult::isVisibleSlow(Sema &SemaRef, NamedDecl *D) { assert(D->isHidden() && "should not call this: not in slow case"); Module *DeclModule = SemaRef.getOwningModule(D); - assert(DeclModule && "hidden decl not from a module"); + if (!DeclModule) { + // A module-private declaration with no owning module means this is in the + // global module in the C++ Modules TS. This is visible within the same + // translation unit only. + // FIXME: Don't assume that "same translation unit" means the same thing + // as "not from an AST file". + assert(D->isModulePrivate() && "hidden decl has no module"); + if (!D->isFromASTFile() || SemaRef.hasMergedDefinitionInCurrentModule(D)) + return true; + } else { + // If the owning module is visible, and the decl is not module private, + // then the decl is visible too. (Module private is ignored within the same + // top-level module.) + if (D->isModulePrivate() + ? DeclModule->getTopLevelModuleName() == + SemaRef.getLangOpts().CurrentModule || + SemaRef.hasMergedDefinitionInCurrentModule(D) + : SemaRef.isModuleVisible(DeclModule) || + SemaRef.hasVisibleMergedDefinition(D)) + return true; + } - // If the owning module is visible, and the decl is not module private, - // then the decl is visible too. (Module private is ignored within the same - // top-level module.) - // FIXME: Check the owning module for module-private declarations rather than - // assuming "same AST file" is the same thing as "same module". - if ((!D->isFromASTFile() || !D->isModulePrivate()) && - (SemaRef.isModuleVisible(DeclModule) || - SemaRef.hasVisibleMergedDefinition(D))) - return true; + // Determine whether a decl context is a file context for the purpose of + // visibility. This looks through some (export and linkage spec) transparent + // contexts, but not others (enums). + auto IsEffectivelyFileContext = [](const DeclContext *DC) { + return DC->isFileContext() || isa(DC) || + isa(DC); + }; - // If this declaration is not at namespace scope nor module-private, + // If this declaration is not at namespace scope // then it is visible if its lexical parent has a visible definition. DeclContext *DC = D->getLexicalDeclContext(); - if (!D->isModulePrivate() && DC && !DC->isFileContext() && - !isa(DC) && !isa(DC)) { + if (DC && !IsEffectivelyFileContext(DC)) { // For a parameter, check whether our current template declaration's // lexical context is visible, not whether there's some other visible // definition of it, because parameters aren't "within" the definition. @@ -1519,32 +1550,45 @@ bool LookupResult::isVisibleSlow(Sema &SemaRef, NamedDecl *D) { // In C++ we need to check for a visible definition due to ODR merging, // and in C we must not because each declaration of a function gets its own // set of declarations for tags in prototype scope. - if ((D->isTemplateParameter() || isa(D) - || (isa(DC) && !SemaRef.getLangOpts().CPlusPlus)) - ? isVisible(SemaRef, cast(DC)) - : SemaRef.hasVisibleDefinition(cast(DC))) { - if (SemaRef.CodeSynthesisContexts.empty() && - // FIXME: Do something better in this case. - !SemaRef.getLangOpts().ModulesLocalVisibility) { - // Cache the fact that this declaration is implicitly visible because - // its parent has a visible definition. - D->setVisibleDespiteOwningModule(); - } - return true; + bool VisibleWithinParent; + if (D->isTemplateParameter() || isa(D) || + (isa(DC) && !SemaRef.getLangOpts().CPlusPlus)) + VisibleWithinParent = isVisible(SemaRef, cast(DC)); + else if (D->isModulePrivate()) { + // A module-private declaration is only visible if an enclosing lexical + // parent was merged with another definition in the current module. + VisibleWithinParent = false; + do { + if (SemaRef.hasMergedDefinitionInCurrentModule(cast(DC))) { + VisibleWithinParent = true; + break; + } + DC = DC->getLexicalParent(); + } while (!IsEffectivelyFileContext(DC)); + } else { + VisibleWithinParent = SemaRef.hasVisibleDefinition(cast(DC)); } - return false; + + if (VisibleWithinParent && SemaRef.CodeSynthesisContexts.empty() && + // FIXME: Do something better in this case. + !SemaRef.getLangOpts().ModulesLocalVisibility) { + // Cache the fact that this declaration is implicitly visible because + // its parent has a visible definition. + D->setVisibleDespiteOwningModule(); + } + return VisibleWithinParent; } + // FIXME: All uses of DeclModule below this point should also check merged + // modules. + if (!DeclModule) + return false; + // Find the extra places where we need to look. llvm::DenseSet &LookupModules = SemaRef.getLookupModules(); if (LookupModules.empty()) return false; - if (!DeclModule) { - DeclModule = SemaRef.getOwningModule(D); - assert(DeclModule && "hidden decl not from a module"); - } - // If our lookup set contains the decl's module, it's visible. if (LookupModules.count(DeclModule)) return true; diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp index 6c5716454874..62a771bcffa0 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaObjCProperty.cpp @@ -1676,8 +1676,9 @@ static bool SuperClassImplementsProperty(ObjCInterfaceDecl *IDecl, /// \brief Default synthesizes all properties which must be synthesized /// in class's \@implementation. -void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl* IMPDecl, - ObjCInterfaceDecl *IDecl) { +void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl, + ObjCInterfaceDecl *IDecl, + SourceLocation AtEnd) { ObjCInterfaceDecl::PropertyMap PropMap; ObjCInterfaceDecl::PropertyDeclOrder PropertyOrder; IDecl->collectPropertiesToImplement(PropMap, PropertyOrder); @@ -1725,6 +1726,10 @@ void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl* IMPDecl, diag::warn_auto_synthesizing_protocol_property) << Prop << Proto; Diag(Prop->getLocation(), diag::note_property_declare); + std::string FixIt = + (Twine("@synthesize ") + Prop->getName() + ";\n\n").str(); + Diag(AtEnd, diag::note_add_synthesize_directive) + << FixItHint::CreateInsertion(AtEnd, FixIt); } continue; } @@ -1764,7 +1769,8 @@ void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl* IMPDecl, } } -void Sema::DefaultSynthesizeProperties(Scope *S, Decl *D) { +void Sema::DefaultSynthesizeProperties(Scope *S, Decl *D, + SourceLocation AtEnd) { if (!LangOpts.ObjCDefaultSynthProperties || LangOpts.ObjCRuntime.isFragile()) return; ObjCImplementationDecl *IC=dyn_cast_or_null(D); @@ -1772,7 +1778,7 @@ void Sema::DefaultSynthesizeProperties(Scope *S, Decl *D) { return; if (ObjCInterfaceDecl* IDecl = IC->getClassInterface()) if (!IDecl->isObjCRequiresPropertyDefs()) - DefaultSynthesizeProperties(S, IC, IDecl); + DefaultSynthesizeProperties(S, IC, IDecl, AtEnd); } static void DiagnoseUnimplementedAccessor( diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaOpenMP.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaOpenMP.cpp index 49da0e499771..1e0b6c158348 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaOpenMP.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaOpenMP.cpp @@ -6106,6 +6106,33 @@ static bool checkGrainsizeNumTasksClauses(Sema &S, return ErrorFound; } +static bool checkReductionClauseWithNogroup(Sema &S, + ArrayRef Clauses) { + OMPClause *ReductionClause = nullptr; + OMPClause *NogroupClause = nullptr; + for (auto *C : Clauses) { + if (C->getClauseKind() == OMPC_reduction) { + ReductionClause = C; + if (NogroupClause) + break; + continue; + } + if (C->getClauseKind() == OMPC_nogroup) { + NogroupClause = C; + if (ReductionClause) + break; + continue; + } + } + if (ReductionClause && NogroupClause) { + S.Diag(ReductionClause->getLocStart(), diag::err_omp_reduction_with_nogroup) + << SourceRange(NogroupClause->getLocStart(), + NogroupClause->getLocEnd()); + return true; + } + return false; +} + StmtResult Sema::ActOnOpenMPTaskLoopDirective( ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc, @@ -6132,6 +6159,11 @@ StmtResult Sema::ActOnOpenMPTaskLoopDirective( // not appear on the same taskloop directive. if (checkGrainsizeNumTasksClauses(*this, Clauses)) return StmtError(); + // OpenMP, [2.9.2 taskloop Construct, Restrictions] + // If a reduction clause is present on the taskloop directive, the nogroup + // clause must not be specified. + if (checkReductionClauseWithNogroup(*this, Clauses)) + return StmtError(); getCurFunction()->setHasBranchProtectedScope(); return OMPTaskLoopDirective::Create(Context, StartLoc, EndLoc, @@ -6175,6 +6207,11 @@ StmtResult Sema::ActOnOpenMPTaskLoopSimdDirective( // not appear on the same taskloop directive. if (checkGrainsizeNumTasksClauses(*this, Clauses)) return StmtError(); + // OpenMP, [2.9.2 taskloop Construct, Restrictions] + // If a reduction clause is present on the taskloop directive, the nogroup + // clause must not be specified. + if (checkReductionClauseWithNogroup(*this, Clauses)) + return StmtError(); getCurFunction()->setHasBranchProtectedScope(); return OMPTaskLoopSimdDirective::Create(Context, StartLoc, EndLoc, @@ -9399,6 +9436,12 @@ OMPClause *Sema::ActOnOpenMPReductionClause( SimpleRefExpr, RefRes.get()); if (!PostUpdateRes.isUsable()) continue; + if (isOpenMPTaskingDirective(DSAStack->getCurrentDirective())) { + Diag(RefExpr->getExprLoc(), + diag::err_omp_reduction_non_addressable_expression) + << RefExpr->getSourceRange(); + continue; + } ExprPostUpdates.push_back( IgnoredValueConversions(PostUpdateRes.get()).get()); } diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp index 5cc13f391d11..36f24fd9c463 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaOverload.cpp @@ -48,7 +48,7 @@ static bool functionHasPassObjectSizeParams(const FunctionDecl *FD) { /// A convenience routine for creating a decayed reference to a function. static ExprResult CreateFunctionRefExpr(Sema &S, FunctionDecl *Fn, NamedDecl *FoundDecl, - bool HadMultipleCandidates, + const Expr *Base, bool HadMultipleCandidates, SourceLocation Loc = SourceLocation(), const DeclarationNameLoc &LocInfo = DeclarationNameLoc()){ if (S.DiagnoseUseOfDecl(FoundDecl, Loc)) @@ -68,7 +68,7 @@ CreateFunctionRefExpr(Sema &S, FunctionDecl *Fn, NamedDecl *FoundDecl, if (HadMultipleCandidates) DRE->setHadMultipleCandidates(true); - S.MarkDeclRefReferenced(DRE); + S.MarkDeclRefReferenced(DRE, Base); return S.ImpCastExprToType(DRE, S.Context.getPointerType(DRE->getType()), CK_FunctionToPointerDecay); } @@ -9830,6 +9830,15 @@ static void DiagnoseBadDeduction(Sema &S, NamedDecl *Found, Decl *Templated, return; } + // We found a specific requirement that disabled the enable_if. + if (PDiag && PDiag->second.getDiagID() == + diag::err_typename_nested_not_found_requirement) { + S.Diag(Templated->getLocation(), + diag::note_ovl_candidate_disabled_by_requirement) + << PDiag->second.getStringArg(0) << TemplateArgString; + return; + } + // Format the SFINAE diagnostic into the argument string. // FIXME: Add a general mechanism to include a PartialDiagnostic *'s // formatted message in another diagnostic. @@ -11937,6 +11946,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc, FunctionDecl *FnDecl = Best->Function; if (FnDecl) { + Expr *Base = nullptr; // We matched an overloaded operator. Build a call to that // operator. @@ -11949,7 +11959,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc, Best->FoundDecl, Method); if (InputRes.isInvalid()) return ExprError(); - Input = InputRes.get(); + Base = Input = InputRes.get(); } else { // Convert the arguments. ExprResult InputInit @@ -11965,7 +11975,8 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc, // Build the actual expression node. ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl, Best->FoundDecl, - HadMultipleCandidates, OpLoc); + Base, HadMultipleCandidates, + OpLoc); if (FnExpr.isInvalid()) return ExprError(); @@ -12150,6 +12161,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc, FunctionDecl *FnDecl = Best->Function; if (FnDecl) { + Expr *Base = nullptr; // We matched an overloaded operator. Build a call to that // operator. @@ -12171,7 +12183,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc, Best->FoundDecl, Method); if (Arg0.isInvalid()) return ExprError(); - Args[0] = Arg0.getAs(); + Base = Args[0] = Arg0.getAs(); Args[1] = RHS = Arg1.getAs(); } else { // Convert the arguments. @@ -12195,7 +12207,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc, // Build the actual expression node. ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl, - Best->FoundDecl, + Best->FoundDecl, Base, HadMultipleCandidates, OpLoc); if (FnExpr.isInvalid()) return ExprError(); @@ -12417,6 +12429,7 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc, OpLocInfo.setCXXOperatorNameRange(SourceRange(LLoc, RLoc)); ExprResult FnExpr = CreateFunctionRefExpr(*this, FnDecl, Best->FoundDecl, + Base, HadMultipleCandidates, OpLocInfo.getLoc(), OpLocInfo.getInfo()); @@ -12975,7 +12988,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj, Context.DeclarationNames.getCXXOperatorName(OO_Call), LParenLoc); OpLocInfo.setCXXOperatorNameRange(SourceRange(LParenLoc, RParenLoc)); ExprResult NewFn = CreateFunctionRefExpr(*this, Method, Best->FoundDecl, - HadMultipleCandidates, + Obj, HadMultipleCandidates, OpLocInfo.getLoc(), OpLocInfo.getInfo()); if (NewFn.isInvalid()) @@ -13166,7 +13179,7 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc, // Build the operator call. ExprResult FnExpr = CreateFunctionRefExpr(*this, Method, Best->FoundDecl, - HadMultipleCandidates, OpLoc); + Base, HadMultipleCandidates, OpLoc); if (FnExpr.isInvalid()) return ExprError(); @@ -13225,7 +13238,7 @@ ExprResult Sema::BuildLiteralOperatorCall(LookupResult &R, FunctionDecl *FD = Best->Function; ExprResult Fn = CreateFunctionRefExpr(*this, FD, Best->FoundDecl, - HadMultipleCandidates, + nullptr, HadMultipleCandidates, SuffixInfo.getLoc(), SuffixInfo.getInfo()); if (Fn.isInvalid()) diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaPseudoObject.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaPseudoObject.cpp index b6b429d1f25c..d159172a6990 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaPseudoObject.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaPseudoObject.cpp @@ -1176,8 +1176,6 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() { AtIndexGetter = S.LookupMethodInObjectType(AtIndexGetterSelector, ResultType, true /*instance*/); - bool receiverIdType = (BaseT->isObjCIdType() || - BaseT->isObjCQualifiedIdType()); if (!AtIndexGetter && S.getLangOpts().DebuggerObjCLiteral) { AtIndexGetter = ObjCMethodDecl::Create(S.Context, SourceLocation(), @@ -1203,7 +1201,7 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() { } if (!AtIndexGetter) { - if (!receiverIdType) { + if (!BaseT->isObjCIdType()) { S.Diag(BaseExpr->getExprLoc(), diag::err_objc_subscript_method_not_found) << BaseExpr->getType() << 0 << arrayRef; return false; @@ -1284,9 +1282,6 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() { } AtIndexSetter = S.LookupMethodInObjectType(AtIndexSetterSelector, ResultType, true /*instance*/); - - bool receiverIdType = (BaseT->isObjCIdType() || - BaseT->isObjCQualifiedIdType()); if (!AtIndexSetter && S.getLangOpts().DebuggerObjCLiteral) { TypeSourceInfo *ReturnTInfo = nullptr; @@ -1321,7 +1316,7 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() { } if (!AtIndexSetter) { - if (!receiverIdType) { + if (!BaseT->isObjCIdType()) { S.Diag(BaseExpr->getExprLoc(), diag::err_objc_subscript_method_not_found) << BaseExpr->getType() << 1 << arrayRef; diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp index eed10b077eb8..2a38a1f8e1d8 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaStmt.cpp @@ -1544,23 +1544,78 @@ namespace { // A visitor to determine if a continue or break statement is a // subexpression. - class BreakContinueFinder : public EvaluatedExprVisitor { + class BreakContinueFinder : public ConstEvaluatedExprVisitor { SourceLocation BreakLoc; SourceLocation ContinueLoc; + bool InSwitch = false; + public: - BreakContinueFinder(Sema &S, Stmt* Body) : + BreakContinueFinder(Sema &S, const Stmt* Body) : Inherited(S.Context) { Visit(Body); } - typedef EvaluatedExprVisitor Inherited; + typedef ConstEvaluatedExprVisitor Inherited; - void VisitContinueStmt(ContinueStmt* E) { + void VisitContinueStmt(const ContinueStmt* E) { ContinueLoc = E->getContinueLoc(); } - void VisitBreakStmt(BreakStmt* E) { - BreakLoc = E->getBreakLoc(); + void VisitBreakStmt(const BreakStmt* E) { + if (!InSwitch) + BreakLoc = E->getBreakLoc(); + } + + void VisitSwitchStmt(const SwitchStmt* S) { + if (const Stmt *Init = S->getInit()) + Visit(Init); + if (const Stmt *CondVar = S->getConditionVariableDeclStmt()) + Visit(CondVar); + if (const Stmt *Cond = S->getCond()) + Visit(Cond); + + // Don't return break statements from the body of a switch. + InSwitch = true; + if (const Stmt *Body = S->getBody()) + Visit(Body); + InSwitch = false; + } + + void VisitForStmt(const ForStmt *S) { + // Only visit the init statement of a for loop; the body + // has a different break/continue scope. + if (const Stmt *Init = S->getInit()) + Visit(Init); + } + + void VisitWhileStmt(const WhileStmt *) { + // Do nothing; the children of a while loop have a different + // break/continue scope. + } + + void VisitDoStmt(const DoStmt *) { + // Do nothing; the children of a while loop have a different + // break/continue scope. + } + + void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { + // Only visit the initialization of a for loop; the body + // has a different break/continue scope. + if (const Stmt *Range = S->getRangeStmt()) + Visit(Range); + if (const Stmt *Begin = S->getBeginStmt()) + Visit(Begin); + if (const Stmt *End = S->getEndStmt()) + Visit(End); + } + + void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { + // Only visit the initialization of a for loop; the body + // has a different break/continue scope. + if (const Stmt *Element = S->getElement()) + Visit(Element); + if (const Stmt *Collection = S->getCollection()) + Visit(Collection); } bool ContinueFound() { return ContinueLoc.isValid(); } diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp index a8923ce9e27d..e9b38551683c 100644 --- a/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp +++ b/contrib/llvm/tools/clang/lib/Sema/SemaTemplate.cpp @@ -2806,6 +2806,101 @@ checkBuiltinTemplateIdType(Sema &SemaRef, BuiltinTemplateDecl *BTD, llvm_unreachable("unexpected BuiltinTemplateDecl!"); } +/// Determine whether this alias template is "enable_if_t". +static bool isEnableIfAliasTemplate(TypeAliasTemplateDecl *AliasTemplate) { + return AliasTemplate->getName().equals("enable_if_t"); +} + +/// Collect all of the separable terms in the given condition, which +/// might be a conjunction. +/// +/// FIXME: The right answer is to convert the logical expression into +/// disjunctive normal form, so we can find the first failed term +/// within each possible clause. +static void collectConjunctionTerms(Expr *Clause, + SmallVectorImpl &Terms) { + if (auto BinOp = dyn_cast(Clause->IgnoreParenImpCasts())) { + if (BinOp->getOpcode() == BO_LAnd) { + collectConjunctionTerms(BinOp->getLHS(), Terms); + collectConjunctionTerms(BinOp->getRHS(), Terms); + } + + return; + } + + Terms.push_back(Clause); +} + +// The ranges-v3 library uses an odd pattern of a top-level "||" with +// a left-hand side that is value-dependent but never true. Identify +// the idiom and ignore that term. +static Expr *lookThroughRangesV3Condition(Preprocessor &PP, Expr *Cond) { + // Top-level '||'. + auto *BinOp = dyn_cast(Cond->IgnoreParenImpCasts()); + if (!BinOp) return Cond; + + if (BinOp->getOpcode() != BO_LOr) return Cond; + + // With an inner '==' that has a literal on the right-hand side. + Expr *LHS = BinOp->getLHS(); + auto *InnerBinOp = dyn_cast(LHS->IgnoreParenImpCasts()); + if (!InnerBinOp) return Cond; + + if (InnerBinOp->getOpcode() != BO_EQ || + !isa(InnerBinOp->getRHS())) + return Cond; + + // If the inner binary operation came from a macro expansion named + // CONCEPT_REQUIRES or CONCEPT_REQUIRES_, return the right-hand side + // of the '||', which is the real, user-provided condition. + SourceLocation Loc = InnerBinOp->getExprLoc(); + if (!Loc.isMacroID()) return Cond; + + StringRef MacroName = PP.getImmediateMacroName(Loc); + if (MacroName == "CONCEPT_REQUIRES" || MacroName == "CONCEPT_REQUIRES_") + return BinOp->getRHS(); + + return Cond; +} + +/// Find the failed subexpression within enable_if, and describe it +/// with a string. +static std::pair +findFailedEnableIfCondition(Sema &S, Expr *Cond) { + Cond = lookThroughRangesV3Condition(S.PP, Cond); + + // Separate out all of the terms in a conjunction. + SmallVector Terms; + collectConjunctionTerms(Cond, Terms); + + // Determine which term failed. + Expr *FailedCond = nullptr; + for (Expr *Term : Terms) { + // The initialization of the parameter from the argument is + // a constant-evaluated context. + EnterExpressionEvaluationContext ConstantEvaluated( + S, Sema::ExpressionEvaluationContext::ConstantEvaluated); + + bool Succeeded; + if (Term->EvaluateAsBooleanCondition(Succeeded, S.Context) && + !Succeeded) { + FailedCond = Term->IgnoreParenImpCasts(); + break; + } + } + + if (!FailedCond) + FailedCond = Cond->IgnoreParenImpCasts(); + + std::string Description; + { + llvm::raw_string_ostream Out(Description); + FailedCond->printPretty(Out, nullptr, + PrintingPolicy(S.Context.getLangOpts())); + } + return { FailedCond, Description }; +} + QualType Sema::CheckTemplateIdType(TemplateName Name, SourceLocation TemplateLoc, TemplateArgumentListInfo &TemplateArgs) { @@ -2852,12 +2947,12 @@ QualType Sema::CheckTemplateIdType(TemplateName Name, if (Pattern->isInvalidDecl()) return QualType(); - TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack, - Converted); + TemplateArgumentList StackTemplateArgs(TemplateArgumentList::OnStack, + Converted); // Only substitute for the innermost template argument list. MultiLevelTemplateArgumentList TemplateArgLists; - TemplateArgLists.addOuterTemplateArguments(&TemplateArgs); + TemplateArgLists.addOuterTemplateArguments(&StackTemplateArgs); unsigned Depth = AliasTemplate->getTemplateParameters()->getDepth(); for (unsigned I = 0; I < Depth; ++I) TemplateArgLists.addOuterTemplateArguments(None); @@ -2870,8 +2965,42 @@ QualType Sema::CheckTemplateIdType(TemplateName Name, CanonType = SubstType(Pattern->getUnderlyingType(), TemplateArgLists, AliasTemplate->getLocation(), AliasTemplate->getDeclName()); - if (CanonType.isNull()) + if (CanonType.isNull()) { + // If this was enable_if and we failed to find the nested type + // within enable_if in a SFINAE context, dig out the specific + // enable_if condition that failed and present that instead. + if (isEnableIfAliasTemplate(AliasTemplate)) { + if (auto DeductionInfo = isSFINAEContext()) { + if (*DeductionInfo && + (*DeductionInfo)->hasSFINAEDiagnostic() && + (*DeductionInfo)->peekSFINAEDiagnostic().second.getDiagID() == + diag::err_typename_nested_not_found_enable_if && + TemplateArgs[0].getArgument().getKind() + == TemplateArgument::Expression) { + Expr *FailedCond; + std::string FailedDescription; + std::tie(FailedCond, FailedDescription) = + findFailedEnableIfCondition( + *this, TemplateArgs[0].getSourceExpression()); + + // Remove the old SFINAE diagnostic. + PartialDiagnosticAt OldDiag = + {SourceLocation(), PartialDiagnostic::NullDiagnostic()}; + (*DeductionInfo)->takeSFINAEDiagnostic(OldDiag); + + // Add a new SFINAE diagnostic specifying which condition + // failed. + (*DeductionInfo)->addSFINAEDiagnostic( + OldDiag.first, + PDiag(diag::err_typename_nested_not_found_requirement) + << FailedDescription + << FailedCond->getSourceRange()); + } + } + } + return QualType(); + } } else if (Name.isDependent() || TemplateSpecializationType::anyDependentTemplateArguments( TemplateArgs, InstantiationDependent)) { @@ -5190,10 +5319,16 @@ enum NullPointerValueKind { /// value of the appropriate type. static NullPointerValueKind isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, - QualType ParamType, Expr *Arg) { + QualType ParamType, Expr *Arg, + Decl *Entity = nullptr) { if (Arg->isValueDependent() || Arg->isTypeDependent()) return NPV_NotNullPointer; + // dllimport'd entities aren't constant but are available inside of template + // arguments. + if (Entity && Entity->hasAttr()) + return NPV_NotNullPointer; + if (!S.isCompleteType(Arg->getExprLoc(), ParamType)) llvm_unreachable( "Incomplete parameter type in isNullPointerValueTemplateArgument!"); @@ -5437,14 +5572,8 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S, // If our parameter has pointer type, check for a null template value. if (ParamType->isPointerType() || ParamType->isNullPtrType()) { - NullPointerValueKind NPV; - // dllimport'd entities aren't constant but are available inside of template - // arguments. - if (Entity && Entity->hasAttr()) - NPV = NPV_NotNullPointer; - else - NPV = isNullPointerValueTemplateArgument(S, Param, ParamType, ArgIn); - switch (NPV) { + switch (isNullPointerValueTemplateArgument(S, Param, ParamType, ArgIn, + Entity)) { case NPV_NullPointer: S.Diag(Arg->getExprLoc(), diag::warn_cxx98_compat_template_arg_null); Converted = TemplateArgument(S.Context.getCanonicalType(ParamType), @@ -5636,39 +5765,8 @@ static bool CheckTemplateArgumentPointerToMember(Sema &S, TemplateArgument &Converted) { bool Invalid = false; - // Check for a null pointer value. Expr *Arg = ResultArg; - switch (isNullPointerValueTemplateArgument(S, Param, ParamType, Arg)) { - case NPV_Error: - return true; - case NPV_NullPointer: - S.Diag(Arg->getExprLoc(), diag::warn_cxx98_compat_template_arg_null); - Converted = TemplateArgument(S.Context.getCanonicalType(ParamType), - /*isNullPtr*/true); - return false; - case NPV_NotNullPointer: - break; - } - bool ObjCLifetimeConversion; - if (S.IsQualificationConversion(Arg->getType(), - ParamType.getNonReferenceType(), - false, ObjCLifetimeConversion)) { - Arg = S.ImpCastExprToType(Arg, ParamType, CK_NoOp, - Arg->getValueKind()).get(); - ResultArg = Arg; - } else if (!S.Context.hasSameUnqualifiedType(Arg->getType(), - ParamType.getNonReferenceType())) { - // We can't perform this conversion. - S.Diag(Arg->getLocStart(), diag::err_template_arg_not_convertible) - << Arg->getType() << ParamType << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); - return true; - } - - // See through any implicit casts we added to fix the type. - while (ImplicitCastExpr *Cast = dyn_cast(Arg)) - Arg = Cast->getSubExpr(); // C++ [temp.arg.nontype]p1: // @@ -5725,6 +5823,37 @@ static bool CheckTemplateArgumentPointerToMember(Sema &S, DRE = nullptr; } + ValueDecl *Entity = DRE ? DRE->getDecl() : nullptr; + + // Check for a null pointer value. + switch (isNullPointerValueTemplateArgument(S, Param, ParamType, ResultArg, + Entity)) { + case NPV_Error: + return true; + case NPV_NullPointer: + S.Diag(ResultArg->getExprLoc(), diag::warn_cxx98_compat_template_arg_null); + Converted = TemplateArgument(S.Context.getCanonicalType(ParamType), + /*isNullPtr*/true); + return false; + case NPV_NotNullPointer: + break; + } + + if (S.IsQualificationConversion(ResultArg->getType(), + ParamType.getNonReferenceType(), false, + ObjCLifetimeConversion)) { + ResultArg = S.ImpCastExprToType(ResultArg, ParamType, CK_NoOp, + ResultArg->getValueKind()) + .get(); + } else if (!S.Context.hasSameUnqualifiedType( + ResultArg->getType(), ParamType.getNonReferenceType())) { + // We can't perform this conversion. + S.Diag(ResultArg->getLocStart(), diag::err_template_arg_not_convertible) + << ResultArg->getType() << ParamType << ResultArg->getSourceRange(); + S.Diag(Param->getLocation(), diag::note_template_param_here); + return true; + } + if (!DRE) return S.Diag(Arg->getLocStart(), diag::err_template_arg_not_pointer_to_member_form) @@ -9290,7 +9419,7 @@ Sema::ActOnTypenameType(Scope *S, /// Determine whether this failed name lookup should be treated as being /// disabled by a usage of std::enable_if. static bool isEnableIf(NestedNameSpecifierLoc NNS, const IdentifierInfo &II, - SourceRange &CondRange) { + SourceRange &CondRange, Expr *&Cond) { // We must be looking for a ::type... if (!II.isStr("type")) return false; @@ -9320,6 +9449,19 @@ static bool isEnableIf(NestedNameSpecifierLoc NNS, const IdentifierInfo &II, // Assume the first template argument is the condition. CondRange = EnableIfTSTLoc.getArgLoc(0).getSourceRange(); + + // Dig out the condition. + Cond = nullptr; + if (EnableIfTSTLoc.getArgLoc(0).getArgument().getKind() + != TemplateArgument::Expression) + return true; + + Cond = EnableIfTSTLoc.getArgLoc(0).getSourceExpression(); + + // Ignore Boolean literals; they add no value. + if (isa(Cond->IgnoreParenCasts())) + Cond = nullptr; + return true; } @@ -9363,9 +9505,25 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword, // If we're looking up 'type' within a template named 'enable_if', produce // a more specific diagnostic. SourceRange CondRange; - if (isEnableIf(QualifierLoc, II, CondRange)) { + Expr *Cond = nullptr; + if (isEnableIf(QualifierLoc, II, CondRange, Cond)) { + // If we have a condition, narrow it down to the specific failed + // condition. + if (Cond) { + Expr *FailedCond; + std::string FailedDescription; + std::tie(FailedCond, FailedDescription) = + findFailedEnableIfCondition(*this, Cond); + + Diag(FailedCond->getExprLoc(), + diag::err_typename_nested_not_found_requirement) + << FailedDescription + << FailedCond->getSourceRange(); + return QualType(); + } + Diag(CondRange.getBegin(), diag::err_typename_nested_not_found_enable_if) - << Ctx << CondRange; + << Ctx << CondRange; return QualType(); } diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp index 3aee3c04001d..678ecfc9a3d9 100644 --- a/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp +++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp @@ -9314,6 +9314,7 @@ void ASTReader::diagnoseOdrViolations() { TypeAlias, TypeDef, Var, + Friend, Other } FirstDiffType = Other, SecondDiffType = Other; @@ -9347,6 +9348,8 @@ void ASTReader::diagnoseOdrViolations() { return TypeDef; case Decl::Var: return Var; + case Decl::Friend: + return Friend; } }; @@ -9463,6 +9466,9 @@ void ASTReader::diagnoseOdrViolations() { VarSingleInitializer, VarDifferentInitializer, VarConstexpr, + FriendTypeFunction, + FriendType, + FriendFunction, }; // These lambdas have the common portions of the ODR diagnostics. This @@ -9973,6 +9979,53 @@ void ASTReader::diagnoseOdrViolations() { } break; } + case Friend: { + FriendDecl *FirstFriend = cast(FirstDecl); + FriendDecl *SecondFriend = cast(SecondDecl); + + NamedDecl *FirstND = FirstFriend->getFriendDecl(); + NamedDecl *SecondND = SecondFriend->getFriendDecl(); + + TypeSourceInfo *FirstTSI = FirstFriend->getFriendType(); + TypeSourceInfo *SecondTSI = SecondFriend->getFriendType(); + + if (FirstND && SecondND) { + ODRDiagError(FirstFriend->getFriendLoc(), + FirstFriend->getSourceRange(), FriendFunction) + << FirstND; + ODRDiagNote(SecondFriend->getFriendLoc(), + SecondFriend->getSourceRange(), FriendFunction) + << SecondND; + + Diagnosed = true; + break; + } + + if (FirstTSI && SecondTSI) { + QualType FirstFriendType = FirstTSI->getType(); + QualType SecondFriendType = SecondTSI->getType(); + assert(ComputeQualTypeODRHash(FirstFriendType) != + ComputeQualTypeODRHash(SecondFriendType)); + ODRDiagError(FirstFriend->getFriendLoc(), + FirstFriend->getSourceRange(), FriendType) + << FirstFriendType; + ODRDiagNote(SecondFriend->getFriendLoc(), + SecondFriend->getSourceRange(), FriendType) + << SecondFriendType; + Diagnosed = true; + break; + } + + ODRDiagError(FirstFriend->getFriendLoc(), FirstFriend->getSourceRange(), + FriendTypeFunction) + << (FirstTSI == nullptr); + ODRDiagNote(SecondFriend->getFriendLoc(), + SecondFriend->getSourceRange(), FriendTypeFunction) + << (SecondTSI == nullptr); + + Diagnosed = true; + break; + } } if (Diagnosed == true) diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp index 4d9ddd2ff506..abed2586561a 100644 --- a/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReaderDecl.cpp @@ -573,6 +573,8 @@ void ASTDeclReader::VisitDecl(Decl *D) { else Reader.HiddenNamesMap[Owner].push_back(D); } + } else if (ModulePrivate) { + D->setModuleOwnershipKind(Decl::ModuleOwnershipKind::ModulePrivate); } } diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp index c6129d326cb6..f7a49e41009d 100644 --- a/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp +++ b/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp @@ -1462,7 +1462,7 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, } // Module map file - if (WritingModule) { + if (WritingModule && WritingModule->Kind == Module::ModuleMapModule) { Record.clear(); auto &Map = PP.getHeaderSearchInfo().getModuleMap(); diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp index 2d648cb103cb..ec21ca2276c1 100644 --- a/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/contrib/llvm/tools/clang/lib/Serialization/ASTWriterDecl.cpp @@ -2233,8 +2233,18 @@ void ASTRecordWriter::AddFunctionDefinition(const FunctionDecl *FD) { Writer->ClearSwitchCaseIDs(); assert(FD->doesThisDeclarationHaveABody()); - bool ModulesCodegen = Writer->Context->getLangOpts().ModulesCodegen && - Writer->WritingModule && !FD->isDependentContext(); + bool ModulesCodegen = false; + if (Writer->WritingModule && !FD->isDependentContext()) { + // Under -fmodules-codegen, codegen is performed for all defined functions. + // When building a C++ Modules TS module interface unit, a strong definition + // in the module interface is provided by the compilation of that module + // interface unit, not by its users. (Inline functions are still emitted + // in module users.) + ModulesCodegen = + Writer->Context->getLangOpts().ModulesCodegen || + (Writer->WritingModule->Kind == Module::ModuleInterfaceUnit && + Writer->Context->GetGVALinkageForFunction(FD) == GVA_StrongExternal); + } Record->push_back(ModulesCodegen); if (ModulesCodegen) Writer->ModularCodegenDecls.push_back(Writer->GetDeclRef(FD)); diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp index 54634fdffeb5..83e67662e614 100644 --- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp +++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalysisManager.cpp @@ -23,9 +23,10 @@ AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags, AnalyzerOptions &Options, CodeInjector *injector) : AnaCtxMgr(Options.UnoptimizedCFG, - /*AddImplicitDtors=*/true, + Options.includeImplicitDtorsInCFG(), /*AddInitializers=*/true, Options.includeTemporaryDtorsInCFG(), + Options.includeLifetimeInCFG(), Options.shouldSynthesizeBodies(), Options.shouldConditionalizeStaticInitializers(), /*addCXXNewAllocator=*/true, diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp index 11b9f8c4f725..6f48fcb9e20c 100644 --- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp +++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp @@ -172,6 +172,17 @@ bool AnalyzerOptions::includeTemporaryDtorsInCFG() { /* Default = */ false); } +bool AnalyzerOptions::includeImplicitDtorsInCFG() { + return getBooleanOption(IncludeImplicitDtorsInCFG, + "cfg-implicit-dtors", + /* Default = */ true); +} + +bool AnalyzerOptions::includeLifetimeInCFG() { + return getBooleanOption(IncludeLifetimeInCFG, "cfg-lifetime", + /* Default = */ false); +} + bool AnalyzerOptions::mayInlineCXXStandardLibrary() { return getBooleanOption(InlineCXXStandardLibrary, "c++-stdlib-inlining", diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp index f84c0ee800a3..eee5400fe177 100644 --- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp +++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp @@ -362,6 +362,8 @@ void ExprEngine::processCFGElement(const CFGElement E, ExplodedNode *Pred, case CFGElement::TemporaryDtor: ProcessImplicitDtor(E.castAs(), Pred); return; + case CFGElement::LifetimeEnds: + return; } } diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp index 6aa6da560e60..d91786f74919 100644 --- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp +++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/PathDiagnostic.cpp @@ -578,6 +578,7 @@ getLocationForCaller(const StackFrameContext *SFC, } case CFGElement::TemporaryDtor: case CFGElement::NewAllocator: + case CFGElement::LifetimeEnds: llvm_unreachable("not yet implemented!"); } diff --git a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp index 9c28457b2139..f09f9696f5ad 100644 --- a/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp +++ b/contrib/llvm/tools/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp @@ -71,18 +71,15 @@ SVal SimpleSValBuilder::dispatchCast(SVal Val, QualType CastTy) { } SVal SimpleSValBuilder::evalCastFromNonLoc(NonLoc val, QualType castTy) { - bool isLocType = Loc::isLocType(castTy); - if (val.getAs()) return val; if (Optional LI = val.getAs()) { if (isLocType) return LI->getLoc(); - // FIXME: Correctly support promotions/truncations. - unsigned castSize = Context.getTypeSize(castTy); + unsigned castSize = Context.getIntWidth(castTy); if (castSize == LI->getNumBits()) return val; return makeLocAsInteger(LI->getLoc(), castSize); @@ -173,7 +170,7 @@ SVal SimpleSValBuilder::evalCastFromLoc(Loc val, QualType castTy) { } if (castTy->isIntegralOrEnumerationType()) { - unsigned BitWidth = Context.getTypeSize(castTy); + unsigned BitWidth = Context.getIntWidth(castTy); if (!val.getAs()) return makeLocAsInteger(val, BitWidth); diff --git a/contrib/llvm/tools/clang/lib/Tooling/ArgumentsAdjusters.cpp b/contrib/llvm/tools/clang/lib/Tooling/ArgumentsAdjusters.cpp index 48b925c698a7..ac9fd3c5cade 100644 --- a/contrib/llvm/tools/clang/lib/Tooling/ArgumentsAdjusters.cpp +++ b/contrib/llvm/tools/clang/lib/Tooling/ArgumentsAdjusters.cpp @@ -42,7 +42,7 @@ ArgumentsAdjuster getClangStripOutputAdjuster() { AdjustedArgs.push_back(Args[i]); if (Arg == "-o") { - // Output is specified as -o foo. Skip the next argument also. + // Output is specified as -o foo. Skip the next argument too. ++i; } // Else, the output is specified as -ofoo. Just do nothing. @@ -51,6 +51,26 @@ ArgumentsAdjuster getClangStripOutputAdjuster() { }; } +ArgumentsAdjuster getClangStripDependencyFileAdjuster() { + return [](const CommandLineArguments &Args, StringRef /*unused*/) { + CommandLineArguments AdjustedArgs; + for (size_t i = 0, e = Args.size(); i < e; ++i) { + StringRef Arg = Args[i]; + // All dependency-file options begin with -M. These include -MM, + // -MF, -MG, -MP, -MT, -MQ, -MD, and -MMD. + if (!Arg.startswith("-M")) + AdjustedArgs.push_back(Args[i]); + + if ((Arg == "-MF") || (Arg == "-MT") || (Arg == "-MQ") || + (Arg == "-MD") || (Arg == "-MMD")) { + // Output is specified as -MX foo. Skip the next argument also. + ++i; + } + } + return AdjustedArgs; + }; +} + ArgumentsAdjuster getInsertArgumentAdjuster(const CommandLineArguments &Extra, ArgumentInsertPosition Pos) { return [Extra, Pos](const CommandLineArguments &Args, StringRef /*unused*/) { @@ -83,4 +103,3 @@ ArgumentsAdjuster combineAdjusters(ArgumentsAdjuster First, } // end namespace tooling } // end namespace clang - diff --git a/contrib/llvm/tools/clang/lib/Tooling/Core/Diagnostic.cpp b/contrib/llvm/tools/clang/lib/Tooling/Core/Diagnostic.cpp index 3bbc2b901e38..9e4833f2eff4 100644 --- a/contrib/llvm/tools/clang/lib/Tooling/Core/Diagnostic.cpp +++ b/contrib/llvm/tools/clang/lib/Tooling/Core/Diagnostic.cpp @@ -35,9 +35,9 @@ Diagnostic::Diagnostic(llvm::StringRef DiagnosticName, BuildDirectory(BuildDirectory) {} Diagnostic::Diagnostic(llvm::StringRef DiagnosticName, - DiagnosticMessage &Message, - llvm::StringMap &Fix, - SmallVector &Notes, + const DiagnosticMessage &Message, + const llvm::StringMap &Fix, + const SmallVector &Notes, Level DiagLevel, llvm::StringRef BuildDirectory) : DiagnosticName(DiagnosticName), Message(Message), Fix(Fix), Notes(Notes), DiagLevel(DiagLevel), BuildDirectory(BuildDirectory) {} diff --git a/contrib/llvm/tools/clang/lib/Tooling/Tooling.cpp b/contrib/llvm/tools/clang/lib/Tooling/Tooling.cpp index 2e093dd9afc8..c84fbf473753 100644 --- a/contrib/llvm/tools/clang/lib/Tooling/Tooling.cpp +++ b/contrib/llvm/tools/clang/lib/Tooling/Tooling.cpp @@ -100,7 +100,6 @@ clang::CompilerInvocation *newInvocation( *Diagnostics); Invocation->getFrontendOpts().DisableFree = false; Invocation->getCodeGenOpts().DisableFree = false; - Invocation->getDependencyOutputOpts() = DependencyOutputOptions(); return Invocation; } @@ -140,9 +139,11 @@ bool runToolOnCodeWithArgs( OverlayFileSystem->pushOverlay(InMemoryFileSystem); llvm::IntrusiveRefCntPtr Files( new FileManager(FileSystemOptions(), OverlayFileSystem)); - ToolInvocation Invocation(getSyntaxOnlyToolArgs(ToolName, Args, FileNameRef), - ToolAction, Files.get(), - std::move(PCHContainerOps)); + ArgumentsAdjuster Adjuster = getClangStripDependencyFileAdjuster(); + ToolInvocation Invocation( + getSyntaxOnlyToolArgs(ToolName, Adjuster(Args, FileNameRef), FileNameRef), + ToolAction, Files.get(), + std::move(PCHContainerOps)); SmallString<1024> CodeStorage; InMemoryFileSystem->addFile(FileNameRef, 0, @@ -510,7 +511,8 @@ buildASTFromCode(const Twine &Code, const Twine &FileName, std::unique_ptr buildASTFromCodeWithArgs( const Twine &Code, const std::vector &Args, const Twine &FileName, const Twine &ToolName, - std::shared_ptr PCHContainerOps) { + std::shared_ptr PCHContainerOps, + ArgumentsAdjuster Adjuster) { SmallString<16> FileNameStorage; StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage); @@ -523,8 +525,10 @@ std::unique_ptr buildASTFromCodeWithArgs( OverlayFileSystem->pushOverlay(InMemoryFileSystem); llvm::IntrusiveRefCntPtr Files( new FileManager(FileSystemOptions(), OverlayFileSystem)); - ToolInvocation Invocation(getSyntaxOnlyToolArgs(ToolName, Args, FileNameRef), - &Action, Files.get(), std::move(PCHContainerOps)); + + ToolInvocation Invocation( + getSyntaxOnlyToolArgs(ToolName, Adjuster(Args, FileNameRef), FileNameRef), + &Action, Files.get(), std::move(PCHContainerOps)); SmallString<1024> CodeStorage; InMemoryFileSystem->addFile(FileNameRef, 0, diff --git a/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp b/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp index 62fcccbacb55..49c1edce3220 100644 --- a/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp +++ b/contrib/llvm/tools/clang/utils/TableGen/NeonEmitter.cpp @@ -860,10 +860,6 @@ void Type::applyModifier(char Mod) { Float = true; ElementBitwidth = 64; break; - case 'H': - Float = true; - ElementBitwidth = 16; - break; case 'g': if (AppliedQuad) Bitwidth /= 2; @@ -1010,7 +1006,7 @@ std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const { } static bool isFloatingPointProtoModifier(char Mod) { - return Mod == 'F' || Mod == 'f' || Mod == 'H'; + return Mod == 'F' || Mod == 'f'; } std::string Intrinsic::getBuiltinTypeStr() { diff --git a/contrib/llvm/tools/lld/COFF/Chunks.cpp b/contrib/llvm/tools/lld/COFF/Chunks.cpp index 9b642dcaf137..c0996f55f9d1 100644 --- a/contrib/llvm/tools/lld/COFF/Chunks.cpp +++ b/contrib/llvm/tools/lld/COFF/Chunks.cpp @@ -52,6 +52,7 @@ static void add16(uint8_t *P, int16_t V) { write16le(P, read16le(P) + V); } static void add32(uint8_t *P, int32_t V) { write32le(P, read32le(P) + V); } static void add64(uint8_t *P, int64_t V) { write64le(P, read64le(P) + V); } static void or16(uint8_t *P, uint16_t V) { write16le(P, read16le(P) | V); } +static void or32(uint8_t *P, uint32_t V) { write32le(P, read32le(P) | V); } static void applySecRel(const SectionChunk *Sec, uint8_t *Off, OutputSection *OS, uint64_t S) { @@ -166,6 +167,41 @@ void SectionChunk::applyRelARM(uint8_t *Off, uint16_t Type, OutputSection *OS, } } +static void applyArm64Addr(uint8_t *Off, uint64_t Imm) { + uint32_t ImmLo = (Imm & 0x3) << 29; + uint32_t ImmHi = (Imm & 0x1FFFFC) << 3; + uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3); + write32le(Off, (read32le(Off) & ~Mask) | ImmLo | ImmHi); +} + +// Update the immediate field in a AARCH64 ldr, str, and add instruction. +static void applyArm64Imm(uint8_t *Off, uint64_t Imm) { + uint32_t Orig = read32le(Off); + Imm += (Orig >> 10) & 0xFFF; + Orig &= ~(0xFFF << 10); + write32le(Off, Orig | ((Imm & 0xFFF) << 10)); +} + +static void applyArm64Ldr(uint8_t *Off, uint64_t Imm) { + int Size = read32le(Off) >> 30; + Imm >>= Size; + applyArm64Imm(Off, Imm); +} + +void SectionChunk::applyRelARM64(uint8_t *Off, uint16_t Type, OutputSection *OS, + uint64_t S, uint64_t P) const { + switch (Type) { + case IMAGE_REL_ARM64_PAGEBASE_REL21: applyArm64Addr(Off, (S >> 12) - (P >> 12)); break; + case IMAGE_REL_ARM64_PAGEOFFSET_12A: applyArm64Imm(Off, S & 0xfff); break; + case IMAGE_REL_ARM64_PAGEOFFSET_12L: applyArm64Ldr(Off, S & 0xfff); break; + case IMAGE_REL_ARM64_BRANCH26: or32(Off, ((S - P) & 0x0FFFFFFC) >> 2); break; + case IMAGE_REL_ARM64_ADDR32: add32(Off, S + Config->ImageBase); break; + case IMAGE_REL_ARM64_ADDR64: add64(Off, S + Config->ImageBase); break; + default: + fatal("unsupported relocation type 0x" + Twine::utohexstr(Type)); + } +} + void SectionChunk::writeTo(uint8_t *Buf) const { if (!hasData()) return; @@ -210,6 +246,9 @@ void SectionChunk::writeTo(uint8_t *Buf) const { case ARMNT: applyRelARM(Off, Rel.Type, OS, S, P); break; + case ARM64: + applyRelARM64(Off, Rel.Type, OS, S, P); + break; default: llvm_unreachable("unknown machine type"); } @@ -236,6 +275,10 @@ static uint8_t getBaserelType(const coff_relocation &Rel) { if (Rel.Type == IMAGE_REL_ARM_MOV32T) return IMAGE_REL_BASED_ARM_MOV32T; return IMAGE_REL_BASED_ABSOLUTE; + case ARM64: + if (Rel.Type == IMAGE_REL_ARM64_ADDR64) + return IMAGE_REL_BASED_DIR64; + return IMAGE_REL_BASED_ABSOLUTE; default: llvm_unreachable("unknown machine type"); } @@ -345,6 +388,14 @@ void ImportThunkChunkARM::writeTo(uint8_t *Buf) const { applyMOV32T(Buf + OutputSectionOff, ImpSymbol->getRVA() + Config->ImageBase); } +void ImportThunkChunkARM64::writeTo(uint8_t *Buf) const { + int64_t PageOff = (ImpSymbol->getRVA() >> 12) - (RVA >> 12); + int64_t Off = ImpSymbol->getRVA() & 0xfff; + memcpy(Buf + OutputSectionOff, ImportThunkARM64, sizeof(ImportThunkARM64)); + applyArm64Addr(Buf + OutputSectionOff, PageOff); + applyArm64Ldr(Buf + OutputSectionOff + 4, Off); +} + void LocalImportChunk::getBaserels(std::vector *Res) { Res->emplace_back(getRVA()); } diff --git a/contrib/llvm/tools/lld/COFF/Chunks.h b/contrib/llvm/tools/lld/COFF/Chunks.h index 6e1bf94da1a5..fc3f5d0df4b6 100644 --- a/contrib/llvm/tools/lld/COFF/Chunks.h +++ b/contrib/llvm/tools/lld/COFF/Chunks.h @@ -151,6 +151,8 @@ class SectionChunk : public Chunk { uint64_t P) const; void applyRelARM(uint8_t *Off, uint16_t Type, OutputSection *OS, uint64_t S, uint64_t P) const; + void applyRelARM64(uint8_t *Off, uint16_t Type, OutputSection *OS, uint64_t S, + uint64_t P) const; // Called if the garbage collector decides to not include this chunk // in a final output. It's supposed to print out a log message to stdout. @@ -264,6 +266,12 @@ static const uint8_t ImportThunkARM[] = { 0xdc, 0xf8, 0x00, 0xf0, // ldr.w pc, [ip] }; +static const uint8_t ImportThunkARM64[] = { + 0x10, 0x00, 0x00, 0x90, // adrp x16, #0 + 0x10, 0x02, 0x40, 0xf9, // ldr x16, [x16] + 0x00, 0x02, 0x1f, 0xd6, // br x16 +}; + // Windows-specific. // A chunk for DLL import jump table entry. In a final output, it's // contents will be a JMP instruction to some __imp_ symbol. @@ -299,6 +307,16 @@ class ImportThunkChunkARM : public Chunk { Defined *ImpSymbol; }; +class ImportThunkChunkARM64 : public Chunk { +public: + explicit ImportThunkChunkARM64(Defined *S) : ImpSymbol(S) {} + size_t getSize() const override { return sizeof(ImportThunkARM64); } + void writeTo(uint8_t *Buf) const override; + +private: + Defined *ImpSymbol; +}; + // Windows-specific. // See comments for DefinedLocalImport class. class LocalImportChunk : public Chunk { diff --git a/contrib/llvm/tools/lld/COFF/Config.h b/contrib/llvm/tools/lld/COFF/Config.h index 9fcea96d65d3..25fdc7abd67b 100644 --- a/contrib/llvm/tools/lld/COFF/Config.h +++ b/contrib/llvm/tools/lld/COFF/Config.h @@ -31,6 +31,7 @@ class SymbolBody; // Short aliases. static const auto AMD64 = llvm::COFF::IMAGE_FILE_MACHINE_AMD64; +static const auto ARM64 = llvm::COFF::IMAGE_FILE_MACHINE_ARM64; static const auto ARMNT = llvm::COFF::IMAGE_FILE_MACHINE_ARMNT; static const auto I386 = llvm::COFF::IMAGE_FILE_MACHINE_I386; @@ -73,7 +74,7 @@ enum class DebugType { // Global configuration. struct Configuration { enum ManifestKind { SideBySide, Embed, No }; - bool is64() { return Machine == AMD64; } + bool is64() { return Machine == AMD64 || Machine == ARM64; } llvm::COFF::MachineTypes Machine = IMAGE_FILE_MACHINE_UNKNOWN; bool Verbose = false; @@ -91,6 +92,7 @@ struct Configuration { bool WriteSymtab = true; unsigned DebugTypes = static_cast(DebugType::None); llvm::SmallString<128> PDBPath; + std::vector Argv; // Symbols in this set are considered as live by the garbage collector. std::set GCRoot; diff --git a/contrib/llvm/tools/lld/COFF/Driver.cpp b/contrib/llvm/tools/lld/COFF/Driver.cpp index 22efb312ae49..3620297b8b94 100644 --- a/contrib/llvm/tools/lld/COFF/Driver.cpp +++ b/contrib/llvm/tools/lld/COFF/Driver.cpp @@ -55,8 +55,8 @@ std::vector SpecificAllocBase::Instances; bool link(ArrayRef Args, raw_ostream &Diag) { ErrorCount = 0; ErrorOS = &Diag; - Argv0 = Args[0]; Config = make(); + Config->Argv = {Args.begin(), Args.end()}; Config->ColorDiagnostics = (ErrorOS == &llvm::errs() && Process::StandardErrHasColors()); Driver = make(); diff --git a/contrib/llvm/tools/lld/COFF/DriverUtils.cpp b/contrib/llvm/tools/lld/COFF/DriverUtils.cpp index d0152b0917b6..39d582469640 100644 --- a/contrib/llvm/tools/lld/COFF/DriverUtils.cpp +++ b/contrib/llvm/tools/lld/COFF/DriverUtils.cpp @@ -85,6 +85,7 @@ MachineTypes getMachineType(StringRef S) { .Cases("x64", "amd64", AMD64) .Cases("x86", "i386", I386) .Case("arm", ARMNT) + .Case("arm64", ARM64) .Default(IMAGE_FILE_MACHINE_UNKNOWN); if (MT != IMAGE_FILE_MACHINE_UNKNOWN) return MT; @@ -95,6 +96,8 @@ StringRef machineToStr(MachineTypes MT) { switch (MT) { case ARMNT: return "arm"; + case ARM64: + return "arm64"; case AMD64: return "x64"; case I386: @@ -378,13 +381,11 @@ static std::string createManifestXml() { static std::unique_ptr createMemoryBufferForManifestRes(size_t ManifestSize) { - size_t ResSize = alignTo(object::WIN_RES_MAGIC_SIZE + - object::WIN_RES_NULL_ENTRY_SIZE + - sizeof(object::WinResHeaderPrefix) + - sizeof(object::WinResIDs) + - sizeof(object::WinResHeaderSuffix) + - ManifestSize, - object::WIN_RES_DATA_ALIGNMENT); + size_t ResSize = alignTo( + object::WIN_RES_MAGIC_SIZE + object::WIN_RES_NULL_ENTRY_SIZE + + sizeof(object::WinResHeaderPrefix) + sizeof(object::WinResIDs) + + sizeof(object::WinResHeaderSuffix) + ManifestSize, + object::WIN_RES_DATA_ALIGNMENT); return MemoryBuffer::getNewMemBuffer(ResSize); } diff --git a/contrib/llvm/tools/lld/COFF/Error.cpp b/contrib/llvm/tools/lld/COFF/Error.cpp index 166b1971e77f..34abc280f6bf 100644 --- a/contrib/llvm/tools/lld/COFF/Error.cpp +++ b/contrib/llvm/tools/lld/COFF/Error.cpp @@ -29,7 +29,6 @@ namespace lld { static std::mutex Mu; namespace coff { -StringRef Argv0; uint64_t ErrorCount; raw_ostream *ErrorOS; @@ -45,7 +44,7 @@ static LLVM_ATTRIBUTE_NORETURN void exitLld(int Val) { } static void print(StringRef S, raw_ostream::Colors C) { - *ErrorOS << Argv0 + ": "; + *ErrorOS << Config->Argv[0] << ": "; if (Config->ColorDiagnostics) { ErrorOS->changeColor(C, true); *ErrorOS << S; @@ -58,7 +57,7 @@ static void print(StringRef S, raw_ostream::Colors C) { void log(const Twine &Msg) { if (Config->Verbose) { std::lock_guard Lock(Mu); - outs() << Argv0 << ": " << Msg << "\n"; + outs() << Config->Argv[0] << ": " << Msg << "\n"; outs().flush(); } } diff --git a/contrib/llvm/tools/lld/COFF/Error.h b/contrib/llvm/tools/lld/COFF/Error.h index a4f44fb1e36c..e1e4c1e5216f 100644 --- a/contrib/llvm/tools/lld/COFF/Error.h +++ b/contrib/llvm/tools/lld/COFF/Error.h @@ -18,7 +18,6 @@ namespace coff { extern uint64_t ErrorCount; extern llvm::raw_ostream *ErrorOS; -extern llvm::StringRef Argv0; void log(const Twine &Msg); void message(const Twine &Msg); diff --git a/contrib/llvm/tools/lld/COFF/InputFiles.cpp b/contrib/llvm/tools/lld/COFF/InputFiles.cpp index c26483e3e368..7d41caebb4b9 100644 --- a/contrib/llvm/tools/lld/COFF/InputFiles.cpp +++ b/contrib/llvm/tools/lld/COFF/InputFiles.cpp @@ -380,6 +380,8 @@ MachineTypes BitcodeFile::getMachineType() { return I386; case Triple::arm: return ARMNT; + case Triple::aarch64: + return ARM64; default: return IMAGE_FILE_MACHINE_UNKNOWN; } diff --git a/contrib/llvm/tools/lld/COFF/PDB.cpp b/contrib/llvm/tools/lld/COFF/PDB.cpp index c9842cfd1b9a..508f59e3af1f 100644 --- a/contrib/llvm/tools/lld/COFF/PDB.cpp +++ b/contrib/llvm/tools/lld/COFF/PDB.cpp @@ -18,19 +18,20 @@ #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h" #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" +#include "llvm/DebugInfo/CodeView/SymbolSerializer.h" #include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h" #include "llvm/DebugInfo/CodeView/TypeIndexDiscovery.h" #include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" #include "llvm/DebugInfo/CodeView/TypeTableBuilder.h" #include "llvm/DebugInfo/MSF/MSFBuilder.h" #include "llvm/DebugInfo/MSF/MSFCommon.h" +#include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h" #include "llvm/DebugInfo/PDB/Native/DbiStream.h" #include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h" #include "llvm/DebugInfo/PDB/Native/InfoStream.h" #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBFileBuilder.h" -#include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" #include "llvm/DebugInfo/PDB/Native/PDBTypeServerHandler.h" #include "llvm/DebugInfo/PDB/Native/TpiStream.h" @@ -124,26 +125,25 @@ static bool remapTypeIndex(TypeIndex &TI, ArrayRef TypeIndexMap) { return true; } -static bool remapTypesInSymbolRecord(ObjectFile *File, +static void remapTypesInSymbolRecord(ObjectFile *File, MutableArrayRef Contents, ArrayRef TypeIndexMap, ArrayRef TypeRefs) { for (const TiReference &Ref : TypeRefs) { unsigned ByteSize = Ref.Count * sizeof(TypeIndex); - if (Contents.size() < Ref.Offset + ByteSize) { - log("ignoring short symbol record"); - return false; - } + if (Contents.size() < Ref.Offset + ByteSize) + fatal("symbol record too short"); MutableArrayRef TIs( reinterpret_cast(Contents.data() + Ref.Offset), Ref.Count); - for (TypeIndex &TI : TIs) + for (TypeIndex &TI : TIs) { if (!remapTypeIndex(TI, TypeIndexMap)) { + TI = TypeIndex(SimpleTypeKind::NotTranslated); log("ignoring symbol record in " + File->getName() + " with bad type index 0x" + utohexstr(TI.getIndex())); - return false; + continue; } + } } - return true; } /// MSVC translates S_PROC_ID_END to S_END. @@ -176,6 +176,70 @@ static MutableArrayRef copySymbolForPdb(const CVSymbol &Sym, return NewData; } +/// Return true if this symbol opens a scope. This implies that the symbol has +/// "parent" and "end" fields, which contain the offset of the S_END or +/// S_INLINESITE_END record. +static bool symbolOpensScope(SymbolKind Kind) { + switch (Kind) { + case SymbolKind::S_GPROC32: + case SymbolKind::S_LPROC32: + case SymbolKind::S_LPROC32_ID: + case SymbolKind::S_GPROC32_ID: + case SymbolKind::S_BLOCK32: + case SymbolKind::S_SEPCODE: + case SymbolKind::S_THUNK32: + case SymbolKind::S_INLINESITE: + case SymbolKind::S_INLINESITE2: + return true; + default: + break; + } + return false; +} + +static bool symbolEndsScope(SymbolKind Kind) { + switch (Kind) { + case SymbolKind::S_END: + case SymbolKind::S_PROC_ID_END: + case SymbolKind::S_INLINESITE_END: + return true; + default: + break; + } + return false; +} + +struct ScopeRecord { + ulittle32_t PtrParent; + ulittle32_t PtrEnd; +}; + +struct SymbolScope { + ScopeRecord *OpeningRecord; + uint32_t ScopeOffset; +}; + +static void scopeStackOpen(SmallVectorImpl &Stack, + uint32_t CurOffset, CVSymbol &Sym) { + assert(symbolOpensScope(Sym.kind())); + SymbolScope S; + S.ScopeOffset = CurOffset; + S.OpeningRecord = const_cast( + reinterpret_cast(Sym.content().data())); + S.OpeningRecord->PtrParent = Stack.empty() ? 0 : Stack.back().ScopeOffset; + Stack.push_back(S); +} + +static void scopeStackClose(SmallVectorImpl &Stack, + uint32_t CurOffset, ObjectFile *File) { + if (Stack.empty()) { + warn("symbol scopes are not balanced in " + File->getName()); + return; + } + SymbolScope S = Stack.pop_back_val(); + S.OpeningRecord->PtrEnd = CurOffset; +} + static void mergeSymbolRecords(BumpPtrAllocator &Alloc, ObjectFile *File, ArrayRef TypeIndexMap, BinaryStreamRef SymData) { @@ -184,6 +248,7 @@ static void mergeSymbolRecords(BumpPtrAllocator &Alloc, ObjectFile *File, CVSymbolArray Syms; BinaryStreamReader Reader(SymData); ExitOnErr(Reader.readArray(Syms, Reader.getLength())); + SmallVector Scopes; for (const CVSymbol &Sym : Syms) { // Discover type index references in the record. Skip it if we don't know // where they are. @@ -199,14 +264,17 @@ static void mergeSymbolRecords(BumpPtrAllocator &Alloc, ObjectFile *File, // Re-map all the type index references. MutableArrayRef Contents = NewData.drop_front(sizeof(RecordPrefix)); - if (!remapTypesInSymbolRecord(File, Contents, TypeIndexMap, TypeRefs)) - continue; + remapTypesInSymbolRecord(File, Contents, TypeIndexMap, TypeRefs); - // FIXME: Fill in "Parent" and "End" fields by maintaining a stack of - // scopes. + // Fill in "Parent" and "End" fields by maintaining a stack of scopes. + CVSymbol NewSym(Sym.kind(), NewData); + if (symbolOpensScope(Sym.kind())) + scopeStackOpen(Scopes, File->ModuleDBI->getNextSymbolOffset(), NewSym); + else if (symbolEndsScope(Sym.kind())) + scopeStackClose(Scopes, File->ModuleDBI->getNextSymbolOffset(), File); // Add the symbol to the module. - File->ModuleDBI->addSymbol(CVSymbol(Sym.kind(), NewData)); + File->ModuleDBI->addSymbol(NewSym); } } @@ -246,7 +314,9 @@ static void addObjectsToPDB(BumpPtrAllocator &Alloc, SymbolTable *Symtab, bool InArchive = !File->ParentName.empty(); SmallString<128> Path = InArchive ? File->ParentName : File->getName(); sys::fs::make_absolute(Path); + sys::path::native(Path, llvm::sys::path::Style::windows); StringRef Name = InArchive ? File->getName() : StringRef(Path); + File->ModuleDBI = &ExitOnErr(Builder.getDbiBuilder().addModuleInfo(Name)); File->ModuleDBI->setObjFileName(Path); @@ -325,9 +395,52 @@ static void addObjectsToPDB(BumpPtrAllocator &Alloc, SymbolTable *Symtab, addTypeInfo(Builder.getIpiBuilder(), IDTable); } +static void addLinkerModuleSymbols(StringRef Path, + pdb::DbiModuleDescriptorBuilder &Mod, + BumpPtrAllocator &Allocator) { + codeview::SymbolSerializer Serializer(Allocator, CodeViewContainer::Pdb); + codeview::ObjNameSym ONS(SymbolRecordKind::ObjNameSym); + codeview::Compile3Sym CS(SymbolRecordKind::Compile3Sym); + codeview::EnvBlockSym EBS(SymbolRecordKind::EnvBlockSym); + + ONS.Name = "* Linker *"; + ONS.Signature = 0; + + CS.Machine = Config->is64() ? CPUType::X64 : CPUType::Intel80386; + CS.Flags = CompileSym3Flags::None; + CS.VersionBackendBuild = 0; + CS.VersionBackendMajor = 0; + CS.VersionBackendMinor = 0; + CS.VersionBackendQFE = 0; + CS.VersionFrontendBuild = 0; + CS.VersionFrontendMajor = 0; + CS.VersionFrontendMinor = 0; + CS.VersionFrontendQFE = 0; + CS.Version = "LLVM Linker"; + CS.setLanguage(SourceLanguage::Link); + + ArrayRef Args = makeArrayRef(Config->Argv).drop_front(); + std::string ArgStr = llvm::join(Args, " "); + EBS.Fields.push_back("cwd"); + SmallString<64> cwd; + llvm::sys::fs::current_path(cwd); + EBS.Fields.push_back(cwd); + EBS.Fields.push_back("exe"); + EBS.Fields.push_back(Config->Argv[0]); + EBS.Fields.push_back("pdb"); + EBS.Fields.push_back(Path); + EBS.Fields.push_back("cmd"); + EBS.Fields.push_back(ArgStr); + Mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( + ONS, Allocator, CodeViewContainer::Pdb)); + Mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( + CS, Allocator, CodeViewContainer::Pdb)); + Mod.addSymbol(codeview::SymbolSerializer::writeOneSymbol( + EBS, Allocator, CodeViewContainer::Pdb)); +} + // Creates a PDB file. -void coff::createPDB(StringRef Path, SymbolTable *Symtab, - ArrayRef SectionTable, +void coff::createPDB(SymbolTable *Symtab, ArrayRef SectionTable, const llvm::codeview::DebugInfo *DI) { BumpPtrAllocator Alloc; pdb::PDBFileBuilder Builder(Alloc); @@ -342,22 +455,37 @@ void coff::createPDB(StringRef Path, SymbolTable *Symtab, auto &InfoBuilder = Builder.getInfoBuilder(); InfoBuilder.setAge(DI ? DI->PDB70.Age : 0); + llvm::SmallString<128> NativePath(Config->PDBPath.begin(), + Config->PDBPath.end()); + llvm::sys::fs::make_absolute(NativePath); + llvm::sys::path::native(NativePath, llvm::sys::path::Style::windows); + pdb::PDB_UniqueId uuid{}; if (DI) memcpy(&uuid, &DI->PDB70.Signature, sizeof(uuid)); InfoBuilder.setGuid(uuid); - // Should be the current time, but set 0 for reproducibilty. - InfoBuilder.setSignature(0); + InfoBuilder.setSignature(time(nullptr)); InfoBuilder.setVersion(pdb::PdbRaw_ImplVer::PdbImplVC70); - // Add an empty DPI stream. + // Add an empty DBI stream. pdb::DbiStreamBuilder &DbiBuilder = Builder.getDbiBuilder(); - DbiBuilder.setVersionHeader(pdb::PdbDbiV110); + DbiBuilder.setVersionHeader(pdb::PdbDbiV70); + ExitOnErr(DbiBuilder.addDbgStream(pdb::DbgHeaderType::NewFPO, {})); + + // It's not entirely clear what this is, but the * Linker * module uses it. + uint32_t PdbFilePathNI = DbiBuilder.addECName(NativePath); TypeTableBuilder TypeTable(BAlloc); TypeTableBuilder IDTable(BAlloc); addObjectsToPDB(Alloc, Symtab, Builder, TypeTable, IDTable); + // Add public and symbol records stream. + + // For now we don't actually write any thing useful to the publics stream, but + // the act of "getting" it also creates it lazily so that we write an empty + // stream. + (void)Builder.getPublicsBuilder(); + // Add Section Contributions. addSectionContribs(Symtab, DbiBuilder); @@ -369,12 +497,14 @@ void coff::createPDB(StringRef Path, SymbolTable *Symtab, pdb::DbiStreamBuilder::createSectionMap(Sections); DbiBuilder.setSectionMap(SectionMap); - ExitOnErr(DbiBuilder.addModuleInfo("* Linker *")); + auto &LinkerModule = ExitOnErr(DbiBuilder.addModuleInfo("* Linker *")); + LinkerModule.setPdbFilePathNI(PdbFilePathNI); + addLinkerModuleSymbols(NativePath, LinkerModule, Alloc); // Add COFF section header stream. ExitOnErr( DbiBuilder.addDbgStream(pdb::DbgHeaderType::SectionHdr, SectionTable)); // Write to a file. - ExitOnErr(Builder.commit(Path)); + ExitOnErr(Builder.commit(Config->PDBPath)); } diff --git a/contrib/llvm/tools/lld/COFF/PDB.h b/contrib/llvm/tools/lld/COFF/PDB.h index c9c37914299a..9aaa3178df21 100644 --- a/contrib/llvm/tools/lld/COFF/PDB.h +++ b/contrib/llvm/tools/lld/COFF/PDB.h @@ -23,8 +23,7 @@ namespace lld { namespace coff { class SymbolTable; -void createPDB(llvm::StringRef Path, SymbolTable *Symtab, - llvm::ArrayRef SectionTable, +void createPDB(SymbolTable *Symtab, llvm::ArrayRef SectionTable, const llvm::codeview::DebugInfo *DI); } } diff --git a/contrib/llvm/tools/lld/COFF/Symbols.cpp b/contrib/llvm/tools/lld/COFF/Symbols.cpp index 1cf2934a355b..9b59079072a8 100644 --- a/contrib/llvm/tools/lld/COFF/Symbols.cpp +++ b/contrib/llvm/tools/lld/COFF/Symbols.cpp @@ -68,6 +68,8 @@ static Chunk *makeImportThunk(DefinedImportData *S, uint16_t Machine) { return make(S); if (Machine == I386) return make(S); + if (Machine == ARM64) + return make(S); assert(Machine == ARMNT); return make(S); } diff --git a/contrib/llvm/tools/lld/COFF/Writer.cpp b/contrib/llvm/tools/lld/COFF/Writer.cpp index 4cf718a48d8b..a6a5e278498a 100644 --- a/contrib/llvm/tools/lld/COFF/Writer.cpp +++ b/contrib/llvm/tools/lld/COFF/Writer.cpp @@ -239,7 +239,7 @@ void Writer::run() { const llvm::codeview::DebugInfo *DI = nullptr; if (Config->DebugTypes & static_cast(coff::DebugType::CV)) DI = BuildId->DI; - createPDB(Config->PDBPath, Symtab, SectionTable, DI); + createPDB(Symtab, SectionTable, DI); } writeMapFile(OutputSections); diff --git a/contrib/llvm/tools/lld/ELF/Config.h b/contrib/llvm/tools/lld/ELF/Config.h index 32e86b0ec7b6..5e3b77637316 100644 --- a/contrib/llvm/tools/lld/ELF/Config.h +++ b/contrib/llvm/tools/lld/ELF/Config.h @@ -97,6 +97,7 @@ struct Configuration { llvm::StringRef ThinLTOCacheDir; std::string Rpath; std::vector VersionDefinitions; + std::vector Argv; std::vector AuxiliaryList; std::vector SearchPaths; std::vector SymbolOrderingFile; diff --git a/contrib/llvm/tools/lld/ELF/Driver.cpp b/contrib/llvm/tools/lld/ELF/Driver.cpp index 5fb33caea46f..10ad13f214d5 100644 --- a/contrib/llvm/tools/lld/ELF/Driver.cpp +++ b/contrib/llvm/tools/lld/ELF/Driver.cpp @@ -74,13 +74,13 @@ bool elf::link(ArrayRef Args, bool CanExitEarly, raw_ostream &Error) { ErrorCount = 0; ErrorOS = &Error; - Argv0 = Args[0]; InputSections.clear(); Tar = nullptr; Config = make(); Driver = make(); Script = make(); + Config->Argv = {Args.begin(), Args.end()}; Driver->main(Args, CanExitEarly); freeArena(); diff --git a/contrib/llvm/tools/lld/ELF/Error.cpp b/contrib/llvm/tools/lld/ELF/Error.cpp index 7a58668bdcc0..224570ea7424 100644 --- a/contrib/llvm/tools/lld/ELF/Error.cpp +++ b/contrib/llvm/tools/lld/ELF/Error.cpp @@ -27,7 +27,6 @@ using namespace lld::elf; uint64_t elf::ErrorCount; raw_ostream *elf::ErrorOS; -StringRef elf::Argv0; // The functions defined in this file can be called from multiple threads, // but outs() or errs() are not thread-safe. We protect them using a mutex. @@ -46,7 +45,7 @@ static void newline(const Twine &Msg) { } static void print(StringRef S, raw_ostream::Colors C) { - *ErrorOS << Argv0 + ": "; + *ErrorOS << Config->Argv[0] << ": "; if (Config->ColorDiagnostics) { ErrorOS->changeColor(C, true); *ErrorOS << S; @@ -59,7 +58,7 @@ static void print(StringRef S, raw_ostream::Colors C) { void elf::log(const Twine &Msg) { if (Config->Verbose) { std::lock_guard Lock(Mu); - outs() << Argv0 << ": " << Msg << "\n"; + outs() << Config->Argv[0] << ": " << Msg << "\n"; outs().flush(); } } diff --git a/contrib/llvm/tools/lld/ELF/Error.h b/contrib/llvm/tools/lld/ELF/Error.h index dd6e37c99b15..89bc2111b44e 100644 --- a/contrib/llvm/tools/lld/ELF/Error.h +++ b/contrib/llvm/tools/lld/ELF/Error.h @@ -37,7 +37,6 @@ namespace elf { extern uint64_t ErrorCount; extern llvm::raw_ostream *ErrorOS; -extern llvm::StringRef Argv0; void log(const Twine &Msg); void message(const Twine &Msg); diff --git a/contrib/llvm/tools/lld/ELF/Filesystem.cpp b/contrib/llvm/tools/lld/ELF/Filesystem.cpp index b63d521a83b0..d468ae0c618a 100644 --- a/contrib/llvm/tools/lld/ELF/Filesystem.cpp +++ b/contrib/llvm/tools/lld/ELF/Filesystem.cpp @@ -38,7 +38,8 @@ using namespace lld::elf; // This function spawns a background thread to call unlink. // The calling thread returns almost immediately. void elf::unlinkAsync(StringRef Path) { - if (!Config->Threads || !sys::fs::exists(Config->OutputFile)) + if (!Config->Threads || !sys::fs::exists(Config->OutputFile) || + !sys::fs::is_regular_file(Config->OutputFile)) return; // First, rename Path to avoid race condition. We cannot remove diff --git a/contrib/llvm/tools/lld/ELF/GdbIndex.h b/contrib/llvm/tools/lld/ELF/GdbIndex.h index 527667f7280e..c49f8946e199 100644 --- a/contrib/llvm/tools/lld/ELF/GdbIndex.h +++ b/contrib/llvm/tools/lld/ELF/GdbIndex.h @@ -45,6 +45,7 @@ struct NameTypeEntry { // debug information performed. That information futher used // for filling gdb index section areas. struct GdbIndexChunk { + InputSection *DebugInfoSec; std::vector AddressArea; std::vector CompilationUnits; std::vector NamesAndTypes; diff --git a/contrib/llvm/tools/lld/ELF/InputFiles.cpp b/contrib/llvm/tools/lld/ELF/InputFiles.cpp index e07f24d665df..d3c307d5cb6b 100644 --- a/contrib/llvm/tools/lld/ELF/InputFiles.cpp +++ b/contrib/llvm/tools/lld/ELF/InputFiles.cpp @@ -45,14 +45,11 @@ namespace { // LLVM DWARF parser will not be able to parse .debug_line correctly, unless // we assign each section some unique address. This callback method assigns // each section an address equal to its offset in ELF object file. -class ObjectInfo : public LoadedObjectInfo { +class ObjectInfo : public LoadedObjectInfoHelper { public: uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const override { return static_cast(Sec).getOffset(); } - std::unique_ptr clone() const override { - return std::unique_ptr(); - } }; } diff --git a/contrib/llvm/tools/lld/ELF/InputSection.cpp b/contrib/llvm/tools/lld/ELF/InputSection.cpp index b1d5e1349460..c6a539b8dfa5 100644 --- a/contrib/llvm/tools/lld/ELF/InputSection.cpp +++ b/contrib/llvm/tools/lld/ELF/InputSection.cpp @@ -276,7 +276,9 @@ template std::string InputSectionBase::getSrcMsg(uint64_t Offset) { template std::string InputSectionBase::getObjMsg(uint64_t Off) { // Synthetic sections don't have input files. elf::ObjectFile *File = getFile(); - std::string Filename = File ? File->getName() : "(internal)"; + if (!File) + return ("(internal):(" + Name + "+0x" + utohexstr(Off) + ")").str(); + std::string Filename = File->getName(); std::string Archive; if (!File->ArchiveName.empty()) @@ -466,7 +468,7 @@ static uint64_t getAArch64UndefinedRelativeWeakVA(uint64_t Type, uint64_t A, static uint64_t getARMStaticBase(const SymbolBody &Body) { OutputSection *OS = Body.getOutputSection(); if (!OS || !OS->FirstInPtLoad) - fatal("SBREL relocation to " + Body.getName() + " without static base\n"); + fatal("SBREL relocation to " + Body.getName() + " without static base"); return OS->FirstInPtLoad->Addr; } diff --git a/contrib/llvm/tools/lld/ELF/LinkerScript.cpp b/contrib/llvm/tools/lld/ELF/LinkerScript.cpp index d369a6f978a2..a182d5a3a096 100644 --- a/contrib/llvm/tools/lld/ELF/LinkerScript.cpp +++ b/contrib/llvm/tools/lld/ELF/LinkerScript.cpp @@ -111,17 +111,13 @@ LinkerScript::getOrCreateOutputSectionCommand(StringRef Name) { void LinkerScript::setDot(Expr E, const Twine &Loc, bool InSec) { uint64_t Val = E().getValue(); - if (Val < Dot) { - if (InSec) - error(Loc + ": unable to move location counter backward for: " + - CurOutSec->Name); - else - error(Loc + ": unable to move location counter backward"); - } + if (Val < Dot && InSec) + error(Loc + ": unable to move location counter backward for: " + + CurAddressState->OutSec->Name); Dot = Val; // Update to location counter means update to section size. if (InSec) - CurOutSec->Size = Dot - CurOutSec->Addr; + CurAddressState->OutSec->Size = Dot - CurAddressState->OutSec->Addr; } // Sets value of a symbol. Two kinds of symbols are processed: synthetic @@ -373,7 +369,13 @@ void LinkerScript::processCommands(OutputSectionFactory &Factory) { // which will map to whatever the first actual section is. Aether = make("", 0, SHF_ALLOC); Aether->SectionIndex = 1; - CurOutSec = Aether; + auto State = make_unique(Opt); + // CurAddressState captures the local AddressState and makes it accessible + // deliberately. This is needed as there are some cases where we cannot just + // thread the current state through to a lambda function created by the + // script parser. + CurAddressState = State.get(); + CurAddressState->OutSec = Aether; Dot = 0; for (size_t I = 0; I < Opt.Commands.size(); ++I) { @@ -435,7 +437,7 @@ void LinkerScript::processCommands(OutputSectionFactory &Factory) { } } } - CurOutSec = nullptr; + CurAddressState = nullptr; } void LinkerScript::fabricateDefaultCommands() { @@ -481,20 +483,31 @@ void LinkerScript::fabricateDefaultCommands() { // Add sections that didn't match any sections command. void LinkerScript::addOrphanSections(OutputSectionFactory &Factory) { + unsigned NumCommands = Opt.Commands.size(); for (InputSectionBase *S : InputSections) { if (!S->Live || S->Parent) continue; StringRef Name = getOutputSectionName(S->Name); - auto I = std::find_if( - Opt.Commands.begin(), Opt.Commands.end(), [&](BaseCommand *Base) { - if (auto *Cmd = dyn_cast(Base)) - return Cmd->Name == Name; - return false; - }); - if (I == Opt.Commands.end()) { + auto End = Opt.Commands.begin() + NumCommands; + auto I = std::find_if(Opt.Commands.begin(), End, [&](BaseCommand *Base) { + if (auto *Cmd = dyn_cast(Base)) + return Cmd->Name == Name; + return false; + }); + OutputSectionCommand *Cmd; + if (I == End) { Factory.addInputSec(S, Name); + OutputSection *Sec = S->getOutputSection(); + assert(Sec->SectionIndex == INT_MAX); + OutputSectionCommand *&CmdRef = SecToCommand[Sec]; + if (!CmdRef) { + CmdRef = createOutputSectionCommand(Sec->Name, ""); + CmdRef->Sec = Sec; + Opt.Commands.push_back(CmdRef); + } + Cmd = CmdRef; } else { - auto *Cmd = cast(*I); + Cmd = cast(*I); Factory.addInputSec(S, Name, Cmd->Sec); if (OutputSection *Sec = Cmd->Sec) { SecToCommand[Sec] = Cmd; @@ -502,21 +515,22 @@ void LinkerScript::addOrphanSections(OutputSectionFactory &Factory) { assert(Sec->SectionIndex == INT_MAX || Sec->SectionIndex == Index); Sec->SectionIndex = Index; } - auto *ISD = make(""); - ISD->Sections.push_back(cast(S)); - Cmd->Commands.push_back(ISD); } + auto *ISD = make(""); + ISD->Sections.push_back(cast(S)); + Cmd->Commands.push_back(ISD); } } uint64_t LinkerScript::advance(uint64_t Size, unsigned Align) { - bool IsTbss = (CurOutSec->Flags & SHF_TLS) && CurOutSec->Type == SHT_NOBITS; - uint64_t Start = IsTbss ? Dot + ThreadBssOffset : Dot; + bool IsTbss = (CurAddressState->OutSec->Flags & SHF_TLS) && + CurAddressState->OutSec->Type == SHT_NOBITS; + uint64_t Start = IsTbss ? Dot + CurAddressState->ThreadBssOffset : Dot; Start = alignTo(Start, Align); uint64_t End = Start + Size; if (IsTbss) - ThreadBssOffset = End - Dot; + CurAddressState->ThreadBssOffset = End - Dot; else Dot = End; return End; @@ -524,40 +538,43 @@ uint64_t LinkerScript::advance(uint64_t Size, unsigned Align) { void LinkerScript::output(InputSection *S) { uint64_t Pos = advance(S->getSize(), S->Alignment); - S->OutSecOff = Pos - S->getSize() - CurOutSec->Addr; + S->OutSecOff = Pos - S->getSize() - CurAddressState->OutSec->Addr; // Update output section size after adding each section. This is so that // SIZEOF works correctly in the case below: // .foo { *(.aaa) a = SIZEOF(.foo); *(.bbb) } - CurOutSec->Size = Pos - CurOutSec->Addr; + CurAddressState->OutSec->Size = Pos - CurAddressState->OutSec->Addr; // If there is a memory region associated with this input section, then // place the section in that region and update the region index. - if (CurMemRegion) { - CurMemRegion->Offset += CurOutSec->Size; - uint64_t CurSize = CurMemRegion->Offset - CurMemRegion->Origin; - if (CurSize > CurMemRegion->Length) { - uint64_t OverflowAmt = CurSize - CurMemRegion->Length; - error("section '" + CurOutSec->Name + "' will not fit in region '" + - CurMemRegion->Name + "': overflowed by " + Twine(OverflowAmt) + - " bytes"); + if (CurAddressState->MemRegion) { + uint64_t &CurOffset = + CurAddressState->MemRegionOffset[CurAddressState->MemRegion]; + CurOffset += CurAddressState->OutSec->Size; + uint64_t CurSize = CurOffset - CurAddressState->MemRegion->Origin; + if (CurSize > CurAddressState->MemRegion->Length) { + uint64_t OverflowAmt = CurSize - CurAddressState->MemRegion->Length; + error("section '" + CurAddressState->OutSec->Name + + "' will not fit in region '" + CurAddressState->MemRegion->Name + + "': overflowed by " + Twine(OverflowAmt) + " bytes"); } } } void LinkerScript::switchTo(OutputSection *Sec) { - if (CurOutSec == Sec) + if (CurAddressState->OutSec == Sec) return; - CurOutSec = Sec; - CurOutSec->Addr = advance(0, CurOutSec->Alignment); + CurAddressState->OutSec = Sec; + CurAddressState->OutSec->Addr = + advance(0, CurAddressState->OutSec->Alignment); // If neither AT nor AT> is specified for an allocatable section, the linker // will set the LMA such that the difference between VMA and LMA for the // section is the same as the preceding output section in the same region // https://sourceware.org/binutils/docs-2.20/ld/Output-Section-LMA.html - if (LMAOffset) - CurOutSec->LMAOffset = LMAOffset(); + if (CurAddressState->LMAOffset) + CurAddressState->OutSec->LMAOffset = CurAddressState->LMAOffset(); } void LinkerScript::process(BaseCommand &Base) { @@ -569,9 +586,9 @@ void LinkerScript::process(BaseCommand &Base) { // Handle BYTE(), SHORT(), LONG(), or QUAD(). if (auto *Cmd = dyn_cast(&Base)) { - Cmd->Offset = Dot - CurOutSec->Addr; + Cmd->Offset = Dot - CurAddressState->OutSec->Addr; Dot += Cmd->Size; - CurOutSec->Size = Dot - CurOutSec->Addr; + CurAddressState->OutSec->Size = Dot - CurAddressState->OutSec->Addr; return; } @@ -596,7 +613,7 @@ void LinkerScript::process(BaseCommand &Base) { if (!Sec->Live) continue; - assert(CurOutSec == Sec->getParent()); + assert(CurAddressState->OutSec == Sec->getParent()); output(Sec); } } @@ -649,17 +666,17 @@ void LinkerScript::assignOffsets(OutputSectionCommand *Cmd) { if (Cmd->LMAExpr) { uint64_t D = Dot; - LMAOffset = [=] { return Cmd->LMAExpr().getValue() - D; }; + CurAddressState->LMAOffset = [=] { return Cmd->LMAExpr().getValue() - D; }; } - CurMemRegion = Cmd->MemRegion; - if (CurMemRegion) - Dot = CurMemRegion->Offset; + CurAddressState->MemRegion = Cmd->MemRegion; + if (CurAddressState->MemRegion) + Dot = CurAddressState->MemRegionOffset[CurAddressState->MemRegion]; switchTo(Sec); // We do not support custom layout for compressed debug sectons. // At this point we already know their size and have compressed content. - if (CurOutSec->Flags & SHF_COMPRESSED) + if (CurAddressState->OutSec->Flags & SHF_COMPRESSED) return; for (BaseCommand *C : Cmd->Commands) @@ -746,30 +763,20 @@ void LinkerScript::adjustSectionsAfterSorting() { if (!Cmd) continue; - if (Cmd->Phdrs.empty()) - Cmd->Phdrs = DefPhdrs; - else + if (Cmd->Phdrs.empty()) { + OutputSection *Sec = Cmd->Sec; + // To match the bfd linker script behaviour, only propagate program + // headers to sections that are allocated. + if (Sec && (Sec->Flags & SHF_ALLOC)) + Cmd->Phdrs = DefPhdrs; + } else { DefPhdrs = Cmd->Phdrs; + } } removeEmptyCommands(); } -void LinkerScript::createOrphanCommands() { - for (OutputSection *Sec : OutputSections) { - if (Sec->SectionIndex != INT_MAX) - continue; - OutputSectionCommand *Cmd = - createOutputSectionCommand(Sec->Name, ""); - Cmd->Sec = Sec; - SecToCommand[Sec] = Cmd; - auto *ISD = make(""); - ISD->Sections = Sec->Sections; - Cmd->Commands.push_back(ISD); - Opt.Commands.push_back(Cmd); - } -} - void LinkerScript::processNonSectionCommands() { for (BaseCommand *Base : Opt.Commands) { if (auto *Cmd = dyn_cast(Base)) @@ -779,22 +786,25 @@ void LinkerScript::processNonSectionCommands() { } } -static bool -allocateHeaders(std::vector &Phdrs, - ArrayRef OutputSectionCommands, - uint64_t Min) { - auto FirstPTLoad = - std::find_if(Phdrs.begin(), Phdrs.end(), - [](const PhdrEntry &E) { return E.p_type == PT_LOAD; }); +void LinkerScript::allocateHeaders(std::vector &Phdrs) { + uint64_t Min = std::numeric_limits::max(); + for (OutputSectionCommand *Cmd : OutputSectionCommands) { + OutputSection *Sec = Cmd->Sec; + if (Sec->Flags & SHF_ALLOC) + Min = std::min(Min, Sec->Addr); + } + + auto FirstPTLoad = llvm::find_if( + Phdrs, [](const PhdrEntry &E) { return E.p_type == PT_LOAD; }); if (FirstPTLoad == Phdrs.end()) - return false; + return; uint64_t HeaderSize = getHeaderSize(); if (HeaderSize <= Min || Script->hasPhdrsCommands()) { Min = alignDown(Min - HeaderSize, Config->MaxPageSize); Out::ElfHeader->Addr = Min; Out::ProgramHeaders->Addr = Min + Out::ElfHeader->Size; - return true; + return; } assert(FirstPTLoad->First == Out::ElfHeader); @@ -817,17 +827,28 @@ allocateHeaders(std::vector &Phdrs, Phdrs.erase(FirstPTLoad); } - auto PhdrI = std::find_if(Phdrs.begin(), Phdrs.end(), [](const PhdrEntry &E) { - return E.p_type == PT_PHDR; - }); + auto PhdrI = llvm::find_if( + Phdrs, [](const PhdrEntry &E) { return E.p_type == PT_PHDR; }); if (PhdrI != Phdrs.end()) Phdrs.erase(PhdrI); - return false; } -void LinkerScript::assignAddresses(std::vector &Phdrs) { +LinkerScript::AddressState::AddressState(const ScriptConfiguration &Opt) { + for (auto &MRI : Opt.MemoryRegions) { + const MemoryRegion *MR = &MRI.second; + MemRegionOffset[MR] = MR->Origin; + } +} + +void LinkerScript::assignAddresses() { // Assign addresses as instructed by linker script SECTIONS sub-commands. Dot = 0; + auto State = make_unique(Opt); + // CurAddressState captures the local AddressState and makes it accessible + // deliberately. This is needed as there are some cases where we cannot just + // thread the current state through to a lambda function created by the + // script parser. + CurAddressState = State.get(); ErrorOnMissingSection = true; switchTo(Aether); @@ -845,15 +866,7 @@ void LinkerScript::assignAddresses(std::vector &Phdrs) { auto *Cmd = cast(Base); assignOffsets(Cmd); } - - uint64_t MinVA = std::numeric_limits::max(); - for (OutputSectionCommand *Cmd : OutputSectionCommands) { - OutputSection *Sec = Cmd->Sec; - if (Sec->Flags & SHF_ALLOC) - MinVA = std::min(MinVA, Sec->Addr); - } - - allocateHeaders(Phdrs, OutputSectionCommands, MinVA); + CurAddressState = nullptr; } // Creates program headers as instructed by PHDRS linker script command. @@ -879,12 +892,9 @@ std::vector LinkerScript::createPhdrs() { // Add output sections to program headers. for (OutputSectionCommand *Cmd : OutputSectionCommands) { - OutputSection *Sec = Cmd->Sec; - if (!(Sec->Flags & SHF_ALLOC)) - break; - // Assign headers specified by linker script - for (size_t Id : getPhdrIndices(Sec)) { + for (size_t Id : getPhdrIndices(Cmd)) { + OutputSection *Sec = Cmd->Sec; Ret[Id].add(Sec); if (Opt.PhdrsCommands[Id].Flags == UINT_MAX) Ret[Id].p_flags |= Sec->getPhdrFlags(); @@ -911,6 +921,92 @@ OutputSectionCommand *LinkerScript::getCmd(OutputSection *Sec) const { return I->second; } +void OutputSectionCommand::sort(std::function Order) { + typedef std::pair Pair; + auto Comp = [](const Pair &A, const Pair &B) { return A.first < B.first; }; + + std::vector V; + assert(Commands.size() == 1); + auto *ISD = cast(Commands[0]); + for (InputSection *S : ISD->Sections) + V.push_back({Order(S), S}); + std::stable_sort(V.begin(), V.end(), Comp); + ISD->Sections.clear(); + for (Pair &P : V) + ISD->Sections.push_back(P.second); +} + +// Returns true if S matches /Filename.?\.o$/. +static bool isCrtBeginEnd(StringRef S, StringRef Filename) { + if (!S.endswith(".o")) + return false; + S = S.drop_back(2); + if (S.endswith(Filename)) + return true; + return !S.empty() && S.drop_back().endswith(Filename); +} + +static bool isCrtbegin(StringRef S) { return isCrtBeginEnd(S, "crtbegin"); } +static bool isCrtend(StringRef S) { return isCrtBeginEnd(S, "crtend"); } + +// .ctors and .dtors are sorted by this priority from highest to lowest. +// +// 1. The section was contained in crtbegin (crtbegin contains +// some sentinel value in its .ctors and .dtors so that the runtime +// can find the beginning of the sections.) +// +// 2. The section has an optional priority value in the form of ".ctors.N" +// or ".dtors.N" where N is a number. Unlike .{init,fini}_array, +// they are compared as string rather than number. +// +// 3. The section is just ".ctors" or ".dtors". +// +// 4. The section was contained in crtend, which contains an end marker. +// +// In an ideal world, we don't need this function because .init_array and +// .ctors are duplicate features (and .init_array is newer.) However, there +// are too many real-world use cases of .ctors, so we had no choice to +// support that with this rather ad-hoc semantics. +static bool compCtors(const InputSection *A, const InputSection *B) { + bool BeginA = isCrtbegin(A->File->getName()); + bool BeginB = isCrtbegin(B->File->getName()); + if (BeginA != BeginB) + return BeginA; + bool EndA = isCrtend(A->File->getName()); + bool EndB = isCrtend(B->File->getName()); + if (EndA != EndB) + return EndB; + StringRef X = A->Name; + StringRef Y = B->Name; + assert(X.startswith(".ctors") || X.startswith(".dtors")); + assert(Y.startswith(".ctors") || Y.startswith(".dtors")); + X = X.substr(6); + Y = Y.substr(6); + if (X.empty() && Y.empty()) + return false; + return X < Y; +} + +// Sorts input sections by the special rules for .ctors and .dtors. +// Unfortunately, the rules are different from the one for .{init,fini}_array. +// Read the comment above. +void OutputSectionCommand::sortCtorsDtors() { + assert(Commands.size() == 1); + auto *ISD = cast(Commands[0]); + std::stable_sort(ISD->Sections.begin(), ISD->Sections.end(), compCtors); +} + +// Sorts input sections by section name suffixes, so that .foo.N comes +// before .foo.M if N < M. Used to sort .{init,fini}_array.N sections. +// We want to keep the original order if the priorities are the same +// because the compiler keeps the original initialization order in a +// translation unit and we need to respect that. +// For more detail, read the section of the GCC's manual about init_priority. +void OutputSectionCommand::sortInitFini() { + // Sort sections by priority. + sort([](InputSectionBase *S) { return getPriority(S->Name); }); +} + uint32_t OutputSectionCommand::getFiller() { if (Filler) return *Filler; @@ -1085,16 +1181,9 @@ template void OutputSectionCommand::writeTo(uint8_t *Buf) { writeInt(Buf + Data->Offset, Data->Expression().getValue(), Data->Size); } -bool LinkerScript::hasLMA(OutputSection *Sec) { - if (OutputSectionCommand *Cmd = getCmd(Sec)) - if (Cmd->LMAExpr) - return true; - return false; -} - ExprValue LinkerScript::getSymbolValue(const Twine &Loc, StringRef S) { if (S == ".") - return {CurOutSec, Dot - CurOutSec->Addr, Loc}; + return {CurAddressState->OutSec, Dot - CurAddressState->OutSec->Addr, Loc}; if (SymbolBody *B = findSymbol(S)) { if (auto *D = dyn_cast(B)) return {D->Section, D->Value, Loc}; @@ -1111,17 +1200,14 @@ static const size_t NoPhdr = -1; // Returns indices of ELF headers containing specific section. Each index is a // zero based number of ELF header listed within PHDRS {} script block. -std::vector LinkerScript::getPhdrIndices(OutputSection *Sec) { - if (OutputSectionCommand *Cmd = getCmd(Sec)) { - std::vector Ret; - for (StringRef PhdrName : Cmd->Phdrs) { - size_t Index = getPhdrIndex(Cmd->Location, PhdrName); - if (Index != NoPhdr) - Ret.push_back(Index); - } - return Ret; +std::vector LinkerScript::getPhdrIndices(OutputSectionCommand *Cmd) { + std::vector Ret; + for (StringRef PhdrName : Cmd->Phdrs) { + size_t Index = getPhdrIndex(Cmd->Location, PhdrName); + if (Index != NoPhdr) + Ret.push_back(Index); } - return {}; + return Ret; } // Returns the index of the segment named PhdrName if found otherwise diff --git a/contrib/llvm/tools/lld/ELF/LinkerScript.h b/contrib/llvm/tools/lld/ELF/LinkerScript.h index f8a34a1e97dd..dd5a7d797f60 100644 --- a/contrib/llvm/tools/lld/ELF/LinkerScript.h +++ b/contrib/llvm/tools/lld/ELF/LinkerScript.h @@ -110,7 +110,6 @@ struct MemoryRegion { std::string Name; uint64_t Origin; uint64_t Length; - uint64_t Offset; uint32_t Flags; uint32_t NegFlags; }; @@ -140,6 +139,10 @@ struct OutputSectionCommand : BaseCommand { template void writeTo(uint8_t *Buf); template void maybeCompress(); uint32_t getFiller(); + + void sort(std::function Order); + void sortInitFini(); + void sortCtorsDtors(); }; // This struct represents one section match pattern in SECTIONS() command. @@ -222,6 +225,17 @@ struct ScriptConfiguration { }; class LinkerScript final { + // Temporary state used in processCommands() and assignAddresses() + // that must be reinitialized for each call to the above functions, and must + // not be used outside of the scope of a call to the above functions. + struct AddressState { + uint64_t ThreadBssOffset = 0; + OutputSection *OutSec = nullptr; + MemoryRegion *MemRegion = nullptr; + llvm::DenseMap MemRegionOffset; + std::function LMAOffset; + AddressState(const ScriptConfiguration &Opt); + }; llvm::DenseMap SecToCommand; llvm::DenseMap NameToOutputSectionCommand; @@ -234,7 +248,7 @@ class LinkerScript final { std::vector createInputSectionList(OutputSectionCommand &Cmd); - std::vector getPhdrIndices(OutputSection *Sec); + std::vector getPhdrIndices(OutputSectionCommand *Cmd); size_t getPhdrIndex(const Twine &Loc, StringRef PhdrName); MemoryRegion *findMemoryRegion(OutputSectionCommand *Cmd); @@ -244,14 +258,10 @@ class LinkerScript final { void output(InputSection *Sec); void process(BaseCommand &Base); + AddressState *CurAddressState = nullptr; OutputSection *Aether; uint64_t Dot; - uint64_t ThreadBssOffset = 0; - - std::function LMAOffset; - OutputSection *CurOutSec = nullptr; - MemoryRegion *CurMemRegion = nullptr; public: bool ErrorOnMissingSection = false; @@ -276,13 +286,11 @@ class LinkerScript final { std::vector createPhdrs(); bool ignoreInterpSection(); - bool hasLMA(OutputSection *Sec); bool shouldKeep(InputSectionBase *S); void assignOffsets(OutputSectionCommand *Cmd); - void createOrphanCommands(); void processNonSectionCommands(); - void assignAddresses(std::vector &Phdrs); - + void assignAddresses(); + void allocateHeaders(std::vector &Phdrs); void addSymbol(SymbolAssignment *Cmd); void processCommands(OutputSectionFactory &Factory); diff --git a/contrib/llvm/tools/lld/ELF/OutputSections.cpp b/contrib/llvm/tools/lld/ELF/OutputSections.cpp index c0bf6b32e6e2..d6ae5dcae167 100644 --- a/contrib/llvm/tools/lld/ELF/OutputSections.cpp +++ b/contrib/llvm/tools/lld/ELF/OutputSections.cpp @@ -101,100 +101,6 @@ void OutputSection::addSection(InputSection *S) { this->Entsize = std::max(this->Entsize, S->Entsize); } -// This function is called after we sort input sections -// and scan relocations to setup sections' offsets. -void OutputSection::assignOffsets() { - OutputSectionCommand *Cmd = Script->getCmd(this); - uint64_t Off = 0; - for (BaseCommand *Base : Cmd->Commands) - if (auto *ISD = dyn_cast(Base)) - for (InputSection *S : ISD->Sections) - Off = updateOffset(Off, S); - this->Size = Off; -} - -void OutputSection::sort(std::function Order) { - typedef std::pair Pair; - auto Comp = [](const Pair &A, const Pair &B) { return A.first < B.first; }; - - std::vector V; - for (InputSection *S : Sections) - V.push_back({Order(S), S}); - std::stable_sort(V.begin(), V.end(), Comp); - Sections.clear(); - for (Pair &P : V) - Sections.push_back(P.second); -} - -// Sorts input sections by section name suffixes, so that .foo.N comes -// before .foo.M if N < M. Used to sort .{init,fini}_array.N sections. -// We want to keep the original order if the priorities are the same -// because the compiler keeps the original initialization order in a -// translation unit and we need to respect that. -// For more detail, read the section of the GCC's manual about init_priority. -void OutputSection::sortInitFini() { - // Sort sections by priority. - sort([](InputSectionBase *S) { return getPriority(S->Name); }); -} - -// Returns true if S matches /Filename.?\.o$/. -static bool isCrtBeginEnd(StringRef S, StringRef Filename) { - if (!S.endswith(".o")) - return false; - S = S.drop_back(2); - if (S.endswith(Filename)) - return true; - return !S.empty() && S.drop_back().endswith(Filename); -} - -static bool isCrtbegin(StringRef S) { return isCrtBeginEnd(S, "crtbegin"); } -static bool isCrtend(StringRef S) { return isCrtBeginEnd(S, "crtend"); } - -// .ctors and .dtors are sorted by this priority from highest to lowest. -// -// 1. The section was contained in crtbegin (crtbegin contains -// some sentinel value in its .ctors and .dtors so that the runtime -// can find the beginning of the sections.) -// -// 2. The section has an optional priority value in the form of ".ctors.N" -// or ".dtors.N" where N is a number. Unlike .{init,fini}_array, -// they are compared as string rather than number. -// -// 3. The section is just ".ctors" or ".dtors". -// -// 4. The section was contained in crtend, which contains an end marker. -// -// In an ideal world, we don't need this function because .init_array and -// .ctors are duplicate features (and .init_array is newer.) However, there -// are too many real-world use cases of .ctors, so we had no choice to -// support that with this rather ad-hoc semantics. -static bool compCtors(const InputSection *A, const InputSection *B) { - bool BeginA = isCrtbegin(A->File->getName()); - bool BeginB = isCrtbegin(B->File->getName()); - if (BeginA != BeginB) - return BeginA; - bool EndA = isCrtend(A->File->getName()); - bool EndB = isCrtend(B->File->getName()); - if (EndA != EndB) - return EndB; - StringRef X = A->Name; - StringRef Y = B->Name; - assert(X.startswith(".ctors") || X.startswith(".dtors")); - assert(Y.startswith(".ctors") || Y.startswith(".dtors")); - X = X.substr(6); - Y = Y.substr(6); - if (X.empty() && Y.empty()) - return false; - return X < Y; -} - -// Sorts input sections by the special rules for .ctors and .dtors. -// Unfortunately, the rules are different from the one for .{init,fini}_array. -// Read the comment above. -void OutputSection::sortCtorsDtors() { - std::stable_sort(Sections.begin(), Sections.end(), compCtors); -} - static SectionKey createKey(InputSectionBase *C, StringRef OutsecName) { // The ELF spec just says // ---------------------------------------------------------------- @@ -249,9 +155,7 @@ static SectionKey createKey(InputSectionBase *C, StringRef OutsecName) { return SectionKey{OutsecName, Flags, Alignment}; } -OutputSectionFactory::OutputSectionFactory( - std::vector &OutputSections) - : OutputSections(OutputSections) {} +OutputSectionFactory::OutputSectionFactory() {} static uint64_t getIncompatibleFlags(uint64_t Flags) { return Flags & (SHF_ALLOC | SHF_TLS); diff --git a/contrib/llvm/tools/lld/ELF/OutputSections.h b/contrib/llvm/tools/lld/ELF/OutputSections.h index d5f77838d530..68ee066a13da 100644 --- a/contrib/llvm/tools/lld/ELF/OutputSections.h +++ b/contrib/llvm/tools/lld/ELF/OutputSections.h @@ -80,10 +80,6 @@ class OutputSection final : public SectionBase { uint32_t ShName = 0; void addSection(InputSection *S); - void sort(std::function Order); - void sortInitFini(); - void sortCtorsDtors(); - void assignOffsets(); std::vector Sections; // Used for implementation of --compress-debug-sections option. @@ -135,7 +131,7 @@ namespace elf { // linker scripts. class OutputSectionFactory { public: - OutputSectionFactory(std::vector &OutputSections); + OutputSectionFactory(); ~OutputSectionFactory(); void addInputSec(InputSectionBase *IS, StringRef OutsecName); @@ -144,7 +140,6 @@ class OutputSectionFactory { private: llvm::SmallDenseMap Map; - std::vector &OutputSections; }; uint64_t getHeaderSize(); diff --git a/contrib/llvm/tools/lld/ELF/Relocations.cpp b/contrib/llvm/tools/lld/ELF/Relocations.cpp index fd823fe0ed42..52dbe4b583d0 100644 --- a/contrib/llvm/tools/lld/ELF/Relocations.cpp +++ b/contrib/llvm/tools/lld/ELF/Relocations.cpp @@ -557,9 +557,9 @@ static RelExpr adjustExpr(SymbolBody &Body, RelExpr Expr, uint32_t Type, // the refered symbol can be preemepted to refer to the executable. if (Config->Shared || (Config->Pic && !isRelExpr(Expr))) { error("can't create dynamic relocation " + toString(Type) + " against " + - (Body.getName().empty() ? "local symbol in readonly segment" + (Body.getName().empty() ? "local symbol" : "symbol: " + toString(Body)) + - getLocation(S, Body, RelOff)); + " in readonly segment" + getLocation(S, Body, RelOff)); return Expr; } @@ -1049,10 +1049,17 @@ ThunkSection *ThunkCreator::addThunkSection(OutputSection *OS, std::pair ThunkCreator::getThunk(SymbolBody &Body, uint32_t Type) { - auto res = ThunkedSymbols.insert({&Body, nullptr}); - if (res.second) - res.first->second = addThunk(Type, Body); - return std::make_pair(res.first->second, res.second); + auto Res = ThunkedSymbols.insert({&Body, std::vector()}); + if (!Res.second) { + // Check existing Thunks for Body to see if they can be reused + for (Thunk *ET : Res.first->second) + if (ET->isCompatibleWith(Type)) + return std::make_pair(ET, false); + } + // No existing compatible Thunk in range, create a new one + Thunk *T = addThunk(Type, Body); + Res.first->second.push_back(T); + return std::make_pair(T, true); } // Call Fn on every executable InputSection accessed via the linker script @@ -1066,13 +1073,12 @@ void ThunkCreator::forEachExecInputSection( OutputSection *OS = Cmd->Sec; if (!(OS->Flags & SHF_ALLOC) || !(OS->Flags & SHF_EXECINSTR)) continue; - if (OutputSectionCommand *C = Script->getCmd(OS)) - for (BaseCommand *BC : C->Commands) - if (auto *ISD = dyn_cast(BC)) { - CurTS = nullptr; - for (InputSection* IS : ISD->Sections) - Fn(OS, &ISD->Sections, IS); - } + for (BaseCommand *BC : Cmd->Commands) + if (auto *ISD = dyn_cast(BC)) { + CurTS = nullptr; + for (InputSection *IS : ISD->Sections) + Fn(OS, &ISD->Sections, IS); + } } } diff --git a/contrib/llvm/tools/lld/ELF/Relocations.h b/contrib/llvm/tools/lld/ELF/Relocations.h index 445308b27cec..fc3e3444ac24 100644 --- a/contrib/llvm/tools/lld/ELF/Relocations.h +++ b/contrib/llvm/tools/lld/ELF/Relocations.h @@ -144,14 +144,17 @@ class ThunkCreator { std::pair getThunk(SymbolBody &Body, uint32_t Type); ThunkSection *addThunkSection(OutputSection *OS, std::vector *, uint64_t Off); - // Track Symbols that already have a Thunk - llvm::DenseMap ThunkedSymbols; + // Record all the available Thunks for a Symbol + llvm::DenseMap> ThunkedSymbols; // Find a Thunk from the Thunks symbol definition, we can use this to find // the Thunk from a relocation to the Thunks symbol definition. llvm::DenseMap Thunks; - // Track InputSections that have a ThunkSection placed in front + // Track InputSections that have an inline ThunkSection placed in front + // an inline ThunkSection may have control fall through to the section below + // so we need to make sure that there is only one of them. + // The Mips LA25 Thunk is an example of an inline ThunkSection. llvm::DenseMap ThunkedSections; // All the ThunkSections that we have created, organised by OutputSection diff --git a/contrib/llvm/tools/lld/ELF/ScriptParser.cpp b/contrib/llvm/tools/lld/ELF/ScriptParser.cpp index 4a44944fe7ed..72940ca0cfd4 100644 --- a/contrib/llvm/tools/lld/ELF/ScriptParser.cpp +++ b/contrib/llvm/tools/lld/ELF/ScriptParser.cpp @@ -1191,8 +1191,7 @@ void ScriptParser::readMemory() { if (It != Script->Opt.MemoryRegions.end()) setError("region '" + Name + "' already defined"); else - Script->Opt.MemoryRegions[Name] = {Name, Origin, Length, - Origin, Flags, NegFlags}; + Script->Opt.MemoryRegions[Name] = {Name, Origin, Length, Flags, NegFlags}; } } diff --git a/contrib/llvm/tools/lld/ELF/SymbolTable.cpp b/contrib/llvm/tools/lld/ELF/SymbolTable.cpp index 0b5450b071f2..c802d74b8ff8 100644 --- a/contrib/llvm/tools/lld/ELF/SymbolTable.cpp +++ b/contrib/llvm/tools/lld/ELF/SymbolTable.cpp @@ -211,6 +211,13 @@ static uint8_t getMinVisibility(uint8_t VA, uint8_t VB) { // Find an existing symbol or create and insert a new one. template std::pair SymbolTable::insert(StringRef Name) { + // @@ means the symbol is the default version. In that + // case symbol must exist and @@ will be used to + // resolve references to . + size_t Pos = Name.find("@@"); + if (Pos != StringRef::npos) + Name = Name.take_front(Pos); + auto P = Symtab.insert( {CachedHashStringRef(Name), SymIndex((int)SymVector.size(), false)}); SymIndex &V = P.first->second; @@ -319,7 +326,7 @@ static int compareDefined(Symbol *S, bool WasInserted, uint8_t Binding) { if (WasInserted) return 1; SymbolBody *Body = S->body(); - if (Body->isLazy() || !Body->isInCurrentDSO()) + if (!Body->isInCurrentDSO()) return 1; if (Binding == STB_WEAK) return -1; @@ -689,6 +696,12 @@ void SymbolTable::assignExactVersion(SymbolVersion Ver, uint16_t VersionId // Assign the version. for (SymbolBody *B : Syms) { + // Skip symbols containing version info because symbol versions + // specified by symbol names take precedence over version scripts. + // See parseSymbolVersion(). + if (B->getName().find('@') != StringRef::npos) + continue; + Symbol *Sym = B->symbol(); if (Sym->InVersionScript) warn("duplicate symbol '" + Ver.Name + "' in version script"); @@ -702,12 +715,11 @@ void SymbolTable::assignWildcardVersion(SymbolVersion Ver, uint16_t VersionId) { if (!Ver.HasWildcard) return; - std::vector Syms = findAllByVersion(Ver); // Exact matching takes precendence over fuzzy matching, // so we set a version to a symbol only if no version has been assigned // to the symbol. This behavior is compatible with GNU. - for (SymbolBody *B : Syms) + for (SymbolBody *B : findAllByVersion(Ver)) if (B->symbol()->VersionId == Config->DefaultSymbolVersion) B->symbol()->VersionId = VersionId; } @@ -715,18 +727,9 @@ void SymbolTable::assignWildcardVersion(SymbolVersion Ver, // This function processes version scripts by updating VersionId // member of symbols. template void SymbolTable::scanVersionScript() { - // Symbol themselves might know their versions because symbols - // can contain versions in the form of @. - // Let them parse their names. - if (!Config->VersionDefinitions.empty()) - for (Symbol *Sym : SymVector) - Sym->body()->parseSymbolVersion(); - // Handle edge cases first. handleAnonymousVersion(); - if (Config->VersionDefinitions.empty()) - return; // Now we have version definitions, so we need to set version ids to symbols. // Each version definition has a glob pattern, and all symbols that match @@ -745,6 +748,12 @@ template void SymbolTable::scanVersionScript() { for (VersionDefinition &V : llvm::reverse(Config->VersionDefinitions)) for (SymbolVersion &Ver : V.Globals) assignWildcardVersion(Ver, V.Id); + + // Symbol themselves might know their versions because symbols + // can contain versions in the form of @. + // Let them parse and update their names to exclude version suffix. + for (Symbol *Sym : SymVector) + Sym->body()->parseSymbolVersion(); } template class elf::SymbolTable; diff --git a/contrib/llvm/tools/lld/ELF/Symbols.cpp b/contrib/llvm/tools/lld/ELF/Symbols.cpp index e8cd662c69ac..1d17f57f0c30 100644 --- a/contrib/llvm/tools/lld/ELF/Symbols.cpp +++ b/contrib/llvm/tools/lld/ELF/Symbols.cpp @@ -159,19 +159,12 @@ bool SymbolBody::isPreemptible() const { return true; } -// Overwrites all attributes except symbol name with Other's so that -// this symbol becomes an alias to Other. This is useful for handling -// some options such as --wrap. -// -// The reason why we want to keep the symbol name is because, if we -// copy symbol names, we'll end up having symbol tables in resulting -// executables or DSOs containing two or more identical symbols, which -// is just inconvenient. +// Overwrites all attributes with Other's so that this symbol becomes +// an alias to Other. This is useful for handling some options such as +// --wrap. void SymbolBody::copy(SymbolBody *Other) { - StringRef S = Name; memcpy(symbol()->Body.buffer, Other->symbol()->Body.buffer, sizeof(Symbol::Body)); - Name = S; } uint64_t SymbolBody::getVA(int64_t Addend) const { @@ -272,7 +265,12 @@ void SymbolBody::parseSymbolVersion() { } // It is an error if the specified version is not defined. - error(toString(File) + ": symbol " + S + " has undefined version " + Verstr); + // Usually version script is not provided when linking executable, + // but we may still want to override a versioned symbol from DSO, + // so we do not report error in this case. + if (Config->Shared) + error(toString(File) + ": symbol " + S + " has undefined version " + + Verstr); } Defined::Defined(Kind K, StringRefZ Name, bool IsLocal, uint8_t StOther, diff --git a/contrib/llvm/tools/lld/ELF/Symbols.h b/contrib/llvm/tools/lld/ELF/Symbols.h index 773e1ad9588a..a1b3a6fba911 100644 --- a/contrib/llvm/tools/lld/ELF/Symbols.h +++ b/contrib/llvm/tools/lld/ELF/Symbols.h @@ -65,7 +65,9 @@ class SymbolBody { return SymbolKind == LazyArchiveKind || SymbolKind == LazyObjectKind; } bool isShared() const { return SymbolKind == SharedKind; } - bool isInCurrentDSO() const { return !isUndefined() && !isShared(); } + bool isInCurrentDSO() const { + return !isUndefined() && !isShared() && !isLazy(); + } bool isLocal() const { return IsLocal; } bool isPreemptible() const; StringRef getName() const { return Name; } @@ -218,7 +220,7 @@ class SharedSymbol : public Defined { Verdef(Verdef), ElfSym(ElfSym) { // IFuncs defined in DSOs are treated as functions by the static linker. if (isGnuIFunc()) - Type = llvm::ELF::STT_FUNC; + this->Type = llvm::ELF::STT_FUNC; this->File = File; } diff --git a/contrib/llvm/tools/lld/ELF/SyntheticSections.cpp b/contrib/llvm/tools/lld/ELF/SyntheticSections.cpp index 995d05692ee2..fd724fac327c 100644 --- a/contrib/llvm/tools/lld/ELF/SyntheticSections.cpp +++ b/contrib/llvm/tools/lld/ELF/SyntheticSections.cpp @@ -1071,10 +1071,11 @@ template void DynamicSection::finalizeContents() { return; // Already finalized. this->Link = InX::DynStrTab->getParent()->SectionIndex; - if (In::RelaDyn->getParent()->Size > 0) { + if (In::RelaDyn->getParent() && !In::RelaDyn->empty()) { bool IsRela = Config->IsRela; add({IsRela ? DT_RELA : DT_REL, In::RelaDyn}); - add({IsRela ? DT_RELASZ : DT_RELSZ, In::RelaDyn->getParent()->Size}); + add({IsRela ? DT_RELASZ : DT_RELSZ, In::RelaDyn->getParent(), + Entry::SecSize}); add({IsRela ? DT_RELAENT : DT_RELENT, uint64_t(IsRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel))}); @@ -1087,9 +1088,9 @@ template void DynamicSection::finalizeContents() { add({IsRela ? DT_RELACOUNT : DT_RELCOUNT, NumRelativeRels}); } } - if (In::RelaPlt->getParent()->Size > 0) { + if (In::RelaPlt->getParent() && !In::RelaPlt->empty()) { add({DT_JMPREL, In::RelaPlt}); - add({DT_PLTRELSZ, In::RelaPlt->getParent()->Size}); + add({DT_PLTRELSZ, In::RelaPlt->getParent(), Entry::SecSize}); switch (Config->EMachine) { case EM_MIPS: add({DT_MIPS_PLTGOT, In::GotPlt}); @@ -1699,9 +1700,9 @@ unsigned PltSection::getPltRelocOff() const { return (HeaderSize == 0) ? InX::Plt->getSize() : 0; } -GdbIndexSection::GdbIndexSection() +GdbIndexSection::GdbIndexSection(std::vector &&Chunks) : SyntheticSection(0, SHT_PROGBITS, 1, ".gdb_index"), - StringPool(llvm::StringTableBuilder::ELF) {} + StringPool(llvm::StringTableBuilder::ELF), Chunks(std::move(Chunks)) {} // Iterative hash function for symbol's name is described in .gdb_index format // specification. Note that we use one for version 5 to 7 here, it is different @@ -1713,11 +1714,10 @@ static uint32_t hash(StringRef Str) { return R; } -static std::vector readCuList(DWARFContext &Dwarf, - InputSection *Sec) { +static std::vector readCuList(DWARFContext &Dwarf) { std::vector Ret; for (std::unique_ptr &CU : Dwarf.compile_units()) - Ret.push_back({Sec->OutSecOff + CU->getOffset(), CU->getLength() + 4}); + Ret.push_back({CU->getOffset(), CU->getLength() + 4}); return Ret; } @@ -1764,19 +1764,15 @@ static std::vector getDebugInfoSections() { std::vector Ret; for (InputSectionBase *S : InputSections) if (InputSection *IS = dyn_cast(S)) - if (IS->getParent() && IS->Name == ".debug_info") + if (IS->Name == ".debug_info") Ret.push_back(IS); return Ret; } void GdbIndexSection::buildIndex() { - std::vector V = getDebugInfoSections(); - if (V.empty()) + if (Chunks.empty()) return; - for (InputSection *Sec : V) - Chunks.push_back(readDwarf(Sec)); - uint32_t CuId = 0; for (GdbIndexChunk &D : Chunks) { for (AddressEntry &E : D.AddressArea) @@ -1802,23 +1798,33 @@ void GdbIndexSection::buildIndex() { } } -GdbIndexChunk GdbIndexSection::readDwarf(InputSection *Sec) { - Expected> Obj = - object::ObjectFile::createObjectFile(Sec->File->MB); - if (!Obj) { - error(toString(Sec->File) + ": error creating DWARF context"); - return {}; - } - - DWARFContextInMemory Dwarf(*Obj.get()); - +static GdbIndexChunk readDwarf(DWARFContextInMemory &Dwarf, InputSection *Sec) { GdbIndexChunk Ret; - Ret.CompilationUnits = readCuList(Dwarf, Sec); + Ret.DebugInfoSec = Sec; + Ret.CompilationUnits = readCuList(Dwarf); Ret.AddressArea = readAddressArea(Dwarf, Sec); Ret.NamesAndTypes = readPubNamesAndTypes(Dwarf, Config->IsLE); return Ret; } +template GdbIndexSection *elf::createGdbIndex() { + std::vector Chunks; + for (InputSection *Sec : getDebugInfoSections()) { + InputFile *F = Sec->File; + std::error_code EC; + ELFObjectFile Obj(F->MB, EC); + if (EC) + fatal(EC.message()); + DWARFContextInMemory Dwarf(Obj, nullptr, [&](Error E) { + error(toString(F) + ": error parsing DWARF data:\n>>> " + + toString(std::move(E))); + return ErrorPolicy::Continue; + }); + Chunks.push_back(readDwarf(Dwarf, Sec)); + } + return make(std::move(Chunks)); +} + static size_t getCuSize(std::vector &C) { size_t Ret = 0; for (GdbIndexChunk &D : C) @@ -1876,7 +1882,7 @@ void GdbIndexSection::writeTo(uint8_t *Buf) { // Write the CU list. for (GdbIndexChunk &D : Chunks) { for (CompilationUnitEntry &Cu : D.CompilationUnits) { - write64le(Buf, Cu.CuOffset); + write64le(Buf, D.DebugInfoSec->OutSecOff + Cu.CuOffset); write64le(Buf + 8, Cu.CuLength); Buf += 16; } @@ -2345,6 +2351,11 @@ StringTableSection *InX::ShStrTab; StringTableSection *InX::StrTab; SymbolTableBaseSection *InX::SymTab; +template GdbIndexSection *elf::createGdbIndex(); +template GdbIndexSection *elf::createGdbIndex(); +template GdbIndexSection *elf::createGdbIndex(); +template GdbIndexSection *elf::createGdbIndex(); + template void PltSection::addEntry(SymbolBody &Sym); template void PltSection::addEntry(SymbolBody &Sym); template void PltSection::addEntry(SymbolBody &Sym); diff --git a/contrib/llvm/tools/lld/ELF/SyntheticSections.h b/contrib/llvm/tools/lld/ELF/SyntheticSections.h index be9a43c8155b..ddd8ca99a61b 100644 --- a/contrib/llvm/tools/lld/ELF/SyntheticSections.h +++ b/contrib/llvm/tools/lld/ELF/SyntheticSections.h @@ -503,7 +503,7 @@ class GdbIndexSection final : public SyntheticSection { const unsigned SymTabEntrySize = 2 * OffsetTypeSize; public: - GdbIndexSection(); + GdbIndexSection(std::vector &&Chunks); void finalizeContents() override; void writeTo(uint8_t *Buf) override; size_t getSize() const override; @@ -524,7 +524,6 @@ class GdbIndexSection final : public SyntheticSection { std::vector Chunks; private: - GdbIndexChunk readDwarf(InputSection *Sec); void buildIndex(); uint32_t CuTypesOffset; @@ -538,6 +537,8 @@ class GdbIndexSection final : public SyntheticSection { bool Finalized = false; }; +template GdbIndexSection *createGdbIndex(); + // --eh-frame-hdr option tells linker to construct a header for all the // .eh_frame sections. This header is placed to a section named .eh_frame_hdr // and also to a PT_GNU_EH_FRAME segment. diff --git a/contrib/llvm/tools/lld/ELF/Thunks.cpp b/contrib/llvm/tools/lld/ELF/Thunks.cpp index 752a881d7867..cae31027e557 100644 --- a/contrib/llvm/tools/lld/ELF/Thunks.cpp +++ b/contrib/llvm/tools/lld/ELF/Thunks.cpp @@ -57,6 +57,7 @@ class ARMV7ABSLongThunk final : public Thunk { uint32_t size() const override { return 12; } void writeTo(uint8_t *Buf, ThunkSection &IS) const override; void addSymbols(ThunkSection &IS) override; + bool isCompatibleWith(uint32_t RelocType) const override; }; class ARMV7PILongThunk final : public Thunk { @@ -66,28 +67,31 @@ class ARMV7PILongThunk final : public Thunk { uint32_t size() const override { return 16; } void writeTo(uint8_t *Buf, ThunkSection &IS) const override; void addSymbols(ThunkSection &IS) override; + bool isCompatibleWith(uint32_t RelocType) const override; }; class ThumbV7ABSLongThunk final : public Thunk { public: ThumbV7ABSLongThunk(const SymbolBody &Dest) : Thunk(Dest) { - this->alignment = 2; + alignment = 2; } uint32_t size() const override { return 10; } void writeTo(uint8_t *Buf, ThunkSection &IS) const override; void addSymbols(ThunkSection &IS) override; + bool isCompatibleWith(uint32_t RelocType) const override; }; class ThumbV7PILongThunk final : public Thunk { public: ThumbV7PILongThunk(const SymbolBody &Dest) : Thunk(Dest) { - this->alignment = 2; + alignment = 2; } uint32_t size() const override { return 12; } void writeTo(uint8_t *Buf, ThunkSection &IS) const override; void addSymbols(ThunkSection &IS) override; + bool isCompatibleWith(uint32_t RelocType) const override; }; // MIPS LA25 thunk @@ -128,6 +132,11 @@ void ARMV7ABSLongThunk::addSymbols(ThunkSection &IS) { addSyntheticLocal("$a", STT_NOTYPE, Offset, 0, &IS); } +bool ARMV7ABSLongThunk::isCompatibleWith(uint32_t RelocType) const { + // Thumb branch relocations can't use BLX + return RelocType != R_ARM_THM_JUMP19 && RelocType != R_ARM_THM_JUMP24; +} + void ThumbV7ABSLongThunk::writeTo(uint8_t *Buf, ThunkSection &IS) const { const uint8_t Data[] = { 0x40, 0xf2, 0x00, 0x0c, // movw ip, :lower16:S @@ -147,6 +156,12 @@ void ThumbV7ABSLongThunk::addSymbols(ThunkSection &IS) { addSyntheticLocal("$t", STT_NOTYPE, Offset, 0, &IS); } +bool ThumbV7ABSLongThunk::isCompatibleWith(uint32_t RelocType) const { + // ARM branch relocations can't use BLX + return RelocType != R_ARM_JUMP24 && RelocType != R_ARM_PC24 && + RelocType != R_ARM_PLT32; +} + void ARMV7PILongThunk::writeTo(uint8_t *Buf, ThunkSection &IS) const { const uint8_t Data[] = { 0xf0, 0xcf, 0x0f, 0xe3, // P: movw ip,:lower16:S - (P + (L1-P) +8) @@ -168,6 +183,11 @@ void ARMV7PILongThunk::addSymbols(ThunkSection &IS) { addSyntheticLocal("$a", STT_NOTYPE, Offset, 0, &IS); } +bool ARMV7PILongThunk::isCompatibleWith(uint32_t RelocType) const { + // Thumb branch relocations can't use BLX + return RelocType != R_ARM_THM_JUMP19 && RelocType != R_ARM_THM_JUMP24; +} + void ThumbV7PILongThunk::writeTo(uint8_t *Buf, ThunkSection &IS) const { const uint8_t Data[] = { 0x4f, 0xf6, 0xf4, 0x7c, // P: movw ip,:lower16:S - (P + (L1-P) + 4) @@ -189,9 +209,15 @@ void ThumbV7PILongThunk::addSymbols(ThunkSection &IS) { addSyntheticLocal("$t", STT_NOTYPE, Offset, 0, &IS); } +bool ThumbV7PILongThunk::isCompatibleWith(uint32_t RelocType) const { + // ARM branch relocations can't use BLX + return RelocType != R_ARM_JUMP24 && RelocType != R_ARM_PC24 && + RelocType != R_ARM_PLT32; +} + // Write MIPS LA25 thunk code to call PIC function from the non-PIC one. void MipsThunk::writeTo(uint8_t *Buf, ThunkSection &) const { - uint64_t S = this->Destination.getVA(); + uint64_t S = Destination.getVA(); write32(Buf, 0x3c190000, Config->Endianness); // lui $25, %hi(func) write32(Buf + 4, 0x08000000 | (S >> 2), Config->Endianness); // j func write32(Buf + 8, 0x27390000, Config->Endianness); // addiu $25, $25, %lo(func) diff --git a/contrib/llvm/tools/lld/ELF/Thunks.h b/contrib/llvm/tools/lld/ELF/Thunks.h index 38ee090e75e1..00b6b2cf2994 100644 --- a/contrib/llvm/tools/lld/ELF/Thunks.h +++ b/contrib/llvm/tools/lld/ELF/Thunks.h @@ -41,6 +41,10 @@ class Thunk { // a branch and fall through to the first Symbol in the Target. virtual InputSection *getTargetInputSection() const { return nullptr; } + // To reuse a Thunk the caller as identified by the RelocType must be + // compatible with it. + virtual bool isCompatibleWith(uint32_t RelocType) const { return true; } + // The alignment requirement for this Thunk, defaults to the size of the // typical code section alignment. const SymbolBody &Destination; diff --git a/contrib/llvm/tools/lld/ELF/Writer.cpp b/contrib/llvm/tools/lld/ELF/Writer.cpp index 551ca260de10..bf43ee5c5f91 100644 --- a/contrib/llvm/tools/lld/ELF/Writer.cpp +++ b/contrib/llvm/tools/lld/ELF/Writer.cpp @@ -73,13 +73,12 @@ template class Writer { std::unique_ptr Buffer; - OutputSectionFactory Factory{OutputSections}; + OutputSectionFactory Factory; void addRelIpltSymbols(); void addStartEndSymbols(); void addStartStopSymbols(OutputSection *Sec); uint64_t getEntryAddr(); - OutputSection *findSection(StringRef Name); OutputSection *findSectionInScript(StringRef Name); OutputSectionCommand *findSectionCommand(StringRef Name); @@ -152,10 +151,6 @@ template static void combineEhFrameSections() { } template void Writer::clearOutputSections() { - if (Script->Opt.HasSections) - Script->createOrphanCommands(); - else - Script->fabricateDefaultCommands(); // Clear the OutputSections to make sure it is not used anymore. Any // code from this point on should be using the linker script // commands. @@ -190,9 +185,10 @@ template void Writer::run() { // output sections by default rules. We still need to give the // linker script a chance to run, because it might contain // non-SECTIONS commands such as ASSERT. - createSections(); Script->processCommands(Factory); + createSections(); } + clearOutputSections(); if (Config->Discard != DiscardPolicy::All) copyLocalSymbols(); @@ -218,7 +214,8 @@ template void Writer::run() { OutputSectionCommands.begin(), OutputSectionCommands.end(), [](OutputSectionCommand *Cmd) { Cmd->maybeCompress(); }); - Script->assignAddresses(Phdrs); + Script->assignAddresses(); + Script->allocateHeaders(Phdrs); // Remove empty PT_LOAD to avoid causing the dynamic linker to try to mmap a // 0 sized region. This has to be done late since only after assignAddresses @@ -383,7 +380,7 @@ template void Writer::createSyntheticSections() { Add(InX::IgotPlt); if (Config->GdbIndex) { - InX::GdbIndex = make(); + InX::GdbIndex = createGdbIndex(); Add(InX::GdbIndex); } @@ -499,11 +496,18 @@ template void Writer::copyLocalSymbols() { template void Writer::addSectionSymbols() { // Create one STT_SECTION symbol for each output section we might // have a relocation with. - for (OutputSection *Sec : OutputSections) { - if (Sec->Sections.empty()) + for (BaseCommand *Base : Script->Opt.Commands) { + auto *Cmd = dyn_cast(Base); + if (!Cmd) continue; - - InputSection *IS = Sec->Sections[0]; + auto I = llvm::find_if(Cmd->Commands, [](BaseCommand *Base) { + if (auto *ISD = dyn_cast(Base)) + return !ISD->Sections.empty(); + return false; + }); + if (I == Cmd->Commands.end()) + continue; + InputSection *IS = cast(*I)->Sections[0]; if (isa(IS) || IS->Type == SHT_REL || IS->Type == SHT_RELA) continue; @@ -864,20 +868,19 @@ template void Writer::addReservedSymbols() { // Sort input sections by section name suffixes for // __attribute__((init_priority(N))). -static void sortInitFini(OutputSection *S) { - if (S) - reinterpret_cast(S)->sortInitFini(); +static void sortInitFini(OutputSectionCommand *Cmd) { + if (Cmd) + Cmd->sortInitFini(); } // Sort input sections by the special rule for .ctors and .dtors. -static void sortCtorsDtors(OutputSection *S) { - if (S) - reinterpret_cast(S)->sortCtorsDtors(); +static void sortCtorsDtors(OutputSectionCommand *Cmd) { + if (Cmd) + Cmd->sortCtorsDtors(); } // Sort input sections using the list provided by --symbol-ordering-file. -template -static void sortBySymbolsOrder(ArrayRef OutputSections) { +template static void sortBySymbolsOrder() { if (Config->SymbolOrderingFile.empty()) return; @@ -902,9 +905,9 @@ static void sortBySymbolsOrder(ArrayRef OutputSections) { } // Sort sections by priority. - for (OutputSection *Base : OutputSections) - if (auto *Sec = dyn_cast(Base)) - Sec->sort([&](InputSectionBase *S) { return SectionOrder.lookup(S); }); + for (BaseCommand *Base : Script->Opt.Commands) + if (auto *Cmd = dyn_cast(Base)) + Cmd->sort([&](InputSectionBase *S) { return SectionOrder.lookup(S); }); } template @@ -934,11 +937,12 @@ template void Writer::createSections() { if (IS) Factory.addInputSec(IS, getOutputSectionName(IS->Name)); - sortBySymbolsOrder(OutputSections); - sortInitFini(findSection(".init_array")); - sortInitFini(findSection(".fini_array")); - sortCtorsDtors(findSection(".ctors")); - sortCtorsDtors(findSection(".dtors")); + Script->fabricateDefaultCommands(); + sortBySymbolsOrder(); + sortInitFini(findSectionCommand(".init_array")); + sortInitFini(findSectionCommand(".fini_array")); + sortCtorsDtors(findSectionCommand(".ctors")); + sortCtorsDtors(findSectionCommand(".dtors")); } // We want to find how similar two ranks are. @@ -1132,7 +1136,7 @@ static void applySynthetic(const std::vector &Sections, // to make them visible from linkescript side. But not all sections are always // required to be in output. For example we don't need dynamic section content // sometimes. This function filters out such unused sections from the output. -static void removeUnusedSyntheticSections(std::vector &V) { +static void removeUnusedSyntheticSections() { // All input synthetic sections that can be empty are placed after // all regular ones. We iterate over them all and exit at first // non-synthetic. @@ -1145,12 +1149,25 @@ static void removeUnusedSyntheticSections(std::vector &V) { continue; if ((SS == InX::Got || SS == InX::MipsGot) && ElfSym::GlobalOffsetTable) continue; - OS->Sections.erase(std::find(OS->Sections.begin(), OS->Sections.end(), SS)); - SS->Live = false; + + OutputSectionCommand *Cmd = Script->getCmd(OS); + std::vector::iterator Empty = Cmd->Commands.end(); + for (auto I = Cmd->Commands.begin(), E = Cmd->Commands.end(); I != E; ++I) { + BaseCommand *B = *I; + if (auto *ISD = dyn_cast(B)) { + auto P = std::find(ISD->Sections.begin(), ISD->Sections.end(), SS); + if (P != ISD->Sections.end()) + ISD->Sections.erase(P); + if (ISD->Sections.empty()) + Empty = I; + } + } + if (Empty != Cmd->Commands.end()) + Cmd->Commands.erase(Empty); + // If there are no other sections in the output section, remove it from the // output. - if (OS->Sections.empty()) { - V.erase(std::find(V.begin(), V.end(), OS)); + if (Cmd->Commands.empty()) { // Also remove script commands matching the output section. auto &Cmds = Script->Opt.Commands; auto I = std::remove_if(Cmds.begin(), Cmds.end(), [&](BaseCommand *Cmd) { @@ -1165,18 +1182,20 @@ static void removeUnusedSyntheticSections(std::vector &V) { // Create output section objects and add them to OutputSections. template void Writer::finalizeSections() { - Out::DebugInfo = findSection(".debug_info"); - Out::PreinitArray = findSection(".preinit_array"); - Out::InitArray = findSection(".init_array"); - Out::FiniArray = findSection(".fini_array"); + Out::DebugInfo = findSectionInScript(".debug_info"); + Out::PreinitArray = findSectionInScript(".preinit_array"); + Out::InitArray = findSectionInScript(".init_array"); + Out::FiniArray = findSectionInScript(".fini_array"); // The linker needs to define SECNAME_start, SECNAME_end and SECNAME_stop // symbols for sections, so that the runtime can get the start and end // addresses of each section by section name. Add such symbols. if (!Config->Relocatable) { addStartEndSymbols(); - for (OutputSection *Sec : OutputSections) - addStartStopSymbols(Sec); + for (BaseCommand *Base : Script->Opt.Commands) + if (auto *Cmd = dyn_cast(Base)) + if (Cmd->Sec) + addStartStopSymbols(Cmd->Sec); } // Add _DYNAMIC symbol. Unlike GNU gold, our _DYNAMIC symbol has no type. @@ -1227,9 +1246,8 @@ template void Writer::finalizeSections() { return; addPredefinedSections(); - removeUnusedSyntheticSections(OutputSections); + removeUnusedSyntheticSections(); - clearOutputSections(); sortSections(); // Now that we have the final list, create a list of all the @@ -1266,12 +1284,6 @@ template void Writer::finalizeSections() { Out::ProgramHeaders->Size = sizeof(Elf_Phdr) * Phdrs.size(); } - // Compute the size of .rela.dyn and .rela.plt early since we need - // them to populate .dynamic. - for (SyntheticSection *SS : {In::RelaDyn, In::RelaPlt}) - if (SS->getParent() && !SS->empty()) - SS->getParent()->assignOffsets(); - // Dynamic section must be the last one in this list and dynamic // symbol table section (DynSymTab) must be the first one. applySynthetic({InX::DynSymTab, InX::Bss, InX::BssRelRo, @@ -1295,6 +1307,7 @@ template void Writer::finalizeSections() { // are out of range. This will need to turn into a loop that converges // when no more Thunks are added ThunkCreator TC; + Script->assignAddresses(); if (TC.createThunks(OutputSectionCommands)) { applySynthetic({InX::MipsGot}, [](SyntheticSection *SS) { SS->updateAllocSize(); }); @@ -1317,21 +1330,18 @@ template void Writer::finalizeSections() { template void Writer::addPredefinedSections() { // ARM ABI requires .ARM.exidx to be terminated by some piece of data. // We have the terminater synthetic section class. Add that at the end. - auto *OS = dyn_cast_or_null(findSection(".ARM.exidx")); - if (!OS || OS->Sections.empty() || Config->Relocatable) + OutputSectionCommand *Cmd = findSectionCommand(".ARM.exidx"); + if (!Cmd || Cmd->Commands.empty() || Config->Relocatable) return; auto *Sentinel = make(); - OS->addSection(Sentinel); - // If there are linker script commands existing at this point then add the - // sentinel to the last of these too. - if (OutputSectionCommand *C = Script->getCmd(OS)) { - auto ISD = std::find_if(C->Commands.rbegin(), C->Commands.rend(), - [](const BaseCommand *Base) { - return isa(Base); - }); - cast(*ISD)->Sections.push_back(Sentinel); - } + Cmd->Sec->addSection(Sentinel); + // Add the sentinel to the last of these too. + auto ISD = std::find_if(Cmd->Commands.rbegin(), Cmd->Commands.rend(), + [](const BaseCommand *Base) { + return isa(Base); + }); + cast(*ISD)->Sections.push_back(Sentinel); } // The linker is expected to define SECNAME_start and SECNAME_end @@ -1355,7 +1365,7 @@ template void Writer::addStartEndSymbols() { Define("__init_array_start", "__init_array_end", Out::InitArray); Define("__fini_array_start", "__fini_array_end", Out::FiniArray); - if (OutputSection *Sec = findSection(".ARM.exidx")) + if (OutputSection *Sec = findSectionInScript(".ARM.exidx")) Define("__exidx_start", "__exidx_end", Sec); } @@ -1375,9 +1385,10 @@ void Writer::addStartStopSymbols(OutputSection *Sec) { template OutputSectionCommand *Writer::findSectionCommand(StringRef Name) { - for (OutputSectionCommand *Cmd : OutputSectionCommands) - if (Cmd->Name == Name) - return Cmd; + for (BaseCommand *Base : Script->Opt.Commands) + if (auto *Cmd = dyn_cast(Base)) + if (Cmd->Name == Name) + return Cmd; return nullptr; } @@ -1387,13 +1398,6 @@ template OutputSection *Writer::findSectionInScript(StringRef return nullptr; } -template OutputSection *Writer::findSection(StringRef Name) { - for (OutputSection *Sec : OutputSections) - if (Sec->Name == Name) - return Sec; - return nullptr; -} - static bool needsPtLoad(OutputSection *Sec) { if (!(Sec->Flags & SHF_ALLOC)) return false; @@ -1455,7 +1459,7 @@ template std::vector Writer::createPhdrs() { // different flags or is loaded at a discontiguous address using AT linker // script command. uint64_t NewFlags = computeFlags(Sec->getPhdrFlags()); - if (Script->hasLMA(Sec) || Flags != NewFlags) { + if (Cmd->LMAExpr || Flags != NewFlags) { Load = AddHdr(PT_LOAD, NewFlags); Flags = NewFlags; } @@ -1523,7 +1527,7 @@ template std::vector Writer::createPhdrs() { for (OutputSectionCommand *Cmd : OutputSectionCommands) { OutputSection *Sec = Cmd->Sec; if (Sec->Type == SHT_NOTE) { - if (!Note || Script->hasLMA(Sec)) + if (!Note || Cmd->LMAExpr) Note = AddHdr(PT_NOTE, PF_R); Note->add(Sec); } else { @@ -1537,11 +1541,9 @@ template void Writer::addPtArmExid(std::vector &Phdrs) { if (Config->EMachine != EM_ARM) return; - auto I = - std::find_if(OutputSectionCommands.begin(), OutputSectionCommands.end(), - [](OutputSectionCommand *Cmd) { - return Cmd->Sec->Type == SHT_ARM_EXIDX; - }); + auto I = llvm::find_if(OutputSectionCommands, [](OutputSectionCommand *Cmd) { + return Cmd->Sec->Type == SHT_ARM_EXIDX; + }); if (I == OutputSectionCommands.end()) return; diff --git a/contrib/llvm/tools/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp b/contrib/llvm/tools/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp index edbe576f0086..b54054726dfe 100644 --- a/contrib/llvm/tools/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp +++ b/contrib/llvm/tools/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp @@ -508,9 +508,9 @@ readBinary(std::unique_ptr &mb, if (dyldInfo) { // If any exports, extract and add to normalized exportInfo vector. if (dyldInfo->export_size) { - const uint8_t *trieStart = reinterpret_cast(start + - dyldInfo->export_off); - ArrayRef trie(trieStart, dyldInfo->export_size); + const uint8_t *trieStart = reinterpret_cast( + start + read32(&dyldInfo->export_off, isBig)); + ArrayRef trie(trieStart, read32(&dyldInfo->export_size, isBig)); for (const ExportEntry &trieExport : MachOObjectFile::exports(trie)) { Export normExport; normExport.name = trieExport.name().copy(f->ownedAllocations); diff --git a/contrib/llvm/tools/lldb/include/lldb/Host/Host.h b/contrib/llvm/tools/lldb/include/lldb/Host/Host.h index c41e4796f532..da0b8e14c4a7 100644 --- a/contrib/llvm/tools/lldb/include/lldb/Host/Host.h +++ b/contrib/llvm/tools/lldb/include/lldb/Host/Host.h @@ -7,14 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef liblldb_Host_h_ -#define liblldb_Host_h_ -#if defined(__cplusplus) - -#include - -#include -#include +#ifndef LLDB_HOST_HOST_H +#define LLDB_HOST_HOST_H #include "lldb/Host/File.h" #include "lldb/Host/HostThread.h" @@ -22,6 +16,11 @@ #include "lldb/Utility/StringList.h" #include "lldb/lldb-private-forward.h" #include "lldb/lldb-private.h" +#include +#include +#include +#include +#include namespace lldb_private { @@ -254,5 +253,4 @@ template <> struct format_provider { }; } // namespace llvm -#endif // #if defined(__cplusplus) -#endif // liblldb_Host_h_ +#endif // LLDB_HOST_HOST_H diff --git a/contrib/llvm/tools/lldb/include/lldb/Host/common/NativeProcessProtocol.h b/contrib/llvm/tools/lldb/include/lldb/Host/common/NativeProcessProtocol.h index a97540499cd3..5f2157510c0a 100644 --- a/contrib/llvm/tools/lldb/include/lldb/Host/common/NativeProcessProtocol.h +++ b/contrib/llvm/tools/lldb/include/lldb/Host/common/NativeProcessProtocol.h @@ -19,6 +19,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" #include @@ -244,68 +245,57 @@ class NativeProcessProtocol virtual Status GetFileLoadAddress(const llvm::StringRef &file_name, lldb::addr_t &load_addr) = 0; - //------------------------------------------------------------------ - /// Launch a process for debugging. This method will create an concrete - /// instance of NativeProcessProtocol, based on the host platform. - /// (e.g. NativeProcessLinux on linux, etc.) - /// - /// @param[in] launch_info - /// Information required to launch the process. - /// - /// @param[in] native_delegate - /// The delegate that will receive messages regarding the - /// inferior. Must outlive the NativeProcessProtocol - /// instance. - /// - /// @param[in] mainloop - /// The mainloop instance with which the process can register - /// callbacks. Must outlive the NativeProcessProtocol - /// instance. - /// - /// @param[out] process_sp - /// On successful return from the method, this parameter - /// contains the shared pointer to the - /// NativeProcessProtocol that can be used to manipulate - /// the native process. - /// - /// @return - /// An error object indicating if the operation succeeded, - /// and if not, what error occurred. - //------------------------------------------------------------------ - static Status Launch(ProcessLaunchInfo &launch_info, - NativeDelegate &native_delegate, MainLoop &mainloop, - NativeProcessProtocolSP &process_sp); + class Factory { + public: + virtual ~Factory(); + //------------------------------------------------------------------ + /// Launch a process for debugging. + /// + /// @param[in] launch_info + /// Information required to launch the process. + /// + /// @param[in] native_delegate + /// The delegate that will receive messages regarding the + /// inferior. Must outlive the NativeProcessProtocol + /// instance. + /// + /// @param[in] mainloop + /// The mainloop instance with which the process can register + /// callbacks. Must outlive the NativeProcessProtocol + /// instance. + /// + /// @return + /// A NativeProcessProtocol shared pointer if the operation succeeded or + /// an error object if it failed. + //------------------------------------------------------------------ + virtual llvm::Expected + Launch(ProcessLaunchInfo &launch_info, NativeDelegate &native_delegate, + MainLoop &mainloop) const = 0; - //------------------------------------------------------------------ - /// Attach to an existing process. This method will create an concrete - /// instance of NativeProcessProtocol, based on the host platform. - /// (e.g. NativeProcessLinux on linux, etc.) - /// - /// @param[in] pid - /// pid of the process locatable - /// - /// @param[in] native_delegate - /// The delegate that will receive messages regarding the - /// inferior. Must outlive the NativeProcessProtocol - /// instance. - /// - /// @param[in] mainloop - /// The mainloop instance with which the process can register - /// callbacks. Must outlive the NativeProcessProtocol - /// instance. - /// - /// @param[out] process_sp - /// On successful return from the method, this parameter - /// contains the shared pointer to the - /// NativeProcessProtocol that can be used to manipulate - /// the native process. - /// - /// @return - /// An error object indicating if the operation succeeded, - /// and if not, what error occurred. - //------------------------------------------------------------------ - static Status Attach(lldb::pid_t pid, NativeDelegate &native_delegate, - MainLoop &mainloop, NativeProcessProtocolSP &process_sp); + //------------------------------------------------------------------ + /// Attach to an existing process. + /// + /// @param[in] pid + /// pid of the process locatable + /// + /// @param[in] native_delegate + /// The delegate that will receive messages regarding the + /// inferior. Must outlive the NativeProcessProtocol + /// instance. + /// + /// @param[in] mainloop + /// The mainloop instance with which the process can register + /// callbacks. Must outlive the NativeProcessProtocol + /// instance. + /// + /// @return + /// A NativeProcessProtocol shared pointer if the operation succeeded or + /// an error object if it failed. + //------------------------------------------------------------------ + virtual llvm::Expected + Attach(lldb::pid_t pid, NativeDelegate &native_delegate, + MainLoop &mainloop) const = 0; + }; //------------------------------------------------------------------ /// StartTracing API for starting a tracing instance with the @@ -413,10 +403,10 @@ class NativeProcessProtocol lldb::pid_t m_pid; std::vector m_threads; - lldb::tid_t m_current_thread_id; + lldb::tid_t m_current_thread_id = LLDB_INVALID_THREAD_ID; mutable std::recursive_mutex m_threads_mutex; - lldb::StateType m_state; + lldb::StateType m_state = lldb::eStateInvalid; mutable std::recursive_mutex m_state_mutex; llvm::Optional m_exit_status; @@ -427,7 +417,7 @@ class NativeProcessProtocol NativeWatchpointList m_watchpoint_list; HardwareBreakpointMap m_hw_breakpoints_map; int m_terminal_fd; - uint32_t m_stop_id; + uint32_t m_stop_id = 0; // Set of signal numbers that LLDB directly injects back to inferior // without stopping it. @@ -438,7 +428,8 @@ class NativeProcessProtocol // then the process should be attached to. When attaching to a process // lldb_private::Host calls should be used to locate the process to attach to, // and then this function should be called. - NativeProcessProtocol(lldb::pid_t pid); + NativeProcessProtocol(lldb::pid_t pid, int terminal_fd, + NativeDelegate &delegate); // ----------------------------------------------------------- // Internal interface for state handling diff --git a/contrib/llvm/tools/lldb/source/Commands/CommandObjectThread.cpp b/contrib/llvm/tools/lldb/source/Commands/CommandObjectThread.cpp index 687187b26ccd..6a933df43e14 100644 --- a/contrib/llvm/tools/lldb/source/Commands/CommandObjectThread.cpp +++ b/contrib/llvm/tools/lldb/source/Commands/CommandObjectThread.cpp @@ -161,9 +161,9 @@ class CommandObjectIterateOverThreads : public CommandObjectParsed { // List the common thread ID's const std::vector &thread_index_ids = stack.GetUniqueThreadIndexIDs(); - strm.Printf("%lu thread(s) ", thread_index_ids.size()); + strm.Format("{0} thread(s) ", thread_index_ids.size()); for (const uint32_t &thread_index_id : thread_index_ids) { - strm.Printf("#%u ", thread_index_id); + strm.Format("#{0} ", thread_index_id); } strm.EOL(); @@ -209,7 +209,7 @@ class CommandObjectIterateOverThreads : public CommandObjectParsed { Process *process = m_exe_ctx.GetProcessPtr(); Thread *thread = process->GetThreadList().FindThreadByID(tid).get(); if (thread == nullptr) { - result.AppendErrorWithFormat("Failed to process thread# %llu.\n", tid); + result.AppendErrorWithFormatv("Failed to process thread #{0}.\n", tid); result.SetStatus(eReturnStatusFailed); return false; } diff --git a/contrib/llvm/tools/lldb/source/Core/DumpDataExtractor.cpp b/contrib/llvm/tools/lldb/source/Core/DumpDataExtractor.cpp index 2b7abd60f8bc..c2a9115c3068 100644 --- a/contrib/llvm/tools/lldb/source/Core/DumpDataExtractor.cpp +++ b/contrib/llvm/tools/lldb/source/Core/DumpDataExtractor.cpp @@ -154,7 +154,8 @@ lldb::offset_t lldb_private::DumpDataExtractor( target_sp = exe_scope->CalculateTarget(); if (target_sp) { DisassemblerSP disassembler_sp(Disassembler::FindPlugin( - target_sp->GetArchitecture(), nullptr, nullptr)); + target_sp->GetArchitecture(), + target_sp->GetDisassemblyFlavor(), nullptr)); if (disassembler_sp) { lldb::addr_t addr = base_addr + start_offset; lldb_private::Address so_addr; diff --git a/contrib/llvm/tools/lldb/source/Host/common/File.cpp b/contrib/llvm/tools/lldb/source/Host/common/File.cpp index 3de93ebc220b..90a4462c6ca9 100644 --- a/contrib/llvm/tools/lldb/source/Host/common/File.cpp +++ b/contrib/llvm/tools/lldb/source/Host/common/File.cpp @@ -24,10 +24,12 @@ #endif #include "llvm/Support/ConvertUTF.h" +#include "llvm/Support/Errno.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Process.h" // for llvm::sys::Process::FileDescriptorHasColors() #include "lldb/Host/Config.h" +#include "lldb/Host/Host.h" #include "lldb/Utility/DataBufferHeap.h" #include "lldb/Utility/FileSpec.h" #include "lldb/Utility/Log.h" @@ -133,9 +135,8 @@ FILE *File::GetStream() { m_should_close_fd = true; } - do { - m_stream = ::fdopen(m_descriptor, mode); - } while (m_stream == NULL && errno == EINTR); + m_stream = + llvm::sys::RetryAfterSignal(nullptr, ::fdopen, m_descriptor, mode); // If we got a stream, then we own the stream and should no // longer own the descriptor because fclose() will close it for us @@ -157,6 +158,19 @@ void File::SetStream(FILE *fh, bool transfer_ownership) { m_own_stream = transfer_ownership; } +static int DoOpen(const char *path, int flags, int mode) { +#ifdef _MSC_VER + std::wstring wpath; + if (!llvm::ConvertUTF8toWide(path, wpath)) + return -1; + int result; + ::_wsopen_s(&result, wpath.c_str(), flags, _SH_DENYNO, mode); + return result; +#else + return ::open(path, flags, mode); +#endif +} + Status File::Open(const char *path, uint32_t options, uint32_t permissions) { Status error; if (IsValid()) @@ -222,20 +236,7 @@ Status File::Open(const char *path, uint32_t options, uint32_t permissions) { mode |= S_IXOTH; } - do { -#ifdef _MSC_VER - std::wstring wpath; - if (!llvm::ConvertUTF8toWide(path, wpath)) { - m_descriptor = -1; - error.SetErrorString("Error converting path to UTF-16"); - return error; - } - ::_wsopen_s(&m_descriptor, wpath.c_str(), oflag, _SH_DENYNO, mode); -#else - m_descriptor = ::open(path, oflag, mode); -#endif - } while (m_descriptor < 0 && errno == EINTR); - + m_descriptor = llvm::sys::RetryAfterSignal(-1, DoOpen, path, oflag, mode); if (!DescriptorIsValid()) error.SetErrorToErrno(); else { @@ -421,12 +422,7 @@ off_t File::SeekFromEnd(off_t offset, Status *error_ptr) { Status File::Flush() { Status error; if (StreamIsValid()) { - int err = 0; - do { - err = ::fflush(m_stream); - } while (err == EOF && errno == EINTR); - - if (err == EOF) + if (llvm::sys::RetryAfterSignal(EOF, ::fflush, m_stream) == EOF) error.SetErrorToErrno(); } else if (!DescriptorIsValid()) { error.SetErrorString("invalid file handle"); @@ -442,12 +438,7 @@ Status File::Sync() { if (err == 0) error.SetErrorToGenericError(); #else - int err = 0; - do { - err = ::fsync(m_descriptor); - } while (err == -1 && errno == EINTR); - - if (err == -1) + if (llvm::sys::RetryAfterSignal(-1, ::fsync, m_descriptor) == -1) error.SetErrorToErrno(); #endif } else { @@ -497,10 +488,7 @@ Status File::Read(void *buf, size_t &num_bytes) { ssize_t bytes_read = -1; if (DescriptorIsValid()) { - do { - bytes_read = ::read(m_descriptor, buf, num_bytes); - } while (bytes_read < 0 && errno == EINTR); - + bytes_read = llvm::sys::RetryAfterSignal(-1, ::read, m_descriptor, buf, num_bytes); if (bytes_read == -1) { error.SetErrorToErrno(); num_bytes = 0; @@ -559,10 +547,8 @@ Status File::Write(const void *buf, size_t &num_bytes) { ssize_t bytes_written = -1; if (DescriptorIsValid()) { - do { - bytes_written = ::write(m_descriptor, buf, num_bytes); - } while (bytes_written < 0 && errno == EINTR); - + bytes_written = + llvm::sys::RetryAfterSignal(-1, ::write, m_descriptor, buf, num_bytes); if (bytes_written == -1) { error.SetErrorToErrno(); num_bytes = 0; @@ -624,11 +610,8 @@ Status File::Read(void *buf, size_t &num_bytes, off_t &offset) { #ifndef _WIN32 int fd = GetDescriptor(); if (fd != kInvalidDescriptor) { - ssize_t bytes_read = -1; - do { - bytes_read = ::pread(fd, buf, num_bytes, offset); - } while (bytes_read < 0 && errno == EINTR); - + ssize_t bytes_read = + llvm::sys::RetryAfterSignal(-1, ::pread, fd, buf, num_bytes, offset); if (bytes_read < 0) { num_bytes = 0; error.SetErrorToErrno(); @@ -730,11 +713,8 @@ Status File::Write(const void *buf, size_t &num_bytes, off_t &offset) { int fd = GetDescriptor(); if (fd != kInvalidDescriptor) { #ifndef _WIN32 - ssize_t bytes_written = -1; - do { - bytes_written = ::pwrite(m_descriptor, buf, num_bytes, offset); - } while (bytes_written < 0 && errno == EINTR); - + ssize_t bytes_written = + llvm::sys::RetryAfterSignal(-1, ::pwrite, m_descriptor, buf, num_bytes, offset); if (bytes_written < 0) { num_bytes = 0; error.SetErrorToErrno(); diff --git a/contrib/llvm/tools/lldb/source/Host/common/MainLoop.cpp b/contrib/llvm/tools/lldb/source/Host/common/MainLoop.cpp index 7de6f7fa865d..c0c4471e735f 100644 --- a/contrib/llvm/tools/lldb/source/Host/common/MainLoop.cpp +++ b/contrib/llvm/tools/lldb/source/Host/common/MainLoop.cpp @@ -193,10 +193,16 @@ Status MainLoop::RunImpl::Poll() { void MainLoop::RunImpl::ProcessEvents() { #ifdef FORCE_PSELECT - for (const auto &fd : loop.m_read_fds) { - if (!FD_ISSET(fd.first, &read_fd_set)) - continue; - IOObject::WaitableHandle handle = fd.first; + // Collect first all readable file descriptors into a separate vector and then + // iterate over it to invoke callbacks. Iterating directly over + // loop.m_read_fds is not possible because the callbacks can modify the + // container which could invalidate the iterator. + std::vector fds; + for (const auto &fd : loop.m_read_fds) + if (FD_ISSET(fd.first, &read_fd_set)) + fds.push_back(fd.first); + + for (const auto &handle : fds) { #else for (const auto &fd : read_fds) { if ((fd.revents & POLLIN) == 0) @@ -209,13 +215,16 @@ void MainLoop::RunImpl::ProcessEvents() { loop.ProcessReadObject(handle); } - for (const auto &entry : loop.m_signals) { + std::vector signals; + for (const auto &entry : loop.m_signals) + if (g_signal_flags[entry.first] != 0) + signals.push_back(entry.first); + + for (const auto &signal : signals) { if (loop.m_terminate_request) return; - if (g_signal_flags[entry.first] == 0) - continue; // No signal - g_signal_flags[entry.first] = 0; - loop.ProcessSignal(entry.first); + g_signal_flags[signal] = 0; + loop.ProcessSignal(signal); } } #endif diff --git a/contrib/llvm/tools/lldb/source/Host/common/NativeProcessProtocol.cpp b/contrib/llvm/tools/lldb/source/Host/common/NativeProcessProtocol.cpp index 341c301dc9c5..b5b6e9d8b923 100644 --- a/contrib/llvm/tools/lldb/source/Host/common/NativeProcessProtocol.cpp +++ b/contrib/llvm/tools/lldb/source/Host/common/NativeProcessProtocol.cpp @@ -29,11 +29,13 @@ using namespace lldb_private; // NativeProcessProtocol Members // ----------------------------------------------------------------------------- -NativeProcessProtocol::NativeProcessProtocol(lldb::pid_t pid) - : m_pid(pid), m_threads(), m_current_thread_id(LLDB_INVALID_THREAD_ID), - m_threads_mutex(), m_state(lldb::eStateInvalid), m_state_mutex(), - m_delegates_mutex(), m_delegates(), m_breakpoint_list(), - m_watchpoint_list(), m_terminal_fd(-1), m_stop_id(0) {} +NativeProcessProtocol::NativeProcessProtocol(lldb::pid_t pid, int terminal_fd, + NativeDelegate &delegate) + : m_pid(pid), m_terminal_fd(terminal_fd) { + bool registered = RegisterNativeDelegate(delegate); + assert(registered); + (void)registered; +} lldb_private::Status NativeProcessProtocol::Interrupt() { Status error; @@ -488,23 +490,4 @@ Status NativeProcessProtocol::ResolveProcessArchitecture(lldb::pid_t pid, "failed to retrieve a valid architecture from the exe module"); } -#if !defined(__linux__) && !defined(__NetBSD__) -// These need to be implemented to support lldb-gdb-server on a given platform. -// Stubs are -// provided to make the rest of the code link on non-supported platforms. - -Status NativeProcessProtocol::Launch(ProcessLaunchInfo &launch_info, - NativeDelegate &native_delegate, - MainLoop &mainloop, - NativeProcessProtocolSP &process_sp) { - llvm_unreachable("Platform has no NativeProcessProtocol support"); -} - -Status NativeProcessProtocol::Attach(lldb::pid_t pid, - NativeDelegate &native_delegate, - MainLoop &mainloop, - NativeProcessProtocolSP &process_sp) { - llvm_unreachable("Platform has no NativeProcessProtocol support"); -} - -#endif +NativeProcessProtocol::Factory::~Factory() = default; diff --git a/contrib/llvm/tools/lldb/source/Host/common/SocketAddress.cpp b/contrib/llvm/tools/lldb/source/Host/common/SocketAddress.cpp index 440ae5d9027f..41150fa7fd74 100644 --- a/contrib/llvm/tools/lldb/source/Host/common/SocketAddress.cpp +++ b/contrib/llvm/tools/lldb/source/Host/common/SocketAddress.cpp @@ -201,7 +201,7 @@ const SocketAddress &SocketAddress:: operator=(const struct addrinfo *addr_info) { Clear(); if (addr_info && addr_info->ai_addr && addr_info->ai_addrlen > 0 && - addr_info->ai_addrlen <= sizeof m_socket_addr) { + size_t(addr_info->ai_addrlen) <= sizeof m_socket_addr) { ::memcpy(&m_socket_addr, addr_info->ai_addr, addr_info->ai_addrlen); } return *this; diff --git a/contrib/llvm/tools/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp b/contrib/llvm/tools/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp index 6b0f069c35a9..105ef0f23d46 100644 --- a/contrib/llvm/tools/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp +++ b/contrib/llvm/tools/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp @@ -245,11 +245,7 @@ ConnectionStatus ConnectionFileDescriptor::Connect(llvm::StringRef path, } else if ((addr = GetURLAddress(path, FILE_SCHEME))) { std::string addr_str = addr->str(); // file:///PATH - int fd = -1; - do { - fd = ::open(addr_str.c_str(), O_RDWR); - } while (fd == -1 && errno == EINTR); - + int fd = llvm::sys::RetryAfterSignal(-1, ::open, addr_str.c_str(), O_RDWR); if (fd == -1) { if (error_ptr) error_ptr->SetErrorToErrno(); @@ -620,20 +616,17 @@ ConnectionFileDescriptor::BytesAvailable(const Timeout &timeout, if (select_helper.FDIsSetRead(pipe_fd)) { // There is an interrupt or exit command in the command pipe // Read the data from that pipe: - char buffer[1]; + char c; - ssize_t bytes_read; - - do { - bytes_read = ::read(pipe_fd, buffer, sizeof(buffer)); - } while (bytes_read < 0 && errno == EINTR); - - switch (buffer[0]) { + ssize_t bytes_read = llvm::sys::RetryAfterSignal(-1, ::read, pipe_fd, &c, 1); + assert(bytes_read == 1); + (void)bytes_read; + switch (c) { case 'q': if (log) log->Printf("%p ConnectionFileDescriptor::BytesAvailable() " "got data: %c from the command channel.", - static_cast(this), buffer[0]); + static_cast(this), c); return eConnectionStatusEndOfFile; case 'i': // Interrupt the current read diff --git a/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.cpp b/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.cpp index d385b78e0ec4..5dbb3bb4ef7e 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.cpp @@ -63,10 +63,10 @@ void AuxVector::ParseAuxv(DataExtractor &data) { if (!ParseAuxvEntry(data, entry, &offset, byte_size)) break; - if (entry.type == AT_NULL) + if (entry.type == AUXV_AT_NULL) break; - if (entry.type == AT_IGNORE) + if (entry.type == AUXV_AT_IGNORE) continue; m_auxv.push_back(entry); @@ -110,43 +110,43 @@ void AuxVector::DumpToLog(Log *log) const { const char *AuxVector::GetEntryName(EntryType type) { const char *name = "AT_???"; -#define ENTRY_NAME(_type) \ - _type: \ - name = #_type +#define ENTRY_NAME(_type) \ + _type: \ + name = #_type + 5 switch (type) { - case ENTRY_NAME(AT_NULL); break; - case ENTRY_NAME(AT_IGNORE); break; - case ENTRY_NAME(AT_EXECFD); break; - case ENTRY_NAME(AT_PHDR); break; - case ENTRY_NAME(AT_PHENT); break; - case ENTRY_NAME(AT_PHNUM); break; - case ENTRY_NAME(AT_PAGESZ); break; - case ENTRY_NAME(AT_BASE); break; - case ENTRY_NAME(AT_FLAGS); break; - case ENTRY_NAME(AT_ENTRY); break; - case ENTRY_NAME(AT_NOTELF); break; - case ENTRY_NAME(AT_UID); break; - case ENTRY_NAME(AT_EUID); break; - case ENTRY_NAME(AT_GID); break; - case ENTRY_NAME(AT_EGID); break; - case ENTRY_NAME(AT_CLKTCK); break; - case ENTRY_NAME(AT_PLATFORM); break; - case ENTRY_NAME(AT_HWCAP); break; - case ENTRY_NAME(AT_FPUCW); break; - case ENTRY_NAME(AT_DCACHEBSIZE); break; - case ENTRY_NAME(AT_ICACHEBSIZE); break; - case ENTRY_NAME(AT_UCACHEBSIZE); break; - case ENTRY_NAME(AT_IGNOREPPC); break; - case ENTRY_NAME(AT_SECURE); break; - case ENTRY_NAME(AT_BASE_PLATFORM); break; - case ENTRY_NAME(AT_RANDOM); break; - case ENTRY_NAME(AT_EXECFN); break; - case ENTRY_NAME(AT_SYSINFO); break; - case ENTRY_NAME(AT_SYSINFO_EHDR); break; - case ENTRY_NAME(AT_L1I_CACHESHAPE); break; - case ENTRY_NAME(AT_L1D_CACHESHAPE); break; - case ENTRY_NAME(AT_L2_CACHESHAPE); break; - case ENTRY_NAME(AT_L3_CACHESHAPE); break; + case ENTRY_NAME(AUXV_AT_NULL); break; + case ENTRY_NAME(AUXV_AT_IGNORE); break; + case ENTRY_NAME(AUXV_AT_EXECFD); break; + case ENTRY_NAME(AUXV_AT_PHDR); break; + case ENTRY_NAME(AUXV_AT_PHENT); break; + case ENTRY_NAME(AUXV_AT_PHNUM); break; + case ENTRY_NAME(AUXV_AT_PAGESZ); break; + case ENTRY_NAME(AUXV_AT_BASE); break; + case ENTRY_NAME(AUXV_AT_FLAGS); break; + case ENTRY_NAME(AUXV_AT_ENTRY); break; + case ENTRY_NAME(AUXV_AT_NOTELF); break; + case ENTRY_NAME(AUXV_AT_UID); break; + case ENTRY_NAME(AUXV_AT_EUID); break; + case ENTRY_NAME(AUXV_AT_GID); break; + case ENTRY_NAME(AUXV_AT_EGID); break; + case ENTRY_NAME(AUXV_AT_CLKTCK); break; + case ENTRY_NAME(AUXV_AT_PLATFORM); break; + case ENTRY_NAME(AUXV_AT_HWCAP); break; + case ENTRY_NAME(AUXV_AT_FPUCW); break; + case ENTRY_NAME(AUXV_AT_DCACHEBSIZE); break; + case ENTRY_NAME(AUXV_AT_ICACHEBSIZE); break; + case ENTRY_NAME(AUXV_AT_UCACHEBSIZE); break; + case ENTRY_NAME(AUXV_AT_IGNOREPPC); break; + case ENTRY_NAME(AUXV_AT_SECURE); break; + case ENTRY_NAME(AUXV_AT_BASE_PLATFORM); break; + case ENTRY_NAME(AUXV_AT_RANDOM); break; + case ENTRY_NAME(AUXV_AT_EXECFN); break; + case ENTRY_NAME(AUXV_AT_SYSINFO); break; + case ENTRY_NAME(AUXV_AT_SYSINFO_EHDR); break; + case ENTRY_NAME(AUXV_AT_L1I_CACHESHAPE); break; + case ENTRY_NAME(AUXV_AT_L1D_CACHESHAPE); break; + case ENTRY_NAME(AUXV_AT_L2_CACHESHAPE); break; + case ENTRY_NAME(AUXV_AT_L3_CACHESHAPE); break; } #undef ENTRY_NAME diff --git a/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.h b/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.h index 9c3e1b002a24..cf9da0808357 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.h +++ b/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/AuxVector.h @@ -42,41 +42,42 @@ class AuxVector { /// Constants describing the type of entry. /// On Linux, running "LD_SHOW_AUXV=1 ./executable" will spew AUX information. + /// Added AUXV prefix to avoid potential conflicts with system-defined macros enum EntryType { - AT_NULL = 0, ///< End of auxv. - AT_IGNORE = 1, ///< Ignore entry. - AT_EXECFD = 2, ///< File descriptor of program. - AT_PHDR = 3, ///< Program headers. - AT_PHENT = 4, ///< Size of program header. - AT_PHNUM = 5, ///< Number of program headers. - AT_PAGESZ = 6, ///< Page size. - AT_BASE = 7, ///< Interpreter base address. - AT_FLAGS = 8, ///< Flags. - AT_ENTRY = 9, ///< Program entry point. - AT_NOTELF = 10, ///< Set if program is not an ELF. - AT_UID = 11, ///< UID. - AT_EUID = 12, ///< Effective UID. - AT_GID = 13, ///< GID. - AT_EGID = 14, ///< Effective GID. - AT_CLKTCK = 17, ///< Clock frequency (e.g. times(2)). - AT_PLATFORM = 15, ///< String identifying platform. - AT_HWCAP = 16, ///< Machine dependent hints about processor capabilities. - AT_FPUCW = 18, ///< Used FPU control word. - AT_DCACHEBSIZE = 19, ///< Data cache block size. - AT_ICACHEBSIZE = 20, ///< Instruction cache block size. - AT_UCACHEBSIZE = 21, ///< Unified cache block size. - AT_IGNOREPPC = 22, ///< Entry should be ignored. - AT_SECURE = 23, ///< Boolean, was exec setuid-like? - AT_BASE_PLATFORM = 24, ///< String identifying real platforms. - AT_RANDOM = 25, ///< Address of 16 random bytes. - AT_EXECFN = 31, ///< Filename of executable. - AT_SYSINFO = 32, ///< Pointer to the global system page used for system - ///calls and other nice things. - AT_SYSINFO_EHDR = 33, - AT_L1I_CACHESHAPE = 34, ///< Shapes of the caches. - AT_L1D_CACHESHAPE = 35, - AT_L2_CACHESHAPE = 36, - AT_L3_CACHESHAPE = 37, + AUXV_AT_NULL = 0, ///< End of auxv. + AUXV_AT_IGNORE = 1, ///< Ignore entry. + AUXV_AT_EXECFD = 2, ///< File descriptor of program. + AUXV_AT_PHDR = 3, ///< Program headers. + AUXV_AT_PHENT = 4, ///< Size of program header. + AUXV_AT_PHNUM = 5, ///< Number of program headers. + AUXV_AT_PAGESZ = 6, ///< Page size. + AUXV_AT_BASE = 7, ///< Interpreter base address. + AUXV_AT_FLAGS = 8, ///< Flags. + AUXV_AT_ENTRY = 9, ///< Program entry point. + AUXV_AT_NOTELF = 10, ///< Set if program is not an ELF. + AUXV_AT_UID = 11, ///< UID. + AUXV_AT_EUID = 12, ///< Effective UID. + AUXV_AT_GID = 13, ///< GID. + AUXV_AT_EGID = 14, ///< Effective GID. + AUXV_AT_CLKTCK = 17, ///< Clock frequency (e.g. times(2)). + AUXV_AT_PLATFORM = 15, ///< String identifying platform. + AUXV_AT_HWCAP = 16, ///< Machine dependent hints about processor capabilities. + AUXV_AT_FPUCW = 18, ///< Used FPU control word. + AUXV_AT_DCACHEBSIZE = 19, ///< Data cache block size. + AUXV_AT_ICACHEBSIZE = 20, ///< Instruction cache block size. + AUXV_AT_UCACHEBSIZE = 21, ///< Unified cache block size. + AUXV_AT_IGNOREPPC = 22, ///< Entry should be ignored. + AUXV_AT_SECURE = 23, ///< Boolean, was exec setuid-like? + AUXV_AT_BASE_PLATFORM = 24, ///< String identifying real platforms. + AUXV_AT_RANDOM = 25, ///< Address of 16 random bytes. + AUXV_AT_EXECFN = 31, ///< Filename of executable. + AUXV_AT_SYSINFO = 32, ///< Pointer to the global system page used for system + ///calls and other nice things. + AUXV_AT_SYSINFO_EHDR = 33, + AUXV_AT_L1I_CACHESHAPE = 34, ///< Shapes of the caches. + AUXV_AT_L1D_CACHESHAPE = 35, + AUXV_AT_L2_CACHESHAPE = 36, + AUXV_AT_L3_CACHESHAPE = 37, }; private: diff --git a/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp index 0092535648bd..a7afeb6d68c3 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp @@ -576,7 +576,7 @@ addr_t DynamicLoaderPOSIXDYLD::ComputeLoadOffset() { } void DynamicLoaderPOSIXDYLD::EvalVdsoStatus() { - AuxVector::iterator I = m_auxv->FindEntry(AuxVector::AT_SYSINFO_EHDR); + AuxVector::iterator I = m_auxv->FindEntry(AuxVector::AUXV_AT_SYSINFO_EHDR); if (I != m_auxv->end()) m_vdso_base = I->value; @@ -589,7 +589,7 @@ addr_t DynamicLoaderPOSIXDYLD::GetEntryPoint() { if (m_auxv.get() == NULL) return LLDB_INVALID_ADDRESS; - AuxVector::iterator I = m_auxv->FindEntry(AuxVector::AT_ENTRY); + AuxVector::iterator I = m_auxv->FindEntry(AuxVector::AUXV_AT_ENTRY); if (I == m_auxv->end()) return LLDB_INVALID_ADDRESS; diff --git a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp index af242d786a5f..9a6e39be0bfd 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp @@ -247,12 +247,16 @@ bool AddressSanitizerRuntime::NotifyBreakpointHit( AddressSanitizerRuntime *const instance = static_cast(baton); + ProcessSP process_sp = instance->GetProcessSP(); + + if (process_sp->GetModIDRef().IsLastResumeForUserExpression()) + return false; + StructuredData::ObjectSP report = instance->RetrieveReportData(); std::string description; if (report) { description = instance->FormatDescription(report); } - ProcessSP process_sp = instance->GetProcessSP(); // Make sure this is the right process if (process_sp && process_sp == context->exe_ctx_ref.GetProcessSP()) { ThreadSP thread_sp = context->exe_ctx_ref.GetThreadSP(); diff --git a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/MainThreadCheckerRuntime.cpp b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/MainThreadCheckerRuntime.cpp index 3c22b81df7a4..eb238419ab18 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/MainThreadCheckerRuntime.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/MainThreadCheckerRuntime.cpp @@ -141,7 +141,7 @@ MainThreadCheckerRuntime::RetrieveReportData(ExecutionContextRef exe_ctx_ref) { d->AddStringItem("class_name", className); d->AddStringItem("selector", selector); d->AddStringItem("description", - apiName + " must be called from main thread only"); + apiName + " must be used from main thread only"); d->AddIntegerItem("tid", thread_sp->GetIndexID()); d->AddItem("trace", trace_sp); return dict_sp; @@ -163,6 +163,9 @@ bool MainThreadCheckerRuntime::NotifyBreakpointHit( process_sp != context->exe_ctx_ref.GetProcessSP()) return false; + if (process_sp->GetModIDRef().IsLastResumeForUserExpression()) + return false; + StructuredData::ObjectSP report = instance->RetrieveReportData(context->exe_ctx_ref); diff --git a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp index f60df0463346..cf9ba60c7b60 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp @@ -803,6 +803,11 @@ bool ThreadSanitizerRuntime::NotifyBreakpointHit( ThreadSanitizerRuntime *const instance = static_cast(baton); + ProcessSP process_sp = instance->GetProcessSP(); + + if (process_sp->GetModIDRef().IsLastResumeForUserExpression()) + return false; + StructuredData::ObjectSP report = instance->RetrieveReportData(context->exe_ctx_ref); std::string stop_reason_description; @@ -851,7 +856,6 @@ bool ThreadSanitizerRuntime::NotifyBreakpointHit( all_addresses_are_same); } - ProcessSP process_sp = instance->GetProcessSP(); // Make sure this is the right process if (process_sp && process_sp == context->exe_ctx_ref.GetProcessSP()) { ThreadSP thread_sp = context->exe_ctx_ref.GetThreadSP(); diff --git a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp index 023af84179aa..28c28e41ef44 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp @@ -217,6 +217,9 @@ bool UndefinedBehaviorSanitizerRuntime::NotifyBreakpointHit( process_sp != context->exe_ctx_ref.GetProcessSP()) return false; + if (process_sp->GetModIDRef().IsLastResumeForUserExpression()) + return false; + StructuredData::ObjectSP report = instance->RetrieveReportData(context->exe_ctx_ref); diff --git a/contrib/llvm/tools/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/contrib/llvm/tools/lldb/source/Plugins/Language/ObjC/Cocoa.cpp index 2ba005e02ad4..8f4997533212 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Language/ObjC/Cocoa.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Language/ObjC/Cocoa.cpp @@ -543,6 +543,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( } uint64_t value = 0; + bool success = false; switch (type_code) { case TypeCodes::sint8: value = process_sp->ReadUnsignedIntegerFromMemory(data_location, 1, 0, @@ -550,6 +551,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( if (error.Fail()) return false; NSNumber_FormatChar(valobj, stream, (char)value, options.GetLanguage()); + success = true; break; case TypeCodes::sint16: value = process_sp->ReadUnsignedIntegerFromMemory(data_location, 2, 0, @@ -558,6 +560,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( return false; NSNumber_FormatShort(valobj, stream, (short)value, options.GetLanguage()); + success = true; break; case TypeCodes::sint32: value = process_sp->ReadUnsignedIntegerFromMemory(data_location, 4, 0, @@ -565,6 +568,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( if (error.Fail()) return false; NSNumber_FormatInt(valobj, stream, (int)value, options.GetLanguage()); + success = true; break; case TypeCodes::sint64: value = process_sp->ReadUnsignedIntegerFromMemory(data_location, 8, 0, @@ -572,6 +576,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( if (error.Fail()) return false; NSNumber_FormatLong(valobj, stream, value, options.GetLanguage()); + success = true; break; case TypeCodes::f32: { @@ -582,6 +587,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( float flt_value = 0.0f; memcpy(&flt_value, &flt_as_int, sizeof(flt_as_int)); NSNumber_FormatFloat(valobj, stream, flt_value, options.GetLanguage()); + success = true; break; } case TypeCodes::f64: @@ -593,6 +599,7 @@ bool lldb_private::formatters::NSNumberSummaryProvider( double dbl_value = 0.0; memcpy(&dbl_value, &dbl_as_lng, sizeof(dbl_as_lng)); NSNumber_FormatDouble(valobj, stream, dbl_value, options.GetLanguage()); + success = true; break; } case TypeCodes::sint128: // internally, this is the same @@ -608,12 +615,11 @@ bool lldb_private::formatters::NSNumberSummaryProvider( return false; llvm::APInt i128_value(128, words); NSNumber_FormatInt128(valobj, stream, i128_value, options.GetLanguage()); + success = true; break; } - default: - return false; } - return true; + return success; } } diff --git a/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp index f4cf22ad7583..013c33def13b 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp @@ -884,7 +884,7 @@ void PlatformPOSIX::CalculateTrapHandlerSymbolNames() { Status PlatformPOSIX::EvaluateLibdlExpression( lldb_private::Process *process, const char *expr_cstr, - const char *expr_prefix, lldb::ValueObjectSP &result_valobj_sp) { + llvm::StringRef expr_prefix, lldb::ValueObjectSP &result_valobj_sp) { DynamicLoader *loader = process->GetDynamicLoader(); if (loader) { Status error = loader->CanLoadImage(); @@ -944,7 +944,7 @@ uint32_t PlatformPOSIX::DoLoadImage(lldb_private::Process *process, the_result; )", path); - const char *prefix = GetLibdlFunctionDeclarations(); + llvm::StringRef prefix = GetLibdlFunctionDeclarations(); lldb::ValueObjectSP result_valobj_sp; error = EvaluateLibdlExpression(process, expr.GetData(), prefix, result_valobj_sp); @@ -992,7 +992,7 @@ Status PlatformPOSIX::UnloadImage(lldb_private::Process *process, StreamString expr; expr.Printf("dlclose((void *)0x%" PRIx64 ")", image_addr); - const char *prefix = GetLibdlFunctionDeclarations(); + llvm::StringRef prefix = GetLibdlFunctionDeclarations(); lldb::ValueObjectSP result_valobj_sp; Status error = EvaluateLibdlExpression(process, expr.GetData(), prefix, result_valobj_sp); @@ -1024,7 +1024,7 @@ lldb::ProcessSP PlatformPOSIX::ConnectProcess(llvm::StringRef connect_url, error); } -const char *PlatformPOSIX::GetLibdlFunctionDeclarations() const { +llvm::StringRef PlatformPOSIX::GetLibdlFunctionDeclarations() { return R"( extern "C" void* dlopen(const char*, int); extern "C" void* dlsym(void*, const char*); diff --git a/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.h b/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.h index 742702b07b88..ebc36c2461db 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.h +++ b/contrib/llvm/tools/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.h @@ -198,10 +198,10 @@ class PlatformPOSIX : public lldb_private::Platform { lldb_private::Status EvaluateLibdlExpression(lldb_private::Process *process, const char *expr_cstr, - const char *expr_prefix, + llvm::StringRef expr_prefix, lldb::ValueObjectSP &result_valobj_sp); - virtual const char *GetLibdlFunctionDeclarations() const; + virtual llvm::StringRef GetLibdlFunctionDeclarations(); private: DISALLOW_COPY_AND_ASSIGN(PlatformPOSIX); diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp index 10dd14753914..a4f5f02dde62 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp @@ -746,15 +746,9 @@ ProcessMonitor::ProcessMonitor( if (!error.Success()) return; -WAIT_AGAIN: - // Wait for the operation thread to initialize. - if (sem_wait(&args->m_semaphore)) { - if (errno == EINTR) - goto WAIT_AGAIN; - else { - error.SetErrorToErrno(); - return; - } + if (llvm::sys::RetryAfterSignal(-1, sem_wait, &args->m_semaphore) == -1) { + error.SetErrorToErrno(); + return; } // Check that the launch was a success. @@ -790,15 +784,9 @@ ProcessMonitor::ProcessMonitor(ProcessFreeBSD *process, lldb::pid_t pid, if (!error.Success()) return; -WAIT_AGAIN: - // Wait for the operation thread to initialize. - if (sem_wait(&args->m_semaphore)) { - if (errno == EINTR) - goto WAIT_AGAIN; - else { - error.SetErrorToErrno(); - return; - } + if (llvm::sys::RetryAfterSignal(-1, sem_wait, &args->m_semaphore) == -1) { + error.SetErrorToErrno(); + return; } // Check that the attach was a success. diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp index a4d775860a65..b9ef02efa65d 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp @@ -21,6 +21,7 @@ #include "lldb/Host/common/NativeRegisterContext.h" #include "lldb/Host/posix/ProcessLauncherPosixFork.h" #include "lldb/Target/Process.h" +#include "llvm/Support/Errno.h" // System includes - They have to be included after framework includes because // they define some @@ -63,81 +64,101 @@ static Status EnsureFDFlags(int fd, int flags) { // Public Static Methods // ----------------------------------------------------------------------------- -Status NativeProcessProtocol::Launch( - ProcessLaunchInfo &launch_info, - NativeProcessProtocol::NativeDelegate &native_delegate, MainLoop &mainloop, - NativeProcessProtocolSP &native_process_sp) { +llvm::Expected +NativeProcessNetBSD::Factory::Launch(ProcessLaunchInfo &launch_info, + NativeDelegate &native_delegate, + MainLoop &mainloop) const { Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - Status error; - - // Verify the working directory is valid if one was specified. - FileSpec working_dir{launch_info.GetWorkingDirectory()}; - if (working_dir && (!working_dir.ResolvePath() || - !llvm::sys::fs::is_directory(working_dir.GetPath()))) { - error.SetErrorStringWithFormat("No such file or directory: %s", - working_dir.GetCString()); - return error; + Status status; + ::pid_t pid = ProcessLauncherPosixFork() + .LaunchProcess(launch_info, status) + .GetProcessId(); + LLDB_LOG(log, "pid = {0:x}", pid); + if (status.Fail()) { + LLDB_LOG(log, "failed to launch process: {0}", status); + return status.ToError(); } - // Create the NativeProcessNetBSD in launch mode. - native_process_sp.reset(new NativeProcessNetBSD()); - - if (!native_process_sp->RegisterNativeDelegate(native_delegate)) { - native_process_sp.reset(); - error.SetErrorStringWithFormat("failed to register the native delegate"); - return error; + // Wait for the child process to trap on its call to execve. + int wstatus; + ::pid_t wpid = llvm::sys::RetryAfterSignal(-1, ::waitpid, pid, &wstatus, 0); + assert(wpid == pid); + (void)wpid; + if (!WIFSTOPPED(wstatus)) { + LLDB_LOG(log, "Could not sync with inferior process: wstatus={1}", + WaitStatus::Decode(wstatus)); + return llvm::make_error("Could not sync with inferior process", + llvm::inconvertibleErrorCode()); } + LLDB_LOG(log, "inferior started, now in stopped state"); - error = std::static_pointer_cast(native_process_sp) - ->LaunchInferior(mainloop, launch_info); + ArchSpec arch; + if ((status = ResolveProcessArchitecture(pid, arch)).Fail()) + return status.ToError(); - if (error.Fail()) { - native_process_sp.reset(); - LLDB_LOG(log, "failed to launch process: {0}", error); - return error; + // Set the architecture to the exe architecture. + LLDB_LOG(log, "pid = {0:x}, detected architecture {1}", pid, + arch.GetArchitectureName()); + + std::shared_ptr process_sp(new NativeProcessNetBSD( + pid, launch_info.GetPTY().ReleaseMasterFileDescriptor(), native_delegate, + arch, mainloop)); + + status = process_sp->ReinitializeThreads(); + if (status.Fail()) + return status.ToError(); + + for (const auto &thread_sp : process_sp->m_threads) { + static_pointer_cast(thread_sp)->SetStoppedBySignal( + SIGSTOP); } + process_sp->SetState(StateType::eStateStopped); - launch_info.SetProcessID(native_process_sp->GetID()); - - return error; + return process_sp; } -Status NativeProcessProtocol::Attach( +llvm::Expected NativeProcessNetBSD::Factory::Attach( lldb::pid_t pid, NativeProcessProtocol::NativeDelegate &native_delegate, - MainLoop &mainloop, NativeProcessProtocolSP &native_process_sp) { + MainLoop &mainloop) const { Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); LLDB_LOG(log, "pid = {0:x}", pid); // Retrieve the architecture for the running process. - ArchSpec process_arch; - Status error = ResolveProcessArchitecture(pid, process_arch); - if (!error.Success()) - return error; + ArchSpec arch; + Status status = ResolveProcessArchitecture(pid, arch); + if (!status.Success()) + return status.ToError(); - std::shared_ptr native_process_netbsd_sp( - new NativeProcessNetBSD()); + std::shared_ptr process_sp( + new NativeProcessNetBSD(pid, -1, native_delegate, arch, mainloop)); - if (!native_process_netbsd_sp->RegisterNativeDelegate(native_delegate)) { - error.SetErrorStringWithFormat("failed to register the native delegate"); - return error; - } + status = process_sp->Attach(); + if (!status.Success()) + return status.ToError(); - native_process_netbsd_sp->AttachToInferior(mainloop, pid, error); - if (!error.Success()) - return error; - - native_process_sp = native_process_netbsd_sp; - return error; + return process_sp; } // ----------------------------------------------------------------------------- // Public Instance Methods // ----------------------------------------------------------------------------- -NativeProcessNetBSD::NativeProcessNetBSD() - : NativeProcessProtocol(LLDB_INVALID_PROCESS_ID), m_arch(), - m_supports_mem_region(eLazyBoolCalculate), m_mem_region_cache() {} +NativeProcessNetBSD::NativeProcessNetBSD(::pid_t pid, int terminal_fd, + NativeDelegate &delegate, + const ArchSpec &arch, + MainLoop &mainloop) + : NativeProcessProtocol(pid, terminal_fd, delegate), m_arch(arch) { + if (m_terminal_fd != -1) { + Status status = EnsureFDFlags(m_terminal_fd, O_NONBLOCK); + assert(status.Success()); + } + + Status status; + m_sigchld_handle = mainloop.RegisterSignal( + SIGCHLD, [this](MainLoopBase &) { SigchldHandler(); }, status); + assert(m_sigchld_handle && status.Success()); +} // Handles all waitpid events from the inferior process. void NativeProcessNetBSD::MonitorCallback(lldb::pid_t pid, int signal) { @@ -709,126 +730,17 @@ Status NativeProcessNetBSD::GetFileLoadAddress(const llvm::StringRef &file_name, return Status(); } -Status NativeProcessNetBSD::LaunchInferior(MainLoop &mainloop, - ProcessLaunchInfo &launch_info) { - Status error; - m_sigchld_handle = mainloop.RegisterSignal( - SIGCHLD, [this](MainLoopBase &) { SigchldHandler(); }, error); - if (!m_sigchld_handle) - return error; - - SetState(eStateLaunching); - - ::pid_t pid = ProcessLauncherPosixFork() - .LaunchProcess(launch_info, error) - .GetProcessId(); - if (error.Fail()) - return error; - - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - - // Wait for the child process to trap on its call to execve. - ::pid_t wpid; - int status; - if ((wpid = waitpid(pid, &status, 0)) < 0) { - error.SetErrorToErrno(); - LLDB_LOG(log, "waitpid for inferior failed with %s", error); - - // Mark the inferior as invalid. - // FIXME this could really use a new state - eStateLaunchFailure. For - // now, using eStateInvalid. - SetState(StateType::eStateInvalid); - - return error; - } - assert(WIFSTOPPED(status) && (wpid == static_cast<::pid_t>(pid)) && - "Could not sync with inferior process."); - - LLDB_LOG(log, "inferior started, now in stopped state"); - - // Release the master terminal descriptor and pass it off to the - // NativeProcessNetBSD instance. Similarly stash the inferior pid. - m_terminal_fd = launch_info.GetPTY().ReleaseMasterFileDescriptor(); - m_pid = pid; - launch_info.SetProcessID(pid); - - if (m_terminal_fd != -1) { - error = EnsureFDFlags(m_terminal_fd, O_NONBLOCK); - if (error.Fail()) { - LLDB_LOG(log, - "inferior EnsureFDFlags failed for ensuring terminal " - "O_NONBLOCK setting: {0}", - error); - - // Mark the inferior as invalid. - // FIXME this could really use a new state - eStateLaunchFailure. For - // now, using eStateInvalid. - SetState(StateType::eStateInvalid); - - return error; - } - } - - LLDB_LOG(log, "adding pid = {0}", pid); - - ResolveProcessArchitecture(m_pid, m_arch); - - error = ReinitializeThreads(); - if (error.Fail()) { - SetState(StateType::eStateInvalid); - return error; - } - - for (const auto &thread_sp : m_threads) { - static_pointer_cast(thread_sp)->SetStoppedBySignal( - SIGSTOP); - } - - /* Set process stopped */ - SetState(StateType::eStateStopped); - - if (error.Fail()) - LLDB_LOG(log, "inferior launching failed {0}", error); - return error; -} - -void NativeProcessNetBSD::AttachToInferior(MainLoop &mainloop, lldb::pid_t pid, - Status &error) { - Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); - LLDB_LOG(log, "pid = {0:x}", pid); - - m_sigchld_handle = mainloop.RegisterSignal( - SIGCHLD, [this](MainLoopBase &) { SigchldHandler(); }, error); - if (!m_sigchld_handle) - return; - - error = ResolveProcessArchitecture(pid, m_arch); - if (!error.Success()) - return; - - // Set the architecture to the exe architecture. - LLDB_LOG(log, "pid = {0:x}, detected architecture {1}", pid, - m_arch.GetArchitectureName()); - - m_pid = pid; - SetState(eStateAttaching); - - Attach(pid, error); -} - void NativeProcessNetBSD::SigchldHandler() { Log *log(ProcessPOSIXLog::GetLogIfAllCategoriesSet(POSIX_LOG_PROCESS)); // Process all pending waitpid notifications. int status; - ::pid_t wait_pid = waitpid(GetID(), &status, WALLSIG | WNOHANG); + ::pid_t wait_pid = + llvm::sys::RetryAfterSignal(-1, waitpid, GetID(), &status, WALLSIG | WNOHANG); if (wait_pid == 0) return; // We are done. if (wait_pid == -1) { - if (errno == EINTR) - return; - Status error(errno, eErrorTypePOSIX); LLDB_LOG(log, "waitpid ({0}, &status, _) failed: {1}", GetID(), error); } @@ -880,33 +792,23 @@ NativeThreadNetBSDSP NativeProcessNetBSD::AddThread(lldb::tid_t thread_id) { return thread_sp; } -::pid_t NativeProcessNetBSD::Attach(lldb::pid_t pid, Status &error) { - if (pid <= 1) { - error.SetErrorToGenericError(); - error.SetErrorString("Attaching to process 1 is not allowed."); - return -1; - } - +Status NativeProcessNetBSD::Attach() { // Attach to the requested process. // An attach will cause the thread to stop with a SIGSTOP. - error = PtraceWrapper(PT_ATTACH, pid); - if (error.Fail()) - return -1; + Status status = PtraceWrapper(PT_ATTACH, m_pid); + if (status.Fail()) + return status; - int status; + int wstatus; // Need to use WALLSIG otherwise we receive an error with errno=ECHLD // At this point we should have a thread stopped if waitpid succeeds. - if ((status = waitpid(pid, NULL, WALLSIG)) < 0) - return -1; - - m_pid = pid; + if ((wstatus = waitpid(m_pid, NULL, WALLSIG)) < 0) + return Status(errno, eErrorTypePOSIX); /* Initialize threads */ - error = ReinitializeThreads(); - if (error.Fail()) { - SetState(StateType::eStateInvalid); - return -1; - } + status = ReinitializeThreads(); + if (status.Fail()) + return status; for (const auto &thread_sp : m_threads) { static_pointer_cast(thread_sp)->SetStoppedBySignal( @@ -915,8 +817,7 @@ ::pid_t NativeProcessNetBSD::Attach(lldb::pid_t pid, Status &error) { // Let our process instance know the thread has stopped. SetState(StateType::eStateStopped); - - return pid; + return Status(); } Status NativeProcessNetBSD::ReadMemory(lldb::addr_t addr, void *buf, diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.h b/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.h index 7a1303faea68..34b892f1fc88 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.h +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.h @@ -31,15 +31,18 @@ namespace process_netbsd { /// /// Changes in the inferior process state are broadcasted. class NativeProcessNetBSD : public NativeProcessProtocol { - friend Status NativeProcessProtocol::Launch( - ProcessLaunchInfo &launch_info, NativeDelegate &native_delegate, - MainLoop &mainloop, NativeProcessProtocolSP &process_sp); - - friend Status NativeProcessProtocol::Attach( - lldb::pid_t pid, NativeProcessProtocol::NativeDelegate &native_delegate, - MainLoop &mainloop, NativeProcessProtocolSP &process_sp); - public: + class Factory : public NativeProcessProtocol::Factory { + public: + llvm::Expected + Launch(ProcessLaunchInfo &launch_info, NativeDelegate &native_delegate, + MainLoop &mainloop) const override; + + llvm::Expected + Attach(lldb::pid_t pid, NativeDelegate &native_delegate, + MainLoop &mainloop) const override; + }; + // --------------------------------------------------------------------- // NativeProcessProtocol Interface // --------------------------------------------------------------------- @@ -107,21 +110,19 @@ class NativeProcessNetBSD : public NativeProcessProtocol { private: MainLoop::SignalHandleUP m_sigchld_handle; ArchSpec m_arch; - LazyBool m_supports_mem_region; + LazyBool m_supports_mem_region = eLazyBoolCalculate; std::vector> m_mem_region_cache; // --------------------------------------------------------------------- // Private Instance Methods // --------------------------------------------------------------------- - NativeProcessNetBSD(); + NativeProcessNetBSD(::pid_t pid, int terminal_fd, NativeDelegate &delegate, + const ArchSpec &arch, MainLoop &mainloop); bool HasThreadNoLock(lldb::tid_t thread_id); NativeThreadNetBSDSP AddThread(lldb::tid_t thread_id); - Status LaunchInferior(MainLoop &mainloop, ProcessLaunchInfo &launch_info); - void AttachToInferior(MainLoop &mainloop, lldb::pid_t pid, Status &error); - void MonitorCallback(lldb::pid_t pid, int signal); void MonitorExited(lldb::pid_t pid, WaitStatus status); void MonitorSIGSTOP(lldb::pid_t pid); @@ -133,8 +134,7 @@ class NativeProcessNetBSD : public NativeProcessProtocol { Status PopulateMemoryRegionCache(); void SigchldHandler(); - ::pid_t Attach(lldb::pid_t pid, Status &error); - + Status Attach(); Status ReinitializeThreads(); }; diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp index 33aed7a43c4a..e6fd386b903b 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp @@ -86,6 +86,7 @@ GDBRemoteCommunicationClient::GDBRemoteCommunicationClient() m_supports_jLoadedDynamicLibrariesInfos(eLazyBoolCalculate), m_supports_jGetSharedCacheInfo(eLazyBoolCalculate), m_supports_QPassSignals(eLazyBoolCalculate), + m_supports_error_string_reply(eLazyBoolCalculate), m_supports_qProcessInfoPID(true), m_supports_qfProcessInfo(true), m_supports_qUserName(true), m_supports_qGroupName(true), m_supports_qThreadStopInfo(true), m_supports_z0(true), @@ -596,6 +597,21 @@ bool GDBRemoteCommunicationClient::GetThreadExtendedInfoSupported() { return m_supports_jThreadExtendedInfo; } +void GDBRemoteCommunicationClient::EnableErrorStringInPacket() { + if (m_supports_error_string_reply == eLazyBoolCalculate) { + StringExtractorGDBRemote response; + // We try to enable error strings in remote packets + // but if we fail, we just work in the older way. + m_supports_error_string_reply = eLazyBoolNo; + if (SendPacketAndWaitForResponse("QEnableErrorStrings", response, false) == + PacketResult::Success) { + if (response.IsOKResponse()) { + m_supports_error_string_reply = eLazyBoolYes; + } + } + } +} + bool GDBRemoteCommunicationClient::GetLoadedDynamicLibrariesInfosSupported() { if (m_supports_jLoadedDynamicLibrariesInfos == eLazyBoolCalculate) { StringExtractorGDBRemote response; @@ -3181,8 +3197,8 @@ GDBRemoteCommunicationClient::SendStartTracePacket(const TraceOptions &options, true) == GDBRemoteCommunication::PacketResult::Success) { if (!response.IsNormalResponse()) { - error.SetError(response.GetError(), eErrorTypeGeneric); - LLDB_LOG(log, "Target does not support Tracing"); + error = response.GetStatus(); + LLDB_LOG(log, "Target does not support Tracing , error {0}", error); } else { ret_uid = response.GetHexMaxU64(false, LLDB_INVALID_UID); } @@ -3219,7 +3235,7 @@ GDBRemoteCommunicationClient::SendStopTracePacket(lldb::user_id_t uid, true) == GDBRemoteCommunication::PacketResult::Success) { if (!response.IsOKResponse()) { - error.SetError(response.GetError(), eErrorTypeGeneric); + error = response.GetStatus(); LLDB_LOG(log, "stop tracing failed"); } } else { @@ -3234,6 +3250,7 @@ GDBRemoteCommunicationClient::SendStopTracePacket(lldb::user_id_t uid, Status GDBRemoteCommunicationClient::SendGetDataPacket( lldb::user_id_t uid, lldb::tid_t thread_id, llvm::MutableArrayRef &buffer, size_t offset) { + StreamGDBRemote escaped_packet; escaped_packet.PutCString("jTraceBufferRead:"); return SendGetTraceDataPacket(escaped_packet, uid, thread_id, buffer, offset); @@ -3242,6 +3259,7 @@ Status GDBRemoteCommunicationClient::SendGetDataPacket( Status GDBRemoteCommunicationClient::SendGetMetaDataPacket( lldb::user_id_t uid, lldb::tid_t thread_id, llvm::MutableArrayRef &buffer, size_t offset) { + StreamGDBRemote escaped_packet; escaped_packet.PutCString("jTraceMetaRead:"); return SendGetTraceDataPacket(escaped_packet, uid, thread_id, buffer, offset); @@ -3308,7 +3326,7 @@ GDBRemoteCommunicationClient::SendGetTraceConfigPacket(lldb::user_id_t uid, custom_params_sp)); } } else { - error.SetError(response.GetError(), eErrorTypeGeneric); + error = response.GetStatus(); } } else { LLDB_LOG(log, "failed to send packet"); @@ -3344,7 +3362,7 @@ Status GDBRemoteCommunicationClient::SendGetTraceDataPacket( size_t filled_size = response.GetHexBytesAvail(buffer); buffer = llvm::MutableArrayRef(buffer.data(), filled_size); } else { - error.SetError(response.GetError(), eErrorTypeGeneric); + error = response.GetStatus(); buffer = buffer.slice(buffer.size()); } } else { diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h index a38110faaec6..712d85eed082 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h @@ -340,6 +340,8 @@ class GDBRemoteCommunicationClient : public GDBRemoteClientBase { bool GetQXferAuxvReadSupported(); + void EnableErrorStringInPacket(); + bool GetQXferLibrariesReadSupported(); bool GetQXferLibrariesSVR4ReadSupported(); @@ -549,6 +551,7 @@ class GDBRemoteCommunicationClient : public GDBRemoteClientBase { LazyBool m_supports_jLoadedDynamicLibrariesInfos; LazyBool m_supports_jGetSharedCacheInfo; LazyBool m_supports_QPassSignals; + LazyBool m_supports_error_string_reply; bool m_supports_qProcessInfoPID : 1, m_supports_qfProcessInfo : 1, m_supports_qUserName : 1, m_supports_qGroupName : 1, diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp index dac675ee9432..4be92b79fd1a 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp @@ -20,6 +20,7 @@ // Project includes #include "ProcessGDBRemoteLog.h" #include "Utility/StringExtractorGDBRemote.h" +#include "lldb/Utility/StreamString.h" using namespace lldb; using namespace lldb_private; @@ -27,7 +28,12 @@ using namespace lldb_private::process_gdb_remote; GDBRemoteCommunicationServer::GDBRemoteCommunicationServer( const char *comm_name, const char *listener_name) - : GDBRemoteCommunication(comm_name, listener_name), m_exit_now(false) {} + : GDBRemoteCommunication(comm_name, listener_name), m_exit_now(false) { + RegisterPacketHandler( + StringExtractorGDBRemote::eServerPacketType_QEnableErrorStrings, + [this](StringExtractorGDBRemote packet, Status &error, bool &interrupt, + bool &quit) { return this->Handle_QErrorStringEnable(packet); }); +} GDBRemoteCommunicationServer::~GDBRemoteCommunicationServer() {} @@ -99,6 +105,24 @@ GDBRemoteCommunicationServer::SendErrorResponse(uint8_t err) { return SendPacketNoLock(llvm::StringRef(packet, packet_len)); } +GDBRemoteCommunication::PacketResult +GDBRemoteCommunicationServer::SendErrorResponse(const Status &error) { + if (m_send_error_strings) { + lldb_private::StreamString packet; + packet.Printf("E%2.2x;", static_cast(error.GetError())); + packet.PutCStringAsRawHex8(error.AsCString()); + return SendPacketNoLock(packet.GetString()); + } else + return SendErrorResponse(error.GetError()); +} + +GDBRemoteCommunication::PacketResult +GDBRemoteCommunicationServer::Handle_QErrorStringEnable( + StringExtractorGDBRemote &packet) { + m_send_error_strings = true; + return SendOKResponse(); +} + GDBRemoteCommunication::PacketResult GDBRemoteCommunicationServer::SendIllFormedResponse( const StringExtractorGDBRemote &failed_packet, const char *message) { diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h index 6eb25f8b9f98..a35352480040 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h @@ -57,6 +57,13 @@ class GDBRemoteCommunicationServer : public GDBRemoteCommunication { bool m_exit_now; // use in asynchronous handling to indicate process should // exit. + bool m_send_error_strings; // If the client enables this then + // we will send error strings as well. + + PacketResult Handle_QErrorStringEnable(StringExtractorGDBRemote &packet); + + PacketResult SendErrorResponse(const Status &error); + PacketResult SendUnimplementedResponse(const char *packet); PacketResult SendErrorResponse(uint8_t error); diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp index de2400c51ba3..f53db502be93 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp @@ -1046,14 +1046,9 @@ GDBRemoteCommunicationServerCommon::Handle_A(StringExtractorGDBRemote &packet) { if (success) { m_process_launch_error = LaunchProcess(); - if (m_process_launch_info.GetProcessID() != LLDB_INVALID_PROCESS_ID) { + if (m_process_launch_error.Success()) return SendOKResponse(); - } else { - Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS)); - if (log) - log->Printf("LLGSPacketHandler::%s failed to launch exe: %s", - __FUNCTION__, m_process_launch_error.AsCString()); - } + LLDB_LOG(log, "failed to launch exe: {0}", m_process_launch_error); } return SendErrorResponse(8); } diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp index 7523260c13e4..a7fe4ee3b147 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp @@ -73,15 +73,11 @@ enum GDBRemoteServerError { // GDBRemoteCommunicationServerLLGS constructor //---------------------------------------------------------------------- GDBRemoteCommunicationServerLLGS::GDBRemoteCommunicationServerLLGS( - MainLoop &mainloop) + MainLoop &mainloop, const NativeProcessProtocol::Factory &process_factory) : GDBRemoteCommunicationServerCommon("gdb-remote.server", "gdb-remote.server.rx_packet"), - m_mainloop(mainloop), m_current_tid(LLDB_INVALID_THREAD_ID), - m_continue_tid(LLDB_INVALID_THREAD_ID), m_debugged_process_mutex(), - m_debugged_process_sp(), m_stdio_communication("process.stdio"), - m_inferior_prev_state(StateType::eStateInvalid), - m_saved_registers_map(), m_next_saved_registers_id(1), - m_handshake_completed(false) { + m_mainloop(mainloop), m_process_factory(process_factory), + m_stdio_communication("process.stdio") { RegisterPacketHandlers(); } @@ -241,19 +237,20 @@ Status GDBRemoteCommunicationServerLLGS::LaunchProcess() { const bool default_to_use_pty = true; m_process_launch_info.FinalizeFileActions(nullptr, default_to_use_pty); - Status error; { std::lock_guard guard(m_debugged_process_mutex); assert(!m_debugged_process_sp && "lldb-server creating debugged " "process but one already exists"); - error = NativeProcessProtocol::Launch(m_process_launch_info, *this, - m_mainloop, m_debugged_process_sp); - } - - if (!error.Success()) { - fprintf(stderr, "%s: failed to launch executable %s", __FUNCTION__, - m_process_launch_info.GetArguments().GetArgumentAtIndex(0)); - return error; + auto process_or = + m_process_factory.Launch(m_process_launch_info, *this, m_mainloop); + if (!process_or) { + Status status(process_or.takeError()); + llvm::errs() << llvm::formatv( + "failed to launch executable `{0}`: {1}", + m_process_launch_info.GetArguments().GetArgumentAtIndex(0), status); + return status; + } + m_debugged_process_sp = *process_or; } // Handle mirroring of inferior stdout/stderr over the gdb-remote protocol @@ -279,9 +276,9 @@ Status GDBRemoteCommunicationServerLLGS::LaunchProcess() { log->Printf("ProcessGDBRemoteCommunicationServerLLGS::%s setting " "inferior STDIO fd to %d", __FUNCTION__, terminal_fd); - error = SetSTDIOFileDescriptor(terminal_fd); - if (error.Fail()) - return error; + Status status = SetSTDIOFileDescriptor(terminal_fd); + if (status.Fail()) + return status; } else { if (log) log->Printf("ProcessGDBRemoteCommunicationServerLLGS::%s ignoring " @@ -298,14 +295,12 @@ Status GDBRemoteCommunicationServerLLGS::LaunchProcess() { printf("Launched '%s' as process %" PRIu64 "...\n", m_process_launch_info.GetArguments().GetArgumentAtIndex(0), - m_process_launch_info.GetProcessID()); + m_debugged_process_sp->GetID()); - return error; + return Status(); } Status GDBRemoteCommunicationServerLLGS::AttachToProcess(lldb::pid_t pid) { - Status error; - Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS)); if (log) log->Printf("GDBRemoteCommunicationServerLLGS::%s pid %" PRIu64, @@ -321,13 +316,14 @@ Status GDBRemoteCommunicationServerLLGS::AttachToProcess(lldb::pid_t pid) { pid, m_debugged_process_sp->GetID()); // Try to attach. - error = NativeProcessProtocol::Attach(pid, *this, m_mainloop, - m_debugged_process_sp); - if (!error.Success()) { - fprintf(stderr, "%s: failed to attach to process %" PRIu64 ": %s", - __FUNCTION__, pid, error.AsCString()); - return error; + auto process_or = m_process_factory.Attach(pid, *this, m_mainloop); + if (!process_or) { + Status status(process_or.takeError()); + llvm::errs() << llvm::formatv("failed to attach to process {0}: {1}", pid, + status); + return status; } + m_debugged_process_sp = *process_or; // Setup stdout/stderr mapping from inferior. auto terminal_fd = m_debugged_process_sp->GetTerminalFileDescriptor(); @@ -336,9 +332,9 @@ Status GDBRemoteCommunicationServerLLGS::AttachToProcess(lldb::pid_t pid) { log->Printf("ProcessGDBRemoteCommunicationServerLLGS::%s setting " "inferior STDIO fd to %d", __FUNCTION__, terminal_fd); - error = SetSTDIOFileDescriptor(terminal_fd); - if (error.Fail()) - return error; + Status status = SetSTDIOFileDescriptor(terminal_fd); + if (status.Fail()) + return status; } else { if (log) log->Printf("ProcessGDBRemoteCommunicationServerLLGS::%s ignoring " @@ -347,8 +343,7 @@ Status GDBRemoteCommunicationServerLLGS::AttachToProcess(lldb::pid_t pid) { } printf("Attached to process %" PRIu64 "...\n", pid); - - return error; + return Status(); } void GDBRemoteCommunicationServerLLGS::InitializeDelegate( @@ -1128,7 +1123,7 @@ GDBRemoteCommunicationServerLLGS::Handle_jTraceStart( uid = m_debugged_process_sp->StartTrace(options, error); LLDB_LOG(log, "uid is {0} , error is {1}", uid, error.GetError()); if (error.Fail()) - return SendErrorResponse(error.GetError()); + return SendErrorResponse(error); StreamGDBRemote response; response.Printf("%" PRIx64, uid); @@ -1165,7 +1160,7 @@ GDBRemoteCommunicationServerLLGS::Handle_jTraceStop( Status error = m_debugged_process_sp->StopTrace(uid, tid); if (error.Fail()) - return SendErrorResponse(error.GetError()); + return SendErrorResponse(error); return SendOKResponse(); } @@ -1208,7 +1203,7 @@ GDBRemoteCommunicationServerLLGS::Handle_jTraceConfigRead( Status error = m_debugged_process_sp->GetTraceConfig(uid, options); if (error.Fail()) - return SendErrorResponse(error.GetError()); + return SendErrorResponse(error); StreamGDBRemote escaped_response; StructuredData::Dictionary json_packet; @@ -1284,7 +1279,7 @@ GDBRemoteCommunicationServerLLGS::Handle_jTraceRead( error = m_debugged_process_sp->GetMetaData(uid, tid, buf, offset); if (error.Fail()) - return SendErrorResponse(error.GetError()); + return SendErrorResponse(error); for (auto i : buf) response.PutHex8(i); diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h index a7d7850d454f..b065642d4aed 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h @@ -39,7 +39,9 @@ class GDBRemoteCommunicationServerLLGS //------------------------------------------------------------------ // Constructors and Destructors //------------------------------------------------------------------ - GDBRemoteCommunicationServerLLGS(MainLoop &mainloop); + GDBRemoteCommunicationServerLLGS( + MainLoop &mainloop, + const NativeProcessProtocol::Factory &process_factory); //------------------------------------------------------------------ /// Specify the program to launch and its arguments. @@ -108,20 +110,21 @@ class GDBRemoteCommunicationServerLLGS protected: MainLoop &m_mainloop; MainLoop::ReadHandleUP m_network_handle_up; - lldb::tid_t m_current_tid; - lldb::tid_t m_continue_tid; + const NativeProcessProtocol::Factory &m_process_factory; + lldb::tid_t m_current_tid = LLDB_INVALID_THREAD_ID; + lldb::tid_t m_continue_tid = LLDB_INVALID_THREAD_ID; std::recursive_mutex m_debugged_process_mutex; NativeProcessProtocolSP m_debugged_process_sp; Communication m_stdio_communication; MainLoop::ReadHandleUP m_stdio_handle_up; - lldb::StateType m_inferior_prev_state; + lldb::StateType m_inferior_prev_state = lldb::StateType::eStateInvalid; std::unique_ptr m_active_auxv_buffer_up; std::mutex m_saved_registers_mutex; std::unordered_map m_saved_registers_map; - uint32_t m_next_saved_registers_id; - bool m_handshake_completed : 1; + uint32_t m_next_saved_registers_id = 1; + bool m_handshake_completed = false; PacketResult SendONotification(const char *buffer, uint32_t len); diff --git a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index 98b64510f59f..8b77a282bd73 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -1031,6 +1031,7 @@ Status ProcessGDBRemote::ConnectToDebugserver(llvm::StringRef connect_url) { m_gdb_comm.GetHostInfo(); m_gdb_comm.GetVContSupported('c'); m_gdb_comm.GetVAttachOrWaitSupported(); + m_gdb_comm.EnableErrorStringInPacket(); // Ask the remote server for the default thread id if (GetTarget().GetNonStopModeEnabled()) diff --git a/contrib/llvm/tools/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/contrib/llvm/tools/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp index b7fbd7347d08..6c39690268c6 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp @@ -1857,14 +1857,12 @@ StructuredData::DictionarySP ScriptInterpreterPython::GetDynamicSettings( return StructuredData::DictionarySP(); PythonObject reply_pyobj; - { - Locker py_lock(this, - Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN); - TargetSP target_sp(target->shared_from_this()); - reply_pyobj.Reset(PyRefType::Owned, - (PyObject *)g_swig_plugin_get(generic->GetValue(), - setting_name, target_sp)); - } + Locker py_lock(this, + Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN); + TargetSP target_sp(target->shared_from_this()); + reply_pyobj.Reset(PyRefType::Owned, + (PyObject *)g_swig_plugin_get(generic->GetValue(), + setting_name, target_sp)); PythonDictionary py_dict(PyRefType::Borrowed, reply_pyobj.get()); return py_dict.CreateStructuredDictionary(); diff --git a/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp b/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp index 9d79a411faad..aa15063ac0dc 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp +++ b/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp @@ -604,9 +604,10 @@ uint32_t x86AssemblyInspectionEngine::extract_4(uint8_t *b) { } bool x86AssemblyInspectionEngine::instruction_length(uint8_t *insn_p, - int &length) { + int &length, + uint32_t buffer_remaining_bytes) { - const uint32_t max_op_byte_size = m_arch.GetMaximumOpcodeByteSize(); + uint32_t max_op_byte_size = std::min(buffer_remaining_bytes, m_arch.GetMaximumOpcodeByteSize()); llvm::SmallVector opcode_data; opcode_data.resize(max_op_byte_size); @@ -698,8 +699,9 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly( bool row_updated = false; // The UnwindPlan::Row 'row' has been updated m_cur_insn = data + current_func_text_offset; - if (!instruction_length(m_cur_insn, insn_len) || insn_len == 0 || - insn_len > kMaxInstructionByteSize) { + if (!instruction_length(m_cur_insn, insn_len, size - current_func_text_offset) + || insn_len == 0 + || insn_len > kMaxInstructionByteSize) { // An unrecognized/junk instruction break; } @@ -1002,8 +1004,9 @@ bool x86AssemblyInspectionEngine::AugmentUnwindPlanFromCallSite( while (offset < size) { m_cur_insn = data + offset; int insn_len; - if (!instruction_length(m_cur_insn, insn_len) || insn_len == 0 || - insn_len > kMaxInstructionByteSize) { + if (!instruction_length(m_cur_insn, insn_len, size - offset) + || insn_len == 0 + || insn_len > kMaxInstructionByteSize) { // An unrecognized/junk instruction. break; } @@ -1214,8 +1217,9 @@ bool x86AssemblyInspectionEngine::FindFirstNonPrologueInstruction( int scratch; m_cur_insn = data + offset; - if (!instruction_length(m_cur_insn, insn_len) || - insn_len > kMaxInstructionByteSize || insn_len == 0) { + if (!instruction_length(m_cur_insn, insn_len, size - offset) + || insn_len > kMaxInstructionByteSize + || insn_len == 0) { // An error parsing the instruction, i.e. probably data/garbage - stop // scanning break; diff --git a/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.h b/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.h index 2e7875966cb6..97441d362973 100644 --- a/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.h +++ b/contrib/llvm/tools/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.h @@ -113,7 +113,7 @@ class x86AssemblyInspectionEngine { bool ret_pattern_p(); uint32_t extract_4(uint8_t *b); - bool instruction_length(uint8_t *insn, int &length); + bool instruction_length(uint8_t *insn, int &length, uint32_t buffer_remaining_bytes); bool machine_regno_to_lldb_regno(int machine_regno, uint32_t &lldb_regno); diff --git a/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.cpp b/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.cpp index 3473a9e96689..8e50c0106a48 100644 --- a/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.cpp +++ b/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.cpp @@ -19,8 +19,18 @@ StringExtractorGDBRemote::GetResponseType() const { switch (m_packet[0]) { case 'E': - if (m_packet.size() == 3 && isxdigit(m_packet[1]) && isxdigit(m_packet[2])) - return eError; + if (isxdigit(m_packet[1]) && isxdigit(m_packet[2])) { + if (m_packet.size() == 3) + return eError; + llvm::StringRef packet_ref(m_packet); + if (packet_ref[3] == ';') { + auto err_string = packet_ref.substr(4); + for (auto e : err_string) + if (!isxdigit(e)) + return eResponse; + return eError; + } + } break; case 'O': @@ -86,6 +96,8 @@ StringExtractorGDBRemote::GetServerPacketType() const { return eServerPacketType_QEnvironment; if (PACKET_STARTS_WITH("QEnvironmentHexEncoded:")) return eServerPacketType_QEnvironmentHexEncoded; + if (PACKET_STARTS_WITH("QEnableErrorStrings")) + return eServerPacketType_QEnableErrorStrings; break; case 'P': @@ -438,8 +450,8 @@ bool StringExtractorGDBRemote::IsNormalResponse() const { } bool StringExtractorGDBRemote::IsErrorResponse() const { - return GetResponseType() == eError && m_packet.size() == 3 && - isxdigit(m_packet[1]) && isxdigit(m_packet[2]); + return GetResponseType() == eError && isxdigit(m_packet[1]) && + isxdigit(m_packet[2]); } uint8_t StringExtractorGDBRemote::GetError() { @@ -450,6 +462,23 @@ uint8_t StringExtractorGDBRemote::GetError() { return 0; } +lldb_private::Status StringExtractorGDBRemote::GetStatus() { + lldb_private::Status error; + if (GetResponseType() == eError) { + SetFilePos(1); + uint8_t errc = GetHexU8(255); + error.SetError(errc, lldb::eErrorTypeGeneric); + + error.SetErrorStringWithFormat("Error %u", errc); + std::string error_messg; + if (GetChar() == ';') { + GetHexByteString(error_messg); + error.SetErrorString(error_messg); + } + } + return error; +} + size_t StringExtractorGDBRemote::GetEscapedBinaryData(std::string &str) { // Just get the data bytes in the string as // GDBRemoteCommunication::CheckForPacket() diff --git a/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.h b/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.h index 473cab04f800..f4ed642a706e 100644 --- a/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.h +++ b/contrib/llvm/tools/lldb/source/Utility/StringExtractorGDBRemote.h @@ -10,6 +10,7 @@ #ifndef utility_StringExtractorGDBRemote_h_ #define utility_StringExtractorGDBRemote_h_ +#include "lldb/Utility/Status.h" #include "lldb/Utility/StringExtractor.h" #include "llvm/ADT/StringRef.h" // for StringRef @@ -72,6 +73,7 @@ class StringExtractorGDBRemote : public StringExtractor { eServerPacketType_qGetWorkingDir, eServerPacketType_qFileLoadAddress, eServerPacketType_QEnvironment, + eServerPacketType_QEnableErrorStrings, eServerPacketType_QLaunchArch, eServerPacketType_QSetDisableASLR, eServerPacketType_QSetDetachOnError, @@ -190,6 +192,8 @@ class StringExtractorGDBRemote : public StringExtractor { // digits. Otherwise the error encoded in XX is returned. uint8_t GetError(); + lldb_private::Status GetStatus(); + size_t GetEscapedBinaryData(std::string &str); protected: diff --git a/contrib/llvm/tools/lldb/tools/lldb-server/lldb-gdbserver.cpp b/contrib/llvm/tools/lldb/tools/lldb-server/lldb-gdbserver.cpp index 412d775e8394..337f244c2c2d 100644 --- a/contrib/llvm/tools/lldb/tools/lldb-server/lldb-gdbserver.cpp +++ b/contrib/llvm/tools/lldb/tools/lldb-server/lldb-gdbserver.cpp @@ -33,10 +33,17 @@ #include "lldb/Host/Pipe.h" #include "lldb/Host/Socket.h" #include "lldb/Host/StringConvert.h" +#include "lldb/Host/common/NativeProcessProtocol.h" #include "lldb/Utility/Status.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Errno.h" +#if defined(__linux__) +#include "Plugins/Process/Linux/NativeProcessLinux.h" +#elif defined(__NetBSD__) +#include "Plugins/Process/NetBSD/NativeProcessNetBSD.h" +#endif + #ifndef LLGS_PROGRAM_NAME #define LLGS_PROGRAM_NAME "lldb-server" #endif @@ -51,6 +58,30 @@ using namespace lldb_private; using namespace lldb_private::lldb_server; using namespace lldb_private::process_gdb_remote; +namespace { +#if defined(__linux__) +typedef process_linux::NativeProcessLinux::Factory NativeProcessFactory; +#elif defined(__NetBSD__) +typedef process_netbsd::NativeProcessNetBSD::Factory NativeProcessFactory; +#else +// Dummy implementation to make sure the code compiles +class NativeProcessFactory : public NativeProcessProtocol::Factory { +public: + llvm::Expected + Launch(ProcessLaunchInfo &launch_info, + NativeProcessProtocol::NativeDelegate &delegate, + MainLoop &mainloop) const override { + llvm_unreachable("Not implemented"); + } + llvm::Expected + Attach(lldb::pid_t pid, NativeProcessProtocol::NativeDelegate &delegate, + MainLoop &mainloop) const override { + llvm_unreachable("Not implemented"); + } +}; +#endif +} + //---------------------------------------------------------------------- // option descriptors for getopt_long_only() //---------------------------------------------------------------------- @@ -446,7 +477,8 @@ int main_gdbserver(int argc, char *argv[]) { exit(255); } - GDBRemoteCommunicationServerLLGS gdb_server(mainloop); + NativeProcessFactory factory; + GDBRemoteCommunicationServerLLGS gdb_server(mainloop, factory); const char *const host_and_port = argv[0]; argc -= 1; diff --git a/contrib/llvm/tools/lldb/tools/lldb-server/lldb-platform.cpp b/contrib/llvm/tools/lldb/tools/lldb-server/lldb-platform.cpp index 8d45682566b9..ec5b781dac48 100644 --- a/contrib/llvm/tools/lldb/tools/lldb-server/lldb-platform.cpp +++ b/contrib/llvm/tools/lldb/tools/lldb-server/lldb-platform.cpp @@ -197,46 +197,41 @@ int main_platform(int argc, char *argv[]) { break; case 'p': { - char *end = NULL; - long tmp_port_offset = strtoul(optarg, &end, 0); - if (end && *end == '\0') { - if (LOW_PORT <= tmp_port_offset && tmp_port_offset <= HIGH_PORT) { - port_offset = (uint16_t)tmp_port_offset; - } else { - fprintf(stderr, "error: port offset %li is not in the valid user " - "port range of %u - %u\n", - tmp_port_offset, LOW_PORT, HIGH_PORT); - option_error = 5; - } - } else { - fprintf(stderr, "error: invalid port offset string %s\n", optarg); + if (!llvm::to_integer(optarg, port_offset)) { + llvm::errs() << "error: invalid port offset string " << optarg << "\n"; option_error = 4; + break; + } + if (port_offset < LOW_PORT || port_offset > HIGH_PORT) { + llvm::errs() << llvm::formatv("error: port offset {0} is not in the " + "valid user port range of {1} - {2}\n", + port_offset, LOW_PORT, HIGH_PORT); + option_error = 5; } } break; case 'P': case 'm': case 'M': { - char *end = NULL; - long portnum = strtoul(optarg, &end, 0); - if (end && *end == '\0') { - if (LOW_PORT <= portnum && portnum <= HIGH_PORT) { - if (ch == 'P') - gdbserver_portmap[(uint16_t)portnum] = LLDB_INVALID_PROCESS_ID; - else if (ch == 'm') - min_gdbserver_port = portnum; - else - max_gdbserver_port = portnum; - } else { - fprintf(stderr, "error: port number %li is not in the valid user " - "port range of %u - %u\n", - portnum, LOW_PORT, HIGH_PORT); - option_error = 1; - } - } else { - fprintf(stderr, "error: invalid port number string %s\n", optarg); + uint16_t portnum; + if (!llvm::to_integer(optarg, portnum)) { + llvm::errs() << "error: invalid port number string " << optarg << "\n"; option_error = 2; + break; } + if (portnum < LOW_PORT || portnum > HIGH_PORT) { + llvm::errs() << llvm::formatv("error: port number {0} is not in the " + "valid user port range of {1} - {2}\n", + portnum, LOW_PORT, HIGH_PORT); + option_error = 1; + break; + } + if (ch == 'P') + gdbserver_portmap[portnum] = LLDB_INVALID_PROCESS_ID; + else if (ch == 'm') + min_gdbserver_port = portnum; + else + max_gdbserver_port = portnum; } break; case 'h': /* fall-through is intentional */ diff --git a/contrib/llvm/tools/lli/OrcLazyJIT.cpp b/contrib/llvm/tools/lli/OrcLazyJIT.cpp index 2e15894152f9..f1a752e0790d 100644 --- a/contrib/llvm/tools/lli/OrcLazyJIT.cpp +++ b/contrib/llvm/tools/lli/OrcLazyJIT.cpp @@ -148,18 +148,19 @@ int llvm::runOrcLazyJIT(std::vector> Ms, // Add the module, look up main and run it. for (auto &M : Ms) - J.addModule(std::shared_ptr(std::move(M))); - auto MainSym = J.findSymbol("main"); + cantFail(J.addModule(std::shared_ptr(std::move(M)))); - if (!MainSym) { + if (auto MainSym = J.findSymbol("main")) { + typedef int (*MainFnPtr)(int, const char*[]); + std::vector ArgV; + for (auto &Arg : Args) + ArgV.push_back(Arg.c_str()); + auto Main = fromTargetAddress(cantFail(MainSym.getAddress())); + return Main(ArgV.size(), (const char**)ArgV.data()); + } else if (auto Err = MainSym.takeError()) + logAllUnhandledErrors(std::move(Err), llvm::errs(), ""); + else errs() << "Could not find main function.\n"; - return 1; - } - using MainFnPtr = int (*)(int, const char*[]); - std::vector ArgV; - for (auto &Arg : Args) - ArgV.push_back(Arg.c_str()); - auto Main = fromTargetAddress(MainSym.getAddress()); - return Main(ArgV.size(), (const char**)ArgV.data()); + return 1; } diff --git a/contrib/llvm/tools/lli/OrcLazyJIT.h b/contrib/llvm/tools/lli/OrcLazyJIT.h index fc02a10b514e..47a2acc4d7e6 100644 --- a/contrib/llvm/tools/lli/OrcLazyJIT.h +++ b/contrib/llvm/tools/lli/OrcLazyJIT.h @@ -61,7 +61,8 @@ class OrcLazyJIT { IndirectStubsManagerBuilder IndirectStubsMgrBuilder, bool InlineStubs) : TM(std::move(TM)), DL(this->TM->createDataLayout()), - CCMgr(std::move(CCMgr)), + CCMgr(std::move(CCMgr)), + ObjectLayer([]() { return std::make_shared(); }), CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)), IRDumpLayer(CompileLayer, createDebugDumper()), CODLayer(IRDumpLayer, extractSingleFunction, *this->CCMgr, @@ -74,10 +75,14 @@ class OrcLazyJIT { CXXRuntimeOverrides.runDestructors(); // Run any IR destructors. for (auto &DtorRunner : IRStaticDestructorRunners) - DtorRunner.runViaLayer(CODLayer); + if (auto Err = DtorRunner.runViaLayer(CODLayer)) { + // FIXME: OrcLazyJIT should probably take a "shutdownError" callback to + // report these errors on. + report_fatal_error(std::move(Err)); + } } - void addModule(std::shared_ptr M) { + Error addModule(std::shared_ptr M) { if (M->getDataLayout().isDefault()) M->setDataLayout(DL); @@ -124,21 +129,27 @@ class OrcLazyJIT { ); // Add the module to the JIT. - ModulesHandle = - CODLayer.addModule(std::move(M), - llvm::make_unique(), - std::move(Resolver)); + if (auto ModulesHandleOrErr = + CODLayer.addModule(std::move(M), std::move(Resolver))) + ModulesHandle = std::move(*ModulesHandleOrErr); + else + return ModulesHandleOrErr.takeError(); + } else - CODLayer.addExtraModule(ModulesHandle, std::move(M)); + if (auto Err = CODLayer.addExtraModule(ModulesHandle, std::move(M))) + return Err; // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. orc::CtorDtorRunner CtorRunner(std::move(CtorNames), ModulesHandle); - CtorRunner.runViaLayer(CODLayer); + if (auto Err = CtorRunner.runViaLayer(CODLayer)) + return Err; IRStaticDestructorRunners.emplace_back(std::move(DtorNames), ModulesHandle); + + return Error::success(); } JITSymbol findSymbol(const std::string &Name) { diff --git a/contrib/llvm/tools/lli/RemoteJITUtils.h b/contrib/llvm/tools/lli/RemoteJITUtils.h index 3c82f73ff072..4e948413865c 100644 --- a/contrib/llvm/tools/lli/RemoteJITUtils.h +++ b/contrib/llvm/tools/lli/RemoteJITUtils.h @@ -84,7 +84,7 @@ class ForwardingMemoryManager : public llvm::RTDyldMemoryManager { this->MemMgr = std::move(MemMgr); } - void setResolver(std::unique_ptr Resolver) { + void setResolver(std::shared_ptr Resolver) { this->Resolver = std::move(Resolver); } @@ -145,7 +145,7 @@ class ForwardingMemoryManager : public llvm::RTDyldMemoryManager { private: std::unique_ptr MemMgr; - std::unique_ptr Resolver; + std::shared_ptr Resolver; }; } diff --git a/contrib/llvm/tools/lli/lli.cpp b/contrib/llvm/tools/lli/lli.cpp index f228a3619457..091ca22b4e82 100644 --- a/contrib/llvm/tools/lli/lli.cpp +++ b/contrib/llvm/tools/lli/lli.cpp @@ -646,7 +646,7 @@ int main(int argc, char **argv, char * const *envp) { // else == "if (RemoteMCJIT)" // Remote target MCJIT doesn't (yet) support static constructors. No reason - // it couldn't. This is a limitation of the LLI implemantation, not the + // it couldn't. This is a limitation of the LLI implementation, not the // MCJIT itself. FIXME. // Lanch the remote process and get a channel to it. diff --git a/contrib/llvm/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp b/contrib/llvm/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp index 528247c2dbc3..529bdf5b7d93 100644 --- a/contrib/llvm/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp +++ b/contrib/llvm/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp @@ -71,6 +71,10 @@ static cl::opt ShowBinaryBlobs("show-binary-blobs", cl::desc("Print binary blobs using hex escapes")); +static cl::opt CheckHash( + "check-hash", + cl::desc("Check module hash using the argument as a string table")); + namespace { /// CurStreamTypeType - A type for CurStreamType @@ -652,13 +656,15 @@ static bool ParseBlock(BitstreamCursor &Stream, BitstreamBlockInfo &BlockInfo, } // If we found a module hash, let's verify that it matches! - if (BlockID == bitc::MODULE_BLOCK_ID && Code == bitc::MODULE_CODE_HASH) { + if (BlockID == bitc::MODULE_BLOCK_ID && Code == bitc::MODULE_CODE_HASH && + !CheckHash.empty()) { if (Record.size() != 5) outs() << " (invalid)"; else { // Recompute the hash and compare it to the one in the bitcode SHA1 Hasher; StringRef Hash; + Hasher.update(CheckHash); { int BlockSize = (CurrentRecordPos / 8) - BlockEntryPos; auto Ptr = Stream.getPointerToByte(BlockEntryPos, BlockSize); diff --git a/contrib/llvm/tools/llvm-cov/CodeCoverage.cpp b/contrib/llvm/tools/llvm-cov/CodeCoverage.cpp index 6179c760d5b2..3cbd6591134b 100644 --- a/contrib/llvm/tools/llvm-cov/CodeCoverage.cpp +++ b/contrib/llvm/tools/llvm-cov/CodeCoverage.cpp @@ -32,6 +32,7 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Program.h" #include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/Threading.h" #include "llvm/Support/ThreadPool.h" #include "llvm/Support/ToolOutputFile.h" #include @@ -705,6 +706,12 @@ int CodeCoverageTool::show(int argc, const char **argv, "project-title", cl::Optional, cl::desc("Set project title for the coverage report")); + cl::opt NumThreads( + "num-threads", cl::init(0), + cl::desc("Number of merge threads to use (default: autodetect)")); + cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"), + cl::aliasopt(NumThreads)); + auto Err = commandLineParser(argc, argv); if (Err) return Err; @@ -790,15 +797,19 @@ int CodeCoverageTool::show(int argc, const char **argv, } } - // FIXME: Sink the hardware_concurrency() == 1 check into ThreadPool. - if (!ViewOpts.hasOutputDirectory() || - std::thread::hardware_concurrency() == 1) { + // If NumThreads is not specified, auto-detect a good default. + if (NumThreads == 0) + NumThreads = + std::max(1U, std::min(llvm::heavyweight_hardware_concurrency(), + unsigned(SourceFiles.size()))); + + if (!ViewOpts.hasOutputDirectory() || NumThreads == 1) { for (const std::string &SourceFile : SourceFiles) writeSourceFileView(SourceFile, Coverage.get(), Printer.get(), ShowFilenames); } else { // In -output-dir mode, it's safe to use multiple threads to print files. - ThreadPool Pool; + ThreadPool Pool(NumThreads); for (const std::string &SourceFile : SourceFiles) Pool.async(&CodeCoverageTool::writeSourceFileView, this, SourceFile, Coverage.get(), Printer.get(), ShowFilenames); diff --git a/contrib/llvm/tools/llvm-lto/llvm-lto.cpp b/contrib/llvm/tools/llvm-lto/llvm-lto.cpp index ccc673be4570..87cd13ad70de 100644 --- a/contrib/llvm/tools/llvm-lto/llvm-lto.cpp +++ b/contrib/llvm/tools/llvm-lto/llvm-lto.cpp @@ -383,7 +383,7 @@ loadAllFilesForIndex(const ModuleSummaryIndex &Index) { for (auto &ModPath : Index.modulePaths()) { const auto &Filename = ModPath.first(); - auto CurrentActivity = "loading file '" + Filename + "'"; + std::string CurrentActivity = ("loading file '" + Filename + "'").str(); auto InputOrErr = MemoryBuffer::getFile(Filename); error(InputOrErr, "error " + CurrentActivity); InputBuffers.push_back(std::move(*InputOrErr)); @@ -475,7 +475,7 @@ class ThinLTOProcessing { std::vector> InputBuffers; for (unsigned i = 0; i < InputFilenames.size(); ++i) { auto &Filename = InputFilenames[i]; - StringRef CurrentActivity = "loading file '" + Filename + "'"; + std::string CurrentActivity = "loading file '" + Filename + "'"; auto InputOrErr = MemoryBuffer::getFile(Filename); error(InputOrErr, "error " + CurrentActivity); InputBuffers.push_back(std::move(*InputOrErr)); @@ -710,7 +710,7 @@ class ThinLTOProcessing { std::vector> InputBuffers; for (unsigned i = 0; i < InputFilenames.size(); ++i) { auto &Filename = InputFilenames[i]; - StringRef CurrentActivity = "loading file '" + Filename + "'"; + std::string CurrentActivity = "loading file '" + Filename + "'"; auto InputOrErr = MemoryBuffer::getFile(Filename); error(InputOrErr, "error " + CurrentActivity); InputBuffers.push_back(std::move(*InputOrErr)); diff --git a/contrib/llvm/tools/llvm-objdump/llvm-objdump.cpp b/contrib/llvm/tools/llvm-objdump/llvm-objdump.cpp index be5635a3d4c6..812f1af3ac68 100644 --- a/contrib/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/contrib/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -1032,7 +1032,7 @@ static std::error_code getRelocationValueString(const MachOObjectFile *Obj, case MachO::ARM_RELOC_HALF_SECTDIFF: { // Half relocations steal a bit from the length field to encode // whether this is an upper16 or a lower16 relocation. - bool isUpper = Obj->getAnyRelocationLength(RE) >> 1; + bool isUpper = (Obj->getAnyRelocationLength(RE) & 0x1) == 1; if (isUpper) fmt << ":upper16:("; diff --git a/contrib/llvm/tools/llvm-pdbutil/Diff.cpp b/contrib/llvm/tools/llvm-pdbutil/Diff.cpp index 9b38ae1d603e..aad4e1bf1427 100644 --- a/contrib/llvm/tools/llvm-pdbutil/Diff.cpp +++ b/contrib/llvm/tools/llvm-pdbutil/Diff.cpp @@ -9,22 +9,162 @@ #include "Diff.h" +#include "DiffPrinter.h" +#include "FormatUtil.h" #include "StreamUtil.h" #include "llvm-pdbutil.h" +#include "llvm/ADT/StringSet.h" + +#include "llvm/DebugInfo/PDB/Native/DbiStream.h" #include "llvm/DebugInfo/PDB/Native/Formatters.h" #include "llvm/DebugInfo/PDB/Native/InfoStream.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/FormatAdapters.h" #include "llvm/Support/FormatProviders.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Path.h" using namespace llvm; using namespace llvm::pdb; +namespace { +// Compare and format two stream numbers. Stream numbers are considered +// identical if they contain the same value, equivalent if they are both +// the invalid stream or neither is the invalid stream, and different if +// one is the invalid stream and another isn't. +struct StreamNumberProvider { + static DiffResult compare(uint16_t L, uint16_t R) { + if (L == R) + return DiffResult::IDENTICAL; + bool LP = L != kInvalidStreamIndex; + bool RP = R != kInvalidStreamIndex; + if (LP != RP) + return DiffResult::DIFFERENT; + return DiffResult::EQUIVALENT; + } + + static std::string format(uint16_t SN, bool Right) { + if (SN == kInvalidStreamIndex) + return "(not present)"; + return formatv("{0}", SN).str(); + } +}; + +// Compares and formats two module indices. Modis are considered identical +// if they are identical, equivalent if they either both contain a value or +// both don't contain a value, and different if one contains a value and the +// other doesn't. +struct ModiProvider { + DiffResult compare(Optional L, Optional R) { + if (L == R) + return DiffResult::IDENTICAL; + if (L.hasValue() != R.hasValue()) + return DiffResult::DIFFERENT; + return DiffResult::EQUIVALENT; + } + + std::string format(Optional Modi, bool Right) { + if (!Modi.hasValue()) + return "(not present)"; + return formatv("{0}", *Modi).str(); + } +}; + +// Compares and formats two paths embedded in the PDB, ignoring the beginning +// of the path if the user specified it as a "root path" on the command line. +struct BinaryPathProvider { + explicit BinaryPathProvider(uint32_t MaxLen) : MaxLen(MaxLen) {} + + DiffResult compare(StringRef L, StringRef R) { + if (L == R) + return DiffResult::IDENTICAL; + + SmallString<64> LN = removeRoot(L, false); + SmallString<64> RN = removeRoot(R, true); + + return (LN.equals_lower(RN)) ? DiffResult::EQUIVALENT + : DiffResult::DIFFERENT; + } + + std::string format(StringRef S, bool Right) { + if (S.empty()) + return "(empty)"; + + SmallString<64> Native = removeRoot(S, Right); + return truncateStringFront(Native.str(), MaxLen); + } + + SmallString<64> removeRoot(StringRef Path, bool IsRight) const { + SmallString<64> Native(Path); + auto &RootOpt = IsRight ? opts::diff::RightRoot : opts::diff::LeftRoot; + SmallString<64> Root(static_cast(RootOpt)); + // pdb paths always use windows syntax, convert slashes to backslashes. + sys::path::native(Root, sys::path::Style::windows); + if (sys::path::has_stem(Root, sys::path::Style::windows)) + sys::path::append(Root, sys::path::Style::windows, + sys::path::get_separator(sys::path::Style::windows)); + + sys::path::replace_path_prefix(Native, Root, "", sys::path::Style::windows); + return Native; + } + uint32_t MaxLen; +}; + +// Compare and format two stream purposes. For general streams, this just +// compares the description. For module streams it uses the path comparison +// algorithm taking into consideration the binary root, described above. +// Formatting stream purposes just prints the stream purpose, except for +// module streams and named streams, where it prefixes the name / module +// with an identifier. Example: +// +// Named Stream "\names" +// Module Stream "foo.obj" +// +// If a named stream is too long to fit in a column, it is truncated at the +// end, and if a module is too long to fit in a column, it is truncated at the +// beginning. Example: +// +// Named Stream "\Really Long Str..." +// Module Stream "...puts\foo.obj" +// +struct StreamPurposeProvider { + explicit StreamPurposeProvider(uint32_t MaxLen) : MaxLen(MaxLen) {} + + DiffResult compare(const std::pair &L, + const std::pair &R) { + if (L.first != R.first) + return DiffResult::DIFFERENT; + if (L.first == StreamPurpose::ModuleStream) { + BinaryPathProvider PathProvider(MaxLen); + return PathProvider.compare(L.second, R.second); + } + return (L.second == R.second) ? DiffResult::IDENTICAL + : DiffResult::DIFFERENT; + } + + std::string format(const std::pair &P, + bool Right) { + if (P.first == StreamPurpose::Other) + return truncateStringBack(P.second, MaxLen); + if (P.first == StreamPurpose::NamedStream) + return truncateQuotedNameBack("Named Stream", P.second, MaxLen); + + assert(P.first == StreamPurpose::ModuleStream); + uint32_t ExtraChars = strlen("Module \"\""); + BinaryPathProvider PathProvider(MaxLen - ExtraChars); + std::string Result = PathProvider.format(P.second, Right); + return formatv("Module \"{0}\"", Result); + } + + uint32_t MaxLen; +}; +} // namespace + namespace llvm { template <> struct format_provider { static void format(const PdbRaw_FeatureSig &Sig, raw_ostream &Stream, @@ -49,47 +189,6 @@ template <> struct format_provider { template using ValueOfRange = llvm::detail::ValueOfRange; -template -static void set_differences(Range &&R1, Range &&R2, - SmallVectorImpl> *OnlyLeft, - SmallVectorImpl> *OnlyRight, - SmallVectorImpl> *Intersection, - Comp Comparator) { - - std::sort(R1.begin(), R1.end(), Comparator); - std::sort(R2.begin(), R2.end(), Comparator); - - if (OnlyLeft) { - OnlyLeft->reserve(R1.size()); - auto End = std::set_difference(R1.begin(), R1.end(), R2.begin(), R2.end(), - OnlyLeft->begin(), Comparator); - OnlyLeft->set_size(std::distance(OnlyLeft->begin(), End)); - } - if (OnlyRight) { - OnlyLeft->reserve(R2.size()); - auto End = std::set_difference(R2.begin(), R2.end(), R1.begin(), R1.end(), - OnlyRight->begin(), Comparator); - OnlyRight->set_size(std::distance(OnlyRight->begin(), End)); - } - if (Intersection) { - Intersection->reserve(std::min(R1.size(), R2.size())); - auto End = std::set_intersection(R1.begin(), R1.end(), R2.begin(), R2.end(), - Intersection->begin(), Comparator); - Intersection->set_size(std::distance(Intersection->begin(), End)); - } -} - -template -static void -set_differences(Range &&R1, Range &&R2, - SmallVectorImpl> *OnlyLeft, - SmallVectorImpl> *OnlyRight, - SmallVectorImpl> *Intersection = nullptr) { - std::less> Comp; - set_differences(std::forward(R1), std::forward(R2), OnlyLeft, - OnlyRight, Intersection, Comp); -} - DiffStyle::DiffStyle(PDBFile &File1, PDBFile &File2) : File1(File1), File2(File2) {} @@ -136,300 +235,363 @@ Error DiffStyle::dump() { return Error::success(); } -template -static bool diffAndPrint(StringRef Label, PDBFile &File1, PDBFile &File2, T V1, - T V2) { - if (V1 == V2) { - outs() << formatv(" {0}: No differences detected!\n", Label); - return false; - } - - outs().indent(2) << Label << "\n"; - outs().indent(4) << formatv("{0}: {1}\n", File1.getFilePath(), V1); - outs().indent(4) << formatv("{0}: {1}\n", File2.getFilePath(), V2); - return true; -} - -template -static bool diffAndPrint(StringRef Label, PDBFile &File1, PDBFile &File2, - ArrayRef V1, ArrayRef V2) { - if (V1 == V2) { - outs() << formatv(" {0}: No differences detected!\n", Label); - return false; - } - - outs().indent(2) << Label << "\n"; - outs().indent(4) << formatv("{0}: {1}\n", File1.getFilePath(), - make_range(V1.begin(), V1.end())); - outs().indent(4) << formatv("{0}: {1}\n", File2.getFilePath(), - make_range(V2.begin(), V2.end())); - return true; -} - -template -static bool printSymmetricDifferences(PDBFile &File1, PDBFile &File2, - T &&OnlyRange1, T &&OnlyRange2, - StringRef Label) { - bool HasDiff = false; - if (!OnlyRange1.empty()) { - HasDiff = true; - outs() << formatv(" {0} {1}(s) only in ({2})\n", OnlyRange1.size(), Label, - File1.getFilePath()); - for (const auto &Item : OnlyRange1) - outs() << formatv(" {0}\n", Label, Item); - } - if (!OnlyRange2.empty()) { - HasDiff = true; - outs() << formatv(" {0} {1}(s) only in ({2})\n", OnlyRange2.size(), - File2.getFilePath()); - for (const auto &Item : OnlyRange2) - outs() << formatv(" {0}\n", Item); - } - return HasDiff; -} - Error DiffStyle::diffSuperBlock() { - outs() << "MSF Super Block: Searching for differences...\n"; - bool Diffs = false; - - Diffs |= diffAndPrint("Block Size", File1, File2, File1.getBlockSize(), - File2.getBlockSize()); - Diffs |= diffAndPrint("Block Count", File1, File2, File1.getBlockCount(), - File2.getBlockCount()); - Diffs |= diffAndPrint("Unknown 1", File1, File2, File1.getUnknown1(), - File2.getUnknown1()); - if (!Diffs) - outs() << "MSF Super Block: No differences detected...\n"; + DiffPrinter D(2, "MSF Super Block", 16, 20, opts::diff::PrintResultColumn, + opts::diff::PrintValueColumns, outs()); + D.printExplicit("File", DiffResult::UNSPECIFIED, + truncateStringFront(File1.getFilePath(), 18), + truncateStringFront(File2.getFilePath(), 18)); + D.print("Block Size", File1.getBlockSize(), File2.getBlockSize()); + D.print("Block Count", File1.getBlockCount(), File2.getBlockCount()); + D.print("Unknown 1", File1.getUnknown1(), File2.getUnknown1()); + D.print("Directory Size", File1.getNumDirectoryBytes(), + File2.getNumDirectoryBytes()); return Error::success(); } Error DiffStyle::diffStreamDirectory() { - SmallVector P; - SmallVector Q; + DiffPrinter D(2, "Stream Directory", 30, 20, opts::diff::PrintResultColumn, + opts::diff::PrintValueColumns, outs()); + D.printExplicit("File", DiffResult::UNSPECIFIED, + truncateStringFront(File1.getFilePath(), 18), + truncateStringFront(File2.getFilePath(), 18)); + + SmallVector, 32> P; + SmallVector, 32> Q; discoverStreamPurposes(File1, P); discoverStreamPurposes(File2, Q); - outs() << "Stream Directory: Searching for differences...\n"; - - bool HasDifferences = false; + D.print("Stream Count", File1.getNumStreams(), File2.getNumStreams()); auto PI = to_vector<32>(enumerate(P)); auto QI = to_vector<32>(enumerate(Q)); - typedef decltype(PI) ContainerType; - typedef typename ContainerType::value_type value_type; + // Scan all streams in the left hand side, looking for ones that are also + // in the right. Each time we find one, remove it. When we're done, Q + // should contain all the streams that are in the right but not in the left. + StreamPurposeProvider StreamProvider(28); + for (const auto &P : PI) { + typedef decltype(PI) ContainerType; + typedef typename ContainerType::value_type value_type; - auto Comparator = [](const value_type &I1, const value_type &I2) { - return I1.value() < I2.value(); - }; + auto Iter = llvm::find_if(QI, [P, &StreamProvider](const value_type &V) { + DiffResult Result = StreamProvider.compare(P.value(), V.value()); + return Result == DiffResult::EQUIVALENT || + Result == DiffResult::IDENTICAL; + }); - decltype(PI) OnlyP; - decltype(QI) OnlyQ; - decltype(PI) Common; - - set_differences(PI, QI, &OnlyP, &OnlyQ, &Common, Comparator); - - if (!OnlyP.empty()) { - HasDifferences = true; - outs().indent(2) << formatv("{0} Stream(s) only in ({1})\n", OnlyP.size(), - File1.getFilePath()); - for (auto &Item : OnlyP) { - outs().indent(4) << formatv("Stream {0} - {1}\n", Item.index(), - Item.value()); + if (Iter == QI.end()) { + D.printExplicit(StreamProvider.format(P.value(), false), + DiffResult::DIFFERENT, P.index(), "(not present)"); + continue; } + + D.print(StreamProvider.format(P.value(), false), + P.index(), Iter->index()); + QI.erase(Iter); } - if (!OnlyQ.empty()) { - HasDifferences = true; - outs().indent(2) << formatv("{0} Streams(s) only in ({1})\n", OnlyQ.size(), - File2.getFilePath()); - for (auto &Item : OnlyQ) { - outs().indent(4) << formatv("Stream {0} - {1}\n", Item.index(), - Item.value()); - } + for (const auto &Q : QI) { + D.printExplicit(StreamProvider.format(Q.value(), true), + DiffResult::DIFFERENT, "(not present)", Q.index()); } - if (!Common.empty()) { - outs().indent(2) << formatv("Found {0} common streams. Searching for " - "intra-stream differences.\n", - Common.size()); - bool HasCommonDifferences = false; - for (const auto &Left : Common) { - // Left was copied from the first range so its index refers to a stream - // index in the first file. Find the corresponding stream index in the - // second file. - auto Range = - std::equal_range(QI.begin(), QI.end(), Left, - [](const value_type &L, const value_type &R) { - return L.value() < R.value(); - }); - const auto &Right = *Range.first; - assert(Left.value() == Right.value()); - uint32_t LeftSize = File1.getStreamByteSize(Left.index()); - uint32_t RightSize = File2.getStreamByteSize(Right.index()); - if (LeftSize != RightSize) { - HasDifferences = true; - HasCommonDifferences = true; - outs().indent(4) << formatv("{0} ({1}: {2} bytes, {3}: {4} bytes)\n", - Left.value(), File1.getFilePath(), LeftSize, - File2.getFilePath(), RightSize); - } - } - if (!HasCommonDifferences) - outs().indent(2) << "Common Streams: No differences detected!\n"; - } - if (!HasDifferences) - outs() << "Stream Directory: No differences detected!\n"; return Error::success(); } Error DiffStyle::diffStringTable() { + DiffPrinter D(2, "String Table", 30, 20, opts::diff::PrintResultColumn, + opts::diff::PrintValueColumns, outs()); + D.printExplicit("File", DiffResult::UNSPECIFIED, + truncateStringFront(File1.getFilePath(), 18), + truncateStringFront(File2.getFilePath(), 18)); + auto ExpectedST1 = File1.getStringTable(); auto ExpectedST2 = File2.getStringTable(); - outs() << "String Table: Searching for differences...\n"; bool Has1 = !!ExpectedST1; bool Has2 = !!ExpectedST2; - if (!(Has1 && Has2)) { - // If one has a string table and the other doesn't, we can print less - // output. - if (Has1 != Has2) { - if (Has1) { - outs() << formatv(" {0}: ({1} strings)\n", File1.getFilePath(), - ExpectedST1->getNameCount()); - outs() << formatv(" {0}: (string table not present)\n", - File2.getFilePath()); - } else { - outs() << formatv(" {0}: (string table not present)\n", - File1.getFilePath()); - outs() << formatv(" {0}: ({1})\n", File2.getFilePath(), - ExpectedST2->getNameCount()); - } - } + std::string Count1 = Has1 ? llvm::utostr(ExpectedST1->getNameCount()) + : "(string table not present)"; + std::string Count2 = Has2 ? llvm::utostr(ExpectedST2->getNameCount()) + : "(string table not present)"; + D.print("Number of Strings", Count1, Count2); + + if (!Has1 || !Has2) { consumeError(ExpectedST1.takeError()); consumeError(ExpectedST2.takeError()); return Error::success(); } - bool HasDiff = false; auto &ST1 = *ExpectedST1; auto &ST2 = *ExpectedST2; - if (ST1.getByteSize() != ST2.getByteSize()) { - outs() << " Stream Size\n"; - outs() << formatv(" {0} - {1} byte(s)\n", File1.getFilePath(), - ST1.getByteSize()); - outs() << formatv(" {0} - {1} byte(s)\n", File2.getFilePath(), - ST2.getByteSize()); - outs() << formatv(" Difference: {0} bytes\n", - AbsoluteDifference(ST1.getByteSize(), ST2.getByteSize())); - HasDiff = true; - } - HasDiff |= diffAndPrint("Hash Version", File1, File2, ST1.getHashVersion(), - ST1.getHashVersion()); - HasDiff |= diffAndPrint("Signature", File1, File2, ST1.getSignature(), - ST1.getSignature()); + D.print("Hash Version", ST1.getHashVersion(), ST2.getHashVersion()); + D.print("Byte Size", ST1.getByteSize(), ST2.getByteSize()); + D.print("Signature", ST1.getSignature(), ST2.getSignature()); // Both have a valid string table, dive in and compare individual strings. auto IdList1 = ST1.name_ids(); auto IdList2 = ST2.name_ids(); - std::vector Strings1, Strings2; - Strings1.reserve(IdList1.size()); - Strings2.reserve(IdList2.size()); + StringSet<> LS; + StringSet<> RS; + uint32_t Empty1 = 0; + uint32_t Empty2 = 0; for (auto ID : IdList1) { auto S = ST1.getStringForID(ID); if (!S) return S.takeError(); - Strings1.push_back(*S); + if (S->empty()) + ++Empty1; + else + LS.insert(*S); } for (auto ID : IdList2) { auto S = ST2.getStringForID(ID); if (!S) return S.takeError(); - Strings2.push_back(*S); + if (S->empty()) + ++Empty2; + else + RS.insert(*S); + } + D.print("Empty Strings", Empty1, Empty2); + + for (const auto &S : LS) { + auto R = RS.find(S.getKey()); + std::string Truncated = truncateStringMiddle(S.getKey(), 28); + uint32_t I = cantFail(ST1.getIDForString(S.getKey())); + if (R == RS.end()) { + D.printExplicit(Truncated, DiffResult::DIFFERENT, I, "(not present)"); + continue; + } + + uint32_t J = cantFail(ST2.getIDForString(R->getKey())); + D.print(Truncated, I, J); + RS.erase(R); } - SmallVector OnlyP; - SmallVector OnlyQ; - auto End1 = std::remove(Strings1.begin(), Strings1.end(), ""); - auto End2 = std::remove(Strings2.begin(), Strings2.end(), ""); - uint32_t Empty1 = std::distance(End1, Strings1.end()); - uint32_t Empty2 = std::distance(End2, Strings2.end()); - Strings1.erase(End1, Strings1.end()); - Strings2.erase(End2, Strings2.end()); - set_differences(Strings1, Strings2, &OnlyP, &OnlyQ); - printSymmetricDifferences(File1, File2, OnlyP, OnlyQ, "String"); + for (const auto &S : RS) { + auto L = LS.find(S.getKey()); + std::string Truncated = truncateStringMiddle(S.getKey(), 28); + uint32_t J = cantFail(ST2.getIDForString(S.getKey())); + if (L == LS.end()) { + D.printExplicit(Truncated, DiffResult::DIFFERENT, "(not present)", J); + continue; + } - if (Empty1 != Empty2) { - PDBFile &MoreF = (Empty1 > Empty2) ? File1 : File2; - PDBFile &LessF = (Empty1 < Empty2) ? File1 : File2; - uint32_t Difference = AbsoluteDifference(Empty1, Empty2); - outs() << formatv(" {0} had {1} more empty strings than {2}\n", - MoreF.getFilePath(), Difference, LessF.getFilePath()); + uint32_t I = cantFail(ST1.getIDForString(L->getKey())); + D.print(Truncated, I, J); } - if (!HasDiff) - outs() << "String Table: No differences detected!\n"; return Error::success(); } Error DiffStyle::diffFreePageMap() { return Error::success(); } Error DiffStyle::diffInfoStream() { + DiffPrinter D(2, "PDB Stream", 22, 40, opts::diff::PrintResultColumn, + opts::diff::PrintValueColumns, outs()); + D.printExplicit("File", DiffResult::UNSPECIFIED, + truncateStringFront(File1.getFilePath(), 38), + truncateStringFront(File2.getFilePath(), 38)); + auto ExpectedInfo1 = File1.getPDBInfoStream(); auto ExpectedInfo2 = File2.getPDBInfoStream(); - outs() << "PDB Stream: Searching for differences...\n"; bool Has1 = !!ExpectedInfo1; bool Has2 = !!ExpectedInfo2; if (!(Has1 && Has2)) { - if (Has1 != Has2) - outs() << formatv("{0} does not have a PDB Stream!\n", - Has1 ? File1.getFilePath() : File2.getFilePath()); - consumeError(ExpectedInfo2.takeError()); + std::string L = Has1 ? "(present)" : "(not present)"; + std::string R = Has2 ? "(present)" : "(not present)"; + D.print("Stream", L, R); + + consumeError(ExpectedInfo1.takeError()); consumeError(ExpectedInfo2.takeError()); return Error::success(); } - bool HasDiff = false; auto &IS1 = *ExpectedInfo1; auto &IS2 = *ExpectedInfo2; - if (IS1.getStreamSize() != IS2.getStreamSize()) { - outs() << " Stream Size\n"; - outs() << formatv(" {0} - {1} byte(s)\n", File1.getFilePath(), - IS1.getStreamSize()); - outs() << formatv(" {0} - {1} byte(s)\n", File2.getFilePath(), - IS2.getStreamSize()); - outs() << formatv( - " Difference: {0} bytes\n", - AbsoluteDifference(IS1.getStreamSize(), IS2.getStreamSize())); - HasDiff = true; - } - HasDiff |= diffAndPrint("Age", File1, File2, IS1.getAge(), IS2.getAge()); - HasDiff |= diffAndPrint("Guid", File1, File2, IS1.getGuid(), IS2.getGuid()); - HasDiff |= diffAndPrint("Signature", File1, File2, IS1.getSignature(), - IS2.getSignature()); - HasDiff |= - diffAndPrint("Version", File1, File2, IS1.getVersion(), IS2.getVersion()); - HasDiff |= diffAndPrint("Features", File1, File2, IS1.getFeatureSignatures(), - IS2.getFeatureSignatures()); - HasDiff |= diffAndPrint("Named Stream Byte Size", File1, File2, - IS1.getNamedStreamMapByteSize(), - IS2.getNamedStreamMapByteSize()); - SmallVector NS1; - SmallVector NS2; - for (const auto &X : IS1.getNamedStreams().entries()) - NS1.push_back(X.getKey()); - for (const auto &X : IS2.getNamedStreams().entries()) - NS2.push_back(X.getKey()); - SmallVector OnlyP; - SmallVector OnlyQ; - set_differences(NS1, NS2, &OnlyP, &OnlyQ); - printSymmetricDifferences(File1, File2, OnlyP, OnlyQ, "Named Streams"); - if (!HasDiff) - outs() << "PDB Stream: No differences detected!\n"; - + D.print("Stream Size", IS1.getStreamSize(), IS2.getStreamSize()); + D.print("Age", IS1.getAge(), IS2.getAge()); + D.print("Guid", IS1.getGuid(), IS2.getGuid()); + D.print("Signature", IS1.getSignature(), IS2.getSignature()); + D.print("Version", IS1.getVersion(), IS2.getVersion()); + D.diffUnorderedArray("Feature", IS1.getFeatureSignatures(), + IS2.getFeatureSignatures()); + D.print("Named Stream Size", IS1.getNamedStreamMapByteSize(), + IS2.getNamedStreamMapByteSize()); + StringMap NSL = IS1.getNamedStreams().getStringMap(); + StringMap NSR = IS2.getNamedStreams().getStringMap(); + D.diffUnorderedMap("Named Stream", NSL, NSR); return Error::success(); } -Error DiffStyle::diffDbiStream() { return Error::success(); } +static std::vector> +getModuleDescriptors(const DbiModuleList &ML) { + std::vector> List; + List.reserve(ML.getModuleCount()); + for (uint32_t I = 0; I < ML.getModuleCount(); ++I) + List.emplace_back(I, ML.getModuleDescriptor(I)); + return List; +} + +static void +diffOneModule(DiffPrinter &D, + const std::pair Item, + std::vector> &Other, + bool ItemIsRight) { + StreamPurposeProvider HeaderProvider(70); + std::pair Header; + Header.first = StreamPurpose::ModuleStream; + Header.second = Item.second.getModuleName(); + D.printFullRow(HeaderProvider.format(Header, ItemIsRight)); + + const auto *L = &Item; + + BinaryPathProvider PathProvider(28); + auto Iter = llvm::find_if( + Other, [&PathProvider, ItemIsRight, + L](const std::pair &Other) { + const auto *Left = L; + const auto *Right = &Other; + if (ItemIsRight) + std::swap(Left, Right); + DiffResult Result = PathProvider.compare(Left->second.getModuleName(), + Right->second.getModuleName()); + return Result == DiffResult::EQUIVALENT || + Result == DiffResult::IDENTICAL; + }); + if (Iter == Other.end()) { + // We didn't find this module at all on the other side. Just print one row + // and continue. + D.print("- Modi", Item.first, None); + return; + } + + // We did find this module. Go through and compare each field. + const auto *R = &*Iter; + if (ItemIsRight) + std::swap(L, R); + + D.print("- Modi", L->first, R->first); + D.print("- Obj File Name", L->second.getObjFileName(), + R->second.getObjFileName(), PathProvider); + D.print("- Debug Stream", + L->second.getModuleStreamIndex(), + R->second.getModuleStreamIndex()); + D.print("- C11 Byte Size", L->second.getC11LineInfoByteSize(), + R->second.getC11LineInfoByteSize()); + D.print("- C13 Byte Size", L->second.getC13LineInfoByteSize(), + R->second.getC13LineInfoByteSize()); + D.print("- # of files", L->second.getNumberOfFiles(), + R->second.getNumberOfFiles()); + D.print("- Pdb File Path Index", L->second.getPdbFilePathNameIndex(), + R->second.getPdbFilePathNameIndex()); + D.print("- Source File Name Index", L->second.getSourceFileNameIndex(), + R->second.getSourceFileNameIndex()); + D.print("- Symbol Byte Size", L->second.getSymbolDebugInfoByteSize(), + R->second.getSymbolDebugInfoByteSize()); + Other.erase(Iter); +} + +Error DiffStyle::diffDbiStream() { + DiffPrinter D(2, "DBI Stream", 40, 30, opts::diff::PrintResultColumn, + opts::diff::PrintValueColumns, outs()); + D.printExplicit("File", DiffResult::UNSPECIFIED, + truncateStringFront(File1.getFilePath(), 28), + truncateStringFront(File2.getFilePath(), 28)); + + auto ExpectedDbi1 = File1.getPDBDbiStream(); + auto ExpectedDbi2 = File2.getPDBDbiStream(); + + bool Has1 = !!ExpectedDbi1; + bool Has2 = !!ExpectedDbi2; + if (!(Has1 && Has2)) { + std::string L = Has1 ? "(present)" : "(not present)"; + std::string R = Has2 ? "(present)" : "(not present)"; + D.print("Stream", L, R); + + consumeError(ExpectedDbi1.takeError()); + consumeError(ExpectedDbi2.takeError()); + return Error::success(); + } + + auto &DL = *ExpectedDbi1; + auto &DR = *ExpectedDbi2; + + D.print("Dbi Version", (uint32_t)DL.getDbiVersion(), + (uint32_t)DR.getDbiVersion()); + D.print("Age", DL.getAge(), DR.getAge()); + D.print("Machine", (uint16_t)DL.getMachineType(), + (uint16_t)DR.getMachineType()); + D.print("Flags", DL.getFlags(), DR.getFlags()); + D.print("Build Major", DL.getBuildMajorVersion(), DR.getBuildMajorVersion()); + D.print("Build Minor", DL.getBuildMinorVersion(), DR.getBuildMinorVersion()); + D.print("Build Number", DL.getBuildNumber(), DR.getBuildNumber()); + D.print("PDB DLL Version", DL.getPdbDllVersion(), DR.getPdbDllVersion()); + D.print("PDB DLL RBLD", DL.getPdbDllRbld(), DR.getPdbDllRbld()); + D.print("DBG (FPO)", + DL.getDebugStreamIndex(DbgHeaderType::FPO), + DR.getDebugStreamIndex(DbgHeaderType::FPO)); + D.print( + "DBG (Exception)", DL.getDebugStreamIndex(DbgHeaderType::Exception), + DR.getDebugStreamIndex(DbgHeaderType::Exception)); + D.print("DBG (Fixup)", + DL.getDebugStreamIndex(DbgHeaderType::Fixup), + DR.getDebugStreamIndex(DbgHeaderType::Fixup)); + D.print( + "DBG (OmapToSrc)", DL.getDebugStreamIndex(DbgHeaderType::OmapToSrc), + DR.getDebugStreamIndex(DbgHeaderType::OmapToSrc)); + D.print( + "DBG (OmapFromSrc)", DL.getDebugStreamIndex(DbgHeaderType::OmapFromSrc), + DR.getDebugStreamIndex(DbgHeaderType::OmapFromSrc)); + D.print( + "DBG (SectionHdr)", DL.getDebugStreamIndex(DbgHeaderType::SectionHdr), + DR.getDebugStreamIndex(DbgHeaderType::SectionHdr)); + D.print( + "DBG (TokenRidMap)", DL.getDebugStreamIndex(DbgHeaderType::TokenRidMap), + DR.getDebugStreamIndex(DbgHeaderType::TokenRidMap)); + D.print("DBG (Xdata)", + DL.getDebugStreamIndex(DbgHeaderType::Xdata), + DR.getDebugStreamIndex(DbgHeaderType::Xdata)); + D.print("DBG (Pdata)", + DL.getDebugStreamIndex(DbgHeaderType::Pdata), + DR.getDebugStreamIndex(DbgHeaderType::Pdata)); + D.print("DBG (NewFPO)", + DL.getDebugStreamIndex(DbgHeaderType::NewFPO), + DR.getDebugStreamIndex(DbgHeaderType::NewFPO)); + D.print( + "DBG (SectionHdrOrig)", + DL.getDebugStreamIndex(DbgHeaderType::SectionHdrOrig), + DR.getDebugStreamIndex(DbgHeaderType::SectionHdrOrig)); + D.print("Globals Stream", + DL.getGlobalSymbolStreamIndex(), + DR.getGlobalSymbolStreamIndex()); + D.print("Publics Stream", + DL.getPublicSymbolStreamIndex(), + DR.getPublicSymbolStreamIndex()); + D.print("Symbol Records", DL.getSymRecordStreamIndex(), + DR.getSymRecordStreamIndex()); + D.print("Has CTypes", DL.hasCTypes(), DR.hasCTypes()); + D.print("Is Incrementally Linked", DL.isIncrementallyLinked(), + DR.isIncrementallyLinked()); + D.print("Is Stripped", DL.isStripped(), DR.isStripped()); + const DbiModuleList &ML = DL.modules(); + const DbiModuleList &MR = DR.modules(); + D.print("Module Count", ML.getModuleCount(), MR.getModuleCount()); + D.print("Source File Count", ML.getSourceFileCount(), + MR.getSourceFileCount()); + auto MDL = getModuleDescriptors(ML); + auto MDR = getModuleDescriptors(MR); + // Scan all module descriptors from the left, and look for corresponding + // module descriptors on the right. + for (const auto &L : MDL) + diffOneModule(D, L, MDR, false); + + for (const auto &R : MDR) + diffOneModule(D, R, MDL, true); + + return Error::success(); +} Error DiffStyle::diffSectionContribs() { return Error::success(); } diff --git a/contrib/llvm/tools/llvm-pdbutil/DiffPrinter.cpp b/contrib/llvm/tools/llvm-pdbutil/DiffPrinter.cpp new file mode 100644 index 000000000000..dd61cc182593 --- /dev/null +++ b/contrib/llvm/tools/llvm-pdbutil/DiffPrinter.cpp @@ -0,0 +1,147 @@ + +#include "DiffPrinter.h" + +#include "llvm/Support/FormatAdapters.h" + +using namespace llvm; +using namespace llvm::pdb; + +namespace { +struct Colorize { + Colorize(raw_ostream &OS, DiffResult Result) : OS(OS) { + if (!OS.has_colors()) + return; + switch (Result) { + case DiffResult::IDENTICAL: + OS.changeColor(raw_ostream::Colors::GREEN, false); + break; + case DiffResult::EQUIVALENT: + OS.changeColor(raw_ostream::Colors::YELLOW, true); + break; + default: + OS.changeColor(raw_ostream::Colors::RED, false); + break; + } + } + + ~Colorize() { + if (OS.has_colors()) + OS.resetColor(); + } + + raw_ostream &OS; +}; +} + +DiffPrinter::DiffPrinter(uint32_t Indent, StringRef Header, + uint32_t PropertyWidth, uint32_t FieldWidth, + bool Result, bool Fields, raw_ostream &Stream) + : PrintResult(Result), PrintValues(Fields), Indent(Indent), + PropertyWidth(PropertyWidth), FieldWidth(FieldWidth), OS(Stream) { + printHeaderRow(); + printFullRow(Header); +} + +DiffPrinter::~DiffPrinter() {} + +uint32_t DiffPrinter::tableWidth() const { + // `|` + uint32_t W = 1; + + // `|` + W += PropertyWidth + 1; + + if (PrintResult) { + // ` I |` + W += 4; + } + + if (PrintValues) { + // `||` + W += 2 * (FieldWidth + 1); + } + return W; +} + +void DiffPrinter::printFullRow(StringRef Text) { + newLine(); + printValue(Text, DiffResult::UNSPECIFIED, AlignStyle::Center, + tableWidth() - 2, true); + printSeparatorRow(); +} + +void DiffPrinter::printSeparatorRow() { + newLine(); + OS << formatv("{0}", fmt_repeat('-', PropertyWidth)); + if (PrintResult) { + OS << '+'; + OS << formatv("{0}", fmt_repeat('-', 3)); + } + if (PrintValues) { + OS << '+'; + OS << formatv("{0}", fmt_repeat('-', FieldWidth)); + OS << '+'; + OS << formatv("{0}", fmt_repeat('-', FieldWidth)); + } + OS << '|'; +} + +void DiffPrinter::printHeaderRow() { + newLine('-'); + OS << formatv("{0}", fmt_repeat('-', tableWidth() - 1)); +} + +void DiffPrinter::newLine(char InitialChar) { + OS << "\n"; + OS.indent(Indent) << InitialChar; +} + +void DiffPrinter::printExplicit(StringRef Property, DiffResult C, + StringRef Left, StringRef Right) { + newLine(); + printValue(Property, DiffResult::UNSPECIFIED, AlignStyle::Right, + PropertyWidth, true); + printResult(C); + printValue(Left, C, AlignStyle::Center, FieldWidth, false); + printValue(Right, C, AlignStyle::Center, FieldWidth, false); + printSeparatorRow(); +} + +void DiffPrinter::printResult(DiffResult Result) { + if (!PrintResult) + return; + switch (Result) { + case DiffResult::DIFFERENT: + printValue("D", Result, AlignStyle::Center, 3, true); + break; + case DiffResult::EQUIVALENT: + printValue("E", Result, AlignStyle::Center, 3, true); + break; + case DiffResult::IDENTICAL: + printValue("I", Result, AlignStyle::Center, 3, true); + break; + case DiffResult::UNSPECIFIED: + printValue(" ", Result, AlignStyle::Center, 3, true); + break; + } +} + +void DiffPrinter::printValue(StringRef Value, DiffResult C, AlignStyle Style, + uint32_t Width, bool Force) { + if (!Force && !PrintValues) + return; + + if (Style == AlignStyle::Right) + --Width; + + std::string FormattedItem = + formatv("{0}", fmt_align(Value, Style, Width)).str(); + if (C != DiffResult::UNSPECIFIED) { + Colorize Color(OS, C); + OS << FormattedItem; + } else + OS << FormattedItem; + if (Style == AlignStyle::Right) + OS << ' '; + OS << '|'; +} diff --git a/contrib/llvm/tools/llvm-pdbutil/DiffPrinter.h b/contrib/llvm/tools/llvm-pdbutil/DiffPrinter.h new file mode 100644 index 000000000000..475747d8dc11 --- /dev/null +++ b/contrib/llvm/tools/llvm-pdbutil/DiffPrinter.h @@ -0,0 +1,172 @@ +//===- DiffPrinter.h ------------------------------------------ *- C++ --*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVMPDBDUMP_DIFFPRINTER_H +#define LLVM_TOOLS_LLVMPDBDUMP_DIFFPRINTER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +namespace std { +template <> struct hash { + typedef llvm::pdb::PdbRaw_FeatureSig argument_type; + typedef std::size_t result_type; + result_type operator()(argument_type Item) const { + return std::hash{}(uint32_t(Item)); + } +}; +} // namespace std + +namespace llvm { +namespace pdb { + +class PDBFile; + +enum class DiffResult { UNSPECIFIED, IDENTICAL, EQUIVALENT, DIFFERENT }; + +struct IdenticalDiffProvider { + template + DiffResult compare(const T &Left, const U &Right) { + return (Left == Right) ? DiffResult::IDENTICAL : DiffResult::DIFFERENT; + } + + template std::string format(const T &Item, bool Right) { + return formatv("{0}", Item).str(); + } +}; + +struct EquivalentDiffProvider { + template + DiffResult compare(const T &Left, const U &Right) { + return (Left == Right) ? DiffResult::IDENTICAL : DiffResult::EQUIVALENT; + } + + template std::string format(const T &Item, bool Right) { + return formatv("{0}", Item).str(); + } +}; + +class DiffPrinter { +public: + DiffPrinter(uint32_t Indent, StringRef Header, uint32_t PropertyWidth, + uint32_t FieldWidth, bool Result, bool Values, + raw_ostream &Stream); + ~DiffPrinter(); + + template struct Identical {}; + + template + void print(StringRef Property, const T &Left, const U &Right, + Provider P = Provider()) { + std::string L = P.format(Left, false); + std::string R = P.format(Right, true); + + DiffResult Result = P.compare(Left, Right); + printExplicit(Property, Result, L, R); + } + + void printExplicit(StringRef Property, DiffResult C, StringRef Left, + StringRef Right); + + template + void printExplicit(StringRef Property, DiffResult C, const T &Left, + const U &Right) { + std::string L = formatv("{0}", Left).str(); + std::string R = formatv("{0}", Right).str(); + printExplicit(Property, C, StringRef(L), StringRef(R)); + } + + template + void diffUnorderedArray(StringRef Property, ArrayRef Left, + ArrayRef Right) { + std::unordered_set LS(Left.begin(), Left.end()); + std::unordered_set RS(Right.begin(), Right.end()); + std::string Count1 = formatv("{0} element(s)", Left.size()); + std::string Count2 = formatv("{0} element(s)", Right.size()); + print(std::string(Property) + "s (set)", Count1, Count2); + for (const auto &L : LS) { + auto Iter = RS.find(L); + std::string Text = formatv("{0}", L).str(); + if (Iter == RS.end()) { + print(Property, Text, "(not present)"); + continue; + } + print(Property, Text, Text); + RS.erase(Iter); + } + for (const auto &R : RS) { + auto Iter = LS.find(R); + std::string Text = formatv("{0}", R).str(); + if (Iter == LS.end()) { + print(Property, "(not present)", Text); + continue; + } + print(Property, Text, Text); + } + } + + template + void diffUnorderedMap(StringRef Property, const StringMap &Left, + const StringMap &Right, + ValueProvider P = ValueProvider()) { + StringMap RightCopy(Right); + + std::string Count1 = formatv("{0} element(s)", Left.size()); + std::string Count2 = formatv("{0} element(s)", Right.size()); + print(std::string(Property) + "s (map)", Count1, Count2); + + for (const auto &L : Left) { + auto Iter = RightCopy.find(L.getKey()); + if (Iter == RightCopy.end()) { + printExplicit(L.getKey(), DiffResult::DIFFERENT, L.getValue(), + "(not present)"); + continue; + } + + print(L.getKey(), L.getValue(), Iter->getValue(), P); + RightCopy.erase(Iter); + } + + for (const auto &R : RightCopy) { + printExplicit(R.getKey(), DiffResult::DIFFERENT, "(not present)", + R.getValue()); + } + } + + void printFullRow(StringRef Text); + +private: + uint32_t tableWidth() const; + + void printHeaderRow(); + void printSeparatorRow(); + void newLine(char InitialChar = '|'); + void printValue(StringRef Value, DiffResult C, AlignStyle Style, + uint32_t Width, bool Force); + void printResult(DiffResult Result); + + bool PrintResult; + bool PrintValues; + uint32_t Indent; + uint32_t PropertyWidth; + uint32_t FieldWidth; + raw_ostream &OS; +}; +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp b/contrib/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp index a1f919b4dd06..0642d841fd9f 100644 --- a/contrib/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp +++ b/contrib/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp @@ -418,6 +418,13 @@ Error DumpOutputStyle::dumpModules() { P.formatLine(" debug stream: {0}, # files: {1}, has ec info: {2}", Modi.getModuleStreamIndex(), Modi.getNumberOfFiles(), Modi.hasECInfo()); + StringRef PdbFilePath = + Err(Stream.getECName(Modi.getPdbFilePathNameIndex())); + StringRef SrcFilePath = + Err(Stream.getECName(Modi.getSourceFileNameIndex())); + P.formatLine(" pdb file ni: {0} `{1}`, src file ni: {2} `{3}`", + Modi.getPdbFilePathNameIndex(), PdbFilePath, + Modi.getSourceFileNameIndex(), SrcFilePath); } return Error::success(); } diff --git a/contrib/llvm/tools/llvm-pdbutil/FormatUtil.cpp b/contrib/llvm/tools/llvm-pdbutil/FormatUtil.cpp index 1bbe2724f0ab..02030272dd4d 100644 --- a/contrib/llvm/tools/llvm-pdbutil/FormatUtil.cpp +++ b/contrib/llvm/tools/llvm-pdbutil/FormatUtil.cpp @@ -16,6 +16,58 @@ using namespace llvm; using namespace llvm::pdb; +std::string llvm::pdb::truncateStringBack(StringRef S, uint32_t MaxLen) { + if (MaxLen == 0 || S.size() <= MaxLen || S.size() <= 3) + return S; + + assert(MaxLen >= 3); + uint32_t FinalLen = std::min(S.size(), MaxLen - 3); + S = S.take_front(FinalLen); + return std::string(S) + std::string("..."); +} + +std::string llvm::pdb::truncateStringMiddle(StringRef S, uint32_t MaxLen) { + if (MaxLen == 0 || S.size() <= MaxLen || S.size() <= 3) + return S; + + assert(MaxLen >= 3); + uint32_t FinalLen = std::min(S.size(), MaxLen - 3); + StringRef Front = S.take_front(FinalLen / 2); + StringRef Back = S.take_back(Front.size()); + return std::string(Front) + std::string("...") + std::string(Back); +} + +std::string llvm::pdb::truncateStringFront(StringRef S, uint32_t MaxLen) { + if (MaxLen == 0 || S.size() <= MaxLen || S.size() <= 3) + return S; + + assert(MaxLen >= 3); + S = S.take_back(MaxLen - 3); + return std::string("...") + std::string(S); +} + +std::string llvm::pdb::truncateQuotedNameFront(StringRef Label, StringRef Name, + uint32_t MaxLen) { + uint32_t RequiredExtraChars = Label.size() + 1 + 2; + if (MaxLen == 0 || RequiredExtraChars + Name.size() <= MaxLen) + return formatv("{0} \"{1}\"", Label, Name).str(); + + assert(MaxLen >= RequiredExtraChars); + std::string TN = truncateStringFront(Name, MaxLen - RequiredExtraChars); + return formatv("{0} \"{1}\"", Label, TN).str(); +} + +std::string llvm::pdb::truncateQuotedNameBack(StringRef Label, StringRef Name, + uint32_t MaxLen) { + uint32_t RequiredExtraChars = Label.size() + 1 + 2; + if (MaxLen == 0 || RequiredExtraChars + Name.size() <= MaxLen) + return formatv("{0} \"{1}\"", Label, Name).str(); + + assert(MaxLen >= RequiredExtraChars); + std::string TN = truncateStringBack(Name, MaxLen - RequiredExtraChars); + return formatv("{0} \"{1}\"", Label, TN).str(); +} + std::string llvm::pdb::typesetItemList(ArrayRef Opts, uint32_t IndentLevel, uint32_t GroupSize, StringRef Sep) { diff --git a/contrib/llvm/tools/llvm-pdbutil/FormatUtil.h b/contrib/llvm/tools/llvm-pdbutil/FormatUtil.h index 3db2dbacc57b..df32ed9360fb 100644 --- a/contrib/llvm/tools/llvm-pdbutil/FormatUtil.h +++ b/contrib/llvm/tools/llvm-pdbutil/FormatUtil.h @@ -22,6 +22,14 @@ namespace llvm { namespace pdb { +std::string truncateStringBack(StringRef S, uint32_t MaxLen); +std::string truncateStringMiddle(StringRef S, uint32_t MaxLen); +std::string truncateStringFront(StringRef S, uint32_t MaxLen); +std::string truncateQuotedNameFront(StringRef Label, StringRef Name, + uint32_t MaxLen); +std::string truncateQuotedNameBack(StringRef Label, StringRef Name, + uint32_t MaxLen); + #define PUSH_MASKED_FLAG(Enum, Mask, TheOpt, Value, Text) \ if (Enum::TheOpt == (Value & Mask)) \ Opts.push_back(Text); @@ -33,7 +41,7 @@ namespace pdb { case Enum::X: \ return Ret; -template static std::string formatUnknownEnum(T Value) { +template std::string formatUnknownEnum(T Value) { return formatv("unknown ({0})", static_cast::type>(Value)) .str(); diff --git a/contrib/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp b/contrib/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp index 1af53e35ed11..9621320ea99a 100644 --- a/contrib/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp +++ b/contrib/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp @@ -299,7 +299,7 @@ Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, ClassRecord &Class) { - P.formatLine("class name: `{0}`", Class.Name); + P.format(" `{0}`", Class.Name); if (Class.hasUniqueName()) P.formatLine("unique name: `{0}`", Class.UniqueName); P.formatLine("vtable: {0}, base list: {1}, field list: {2}", @@ -311,7 +311,7 @@ Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, UnionRecord &Union) { - P.formatLine("class name: `{0}`", Union.Name); + P.format(" `{0}`", Union.Name); if (Union.hasUniqueName()) P.formatLine("unique name: `{0}`", Union.UniqueName); P.formatLine("field list: {0}", Union.FieldList); @@ -321,7 +321,7 @@ Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, } Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, EnumRecord &Enum) { - P.formatLine("name: `{0}`", Enum.Name); + P.format(" `{0}`", Enum.Name); if (Enum.hasUniqueName()) P.formatLine("unique name: `{0}`", Enum.UniqueName); P.formatLine("field list: {0}, underlying type: {1}", Enum.FieldList, diff --git a/contrib/llvm/tools/llvm-pdbutil/StreamUtil.cpp b/contrib/llvm/tools/llvm-pdbutil/StreamUtil.cpp index 81aa256b5002..4d352004dec3 100644 --- a/contrib/llvm/tools/llvm-pdbutil/StreamUtil.cpp +++ b/contrib/llvm/tools/llvm-pdbutil/StreamUtil.cpp @@ -8,6 +8,7 @@ //===----------------------------------------------------------------------===// #include "StreamUtil.h" +#include "FormatUtil.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" @@ -18,11 +19,12 @@ #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/TpiStream.h" -namespace llvm { -namespace pdb { -void discoverStreamPurposes(PDBFile &File, - SmallVectorImpl &Purposes) { +using namespace llvm; +using namespace llvm::pdb; +void llvm::pdb::discoverStreamPurposes( + PDBFile &File, + SmallVectorImpl> &Purposes) { // It's OK if we fail to load some of these streams, we still attempt to print // what we can. auto Dbi = File.getPDBDbiStream(); @@ -52,74 +54,72 @@ void discoverStreamPurposes(PDBFile &File, Purposes.resize(StreamCount); for (uint16_t StreamIdx = 0; StreamIdx < StreamCount; ++StreamIdx) { - std::string Value; + std::pair Value; if (StreamIdx == OldMSFDirectory) - Value = "Old MSF Directory"; + Value = std::make_pair(StreamPurpose::Other, "Old MSF Directory"); else if (StreamIdx == StreamPDB) - Value = "PDB Stream"; + Value = std::make_pair(StreamPurpose::Other, "PDB Stream"); else if (StreamIdx == StreamDBI) - Value = "DBI Stream"; + Value = std::make_pair(StreamPurpose::Other, "DBI Stream"); else if (StreamIdx == StreamTPI) - Value = "TPI Stream"; + Value = std::make_pair(StreamPurpose::Other, "TPI Stream"); else if (StreamIdx == StreamIPI) - Value = "IPI Stream"; + Value = std::make_pair(StreamPurpose::Other, "IPI Stream"); else if (Dbi && StreamIdx == Dbi->getGlobalSymbolStreamIndex()) - Value = "Global Symbol Hash"; + Value = std::make_pair(StreamPurpose::Other, "Global Symbol Hash"); else if (Dbi && StreamIdx == Dbi->getPublicSymbolStreamIndex()) - Value = "Public Symbol Hash"; + Value = std::make_pair(StreamPurpose::Other, "Public Symbol Hash"); else if (Dbi && StreamIdx == Dbi->getSymRecordStreamIndex()) - Value = "Public Symbol Records"; + Value = std::make_pair(StreamPurpose::Other, "Public Symbol Records"); else if (Tpi && StreamIdx == Tpi->getTypeHashStreamIndex()) - Value = "TPI Hash"; + Value = std::make_pair(StreamPurpose::Other, "TPI Hash"); else if (Tpi && StreamIdx == Tpi->getTypeHashStreamAuxIndex()) - Value = "TPI Aux Hash"; + Value = std::make_pair(StreamPurpose::Other, "TPI Aux Hash"); else if (Ipi && StreamIdx == Ipi->getTypeHashStreamIndex()) - Value = "IPI Hash"; + Value = std::make_pair(StreamPurpose::Other, "IPI Hash"); else if (Ipi && StreamIdx == Ipi->getTypeHashStreamAuxIndex()) - Value = "IPI Aux Hash"; + Value = std::make_pair(StreamPurpose::Other, "IPI Aux Hash"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Exception)) - Value = "Exception Data"; + Value = std::make_pair(StreamPurpose::Other, "Exception Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Fixup)) - Value = "Fixup Data"; + Value = std::make_pair(StreamPurpose::Other, "Fixup Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::FPO)) - Value = "FPO Data"; + Value = std::make_pair(StreamPurpose::Other, "FPO Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::NewFPO)) - Value = "New FPO Data"; + Value = std::make_pair(StreamPurpose::Other, "New FPO Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::OmapFromSrc)) - Value = "Omap From Source Data"; + Value = std::make_pair(StreamPurpose::Other, "Omap From Source Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::OmapToSrc)) - Value = "Omap To Source Data"; + Value = std::make_pair(StreamPurpose::Other, "Omap To Source Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Pdata)) - Value = "Pdata"; + Value = std::make_pair(StreamPurpose::Other, "Pdata"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::SectionHdr)) - Value = "Section Header Data"; + Value = std::make_pair(StreamPurpose::Other, "Section Header Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::SectionHdrOrig)) - Value = "Section Header Original Data"; + Value = + std::make_pair(StreamPurpose::Other, "Section Header Original Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::TokenRidMap)) - Value = "Token Rid Data"; + Value = std::make_pair(StreamPurpose::Other, "Token Rid Data"); else if (Dbi && StreamIdx == Dbi->getDebugStreamIndex(DbgHeaderType::Xdata)) - Value = "Xdata"; + Value = std::make_pair(StreamPurpose::Other, "Xdata"); else { auto ModIter = ModStreams.find(StreamIdx); auto NSIter = NamedStreams.find(StreamIdx); if (ModIter != ModStreams.end()) { - Value = "Module \""; - Value += ModIter->second.getModuleName(); - Value += "\""; + Value = std::make_pair(StreamPurpose::ModuleStream, + ModIter->second.getModuleName()); } else if (NSIter != NamedStreams.end()) { - Value = "Named Stream \""; - Value += NSIter->second; - Value += "\""; + Value = std::make_pair(StreamPurpose::NamedStream, NSIter->second); } else { - Value = "???"; + Value = std::make_pair(StreamPurpose::Other, "???"); } } Purposes[StreamIdx] = Value; @@ -135,5 +135,18 @@ void discoverStreamPurposes(PDBFile &File, if (!Info) consumeError(Info.takeError()); } -} + +void llvm::pdb::discoverStreamPurposes(PDBFile &File, + SmallVectorImpl &Purposes) { + SmallVector, 24> SP; + discoverStreamPurposes(File, SP); + Purposes.reserve(SP.size()); + for (const auto &P : SP) { + if (P.first == StreamPurpose::NamedStream) + Purposes.push_back(formatv("Named Stream \"{0}\"", P.second)); + else if (P.first == StreamPurpose::ModuleStream) + Purposes.push_back(formatv("Module \"{0}\"", P.second)); + else + Purposes.push_back(P.second); + } } diff --git a/contrib/llvm/tools/llvm-pdbutil/StreamUtil.h b/contrib/llvm/tools/llvm-pdbutil/StreamUtil.h index b5c0beba44fe..f49c0a0eceb6 100644 --- a/contrib/llvm/tools/llvm-pdbutil/StreamUtil.h +++ b/contrib/llvm/tools/llvm-pdbutil/StreamUtil.h @@ -17,8 +17,13 @@ namespace llvm { namespace pdb { class PDBFile; +enum class StreamPurpose { NamedStream, ModuleStream, Other }; + void discoverStreamPurposes(PDBFile &File, SmallVectorImpl &Purposes); +void discoverStreamPurposes( + PDBFile &File, + SmallVectorImpl> &Purposes); } } diff --git a/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp b/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp index ad11ad498000..6aa08ff3cd87 100644 --- a/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp +++ b/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp @@ -284,9 +284,32 @@ cl::opt NoEnumDefs("no-enum-definitions", } namespace diff { -cl::list InputFilenames(cl::Positional, - cl::desc(" "), - cl::OneOrMore, cl::sub(DiffSubcommand)); +cl::opt PrintValueColumns( + "values", cl::init(true), + cl::desc("Print one column for each PDB with the field value"), + cl::Optional, cl::sub(DiffSubcommand)); +cl::opt + PrintResultColumn("result", cl::init(false), + cl::desc("Print a column with the result status"), + cl::Optional, cl::sub(DiffSubcommand)); + +cl::opt LeftRoot( + "left-bin-root", cl::Optional, + cl::desc("Treats the specified path as the root of the tree containing " + "binaries referenced by the left PDB. The root is stripped from " + "embedded paths when doing equality comparisons."), + cl::sub(DiffSubcommand)); +cl::opt RightRoot( + "right-bin-root", cl::Optional, + cl::desc("Treats the specified path as the root of the tree containing " + "binaries referenced by the right PDB. The root is stripped from " + "embedded paths when doing equality comparisons"), + cl::sub(DiffSubcommand)); + +cl::opt Left(cl::Positional, cl::desc(""), + cl::sub(DiffSubcommand)); +cl::opt Right(cl::Positional, cl::desc(""), + cl::sub(DiffSubcommand)); } cl::OptionCategory FileOptions("Module & File Options"); @@ -399,7 +422,7 @@ cl::opt DumpTypeExtras("type-extras", cl::cat(TypeOptions), cl::sub(DumpSubcommand)); cl::list DumpTypeIndex( - "type-index", cl::ZeroOrMore, + "type-index", cl::ZeroOrMore, cl::CommaSeparated, cl::desc("only dump types with the specified hexadecimal type index"), cl::cat(TypeOptions), cl::sub(DumpSubcommand)); @@ -415,7 +438,7 @@ cl::opt DumpIdExtras("id-extras", cl::desc("dump id hashes and index offsets"), cl::cat(TypeOptions), cl::sub(DumpSubcommand)); cl::list DumpIdIndex( - "id-index", cl::ZeroOrMore, + "id-index", cl::ZeroOrMore, cl::CommaSeparated, cl::desc("only dump ids with the specified hexadecimal type index"), cl::cat(TypeOptions), cl::sub(DumpSubcommand)); @@ -1079,6 +1102,11 @@ int main(int argc_, const char *argv_[]) { if (opts::pdb2yaml::DumpModules) opts::pdb2yaml::DbiStream = true; } + if (opts::DiffSubcommand) { + if (!opts::diff::PrintResultColumn && !opts::diff::PrintValueColumns) { + llvm::errs() << "WARNING: No diff columns specified\n"; + } + } llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded); @@ -1137,11 +1165,7 @@ int main(int argc_, const char *argv_[]) { std::for_each(opts::bytes::InputFilenames.begin(), opts::bytes::InputFilenames.end(), dumpBytes); } else if (opts::DiffSubcommand) { - if (opts::diff::InputFilenames.size() != 2) { - errs() << "diff subcommand expects exactly 2 arguments.\n"; - exit(1); - } - diff(opts::diff::InputFilenames[0], opts::diff::InputFilenames[1]); + diff(opts::diff::Left, opts::diff::Right); } else if (opts::MergeSubcommand) { if (opts::merge::InputFilenames.size() < 2) { errs() << "merge subcommand requires at least 2 input files.\n"; diff --git a/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.h b/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.h index 9ee5866bbeff..4e92e639a127 100644 --- a/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.h +++ b/contrib/llvm/tools/llvm-pdbutil/llvm-pdbutil.h @@ -168,6 +168,13 @@ extern llvm::cl::opt DumpModuleFiles; extern llvm::cl::list DumpModuleSubsections; extern llvm::cl::opt DumpModuleSyms; } // namespace pdb2yaml + +namespace diff { +extern llvm::cl::opt PrintValueColumns; +extern llvm::cl::opt PrintResultColumn; +extern llvm::cl::opt LeftRoot; +extern llvm::cl::opt RightRoot; +} // namespace diff } #endif diff --git a/contrib/llvm/tools/llvm-profdata/llvm-profdata.cpp b/contrib/llvm/tools/llvm-profdata/llvm-profdata.cpp index e9bc2de82bdf..eee242107dab 100644 --- a/contrib/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/contrib/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -159,14 +159,20 @@ static void loadInput(const WeightedFile &Input, WriterContext *WC) { for (auto &I : *Reader) { const StringRef FuncName = I.Name; - if (Error E = WC->Writer.addRecord(std::move(I), Input.Weight)) { + bool Reported = false; + WC->Writer.addRecord(std::move(I), Input.Weight, [&](Error E) { + if (Reported) { + consumeError(std::move(E)); + return; + } + Reported = true; // Only show hint the first time an error occurs. instrprof_error IPE = InstrProfError::take(std::move(E)); std::unique_lock ErrGuard{WC->ErrLock}; bool firstTime = WC->WriterErrorCodes.insert(IPE).second; handleMergeWriterError(make_error(IPE), Input.Filename, FuncName, firstTime); - } + }); } if (Reader->hasError()) WC->Err = Reader->getError(); @@ -174,8 +180,15 @@ static void loadInput(const WeightedFile &Input, WriterContext *WC) { /// Merge the \p Src writer context into \p Dst. static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) { - if (Error E = Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer))) + bool Reported = false; + Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) { + if (Reported) { + consumeError(std::move(E)); + return; + } + Reported = true; Dst->Err = std::move(E); + }); } static void mergeInstrProfile(const WeightedFileVector &Inputs, @@ -499,8 +512,8 @@ static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK, } static int showInstrProfile(const std::string &Filename, bool ShowCounts, - bool ShowIndirectCallTargets, bool ShowMemOPSizes, - bool ShowDetailedSummary, + uint32_t TopN, bool ShowIndirectCallTargets, + bool ShowMemOPSizes, bool ShowDetailedSummary, std::vector DetailedSummaryCutoffs, bool ShowAllFunctions, const std::string &ShowFunction, bool TextFormat, @@ -519,6 +532,17 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts, size_t ShownFunctions = 0; int NumVPKind = IPVK_Last - IPVK_First + 1; std::vector VPStats(NumVPKind); + + auto MinCmp = [](const std::pair &v1, + const std::pair &v2) { + return v1.second > v2.second; + }; + + std::priority_queue, + std::vector>, + decltype(MinCmp)> + HottestFuncs(MinCmp); + for (const auto &Func : *Reader) { bool Show = ShowAllFunctions || (!ShowFunction.empty() && @@ -528,13 +552,28 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts, if (doTextFormatDump) { InstrProfSymtab &Symtab = Reader->getSymtab(); - InstrProfWriter::writeRecordInText(Func, Symtab, OS); + InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab, + OS); continue; } assert(Func.Counts.size() > 0 && "function missing entry counter"); Builder.addRecord(Func); + if (TopN) { + uint64_t FuncMax = 0; + for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) + FuncMax = std::max(FuncMax, Func.Counts[I]); + + if (HottestFuncs.size() == TopN) { + if (HottestFuncs.top().second < FuncMax) { + HottestFuncs.pop(); + HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); + } + } else + HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); + } + if (Show) { if (!ShownFunctions) @@ -592,6 +631,18 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts, OS << "Maximum function count: " << PS->getMaxFunctionCount() << "\n"; OS << "Maximum internal block count: " << PS->getMaxInternalCount() << "\n"; + if (TopN) { + std::vector> SortedHottestFuncs; + while (!HottestFuncs.empty()) { + SortedHottestFuncs.emplace_back(HottestFuncs.top()); + HottestFuncs.pop(); + } + OS << "Top " << TopN + << " functions with the largest internal block counts: \n"; + for (auto &hotfunc : llvm::reverse(SortedHottestFuncs)) + OS << " " << hotfunc.first << ", max count = " << hotfunc.second << "\n"; + } + if (ShownFunctions && ShowIndirectCallTargets) { OS << "Statistics for indirect call sites profile:\n"; showValueSitesStats(OS, IPVK_IndirectCallTarget, @@ -675,6 +726,9 @@ static int show_main(int argc, const char *argv[]) { cl::desc("Profile kind:"), cl::init(instr), cl::values(clEnumVal(instr, "Instrumentation profile (default)"), clEnumVal(sample, "Sample profile"))); + cl::opt TopNFunctions( + "topn", cl::init(0), + cl::desc("Show the list of functions with the largest internal counts")); cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n"); @@ -692,10 +746,10 @@ static int show_main(int argc, const char *argv[]) { std::vector Cutoffs(DetailedSummaryCutoffs.begin(), DetailedSummaryCutoffs.end()); if (ProfileKind == instr) - return showInstrProfile(Filename, ShowCounts, ShowIndirectCallTargets, - ShowMemOPSizes, ShowDetailedSummary, - DetailedSummaryCutoffs, ShowAllFunctions, - ShowFunction, TextFormat, OS); + return showInstrProfile(Filename, ShowCounts, TopNFunctions, + ShowIndirectCallTargets, ShowMemOPSizes, + ShowDetailedSummary, DetailedSummaryCutoffs, + ShowAllFunctions, ShowFunction, TextFormat, OS); else return showSampleProfile(Filename, ShowCounts, ShowAllFunctions, ShowFunction, OS); diff --git a/contrib/llvm/tools/llvm-readobj/COFFDumper.cpp b/contrib/llvm/tools/llvm-readobj/COFFDumper.cpp index e5ff3e4186de..9fb3267e2f9d 100644 --- a/contrib/llvm/tools/llvm-readobj/COFFDumper.cpp +++ b/contrib/llvm/tools/llvm-readobj/COFFDumper.cpp @@ -1637,7 +1637,11 @@ static StringRef getBaseRelocTypeName(uint8_t Type) { case COFF::IMAGE_REL_BASED_HIGHADJ: return "HIGHADJ"; case COFF::IMAGE_REL_BASED_ARM_MOV32T: return "ARM_MOV32(T)"; case COFF::IMAGE_REL_BASED_DIR64: return "DIR64"; - default: return "unknown (" + llvm::utostr(Type) + ")"; + default: { + static std::string Result; + Result = "unknown (" + llvm::utostr(Type) + ")"; + return Result; + } } } diff --git a/contrib/llvm/tools/llvm-readobj/WasmDumper.cpp b/contrib/llvm/tools/llvm-readobj/WasmDumper.cpp index 14603f8a2b09..266226d59ee8 100644 --- a/contrib/llvm/tools/llvm-readobj/WasmDumper.cpp +++ b/contrib/llvm/tools/llvm-readobj/WasmDumper.cpp @@ -153,6 +153,12 @@ void WasmDumper::printSections() { switch (WasmSec.Type) { case wasm::WASM_SEC_CUSTOM: W.printString("Name", WasmSec.Name); + if (WasmSec.Name == "linking") { + const wasm::WasmLinkingData &LinkingData = Obj->linkingData(); + W.printNumber("DataSize", LinkingData.DataSize); + if (LinkingData.DataAlignment) + W.printNumber("DataAlignment", LinkingData.DataAlignment); + } break; case wasm::WASM_SEC_MEMORY: ListScope Group(W, "Memories"); diff --git a/contrib/llvm/tools/llvm-stress/llvm-stress.cpp b/contrib/llvm/tools/llvm-stress/llvm-stress.cpp index 3cf8b37bc2e2..3945da7020b0 100644 --- a/contrib/llvm/tools/llvm-stress/llvm-stress.cpp +++ b/contrib/llvm/tools/llvm-stress/llvm-stress.cpp @@ -385,10 +385,10 @@ struct ConstModifier: public Modifier { if (Ty->isVectorTy()) { switch (getRandom() % 2) { - case 0: if (Ty->getScalarType()->isIntegerTy()) + case 0: if (Ty->isIntOrIntVectorTy()) return PT->push_back(ConstantVector::getAllOnesValue(Ty)); break; - case 1: if (Ty->getScalarType()->isIntegerTy()) + case 1: if (Ty->isIntOrIntVectorTy()) return PT->push_back(ConstantVector::getNullValue(Ty)); } } @@ -531,8 +531,7 @@ struct CastModifier: public Modifier { } // Both types are integers: - if (VTy->getScalarType()->isIntegerTy() && - DestTy->getScalarType()->isIntegerTy()) { + if (VTy->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy()) { if (VSize > DestSize) { return PT->push_back( new TruncInst(V, DestTy, "Tr", BB->getTerminator())); @@ -546,8 +545,7 @@ struct CastModifier: public Modifier { } // Fp to int. - if (VTy->getScalarType()->isFloatingPointTy() && - DestTy->getScalarType()->isIntegerTy()) { + if (VTy->isFPOrFPVectorTy() && DestTy->isIntOrIntVectorTy()) { if (getRandom() & 1) return PT->push_back( new FPToSIInst(V, DestTy, "FC", BB->getTerminator())); @@ -555,8 +553,7 @@ struct CastModifier: public Modifier { } // Int to fp. - if (VTy->getScalarType()->isIntegerTy() && - DestTy->getScalarType()->isFloatingPointTy()) { + if (VTy->isIntOrIntVectorTy() && DestTy->isFPOrFPVectorTy()) { if (getRandom() & 1) return PT->push_back( new SIToFPInst(V, DestTy, "FC", BB->getTerminator())); @@ -565,8 +562,7 @@ struct CastModifier: public Modifier { } // Both floats. - if (VTy->getScalarType()->isFloatingPointTy() && - DestTy->getScalarType()->isFloatingPointTy()) { + if (VTy->isFPOrFPVectorTy() && DestTy->isFPOrFPVectorTy()) { if (VSize > DestSize) { return PT->push_back( new FPTruncInst(V, DestTy, "Tr", BB->getTerminator())); diff --git a/contrib/llvm/tools/opt/NewPMDriver.cpp b/contrib/llvm/tools/opt/NewPMDriver.cpp index 58e9caeff0fb..94242d795aae 100644 --- a/contrib/llvm/tools/opt/NewPMDriver.cpp +++ b/contrib/llvm/tools/opt/NewPMDriver.cpp @@ -48,6 +48,102 @@ static cl::opt "pipeline for handling managed aliasing queries"), cl::Hidden); +/// {{@ These options accept textual pipeline descriptions which will be +/// inserted into default pipelines at the respective extension points +static cl::opt PeepholeEPPipeline( + "passes-ep-peephole", + cl::desc("A textual description of the function pass pipeline inserted at " + "the Peephole extension points into default pipelines"), + cl::Hidden); +static cl::opt LateLoopOptimizationsEPPipeline( + "passes-ep-late-loop-optimizations", + cl::desc( + "A textual description of the loop pass pipeline inserted at " + "the LateLoopOptimizations extension point into default pipelines"), + cl::Hidden); +static cl::opt LoopOptimizerEndEPPipeline( + "passes-ep-loop-optimizer-end", + cl::desc("A textual description of the loop pass pipeline inserted at " + "the LoopOptimizerEnd extension point into default pipelines"), + cl::Hidden); +static cl::opt ScalarOptimizerLateEPPipeline( + "passes-ep-scalar-optimizer-late", + cl::desc("A textual description of the function pass pipeline inserted at " + "the ScalarOptimizerLate extension point into default pipelines"), + cl::Hidden); +static cl::opt CGSCCOptimizerLateEPPipeline( + "passes-ep-cgscc-optimizer-late", + cl::desc("A textual description of the cgscc pass pipeline inserted at " + "the CGSCCOptimizerLate extension point into default pipelines"), + cl::Hidden); +static cl::opt VectorizerStartEPPipeline( + "passes-ep-vectorizer-start", + cl::desc("A textual description of the function pass pipeline inserted at " + "the VectorizerStart extension point into default pipelines"), + cl::Hidden); +/// @}} + +template +bool tryParsePipelineText(PassBuilder &PB, StringRef PipelineText) { + if (PipelineText.empty()) + return false; + + // Verify the pipeline is parseable: + PassManagerT PM; + if (PB.parsePassPipeline(PM, PipelineText)) + return true; + + errs() << "Could not parse pipeline '" << PipelineText + << "'. I'm going to igore it.\n"; + return false; +} + +/// If one of the EPPipeline command line options was given, register callbacks +/// for parsing and inserting the given pipeline +static void registerEPCallbacks(PassBuilder &PB, bool VerifyEachPass, + bool DebugLogging) { + if (tryParsePipelineText(PB, PeepholeEPPipeline)) + PB.registerPeepholeEPCallback([&PB, VerifyEachPass, DebugLogging]( + FunctionPassManager &PM, PassBuilder::OptimizationLevel Level) { + PB.parsePassPipeline(PM, PeepholeEPPipeline, VerifyEachPass, + DebugLogging); + }); + if (tryParsePipelineText(PB, + LateLoopOptimizationsEPPipeline)) + PB.registerLateLoopOptimizationsEPCallback( + [&PB, VerifyEachPass, DebugLogging]( + LoopPassManager &PM, PassBuilder::OptimizationLevel Level) { + PB.parsePassPipeline(PM, LateLoopOptimizationsEPPipeline, + VerifyEachPass, DebugLogging); + }); + if (tryParsePipelineText(PB, LoopOptimizerEndEPPipeline)) + PB.registerLoopOptimizerEndEPCallback([&PB, VerifyEachPass, DebugLogging]( + LoopPassManager &PM, PassBuilder::OptimizationLevel Level) { + PB.parsePassPipeline(PM, LoopOptimizerEndEPPipeline, VerifyEachPass, + DebugLogging); + }); + if (tryParsePipelineText(PB, + ScalarOptimizerLateEPPipeline)) + PB.registerScalarOptimizerLateEPCallback( + [&PB, VerifyEachPass, DebugLogging]( + FunctionPassManager &PM, PassBuilder::OptimizationLevel Level) { + PB.parsePassPipeline(PM, ScalarOptimizerLateEPPipeline, + VerifyEachPass, DebugLogging); + }); + if (tryParsePipelineText(PB, CGSCCOptimizerLateEPPipeline)) + PB.registerCGSCCOptimizerLateEPCallback([&PB, VerifyEachPass, DebugLogging]( + CGSCCPassManager &PM, PassBuilder::OptimizationLevel Level) { + PB.parsePassPipeline(PM, CGSCCOptimizerLateEPPipeline, VerifyEachPass, + DebugLogging); + }); + if (tryParsePipelineText(PB, VectorizerStartEPPipeline)) + PB.registerVectorizerStartEPCallback([&PB, VerifyEachPass, DebugLogging]( + FunctionPassManager &PM, PassBuilder::OptimizationLevel Level) { + PB.parsePassPipeline(PM, VectorizerStartEPPipeline, VerifyEachPass, + DebugLogging); + }); +} + bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM, tool_output_file *Out, tool_output_file *ThinLTOLinkOut, @@ -56,7 +152,9 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM, bool ShouldPreserveAssemblyUseListOrder, bool ShouldPreserveBitcodeUseListOrder, bool EmitSummaryIndex, bool EmitModuleHash) { + bool VerifyEachPass = VK == VK_VerifyEachPass; PassBuilder PB(TM); + registerEPCallbacks(PB, VerifyEachPass, DebugPM); // Specially handle the alias analysis manager so that we can register // a custom pipeline of AA passes with it. @@ -85,8 +183,7 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM, if (VK > VK_NoVerifier) MPM.addPass(VerifierPass()); - if (!PB.parsePassPipeline(MPM, PassPipeline, VK == VK_VerifyEachPass, - DebugPM)) { + if (!PB.parsePassPipeline(MPM, PassPipeline, VerifyEachPass, DebugPM)) { errs() << Arg0 << ": unable to parse pass pipeline description.\n"; return false; } diff --git a/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 0980e08f67f7..1f8e1b125889 100644 --- a/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/contrib/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -2222,7 +2222,7 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, OS << " switch (Operand.getReg()) {\n"; OS << " default: OpKind = InvalidMatchClass; break;\n"; for (const auto &RC : Info.RegisterClasses) - OS << " case " << Info.Target.getName() << "::" + OS << " case " << RC.first->getValueAsString("Namespace") << "::" << RC.first->getName() << ": OpKind = " << RC.second->Name << "; break;\n"; OS << " }\n"; @@ -2711,6 +2711,47 @@ static void emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target, OS << "}\n\n"; } +static void emitMnemonicSpellChecker(raw_ostream &OS, CodeGenTarget &Target, + unsigned VariantCount) { + OS << "std::string " << Target.getName() << "MnemonicSpellCheck(StringRef S, uint64_t FBS) {\n"; + if (!VariantCount) + OS << " return \"\";"; + else { + OS << " const unsigned MaxEditDist = 2;\n"; + OS << " std::vector Candidates;\n"; + OS << " StringRef Prev = \"\";\n"; + OS << " auto End = std::end(MatchTable0);\n"; + OS << "\n"; + OS << " for (auto I = std::begin(MatchTable0); I < End; I++) {\n"; + OS << " // Ignore unsupported instructions.\n"; + OS << " if ((FBS & I->RequiredFeatures) != I->RequiredFeatures)\n"; + OS << " continue;\n"; + OS << "\n"; + OS << " StringRef T = I->getMnemonic();\n"; + OS << " // Avoid recomputing the edit distance for the same string.\n"; + OS << " if (T.equals(Prev))\n"; + OS << " continue;\n"; + OS << "\n"; + OS << " Prev = T;\n"; + OS << " unsigned Dist = S.edit_distance(T, false, MaxEditDist);\n"; + OS << " if (Dist <= MaxEditDist)\n"; + OS << " Candidates.push_back(T);\n"; + OS << " }\n"; + OS << "\n"; + OS << " if (Candidates.empty())\n"; + OS << " return \"\";\n"; + OS << "\n"; + OS << " std::string Res = \", did you mean: \";\n"; + OS << " unsigned i = 0;\n"; + OS << " for( ; i < Candidates.size() - 1; i++)\n"; + OS << " Res += Candidates[i].str() + \", \";\n"; + OS << " return Res + Candidates[i].str() + \"?\";\n"; + } + OS << "}\n"; + OS << "\n"; +} + + void AsmMatcherEmitter::run(raw_ostream &OS) { CodeGenTarget Target(Records); Record *AsmParser = Target.getAsmParser(); @@ -2948,7 +2989,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { std::string LenMnemonic = char(MI->Mnemonic.size()) + MI->Mnemonic.str(); OS << " { " << StringTable.GetOrAddStringOffset(LenMnemonic, false) << " /* " << MI->Mnemonic << " */, " - << Target.getName() << "::" + << Target.getInstNamespace() << "::" << MI->getResultInst()->TheDef->getName() << ", " << MI->ConversionFnKind << ", "; @@ -2974,6 +3015,8 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { OS << "};\n\n"; } + emitMnemonicSpellChecker(OS, Target, VariantCount); + // Finally, build the match function. OS << "unsigned " << Target.getName() << ClassName << "::\n" << "MatchInstructionImpl(const OperandVector &Operands,\n"; diff --git a/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp b/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp index 30d21984c4d3..75b9bc6cca40 100644 --- a/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/contrib/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -137,12 +137,12 @@ static void EmitInstructions(std::vector &Insts, O << " switch (MI->getOpcode()) {\n"; O << " default: llvm_unreachable(\"Unexpected opcode.\");\n"; std::vector> OpsToPrint; - OpsToPrint.push_back(std::make_pair(FirstInst.CGI->Namespace + "::" + + OpsToPrint.push_back(std::make_pair(FirstInst.CGI->Namespace.str() + "::" + FirstInst.CGI->TheDef->getName().str(), FirstInst.Operands[i])); for (const AsmWriterInst &AWI : SimilarInsts) { - OpsToPrint.push_back(std::make_pair(AWI.CGI->Namespace+"::" + + OpsToPrint.push_back(std::make_pair(AWI.CGI->Namespace.str()+"::" + AWI.CGI->TheDef->getName().str(), AWI.Operands[i])); } diff --git a/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp b/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp index b80dd5daefe0..23751a2cbfba 100644 --- a/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp +++ b/contrib/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -187,20 +187,18 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, std::string CodeEmitterGen::getInstructionCase(Record *R, CodeGenTarget &Target) { std::string Case; - BitsInit *BI = R->getValueAsBitsInit("Inst"); - const std::vector &Vals = R->getValues(); unsigned NumberedOp = 0; - std::set NamedOpIndices; + // Collect the set of operand indices that might correspond to named // operand, and skip these when assigning operands based on position. if (Target.getInstructionSet()-> getValueAsBit("noNamedPositionallyEncodedOperands")) { CodeGenInstruction &CGI = Target.getInstruction(R); - for (unsigned i = 0, e = Vals.size(); i != e; ++i) { + for (const RecordVal &RV : R->getValues()) { unsigned OpIdx; - if (!CGI.Operands.hasOperandNamed(Vals[i].getName(), OpIdx)) + if (!CGI.Operands.hasOperandNamed(RV.getName(), OpIdx)) continue; NamedOpIndices.insert(OpIdx); @@ -209,13 +207,13 @@ std::string CodeEmitterGen::getInstructionCase(Record *R, // Loop over all of the fields in the instruction, determining which are the // operands to the instruction. - for (unsigned i = 0, e = Vals.size(); i != e; ++i) { + for (const RecordVal &RV : R->getValues()) { // Ignore fixed fields in the record, we're looking for values like: // bits<5> RST = { ?, ?, ?, ?, ? }; - if (Vals[i].getPrefix() || Vals[i].getValue()->isComplete()) + if (RV.getPrefix() || RV.getValue()->isComplete()) continue; - AddCodeToMergeInOperand(R, BI, Vals[i].getName(), NumberedOp, + AddCodeToMergeInOperand(R, BI, RV.getName(), NumberedOp, NamedOpIndices, Case, Target); } diff --git a/contrib/llvm/utils/TableGen/CodeGenInstruction.h b/contrib/llvm/utils/TableGen/CodeGenInstruction.h index 75db17b59ac3..e173e153879c 100644 --- a/contrib/llvm/utils/TableGen/CodeGenInstruction.h +++ b/contrib/llvm/utils/TableGen/CodeGenInstruction.h @@ -206,7 +206,7 @@ template class ArrayRef; class CodeGenInstruction { public: Record *TheDef; // The actual record defining this instruction. - std::string Namespace; // The namespace the instruction is in. + StringRef Namespace; // The namespace the instruction is in. /// AsmString - The format string used to emit a .s file for the /// instruction. diff --git a/contrib/llvm/utils/TableGen/CodeGenMapTable.cpp b/contrib/llvm/utils/TableGen/CodeGenMapTable.cpp index 60db6c267ad7..43348b622a74 100644 --- a/contrib/llvm/utils/TableGen/CodeGenMapTable.cpp +++ b/contrib/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -367,7 +367,7 @@ unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) { ArrayRef NumberedInstructions = Target.getInstructionsByEnumValue(); - std::string Namespace = Target.getInstNamespace(); + StringRef Namespace = Target.getInstNamespace(); const std::vector &ValueCols = InstrMapDesc.getValueCols(); unsigned NumCol = ValueCols.size(); unsigned TotalNumInstr = NumberedInstructions.size(); @@ -567,7 +567,7 @@ namespace llvm { //===----------------------------------------------------------------------===// void EmitMapTable(RecordKeeper &Records, raw_ostream &OS) { CodeGenTarget Target(Records); - std::string NameSpace = Target.getInstNamespace(); + StringRef NameSpace = Target.getInstNamespace(); std::vector InstrMapVec; InstrMapVec = Records.getAllDerivedDefinitions("InstrMapping"); diff --git a/contrib/llvm/utils/TableGen/CodeGenTarget.cpp b/contrib/llvm/utils/TableGen/CodeGenTarget.cpp index eb277f3298f9..58df3ceceee7 100644 --- a/contrib/llvm/utils/TableGen/CodeGenTarget.cpp +++ b/contrib/llvm/utils/TableGen/CodeGenTarget.cpp @@ -207,7 +207,7 @@ const StringRef CodeGenTarget::getName() const { return TargetRec->getName(); } -std::string CodeGenTarget::getInstNamespace() const { +StringRef CodeGenTarget::getInstNamespace() const { for (const CodeGenInstruction *Inst : getInstructionsByEnumValue()) { // Make sure not to pick up "TargetOpcode" by accidentally getting // the namespace off the PHI instruction or something. diff --git a/contrib/llvm/utils/TableGen/CodeGenTarget.h b/contrib/llvm/utils/TableGen/CodeGenTarget.h index c822e940ffae..ff624ea559e5 100644 --- a/contrib/llvm/utils/TableGen/CodeGenTarget.h +++ b/contrib/llvm/utils/TableGen/CodeGenTarget.h @@ -86,7 +86,7 @@ class CodeGenTarget { /// getInstNamespace - Return the target-specific instruction namespace. /// - std::string getInstNamespace() const; + StringRef getInstNamespace() const; /// getInstructionSet - Return the InstructionSet object. /// diff --git a/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp index d239f96d2a60..d4a56a64324f 100644 --- a/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/contrib/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -886,7 +886,7 @@ EmitResultInstructionAsOperand(const TreePatternNode *N, assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) && "Node has no result"); - AddMatcher(new EmitNodeMatcher(II.Namespace+"::"+II.TheDef->getName().str(), + AddMatcher(new EmitNodeMatcher(II.Namespace.str()+"::"+II.TheDef->getName().str(), ResultVTs, InstOps, NodeHasChain, TreeHasInGlue, TreeHasOutGlue, NodeHasMemRefs, NumFixedArityOperands, diff --git a/contrib/llvm/utils/TableGen/FastISelEmitter.cpp b/contrib/llvm/utils/TableGen/FastISelEmitter.cpp index 0e7b0dc09442..25388b75cc0d 100644 --- a/contrib/llvm/utils/TableGen/FastISelEmitter.cpp +++ b/contrib/llvm/utils/TableGen/FastISelEmitter.cpp @@ -390,10 +390,10 @@ class FastISelMap { std::map > SignaturesWithConstantForms; - std::string InstNS; + StringRef InstNS; ImmPredicateSet ImmediatePredicates; public: - explicit FastISelMap(std::string InstNS); + explicit FastISelMap(StringRef InstNS); void collectPatterns(CodeGenDAGPatterns &CGP); void printImmediatePredicates(raw_ostream &OS); @@ -417,7 +417,7 @@ static std::string getLegalCName(std::string OpName) { return OpName; } -FastISelMap::FastISelMap(std::string instns) : InstNS(std::move(instns)) {} +FastISelMap::FastISelMap(StringRef instns) : InstNS(instns) {} static std::string PhyRegForNode(TreePatternNode *Op, const CodeGenTarget &Target) { @@ -440,10 +440,6 @@ static std::string PhyRegForNode(TreePatternNode *Op, void FastISelMap::collectPatterns(CodeGenDAGPatterns &CGP) { const CodeGenTarget &Target = CGP.getTargetInfo(); - // Determine the target's namespace name. - InstNS = Target.getInstNamespace() + "::"; - assert(InstNS.size() > 2 && "Can't determine target-specific namespace!"); - // Scan through all the patterns and record the simple ones. for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), E = CGP.ptm_end(); I != E; ++I) { @@ -659,8 +655,8 @@ void FastISelMap::emitInstructionCode(raw_ostream &OS, if (Memo.SubRegNo.empty()) { Operands.PrintManglingSuffix(OS, *Memo.PhysRegs, ImmediatePredicates, true); - OS << "(" << InstNS << Memo.Name << ", "; - OS << "&" << InstNS << Memo.RC->getName() << "RegClass"; + OS << "(" << InstNS << "::" << Memo.Name << ", "; + OS << "&" << InstNS << "::" << Memo.RC->getName() << "RegClass"; if (!Operands.empty()) OS << ", "; Operands.PrintArguments(OS, *Memo.PhysRegs); @@ -873,8 +869,8 @@ void EmitFastISel(RecordKeeper &RK, raw_ostream &OS) { Target.getName().str() + " target", OS); // Determine the target's namespace name. - std::string InstNS = Target.getInstNamespace() + "::"; - assert(InstNS.size() > 2 && "Can't determine target-specific namespace!"); + StringRef InstNS = Target.getInstNamespace(); + assert(!InstNS.empty() && "Can't determine target-specific namespace!"); FastISelMap F(InstNS); F.collectPatterns(CGP); diff --git a/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp index 75fd73082b9a..03930d7132df 100644 --- a/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp +++ b/contrib/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp @@ -1691,9 +1691,7 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const { dumpStack(errs(), "\t\t"); for (unsigned i = 0; i < Opcodes.size(); ++i) { - const std::string &Name = nameWithID(Opcodes[i]); - - errs() << '\t' << Name << " "; + errs() << '\t' << nameWithID(Opcodes[i]) << " "; dumpBits(errs(), getBitsField(*AllInstructions[Opcodes[i]]->TheDef, "Inst")); errs() << '\n'; diff --git a/contrib/llvm/utils/TableGen/GlobalISelEmitter.cpp b/contrib/llvm/utils/TableGen/GlobalISelEmitter.cpp index 924ed8f65c2c..cafcbeb57de5 100644 --- a/contrib/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/contrib/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -53,6 +53,8 @@ STATISTIC(NumPatternTotal, "Total number of patterns"); STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG"); STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped"); STATISTIC(NumPatternEmitted, "Number of patterns emitted"); +/// A unique identifier for a MatchTable. +static unsigned CurrentMatchTableID = 0; cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel"); @@ -74,6 +76,18 @@ class LLTCodeGen { public: LLTCodeGen(const LLT &Ty) : Ty(Ty) {} + void emitCxxEnumValue(raw_ostream &OS) const { + if (Ty.isScalar()) { + OS << "GILLT_s" << Ty.getSizeInBits(); + return; + } + if (Ty.isVector()) { + OS << "GILLT_v" << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits(); + return; + } + llvm_unreachable("Unhandled LLT"); + } + void emitCxxConstructorCall(raw_ostream &OS) const { if (Ty.isScalar()) { OS << "LLT::scalar(" << Ty.getSizeInBits() << ")"; @@ -88,6 +102,33 @@ class LLTCodeGen { } const LLT &get() const { return Ty; } + + /// This ordering is used for std::unique() and std::sort(). There's no + /// particular logic behind the order. + bool operator<(const LLTCodeGen &Other) const { + if (!Ty.isValid()) + return Other.Ty.isValid(); + if (Ty.isScalar()) { + if (!Other.Ty.isValid()) + return false; + if (Other.Ty.isScalar()) + return Ty.getSizeInBits() < Other.Ty.getSizeInBits(); + return false; + } + if (Ty.isVector()) { + if (!Other.Ty.isValid() || Other.Ty.isScalar()) + return false; + if (Other.Ty.isVector()) { + if (Ty.getNumElements() < Other.Ty.getNumElements()) + return true; + if (Ty.getNumElements() > Other.Ty.getNumElements()) + return false; + return Ty.getSizeInBits() < Other.Ty.getSizeInBits(); + } + return false; + } + llvm_unreachable("Unhandled LLT"); + } }; class InstructionMatcher; @@ -169,6 +210,13 @@ static Record *getInitValueAsRegClass(Init *V) { return nullptr; } +std::string +getNameForFeatureBitset(const std::vector &FeatureBitset) { + std::string Name = "GIFBS"; + for (const auto &Feature : FeatureBitset) + Name += ("_" + Feature->getName()).str(); + return Name; +} //===- Matchers -----------------------------------------------------------===// class OperandMatcher; @@ -187,8 +235,8 @@ class RuleMatcher { std::vector> Actions; /// A map of instruction matchers to the local variables created by - /// emitCxxCaptureStmts(). - std::map InsnVariableNames; + /// emitCaptureOpcodes(). + std::map InsnVariableIDs; /// ID for the next instruction variable defined with defineInsnVar() unsigned NextInsnVarID; @@ -197,35 +245,39 @@ class RuleMatcher { public: RuleMatcher() - : Matchers(), Actions(), InsnVariableNames(), NextInsnVarID(0) {} + : Matchers(), Actions(), InsnVariableIDs(), NextInsnVarID(0) {} RuleMatcher(RuleMatcher &&Other) = default; RuleMatcher &operator=(RuleMatcher &&Other) = default; InstructionMatcher &addInstructionMatcher(); void addRequiredFeature(Record *Feature); + const std::vector &getRequiredFeatures() const; template Kind &addAction(Args &&... args); - std::string defineInsnVar(raw_ostream &OS, const InstructionMatcher &Matcher, - StringRef Value); - StringRef getInsnVarName(const InstructionMatcher &InsnMatcher) const; + /// Define an instruction without emitting any code to do so. + /// This is used for the root of the match. + unsigned implicitlyDefineInsnVar(const InstructionMatcher &Matcher); + /// Define an instruction and emit corresponding state-machine opcodes. + unsigned defineInsnVar(raw_ostream &OS, const InstructionMatcher &Matcher, + unsigned InsnVarID, unsigned OpIdx); + unsigned getInsnVarID(const InstructionMatcher &InsnMatcher) const; - void emitCxxCapturedInsnList(raw_ostream &OS); - void emitCxxCaptureStmts(raw_ostream &OS, StringRef Expr); + void emitCaptureOpcodes(raw_ostream &OS); -void emit(raw_ostream &OS, SubtargetFeatureInfoMap SubtargetFeatures); + void emit(raw_ostream &OS); -/// Compare the priority of this object and B. -/// -/// Returns true if this object is more important than B. -bool isHigherPriorityThan(const RuleMatcher &B) const; + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(const RuleMatcher &B) const; -/// Report the maximum number of temporary operands needed by the rule -/// matcher. -unsigned countRendererFns() const; + /// Report the maximum number of temporary operands needed by the rule + /// matcher. + unsigned countRendererFns() const; -// FIXME: Remove this as soon as possible -InstructionMatcher &insnmatcher_front() const { return *Matchers.front(); } + // FIXME: Remove this as soon as possible + InstructionMatcher &insnmatcher_front() const { return *Matchers.front(); } }; template class PredicateListMatcher { @@ -255,21 +307,16 @@ template class PredicateListMatcher { return Predicates.size(); } - /// Emit a C++ expression that tests whether all the predicates are met. + /// Emit MatchTable opcodes that tests whether all the predicates are met. template - void emitCxxPredicateListExpr(raw_ostream &OS, Args &&... args) const { + void emitPredicateListOpcodes(raw_ostream &OS, Args &&... args) const { if (Predicates.empty()) { - OS << "true"; + OS << "// No predicates\n"; return; } - StringRef Separator = ""; - for (const auto &Predicate : predicates()) { - OS << Separator << "("; - Predicate->emitCxxPredicateExpr(OS, std::forward(args)...); - OS << ")"; - Separator = " &&\n"; - } + for (const auto &Predicate : predicates()) + Predicate->emitPredicateOpcodes(OS, std::forward(args)...); } }; @@ -291,6 +338,7 @@ class OperandPredicateMatcher { enum PredicateKind { OPM_ComplexPattern, OPM_Instruction, + OPM_IntrinsicID, OPM_Int, OPM_LiteralInt, OPM_LLT, @@ -318,15 +366,17 @@ class OperandPredicateMatcher { return None; } - /// Emit C++ statements to capture instructions into local variables. + /// Emit MatchTable opcodes to capture instructions into the MIs table. /// - /// Only InstructionOperandMatcher needs to do anything for this method. - virtual void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef Expr) const {} + /// Only InstructionOperandMatcher needs to do anything for this method the + /// rest just walk the tree. + virtual void emitCaptureOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const {} - /// Emit a C++ expression that checks the predicate for the given operand. - virtual void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const = 0; + /// Emit MatchTable opcodes that check the predicate for the given operand. + virtual void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, + unsigned OpIdx) const = 0; /// Compare the priority of this object and B. /// @@ -353,11 +403,12 @@ class LLTOperandMatcher : public OperandPredicateMatcher { return P->getKind() == OPM_LLT; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OS << "MRI.getType(" << OperandExpr << ".getReg()) == ("; - Ty.emitCxxConstructorCall(OS); - OS << ")"; + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { + OS << " GIM_CheckType, /*MI*/" << InsnVarID << ", /*Op*/" << OpIdx + << ", /*Type*/"; + Ty.emitCxxEnumValue(OS); + OS << ", \n"; } }; @@ -379,11 +430,12 @@ class ComplexPatternOperandMatcher : public OperandPredicateMatcher { return P->getKind() == OPM_ComplexPattern; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { unsigned ID = getAllocatedTemporariesBaseID(); - OS << "(Renderer" << ID << " = " << TheDef.getValueAsString("MatcherFn") - << "(" << OperandExpr << "))"; + OS << " GIM_CheckComplexPattern, /*MI*/" << InsnVarID << ", /*Op*/" + << OpIdx << ", /*Renderer*/" << ID << ", GICP_" + << TheDef.getName() << ",\n"; } unsigned countRendererFns() const override { @@ -404,11 +456,10 @@ class RegisterBankOperandMatcher : public OperandPredicateMatcher { return P->getKind() == OPM_RegBank; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OS << "(&RBI.getRegBankFromRegClass(" << RC.getQualifiedName() - << "RegClass) == RBI.getRegBank(" << OperandExpr - << ".getReg(), MRI, TRI))"; + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { + OS << " GIM_CheckRegBankForClass, /*MI*/" << InsnVarID << ", /*Op*/" + << OpIdx << ", /*RC*/" << RC.getQualifiedName() << "RegClassID,\n"; } }; @@ -421,9 +472,9 @@ class MBBOperandMatcher : public OperandPredicateMatcher { return P->getKind() == OPM_MBB; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OS << OperandExpr << ".isMBB()"; + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { + OS << " GIM_CheckIsMBB, /*MI*/" << InsnVarID << ", /*Op*/" << OpIdx << ",\n"; } }; @@ -441,9 +492,10 @@ class ConstantIntOperandMatcher : public OperandPredicateMatcher { return P->getKind() == OPM_Int; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OS << "isOperandImmEqual(" << OperandExpr << ", " << Value << ", MRI)"; + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { + OS << " GIM_CheckConstantInt, /*MI*/" << InsnVarID << ", /*Op*/" + << OpIdx << ", " << Value << ",\n"; } }; @@ -461,10 +513,30 @@ class LiteralIntOperandMatcher : public OperandPredicateMatcher { return P->getKind() == OPM_LiteralInt; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OS << OperandExpr << ".isCImm() && " << OperandExpr - << ".getCImm()->equalsInt(" << Value << ")"; + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { + OS << " GIM_CheckLiteralInt, /*MI*/" << InsnVarID << ", /*Op*/" + << OpIdx << ", " << Value << ",\n"; + } +}; + +/// Generates code to check that an operand is an intrinsic ID. +class IntrinsicIDOperandMatcher : public OperandPredicateMatcher { +protected: + const CodeGenIntrinsic *II; + +public: + IntrinsicIDOperandMatcher(const CodeGenIntrinsic *II) + : OperandPredicateMatcher(OPM_IntrinsicID), II(II) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_IntrinsicID; + } + + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID, unsigned OpIdx) const override { + OS << " GIM_CheckIntrinsicID, /*MI*/" << InsnVarID << ", /*Op*/" + << OpIdx << ", Intrinsic::" << II->EnumName << ",\n"; } }; @@ -496,8 +568,9 @@ class OperandMatcher : public PredicateListMatcher { } unsigned getOperandIndex() const { return OpIdx; } - std::string getOperandExpr(StringRef InsnVarName) const { - return (InsnVarName + ".getOperand(" + llvm::to_string(OpIdx) + ")").str(); + std::string getOperandExpr(unsigned InsnVarID) const { + return "State.MIs[" + llvm::to_string(InsnVarID) + "]->getOperand(" + + llvm::to_string(OpIdx) + ")"; } Optional @@ -515,25 +588,24 @@ class OperandMatcher : public PredicateListMatcher { InstructionMatcher &getInstructionMatcher() const { return Insn; } - /// Emit C++ statements to capture instructions into local variables. - void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const { + /// Emit MatchTable opcodes to capture instructions into the MIs table. + void emitCaptureOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID) const { for (const auto &Predicate : predicates()) - Predicate->emitCxxCaptureStmts(OS, Rule, OperandExpr); + Predicate->emitCaptureOpcodes(OS, Rule, InsnVarID, OpIdx); } - /// Emit a C++ expression that tests whether the instruction named in - /// InsnVarName matches all the predicate and all the operands. - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef InsnVarName) const { - OS << "(/* "; + /// Emit MatchTable opcodes that test whether the instruction named in + /// InsnVarID matches all the predicates and all the operands. + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID) const { + OS << " // MIs[" << InsnVarID << "] "; if (SymbolicName.empty()) OS << "Operand " << OpIdx; else OS << SymbolicName; - OS << " */ "; - emitCxxPredicateListExpr(OS, Rule, getOperandExpr(InsnVarName)); - OS << ")"; + OS << "\n"; + emitPredicateListOpcodes(OS, Rule, InsnVarID, OpIdx); } /// Compare the priority of this object and B. @@ -599,10 +671,10 @@ class InstructionPredicateMatcher { PredicateKind getKind() const { return Kind; } - /// Emit a C++ expression that tests whether the instruction named in - /// InsnVarName matches the predicate. - virtual void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef InsnVarName) const = 0; + /// Emit MatchTable opcodes that test whether the instruction named in + /// InsnVarID matches the predicate. + virtual void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID) const = 0; /// Compare the priority of this object and B. /// @@ -630,10 +702,10 @@ class InstructionOpcodeMatcher : public InstructionPredicateMatcher { return P->getKind() == IPM_Opcode; } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef InsnVarName) const override { - OS << InsnVarName << ".getOpcode() == " << I->Namespace - << "::" << I->TheDef->getName(); + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID) const override { + OS << " GIM_CheckOpcode, /*MI*/" << InsnVarID << ", " << I->Namespace + << "::" << I->TheDef->getName() << ",\n"; } /// Compare the priority of this object and B. @@ -721,26 +793,23 @@ class InstructionMatcher return make_range(operands_begin(), operands_end()); } - /// Emit C++ statements to check the shape of the match and capture - /// instructions into local variables. - void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, StringRef Expr) { - OS << "if (" << Expr << ".getNumOperands() < " << getNumOperands() << ")\n" - << " return false;\n"; - for (const auto &Operand : Operands) { - Operand->emitCxxCaptureStmts(OS, Rule, Operand->getOperandExpr(Expr)); - } + /// Emit MatchTable opcodes to check the shape of the match and capture + /// instructions into the MIs table. + void emitCaptureOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnID) { + OS << " GIM_CheckNumOperands, /*MI*/" << InsnID << ", /*Expected*/" + << getNumOperands() << ",\n"; + for (const auto &Operand : Operands) + Operand->emitCaptureOpcodes(OS, Rule, InsnID); } - /// Emit a C++ expression that tests whether the instruction named in + /// Emit MatchTable opcodes that test whether the instruction named in /// InsnVarName matches all the predicates and all the operands. - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef InsnVarName) const { - emitCxxPredicateListExpr(OS, Rule, InsnVarName); - for (const auto &Operand : Operands) { - OS << " &&\n("; - Operand->emitCxxPredicateExpr(OS, Rule, InsnVarName); - OS << ")"; - } + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID) const { + emitPredicateListOpcodes(OS, Rule, InsnVarID); + for (const auto &Operand : Operands) + Operand->emitPredicateOpcodes(OS, Rule, InsnVarID); } /// Compare the priority of this object and B. @@ -817,24 +886,17 @@ class InstructionOperandMatcher : public OperandPredicateMatcher { return InsnMatcher->getOptionalOperand(SymbolicName); } - void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OS << "if (!" << OperandExpr + ".isReg())\n" - << " return false;\n" - << "if (TRI.isPhysicalRegister(" << OperandExpr + ".getReg()))\n" - << " return false;\n"; - std::string InsnVarName = Rule.defineInsnVar( - OS, *InsnMatcher, - ("*MRI.getVRegDef(" + OperandExpr + ".getReg())").str()); - InsnMatcher->emitCxxCaptureStmts(OS, Rule, InsnVarName); + void emitCaptureOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnID, unsigned OpIdx) const override { + unsigned InsnVarID = Rule.defineInsnVar(OS, *InsnMatcher, InsnID, OpIdx); + InsnMatcher->emitCaptureOpcodes(OS, Rule, InsnVarID); } - void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, - StringRef OperandExpr) const override { - OperandExpr = Rule.getInsnVarName(*InsnMatcher); - OS << "("; - InsnMatcher->emitCxxPredicateExpr(OS, Rule, OperandExpr); - OS << ")\n"; + void emitPredicateOpcodes(raw_ostream &OS, RuleMatcher &Rule, + unsigned InsnVarID_, + unsigned OpIdx_) const override { + unsigned InsnVarID = Rule.getInsnVarID(*InsnMatcher); + InsnMatcher->emitPredicateOpcodes(OS, Rule, InsnVarID); } }; @@ -858,13 +920,14 @@ class OperandRenderer { RendererKind getKind() const { return Kind; } - virtual void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const = 0; + virtual void emitRenderOpcodes(raw_ostream &OS, RuleMatcher &Rule) const = 0; }; /// A CopyRenderer emits code to copy a single operand from an existing /// instruction to the one being built. class CopyRenderer : public OperandRenderer { protected: + unsigned NewInsnID; /// The matcher for the instruction that this operand is copied from. /// This provides the facility for looking up an a operand by it's name so /// that it can be used as a source for the instruction being built. @@ -873,9 +936,10 @@ class CopyRenderer : public OperandRenderer { const StringRef SymbolicName; public: - CopyRenderer(const InstructionMatcher &Matched, StringRef SymbolicName) - : OperandRenderer(OR_Copy), Matched(Matched), SymbolicName(SymbolicName) { - } + CopyRenderer(unsigned NewInsnID, const InstructionMatcher &Matched, + StringRef SymbolicName) + : OperandRenderer(OR_Copy), NewInsnID(NewInsnID), Matched(Matched), + SymbolicName(SymbolicName) {} static bool classof(const OperandRenderer *R) { return R->getKind() == OR_Copy; @@ -883,12 +947,12 @@ class CopyRenderer : public OperandRenderer { const StringRef getSymbolicName() const { return SymbolicName; } - void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { + void emitRenderOpcodes(raw_ostream &OS, RuleMatcher &Rule) const override { const OperandMatcher &Operand = Matched.getOperand(SymbolicName); - StringRef InsnVarName = - Rule.getInsnVarName(Operand.getInstructionMatcher()); - std::string OperandExpr = Operand.getOperandExpr(InsnVarName); - OS << " MIB.add(" << OperandExpr << "/*" << SymbolicName << "*/);\n"; + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + OS << " GIR_Copy, /*NewInsnID*/" << NewInsnID << ", /*OldInsnID*/" + << OldInsnVarID << ", /*OpIdx*/" << Operand.getOperandIndex() << ", // " + << SymbolicName << "\n"; } }; @@ -897,6 +961,7 @@ class CopyRenderer : public OperandRenderer { /// subregister should be copied. class CopySubRegRenderer : public OperandRenderer { protected: + unsigned NewInsnID; /// The matcher for the instruction that this operand is copied from. /// This provides the facility for looking up an a operand by it's name so /// that it can be used as a source for the instruction being built. @@ -907,9 +972,9 @@ class CopySubRegRenderer : public OperandRenderer { const CodeGenSubRegIndex *SubReg; public: - CopySubRegRenderer(const InstructionMatcher &Matched, StringRef SymbolicName, - const CodeGenSubRegIndex *SubReg) - : OperandRenderer(OR_CopySubReg), Matched(Matched), + CopySubRegRenderer(unsigned NewInsnID, const InstructionMatcher &Matched, + StringRef SymbolicName, const CodeGenSubRegIndex *SubReg) + : OperandRenderer(OR_CopySubReg), NewInsnID(NewInsnID), Matched(Matched), SymbolicName(SymbolicName), SubReg(SubReg) {} static bool classof(const OperandRenderer *R) { @@ -918,13 +983,13 @@ class CopySubRegRenderer : public OperandRenderer { const StringRef getSymbolicName() const { return SymbolicName; } - void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { + void emitRenderOpcodes(raw_ostream &OS, RuleMatcher &Rule) const override { const OperandMatcher &Operand = Matched.getOperand(SymbolicName); - StringRef InsnVarName = - Rule.getInsnVarName(Operand.getInstructionMatcher()); - std::string OperandExpr = Operand.getOperandExpr(InsnVarName); - OS << " MIB.addReg(" << OperandExpr << ".getReg() /*" << SymbolicName - << "*/, 0, " << SubReg->EnumValue << ");\n"; + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + OS << " GIR_CopySubReg, /*NewInsnID*/" << NewInsnID + << ", /*OldInsnID*/" << OldInsnVarID << ", /*OpIdx*/" + << Operand.getOperandIndex() << ", /*SubRegIdx*/" << SubReg->EnumValue + << ", // " << SymbolicName << "\n"; } }; @@ -932,39 +997,44 @@ class CopySubRegRenderer : public OperandRenderer { /// This is typically useful for WZR/XZR on AArch64. class AddRegisterRenderer : public OperandRenderer { protected: + unsigned InsnID; const Record *RegisterDef; public: - AddRegisterRenderer(const Record *RegisterDef) - : OperandRenderer(OR_Register), RegisterDef(RegisterDef) {} + AddRegisterRenderer(unsigned InsnID, const Record *RegisterDef) + : OperandRenderer(OR_Register), InsnID(InsnID), RegisterDef(RegisterDef) { + } static bool classof(const OperandRenderer *R) { return R->getKind() == OR_Register; } - void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { - OS << " MIB.addReg(" << (RegisterDef->getValue("Namespace") - ? RegisterDef->getValueAsString("Namespace") - : "") - << "::" << RegisterDef->getName() << ");\n"; + void emitRenderOpcodes(raw_ostream &OS, RuleMatcher &Rule) const override { + OS << " GIR_AddRegister, /*InsnID*/" << InsnID << ", " + << (RegisterDef->getValue("Namespace") + ? RegisterDef->getValueAsString("Namespace") + : "") + << "::" << RegisterDef->getName() << ",\n"; } }; /// Adds a specific immediate to the instruction being built. class ImmRenderer : public OperandRenderer { protected: + unsigned InsnID; int64_t Imm; public: - ImmRenderer(int64_t Imm) - : OperandRenderer(OR_Imm), Imm(Imm) {} + ImmRenderer(unsigned InsnID, int64_t Imm) + : OperandRenderer(OR_Imm), InsnID(InsnID), Imm(Imm) {} static bool classof(const OperandRenderer *R) { return R->getKind() == OR_Imm; } - void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { - OS << " MIB.addImm(" << Imm << ");\n"; + void emitRenderOpcodes(raw_ostream &OS, RuleMatcher &Rule) const override { + OS << " GIR_AddImm, /*InsnID*/" << InsnID << ", /*Imm*/" << Imm + << ",\n"; } }; @@ -972,6 +1042,7 @@ class ImmRenderer : public OperandRenderer { /// matcher function. class RenderComplexPatternOperand : public OperandRenderer { private: + unsigned InsnID; const Record &TheDef; /// The name of the operand. const StringRef SymbolicName; @@ -984,17 +1055,18 @@ class RenderComplexPatternOperand : public OperandRenderer { } public: - RenderComplexPatternOperand(const Record &TheDef, StringRef SymbolicName, - unsigned RendererID) - : OperandRenderer(OR_ComplexPattern), TheDef(TheDef), + RenderComplexPatternOperand(unsigned InsnID, const Record &TheDef, + StringRef SymbolicName, unsigned RendererID) + : OperandRenderer(OR_ComplexPattern), InsnID(InsnID), TheDef(TheDef), SymbolicName(SymbolicName), RendererID(RendererID) {} static bool classof(const OperandRenderer *R) { return R->getKind() == OR_ComplexPattern; } - void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { - OS << "Renderer" << RendererID << "(MIB);\n"; + void emitRenderOpcodes(raw_ostream &OS, RuleMatcher &Rule) const override { + OS << " GIR_ComplexRenderer, /*InsnID*/" << InsnID << ", /*RendererID*/" + << RendererID << ",\n"; } }; @@ -1009,11 +1081,11 @@ class MatchAction { /// Emit the C++ statements to implement the action. /// - /// \param RecycleVarName If given, it's an instruction to recycle. The - /// requirements on the instruction vary from action to - /// action. + /// \param RecycleInsnID If given, it's an instruction to recycle. The + /// requirements on the instruction vary from action to + /// action. virtual void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef RecycleVarName) const = 0; + unsigned RecycleInsnID) const = 0; }; /// Generates a comment describing the matched rule being acted upon. @@ -1025,8 +1097,9 @@ class DebugCommentAction : public MatchAction { DebugCommentAction(const PatternToMatch &P) : P(P) {} void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef RecycleVarName) const override { - OS << "// " << *P.getSrcPattern() << " => " << *P.getDstPattern() << "\n"; + unsigned RecycleInsnID) const override { + OS << " // " << *P.getSrcPattern() << " => " << *P.getDstPattern() + << "\n"; } }; @@ -1034,7 +1107,7 @@ class DebugCommentAction : public MatchAction { /// into the desired instruction when this is possible. class BuildMIAction : public MatchAction { private: - std::string Name; + unsigned InsnID; const CodeGenInstruction *I; const InstructionMatcher &Matched; std::vector> OperandRenderers; @@ -1058,9 +1131,9 @@ class BuildMIAction : public MatchAction { } public: - BuildMIAction(const StringRef Name, const CodeGenInstruction *I, + BuildMIAction(unsigned InsnID, const CodeGenInstruction *I, const InstructionMatcher &Matched) - : Name(Name), I(I), Matched(Matched) {} + : InsnID(InsnID), I(I), Matched(Matched) {} template Kind &addRenderer(Args&&... args) { @@ -1070,84 +1143,74 @@ class BuildMIAction : public MatchAction { } void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef RecycleVarName) const override { + unsigned RecycleInsnID) const override { if (canMutate()) { - OS << " " << RecycleVarName << ".setDesc(TII.get(" << I->Namespace - << "::" << I->TheDef->getName() << "));\n"; + OS << " GIR_MutateOpcode, /*InsnID*/" << InsnID + << ", /*RecycleInsnID*/ " << RecycleInsnID << ", /*Opcode*/" + << I->Namespace << "::" << I->TheDef->getName() << ",\n"; if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) { - OS << " auto MIB = MachineInstrBuilder(MF, &" << RecycleVarName - << ");\n"; - for (auto Def : I->ImplicitDefs) { auto Namespace = Def->getValue("Namespace") ? Def->getValueAsString("Namespace") : ""; - OS << " MIB.addDef(" << Namespace << "::" << Def->getName() - << ", RegState::Implicit);\n"; + OS << " GIR_AddImplicitDef, " << InsnID << ", " << Namespace + << "::" << Def->getName() << ",\n"; } for (auto Use : I->ImplicitUses) { auto Namespace = Use->getValue("Namespace") ? Use->getValueAsString("Namespace") : ""; - OS << " MIB.addUse(" << Namespace << "::" << Use->getName() - << ", RegState::Implicit);\n"; + OS << " GIR_AddImplicitUse, " << InsnID << ", " << Namespace + << "::" << Use->getName() << ",\n"; } } - - OS << " MachineInstr &" << Name << " = " << RecycleVarName << ";\n"; return; } // TODO: Simple permutation looks like it could be almost as common as // mutation due to commutative operations. - OS << "MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, " - "I.getDebugLoc(), TII.get(" - << I->Namespace << "::" << I->TheDef->getName() << "));\n"; + OS << " GIR_BuildMI, /*InsnID*/" << InsnID << ", /*Opcode*/" + << I->Namespace << "::" << I->TheDef->getName() << ",\n"; for (const auto &Renderer : OperandRenderers) - Renderer->emitCxxRenderStmts(OS, Rule); - OS << " for (const auto *FromMI : "; - Rule.emitCxxCapturedInsnList(OS); - OS << ")\n"; - OS << " for (const auto &MMO : FromMI->memoperands())\n"; - OS << " MIB.addMemOperand(MMO);\n"; - OS << " " << RecycleVarName << ".eraseFromParent();\n"; - OS << " MachineInstr &" << Name << " = *MIB;\n"; + Renderer->emitRenderOpcodes(OS, Rule); + + OS << " GIR_MergeMemOperands, /*InsnID*/" << InsnID << ",\n" + << " GIR_EraseFromParent, /*InsnID*/" << RecycleInsnID << ",\n"; } }; /// Generates code to constrain the operands of an output instruction to the /// register classes specified by the definition of that instruction. class ConstrainOperandsToDefinitionAction : public MatchAction { - std::string Name; + unsigned InsnID; public: - ConstrainOperandsToDefinitionAction(const StringRef Name) : Name(Name) {} + ConstrainOperandsToDefinitionAction(unsigned InsnID) : InsnID(InsnID) {} void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef RecycleVarName) const override { - OS << " constrainSelectedInstRegOperands(" << Name - << ", TII, TRI, RBI);\n"; + unsigned RecycleInsnID) const override { + OS << " GIR_ConstrainSelectedInstOperands, /*InsnID*/" << InsnID << ",\n"; } }; /// Generates code to constrain the specified operand of an output instruction /// to the specified register class. class ConstrainOperandToRegClassAction : public MatchAction { - std::string Name; + unsigned InsnID; unsigned OpIdx; const CodeGenRegisterClass &RC; public: - ConstrainOperandToRegClassAction(const StringRef Name, unsigned OpIdx, + ConstrainOperandToRegClassAction(unsigned InsnID, unsigned OpIdx, const CodeGenRegisterClass &RC) - : Name(Name), OpIdx(OpIdx), RC(RC) {} + : InsnID(InsnID), OpIdx(OpIdx), RC(RC) {} void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, - StringRef RecycleVarName) const override { - OS << " constrainOperandRegToRegClass(" << Name << ", " << OpIdx - << ", " << RC.getQualifiedName() << "RegClass, TII, TRI, RBI);\n"; + unsigned RecycleInsnID) const override { + OS << " GIR_ConstrainOperandRC, /*InsnID*/" << InsnID << ", /*Op*/" + << OpIdx << ", /*RC " << RC.getName() << "*/ " << RC.EnumValue << ",\n"; } }; @@ -1160,53 +1223,49 @@ void RuleMatcher::addRequiredFeature(Record *Feature) { RequiredFeatures.push_back(Feature); } +const std::vector &RuleMatcher::getRequiredFeatures() const { + return RequiredFeatures; +} + template Kind &RuleMatcher::addAction(Args &&... args) { Actions.emplace_back(llvm::make_unique(std::forward(args)...)); return *static_cast(Actions.back().get()); } -std::string RuleMatcher::defineInsnVar(raw_ostream &OS, - const InstructionMatcher &Matcher, - StringRef Value) { - std::string InsnVarName = "MI" + llvm::to_string(NextInsnVarID++); - OS << "MachineInstr &" << InsnVarName << " = " << Value << ";\n"; - InsnVariableNames[&Matcher] = InsnVarName; - return InsnVarName; +unsigned +RuleMatcher::implicitlyDefineInsnVar(const InstructionMatcher &Matcher) { + unsigned NewInsnVarID = NextInsnVarID++; + InsnVariableIDs[&Matcher] = NewInsnVarID; + return NewInsnVarID; } -StringRef -RuleMatcher::getInsnVarName(const InstructionMatcher &InsnMatcher) const { - const auto &I = InsnVariableNames.find(&InsnMatcher); - if (I != InsnVariableNames.end()) +unsigned RuleMatcher::defineInsnVar(raw_ostream &OS, + const InstructionMatcher &Matcher, + unsigned InsnID, unsigned OpIdx) { + unsigned NewInsnVarID = implicitlyDefineInsnVar(Matcher); + OS << " GIM_RecordInsn, /*DefineMI*/" << NewInsnVarID << ", /*MI*/" + << InsnID << ", /*OpIdx*/" << OpIdx << ", // MIs[" << NewInsnVarID + << "]\n"; + return NewInsnVarID; +} + +unsigned RuleMatcher::getInsnVarID(const InstructionMatcher &InsnMatcher) const { + const auto &I = InsnVariableIDs.find(&InsnMatcher); + if (I != InsnVariableIDs.end()) return I->second; llvm_unreachable("Matched Insn was not captured in a local variable"); } -/// Emit a C++ initializer_list containing references to every matched -/// instruction. -void RuleMatcher::emitCxxCapturedInsnList(raw_ostream &OS) { - SmallVector Names; - for (const auto &Pair : InsnVariableNames) - Names.push_back(Pair.second); - std::sort(Names.begin(), Names.end()); - - OS << "{"; - for (const auto &Name : Names) - OS << "&" << Name << ", "; - OS << "}"; -} - -/// Emit C++ statements to check the shape of the match and capture +/// Emit MatchTable opcodes to check the shape of the match and capture /// instructions into local variables. -void RuleMatcher::emitCxxCaptureStmts(raw_ostream &OS, StringRef Expr) { +void RuleMatcher::emitCaptureOpcodes(raw_ostream &OS) { assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet"); - std::string InsnVarName = defineInsnVar(OS, *Matchers.front(), Expr); - Matchers.front()->emitCxxCaptureStmts(OS, *this, InsnVarName); + unsigned InsnVarID = implicitlyDefineInsnVar(*Matchers.front()); + Matchers.front()->emitCaptureOpcodes(OS, *this, InsnVarID); } -void RuleMatcher::emit(raw_ostream &OS, - SubtargetFeatureInfoMap SubtargetFeatures) { +void RuleMatcher::emit(raw_ostream &OS) { if (Matchers.empty()) llvm_unreachable("Unexpected empty matcher!"); @@ -1221,47 +1280,34 @@ void RuleMatcher::emit(raw_ostream &OS, // on some targets but we don't need to make use of that yet. assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet"); - OS << "if ("; - OS << "[&]() {\n"; + OS << " const static int64_t MatchTable" << CurrentMatchTableID << "[] = {\n"; if (!RequiredFeatures.empty()) { - OS << " PredicateBitset ExpectedFeatures = {"; - StringRef Separator = ""; - for (const auto &Predicate : RequiredFeatures) { - const auto &I = SubtargetFeatures.find(Predicate); - assert(I != SubtargetFeatures.end() && "Didn't import predicate?"); - OS << Separator << I->second.getEnumBitName(); - Separator = ", "; - } - OS << "};\n"; - OS << "if ((AvailableFeatures & ExpectedFeatures) != ExpectedFeatures)\n" - << " return false;\n"; + OS << " GIM_CheckFeatures, " << getNameForFeatureBitset(RequiredFeatures) + << ",\n"; } - emitCxxCaptureStmts(OS, "I"); + emitCaptureOpcodes(OS); - OS << " if ("; - Matchers.front()->emitCxxPredicateExpr(OS, *this, - getInsnVarName(*Matchers.front())); - OS << ") {\n"; + Matchers.front()->emitPredicateOpcodes(OS, *this, + getInsnVarID(*Matchers.front())); // We must also check if it's safe to fold the matched instructions. - if (InsnVariableNames.size() >= 2) { + if (InsnVariableIDs.size() >= 2) { // Invert the map to create stable ordering (by var names) - SmallVector Names; - for (const auto &Pair : InsnVariableNames) { + SmallVector InsnIDs; + for (const auto &Pair : InsnVariableIDs) { // Skip the root node since it isn't moving anywhere. Everything else is // sinking to meet it. if (Pair.first == Matchers.front().get()) continue; - Names.push_back(Pair.second); + InsnIDs.push_back(Pair.second); } - std::sort(Names.begin(), Names.end()); + std::sort(InsnIDs.begin(), InsnIDs.end()); - for (const auto &Name : Names) { + for (const auto &InsnID : InsnIDs) { // Reject the difficult cases until we have a more accurate check. - OS << " if (!isObviouslySafeToFold(" << Name - << ")) return false;\n"; + OS << " GIM_CheckIsSafeToFold, /*InsnID*/" << InsnID << ",\n"; // FIXME: Emit checks to determine it's _actually_ safe to fold and/or // account for unsafe cases. @@ -1300,14 +1346,17 @@ void RuleMatcher::emit(raw_ostream &OS, } } - for (const auto &MA : Actions) { - MA->emitCxxActionStmts(OS, *this, "I"); - } - - OS << " return true;\n"; - OS << " }\n"; - OS << " return false;\n"; - OS << " }()) { return true; }\n\n"; + for (const auto &MA : Actions) + MA->emitCxxActionStmts(OS, *this, 0); + OS << " GIR_Done,\n" + << " };\n" + << " State.MIs.resize(1);\n" + << " DEBUG(dbgs() << \"Processing MatchTable" << CurrentMatchTableID + << "\\n\");\n" + << " if (executeMatchTable(*this, OutMIs, State, MatcherInfo, MatchTable" + << CurrentMatchTableID << ", TII, MRI, TRI, RBI, AvailableFeatures)) {\n" + << " return true;\n" + << " }\n\n"; } bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const { @@ -1366,7 +1415,8 @@ class GlobalISelEmitter { Error importRulePredicates(RuleMatcher &M, ArrayRef Predicates); Expected createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, - const TreePatternNode *Src) const; + const TreePatternNode *Src, + unsigned &TempOpIdx) const; Error importChildMatcher(InstructionMatcher &InsnMatcher, const TreePatternNode *SrcChild, unsigned OpIdx, unsigned &TempOpIdx) const; @@ -1425,8 +1475,12 @@ GlobalISelEmitter::importRulePredicates(RuleMatcher &M, return Error::success(); } -Expected GlobalISelEmitter::createAndImportSelDAGMatcher( - InstructionMatcher &InsnMatcher, const TreePatternNode *Src) const { +Expected +GlobalISelEmitter::createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, + unsigned &TempOpIdx) const { + const CodeGenInstruction *SrcGIOrNull = nullptr; + // Start with the defined operands (i.e., the results of the root operator). if (Src->getExtTypes().size() > 1) return failedImport("Src pattern has multiple results"); @@ -1440,7 +1494,7 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( return failedImport( "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); } else { - auto SrcGIOrNull = findNodeEquiv(Src->getOperator()); + SrcGIOrNull = findNodeEquiv(Src->getOperator()); if (!SrcGIOrNull) return failedImport("Pattern operator lacks an equivalent Instruction" + explainOperator(Src->getOperator())); @@ -1451,7 +1505,6 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( } unsigned OpIdx = 0; - unsigned TempOpIdx = 0; for (const EEVT::TypeSet &Ty : Src->getExtTypes()) { auto OpTyOrNone = MVTToLLT(Ty.getConcrete()); @@ -1474,10 +1527,27 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( return failedImport( "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); } else { + assert(SrcGIOrNull && + "Expected to have already found an equivalent Instruction"); // Match the used operands (i.e. the children of the operator). for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) { - if (auto Error = importChildMatcher(InsnMatcher, Src->getChild(i), - OpIdx++, TempOpIdx)) + TreePatternNode *SrcChild = Src->getChild(i); + + // For G_INTRINSIC, the operand immediately following the defs is an + // intrinsic ID. + if (SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" && i == 0) { + if (const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP)) { + OperandMatcher &OM = + InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); + OM.addPredicate(II); + continue; + } + + return failedImport("Expected IntInit containing instrinsic ID)"); + } + + if (auto Error = + importChildMatcher(InsnMatcher, SrcChild, OpIdx++, TempOpIdx)) return std::move(Error); } } @@ -1513,7 +1583,7 @@ Error GlobalISelEmitter::importChildMatcher(InstructionMatcher &InsnMatcher, auto OpTyOrNone = MVTToLLT(ChildTypes.front().getConcrete()); if (!OpTyOrNone) - return failedImport("Src operand has an unsupported type"); + return failedImport("Src operand has an unsupported type (" + to_string(*SrcChild) + ")"); OM.addPredicate(*OpTyOrNone); // Check for nested instructions. @@ -1521,8 +1591,8 @@ Error GlobalISelEmitter::importChildMatcher(InstructionMatcher &InsnMatcher, // Map the node to a gMIR instruction. InstructionOperandMatcher &InsnOperand = OM.addPredicate(); - auto InsnMatcherOrError = - createAndImportSelDAGMatcher(InsnOperand.getInsnMatcher(), SrcChild); + auto InsnMatcherOrError = createAndImportSelDAGMatcher( + InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx); if (auto Error = InsnMatcherOrError.takeError()) return Error; @@ -1581,7 +1651,7 @@ Error GlobalISelEmitter::importExplicitUseRenderer( if (DstChild->getOperator()->isSubClassOf("SDNode")) { auto &ChildSDNI = CGP.getSDNodeInfo(DstChild->getOperator()); if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { - DstMIBuilder.addRenderer(InsnMatcher, + DstMIBuilder.addRenderer(0, InsnMatcher, DstChild->getName()); return Error::success(); } @@ -1606,13 +1676,14 @@ Error GlobalISelEmitter::importExplicitUseRenderer( return failedImport("Dst operand has an unsupported type"); if (ChildRec->isSubClassOf("Register")) { - DstMIBuilder.addRenderer(ChildRec); + DstMIBuilder.addRenderer(0, ChildRec); return Error::success(); } if (ChildRec->isSubClassOf("RegisterClass") || ChildRec->isSubClassOf("RegisterOperand")) { - DstMIBuilder.addRenderer(InsnMatcher, DstChild->getName()); + DstMIBuilder.addRenderer(0, InsnMatcher, + DstChild->getName()); return Error::success(); } @@ -1624,7 +1695,7 @@ Error GlobalISelEmitter::importExplicitUseRenderer( const OperandMatcher &OM = InsnMatcher.getOperand(DstChild->getName()); DstMIBuilder.addRenderer( - *ComplexPattern->second, DstChild->getName(), + 0, *ComplexPattern->second, DstChild->getName(), OM.getAllocatedTemporariesBaseID()); return Error::success(); } @@ -1667,12 +1738,12 @@ Expected GlobalISelEmitter::createAndImportInstructionRenderer( IsExtractSubReg = true; } - auto &DstMIBuilder = M.addAction("NewI", DstI, InsnMatcher); + auto &DstMIBuilder = M.addAction(0, DstI, InsnMatcher); // Render the explicit defs. for (unsigned I = 0; I < DstI->Operands.NumDefs; ++I) { const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[I]; - DstMIBuilder.addRenderer(InsnMatcher, DstIOperand.Name); + DstMIBuilder.addRenderer(0, InsnMatcher, DstIOperand.Name); } // EXTRACT_SUBREG needs to use a subregister COPY. @@ -1695,7 +1766,7 @@ Expected GlobalISelEmitter::createAndImportInstructionRenderer( } DstMIBuilder.addRenderer( - InsnMatcher, Dst->getChild(0)->getName(), SubIdx); + 0, InsnMatcher, Dst->getChild(0)->getName(), SubIdx); return DstMIBuilder; } @@ -1751,12 +1822,12 @@ Error GlobalISelEmitter::importDefaultOperandRenderers( } if (const DefInit *DefaultDefOp = dyn_cast(DefaultOp)) { - DstMIBuilder.addRenderer(DefaultDefOp->getDef()); + DstMIBuilder.addRenderer(0, DefaultDefOp->getDef()); continue; } if (const IntInit *DefaultIntOp = dyn_cast(DefaultOp)) { - DstMIBuilder.addRenderer(DefaultIntOp->getValue()); + DstMIBuilder.addRenderer(0, DefaultIntOp->getValue()); continue; } @@ -1809,7 +1880,9 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { to_string(DstI.Operands.NumDefs) + " def(s))"); InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(); - auto InsnMatcherOrError = createAndImportSelDAGMatcher(InsnMatcherTemp, Src); + unsigned TempOpIdx = 0; + auto InsnMatcherOrError = + createAndImportSelDAGMatcher(InsnMatcherTemp, Src, TempOpIdx); if (auto Error = InsnMatcherOrError.takeError()) return std::move(Error); InstructionMatcher &InsnMatcher = InsnMatcherOrError.get(); @@ -1875,7 +1948,7 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { return failedImport("COPY_TO_REGCLASS operand #1 isn't a register class"); M.addAction( - "NewI", 0, Target.getRegisterClass(DstIOpRec)); + 0, 0, Target.getRegisterClass(DstIOpRec)); // We're done with this pattern! It's eligible for GISel emission; return // it. @@ -1903,8 +1976,7 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { return failedImport("EXTRACT_SUBREG operand #1 isn't a register class"); CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); - CodeGenRegisterClass *SrcRC = CGRegs.getRegClass( - getInitValueAsRegClass(Dst->getChild(0)->getLeafValue())); + CodeGenRegisterClass *SrcRC = CGRegs.getRegClass(DstIOpRec); // It would be nice to leave this constraint implicit but we're required // to pick a register class so constrain the result to a register class @@ -1918,12 +1990,16 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { const auto &SrcRCDstRCPair = SrcRC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx); assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); - M.addAction("NewI", 0, - *SrcRCDstRCPair->second); - M.addAction("NewI", 1, - *SrcRCDstRCPair->first); - } else - M.addAction("NewI"); + M.addAction(0, 0, *SrcRCDstRCPair->second); + M.addAction(0, 1, *SrcRCDstRCPair->first); + + // We're done with this pattern! It's eligible for GISel emission; return + // it. + ++NumPatternImported; + return std::move(M); + } + + M.addAction(0); // We're done with this pattern! It's eligible for GISel emission; return it. ++NumPatternImported; @@ -1969,6 +2045,14 @@ void GlobalISelEmitter::run(raw_ostream &OS) { return false; }); + std::vector ComplexPredicates = + RK.getAllDerivedDefinitions("GIComplexOperandMatcher"); + std::sort(ComplexPredicates.begin(), ComplexPredicates.end(), + [](const Record *A, const Record *B) { + if (A->getName() < B->getName()) + return true; + return false; + }); unsigned MaxTemporaries = 0; for (const auto &Rule : Rules) MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns()); @@ -1980,15 +2064,26 @@ void GlobalISelEmitter::run(raw_ostream &OS) { "llvm::PredicateBitsetImpl;\n" << "#endif // ifdef GET_GLOBALISEL_PREDICATE_BITSET\n\n"; - OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n"; - for (unsigned I = 0; I < MaxTemporaries; ++I) - OS << " mutable ComplexRendererFn Renderer" << I << ";\n"; - OS << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n"; + OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n" + << " mutable MatcherState State;\n" + << " typedef " + "ComplexRendererFn(" + << Target.getName() + << "InstructionSelector::*ComplexMatcherMemFn)(MachineOperand &) const;\n" + << "const MatcherInfoTy " + "MatcherInfo;\n" + << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n"; - OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n"; - for (unsigned I = 0; I < MaxTemporaries; ++I) - OS << ", Renderer" << I << "(nullptr)\n"; - OS << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n\n"; + OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n" + << ", State(" << MaxTemporaries << "),\n" + << "MatcherInfo({TypeObjects, FeatureBitsets, {\n" + << " nullptr, // GICP_Invalid\n"; + for (const auto &Record : ComplexPredicates) + OS << " &" << Target.getName() + << "InstructionSelector::" << Record->getValueAsString("MatcherFn") + << ", // " << Record->getName() << "\n"; + OS << "}})\n" + << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n\n"; OS << "#ifdef GET_GLOBALISEL_IMPL\n"; SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures, @@ -2016,19 +2111,107 @@ void GlobalISelEmitter::run(raw_ostream &OS) { "computeAvailableFunctionFeatures", FunctionFeatures, OS, "const MachineFunction *MF"); + // Emit a table containing the LLT objects needed by the matcher and an enum + // for the matcher to reference them with. + std::vector TypeObjects = { + LLT::scalar(8), LLT::scalar(16), LLT::scalar(32), + LLT::scalar(64), LLT::scalar(80), LLT::vector(8, 1), + LLT::vector(16, 1), LLT::vector(32, 1), LLT::vector(64, 1), + LLT::vector(8, 8), LLT::vector(16, 8), LLT::vector(32, 8), + LLT::vector(64, 8), LLT::vector(4, 16), LLT::vector(8, 16), + LLT::vector(16, 16), LLT::vector(32, 16), LLT::vector(2, 32), + LLT::vector(4, 32), LLT::vector(8, 32), LLT::vector(16, 32), + LLT::vector(2, 64), LLT::vector(4, 64), LLT::vector(8, 64), + }; + std::sort(TypeObjects.begin(), TypeObjects.end()); + OS << "enum {\n"; + for (const auto &TypeObject : TypeObjects) { + OS << " "; + TypeObject.emitCxxEnumValue(OS); + OS << ",\n"; + } + OS << "};\n" + << "const static LLT TypeObjects[] = {\n"; + for (const auto &TypeObject : TypeObjects) { + OS << " "; + TypeObject.emitCxxConstructorCall(OS); + OS << ",\n"; + } + OS << "};\n\n"; + + // Emit a table containing the PredicateBitsets objects needed by the matcher + // and an enum for the matcher to reference them with. + std::vector> FeatureBitsets; + for (auto &Rule : Rules) + FeatureBitsets.push_back(Rule.getRequiredFeatures()); + std::sort( + FeatureBitsets.begin(), FeatureBitsets.end(), + [&](const std::vector &A, const std::vector &B) { + if (A.size() < B.size()) + return true; + if (A.size() > B.size()) + return false; + for (const auto &Pair : zip(A, B)) { + if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) + return true; + if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) + return false; + } + return false; + }); + FeatureBitsets.erase( + std::unique(FeatureBitsets.begin(), FeatureBitsets.end()), + FeatureBitsets.end()); + OS << "enum {\n" + << " GIFBS_Invalid,\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + OS << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; + } + OS << "};\n" + << "const static PredicateBitset FeatureBitsets[] {\n" + << " {}, // GIFBS_Invalid\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + OS << " {"; + for (const auto &Feature : FeatureBitset) { + const auto &I = SubtargetFeatures.find(Feature); + assert(I != SubtargetFeatures.end() && "Didn't import predicate?"); + OS << I->second.getEnumBitName() << ", "; + } + OS << "},\n"; + } + OS << "};\n\n"; + + // Emit complex predicate table and an enum to reference them with. + OS << "enum {\n" + << " GICP_Invalid,\n"; + for (const auto &Record : ComplexPredicates) + OS << " GICP_" << Record->getName() << ",\n"; + OS << "};\n" + << "// See constructor for table contents\n\n"; + OS << "bool " << Target.getName() << "InstructionSelector::selectImpl(MachineInstr &I) const {\n" << " MachineFunction &MF = *I.getParent()->getParent();\n" - << " const MachineRegisterInfo &MRI = MF.getRegInfo();\n" + << " MachineRegisterInfo &MRI = MF.getRegInfo();\n" << " // FIXME: This should be computed on a per-function basis rather " "than per-insn.\n" << " AvailableFunctionFeatures = computeAvailableFunctionFeatures(&STI, " "&MF);\n" - << " const PredicateBitset AvailableFeatures = getAvailableFeatures();\n"; + << " const PredicateBitset AvailableFeatures = getAvailableFeatures();\n" + << " NewMIVector OutMIs;\n" + << " State.MIs.clear();\n" + << " State.MIs.push_back(&I);\n\n"; for (auto &Rule : Rules) { - Rule.emit(OS, SubtargetFeatures); + Rule.emit(OS); + ++CurrentMatchTableID; ++NumPatternEmitted; + assert(CurrentMatchTableID == NumPatternEmitted && + "Statistic deviates from number of emitted tables"); } OS << " return false;\n" diff --git a/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp b/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp index ab7d964cd671..e270a17356f7 100644 --- a/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/contrib/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -67,7 +67,7 @@ class InstrInfoEmitter { void emitOperandTypesEnum(raw_ostream &OS, const CodeGenTarget &Target); void initOperandMapData( ArrayRef NumberedInstructions, - const std::string &Namespace, + StringRef Namespace, std::map &Operands, OpNameMapTy &OperandMap); void emitOperandNameMappings(raw_ostream &OS, const CodeGenTarget &Target, @@ -207,7 +207,7 @@ void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS, /// well as the getNamedOperandIdx() function. void InstrInfoEmitter::initOperandMapData( ArrayRef NumberedInstructions, - const std::string &Namespace, + StringRef Namespace, std::map &Operands, OpNameMapTy &OperandMap) { unsigned NumOperands = 0; @@ -224,7 +224,7 @@ void InstrInfoEmitter::initOperandMapData( } OpList[I->second] = Info.MIOperandNo; } - OperandMap[OpList].push_back(Namespace + "::" + + OperandMap[OpList].push_back(Namespace.str() + "::" + Inst->TheDef->getName().str()); } } @@ -243,7 +243,7 @@ void InstrInfoEmitter::initOperandMapData( void InstrInfoEmitter::emitOperandNameMappings(raw_ostream &OS, const CodeGenTarget &Target, ArrayRef NumberedInstructions) { - const std::string &Namespace = Target.getInstNamespace(); + StringRef Namespace = Target.getInstNamespace(); std::string OpNameNS = "OpName"; // Map of operand names to their enumeration value. This will be used to // generate the OpName enum. @@ -315,7 +315,7 @@ void InstrInfoEmitter::emitOperandNameMappings(raw_ostream &OS, void InstrInfoEmitter::emitOperandTypesEnum(raw_ostream &OS, const CodeGenTarget &Target) { - const std::string &Namespace = Target.getInstNamespace(); + StringRef Namespace = Target.getInstNamespace(); std::vector Operands = Records.getAllDerivedDefinitions("Operand"); OS << "#ifdef GET_INSTRINFO_OPERAND_TYPES_ENUM\n"; @@ -576,7 +576,7 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) { CodeGenTarget Target(Records); // We must emit the PHI opcode first... - std::string Namespace = Target.getInstNamespace(); + StringRef Namespace = Target.getInstNamespace(); if (Namespace.empty()) PrintFatalError("No instructions defined!"); diff --git a/contrib/llvm/utils/TableGen/RegisterBankEmitter.cpp b/contrib/llvm/utils/TableGen/RegisterBankEmitter.cpp index 3f11eff1d371..880d075da427 100644 --- a/contrib/llvm/utils/TableGen/RegisterBankEmitter.cpp +++ b/contrib/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -227,7 +227,7 @@ void RegisterBankEmitter::emitBaseClassImplementation( OS << " // " << LowestIdxInWord << "-" << (LowestIdxInWord + 31) << "\n"; for (const auto &RC : RCs) { std::string QualifiedRegClassID = - (Twine(TargetName) + "::" + RC->getName() + "RegClassID").str(); + (Twine(RC->Namespace) + "::" + RC->getName() + "RegClassID").str(); OS << " (1u << (" << QualifiedRegClassID << " - " << LowestIdxInWord << ")) |\n"; } diff --git a/contrib/llvm/utils/TableGen/SearchableTableEmitter.cpp b/contrib/llvm/utils/TableGen/SearchableTableEmitter.cpp index efd4e83eca90..f73c197dee5a 100644 --- a/contrib/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/contrib/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -230,7 +230,7 @@ void SearchableTableEmitter::emitLookupDeclaration(StringRef Name, void SearchableTableEmitter::emitMapping(Record *InstanceClass, raw_ostream &OS) { - const std::string &TableName = InstanceClass->getName(); + StringRef TableName = InstanceClass->getName(); std::vector Items = Records.getAllDerivedDefinitions(TableName); // Gather all the records we're going to need for this particular mapping. @@ -265,8 +265,8 @@ void SearchableTableEmitter::emitMapping(Record *InstanceClass, ++Idx; } - OS << "#ifdef GET_" << StringRef(TableName).upper() << "_DECL\n"; - OS << "#undef GET_" << StringRef(TableName).upper() << "_DECL\n"; + OS << "#ifdef GET_" << TableName.upper() << "_DECL\n"; + OS << "#undef GET_" << TableName.upper() << "_DECL\n"; // Next emit the enum containing the top-level names for use in C++ code if // requested @@ -281,8 +281,8 @@ void SearchableTableEmitter::emitMapping(Record *InstanceClass, OS << "#endif\n\n"; - OS << "#ifdef GET_" << StringRef(TableName).upper() << "_IMPL\n"; - OS << "#undef GET_" << StringRef(TableName).upper() << "_IMPL\n"; + OS << "#ifdef GET_" << TableName.upper() << "_IMPL\n"; + OS << "#undef GET_" << TableName.upper() << "_IMPL\n"; // The primary data table contains all the fields defined for this map. emitPrimaryTable(TableName, FieldNames, SearchFieldNames, SearchTables, Items, diff --git a/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp b/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp index 16d5740b79a3..d1d873b66aaa 100644 --- a/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/contrib/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -375,7 +375,7 @@ EmitStageAndOperandCycleData(raw_ostream &OS, if (FUs.empty()) continue; - const std::string &Name = ProcModel.ItinsDef->getName(); + StringRef Name = ProcModel.ItinsDef->getName(); OS << "\n// Functional units for \"" << Name << "\"\n" << "namespace " << Name << "FU {\n"; @@ -429,7 +429,7 @@ EmitStageAndOperandCycleData(raw_ostream &OS, if (!ProcModel.hasItineraries()) continue; - const std::string &Name = ProcModel.ItinsDef->getName(); + StringRef Name = ProcModel.ItinsDef->getName(); ItinList.resize(SchedModels.numInstrSchedClasses()); assert(ProcModel.ItinDefList.size() == ItinList.size() && "bad Itins"); @@ -546,9 +546,6 @@ EmitItineraries(raw_ostream &OS, if (!ItinsDefSet.insert(ItinsDef).second) continue; - // Get processor itinerary name - const std::string &Name = ItinsDef->getName(); - // Get the itinerary list for the processor. assert(ProcItinListsIter != ProcItinLists.end() && "bad iterator"); std::vector &ItinList = *ProcItinListsIter; @@ -562,7 +559,7 @@ EmitItineraries(raw_ostream &OS, OS << "static const llvm::InstrItinerary "; // Begin processor itinerary table - OS << Name << "[] = {\n"; + OS << ItinsDef->getName() << "[] = {\n"; // For each itinerary class in CodeGenSchedClass::Index order. for (unsigned j = 0, M = ItinList.size(); j < M; ++j) { diff --git a/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp b/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp index c9e36f96736a..c80b96905b30 100644 --- a/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp +++ b/contrib/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -10,7 +10,7 @@ // This file is part of the X86 Disassembler Emitter. // It contains the implementation of the disassembler tables. // Documentation for the disassembler emitter in general can be found in -// X86DisasemblerEmitter.h. +// X86DisassemblerEmitter.h. // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/utils/TableGen/X86DisassemblerTables.h b/contrib/llvm/utils/TableGen/X86DisassemblerTables.h index 5a8688be0819..1171c7980f42 100644 --- a/contrib/llvm/utils/TableGen/X86DisassemblerTables.h +++ b/contrib/llvm/utils/TableGen/X86DisassemblerTables.h @@ -10,7 +10,7 @@ // This file is part of the X86 Disassembler Emitter. // It contains the interface of the disassembler tables. // Documentation for the disassembler emitter in general can be found in -// X86DisasemblerEmitter.h. +// X86DisassemblerEmitter.h. // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/utils/TableGen/X86ModRMFilters.h b/contrib/llvm/utils/TableGen/X86ModRMFilters.h index d919c588c644..73d5602fd91c 100644 --- a/contrib/llvm/utils/TableGen/X86ModRMFilters.h +++ b/contrib/llvm/utils/TableGen/X86ModRMFilters.h @@ -11,7 +11,7 @@ // It contains ModR/M filters that determine which values of the ModR/M byte // are valid for a partiuclar instruction. // Documentation for the disassembler emitter in general can be found in -// X86DisasemblerEmitter.h. +// X86DisassemblerEmitter.h. // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp b/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp index 55e75763ad69..202a71ae4dc4 100644 --- a/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp +++ b/contrib/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -10,7 +10,7 @@ // This file is part of the X86 Disassembler Emitter. // It contains the implementation of a single recognizable instruction. // Documentation for the disassembler emitter in general can be found in -// X86DisasemblerEmitter.h. +// X86DisassemblerEmitter.h. // //===----------------------------------------------------------------------===// @@ -367,7 +367,7 @@ void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex, ++operandIndex; } - const std::string &typeName = (*Operands)[operandIndex].Rec->getName(); + StringRef typeName = (*Operands)[operandIndex].Rec->getName(); OperandEncoding encoding = encodingFromString(typeName, OpSize); // Adjust the encoding type for an operand based on the instruction. diff --git a/contrib/llvm/utils/TableGen/X86RecognizableInstr.h b/contrib/llvm/utils/TableGen/X86RecognizableInstr.h index 7fe731ec8b1c..ea99935f8790 100644 --- a/contrib/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/contrib/llvm/utils/TableGen/X86RecognizableInstr.h @@ -10,7 +10,7 @@ // This file is part of the X86 Disassembler Emitter. // It contains the interface of a single recognizable instruction. // Documentation for the disassembler emitter in general can be found in -// X86DisasemblerEmitter.h. +// X86DisassemblerEmitter.h. // //===----------------------------------------------------------------------===// diff --git a/lib/clang/include/clang/Basic/Version.inc b/lib/clang/include/clang/Basic/Version.inc index 01561d35e5ec..67eef20f8640 100644 --- a/lib/clang/include/clang/Basic/Version.inc +++ b/lib/clang/include/clang/Basic/Version.inc @@ -8,4 +8,4 @@ #define CLANG_VENDOR "FreeBSD " -#define SVN_REVISION "306956" +#define SVN_REVISION "307894" diff --git a/lib/clang/include/lld/Config/Version.inc b/lib/clang/include/lld/Config/Version.inc index 57dec49b9db5..b8450b7c0456 100644 --- a/lib/clang/include/lld/Config/Version.inc +++ b/lib/clang/include/lld/Config/Version.inc @@ -4,5 +4,5 @@ #define LLD_VERSION_STRING "5.0.0" #define LLD_VERSION_MAJOR 5 #define LLD_VERSION_MINOR 0 -#define LLD_REVISION_STRING "306956" +#define LLD_REVISION_STRING "307894" #define LLD_REPOSITORY_STRING "FreeBSD" diff --git a/lib/clang/include/llvm/Support/VCSRevision.h b/lib/clang/include/llvm/Support/VCSRevision.h index 75b10b66d2d4..b40ecf0c5c10 100644 --- a/lib/clang/include/llvm/Support/VCSRevision.h +++ b/lib/clang/include/llvm/Support/VCSRevision.h @@ -1,2 +1,2 @@ /* $FreeBSD$ */ -#define LLVM_REVISION "svn-r306956" +#define LLVM_REVISION "svn-r307894" diff --git a/lib/clang/libllvm/Makefile b/lib/clang/libllvm/Makefile index 2372bfc0c9ec..c3a0e52737b8 100644 --- a/lib/clang/libllvm/Makefile +++ b/lib/clang/libllvm/Makefile @@ -405,6 +405,7 @@ SRCS_EXT+= DebugInfo/PDB/Native/InfoStream.cpp SRCS_EXT+= DebugInfo/PDB/Native/InfoStreamBuilder.cpp SRCS_EXT+= DebugInfo/PDB/Native/ModuleDebugStream.cpp SRCS_EXT+= DebugInfo/PDB/Native/NamedStreamMap.cpp +SRCS_EXT+= DebugInfo/PDB/Native/NativeBuiltinSymbol.cpp SRCS_EXT+= DebugInfo/PDB/Native/NativeCompilandSymbol.cpp SRCS_EXT+= DebugInfo/PDB/Native/NativeEnumModules.cpp SRCS_EXT+= DebugInfo/PDB/Native/NativeExeSymbol.cpp @@ -415,6 +416,7 @@ SRCS_EXT+= DebugInfo/PDB/Native/PDBFileBuilder.cpp SRCS_EXT+= DebugInfo/PDB/Native/PDBStringTable.cpp SRCS_EXT+= DebugInfo/PDB/Native/PDBStringTableBuilder.cpp SRCS_EXT+= DebugInfo/PDB/Native/PublicsStream.cpp +SRCS_EXT+= DebugInfo/PDB/Native/PublicsStreamBuilder.cpp SRCS_EXT+= DebugInfo/PDB/Native/RawError.cpp SRCS_EXT+= DebugInfo/PDB/Native/SymbolStream.cpp SRCS_EXT+= DebugInfo/PDB/Native/TpiHashing.cpp @@ -528,6 +530,7 @@ SRCS_MIN+= IR/Pass.cpp SRCS_MIN+= IR/PassManager.cpp SRCS_MIN+= IR/PassRegistry.cpp SRCS_MIN+= IR/ProfileSummary.cpp +SRCS_MIN+= IR/SafepointIRVerifier.cpp SRCS_MIN+= IR/Statepoint.cpp SRCS_MIN+= IR/Type.cpp SRCS_MIN+= IR/TypeFinder.cpp diff --git a/lib/libc++/Makefile b/lib/libc++/Makefile index beab0df3d422..db2d3144f901 100644 --- a/lib/libc++/Makefile +++ b/lib/libc++/Makefile @@ -92,7 +92,6 @@ STD_HEADERS+= __libcpp_version STD_HEADERS+= __locale STD_HEADERS+= __mutex_base STD_HEADERS+= __nullptr -STD_HEADERS+= __refstring STD_HEADERS+= __split_buffer STD_HEADERS+= __sso_allocator STD_HEADERS+= __std_stream diff --git a/lib/libclang_rt/asan/Makefile b/lib/libclang_rt/asan/Makefile index 359b4fa60c30..8c11338735ff 100644 --- a/lib/libclang_rt/asan/Makefile +++ b/lib/libclang_rt/asan/Makefile @@ -40,13 +40,13 @@ SRCS+= sanitizer_common/sanitizer_common_libcdep.cc SRCS+= sanitizer_common/sanitizer_coverage_libcdep_new.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector1.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector2.cc +SRCS+= sanitizer_common/sanitizer_errno.cc SRCS+= sanitizer_common/sanitizer_flag_parser.cc SRCS+= sanitizer_common/sanitizer_flags.cc SRCS+= sanitizer_common/sanitizer_libc.cc SRCS+= sanitizer_common/sanitizer_libignore.cc SRCS+= sanitizer_common/sanitizer_linux.cc SRCS+= sanitizer_common/sanitizer_linux_libcdep.cc -SRCS+= sanitizer_common/sanitizer_linux_x86_64.S SRCS+= sanitizer_common/sanitizer_mac.cc SRCS+= sanitizer_common/sanitizer_persistent_allocator.cc SRCS+= sanitizer_common/sanitizer_platform_limits_linux.cc diff --git a/lib/libclang_rt/asan_dynamic/Makefile b/lib/libclang_rt/asan_dynamic/Makefile index 93e4f1017d6f..9ef7dc4c846c 100644 --- a/lib/libclang_rt/asan_dynamic/Makefile +++ b/lib/libclang_rt/asan_dynamic/Makefile @@ -47,7 +47,6 @@ SRCS+= sanitizer_common/sanitizer_libc.cc SRCS+= sanitizer_common/sanitizer_libignore.cc SRCS+= sanitizer_common/sanitizer_linux.cc SRCS+= sanitizer_common/sanitizer_linux_libcdep.cc -SRCS+= sanitizer_common/sanitizer_linux_x86_64.S SRCS+= sanitizer_common/sanitizer_mac.cc SRCS+= sanitizer_common/sanitizer_persistent_allocator.cc SRCS+= sanitizer_common/sanitizer_platform_limits_linux.cc diff --git a/lib/libclang_rt/safestack/Makefile b/lib/libclang_rt/safestack/Makefile index 661212075ab4..fd305e6b8cb6 100644 --- a/lib/libclang_rt/safestack/Makefile +++ b/lib/libclang_rt/safestack/Makefile @@ -14,12 +14,12 @@ SRCS+= sanitizer_common/sanitizer_common.cc SRCS+= sanitizer_common/sanitizer_common_nolibc.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector1.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector2.cc +SRCS+= sanitizer_common/sanitizer_errno.cc SRCS+= sanitizer_common/sanitizer_flag_parser.cc SRCS+= sanitizer_common/sanitizer_flags.cc SRCS+= sanitizer_common/sanitizer_libc.cc SRCS+= sanitizer_common/sanitizer_libignore.cc SRCS+= sanitizer_common/sanitizer_linux.cc -SRCS+= sanitizer_common/sanitizer_linux_x86_64.S SRCS+= sanitizer_common/sanitizer_mac.cc SRCS+= sanitizer_common/sanitizer_persistent_allocator.cc SRCS+= sanitizer_common/sanitizer_platform_limits_linux.cc diff --git a/lib/libclang_rt/stats/Makefile b/lib/libclang_rt/stats/Makefile index 378f3a943379..dd214cfc4b28 100644 --- a/lib/libclang_rt/stats/Makefile +++ b/lib/libclang_rt/stats/Makefile @@ -11,13 +11,13 @@ SRCS+= sanitizer_common/sanitizer_common_libcdep.cc SRCS+= sanitizer_common/sanitizer_coverage_libcdep_new.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector1.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector2.cc +SRCS+= sanitizer_common/sanitizer_errno.cc SRCS+= sanitizer_common/sanitizer_flag_parser.cc SRCS+= sanitizer_common/sanitizer_flags.cc SRCS+= sanitizer_common/sanitizer_libc.cc SRCS+= sanitizer_common/sanitizer_libignore.cc SRCS+= sanitizer_common/sanitizer_linux.cc SRCS+= sanitizer_common/sanitizer_linux_libcdep.cc -SRCS+= sanitizer_common/sanitizer_linux_x86_64.S SRCS+= sanitizer_common/sanitizer_mac.cc SRCS+= sanitizer_common/sanitizer_persistent_allocator.cc SRCS+= sanitizer_common/sanitizer_platform_limits_linux.cc diff --git a/lib/libclang_rt/ubsan_standalone/Makefile b/lib/libclang_rt/ubsan_standalone/Makefile index 01b8f20c634d..75f5268b8efd 100644 --- a/lib/libclang_rt/ubsan_standalone/Makefile +++ b/lib/libclang_rt/ubsan_standalone/Makefile @@ -11,13 +11,13 @@ SRCS+= sanitizer_common/sanitizer_common_libcdep.cc SRCS+= sanitizer_common/sanitizer_coverage_libcdep_new.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector1.cc SRCS+= sanitizer_common/sanitizer_deadlock_detector2.cc +SRCS+= sanitizer_common/sanitizer_errno.cc SRCS+= sanitizer_common/sanitizer_flag_parser.cc SRCS+= sanitizer_common/sanitizer_flags.cc SRCS+= sanitizer_common/sanitizer_libc.cc SRCS+= sanitizer_common/sanitizer_libignore.cc SRCS+= sanitizer_common/sanitizer_linux.cc SRCS+= sanitizer_common/sanitizer_linux_libcdep.cc -SRCS+= sanitizer_common/sanitizer_linux_x86_64.S SRCS+= sanitizer_common/sanitizer_mac.cc SRCS+= sanitizer_common/sanitizer_persistent_allocator.cc SRCS+= sanitizer_common/sanitizer_platform_limits_linux.cc diff --git a/usr.bin/clang/llvm-pdbutil/Makefile b/usr.bin/clang/llvm-pdbutil/Makefile index 46b6de22c7b1..c78fc22c248a 100644 --- a/usr.bin/clang/llvm-pdbutil/Makefile +++ b/usr.bin/clang/llvm-pdbutil/Makefile @@ -7,6 +7,7 @@ SRCDIR= tools/llvm-pdbutil SRCS+= Analyze.cpp SRCS+= BytesOutputStyle.cpp SRCS+= Diff.cpp +SRCS+= DiffPrinter.cpp SRCS+= DumpOutputStyle.cpp SRCS+= FormatUtil.cpp SRCS+= LinePrinter.cpp