From 7eff647615f93a9aaff1997e1880b195dc3aabe6 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Fri, 18 Feb 2022 23:37:06 +0100 Subject: [PATCH] Vendor import of llvm-project branch release/14.x llvmorg-14.0.0-rc1-74-g4dc3cb8e3255. --- clang/include/clang/Basic/Builtins.def | 56 +-- clang/include/clang/Basic/Builtins.h | 25 +- .../clang/Basic/DiagnosticCommonKinds.td | 4 +- clang/include/clang/Basic/TargetInfo.h | 4 +- clang/lib/Basic/Builtins.cpp | 26 +- clang/lib/CodeGen/CodeGenModule.cpp | 8 + clang/lib/CodeGen/TargetInfo.cpp | 34 ++ clang/lib/Driver/ToolChains/Clang.cpp | 37 +- clang/lib/Driver/ToolChains/Hexagon.cpp | 34 +- clang/lib/Driver/ToolChains/MSVC.cpp | 4 +- clang/lib/Driver/ToolChains/MinGW.cpp | 4 +- clang/lib/Driver/ToolChains/MinGW.h | 2 + clang/lib/Driver/ToolChains/OpenBSD.cpp | 9 +- clang/lib/Format/DefinitionBlockSeparator.cpp | 49 ++- clang/lib/Format/QualifierAlignmentFixer.cpp | 4 + clang/lib/Format/UnwrappedLineParser.cpp | 3 +- clang/lib/Headers/opencl-c-base.h | 2 + clang/lib/Headers/opencl-c.h | 6 + clang/lib/Headers/ppc_wrappers/mm_malloc.h | 2 +- clang/lib/Sema/OpenCLBuiltins.td | 414 ++++++------------ clang/lib/Sema/SemaChecking.cpp | 10 +- clang/lib/Sema/SemaDecl.cpp | 17 - compiler-rt/lib/asan/asan_errors.h | 4 +- .../lib/sanitizer_common/sanitizer_common.h | 2 +- .../lib/sanitizer_common/sanitizer_linux.cpp | 46 +- .../sanitizer_linux_libcdep.cpp | 27 +- .../lib/sanitizer_common/sanitizer_mac.cpp | 4 +- .../sanitizer_symbolizer_report.cpp | 4 +- .../lib/sanitizer_common/sanitizer_win.cpp | 10 +- compiler-rt/lib/tsan/rtl/tsan_interceptors.h | 8 + .../lib/tsan/rtl/tsan_interceptors_posix.cpp | 83 +++- libcxx/include/__algorithm/in_in_out_result.h | 4 +- libcxx/include/__algorithm/in_out_result.h | 4 +- libcxx/include/__config | 2 + .../include/__filesystem/directory_iterator.h | 2 +- .../recursive_directory_iterator.h | 2 +- .../include/__functional/ranges_operations.h | 5 +- libcxx/include/__iterator/advance.h | 4 +- libcxx/include/__iterator/distance.h | 4 +- libcxx/include/__iterator/insert_iterator.h | 2 +- libcxx/include/__iterator/iter_move.h | 2 +- libcxx/include/__iterator/iter_swap.h | 1 - libcxx/include/__iterator/next.h | 4 +- libcxx/include/__iterator/prev.h | 4 +- libcxx/include/__memory/concepts.h | 6 +- libcxx/include/__memory/ranges_construct_at.h | 4 +- .../ranges_uninitialized_algorithms.h | 6 +- libcxx/include/__ranges/all.h | 4 +- libcxx/include/__ranges/common_view.h | 4 +- libcxx/include/__ranges/concepts.h | 12 +- libcxx/include/__ranges/copyable_box.h | 4 +- libcxx/include/__ranges/counted.h | 4 +- libcxx/include/__ranges/dangling.h | 2 +- libcxx/include/__ranges/data.h | 4 +- libcxx/include/__ranges/drop_view.h | 4 +- libcxx/include/__ranges/empty.h | 4 +- libcxx/include/__ranges/empty_view.h | 4 +- .../include/__ranges/enable_borrowed_range.h | 3 +- libcxx/include/__ranges/enable_view.h | 2 +- libcxx/include/__ranges/iota_view.h | 4 +- libcxx/include/__ranges/join_view.h | 4 +- .../include/__ranges/non_propagating_cache.h | 4 +- libcxx/include/__ranges/owning_view.h | 4 +- libcxx/include/__ranges/range_adaptor.h | 4 +- libcxx/include/__ranges/ref_view.h | 4 +- libcxx/include/__ranges/reverse_view.h | 4 +- libcxx/include/__ranges/single_view.h | 4 +- libcxx/include/__ranges/size.h | 4 +- libcxx/include/__ranges/subrange.h | 4 +- libcxx/include/__ranges/take_view.h | 4 +- libcxx/include/__ranges/transform_view.h | 4 +- libcxx/include/__ranges/view_interface.h | 4 +- libcxx/include/__threading_support | 56 --- libcxx/include/ranges | 12 +- libcxx/include/span | 19 +- libcxx/include/string | 4 + libcxx/include/string_view | 8 +- libcxx/include/vector | 93 ++-- libcxx/src/vector.cpp | 17 +- libunwind/include/__libunwind_config.h | 8 + libunwind/src/DwarfInstructions.hpp | 19 + libunwind/src/DwarfParser.hpp | 27 +- libunwind/src/Registers.hpp | 186 ++++++++ libunwind/src/UnwindCursor.hpp | 16 + libunwind/src/UnwindRegistersRestore.S | 47 ++ libunwind/src/UnwindRegistersSave.S | 58 +++ libunwind/src/config.h | 11 +- libunwind/src/libunwind.cpp | 2 + lld/ELF/Arch/PPC64.cpp | 7 +- lld/ELF/Driver.cpp | 35 +- lld/ELF/InputFiles.cpp | 11 +- lld/ELF/OutputSections.cpp | 2 +- lld/docs/ReleaseNotes.rst | 148 ++++++- lldb/include/lldb/Core/Mangled.h | 11 +- lldb/include/lldb/Core/RichManglingContext.h | 27 +- lldb/include/lldb/Target/Process.h | 38 +- lldb/source/Commands/CommandObjectMemory.cpp | 20 +- lldb/source/Commands/CommandObjectThread.cpp | 49 +++ lldb/source/Core/Mangled.cpp | 21 +- lldb/source/Core/RichManglingContext.cpp | 46 +- .../ASan/InstrumentationRuntimeASan.cpp | 2 +- .../Process/elf-core/ProcessElfCore.cpp | 4 +- .../Plugins/Process/elf-core/ProcessElfCore.h | 8 +- .../Process/gdb-remote/ProcessGDBRemote.cpp | 4 +- .../Process/gdb-remote/ProcessGDBRemote.h | 6 +- .../Process/minidump/ProcessMinidump.cpp | 4 +- .../Process/minidump/ProcessMinidump.h | 6 +- .../Process/scripted/ScriptedProcess.cpp | 4 +- .../Process/scripted/ScriptedProcess.h | 6 +- lldb/source/Symbol/Symtab.cpp | 11 +- lldb/source/Target/Process.cpp | 23 +- llvm/include/llvm/CodeGen/TargetLowering.h | 8 + .../llvm/DebugInfo/Symbolize/DIFetcher.h | 51 +++ .../llvm/DebugInfo/Symbolize/Symbolize.h | 13 + llvm/include/llvm/Debuginfod/DIFetcher.h | 34 ++ llvm/include/llvm/Support/Compiler.h | 2 + .../LiveDebugValues/InstrRefBasedImpl.cpp | 376 ++++++++++++---- .../LiveDebugValues/InstrRefBasedImpl.h | 63 ++- .../LiveDebugValues/LiveDebugValues.cpp | 5 + llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 73 ++- llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp | 58 +++ llvm/lib/DebugInfo/Symbolize/Symbolize.cpp | 161 +++---- llvm/lib/Debuginfod/DIFetcher.cpp | 28 ++ llvm/lib/Support/RISCVISAInfo.cpp | 23 +- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 53 +-- llvm/lib/Target/AArch64/SVEInstrFormats.td | 25 +- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 35 +- .../RISCV/Disassembler/RISCVDisassembler.cpp | 32 ++ llvm/lib/Target/RISCV/RISCV.td | 37 ++ llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 8 - llvm/lib/Target/RISCV/RISCVFrameLowering.h | 2 - llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 36 +- llvm/lib/Target/RISCV/RISCVInstrInfoD.td | 209 ++++++--- llvm/lib/Target/RISCV/RISCVInstrInfoF.td | 307 +++++++++---- llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td | 207 +++++---- llvm/lib/Target/RISCV/RISCVRegisterInfo.td | 30 ++ llvm/lib/Target/RISCV/RISCVSubtarget.h | 8 + llvm/lib/Target/X86/X86.td | 2 + llvm/lib/Target/X86/X86FrameLowering.cpp | 5 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 46 +- llvm/lib/Target/X86/X86ISelLowering.h | 3 + .../Transforms/IPO/AttributorAttributes.cpp | 82 ++-- llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 1 - .../Transforms/Scalar/InferAddressSpaces.cpp | 11 +- .../Transforms/Vectorize/LoopVectorize.cpp | 35 +- .../Transforms/Vectorize/SLPVectorizer.cpp | 179 +------- .../tools/llvm-symbolizer/llvm-symbolizer.cpp | 9 +- openmp/runtime/src/kmp_affinity.cpp | 2 +- 148 files changed, 2843 insertions(+), 1593 deletions(-) create mode 100644 llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h create mode 100644 llvm/include/llvm/Debuginfod/DIFetcher.h create mode 100644 llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp create mode 100644 llvm/lib/Debuginfod/DIFetcher.cpp diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def index d2cb14d2fd8c..a17d2e59f5d8 100644 --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -1625,50 +1625,50 @@ LANGBUILTIN(__builtin_coro_suspend, "cIb", "n", COR_LANG) // OpenCL v2.0 s6.13.16, s9.17.3.5 - Pipe functions. // We need the generic prototype, since the packet type could be anything. -LANGBUILTIN(read_pipe, "i.", "tn", OCLC20_LANG) -LANGBUILTIN(write_pipe, "i.", "tn", OCLC20_LANG) +LANGBUILTIN(read_pipe, "i.", "tn", OCL_PIPE) +LANGBUILTIN(write_pipe, "i.", "tn", OCL_PIPE) -LANGBUILTIN(reserve_read_pipe, "i.", "tn", OCLC20_LANG) -LANGBUILTIN(reserve_write_pipe, "i.", "tn", OCLC20_LANG) +LANGBUILTIN(reserve_read_pipe, "i.", "tn", OCL_PIPE) +LANGBUILTIN(reserve_write_pipe, "i.", "tn", OCL_PIPE) -LANGBUILTIN(commit_write_pipe, "v.", "tn", OCLC20_LANG) -LANGBUILTIN(commit_read_pipe, "v.", "tn", OCLC20_LANG) +LANGBUILTIN(commit_write_pipe, "v.", "tn", OCL_PIPE) +LANGBUILTIN(commit_read_pipe, "v.", "tn", OCL_PIPE) -LANGBUILTIN(sub_group_reserve_read_pipe, "i.", "tn", OCLC20_LANG) -LANGBUILTIN(sub_group_reserve_write_pipe, "i.", "tn", OCLC20_LANG) +LANGBUILTIN(sub_group_reserve_read_pipe, "i.", "tn", OCL_PIPE) +LANGBUILTIN(sub_group_reserve_write_pipe, "i.", "tn", OCL_PIPE) -LANGBUILTIN(sub_group_commit_read_pipe, "v.", "tn", OCLC20_LANG) -LANGBUILTIN(sub_group_commit_write_pipe, "v.", "tn", OCLC20_LANG) +LANGBUILTIN(sub_group_commit_read_pipe, "v.", "tn", OCL_PIPE) +LANGBUILTIN(sub_group_commit_write_pipe, "v.", "tn", OCL_PIPE) -LANGBUILTIN(work_group_reserve_read_pipe, "i.", "tn", OCLC20_LANG) -LANGBUILTIN(work_group_reserve_write_pipe, "i.", "tn", OCLC20_LANG) +LANGBUILTIN(work_group_reserve_read_pipe, "i.", "tn", OCL_PIPE) +LANGBUILTIN(work_group_reserve_write_pipe, "i.", "tn", OCL_PIPE) -LANGBUILTIN(work_group_commit_read_pipe, "v.", "tn", OCLC20_LANG) -LANGBUILTIN(work_group_commit_write_pipe, "v.", "tn", OCLC20_LANG) +LANGBUILTIN(work_group_commit_read_pipe, "v.", "tn", OCL_PIPE) +LANGBUILTIN(work_group_commit_write_pipe, "v.", "tn", OCL_PIPE) -LANGBUILTIN(get_pipe_num_packets, "Ui.", "tn", OCLC20_LANG) -LANGBUILTIN(get_pipe_max_packets, "Ui.", "tn", OCLC20_LANG) +LANGBUILTIN(get_pipe_num_packets, "Ui.", "tn", OCL_PIPE) +LANGBUILTIN(get_pipe_max_packets, "Ui.", "tn", OCL_PIPE) // OpenCL v2.0 s6.13.17 - Enqueue kernel functions. // Custom builtin check allows to perform special check of passed block arguments. -LANGBUILTIN(enqueue_kernel, "i.", "tn", OCLC20_LANG) -LANGBUILTIN(get_kernel_work_group_size, "Ui.", "tn", OCLC20_LANG) -LANGBUILTIN(get_kernel_preferred_work_group_size_multiple, "Ui.", "tn", OCLC20_LANG) -LANGBUILTIN(get_kernel_max_sub_group_size_for_ndrange, "Ui.", "tn", OCLC20_LANG) -LANGBUILTIN(get_kernel_sub_group_count_for_ndrange, "Ui.", "tn", OCLC20_LANG) +LANGBUILTIN(enqueue_kernel, "i.", "tn", OCL_DSE) +LANGBUILTIN(get_kernel_work_group_size, "Ui.", "tn", OCL_DSE) +LANGBUILTIN(get_kernel_preferred_work_group_size_multiple, "Ui.", "tn", OCL_DSE) +LANGBUILTIN(get_kernel_max_sub_group_size_for_ndrange, "Ui.", "tn", OCL_DSE) +LANGBUILTIN(get_kernel_sub_group_count_for_ndrange, "Ui.", "tn", OCL_DSE) // OpenCL v2.0 s6.13.9 - Address space qualifier functions. // FIXME: Pointer parameters of OpenCL builtins should have their address space // requirement defined. -LANGBUILTIN(to_global, "v*v*", "tn", OCLC20_LANG) -LANGBUILTIN(to_local, "v*v*", "tn", OCLC20_LANG) -LANGBUILTIN(to_private, "v*v*", "tn", OCLC20_LANG) +LANGBUILTIN(to_global, "v*v*", "tn", OCL_GAS) +LANGBUILTIN(to_local, "v*v*", "tn", OCL_GAS) +LANGBUILTIN(to_private, "v*v*", "tn", OCL_GAS) // OpenCL half load/store builtin -LANGBUILTIN(__builtin_store_half, "vdh*", "n", ALL_OCLC_LANGUAGES) -LANGBUILTIN(__builtin_store_halff, "vfh*", "n", ALL_OCLC_LANGUAGES) -LANGBUILTIN(__builtin_load_half, "dhC*", "nc", ALL_OCLC_LANGUAGES) -LANGBUILTIN(__builtin_load_halff, "fhC*", "nc", ALL_OCLC_LANGUAGES) +LANGBUILTIN(__builtin_store_half, "vdh*", "n", ALL_OCL_LANGUAGES) +LANGBUILTIN(__builtin_store_halff, "vfh*", "n", ALL_OCL_LANGUAGES) +LANGBUILTIN(__builtin_load_half, "dhC*", "nc", ALL_OCL_LANGUAGES) +LANGBUILTIN(__builtin_load_halff, "fhC*", "nc", ALL_OCL_LANGUAGES) // Builtins for os_log/os_trace BUILTIN(__builtin_os_log_format_buffer_size, "zcC*.", "p:0:nut") diff --git a/clang/include/clang/Basic/Builtins.h b/clang/include/clang/Basic/Builtins.h index cdaaee48c32d..1dabafce54f3 100644 --- a/clang/include/clang/Basic/Builtins.h +++ b/clang/include/clang/Basic/Builtins.h @@ -28,20 +28,21 @@ class IdentifierTable; class LangOptions; enum LanguageID { - GNU_LANG = 0x1, // builtin requires GNU mode. - C_LANG = 0x2, // builtin for c only. - CXX_LANG = 0x4, // builtin for cplusplus only. - OBJC_LANG = 0x8, // builtin for objective-c and objective-c++ - MS_LANG = 0x10, // builtin requires MS mode. - OCLC20_LANG = 0x20, // builtin for OpenCL C 2.0 only. - OCLC1X_LANG = 0x40, // builtin for OpenCL C 1.x only. - OMP_LANG = 0x80, // builtin requires OpenMP. - CUDA_LANG = 0x100, // builtin requires CUDA. - COR_LANG = 0x200, // builtin requires use of 'fcoroutine-ts' option. + GNU_LANG = 0x1, // builtin requires GNU mode. + C_LANG = 0x2, // builtin for c only. + CXX_LANG = 0x4, // builtin for cplusplus only. + OBJC_LANG = 0x8, // builtin for objective-c and objective-c++ + MS_LANG = 0x10, // builtin requires MS mode. + OMP_LANG = 0x20, // builtin requires OpenMP. + CUDA_LANG = 0x40, // builtin requires CUDA. + COR_LANG = 0x80, // builtin requires use of 'fcoroutine-ts' option. + OCL_GAS = 0x100, // builtin requires OpenCL generic address space. + OCL_PIPE = 0x200, // builtin requires OpenCL pipe. + OCL_DSE = 0x400, // builtin requires OpenCL device side enqueue. + ALL_OCL_LANGUAGES = 0x800, // builtin for OCL languages. ALL_LANGUAGES = C_LANG | CXX_LANG | OBJC_LANG, // builtin for all languages. ALL_GNU_LANGUAGES = ALL_LANGUAGES | GNU_LANG, // builtin requires GNU mode. - ALL_MS_LANGUAGES = ALL_LANGUAGES | MS_LANG, // builtin requires MS mode. - ALL_OCLC_LANGUAGES = OCLC1X_LANG | OCLC20_LANG // builtin for OCLC languages. + ALL_MS_LANGUAGES = ALL_LANGUAGES | MS_LANG // builtin requires MS mode. }; namespace Builtin { diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index 5ea55b0fd31b..421527827a4b 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -145,8 +145,8 @@ def warn_conflicting_nullability_attr_overriding_param_types : Warning< def err_nullability_conflicting : Error< "nullability specifier %0 conflicts with existing specifier %1">; -def warn_target_unsupported_branch_protection_option: Warning < - "ignoring '-mbranch-protection=' option because the '%0' architecture does not support it">, +def warn_incompatible_branch_protection_option: Warning < + "'-mbranch-protection=' option is incompatible with the '%0' architecture">, InGroup; def warn_target_unsupported_branch_protection_attribute: Warning < diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h index a49342a34f3e..e7db877f4e2b 100644 --- a/clang/include/clang/Basic/TargetInfo.h +++ b/clang/include/clang/Basic/TargetInfo.h @@ -644,8 +644,8 @@ class TargetInfo : public virtual TransferrableTargetInfo, } /// Return the largest alignment for which a suitably-sized allocation with - /// '::operator new(size_t)' or 'malloc' is guaranteed to produce a - /// correctly-aligned pointer. + /// '::operator new(size_t)' is guaranteed to produce a correctly-aligned + /// pointer. unsigned getNewAlign() const { return NewAlign ? NewAlign : std::max(LongDoubleAlign, LongLongAlign); } diff --git a/clang/lib/Basic/Builtins.cpp b/clang/lib/Basic/Builtins.cpp index 2b0f4071662c..6d278e9c4a22 100644 --- a/clang/lib/Basic/Builtins.cpp +++ b/clang/lib/Basic/Builtins.cpp @@ -69,22 +69,26 @@ bool Builtin::Context::builtinIsSupported(const Builtin::Info &BuiltinInfo, bool MSModeUnsupported = !LangOpts.MicrosoftExt && (BuiltinInfo.Langs & MS_LANG); bool ObjCUnsupported = !LangOpts.ObjC && BuiltinInfo.Langs == OBJC_LANG; - bool OclC1Unsupported = (LangOpts.OpenCLVersion / 100) != 1 && - (BuiltinInfo.Langs & ALL_OCLC_LANGUAGES ) == OCLC1X_LANG; - bool OclC2Unsupported = - (LangOpts.getOpenCLCompatibleVersion() != 200) && - (BuiltinInfo.Langs & ALL_OCLC_LANGUAGES) == OCLC20_LANG; - bool OclCUnsupported = !LangOpts.OpenCL && - (BuiltinInfo.Langs & ALL_OCLC_LANGUAGES); + bool OclCUnsupported = + !LangOpts.OpenCL && (BuiltinInfo.Langs & ALL_OCL_LANGUAGES); + bool OclGASUnsupported = + !LangOpts.OpenCLGenericAddressSpace && (BuiltinInfo.Langs & OCL_GAS); + bool OclPipeUnsupported = + !LangOpts.OpenCLPipes && (BuiltinInfo.Langs & OCL_PIPE); + // Device side enqueue is not supported until OpenCL 2.0. In 2.0 and higher + // support is indicated with language option for blocks. + bool OclDSEUnsupported = + (LangOpts.getOpenCLCompatibleVersion() < 200 || !LangOpts.Blocks) && + (BuiltinInfo.Langs & OCL_DSE); bool OpenMPUnsupported = !LangOpts.OpenMP && BuiltinInfo.Langs == OMP_LANG; bool CUDAUnsupported = !LangOpts.CUDA && BuiltinInfo.Langs == CUDA_LANG; bool CPlusPlusUnsupported = !LangOpts.CPlusPlus && BuiltinInfo.Langs == CXX_LANG; return !BuiltinsUnsupported && !CorBuiltinsUnsupported && - !MathBuiltinsUnsupported && !OclCUnsupported && !OclC1Unsupported && - !OclC2Unsupported && !OpenMPUnsupported && !GnuModeUnsupported && - !MSModeUnsupported && !ObjCUnsupported && !CPlusPlusUnsupported && - !CUDAUnsupported; + !MathBuiltinsUnsupported && !OclCUnsupported && !OclGASUnsupported && + !OclPipeUnsupported && !OclDSEUnsupported && !OpenMPUnsupported && + !GnuModeUnsupported && !MSModeUnsupported && !ObjCUnsupported && + !CPlusPlusUnsupported && !CUDAUnsupported; } /// initializeBuiltins - Mark the identifiers for all the builtins with their diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 2346176a1562..29806b65e984 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -4381,6 +4381,14 @@ LangAS CodeGenModule::GetGlobalConstantAddressSpace() const { return LangAS::opencl_constant; if (LangOpts.SYCLIsDevice) return LangAS::sycl_global; + if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV()) + // For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V) + // instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up + // with OpVariable instructions with Generic storage class which is not + // allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V + // UniformConstant storage class is not viable as pointers to it may not be + // casted to Generic pointers which are used to model HIP's "flat" pointers. + return LangAS::cuda_device; if (auto AS = getTarget().getConstantAddressSpace()) return AS.getValue(); return LangAS::Default; diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp index 8a0150218a7a..9af3004ebcc5 100644 --- a/clang/lib/CodeGen/TargetInfo.cpp +++ b/clang/lib/CodeGen/TargetInfo.cpp @@ -9474,6 +9474,28 @@ class SparcV8TargetCodeGenInfo : public TargetCodeGenInfo { public: SparcV8TargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(std::make_unique(CGT)) {} + + llvm::Value *decodeReturnAddress(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address) const override { + int Offset; + if (isAggregateTypeForABI(CGF.CurFnInfo->getReturnType())) + Offset = 12; + else + Offset = 8; + return CGF.Builder.CreateGEP(CGF.Int8Ty, Address, + llvm::ConstantInt::get(CGF.Int32Ty, Offset)); + } + + llvm::Value *encodeReturnAddress(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address) const override { + int Offset; + if (isAggregateTypeForABI(CGF.CurFnInfo->getReturnType())) + Offset = -12; + else + Offset = -8; + return CGF.Builder.CreateGEP(CGF.Int8Ty, Address, + llvm::ConstantInt::get(CGF.Int32Ty, Offset)); + } }; } // end anonymous namespace @@ -9748,6 +9770,18 @@ class SparcV9TargetCodeGenInfo : public TargetCodeGenInfo { bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; + + llvm::Value *decodeReturnAddress(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address) const override { + return CGF.Builder.CreateGEP(CGF.Int8Ty, Address, + llvm::ConstantInt::get(CGF.Int32Ty, 8)); + } + + llvm::Value *encodeReturnAddress(CodeGen::CodeGenFunction &CGF, + llvm::Value *Address) const override { + return CGF.Builder.CreateGEP(CGF.Int8Ty, Address, + llvm::ConstantInt::get(CGF.Int32Ty, -8)); + } }; } // end anonymous namespace diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 7aac977209eb..f2f18e901ab0 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1639,7 +1639,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, const Driver &D = TC.getDriver(); const llvm::Triple &Triple = TC.getEffectiveTriple(); if (!(isAArch64 || (Triple.isArmT32() && Triple.isArmMClass()))) - D.Diag(diag::warn_target_unsupported_branch_protection_option) + D.Diag(diag::warn_incompatible_branch_protection_option) << Triple.getArchName(); StringRef Scope, Key; @@ -8148,11 +8148,25 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, const InputInfoList &Inputs, const ArgList &Args, const char *LinkingOutput) const { + const Driver &D = getToolChain().getDriver(); + const llvm::Triple TheTriple = getToolChain().getTriple(); + auto OpenMPTCRange = C.getOffloadToolChains(); ArgStringList CmdArgs; - if (getToolChain().getDriver().isUsingLTO(/* IsOffload */ true)) { + // Pass the CUDA path to the linker wrapper tool. + for (auto &I : llvm::make_range(OpenMPTCRange.first, OpenMPTCRange.second)) { + const ToolChain *TC = I.second; + if (TC->getTriple().isNVPTX()) { + CudaInstallationDetector CudaInstallation(D, TheTriple, Args); + if (CudaInstallation.isValid()) + CmdArgs.push_back(Args.MakeArgString( + "--cuda-path=" + CudaInstallation.getInstallPath())); + break; + } + } + + if (D.isUsingLTO(/* IsOffload */ true)) { // Pass in target features for each toolchain. - auto OpenMPTCRange = C.getOffloadToolChains(); for (auto &I : llvm::make_range(OpenMPTCRange.first, OpenMPTCRange.second)) { const ToolChain *TC = I.second; @@ -8165,9 +8179,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, } // Pass in the bitcode library to be linked during LTO. - for (auto &I : llvm::make_range(OpenMPTCRange.first, OpenMPTCRange.second)) { + for (auto &I : + llvm::make_range(OpenMPTCRange.first, OpenMPTCRange.second)) { const ToolChain *TC = I.second; - const Driver &D = TC->getDriver(); + const Driver &TCDriver = TC->getDriver(); const ArgList &TCArgs = C.getArgsForToolChain(TC, "", Action::OFK_OpenMP); StringRef Arch = TCArgs.getLastArgValue(options::OPT_march_EQ); @@ -8182,7 +8197,7 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, BitcodeSuffix += Arch; ArgStringList BitcodeLibrary; - addOpenMPDeviceRTL(D, TCArgs, BitcodeLibrary, BitcodeSuffix, + addOpenMPDeviceRTL(TCDriver, TCArgs, BitcodeLibrary, BitcodeSuffix, TC->getTriple()); if (!BitcodeLibrary.empty()) @@ -8210,12 +8225,8 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, } } - // Construct the link job so we can wrap around it. - Linker->ConstructJob(C, JA, Output, Inputs, Args, LinkingOutput); - const auto &LinkCommand = C.getJobs().getJobs().back(); - CmdArgs.push_back("-host-triple"); - CmdArgs.push_back(Args.MakeArgString(getToolChain().getTripleString())); + CmdArgs.push_back(Args.MakeArgString(TheTriple.getTriple())); if (Args.hasArg(options::OPT_v)) CmdArgs.push_back("-v"); @@ -8246,6 +8257,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, if (Args.getLastArg(options::OPT_save_temps_EQ)) CmdArgs.push_back("-save-temps"); + // Construct the link job so we can wrap around it. + Linker->ConstructJob(C, JA, Output, Inputs, Args, LinkingOutput); + const auto &LinkCommand = C.getJobs().getJobs().back(); + // Add the linker arguments to be forwarded by the wrapper. CmdArgs.push_back("-linker-path"); CmdArgs.push_back(LinkCommand->getExecutable()); diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp index ba3040636604..e772122f5ff5 100644 --- a/clang/lib/Driver/ToolChains/Hexagon.cpp +++ b/clang/lib/Driver/ToolChains/Hexagon.cpp @@ -72,23 +72,25 @@ static void handleHVXTargetFeatures(const Driver &D, const ArgList &Args, (Cpu.back() == 'T' || Cpu.back() == 't' ? Cpu.drop_back(1) : Cpu).str(); HasHVX = false; - // Handle -mhvx, -mhvx=, -mno-hvx. If both present, -mhvx= wins over -mhvx. - auto argOrNull = [&Args](auto FlagOn, auto FlagOff) -> Arg* { - if (Arg *A = Args.getLastArg(FlagOn, FlagOff)) { - if (A->getOption().matches(FlagOn)) - return A; + // Handle -mhvx, -mhvx=, -mno-hvx. If versioned and versionless flags + // are both present, the last one wins. + Arg *HvxEnablingArg = + Args.getLastArg(options::OPT_mhexagon_hvx, options::OPT_mhexagon_hvx_EQ, + options::OPT_mno_hexagon_hvx); + if (HvxEnablingArg) { + if (HvxEnablingArg->getOption().matches(options::OPT_mno_hexagon_hvx)) + HvxEnablingArg = nullptr; + } + + if (HvxEnablingArg) { + // If -mhvx[=] was given, it takes precedence. + if (Arg *A = Args.getLastArg(options::OPT_mhexagon_hvx, + options::OPT_mhexagon_hvx_EQ)) { + // If the version was given, set HvxVer. Otherwise HvxVer + // will remain equal to the CPU version. + if (A->getOption().matches(options::OPT_mhexagon_hvx_EQ)) + HvxVer = StringRef(A->getValue()).lower(); } - return nullptr; - }; - - Arg *HvxBareA = - argOrNull(options::OPT_mhexagon_hvx, options::OPT_mno_hexagon_hvx); - Arg *HvxVerA = - argOrNull(options::OPT_mhexagon_hvx_EQ, options::OPT_mno_hexagon_hvx); - - if (Arg *A = HvxVerA ? HvxVerA : HvxBareA) { - if (A->getOption().matches(options::OPT_mhexagon_hvx_EQ)) - HvxVer = StringRef(A->getValue()).lower(); // lower produces std:string HasHVX = true; Features.push_back(makeFeature(Twine("hvx") + HvxVer, true)); } else if (Arg *A = Args.getLastArg(options::OPT_mno_hexagon_hvx)) { diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp index 18cef288f018..9f4751167ac1 100644 --- a/clang/lib/Driver/ToolChains/MSVC.cpp +++ b/clang/lib/Driver/ToolChains/MSVC.cpp @@ -1393,8 +1393,8 @@ VersionTuple MSVCToolChain::computeMSVCVersion(const Driver *D, if (MSVT.empty() && Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions, IsWindowsMSVC)) { - // -fms-compatibility-version=19.14 is default, aka 2017, 15.7 - MSVT = VersionTuple(19, 14); + // -fms-compatibility-version=19.20 is default, aka 2019, 16.x + MSVT = VersionTuple(19, 20); } return MSVT; } diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp index 0501f9737404..ceeaa79bc202 100644 --- a/clang/lib/Driver/ToolChains/MinGW.cpp +++ b/clang/lib/Driver/ToolChains/MinGW.cpp @@ -86,7 +86,9 @@ void tools::MinGW::Linker::AddLibGCC(const ArgList &Args, CmdArgs.push_back("-lmoldname"); CmdArgs.push_back("-lmingwex"); for (auto Lib : Args.getAllArgValues(options::OPT_l)) - if (StringRef(Lib).startswith("msvcr") || StringRef(Lib).startswith("ucrt")) + if (StringRef(Lib).startswith("msvcr") || + StringRef(Lib).startswith("ucrt") || + StringRef(Lib).startswith("crtdll")) return; CmdArgs.push_back("-lmsvcrt"); } diff --git a/clang/lib/Driver/ToolChains/MinGW.h b/clang/lib/Driver/ToolChains/MinGW.h index c3de19b97724..c9553b4f4652 100644 --- a/clang/lib/Driver/ToolChains/MinGW.h +++ b/clang/lib/Driver/ToolChains/MinGW.h @@ -90,6 +90,8 @@ class LLVM_LIBRARY_VISIBILITY MinGW : public ToolChain { void printVerboseInfo(raw_ostream &OS) const override; + unsigned GetDefaultDwarfVersion() const override { return 4; } + protected: Tool *getTool(Action::ActionClass AC) const override; Tool *buildLinker() const override; diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp index 96abac57764f..bcd54bedfa89 100644 --- a/clang/lib/Driver/ToolChains/OpenBSD.cpp +++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp @@ -160,7 +160,8 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, assert(Output.isNothing() && "Invalid output."); } - if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles)) { + if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles, + options::OPT_r)) { const char *crt0 = nullptr; const char *crtbegin = nullptr; if (!Args.hasArg(options::OPT_shared)) { @@ -191,7 +192,8 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, bool NeedsXRayDeps = addXRayRuntime(ToolChain, Args, CmdArgs); AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA); - if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) { + if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs, + options::OPT_r)) { // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Args.hasArg(options::OPT_static); @@ -234,7 +236,8 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back("-lcompiler_rt"); } - if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles)) { + if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles, + options::OPT_r)) { const char *crtend = nullptr; if (!Args.hasArg(options::OPT_shared)) crtend = "crtend.o"; diff --git a/clang/lib/Format/DefinitionBlockSeparator.cpp b/clang/lib/Format/DefinitionBlockSeparator.cpp index 827564357f78..cfb019a471dc 100644 --- a/clang/lib/Format/DefinitionBlockSeparator.cpp +++ b/clang/lib/Format/DefinitionBlockSeparator.cpp @@ -35,19 +35,31 @@ void DefinitionBlockSeparator::separateBlocks( const bool IsNeverStyle = Style.SeparateDefinitionBlocks == FormatStyle::SDS_Never; const AdditionalKeywords &ExtraKeywords = Tokens.getKeywords(); - auto LikelyDefinition = [this, ExtraKeywords](const AnnotatedLine *Line, - bool ExcludeEnum = false) { + auto GetBracketLevelChange = [](const FormatToken *Tok) { + if (Tok->isOneOf(tok::l_brace, tok::l_paren, tok::l_square)) + return 1; + if (Tok->isOneOf(tok::r_brace, tok::r_paren, tok::r_square)) + return -1; + return 0; + }; + auto LikelyDefinition = [&](const AnnotatedLine *Line, + bool ExcludeEnum = false) { if ((Line->MightBeFunctionDecl && Line->mightBeFunctionDefinition()) || Line->startsWithNamespace()) return true; - FormatToken *CurrentToken = Line->First; - while (CurrentToken) { - if (CurrentToken->isOneOf(tok::kw_class, tok::kw_struct) || - (Style.isJavaScript() && CurrentToken->is(ExtraKeywords.kw_function))) - return true; - if (!ExcludeEnum && CurrentToken->is(tok::kw_enum)) - return true; - CurrentToken = CurrentToken->Next; + int BracketLevel = 0; + for (const FormatToken *CurrentToken = Line->First; CurrentToken; + CurrentToken = CurrentToken->Next) { + if (BracketLevel == 0) { + if ((CurrentToken->isOneOf(tok::kw_class, tok::kw_struct, + tok::kw_union) || + (Style.isJavaScript() && + CurrentToken->is(ExtraKeywords.kw_function)))) + return true; + if (!ExcludeEnum && CurrentToken->is(tok::kw_enum)) + return true; + } + BracketLevel += GetBracketLevelChange(CurrentToken); } return false; }; @@ -102,14 +114,17 @@ void DefinitionBlockSeparator::separateBlocks( IsPPConditional(OpeningLineIndex - 1); }; const auto HasEnumOnLine = [&]() { - FormatToken *CurrentToken = CurrentLine->First; bool FoundEnumKeyword = false; - while (CurrentToken) { - if (CurrentToken->is(tok::kw_enum)) - FoundEnumKeyword = true; - else if (FoundEnumKeyword && CurrentToken->is(tok::l_brace)) - return true; - CurrentToken = CurrentToken->Next; + int BracketLevel = 0; + for (const FormatToken *CurrentToken = CurrentLine->First; CurrentToken; + CurrentToken = CurrentToken->Next) { + if (BracketLevel == 0) { + if (CurrentToken->is(tok::kw_enum)) + FoundEnumKeyword = true; + else if (FoundEnumKeyword && CurrentToken->is(tok::l_brace)) + return true; + } + BracketLevel += GetBracketLevelChange(CurrentToken); } return FoundEnumKeyword && I + 1 < Lines.size() && Lines[I + 1]->First->is(tok::l_brace); diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp index b3a4684bead1..0142a6c08ed3 100644 --- a/clang/lib/Format/QualifierAlignmentFixer.cpp +++ b/clang/lib/Format/QualifierAlignmentFixer.cpp @@ -396,6 +396,10 @@ LeftRightQualifierAlignmentFixer::analyze( for (AnnotatedLine *Line : AnnotatedLines) { FormatToken *First = Line->First; + assert(First); + if (First->Finalized) + continue; + const auto *Last = Line->Last; for (const auto *Tok = First; Tok && Tok != Last && Tok->Next; diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index 642679128409..2e2293c6d58b 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -3102,7 +3102,8 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) { } if (FormatTok->is(tok::l_square)) { FormatToken *Previous = FormatTok->Previous; - if (!Previous || Previous->isNot(tok::r_paren)) { + if (!Previous || + !(Previous->is(tok::r_paren) || Previous->isTypeOrIdentifier())) { // Don't try parsing a lambda if we had a closing parenthesis before, // it was probably a pointer to an array: int (*)[]. if (!tryToParseLambda()) diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h index ad276dc0f6aa..5191c41bcd05 100644 --- a/clang/lib/Headers/opencl-c-base.h +++ b/clang/lib/Headers/opencl-c-base.h @@ -67,6 +67,8 @@ #if (__OPENCL_CPP_VERSION__ == 202100 || __OPENCL_C_VERSION__ == 300) // For the SPIR and SPIR-V target all features are supported. #if defined(__SPIR__) || defined(__SPIRV__) +#define __opencl_c_atomic_order_seq_cst 1 +#define __opencl_c_atomic_scope_device 1 #define __opencl_c_atomic_scope_all_devices 1 #define __opencl_c_read_write_images 1 #endif // defined(__SPIR__) diff --git a/clang/lib/Headers/opencl-c.h b/clang/lib/Headers/opencl-c.h index 059a2ec2371b..c7bb77716ac4 100644 --- a/clang/lib/Headers/opencl-c.h +++ b/clang/lib/Headers/opencl-c.h @@ -13832,6 +13832,7 @@ float __ovld atomic_fetch_max_explicit(volatile atomic_float *object, #endif // defined(__opencl_c_ext_fp32_global_atomic_min_max) && \ defined(__opencl_c_ext_fp32_local_atomic_min_max) +#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics) #if defined(__opencl_c_ext_fp64_global_atomic_min_max) double __ovld atomic_fetch_min(volatile __global atomic_double *object, double operand); @@ -13882,6 +13883,8 @@ double __ovld atomic_fetch_max_explicit(volatile atomic_double *object, memory_scope scope); #endif // defined(__opencl_c_ext_fp64_global_atomic_min_max) && \ defined(__opencl_c_ext_fp64_local_atomic_min_max) +#endif // defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics) #if defined(__opencl_c_ext_fp16_global_atomic_add) half __ovld atomic_fetch_add(volatile __global atomic_half *object, @@ -13985,6 +13988,7 @@ float __ovld atomic_fetch_sub_explicit(volatile atomic_float *object, #endif // defined(__opencl_c_ext_fp32_global_atomic_add) && \ defined(__opencl_c_ext_fp32_local_atomic_add) +#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics) #if defined(__opencl_c_ext_fp64_global_atomic_add) double __ovld atomic_fetch_add(volatile __global atomic_double *object, double operand); @@ -14035,6 +14039,8 @@ double __ovld atomic_fetch_sub_explicit(volatile atomic_double *object, memory_scope scope); #endif // defined(__opencl_c_ext_fp64_global_atomic_add) && \ defined(__opencl_c_ext_fp64_local_atomic_add) +#endif // defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics) #endif // cl_ext_float_atomics diff --git a/clang/lib/Headers/ppc_wrappers/mm_malloc.h b/clang/lib/Headers/ppc_wrappers/mm_malloc.h index 86cf1a0f7618..b6bf22f92887 100644 --- a/clang/lib/Headers/ppc_wrappers/mm_malloc.h +++ b/clang/lib/Headers/ppc_wrappers/mm_malloc.h @@ -19,7 +19,7 @@ #ifndef __cplusplus extern int posix_memalign (void **, size_t, size_t); #else -extern "C" int posix_memalign (void **, size_t, size_t) throw (); +extern "C" int posix_memalign (void **, size_t, size_t); #endif static __inline void * diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td index cd704ba2df13..ab3055300572 100644 --- a/clang/lib/Sema/OpenCLBuiltins.td +++ b/clang/lib/Sema/OpenCLBuiltins.td @@ -57,6 +57,23 @@ class FunctionExtension : AbstractExtension<_Ext>; // disabled. class TypeExtension : AbstractExtension<_Ext>; +// Concatenate zero or more space-separated extensions in NewExts to Base and +// return the resulting FunctionExtension in ret. +class concatExtension { + FunctionExtension ret = FunctionExtension< + !cond( + // Return Base extension if NewExts is empty, + !empty(NewExts) : Base.ExtName, + + // otherwise, return NewExts if Base extension is empty, + !empty(Base.ExtName) : NewExts, + + // otherwise, concatenate NewExts to Base. + true : Base.ExtName # " " # NewExts + ) + >; +} + // TypeExtension definitions. def NoTypeExt : TypeExtension<"">; def Fp16TypeExt : TypeExtension<"cl_khr_fp16">; @@ -90,27 +107,27 @@ def FuncExtOpenCLCNamedAddressSpaceBuiltins : FunctionExtension<"__opencl_c_name def FuncExtOpenCLCPipes : FunctionExtension<"__opencl_c_pipes">; def FuncExtOpenCLCWGCollectiveFunctions : FunctionExtension<"__opencl_c_work_group_collective_functions">; def FuncExtOpenCLCReadWriteImages : FunctionExtension<"__opencl_c_read_write_images">; -def FuncExtFloatAtomicsFp16GlobalLoadStore : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_load_store">; -def FuncExtFloatAtomicsFp16LocalLoadStore : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_load_store">; -def FuncExtFloatAtomicsFp16GenericLoadStore : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_load_store __opencl_c_ext_fp16_local_atomic_load_store">; -def FuncExtFloatAtomicsFp16GlobalAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_add">; -def FuncExtFloatAtomicsFp32GlobalAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_global_atomic_add">; -def FuncExtFloatAtomicsFp64GlobalAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_global_atomic_add">; -def FuncExtFloatAtomicsFp16LocalAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_add">; -def FuncExtFloatAtomicsFp32LocalAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_add">; -def FuncExtFloatAtomicsFp64LocalAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_add">; -def FuncExtFloatAtomicsFp16GenericAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_add __opencl_c_ext_fp16_global_atomic_add">; -def FuncExtFloatAtomicsFp32GenericAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_add __opencl_c_ext_fp32_global_atomic_add">; -def FuncExtFloatAtomicsFp64GenericAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_add __opencl_c_ext_fp64_global_atomic_add">; -def FuncExtFloatAtomicsFp16GlobalMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_min_max">; -def FuncExtFloatAtomicsFp32GlobalMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_global_atomic_min_max">; -def FuncExtFloatAtomicsFp64GlobalMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_global_atomic_min_max">; -def FuncExtFloatAtomicsFp16LocalMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_min_max">; -def FuncExtFloatAtomicsFp32LocalMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_min_max">; -def FuncExtFloatAtomicsFp64LocalMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_min_max">; -def FuncExtFloatAtomicsFp16GenericMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_min_max __opencl_c_ext_fp16_global_atomic_min_max">; -def FuncExtFloatAtomicsFp32GenericMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_min_max __opencl_c_ext_fp32_global_atomic_min_max">; -def FuncExtFloatAtomicsFp64GenericMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_min_max __opencl_c_ext_fp64_global_atomic_min_max">; +def FuncExtFloatAtomicsFp16GlobalASLoadStore : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_load_store">; +def FuncExtFloatAtomicsFp16LocalASLoadStore : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_load_store">; +def FuncExtFloatAtomicsFp16GenericASLoadStore : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_load_store __opencl_c_ext_fp16_local_atomic_load_store">; +def FuncExtFloatAtomicsFp16GlobalASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_add">; +def FuncExtFloatAtomicsFp32GlobalASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_global_atomic_add">; +def FuncExtFloatAtomicsFp64GlobalASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_global_atomic_add">; +def FuncExtFloatAtomicsFp16LocalASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_add">; +def FuncExtFloatAtomicsFp32LocalASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_add">; +def FuncExtFloatAtomicsFp64LocalASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_add">; +def FuncExtFloatAtomicsFp16GenericASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_add __opencl_c_ext_fp16_global_atomic_add">; +def FuncExtFloatAtomicsFp32GenericASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_add __opencl_c_ext_fp32_global_atomic_add">; +def FuncExtFloatAtomicsFp64GenericASAdd : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_add __opencl_c_ext_fp64_global_atomic_add">; +def FuncExtFloatAtomicsFp16GlobalASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_global_atomic_min_max">; +def FuncExtFloatAtomicsFp32GlobalASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_global_atomic_min_max">; +def FuncExtFloatAtomicsFp64GlobalASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_global_atomic_min_max">; +def FuncExtFloatAtomicsFp16LocalASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_min_max">; +def FuncExtFloatAtomicsFp32LocalASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_min_max">; +def FuncExtFloatAtomicsFp64LocalASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_min_max">; +def FuncExtFloatAtomicsFp16GenericASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp16_local_atomic_min_max __opencl_c_ext_fp16_global_atomic_min_max">; +def FuncExtFloatAtomicsFp32GenericASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp32_local_atomic_min_max __opencl_c_ext_fp32_global_atomic_min_max">; +def FuncExtFloatAtomicsFp64GenericASMinMax : FunctionExtension<"cl_ext_float_atomics __opencl_c_ext_fp64_local_atomic_min_max __opencl_c_ext_fp64_global_atomic_min_max">; // Not a real extension, but a workaround to add C++ for OpenCL specific builtins. def FuncExtOpenCLCxx : FunctionExtension<"__cplusplus">; @@ -1041,42 +1058,59 @@ let Extension = FuncExtOpenCLCxx in { } // OpenCL v2.0 s6.13.11 - Atomic Functions. -let MinVersion = CL20 in { - def : Builtin<"atomic_work_item_fence", [Void, MemFenceFlags, MemoryOrder, MemoryScope]>; +// An atomic builtin with 2 additional _explicit variants. +multiclass BuiltinAtomicExplicit Types, FunctionExtension BaseExt> { + // Without explicit MemoryOrder or MemoryScope. + let Extension = concatExtension.ret in { + def : Builtin; + } + + // With an explicit MemoryOrder argument. + let Extension = concatExtension.ret in { + def : Builtin; + } + + // With explicit MemoryOrder and MemoryScope arguments. + let Extension = BaseExt in { + def : Builtin; + } +} + +// OpenCL 2.0 atomic functions that have a pointer argument in a given address space. +multiclass OpenCL2Atomics { foreach TypePair = [[AtomicInt, Int], [AtomicUInt, UInt], [AtomicLong, Long], [AtomicULong, ULong], [AtomicFloat, Float], [AtomicDouble, Double]] in { def : Builtin<"atomic_init", - [Void, PointerType, GenericAS>, TypePair[1]]>; - def : Builtin<"atomic_store", - [Void, PointerType, GenericAS>, TypePair[1]]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, GenericAS>, TypePair[1], MemoryOrder]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, GenericAS>, TypePair[1], MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_load", - [TypePair[1], PointerType, GenericAS>]>; - def : Builtin<"atomic_load_explicit", - [TypePair[1], PointerType, GenericAS>, MemoryOrder]>; - def : Builtin<"atomic_load_explicit", - [TypePair[1], PointerType, GenericAS>, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_exchange", - [TypePair[1], PointerType, GenericAS>, TypePair[1]]>; - def : Builtin<"atomic_exchange_explicit", - [TypePair[1], PointerType, GenericAS>, TypePair[1], MemoryOrder]>; - def : Builtin<"atomic_exchange_explicit", - [TypePair[1], PointerType, GenericAS>, TypePair[1], MemoryOrder, MemoryScope]>; + [Void, PointerType, addrspace>, TypePair[1]]>; + defm : BuiltinAtomicExplicit<"atomic_store", + [Void, PointerType, addrspace>, TypePair[1]], BaseExt>; + defm : BuiltinAtomicExplicit<"atomic_load", + [TypePair[1], PointerType, addrspace>], BaseExt>; + defm : BuiltinAtomicExplicit<"atomic_exchange", + [TypePair[1], PointerType, addrspace>, TypePair[1]], BaseExt>; foreach Variant = ["weak", "strong"] in { - def : Builtin<"atomic_compare_exchange_" # Variant, - [Bool, PointerType, GenericAS>, - PointerType, TypePair[1]]>; - def : Builtin<"atomic_compare_exchange_" # Variant # "_explicit", - [Bool, PointerType, GenericAS>, - PointerType, TypePair[1], MemoryOrder, MemoryOrder]>; - def : Builtin<"atomic_compare_exchange_" # Variant # "_explicit", - [Bool, PointerType, GenericAS>, - PointerType, TypePair[1], MemoryOrder, MemoryOrder, MemoryScope]>; + foreach exp_ptr_addrspace = !cond( + !eq(BaseExt, FuncExtOpenCLCGenericAddressSpace): [GenericAS], + !eq(BaseExt, FuncExtOpenCLCNamedAddressSpaceBuiltins): [GlobalAS, LocalAS, PrivateAS]) + in { + let Extension = concatExtension.ret in { + def : Builtin<"atomic_compare_exchange_" # Variant, + [Bool, PointerType, addrspace>, + PointerType, TypePair[1]]>; + } + let Extension = concatExtension.ret in { + def : Builtin<"atomic_compare_exchange_" # Variant # "_explicit", + [Bool, PointerType, addrspace>, + PointerType, TypePair[1], MemoryOrder, MemoryOrder]>; + } + let Extension = BaseExt in { + def : Builtin<"atomic_compare_exchange_" # Variant # "_explicit", + [Bool, PointerType, addrspace>, + PointerType, TypePair[1], MemoryOrder, MemoryOrder, MemoryScope]>; + } + } } } @@ -1084,249 +1118,69 @@ let MinVersion = CL20 in { [AtomicLong, Long, Long], [AtomicULong, ULong, ULong], [AtomicUIntPtr, UIntPtr, PtrDiff]] in { foreach ModOp = ["add", "sub"] in { - def : Builtin<"atomic_fetch_" # ModOp, - [TypePair[1], PointerType, GenericAS>, TypePair[2]]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [TypePair[1], PointerType, GenericAS>, TypePair[2], MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [TypePair[1], PointerType, GenericAS>, TypePair[2], MemoryOrder, MemoryScope]>; + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [TypePair[1], PointerType, addrspace>, TypePair[2]], BaseExt>; } } foreach TypePair = [[AtomicInt, Int, Int], [AtomicUInt, UInt, UInt], [AtomicLong, Long, Long], [AtomicULong, ULong, ULong]] in { foreach ModOp = ["or", "xor", "and", "min", "max"] in { - def : Builtin<"atomic_fetch_" # ModOp, - [TypePair[1], PointerType, GenericAS>, TypePair[2]]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [TypePair[1], PointerType, GenericAS>, TypePair[2], MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [TypePair[1], PointerType, GenericAS>, TypePair[2], MemoryOrder, MemoryScope]>; + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [TypePair[1], PointerType, addrspace>, TypePair[2]], BaseExt>; } } - def : Builtin<"atomic_flag_clear", - [Void, PointerType, GenericAS>]>; - def : Builtin<"atomic_flag_clear_explicit", - [Void, PointerType, GenericAS>, MemoryOrder]>; - def : Builtin<"atomic_flag_clear_explicit", - [Void, PointerType, GenericAS>, MemoryOrder, MemoryScope]>; + defm : BuiltinAtomicExplicit<"atomic_flag_clear", + [Void, PointerType, addrspace>], BaseExt>; - def : Builtin<"atomic_flag_test_and_set", - [Bool, PointerType, GenericAS>]>; - def : Builtin<"atomic_flag_test_and_set_explicit", - [Bool, PointerType, GenericAS>, MemoryOrder]>; - def : Builtin<"atomic_flag_test_and_set_explicit", - [Bool, PointerType, GenericAS>, MemoryOrder, MemoryScope]>; + defm : BuiltinAtomicExplicit<"atomic_flag_test_and_set", + [Bool, PointerType, addrspace>], BaseExt>; +} + +let MinVersion = CL20 in { + def : Builtin<"atomic_work_item_fence", [Void, MemFenceFlags, MemoryOrder, MemoryScope]>; + + defm : OpenCL2Atomics; + defm : OpenCL2Atomics; + defm : OpenCL2Atomics; } // The functionality added by cl_ext_float_atomics extension let MinVersion = CL20 in { - let Extension = FuncExtFloatAtomicsFp16GlobalLoadStore in { - def : Builtin<"atomic_store", - [Void, PointerType, GlobalAS>, AtomicHalf]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, GlobalAS>, AtomicHalf, MemoryOrder]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, GlobalAS>, AtomicHalf, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_load", - [Half, PointerType, GlobalAS>]>; - def : Builtin<"atomic_load_explicit", - [Half, PointerType, GlobalAS>, MemoryOrder]>; - def : Builtin<"atomic_load_explicit", - [Half, PointerType, GlobalAS>, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_exchange", - [Half, PointerType, GlobalAS>, Half]>; - def : Builtin<"atomic_exchange_explicit", - [Half, PointerType, GlobalAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_exchange_explicit", - [Half, PointerType, GlobalAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp16LocalLoadStore in { - def : Builtin<"atomic_store", - [Void, PointerType, LocalAS>, AtomicHalf]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, LocalAS>, AtomicHalf, MemoryOrder]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, LocalAS>, AtomicHalf, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_load", - [Half, PointerType, LocalAS>]>; - def : Builtin<"atomic_load_explicit", - [Half, PointerType, LocalAS>, MemoryOrder]>; - def : Builtin<"atomic_load_explicit", - [Half, PointerType, LocalAS>, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_exchange", - [Half, PointerType, LocalAS>, Half]>; - def : Builtin<"atomic_exchange_explicit", - [Half, PointerType, LocalAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_exchange_explicit", - [Half, PointerType, LocalAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp16GenericLoadStore in { - def : Builtin<"atomic_store", - [Void, PointerType, GenericAS>, AtomicHalf]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, GenericAS>, AtomicHalf, MemoryOrder]>; - def : Builtin<"atomic_store_explicit", - [Void, PointerType, GenericAS>, AtomicHalf, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_load", - [Half, PointerType, GenericAS>]>; - def : Builtin<"atomic_load_explicit", - [Half, PointerType, GenericAS>, MemoryOrder]>; - def : Builtin<"atomic_load_explicit", - [Half, PointerType, GenericAS>, MemoryOrder, MemoryScope]>; - def : Builtin<"atomic_exchange", - [Half, PointerType, GenericAS>, Half]>; - def : Builtin<"atomic_exchange_explicit", - [Half, PointerType, GenericAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_exchange_explicit", - [Half, PointerType, GenericAS>, Half, MemoryOrder, MemoryScope]>; - } - foreach ModOp = ["add", "sub"] in { - let Extension = FuncExtFloatAtomicsFp16GlobalAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Half, PointerType, GlobalAS>, Half]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GlobalAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GlobalAS>, Half, MemoryOrder, MemoryScope]>; + foreach addrspace = [GlobalAS, LocalAS, GenericAS] in { + defvar extension_fp16 = !cast("FuncExtFloatAtomicsFp16" # addrspace # "LoadStore"); + + defm : BuiltinAtomicExplicit<"atomic_store", + [Void, PointerType, addrspace>, AtomicHalf], extension_fp16>; + defm : BuiltinAtomicExplicit<"atomic_load", + [Half, PointerType, addrspace>], extension_fp16>; + defm : BuiltinAtomicExplicit<"atomic_exchange", + [Half, PointerType, addrspace>, Half], extension_fp16>; + + foreach ModOp = ["add", "sub"] in { + defvar extension_fp16 = !cast("FuncExtFloatAtomicsFp16" # addrspace # "Add"); + defvar extension_fp32 = !cast("FuncExtFloatAtomicsFp32" # addrspace # "Add"); + defvar extension_fp64 = !cast("FuncExtFloatAtomicsFp64" # addrspace # "Add"); + + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [Half, PointerType, addrspace>, Half], extension_fp16>; + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [Float, PointerType, addrspace>, Float], extension_fp32>; + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [Double, PointerType, addrspace>, Double], extension_fp64>; } - let Extension = FuncExtFloatAtomicsFp32GlobalAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Float, PointerType, GlobalAS>, Float]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GlobalAS>, Float, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GlobalAS>, Float, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp64GlobalAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Double, PointerType, GlobalAS>, Double]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GlobalAS>, Double, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GlobalAS>, Double, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp16LocalAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Half, PointerType, LocalAS>, Half]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, LocalAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, LocalAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp32LocalAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Float, PointerType, LocalAS>, Float]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, LocalAS>, Float, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, LocalAS>, Float, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp64LocalAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Double, PointerType, LocalAS>, Double]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, LocalAS>, Double, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, LocalAS>, Double, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp16GenericAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Half, PointerType, GenericAS>, Half]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GenericAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GenericAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp32GenericAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Float, PointerType, GenericAS>, Float]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GenericAS>, Float, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GenericAS>, Float, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp64GenericAdd in { - def : Builtin<"atomic_fetch_" # ModOp, - [Double, PointerType, GenericAS>, Double]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GenericAS>, Double, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GenericAS>, Double, MemoryOrder, MemoryScope]>; - } - } - foreach ModOp = ["min", "max"] in { - let Extension = FuncExtFloatAtomicsFp16GlobalMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Half, PointerType, GlobalAS>, Half]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GlobalAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GlobalAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp32GlobalMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Float, PointerType, GlobalAS>, Float]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GlobalAS>, Float, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GlobalAS>, Float, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp64GlobalMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Double, PointerType, GlobalAS>, Double]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GlobalAS>, Double, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GlobalAS>, Double, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp16LocalMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Half, PointerType, LocalAS>, Half]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, LocalAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, LocalAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp32LocalMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Float, PointerType, LocalAS>, Float]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, LocalAS>, Float, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, LocalAS>, Float, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp64LocalMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Double, PointerType, LocalAS>, Double]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, LocalAS>, Double, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, LocalAS>, Double, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp16GenericMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Half, PointerType, GenericAS>, Half]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GenericAS>, Half, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Half, PointerType, GenericAS>, Half, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp32GenericMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Float, PointerType, GenericAS>, Float]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GenericAS>, Float, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Float, PointerType, GenericAS>, Float, MemoryOrder, MemoryScope]>; - } - let Extension = FuncExtFloatAtomicsFp64GenericMinMax in { - def : Builtin<"atomic_fetch_" # ModOp, - [Double, PointerType, GenericAS>, Double]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GenericAS>, Double, MemoryOrder]>; - def : Builtin<"atomic_fetch_" # ModOp # "_explicit", - [Double, PointerType, GenericAS>, Double, MemoryOrder, MemoryScope]>; + + foreach ModOp = ["min", "max"] in { + defvar extension_fp16 = !cast("FuncExtFloatAtomicsFp16" # addrspace # "MinMax"); + defvar extension_fp32 = !cast("FuncExtFloatAtomicsFp32" # addrspace # "MinMax"); + defvar extension_fp64 = !cast("FuncExtFloatAtomicsFp64" # addrspace # "MinMax"); + + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [Half, PointerType, addrspace>, Half], extension_fp16>; + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [Float, PointerType, addrspace>, Float], extension_fp32>; + defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp, + [Double, PointerType, addrspace>, Double], extension_fp64>; } } } diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index dfbf4cdc89cb..69dcc3aaaaf3 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -1042,9 +1042,15 @@ static bool checkOpenCLBlockArgs(Sema &S, Expr *BlockArg) { } static bool checkOpenCLSubgroupExt(Sema &S, CallExpr *Call) { - if (!S.getOpenCLOptions().isSupported("cl_khr_subgroups", S.getLangOpts())) { + // OpenCL device can support extension but not the feature as extension + // requires subgroup independent forward progress, but subgroup independent + // forward progress is optional in OpenCL C 3.0 __opencl_c_subgroups feature. + if (!S.getOpenCLOptions().isSupported("cl_khr_subgroups", S.getLangOpts()) && + !S.getOpenCLOptions().isSupported("__opencl_c_subgroups", + S.getLangOpts())) { S.Diag(Call->getBeginLoc(), diag::err_opencl_requires_extension) - << 1 << Call->getDirectCallee() << "cl_khr_subgroups"; + << 1 << Call->getDirectCallee() + << "cl_khr_subgroups or __opencl_c_subgroups"; return true; } return false; diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index cbd9df4d6a7b..bcadf4139046 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -15319,24 +15319,7 @@ void Sema::AddKnownFunctionAttributes(FunctionDecl *FD) { if (!FD->hasAttr()) FD->addAttr(AllocAlignAttr::CreateImplicit(Context, ParamIdx(1, FD), FD->getLocation())); - LLVM_FALLTHROUGH; - case Builtin::BIcalloc: - case Builtin::BImalloc: - case Builtin::BImemalign: - case Builtin::BIrealloc: - case Builtin::BIstrdup: - case Builtin::BIstrndup: { - if (!FD->hasAttr()) { - unsigned NewAlign = Context.getTargetInfo().getNewAlign() / - Context.getTargetInfo().getCharWidth(); - IntegerLiteral *Alignment = IntegerLiteral::Create( - Context, Context.MakeIntValue(NewAlign, Context.UnsignedIntTy), - Context.UnsignedIntTy, FD->getLocation()); - FD->addAttr(AssumeAlignedAttr::CreateImplicit( - Context, Alignment, /*Offset=*/nullptr, FD->getLocation())); - } break; - } default: break; } diff --git a/compiler-rt/lib/asan/asan_errors.h b/compiler-rt/lib/asan/asan_errors.h index a7fda2fd9f5d..af6d1f295eb2 100644 --- a/compiler-rt/lib/asan/asan_errors.h +++ b/compiler-rt/lib/asan/asan_errors.h @@ -53,9 +53,9 @@ struct ErrorDeadlySignal : ErrorBase { scariness.Scare(10, "null-deref"); } else if (signal.addr == signal.pc) { scariness.Scare(60, "wild-jump"); - } else if (signal.write_flag == SignalContext::WRITE) { + } else if (signal.write_flag == SignalContext::Write) { scariness.Scare(30, "wild-addr-write"); - } else if (signal.write_flag == SignalContext::READ) { + } else if (signal.write_flag == SignalContext::Read) { scariness.Scare(20, "wild-addr-read"); } else { scariness.Scare(25, "wild-addr"); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 139d5a066664..3302590c6771 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -956,7 +956,7 @@ struct SignalContext { uptr sp; uptr bp; bool is_memory_access; - enum WriteFlag { UNKNOWN, READ, WRITE } write_flag; + enum WriteFlag { Unknown, Read, Write } write_flag; // In some cases the kernel cannot provide the true faulting address; `addr` // will be zero then. This field allows to distinguish between these cases diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp index 29ff7c041729..2e4d57d87f58 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp @@ -1825,7 +1825,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #else uptr err = ucontext->uc_mcontext.gregs[REG_ERR]; #endif // SANITIZER_FREEBSD - return err & PF_WRITE ? WRITE : READ; + return err & PF_WRITE ? Write : Read; #elif defined(__mips__) uint32_t *exception_source; uint32_t faulty_instruction; @@ -1848,7 +1848,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { case 0x2a: // swl case 0x2e: // swr #endif - return SignalContext::WRITE; + return SignalContext::Write; case 0x20: // lb case 0x24: // lbu @@ -1863,27 +1863,27 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { case 0x22: // lwl case 0x26: // lwr #endif - return SignalContext::READ; + return SignalContext::Read; #if __mips_isa_rev == 6 case 0x3b: // pcrel op_code = (faulty_instruction >> 19) & 0x3; switch (op_code) { case 0x1: // lwpc case 0x2: // lwupc - return SignalContext::READ; + return SignalContext::Read; } #endif } - return SignalContext::UNKNOWN; + return SignalContext::Unknown; #elif defined(__arm__) static const uptr FSR_WRITE = 1U << 11; uptr fsr = ucontext->uc_mcontext.error_code; - return fsr & FSR_WRITE ? WRITE : READ; + return fsr & FSR_WRITE ? Write : Read; #elif defined(__aarch64__) static const u64 ESR_ELx_WNR = 1U << 6; u64 esr; - if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN; - return esr & ESR_ELx_WNR ? WRITE : READ; + if (!Aarch64GetESR(ucontext, &esr)) return Unknown; + return esr & ESR_ELx_WNR ? Write : Read; #elif defined(__sparc__) // Decode the instruction to determine the access type. // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype). @@ -1899,7 +1899,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #endif #endif u32 instr = *(u32 *)pc; - return (instr >> 21) & 1 ? WRITE: READ; + return (instr >> 21) & 1 ? Write: Read; #elif defined(__riscv) #if SANITIZER_FREEBSD unsigned long pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc; @@ -1919,7 +1919,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if __riscv_xlen == 64 case 0b10'011: // c.ldsp (rd != x0) #endif - return rd ? SignalContext::READ : SignalContext::UNKNOWN; + return rd ? SignalContext::Read : SignalContext::Unknown; case 0b00'010: // c.lw #if __riscv_flen >= 32 && __riscv_xlen == 32 case 0b10'011: // c.flwsp @@ -1931,7 +1931,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { case 0b00'001: // c.fld case 0b10'001: // c.fldsp #endif - return SignalContext::READ; + return SignalContext::Read; case 0b00'110: // c.sw case 0b10'110: // c.swsp #if __riscv_flen >= 32 || __riscv_xlen == 64 @@ -1942,9 +1942,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { case 0b00'101: // c.fsd case 0b10'101: // c.fsdsp #endif - return SignalContext::WRITE; + return SignalContext::Write; default: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } } #endif @@ -1962,9 +1962,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #endif case 0b100: // lbu case 0b101: // lhu - return SignalContext::READ; + return SignalContext::Read; default: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } case 0b0100011: // stores switch (funct3) { @@ -1974,9 +1974,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if __riscv_xlen == 64 case 0b011: // sd #endif - return SignalContext::WRITE; + return SignalContext::Write; default: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } #if __riscv_flen >= 32 case 0b0000111: // floating-point loads @@ -1985,9 +1985,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if __riscv_flen == 64 case 0b011: // fld #endif - return SignalContext::READ; + return SignalContext::Read; default: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } case 0b0100111: // floating-point stores switch (funct3) { @@ -1995,17 +1995,17 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if __riscv_flen == 64 case 0b011: // fsd #endif - return SignalContext::WRITE; + return SignalContext::Write; default: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } #endif default: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } #else (void)ucontext; - return UNKNOWN; // FIXME: Implement. + return Unknown; // FIXME: Implement. #endif } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp index b025a5e4fb64..4ccd2e828108 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -216,14 +216,12 @@ void InitTlsSize() { } // On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage // of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan // to get the pointer to thread-specific data keys in the thread control block. -#if (SANITIZER_FREEBSD || SANITIZER_LINUX) && !SANITIZER_ANDROID +#if (SANITIZER_FREEBSD || SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO // sizeof(struct pthread) from glibc. static atomic_uintptr_t thread_descriptor_size; -uptr ThreadDescriptorSize() { - uptr val = atomic_load_relaxed(&thread_descriptor_size); - if (val) - return val; +static uptr ThreadDescriptorSizeFallback() { + uptr val = 0; #if defined(__x86_64__) || defined(__i386__) || defined(__arm__) int major; int minor; @@ -285,8 +283,21 @@ uptr ThreadDescriptorSize() { #elif defined(__powerpc64__) val = 1776; // from glibc.ppc64le 2.20-8.fc21 #endif + return val; +} + +uptr ThreadDescriptorSize() { + uptr val = atomic_load_relaxed(&thread_descriptor_size); if (val) - atomic_store_relaxed(&thread_descriptor_size, val); + return val; + // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in + // glibc 2.34 and later. + if (unsigned *psizeof = static_cast( + dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) + val = *psizeof; + if (!val) + val = ThreadDescriptorSizeFallback(); + atomic_store_relaxed(&thread_descriptor_size, val); return val; } @@ -308,7 +319,6 @@ static uptr TlsPreTcbSize() { } #endif -#if !SANITIZER_GO namespace { struct TlsBlock { uptr begin, end, align; @@ -396,9 +406,8 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size, *addr = ranges[l].begin; *size = ranges[r - 1].end - ranges[l].begin; } -#endif // !SANITIZER_GO #endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD || - // SANITIZER_LINUX) && !SANITIZER_ANDROID + // SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO #if SANITIZER_NETBSD static struct tls_tcb * ThreadSelfTlsTcb() { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp index a2fc310ad1a2..3b20e1c5e2af 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp @@ -871,9 +871,9 @@ void LogFullErrorReport(const char *buffer) { SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if defined(__x86_64__) || defined(__i386__) ucontext_t *ucontext = static_cast(context); - return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ; + return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? Write : Read; #else - return UNKNOWN; + return Unknown; #endif } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index 869c8935330d..ac855c8be1c8 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -211,9 +211,9 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid, Report("Hint: pc points to the zero page.\n"); if (sig.is_memory_access) { const char *access_type = - sig.write_flag == SignalContext::WRITE + sig.write_flag == SignalContext::Write ? "WRITE" - : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN"); + : (sig.write_flag == SignalContext::Read ? "READ" : "UNKNOWN"); Report("The signal is caused by a %s memory access.\n", access_type); if (!sig.is_true_faulting_addr) Report("Hint: this fault was caused by a dereference of a high value " diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp index 87758a4904ab..7c84cdc22ce4 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp @@ -983,7 +983,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { // The write flag is only available for access violation exceptions. if (exception_record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) - return SignalContext::UNKNOWN; + return SignalContext::Unknown; // The contents of this array are documented at // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record @@ -991,13 +991,13 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { // second element is the faulting address. switch (exception_record->ExceptionInformation[0]) { case 0: - return SignalContext::READ; + return SignalContext::Read; case 1: - return SignalContext::WRITE; + return SignalContext::Write; case 8: - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } - return SignalContext::UNKNOWN; + return SignalContext::Unknown; } void SignalContext::DumpAllRegisters(void *context) { diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index 88a54b554421..3091ad809c40 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -78,6 +78,14 @@ inline bool MustIgnoreInterceptor(ThreadState *thr) { #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) +#if SANITIZER_FREEBSD +# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, _pthread_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +#else +# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) +#endif + #if SANITIZER_NETBSD # define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \ TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \ diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 733ae5ee87ef..ea99c3843075 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -289,20 +289,25 @@ void ScopedInterceptor::DisableIgnoresImpl() { } #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) -#if SANITIZER_FREEBSD -# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) -# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) -# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) -#elif SANITIZER_NETBSD -# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) -# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ - INTERCEPT_FUNCTION(__libc_##func) -# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ - INTERCEPT_FUNCTION(__libc_thr_##func) +#if SANITIZER_FREEBSD || SANITIZER_NETBSD +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) #else -# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) -# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) -# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) +#endif +#if SANITIZER_FREEBSD +# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \ + INTERCEPT_FUNCTION(_pthread_##func) +#else +# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) +#endif +#if SANITIZER_NETBSD +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ + INTERCEPT_FUNCTION(__libc_##func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ + INTERCEPT_FUNCTION(__libc_thr_##func) +#else +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) #endif #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ @@ -1549,16 +1554,16 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD - SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); - if (fd > 0) - FdAccess(thr, pc, fd); - return REAL(fstat)(fd, buf); -#else +#if SANITIZER_GLIBC SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat)(0, fd, buf); +#else + SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); + if (fd > 0) + FdAccess(thr, pc, fd); + return REAL(fstat)(fd, buf); #endif } @@ -2713,6 +2718,26 @@ TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) { #define TSAN_MAYBE_INTERCEPT_THR_EXIT #endif +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void *c, void *a) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void *c, void *m) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void *m, void *a) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void *l, void *a) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void *o, void (*i)()) +TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void *n, void *o) + TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c) @@ -2941,6 +2966,26 @@ void InitializeInterceptors() { } #endif + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once); + TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); diff --git a/libcxx/include/__algorithm/in_in_out_result.h b/libcxx/include/__algorithm/in_in_out_result.h index a492d2735229..8d29b7b69b47 100644 --- a/libcxx/include/__algorithm/in_in_out_result.h +++ b/libcxx/include/__algorithm/in_in_out_result.h @@ -20,7 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { @@ -47,7 +47,7 @@ struct in_in_out_result { } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/in_out_result.h b/libcxx/include/__algorithm/in_out_result.h index d3c16e4acd45..dcf72d08dd8a 100644 --- a/libcxx/include/__algorithm/in_out_result.h +++ b/libcxx/include/__algorithm/in_out_result.h @@ -20,7 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { @@ -47,7 +47,7 @@ struct in_out_result { } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__config b/libcxx/include/__config index d2d70c4ed769..458d0c1b897a 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -116,6 +116,8 @@ # define _LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT // Remove basic_string common base # define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON +// Remove vector base class +# define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support diff --git a/libcxx/include/__filesystem/directory_iterator.h b/libcxx/include/__filesystem/directory_iterator.h index 1cb1794efdae..cfaf2064bead 100644 --- a/libcxx/include/__filesystem/directory_iterator.h +++ b/libcxx/include/__filesystem/directory_iterator.h @@ -143,7 +143,7 @@ template <> _LIBCPP_AVAILABILITY_FILESYSTEM inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::directory_iterator> = true; -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) #endif // _LIBCPP_CXX03_LANG diff --git a/libcxx/include/__filesystem/recursive_directory_iterator.h b/libcxx/include/__filesystem/recursive_directory_iterator.h index c26d0a381159..b80d59750ed0 100644 --- a/libcxx/include/__filesystem/recursive_directory_iterator.h +++ b/libcxx/include/__filesystem/recursive_directory_iterator.h @@ -174,7 +174,7 @@ template <> _LIBCPP_AVAILABILITY_FILESYSTEM inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::recursive_directory_iterator> = true; -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) #endif // _LIBCPP_CXX03_LANG diff --git a/libcxx/include/__functional/ranges_operations.h b/libcxx/include/__functional/ranges_operations.h index 8b06240e46a7..a0ea7ccaffef 100644 --- a/libcxx/include/__functional/ranges_operations.h +++ b/libcxx/include/__functional/ranges_operations.h @@ -20,7 +20,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) + namespace ranges { struct equal_to { @@ -90,7 +91,7 @@ struct greater_equal { }; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/advance.h b/libcxx/include/__iterator/advance.h index 5b0e97d76be9..d74d6b8b929e 100644 --- a/libcxx/include/__iterator/advance.h +++ b/libcxx/include/__iterator/advance.h @@ -64,7 +64,7 @@ void advance(_InputIter& __i, _Distance __orig_n) { _VSTD::__advance(__i, __n, typename iterator_traits<_InputIter>::iterator_category()); } -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // [range.iter.op.advance] @@ -192,7 +192,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/distance.h b/libcxx/include/__iterator/distance.h index 85309f8d47d6..50ed76a4e53a 100644 --- a/libcxx/include/__iterator/distance.h +++ b/libcxx/include/__iterator/distance.h @@ -52,7 +52,7 @@ distance(_InputIter __first, _InputIter __last) return _VSTD::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category()); } -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // [range.iter.op.distance] @@ -100,7 +100,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/insert_iterator.h b/libcxx/include/__iterator/insert_iterator.h index d3cd5ad6f9c5..dc617c1c93d7 100644 --- a/libcxx/include/__iterator/insert_iterator.h +++ b/libcxx/include/__iterator/insert_iterator.h @@ -24,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template using __insert_iterator_iter_t = ranges::iterator_t<_Container>; #else diff --git a/libcxx/include/__iterator/iter_move.h b/libcxx/include/__iterator/iter_move.h index dfcf8e6c8308..45d9ade68227 100644 --- a/libcxx/include/__iterator/iter_move.h +++ b/libcxx/include/__iterator/iter_move.h @@ -86,7 +86,7 @@ template<__dereferenceable _Tp> requires requires(_Tp& __t) { { ranges::iter_move(__t) } -> __referenceable; } using iter_rvalue_reference_t = decltype(ranges::iter_move(declval<_Tp&>())); -#endif // !_LIBCPP_HAS_NO_CONCEPTS +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/iter_swap.h b/libcxx/include/__iterator/iter_swap.h index 0179546667b7..0b290520b702 100644 --- a/libcxx/include/__iterator/iter_swap.h +++ b/libcxx/include/__iterator/iter_swap.h @@ -14,7 +14,6 @@ #include <__iterator/iter_move.h> #include <__iterator/iterator_traits.h> #include <__iterator/readable_traits.h> -#include <__ranges/access.h> #include <__utility/forward.h> #include <__utility/move.h> #include diff --git a/libcxx/include/__iterator/next.h b/libcxx/include/__iterator/next.h index 8683e2210e95..5dda0674d57e 100644 --- a/libcxx/include/__iterator/next.h +++ b/libcxx/include/__iterator/next.h @@ -35,7 +35,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 return __x; } -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // [range.iter.op.next] @@ -79,7 +79,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/prev.h b/libcxx/include/__iterator/prev.h index 5a58dc0e4b43..2b8c15fe5726 100644 --- a/libcxx/include/__iterator/prev.h +++ b/libcxx/include/__iterator/prev.h @@ -34,7 +34,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 return __x; } -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // [range.iter.op.prev] @@ -71,7 +71,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__memory/concepts.h b/libcxx/include/__memory/concepts.h index dced563f38b6..eabc245c915e 100644 --- a/libcxx/include/__memory/concepts.h +++ b/libcxx/include/__memory/concepts.h @@ -25,7 +25,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) + namespace ranges { // [special.mem.concepts] @@ -59,7 +60,8 @@ concept __nothrow_forward_range = __nothrow_forward_iterator>; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) + +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__memory/ranges_construct_at.h b/libcxx/include/__memory/ranges_construct_at.h index b1821c8cc29e..d26f726836e1 100644 --- a/libcxx/include/__memory/ranges_construct_at.h +++ b/libcxx/include/__memory/ranges_construct_at.h @@ -29,7 +29,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { // construct_at @@ -117,7 +117,7 @@ inline namespace __cpo { } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__memory/ranges_uninitialized_algorithms.h b/libcxx/include/__memory/ranges_uninitialized_algorithms.h index 70d901a5ad03..4a6123ec6946 100644 --- a/libcxx/include/__memory/ranges_uninitialized_algorithms.h +++ b/libcxx/include/__memory/ranges_uninitialized_algorithms.h @@ -32,7 +32,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) + namespace ranges { // uninitialized_default_construct @@ -309,7 +310,8 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) + +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/all.h b/libcxx/include/__ranges/all.h index b0a58e9bad82..acdf30a503a8 100644 --- a/libcxx/include/__ranges/all.h +++ b/libcxx/include/__ranges/all.h @@ -28,7 +28,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges::views { @@ -75,7 +75,7 @@ using all_t = decltype(views::all(declval<_Range>())); } // namespace ranges::views -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/common_view.h b/libcxx/include/__ranges/common_view.h index fb8078d65558..3be49e8dbe71 100644 --- a/libcxx/include/__ranges/common_view.h +++ b/libcxx/include/__ranges/common_view.h @@ -30,7 +30,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { @@ -128,7 +128,7 @@ inline namespace __cpo { } // namespace views } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/concepts.h b/libcxx/include/__ranges/concepts.h index 0590863138ed..5f1fa834d409 100644 --- a/libcxx/include/__ranges/concepts.h +++ b/libcxx/include/__ranges/concepts.h @@ -35,13 +35,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_CONCEPTS) namespace ranges { + // [range.range] + template concept range = requires(_Tp& __t) { ranges::begin(__t); // sometimes equality-preserving ranges::end(__t); }; + template + concept input_range = range<_Tp> && input_iterator>; + template concept borrowed_range = range<_Range> && (is_lvalue_reference_v<_Range> || enable_borrowed_range>); @@ -63,6 +68,8 @@ namespace ranges { template using range_rvalue_reference_t = iter_rvalue_reference_t>; +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) + // [range.sized] template concept sized_range = range<_Tp> && requires(_Tp& __t) { ranges::size(__t); }; @@ -93,9 +100,6 @@ namespace ranges { template concept output_range = range<_Rp> && output_iterator, _Tp>; - template - concept input_range = range<_Tp> && input_iterator>; - template concept forward_range = input_range<_Tp> && forward_iterator>; @@ -131,6 +135,8 @@ namespace ranges { (is_lvalue_reference_v<_Tp> || (movable> && !__is_std_initializer_list>)))); +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) + } // namespace ranges #endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) diff --git a/libcxx/include/__ranges/copyable_box.h b/libcxx/include/__ranges/copyable_box.h index 608db55dbc5f..77e47b1216b4 100644 --- a/libcxx/include/__ranges/copyable_box.h +++ b/libcxx/include/__ranges/copyable_box.h @@ -24,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // __copyable_box allows turning a type that is copy-constructible (but maybe not copy-assignable) into // a type that is both copy-constructible and copy-assignable. It does that by introducing an empty state @@ -171,7 +171,7 @@ namespace ranges { }; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/counted.h b/libcxx/include/__ranges/counted.h index 92bcf06be3ad..2eb44035cbb2 100644 --- a/libcxx/include/__ranges/counted.h +++ b/libcxx/include/__ranges/counted.h @@ -29,7 +29,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges::views { @@ -74,7 +74,7 @@ inline namespace __cpo { } // namespace ranges::views -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/dangling.h b/libcxx/include/__ranges/dangling.h index b6b733a6eaa9..ecc7c093c92b 100644 --- a/libcxx/include/__ranges/dangling.h +++ b/libcxx/include/__ranges/dangling.h @@ -35,7 +35,7 @@ using borrowed_iterator_t = _If, iterator_t<_Rp>, dangling>; // borrowed_subrange_t defined in <__ranges/subrange.h> } // namespace ranges -#endif // !_LIBCPP_HAS_NO_CONCEPTS +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/data.h b/libcxx/include/__ranges/data.h index f97ec8033297..f8d92cbc7520 100644 --- a/libcxx/include/__ranges/data.h +++ b/libcxx/include/__ranges/data.h @@ -24,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // [range.prim.data] @@ -99,7 +99,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/drop_view.h b/libcxx/include/__ranges/drop_view.h index 1e2bfa40b325..8ff895863e2a 100644 --- a/libcxx/include/__ranges/drop_view.h +++ b/libcxx/include/__ranges/drop_view.h @@ -31,7 +31,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -120,7 +120,7 @@ namespace ranges { inline constexpr bool enable_borrowed_range> = enable_borrowed_range<_Tp>; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/empty.h b/libcxx/include/__ranges/empty.h index b06a81c98e8f..1be55c192f60 100644 --- a/libcxx/include/__ranges/empty.h +++ b/libcxx/include/__ranges/empty.h @@ -22,7 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // [range.prim.empty] @@ -75,7 +75,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/empty_view.h b/libcxx/include/__ranges/empty_view.h index e0116b933797..b5509098da27 100644 --- a/libcxx/include/__ranges/empty_view.h +++ b/libcxx/include/__ranges/empty_view.h @@ -20,7 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -38,7 +38,7 @@ namespace ranges { inline constexpr bool enable_borrowed_range> = true; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/enable_borrowed_range.h b/libcxx/include/__ranges/enable_borrowed_range.h index f9985dfd0061..5523285580ea 100644 --- a/libcxx/include/__ranges/enable_borrowed_range.h +++ b/libcxx/include/__ranges/enable_borrowed_range.h @@ -24,8 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_CONCEPTS) -namespace ranges -{ +namespace ranges { // [range.range], ranges diff --git a/libcxx/include/__ranges/enable_view.h b/libcxx/include/__ranges/enable_view.h index 87d53f3fcc8c..837d44908bf5 100644 --- a/libcxx/include/__ranges/enable_view.h +++ b/libcxx/include/__ranges/enable_view.h @@ -40,7 +40,7 @@ inline constexpr bool enable_view = derived_from<_Tp, view_base> || } // namespace ranges -#endif // !_LIBCPP_HAS_NO_CONCEPTS +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/iota_view.h b/libcxx/include/__ranges/iota_view.h index 17f6021f7dd4..b74f57d7cd68 100644 --- a/libcxx/include/__ranges/iota_view.h +++ b/libcxx/include/__ranges/iota_view.h @@ -39,7 +39,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -401,7 +401,7 @@ inline namespace __cpo { } // namespace views } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/join_view.h b/libcxx/include/__ranges/join_view.h index 7efbcfdf8788..26674b30f610 100644 --- a/libcxx/include/__ranges/join_view.h +++ b/libcxx/include/__ranges/join_view.h @@ -30,7 +30,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -343,7 +343,7 @@ namespace ranges { #undef _CONSTEXPR_TERNARY -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/non_propagating_cache.h b/libcxx/include/__ranges/non_propagating_cache.h index 2d3a9408713e..0e464e169088 100644 --- a/libcxx/include/__ranges/non_propagating_cache.h +++ b/libcxx/include/__ranges/non_propagating_cache.h @@ -24,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { // __non_propagating_cache is a helper type that allows storing an optional value in it, @@ -107,7 +107,7 @@ namespace ranges { struct __empty_cache { }; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/owning_view.h b/libcxx/include/__ranges/owning_view.h index 9c038cbd7580..7206f813b1dd 100644 --- a/libcxx/include/__ranges/owning_view.h +++ b/libcxx/include/__ranges/owning_view.h @@ -28,7 +28,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -74,7 +74,7 @@ namespace ranges { } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/range_adaptor.h b/libcxx/include/__ranges/range_adaptor.h index d037585d2679..fbe3b888668a 100644 --- a/libcxx/include/__ranges/range_adaptor.h +++ b/libcxx/include/__ranges/range_adaptor.h @@ -25,7 +25,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) // CRTP base that one can derive from in order to be considered a range adaptor closure // by the library. When deriving from this class, a pipe operator will be provided to @@ -66,7 +66,7 @@ struct __range_adaptor_closure { { return __range_adaptor_closure_t(_VSTD::__compose(_VSTD::forward<_OtherClosure>(__c2), _VSTD::forward<_Closure>(__c1))); } }; -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/ref_view.h b/libcxx/include/__ranges/ref_view.h index 4d12759e614f..8777b71ac5e2 100644 --- a/libcxx/include/__ranges/ref_view.h +++ b/libcxx/include/__ranges/ref_view.h @@ -31,7 +31,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -79,7 +79,7 @@ namespace ranges { inline constexpr bool enable_borrowed_range> = true; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/reverse_view.h b/libcxx/include/__ranges/reverse_view.h index 04ac7f29aa7c..a2519e3cb1a6 100644 --- a/libcxx/include/__ranges/reverse_view.h +++ b/libcxx/include/__ranges/reverse_view.h @@ -33,7 +33,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -183,7 +183,7 @@ namespace ranges { } // namespace views } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/single_view.h b/libcxx/include/__ranges/single_view.h index c6f0746ea795..bf5a5e2700ef 100644 --- a/libcxx/include/__ranges/single_view.h +++ b/libcxx/include/__ranges/single_view.h @@ -24,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -74,7 +74,7 @@ namespace ranges { single_view(_Tp) -> single_view<_Tp>; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/size.h b/libcxx/include/__ranges/size.h index e1aaf7eba898..2b71c03fb399 100644 --- a/libcxx/include/__ranges/size.h +++ b/libcxx/include/__ranges/size.h @@ -24,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -128,7 +128,7 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/subrange.h b/libcxx/include/__ranges/subrange.h index c6977cec3672..47752a9d05c8 100644 --- a/libcxx/include/__ranges/subrange.h +++ b/libcxx/include/__ranges/subrange.h @@ -36,7 +36,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -282,7 +282,7 @@ struct tuple_element<1, const ranges::subrange<_Ip, _Sp, _Kp>> { using type = _Sp; }; -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/take_view.h b/libcxx/include/__ranges/take_view.h index 4d45219020da..16b9faf168b9 100644 --- a/libcxx/include/__ranges/take_view.h +++ b/libcxx/include/__ranges/take_view.h @@ -34,7 +34,7 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { template @@ -176,7 +176,7 @@ namespace ranges { inline constexpr bool enable_borrowed_range> = enable_borrowed_range<_Tp>; } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/transform_view.h b/libcxx/include/__ranges/transform_view.h index d39dd4097290..34a64f623a3b 100644 --- a/libcxx/include/__ranges/transform_view.h +++ b/libcxx/include/__ranges/transform_view.h @@ -41,7 +41,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { @@ -433,7 +433,7 @@ inline namespace __cpo { } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/view_interface.h b/libcxx/include/__ranges/view_interface.h index 0a37323df817..63417652c122 100644 --- a/libcxx/include/__ranges/view_interface.h +++ b/libcxx/include/__ranges/view_interface.h @@ -27,7 +27,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) namespace ranges { @@ -188,7 +188,7 @@ class view_interface { } // namespace ranges -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__threading_support b/libcxx/include/__threading_support index bf85d5f5d9f0..2db571d84c8f 100644 --- a/libcxx/include/__threading_support +++ b/libcxx/include/__threading_support @@ -254,7 +254,6 @@ int __libcpp_tls_set(__libcpp_tls_key __key, void *__p); #if defined(_LIBCPP_HAS_THREAD_API_PTHREAD) -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m) { pthread_mutexattr_t attr; @@ -279,88 +278,74 @@ int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m) return 0; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_lock(__m); } -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_trylock(__m) == 0; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_unlock(__m); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_destroy(__m); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_lock(__libcpp_mutex_t *__m) { return pthread_mutex_lock(__m); } -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_mutex_trylock(__libcpp_mutex_t *__m) { return pthread_mutex_trylock(__m) == 0; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_unlock(__libcpp_mutex_t *__m) { return pthread_mutex_unlock(__m); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_destroy(__libcpp_mutex_t *__m) { return pthread_mutex_destroy(__m); } // Condition Variable -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_signal(__libcpp_condvar_t *__cv) { return pthread_cond_signal(__cv); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_broadcast(__libcpp_condvar_t *__cv) { return pthread_cond_broadcast(__cv); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_wait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m) { return pthread_cond_wait(__cv, __m); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m, __libcpp_timespec_t *__ts) { return pthread_cond_timedwait(__cv, __m, __ts); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv) { return pthread_cond_destroy(__cv); } // Execute once -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_execute_once(__libcpp_exec_once_flag *flag, void (*init_routine)()) { return pthread_once(flag, init_routine); @@ -368,40 +353,34 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag, // Thread id // Returns non-zero if the thread ids are equal, otherwise 0 -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2) { return t1 == t2; } // Returns non-zero if t1 < t2, otherwise 0 -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2) { return t1 < t2; } // Thread -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) { return __libcpp_thread_get_id(__t) == 0; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *), void *__arg) { return pthread_create(__t, nullptr, __func, __arg); } -_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_current_id() { const __libcpp_thread_t thread = pthread_self(); return __libcpp_thread_get_id(&thread); } -_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t) { #if defined(__MVS__) @@ -411,25 +390,21 @@ __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t) #endif } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_join(__libcpp_thread_t *__t) { return pthread_join(*__t, nullptr); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_detach(__libcpp_thread_t *__t) { return pthread_detach(*__t); } -_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_yield() { sched_yield(); } -_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) { __libcpp_timespec_t __ts = _VSTD::__convert_to_timespec<__libcpp_timespec_t>(__ns); @@ -437,19 +412,16 @@ void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) } // Thread local storage -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_create(__libcpp_tls_key *__key, void (*__at_exit)(void *)) { return pthread_key_create(__key, __at_exit); } -_LIBCPP_HIDE_FROM_ABI inline void *__libcpp_tls_get(__libcpp_tls_key __key) { return pthread_getspecific(__key); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_set(__libcpp_tls_key __key, void *__p) { return pthread_setspecific(__key, __p); @@ -457,56 +429,47 @@ int __libcpp_tls_set(__libcpp_tls_key __key, void *__p) #elif defined(_LIBCPP_HAS_THREAD_API_C11) -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m) { return mtx_init(__m, mtx_plain | mtx_recursive) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t *__m) { return mtx_lock(__m) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t *__m) { return mtx_trylock(__m) == thrd_success; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t *__m) { return mtx_unlock(__m) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t *__m) { mtx_destroy(__m); return 0; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_lock(__libcpp_mutex_t *__m) { return mtx_lock(__m) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_mutex_trylock(__libcpp_mutex_t *__m) { return mtx_trylock(__m) == thrd_success; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_unlock(__libcpp_mutex_t *__m) { return mtx_unlock(__m) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_destroy(__libcpp_mutex_t *__m) { mtx_destroy(__m); @@ -514,25 +477,21 @@ int __libcpp_mutex_destroy(__libcpp_mutex_t *__m) } // Condition Variable -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_signal(__libcpp_condvar_t *__cv) { return cnd_signal(__cv) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_broadcast(__libcpp_condvar_t *__cv) { return cnd_broadcast(__cv) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_wait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m) { return cnd_wait(__cv, __m) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m, timespec *__ts) { @@ -540,7 +499,6 @@ int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m, return __ec == thrd_timedout ? ETIMEDOUT : __ec; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv) { cnd_destroy(__cv); @@ -548,7 +506,6 @@ int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv) } // Execute once -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_execute_once(__libcpp_exec_once_flag *flag, void (*init_routine)(void)) { ::call_once(flag, init_routine); @@ -557,26 +514,22 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag, // Thread id // Returns non-zero if the thread ids are equal, otherwise 0 -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2) { return thrd_equal(t1, t2) != 0; } // Returns non-zero if t1 < t2, otherwise 0 -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2) { return t1 < t2; } // Thread -_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) { return __libcpp_thread_get_id(__t) == 0; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *), void *__arg) { @@ -584,37 +537,31 @@ int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *), return __ec == thrd_nomem ? ENOMEM : __ec; } -_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_current_id() { return thrd_current(); } -_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t) { return *__t; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_join(__libcpp_thread_t *__t) { return thrd_join(*__t, nullptr) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_detach(__libcpp_thread_t *__t) { return thrd_detach(*__t) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_yield() { thrd_yield(); } -_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) { __libcpp_timespec_t __ts = _VSTD::__convert_to_timespec<__libcpp_timespec_t>(__ns); @@ -622,19 +569,16 @@ void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) } // Thread local storage -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_create(__libcpp_tls_key *__key, void (*__at_exit)(void *)) { return tss_create(__key, __at_exit) == thrd_success ? 0 : EINVAL; } -_LIBCPP_HIDE_FROM_ABI inline void *__libcpp_tls_get(__libcpp_tls_key __key) { return tss_get(__key); } -_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_set(__libcpp_tls_key __key, void *__p) { return tss_set(__key, __p) == thrd_success ? 0 : EINVAL; diff --git a/libcxx/include/ranges b/libcxx/include/ranges index 2d79d87eef89..82ad7f4d507f 100644 --- a/libcxx/include/ranges +++ b/libcxx/include/ranges @@ -200,11 +200,6 @@ namespace std::ranges { */ -// Make sure all feature-test macros are available. -#include -// Enable the contents of the header only when libc++ was built with LIBCXX_ENABLE_INCOMPLETE_FEATURES. -#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) - #include <__config> #include <__ranges/access.h> #include <__ranges/all.h> @@ -232,19 +227,18 @@ namespace std::ranges { #include // Required by the standard. #include // Required by the standard. #include +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) #pragma GCC system_header #endif +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) + _LIBCPP_BEGIN_NAMESPACE_STD -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) - namespace views = ranges::views; -#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) - _LIBCPP_END_NAMESPACE_STD #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) diff --git a/libcxx/include/span b/libcxx/include/span index bd11330e69d9..fd95ecca17f7 100644 --- a/libcxx/include/span +++ b/libcxx/include/span @@ -170,7 +170,7 @@ struct __is_std_span : false_type {}; template struct __is_std_span> : true_type {}; -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template concept __span_compatible_range = ranges::contiguous_range<_Range> && @@ -234,7 +234,7 @@ public: _LIBCPP_ASSERT(__last - __first == _Extent, "invalid range in span's constructor (iterator, sentinel): last - first != extent"); } -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) _LIBCPP_INLINE_VISIBILITY constexpr span(type_identity_t (&__arr)[_Extent]) noexcept : __data{__arr} {} @@ -248,13 +248,13 @@ public: _LIBCPP_INLINE_VISIBILITY constexpr span(const array<_OtherElementType, _Extent>& __arr) noexcept : __data{__arr.data()} {} -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template <__span_compatible_range _Range> _LIBCPP_INLINE_VISIBILITY constexpr explicit span(_Range&& __r) : __data{ranges::data(__r)} { _LIBCPP_ASSERT(ranges::size(__r) == _Extent, "size mismatch in span's constructor (range)"); } -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template _LIBCPP_INLINE_VISIBILITY @@ -418,7 +418,7 @@ public: _LIBCPP_INLINE_VISIBILITY constexpr span(_It __first, _End __last) : __data(_VSTD::to_address(__first)), __size(__last - __first) {} -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) template _LIBCPP_INLINE_VISIBILITY @@ -434,11 +434,11 @@ public: _LIBCPP_INLINE_VISIBILITY constexpr span(const array<_OtherElementType, _Sz>& __arr) noexcept : __data{__arr.data()}, __size{_Sz} {} -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template <__span_compatible_range _Range> _LIBCPP_INLINE_VISIBILITY constexpr span(_Range&& __r) : __data(ranges::data(__r)), __size{ranges::size(__r)} {} -# endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template _LIBCPP_INLINE_VISIBILITY @@ -565,10 +565,9 @@ auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept { return __s.__as_writable_bytes(); } #if !defined(_LIBCPP_HAS_NO_CONCEPTS) -// Deduction guides template span(_It, _EndOrSize) -> span>>; -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) template span(_Tp (&)[_Sz]) -> span<_Tp, _Sz>; @@ -579,7 +578,7 @@ template template span(const array<_Tp, _Sz>&) -> span; -#if !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template span(_Range&&) -> span>>; #endif diff --git a/libcxx/include/string b/libcxx/include/string index 01cff902e07d..fd6a7424c815 100644 --- a/libcxx/include/string +++ b/libcxx/include/string @@ -2897,6 +2897,10 @@ template typename basic_string<_CharT, _Traits, _Allocator>::iterator basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, value_type __c) { + _LIBCPP_DEBUG_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this, + "string::insert(iterator, character) called with an iterator not" + " referring to this string"); + size_type __ip = static_cast(__pos - begin()); size_type __sz = size(); size_type __cap = capacity(); diff --git a/libcxx/include/string_view b/libcxx/include/string_view index 5d0a790dd13d..97b24eb44108 100644 --- a/libcxx/include/string_view +++ b/libcxx/include/string_view @@ -294,9 +294,9 @@ public: { _LIBCPP_ASSERT((__end - __begin) >= 0, "std::string_view::string_view(iterator, sentinel) received invalid range"); } -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) -#if _LIBCPP_STD_VER > 20 && !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if _LIBCPP_STD_VER > 20 && !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template requires ( !is_same_v, basic_string_view> && @@ -720,10 +720,10 @@ inline constexpr bool ranges::enable_borrowed_range _End> basic_string_view(_It, _End) -> basic_string_view>; -#endif +#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) -#if _LIBCPP_STD_VER > 20 && !defined(_LIBCPP_HAS_NO_CONCEPTS) +#if _LIBCPP_STD_VER > 20 && !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES) template basic_string_view(_Range) -> basic_string_view>; #endif diff --git a/libcxx/include/vector b/libcxx/include/vector index fd0fb0db2756..28a7ed9bdf36 100644 --- a/libcxx/include/vector +++ b/libcxx/include/vector @@ -302,55 +302,10 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -template -struct __vector_base_common; - -template <> -struct __vector_base_common { - // Both are defined in vector.cpp - _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_length_error() const; - _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_out_of_range() const; -}; - -template -class __vector_base - : protected __vector_base_common // This base class is historical, but it needs to remain for ABI compatibility -{ - typedef _Allocator allocator_type; - typedef typename allocator_traits::pointer pointer; - -protected: - pointer __begin_; - pointer __end_; - __compressed_pair __end_cap_; - - _LIBCPP_INLINE_VISIBILITY - __vector_base() - _NOEXCEPT_(is_nothrow_default_constructible::value) - : __begin_(nullptr), - __end_(nullptr), - __end_cap_(nullptr, __default_init_tag()) {} - - _LIBCPP_INLINE_VISIBILITY __vector_base(const allocator_type& __a) - : __begin_(nullptr), - __end_(nullptr), - __end_cap_(nullptr, __a) {} - -#ifndef _LIBCPP_CXX03_LANG - _LIBCPP_INLINE_VISIBILITY __vector_base(allocator_type&& __a) _NOEXCEPT - : __begin_(nullptr), - __end_(nullptr), - __end_cap_(nullptr, _VSTD::move(__a)) {} -#endif -}; - template */> class _LIBCPP_TEMPLATE_VIS vector - // This base class is historical, but it needs to remain for ABI compatibility. - : private __vector_base<_Tp, _Allocator> { private: - typedef __vector_base<_Tp, _Allocator> __base; typedef allocator<_Tp> __default_allocator_type; public: typedef vector __self; @@ -382,7 +337,7 @@ public: #else _NOEXCEPT #endif - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); } @@ -394,7 +349,7 @@ public: template ::value> > vector(size_type __n, const value_type& __x, const allocator_type& __a) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); if (__n > 0) @@ -691,6 +646,11 @@ public: #endif // _LIBCPP_DEBUG_LEVEL == 2 private: + pointer __begin_ = nullptr; + pointer __end_ = nullptr; + __compressed_pair __end_cap_ = + __compressed_pair(nullptr, __default_init_tag()); + _LIBCPP_INLINE_VISIBILITY void __invalidate_all_iterators(); _LIBCPP_INLINE_VISIBILITY void __invalidate_iterators_past(pointer __new_last); void __vallocate(size_type __n); @@ -859,20 +819,12 @@ private: _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_length_error() const { -#ifndef _LIBCPP_NO_EXCEPTIONS - __vector_base_common::__throw_length_error(); -#else - _VSTD::abort(); -#endif + _VSTD::__throw_length_error("vector"); } _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range() const { -#ifndef _LIBCPP_NO_EXCEPTIONS - __vector_base_common::__throw_out_of_range(); -#else - _VSTD::abort(); -#endif + _VSTD::__throw_out_of_range("vector"); } _LIBCPP_INLINE_VISIBILITY @@ -1106,7 +1058,7 @@ vector<_Tp, _Allocator>::vector(size_type __n) #if _LIBCPP_STD_VER > 11 template vector<_Tp, _Allocator>::vector(size_type __n, const allocator_type& __a) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); if (__n > 0) @@ -1151,7 +1103,7 @@ vector<_Tp, _Allocator>::vector(_InputIterator __first, _InputIterator __last, c is_constructible< value_type, typename iterator_traits<_InputIterator>::reference>::value>::type*) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); for (; __first != __last; ++__first) @@ -1183,7 +1135,7 @@ vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __las is_constructible< value_type, typename iterator_traits<_ForwardIterator>::reference>::value>::type*) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); size_type __n = static_cast(_VSTD::distance(__first, __last)); @@ -1196,7 +1148,7 @@ vector<_Tp, _Allocator>::vector(_ForwardIterator __first, _ForwardIterator __las template vector<_Tp, _Allocator>::vector(const vector& __x) - : __base(__alloc_traits::select_on_container_copy_construction(__x.__alloc())) + : __end_cap_(nullptr, __alloc_traits::select_on_container_copy_construction(__x.__alloc())) { _VSTD::__debug_db_insert_c(this); size_type __n = __x.size(); @@ -1209,7 +1161,7 @@ vector<_Tp, _Allocator>::vector(const vector& __x) template vector<_Tp, _Allocator>::vector(const vector& __x, const __identity_t& __a) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); size_type __n = __x.size(); @@ -1230,7 +1182,7 @@ vector<_Tp, _Allocator>::vector(vector&& __x) #else _NOEXCEPT_(is_nothrow_move_constructible::value) #endif - : __base(_VSTD::move(__x.__alloc())) + : __end_cap_(nullptr, _VSTD::move(__x.__alloc())) { _VSTD::__debug_db_insert_c(this); #if _LIBCPP_DEBUG_LEVEL == 2 @@ -1245,7 +1197,7 @@ vector<_Tp, _Allocator>::vector(vector&& __x) template inline _LIBCPP_INLINE_VISIBILITY vector<_Tp, _Allocator>::vector(vector&& __x, const __identity_t& __a) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); if (__a == __x.__alloc()) @@ -1280,7 +1232,7 @@ vector<_Tp, _Allocator>::vector(initializer_list __il) template inline _LIBCPP_INLINE_VISIBILITY vector<_Tp, _Allocator>::vector(initializer_list __il, const allocator_type& __a) - : __base(__a) + : __end_cap_(nullptr, __a) { _VSTD::__debug_db_insert_c(this); if (__il.size() > 0) @@ -2079,7 +2031,6 @@ struct __has_storage_type > template class _LIBCPP_TEMPLATE_VIS vector - : private __vector_base_common { public: typedef vector __self; @@ -2348,6 +2299,16 @@ public: bool __invariants() const; private: + _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI + void __throw_length_error() const { + _VSTD::__throw_length_error("vector"); + } + + _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI + void __throw_out_of_range() const { + _VSTD::__throw_out_of_range("vector"); + } + _LIBCPP_INLINE_VISIBILITY void __invalidate_all_iterators(); void __vallocate(size_type __n); void __vdeallocate() _NOEXCEPT; diff --git a/libcxx/src/vector.cpp b/libcxx/src/vector.cpp index cc3d291e350e..201d4e3bf09a 100644 --- a/libcxx/src/vector.cpp +++ b/libcxx/src/vector.cpp @@ -10,12 +10,25 @@ _LIBCPP_BEGIN_NAMESPACE_STD +#ifndef _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON + +template +struct __vector_base_common; + +template <> +struct __vector_base_common { + _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_length_error() const; + _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_out_of_range() const; +}; + void __vector_base_common::__throw_length_error() const { - _VSTD::__throw_length_error("vector"); + _VSTD::__throw_length_error("vector"); } void __vector_base_common::__throw_out_of_range() const { - _VSTD::__throw_out_of_range("vector"); + _VSTD::__throw_out_of_range("vector"); } +#endif // _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON + _LIBCPP_END_NAMESPACE_STD diff --git a/libunwind/include/__libunwind_config.h b/libunwind/include/__libunwind_config.h index 67527d9da404..e87bcf40034f 100644 --- a/libunwind/include/__libunwind_config.h +++ b/libunwind/include/__libunwind_config.h @@ -23,6 +23,7 @@ #define _LIBUNWIND_HIGHEST_DWARF_REGISTER_OR1K 32 #define _LIBUNWIND_HIGHEST_DWARF_REGISTER_MIPS 65 #define _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC 31 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC64 31 #define _LIBUNWIND_HIGHEST_DWARF_REGISTER_HEXAGON 34 #define _LIBUNWIND_HIGHEST_DWARF_REGISTER_RISCV 64 #define _LIBUNWIND_HIGHEST_DWARF_REGISTER_VE 143 @@ -125,6 +126,12 @@ # error "Unsupported MIPS ABI and/or environment" # endif # define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_MIPS +#elif defined(__sparc__) && defined(__arch64__) +#define _LIBUNWIND_TARGET_SPARC64 1 +#define _LIBUNWIND_HIGHEST_DWARF_REGISTER \ + _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC64 +#define _LIBUNWIND_CONTEXT_SIZE 33 +#define _LIBUNWIND_CURSOR_SIZE 45 # elif defined(__sparc__) #define _LIBUNWIND_TARGET_SPARC 1 #define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC @@ -165,6 +172,7 @@ # define _LIBUNWIND_TARGET_MIPS_O32 1 # define _LIBUNWIND_TARGET_MIPS_NEWABI 1 # define _LIBUNWIND_TARGET_SPARC 1 +# define _LIBUNWIND_TARGET_SPARC64 1 # define _LIBUNWIND_TARGET_HEXAGON 1 # define _LIBUNWIND_TARGET_RISCV 1 # define _LIBUNWIND_TARGET_VE 1 diff --git a/libunwind/src/DwarfInstructions.hpp b/libunwind/src/DwarfInstructions.hpp index 1c50941680b3..ab83b0c87acd 100644 --- a/libunwind/src/DwarfInstructions.hpp +++ b/libunwind/src/DwarfInstructions.hpp @@ -74,6 +74,13 @@ class DwarfInstructions { } }; +template +auto getSparcWCookie(const R &r, int) -> decltype(r.getWCookie()) { + return r.getWCookie(); +} +template uint64_t getSparcWCookie(const R &, long) { + return 0; +} template typename A::pint_t DwarfInstructions::getSavedRegister( @@ -83,6 +90,10 @@ typename A::pint_t DwarfInstructions::getSavedRegister( case CFI_Parser::kRegisterInCFA: return (pint_t)addressSpace.getRegister(cfa + (pint_t)savedReg.value); + case CFI_Parser::kRegisterInCFADecrypt: // sparc64 specific + return (pint_t)(addressSpace.getP(cfa + (pint_t)savedReg.value) ^ + getSparcWCookie(registers, 0)); + case CFI_Parser::kRegisterAtExpression: return (pint_t)addressSpace.getRegister(evaluateExpression( (pint_t)savedReg.value, addressSpace, registers, cfa)); @@ -124,6 +135,7 @@ double DwarfInstructions::getSavedFloatRegister( case CFI_Parser::kRegisterIsExpression: case CFI_Parser::kRegisterUnused: case CFI_Parser::kRegisterOffsetFromCFA: + case CFI_Parser::kRegisterInCFADecrypt: // FIX ME break; } @@ -148,6 +160,7 @@ v128 DwarfInstructions::getSavedVectorRegister( case CFI_Parser::kRegisterUndefined: case CFI_Parser::kRegisterOffsetFromCFA: case CFI_Parser::kRegisterInRegister: + case CFI_Parser::kRegisterInCFADecrypt: // FIX ME break; } @@ -266,6 +279,12 @@ int DwarfInstructions::stepWithDwarf(A &addressSpace, pint_t pc, } #endif +#if defined(_LIBUNWIND_TARGET_SPARC64) + // Skip call site instruction and delay slot. + if (R::getArch() == REGISTERS_SPARC64) + returnAddress += 8; +#endif + #if defined(_LIBUNWIND_TARGET_PPC64) #define PPC64_ELFV1_R2_LOAD_INST_ENCODING 0xe8410028u // ld r2,40(r1) #define PPC64_ELFV1_R2_OFFSET 40 diff --git a/libunwind/src/DwarfParser.hpp b/libunwind/src/DwarfParser.hpp index 8630178777e3..b5a53166fc3f 100644 --- a/libunwind/src/DwarfParser.hpp +++ b/libunwind/src/DwarfParser.hpp @@ -71,6 +71,7 @@ class CFI_Parser { kRegisterUnused, kRegisterUndefined, kRegisterInCFA, + kRegisterInCFADecrypt, // sparc64 specific kRegisterOffsetFromCFA, kRegisterInRegister, kRegisterAtExpression, @@ -733,7 +734,8 @@ bool CFI_Parser::parseFDEInstructions(A &addressSpace, "DW_CFA_GNU_negative_offset_extended(%" PRId64 ")\n", offset); break; -#if defined(_LIBUNWIND_TARGET_AARCH64) || defined(_LIBUNWIND_TARGET_SPARC) +#if defined(_LIBUNWIND_TARGET_AARCH64) || defined(_LIBUNWIND_TARGET_SPARC) || \ + defined(_LIBUNWIND_TARGET_SPARC64) // The same constant is used to represent different instructions on // AArch64 (negate_ra_state) and SPARC (window_save). static_assert(DW_CFA_AARCH64_negate_ra_state == DW_CFA_GNU_window_save, @@ -767,8 +769,31 @@ bool CFI_Parser::parseFDEInstructions(A &addressSpace, } break; #endif + +#if defined(_LIBUNWIND_TARGET_SPARC64) + // case DW_CFA_GNU_window_save: + case REGISTERS_SPARC64: + // Don't save %o0-%o7 on sparc64. + // https://reviews.llvm.org/D32450#736405 + + for (reg = UNW_SPARC_L0; reg <= UNW_SPARC_I7; reg++) { + if (reg == UNW_SPARC_I7) + results->setRegister( + reg, kRegisterInCFADecrypt, + static_cast((reg - UNW_SPARC_L0) * sizeof(pint_t)), + initialState); + else + results->setRegister( + reg, kRegisterInCFA, + static_cast((reg - UNW_SPARC_L0) * sizeof(pint_t)), + initialState); + } + _LIBUNWIND_TRACE_DWARF("DW_CFA_GNU_window_save\n"); + break; +#endif } break; + #else (void)arch; #endif diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp index 373d9e49018e..cbc3876d672e 100644 --- a/libunwind/src/Registers.hpp +++ b/libunwind/src/Registers.hpp @@ -35,6 +35,7 @@ enum { REGISTERS_MIPS_O32, REGISTERS_MIPS_NEWABI, REGISTERS_SPARC, + REGISTERS_SPARC64, REGISTERS_HEXAGON, REGISTERS_RISCV, REGISTERS_VE, @@ -3586,6 +3587,191 @@ inline const char *Registers_sparc::getRegisterName(int regNum) { } #endif // _LIBUNWIND_TARGET_SPARC +#if defined(_LIBUNWIND_TARGET_SPARC64) +/// Registers_sparc64 holds the register state of a thread in a 64-bit +/// sparc process. +class _LIBUNWIND_HIDDEN Registers_sparc64 { +public: + Registers_sparc64() = default; + Registers_sparc64(const void *registers); + + bool validRegister(int num) const; + uint64_t getRegister(int num) const; + void setRegister(int num, uint64_t value); + bool validFloatRegister(int num) const; + double getFloatRegister(int num) const; + void setFloatRegister(int num, double value); + bool validVectorRegister(int num) const; + v128 getVectorRegister(int num) const; + void setVectorRegister(int num, v128 value); + const char *getRegisterName(int num); + void jumpto(); + static int lastDwarfRegNum() { + return _LIBUNWIND_HIGHEST_DWARF_REGISTER_SPARC64; + } + static int getArch() { return REGISTERS_SPARC64; } + + uint64_t getSP() const { return _registers.__regs[UNW_SPARC_O6] + 2047; } + void setSP(uint64_t value) { _registers.__regs[UNW_SPARC_O6] = value - 2047; } + uint64_t getIP() const { return _registers.__regs[UNW_SPARC_O7]; } + void setIP(uint64_t value) { _registers.__regs[UNW_SPARC_O7] = value; } + uint64_t getWCookie() const { return _wcookie; } + +private: + struct sparc64_thread_state_t { + uint64_t __regs[32]; + }; + + sparc64_thread_state_t _registers{}; + uint64_t _wcookie = 0; +}; + +inline Registers_sparc64::Registers_sparc64(const void *registers) { + static_assert((check_fit::does_fit), + "sparc64 registers do not fit into unw_context_t"); + memcpy(&_registers, registers, sizeof(_registers)); + memcpy(&_wcookie, + static_cast(registers) + sizeof(_registers), + sizeof(_wcookie)); +} + +inline bool Registers_sparc64::validRegister(int regNum) const { + if (regNum == UNW_REG_IP) + return true; + if (regNum == UNW_REG_SP) + return true; + if (regNum < 0) + return false; + if (regNum <= UNW_SPARC_I7) + return true; + return false; +} + +inline uint64_t Registers_sparc64::getRegister(int regNum) const { + if (regNum >= UNW_SPARC_G0 && regNum <= UNW_SPARC_I7) + return _registers.__regs[regNum]; + + switch (regNum) { + case UNW_REG_IP: + return _registers.__regs[UNW_SPARC_O7]; + case UNW_REG_SP: + return _registers.__regs[UNW_SPARC_O6] + 2047; + } + _LIBUNWIND_ABORT("unsupported sparc64 register"); +} + +inline void Registers_sparc64::setRegister(int regNum, uint64_t value) { + if (regNum >= UNW_SPARC_G0 && regNum <= UNW_SPARC_I7) { + _registers.__regs[regNum] = value; + return; + } + + switch (regNum) { + case UNW_REG_IP: + _registers.__regs[UNW_SPARC_O7] = value; + return; + case UNW_REG_SP: + _registers.__regs[UNW_SPARC_O6] = value - 2047; + return; + } + _LIBUNWIND_ABORT("unsupported sparc64 register"); +} + +inline bool Registers_sparc64::validFloatRegister(int) const { return false; } + +inline double Registers_sparc64::getFloatRegister(int) const { + _LIBUNWIND_ABORT("no sparc64 float registers"); +} + +inline void Registers_sparc64::setFloatRegister(int, double) { + _LIBUNWIND_ABORT("no sparc64 float registers"); +} + +inline bool Registers_sparc64::validVectorRegister(int) const { return false; } + +inline v128 Registers_sparc64::getVectorRegister(int) const { + _LIBUNWIND_ABORT("no sparc64 vector registers"); +} + +inline void Registers_sparc64::setVectorRegister(int, v128) { + _LIBUNWIND_ABORT("no sparc64 vector registers"); +} + +inline const char *Registers_sparc64::getRegisterName(int regNum) { + switch (regNum) { + case UNW_REG_IP: + return "pc"; + case UNW_SPARC_G0: + return "g0"; + case UNW_SPARC_G1: + return "g1"; + case UNW_SPARC_G2: + return "g2"; + case UNW_SPARC_G3: + return "g3"; + case UNW_SPARC_G4: + return "g4"; + case UNW_SPARC_G5: + return "g5"; + case UNW_SPARC_G6: + return "g6"; + case UNW_SPARC_G7: + return "g7"; + case UNW_SPARC_O0: + return "o0"; + case UNW_SPARC_O1: + return "o1"; + case UNW_SPARC_O2: + return "o2"; + case UNW_SPARC_O3: + return "o3"; + case UNW_SPARC_O4: + return "o4"; + case UNW_SPARC_O5: + return "o5"; + case UNW_REG_SP: + case UNW_SPARC_O6: + return "o6"; + case UNW_SPARC_O7: + return "o7"; + case UNW_SPARC_L0: + return "l0"; + case UNW_SPARC_L1: + return "l1"; + case UNW_SPARC_L2: + return "l2"; + case UNW_SPARC_L3: + return "l3"; + case UNW_SPARC_L4: + return "l4"; + case UNW_SPARC_L5: + return "l5"; + case UNW_SPARC_L6: + return "l6"; + case UNW_SPARC_L7: + return "l7"; + case UNW_SPARC_I0: + return "i0"; + case UNW_SPARC_I1: + return "i1"; + case UNW_SPARC_I2: + return "i2"; + case UNW_SPARC_I3: + return "i3"; + case UNW_SPARC_I4: + return "i4"; + case UNW_SPARC_I5: + return "i5"; + case UNW_SPARC_I6: + return "i6"; + case UNW_SPARC_I7: + return "i7"; + default: + return "unknown register"; + } +} +#endif // _LIBUNWIND_TARGET_SPARC64 + #if defined(_LIBUNWIND_TARGET_HEXAGON) /// Registers_hexagon holds the register state of a thread in a Hexagon QDSP6 /// process. diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp index c400fdf33d8a..1ca842f33aa5 100644 --- a/libunwind/src/UnwindCursor.hpp +++ b/libunwind/src/UnwindCursor.hpp @@ -1032,6 +1032,10 @@ class UnwindCursor : public AbstractUnwindCursor{ int stepWithCompactEncoding(Registers_sparc &) { return UNW_EINVAL; } #endif +#if defined(_LIBUNWIND_TARGET_SPARC64) + int stepWithCompactEncoding(Registers_sparc64 &) { return UNW_EINVAL; } +#endif + #if defined (_LIBUNWIND_TARGET_RISCV) int stepWithCompactEncoding(Registers_riscv &) { return UNW_EINVAL; @@ -1104,6 +1108,12 @@ class UnwindCursor : public AbstractUnwindCursor{ bool compactSaysUseDwarf(Registers_sparc &, uint32_t *) const { return true; } #endif +#if defined(_LIBUNWIND_TARGET_SPARC64) + bool compactSaysUseDwarf(Registers_sparc64 &, uint32_t *) const { + return true; + } +#endif + #if defined (_LIBUNWIND_TARGET_RISCV) bool compactSaysUseDwarf(Registers_riscv &, uint32_t *) const { return true; @@ -1182,6 +1192,12 @@ class UnwindCursor : public AbstractUnwindCursor{ compact_unwind_encoding_t dwarfEncoding(Registers_sparc &) const { return 0; } #endif +#if defined(_LIBUNWIND_TARGET_SPARC64) + compact_unwind_encoding_t dwarfEncoding(Registers_sparc64 &) const { + return 0; + } +#endif + #if defined (_LIBUNWIND_TARGET_RISCV) compact_unwind_encoding_t dwarfEncoding(Registers_riscv &) const { return 0; diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S index f3d9dd31683e..1df97f5fc41c 100644 --- a/libunwind/src/UnwindRegistersRestore.S +++ b/libunwind/src/UnwindRegistersRestore.S @@ -1062,6 +1062,53 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind21Registers_mips_newabi6jumptoEv) ld $4, (8 * 4)($4) .set pop +#elif defined(__sparc__) && defined(__arch64__) + +DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind17Registers_sparc646jumptoEv) +// +// void libunwind::Registers_sparc64::jumpto() +// +// On entry: +// thread_state pointer is in %o0 +// + .register %g2, #scratch + .register %g3, #scratch + .register %g6, #scratch + .register %g7, #scratch + flushw + ldx [%o0 + 0x08], %g1 + ldx [%o0 + 0x10], %g2 + ldx [%o0 + 0x18], %g3 + ldx [%o0 + 0x20], %g4 + ldx [%o0 + 0x28], %g5 + ldx [%o0 + 0x30], %g6 + ldx [%o0 + 0x38], %g7 + ldx [%o0 + 0x48], %o1 + ldx [%o0 + 0x50], %o2 + ldx [%o0 + 0x58], %o3 + ldx [%o0 + 0x60], %o4 + ldx [%o0 + 0x68], %o5 + ldx [%o0 + 0x70], %o6 + ldx [%o0 + 0x78], %o7 + ldx [%o0 + 0x80], %l0 + ldx [%o0 + 0x88], %l1 + ldx [%o0 + 0x90], %l2 + ldx [%o0 + 0x98], %l3 + ldx [%o0 + 0xa0], %l4 + ldx [%o0 + 0xa8], %l5 + ldx [%o0 + 0xb0], %l6 + ldx [%o0 + 0xb8], %l7 + ldx [%o0 + 0xc0], %i0 + ldx [%o0 + 0xc8], %i1 + ldx [%o0 + 0xd0], %i2 + ldx [%o0 + 0xd8], %i3 + ldx [%o0 + 0xe0], %i4 + ldx [%o0 + 0xe8], %i5 + ldx [%o0 + 0xf0], %i6 + ldx [%o0 + 0xf8], %i7 + jmp %o7 + ldx [%o0 + 0x40], %o0 + #elif defined(__sparc__) // diff --git a/libunwind/src/UnwindRegistersSave.S b/libunwind/src/UnwindRegistersSave.S index 7af5c9d154fb..9566bb0335fe 100644 --- a/libunwind/src/UnwindRegistersSave.S +++ b/libunwind/src/UnwindRegistersSave.S @@ -999,6 +999,64 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext) jumpr r31 +#elif defined(__sparc__) && defined(__arch64__) + +# +# extern int __unw_getcontext(unw_context_t* thread_state) +# +# On entry: +# thread_state pointer is in %o0 +# +DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext) + .register %g2, #scratch + .register %g3, #scratch + .register %g6, #scratch + .register %g7, #scratch + stx %g1, [%o0 + 0x08] + stx %g2, [%o0 + 0x10] + stx %g3, [%o0 + 0x18] + stx %g4, [%o0 + 0x20] + stx %g5, [%o0 + 0x28] + stx %g6, [%o0 + 0x30] + stx %g7, [%o0 + 0x38] + stx %o0, [%o0 + 0x40] + stx %o1, [%o0 + 0x48] + stx %o2, [%o0 + 0x50] + stx %o3, [%o0 + 0x58] + stx %o4, [%o0 + 0x60] + stx %o5, [%o0 + 0x68] + stx %o6, [%o0 + 0x70] + stx %o7, [%o0 + 0x78] + stx %l0, [%o0 + 0x80] + stx %l1, [%o0 + 0x88] + stx %l2, [%o0 + 0x90] + stx %l3, [%o0 + 0x98] + stx %l4, [%o0 + 0xa0] + stx %l5, [%o0 + 0xa8] + stx %l6, [%o0 + 0xb0] + stx %l7, [%o0 + 0xb8] + stx %i0, [%o0 + 0xc0] + stx %i1, [%o0 + 0xc8] + stx %i2, [%o0 + 0xd0] + stx %i3, [%o0 + 0xd8] + stx %i4, [%o0 + 0xe0] + stx %i5, [%o0 + 0xe8] + stx %i6, [%o0 + 0xf0] + stx %i7, [%o0 + 0xf8] + + # save StackGhost cookie + mov %i7, %g4 + save %sp, -176, %sp + # register window flush necessary even without StackGhost + flushw + restore + ldx [%sp + 2047 + 0x78], %g5 + xor %g4, %g5, %g4 + stx %g4, [%o0 + 0x100] + retl + # return UNW_ESUCCESS + clr %o0 + #elif defined(__sparc__) # diff --git a/libunwind/src/config.h b/libunwind/src/config.h index 560edda04eaa..5ae1604f657d 100644 --- a/libunwind/src/config.h +++ b/libunwind/src/config.h @@ -109,13 +109,10 @@ #define _LIBUNWIND_SUPPORT_FRAME_APIS #endif -#if defined(__i386__) || defined(__x86_64__) || \ - defined(__ppc__) || defined(__ppc64__) || defined(__powerpc64__) || \ - (!defined(__APPLE__) && defined(__arm__)) || \ - defined(__aarch64__) || \ - defined(__mips__) || \ - defined(__riscv) || \ - defined(__hexagon__) +#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \ + (!defined(__APPLE__) && defined(__arm__)) || defined(__aarch64__) || \ + defined(__mips__) || defined(__riscv) || defined(__hexagon__) || \ + defined(__sparc__) #if !defined(_LIBUNWIND_BUILD_SJLJ_APIS) #define _LIBUNWIND_BUILD_ZERO_COST_APIS #endif diff --git a/libunwind/src/libunwind.cpp b/libunwind/src/libunwind.cpp index ef48cbe860c7..03f8b75b5bba 100644 --- a/libunwind/src/libunwind.cpp +++ b/libunwind/src/libunwind.cpp @@ -67,6 +67,8 @@ _LIBUNWIND_HIDDEN int __unw_init_local(unw_cursor_t *cursor, # define REGISTER_KIND Registers_mips_newabi #elif defined(__mips__) # warning The MIPS architecture is not supported with this ABI and environment! +#elif defined(__sparc__) && defined(__arch64__) +#define REGISTER_KIND Registers_sparc64 #elif defined(__sparc__) # define REGISTER_KIND Registers_sparc #elif defined(__riscv) diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp index c083484f4280..7d051c43af0d 100644 --- a/lld/ELF/Arch/PPC64.cpp +++ b/lld/ELF/Arch/PPC64.cpp @@ -1380,9 +1380,10 @@ bool PPC64::needsThunk(RelExpr expr, RelType type, const InputFile *file, if (type == R_PPC64_REL24_NOTOC && (s.stOther >> 5) > 1) return true; - // If a symbol is a weak undefined and we are compiling an executable - // it doesn't need a range-extending thunk since it can't be called. - if (s.isUndefWeak() && !config->shared) + // An undefined weak symbol not in a PLT does not need a thunk. If it is + // hidden, its binding has been converted to local, so we just check + // isUndefined() here. A undefined non-weak symbol has been errored. + if (s.isUndefined()) return false; // If the offset exceeds the range of the branch type then it will need diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 406571e4e9fd..54b487bdc243 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -235,22 +235,28 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) { // user is attempting LTO and using a default ar command that doesn't // understand the LLVM bitcode file. Treat the archive as a group of lazy // object files. - if (!file->isEmpty() && !file->hasSymbolTable()) { - for (const std::pair &p : - getArchiveMembers(mbref)) { - auto magic = identify_magic(p.first.getBuffer()); - if (magic == file_magic::bitcode || - magic == file_magic::elf_relocatable) - files.push_back(createLazyFile(p.first, path, p.second)); - else - error(path + ": archive member '" + p.first.getBufferIdentifier() + - "' is neither ET_REL nor LLVM bitcode"); - } + if (file->isEmpty() || file->hasSymbolTable()) { + // Handle the regular case. + files.push_back(make(std::move(file))); return; } - // Handle the regular case. - files.push_back(make(std::move(file))); + // All files within the archive get the same group ID to allow mutual + // references for --warn-backrefs. + bool saved = InputFile::isInGroup; + InputFile::isInGroup = true; + for (const std::pair &p : + getArchiveMembers(mbref)) { + auto magic = identify_magic(p.first.getBuffer()); + if (magic == file_magic::bitcode || magic == file_magic::elf_relocatable) + files.push_back(createLazyFile(p.first, path, p.second)); + else + error(path + ": archive member '" + p.first.getBufferIdentifier() + + "' is neither ET_REL nor LLVM bitcode"); + } + InputFile::isInGroup = saved; + if (!saved) + ++InputFile::nextGroupId; return; } case file_magic::elf_shared_object: @@ -1234,6 +1240,9 @@ static void readConfigs(opt::InputArgList &args) { error(errPrefix + toString(pat.takeError())); } + if (args.hasArg(OPT_define_common, OPT_no_define_common)) + warn("-d, -dc, -dp, and --[no-]define-common will be removed. See https://github.com/llvm/llvm-project/issues/53660"); + cl::ResetAllOptionOccurrences(); // Parse LTO options. diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index b5510b3b2736..775999a5f534 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -152,18 +152,17 @@ static bool isCompatible(InputFile *file) { return false; } - InputFile *existing; + InputFile *existing = nullptr; if (!objectFiles.empty()) existing = objectFiles[0]; else if (!sharedFiles.empty()) existing = sharedFiles[0]; else if (!bitcodeFiles.empty()) existing = bitcodeFiles[0]; - else - llvm_unreachable("Must have -m, OUTPUT_FORMAT or existing input file to " - "determine target emulation"); - - error(toString(file) + " is incompatible with " + toString(existing)); + std::string with; + if (existing) + with = " with " + toString(existing); + error(toString(file) + " is incompatible" + with); return false; } diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp index 241b3ea3b418..53cdcee1bfe8 100644 --- a/lld/ELF/OutputSections.cpp +++ b/lld/ELF/OutputSections.cpp @@ -16,7 +16,7 @@ #include "lld/Common/Memory.h" #include "lld/Common/Strings.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/Config/config.h" // LLVM_ENABLE_ZLIB +#include "llvm/Config/llvm-config.h" // LLVM_ENABLE_ZLIB #include "llvm/Support/MD5.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Parallel.h" diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst index 82683801fad9..672759fdf6b3 100644 --- a/lld/docs/ReleaseNotes.rst +++ b/lld/docs/ReleaseNotes.rst @@ -30,45 +30,173 @@ ELF Improvements (`D107317 `_) * ``--why-extract`` has been added to query why archive members/lazy object files are extracted. (`D109572 `_) -* ``e_entry`` no longer falls back to the address of ``.text`` if the entry symbol does not exist. - Instead, a value of 0 will be written. - (`D110014 `_) * If ``-Map`` is specified, ``--cref`` will be printed to the specified file. (`D114663 `_) +* ``-z bti-report`` and ``-z cet-report`` are now supported. + (`D113901 `_) +* ``--lto-pgo-warn-mismatch`` has been added. + (`D104431 `_) +* Archives without an index (symbol table) are now supported and work with + ``--warn-backrefs``. One may build such an archive with ``llvm-ar rcS + [--thin]`` to save space. + (`D117284 `_) + The archive index may be `entirely ignored `_ + in a future release. * No longer deduplicate local symbol names at the default optimization level of ``-O1``. This results in a larger ``.strtab`` (usually less than 1%) but a faster link - time. Use optimization level ``-O2`` to restore the deduplication. + time. Use optimization level ``-O2`` to restore the deduplication. The ``-O2`` + deduplication may be dropped in the future to help parallel ``.symtab`` write. +* In relocatable output, relocations to discarded symbols now use tombstone + values. + (`D116946 `_) +* Orphan section placement now picks a more suitable segment. Previously the + algorithm might pick a readonly segment for a writable orphan section and make + the segment writable. + (`D111717 `_) +* An empty output section moved by an ``INSERT`` comment now gets appropriate + flags. + (`D118529 `_) +* Negation in a memory region attribute is now correctly handled. + (`D113771 `_) +* ``--compress-debug-sections=zlib`` is now run in parallel. ``{clang,gcc} -gz`` link + actions are significantly faster. + (`D117853 `_) +* "relocation out of range" diagnostics and a few uncommon diagnostics + now report an object file location beside a source file location. + (`D112518 `_) +* The write of ``.rela.dyn`` and ``SHF_MERGE|SHF_STRINGS`` sections (e.g. + ``.debug_str``) is now run in parallel. Architecture specific changes: +* The AArch64 port now supports adrp+ldr and adrp+add optimizations. + ``--no-relax`` can suppress the optimization. + (`D112063 `_) + (`D117614 `_) * The x86-32 port now supports TLSDESC (``-mtls-dialect=gnu2``). (`D112582 `_) * The x86-64 port now handles non-RAX/non-adjacent ``R_X86_64_GOTPC32_TLSDESC`` and ``R_X86_64_TLSDESC_CALL`` (``-mtls-dialect=gnu2``). (`D114416 `_) +* The x86-32 and x86-64 ports now support mixed TLSDESC and TLS GD, i.e. mixing + objects compiled with and without ``-mtls-dialect=gnu2`` referencing the same + TLS variable is now supported. + (`D114416 `_) * For x86-64, ``--no-relax`` now suppresses ``R_X86_64_GOTPCRELX`` and ``R_X86_64_REX_GOTPCRELX`` GOT optimization (`D113615 `_) +* ``R_X86_64_PLTOFF64`` is now supported. + (`D112386 `_) +* ``R_AARCH64_NONE``, ``R_PPC_NONE``, and ``R_PPC64_NONE`` in input REL + relocation sections are now supported. Breaking changes ---------------- -* ... +* ``e_entry`` no longer falls back to the address of ``.text`` if the entry symbol does not exist. + Instead, a value of 0 will be written. + (`D110014 `_) +* ``--lto-pseudo-probe-for-profiling`` has been removed. In LTO, the compiler + enables this feature automatically. + (`D110209 `_) +* Use of ``--[no-]define-common``, ``-d``, ``-dc``, and ``-dp`` will now get a + warning. They will be removed or ignored in 15.0.0. + (`llvm-project#53660 `_) COFF Improvements ----------------- -* ... +* Correctly handle a signed immediate offset in ARM64 adr/adrp relocations. + (`D114347 `_) + +* Omit section and label symbols from the symbol table. + (`D113866 `_) MinGW Improvements ------------------ -* ... +* ``--heap`` is now handled. + (`D118405 `_) -MachO Improvements ------------------- +Mach-O Improvements +------------------- -* Item 1. +* The ``ld64.lld.darwinnew`` symlink has been removed. Use ``ld64.lld`` to + invoke the Mach-O backend from now on. Moreover, the symlink + ``ld64.lld.darwinold`` -- and the old Mach-O LLD code that it pointed to -- + have been removed. (`D114842 `_) +* The "cannot export hidden symbol" error has been downgraded to a warning. + (`D107011 `_) +* Common bitcode symbols are now supported. + (`D107027 `_) +* Weak references in bitcode are now supported. + (`D115949 `_) +* Thunk insertion now works more reliably. + (`D108897 `_, + `D108924 `_, + `D109079 `_, + `D116705 `_) +* ``-ObjC`` now loads archive members before the symbol resolution phase. + (`D108781 `_) +* ``-oso_prefix`` is now supported. + (`D112291 `_) +* Properly encode binaries with zero exported symbols. Tools like + ``codesign_allocate`` no longer choke on those binaries. + (`D112589 `_) +* We no longer treat architecture mismatches as a fatal error. Use + ``-arch_errors_fatal`` if that behavior is still desired. + (`D113082 `_) +* Several performance improvements were done to speed LLD up on projects with + a lot of framework flags and library lookups. Large Swift-based projects + will benefit significantly. + (`D113073 `_, + `D113063 `_, + `D113153 `_, + `D113235 `_) +* Several memory-reduction optimizations were done to reduce LLD's RSS + footprint. + (`D113813 `_, + `D113818 `_) +* Symbol patterns from ``-[un]exported_symbols_list`` are now processed in + parallel. (`D113820 `_) +* ``.weak_def_can_be_hidden`` symbols can now be exported. + (`D113167 `_) +* ``-S`` -- to omit debug info -- is now handled. + (`D112594 `_) +* ``-v`` now writes to stderr instead of stdout. + (`D113020 `_) +* Private externs with GOT relocations are now marked as LOCAL in the indirect + symbol table. This allows ``strip -x`` to remove more symbols. + (`D111852 `_) +* We no longer generate bogus addresses when ``__TEXT,__gcc_except_tab`` is + renamed. (`D113582 `_) +* Unreferenced weak dylib symbols no longer trigger fetches from an archive. + (`D115092 `_) +* ``$ld$hide`` symbols are now supported. + (`D115775 `_) +* Symbols imported via ``-weak_framework`` are now properly marked as weak refs. + (`D114397 `_) +* ``--warn-dylib-install-name`` and ``--no-warn-dylib-install-name`` were added + to toggle LLD-specific warnings around the use of ``-install_name``. + (`D113534 `_) +* Passing both ``--icf=all`` and ``-no_deduplicate`` no longer results in a + warning. (`D110672 `_) +* ICF now deduplicates functions with (identical) unwind info too. + (`D109946 `_) +* We now support ordering sections based on call graph profile data. + (`D112164 `_) +* Map file output now proceeds in parallel with output of the binary. + (`D117069 `_) +* The map file now contains dead-stripped symbols too. + (`D114737 `_) +* Multiple TLV sections with different alignments are now handled properly. + (`D116263 `_) +* ``--start-lib`` and ``--end-lib`` are now supported. + (`D116913 `_) +* ``-noall_load`` is now supported. + (`D117629 `_) +* ``-add_empty_section`` is now supported. + (`D117749 `_) WebAssembly Improvements ------------------------ diff --git a/lldb/include/lldb/Core/Mangled.h b/lldb/include/lldb/Core/Mangled.h index 4e4f75e18e34..aaefb69f017b 100644 --- a/lldb/include/lldb/Core/Mangled.h +++ b/lldb/include/lldb/Core/Mangled.h @@ -226,10 +226,9 @@ class Mangled { /// Function signature for filtering mangled names. using SkipMangledNameFn = bool(llvm::StringRef, ManglingScheme); - /// Trigger explicit demangling to obtain rich mangling information. This is - /// optimized for batch processing while populating a name index. To get the - /// pure demangled name string for a single entity, use GetDemangledName() - /// instead. + /// Get rich mangling information. This is optimized for batch processing + /// while populating a name index. To get the pure demangled name string for + /// a single entity, use GetDemangledName() instead. /// /// For names that match the Itanium mangling scheme, this uses LLVM's /// ItaniumPartialDemangler. All other names fall back to LLDB's builtin @@ -248,8 +247,8 @@ class Mangled { /// /// \return /// True on success, false otherwise. - bool DemangleWithRichManglingInfo(RichManglingContext &context, - SkipMangledNameFn *skip_mangled_name); + bool GetRichManglingInfo(RichManglingContext &context, + SkipMangledNameFn *skip_mangled_name); /// Try to identify the mangling scheme used. /// \param[in] name diff --git a/lldb/include/lldb/Core/RichManglingContext.h b/lldb/include/lldb/Core/RichManglingContext.h index a6b7af8d8d7e..ecd11e93f044 100644 --- a/lldb/include/lldb/Core/RichManglingContext.h +++ b/lldb/include/lldb/Core/RichManglingContext.h @@ -43,25 +43,15 @@ class RichManglingContext { bool IsCtorOrDtor() const; /// Get the base name of a function. This doesn't include trailing template - /// arguments, ie "a::b" gives "b". The result will overwrite the - /// internal buffer. It can be obtained via GetBufferRef(). - void ParseFunctionBaseName(); + /// arguments, ie "a::b" gives "b". + llvm::StringRef ParseFunctionBaseName(); /// Get the context name for a function. For "a::b::c", this function returns - /// "a::b". The result will overwrite the internal buffer. It can be obtained - /// via GetBufferRef(). - void ParseFunctionDeclContextName(); + /// "a::b". + llvm::StringRef ParseFunctionDeclContextName(); - /// Get the entire demangled name. The result will overwrite the internal - /// buffer. It can be obtained via GetBufferRef(). - void ParseFullName(); - - /// Obtain a StringRef to the internal buffer that holds the result of the - /// most recent ParseXy() operation. The next ParseXy() call invalidates it. - llvm::StringRef GetBufferRef() const { - assert(m_provider != None && "Initialize a provider first"); - return m_buffer; - } + /// Get the entire demangled name. + llvm::StringRef ParseFullName(); private: enum InfoProvider { None, ItaniumPartialDemangler, PluginCxxLanguage }; @@ -69,9 +59,6 @@ class RichManglingContext { /// Selects the rich mangling info provider. InfoProvider m_provider = None; - /// Reference to the buffer used for results of ParseXy() operations. - llvm::StringRef m_buffer; - /// Members for ItaniumPartialDemangler llvm::ItaniumPartialDemangler m_ipd; /// Note: m_ipd_buf is a raw pointer due to being resized by realloc via @@ -93,7 +80,7 @@ class RichManglingContext { void ResetProvider(InfoProvider new_provider); /// Uniform handling of string buffers for ItaniumPartialDemangler. - void processIPDStrResult(char *ipd_res, size_t res_len); + llvm::StringRef processIPDStrResult(char *ipd_res, size_t res_len); /// Cast the given parser to the given type. Ideally we would have a type /// trait to deduce \a ParserT from a given InfoProvider, but unfortunately we diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h index 12ed1e09227c..7911dac40b70 100644 --- a/lldb/include/lldb/Target/Process.h +++ b/lldb/include/lldb/Target/Process.h @@ -1759,7 +1759,7 @@ class Process : public std::enable_shared_from_this, /// /// If load_addr is within the address space the process has mapped /// range_info will be filled in with the start and end of that range as - /// well as the permissions for that range and range_info.GetMapped will + /// well as the permissions for that range and range_info. GetMapped will /// return true. /// /// If load_addr is outside any mapped region then range_info will have its @@ -1768,23 +1768,21 @@ class Process : public std::enable_shared_from_this, /// there are no valid mapped ranges between load_addr and the end of the /// process address space. /// - /// GetMemoryRegionInfo will only return an error if it is unimplemented for - /// the current process. + /// GetMemoryRegionInfo calls DoGetMemoryRegionInfo. Override that function in + /// process subclasses. /// /// \param[in] load_addr - /// The load address to query the range_info for. + /// The load address to query the range_info for. May include non + /// address bits, these will be removed by the the ABI plugin if there is + /// one. /// /// \param[out] range_info /// An range_info value containing the details of the range. /// /// \return /// An error value. - virtual Status GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo &range_info) { - Status error; - error.SetErrorString("Process::GetMemoryRegionInfo() not supported"); - return error; - } + Status GetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo &range_info); /// Obtain all the mapped memory regions within this process. /// @@ -2604,6 +2602,26 @@ void PruneThreadPlans(); virtual size_t DoReadMemory(lldb::addr_t vm_addr, void *buf, size_t size, Status &error) = 0; + /// DoGetMemoryRegionInfo is called by GetMemoryRegionInfo after it has + /// removed non address bits from load_addr. Override this method in + /// subclasses of Process. + /// + /// See GetMemoryRegionInfo for details of the logic. + /// + /// \param[in] load_addr + /// The load address to query the range_info for. (non address bits + /// removed) + /// + /// \param[out] range_info + /// An range_info value containing the details of the range. + /// + /// \return + /// An error value. + virtual Status DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo &range_info) { + return Status("Process::DoGetMemoryRegionInfo() not supported"); + } + lldb::StateType GetPrivateState(); /// The "private" side of resuming a process. This doesn't alter the state diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp index 1b44a1bd709a..1033d13f9f26 100644 --- a/lldb/source/Commands/CommandObjectMemory.cpp +++ b/lldb/source/Commands/CommandObjectMemory.cpp @@ -1665,14 +1665,11 @@ class CommandObjectMemoryRegion : public CommandObjectParsed { m_prev_end_addr = LLDB_INVALID_ADDRESS; const size_t argc = command.GetArgumentCount(); - if (argc > 1 || (argc == 0 && load_addr == LLDB_INVALID_ADDRESS)) { - result.AppendErrorWithFormat("'%s' takes one argument:\nUsage: %s\n", - m_cmd_name.c_str(), m_cmd_syntax.c_str()); - return false; - } + const lldb::ABISP &abi = process_sp->GetABI(); if (argc == 1) { auto load_addr_str = command[0].ref(); + // Non-address bits in this will be handled later by GetMemoryRegion load_addr = OptionArgParser::ToAddress(&m_exe_ctx, load_addr_str, LLDB_INVALID_ADDRESS, &error); if (error.Fail() || load_addr == LLDB_INVALID_ADDRESS) { @@ -1680,6 +1677,19 @@ class CommandObjectMemoryRegion : public CommandObjectParsed { command[0].c_str(), error.AsCString()); return false; } + } else if (argc > 1 || + // When we're repeating the command, the previous end address is + // used for load_addr. If that was 0xF...F then we must have + // reached the end of memory. + (argc == 0 && load_addr == LLDB_INVALID_ADDRESS) || + // If the target has non-address bits (tags, limited virtual + // address size, etc.), the end of mappable memory will be lower + // than that. So if we find any non-address bit set, we must be + // at the end of the mappable range. + (abi && (abi->FixDataAddress(load_addr) != load_addr))) { + result.AppendErrorWithFormat("'%s' takes one argument:\nUsage: %s\n", + m_cmd_name.c_str(), m_cmd_syntax.c_str()); + return false; } lldb_private::MemoryRegionInfo range_info; diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp index 137aaa81c61a..f6042937a4ff 100644 --- a/lldb/source/Commands/CommandObjectThread.cpp +++ b/lldb/source/Commands/CommandObjectThread.cpp @@ -1320,6 +1320,53 @@ class CommandObjectThreadException : public CommandObjectIterateOverThreads { } }; +class CommandObjectThreadSiginfo : public CommandObjectIterateOverThreads { +public: + CommandObjectThreadSiginfo(CommandInterpreter &interpreter) + : CommandObjectIterateOverThreads( + interpreter, "thread siginfo", + "Display the current siginfo object for a thread. Defaults to " + "the current thread.", + "thread siginfo", + eCommandRequiresProcess | eCommandTryTargetAPILock | + eCommandProcessMustBeLaunched | eCommandProcessMustBePaused) {} + + ~CommandObjectThreadSiginfo() override = default; + + void + HandleArgumentCompletion(CompletionRequest &request, + OptionElementVector &opt_element_vector) override { + CommandCompletions::InvokeCommonCompletionCallbacks( + GetCommandInterpreter(), CommandCompletions::eThreadIndexCompletion, + request, nullptr); + } + + bool HandleOneThread(lldb::tid_t tid, CommandReturnObject &result) override { + ThreadSP thread_sp = + m_exe_ctx.GetProcessPtr()->GetThreadList().FindThreadByID(tid); + if (!thread_sp) { + result.AppendErrorWithFormat("thread no longer exists: 0x%" PRIx64 "\n", + tid); + return false; + } + + Stream &strm = result.GetOutputStream(); + if (!thread_sp->GetDescription(strm, eDescriptionLevelFull, false, false)) { + result.AppendErrorWithFormat("error displaying info for thread: \"%d\"\n", + thread_sp->GetIndexID()); + return false; + } + ValueObjectSP exception_object_sp = thread_sp->GetSiginfoValue(); + if (exception_object_sp) + exception_object_sp->Dump(strm); + else + strm.Printf("(no siginfo)\n"); + strm.PutChar('\n'); + + return true; + } +}; + // CommandObjectThreadReturn #define LLDB_OPTIONS_thread_return #include "CommandOptions.inc" @@ -2293,6 +2340,8 @@ CommandObjectMultiwordThread::CommandObjectMultiwordThread( CommandObjectSP(new CommandObjectThreadInfo(interpreter))); LoadSubCommand("exception", CommandObjectSP(new CommandObjectThreadException( interpreter))); + LoadSubCommand("siginfo", + CommandObjectSP(new CommandObjectThreadSiginfo(interpreter))); LoadSubCommand("step-in", CommandObjectSP(new CommandObjectThreadStepWithTypeAndScope( interpreter, "thread step-in", diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp index 4e10324401dc..b8e405544b33 100644 --- a/lldb/source/Core/Mangled.cpp +++ b/lldb/source/Core/Mangled.cpp @@ -195,8 +195,8 @@ static char *GetDLangDemangledStr(const char *M) { // Explicit demangling for scheduled requests during batch processing. This // makes use of ItaniumPartialDemangler's rich demangle info -bool Mangled::DemangleWithRichManglingInfo( - RichManglingContext &context, SkipMangledNameFn *skip_mangled_name) { +bool Mangled::GetRichManglingInfo(RichManglingContext &context, + SkipMangledNameFn *skip_mangled_name) { // Others are not meant to arrive here. ObjC names or C's main() for example // have their names stored in m_demangled, while m_mangled is empty. assert(m_mangled); @@ -214,25 +214,16 @@ bool Mangled::DemangleWithRichManglingInfo( case eManglingSchemeItanium: // We want the rich mangling info here, so we don't care whether or not // there is a demangled string in the pool already. - if (context.FromItaniumName(m_mangled)) { - // If we got an info, we have a name. Copy to string pool and connect the - // counterparts to accelerate later access in GetDemangledName(). - context.ParseFullName(); - m_demangled.SetStringWithMangledCounterpart(context.GetBufferRef(), - m_mangled); - return true; - } else { - m_demangled.SetCString(""); - return false; - } + return context.FromItaniumName(m_mangled); case eManglingSchemeMSVC: { // We have no rich mangling for MSVC-mangled names yet, so first try to // demangle it if necessary. if (!m_demangled && !m_mangled.GetMangledCounterpart(m_demangled)) { if (char *d = GetMSVCDemangledStr(m_mangled.GetCString())) { - // If we got an info, we have a name. Copy to string pool and connect - // the counterparts to accelerate later access in GetDemangledName(). + // Without the rich mangling info we have to demangle the full name. + // Copy it to string pool and connect the counterparts to accelerate + // later access in GetDemangledName(). m_demangled.SetStringWithMangledCounterpart(llvm::StringRef(d), m_mangled); ::free(d); diff --git a/lldb/source/Core/RichManglingContext.cpp b/lldb/source/Core/RichManglingContext.cpp index 63170feb6231..cecb5c6a2e54 100644 --- a/lldb/source/Core/RichManglingContext.cpp +++ b/lldb/source/Core/RichManglingContext.cpp @@ -83,15 +83,15 @@ bool RichManglingContext::IsCtorOrDtor() const { llvm_unreachable("Fully covered switch above!"); } -void RichManglingContext::processIPDStrResult(char *ipd_res, size_t res_size) { +llvm::StringRef RichManglingContext::processIPDStrResult(char *ipd_res, + size_t res_size) { // Error case: Clear the buffer. if (LLVM_UNLIKELY(ipd_res == nullptr)) { assert(res_size == m_ipd_buf_size && "Failed IPD queries keep the original size in the N parameter"); m_ipd_buf[0] = '\0'; - m_buffer = llvm::StringRef(m_ipd_buf, 0); - return; + return llvm::StringRef(m_ipd_buf, 0); } // IPD's res_size includes null terminator. @@ -109,60 +109,54 @@ void RichManglingContext::processIPDStrResult(char *ipd_res, size_t res_size) { } // 99% case: Just remember the string length. - m_buffer = llvm::StringRef(m_ipd_buf, res_size - 1); + return llvm::StringRef(m_ipd_buf, res_size - 1); } -void RichManglingContext::ParseFunctionBaseName() { +llvm::StringRef RichManglingContext::ParseFunctionBaseName() { assert(m_provider != None && "Initialize a provider first"); switch (m_provider) { case ItaniumPartialDemangler: { auto n = m_ipd_buf_size; auto buf = m_ipd.getFunctionBaseName(m_ipd_buf, &n); - processIPDStrResult(buf, n); - return; + return processIPDStrResult(buf, n); } case PluginCxxLanguage: - m_buffer = - get(m_cxx_method_parser)->GetBasename(); - return; + return get(m_cxx_method_parser) + ->GetBasename(); case None: - return; + return {}; } } -void RichManglingContext::ParseFunctionDeclContextName() { +llvm::StringRef RichManglingContext::ParseFunctionDeclContextName() { assert(m_provider != None && "Initialize a provider first"); switch (m_provider) { case ItaniumPartialDemangler: { auto n = m_ipd_buf_size; auto buf = m_ipd.getFunctionDeclContextName(m_ipd_buf, &n); - processIPDStrResult(buf, n); - return; + return processIPDStrResult(buf, n); } case PluginCxxLanguage: - m_buffer = - get(m_cxx_method_parser)->GetContext(); - return; + return get(m_cxx_method_parser) + ->GetContext(); case None: - return; + return {}; } } -void RichManglingContext::ParseFullName() { +llvm::StringRef RichManglingContext::ParseFullName() { assert(m_provider != None && "Initialize a provider first"); switch (m_provider) { case ItaniumPartialDemangler: { auto n = m_ipd_buf_size; auto buf = m_ipd.finishDemangle(m_ipd_buf, &n); - processIPDStrResult(buf, n); - return; + return processIPDStrResult(buf, n); } case PluginCxxLanguage: - m_buffer = get(m_cxx_method_parser) - ->GetFullName() - .GetStringRef(); - return; + return get(m_cxx_method_parser) + ->GetFullName() + .GetStringRef(); case None: - return; + return {}; } } diff --git a/lldb/source/Plugins/InstrumentationRuntime/ASan/InstrumentationRuntimeASan.cpp b/lldb/source/Plugins/InstrumentationRuntime/ASan/InstrumentationRuntimeASan.cpp index 8d8b5c68e41b..33b2b5dd5155 100644 --- a/lldb/source/Plugins/InstrumentationRuntime/ASan/InstrumentationRuntimeASan.cpp +++ b/lldb/source/Plugins/InstrumentationRuntime/ASan/InstrumentationRuntimeASan.cpp @@ -281,7 +281,7 @@ void InstrumentationRuntimeASan::Activate() { if (!process_sp) return; - ConstString symbol_name("__asan::AsanDie()"); + ConstString symbol_name("_ZN6__asanL7AsanDieEv"); const Symbol *symbol = GetRuntimeModuleSP()->FindFirstSymbolWithNameAndType( symbol_name, eSymbolTypeCode); diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp index 65dbc8ea95b3..043544f05665 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp @@ -285,8 +285,8 @@ size_t ProcessElfCore::ReadMemory(lldb::addr_t addr, void *buf, size_t size, return DoReadMemory(addr, buf, size, error); } -Status ProcessElfCore::GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo ®ion_info) { +Status ProcessElfCore::DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo ®ion_info) { region_info.Clear(); const VMRangeToPermissions::Entry *permission_entry = m_core_range_infos.FindEntryThatContainsOrFollows(load_addr); diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h index 67df3c5fac76..fd36e5027816 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h @@ -86,10 +86,6 @@ class ProcessElfCore : public lldb_private::PostMortemProcess { size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, lldb_private::Status &error) override; - lldb_private::Status - GetMemoryRegionInfo(lldb::addr_t load_addr, - lldb_private::MemoryRegionInfo ®ion_info) override; - lldb::addr_t GetImageInfoAddress() override; lldb_private::ArchSpec GetArchitecture(); @@ -105,6 +101,10 @@ class ProcessElfCore : public lldb_private::PostMortemProcess { bool DoUpdateThreadList(lldb_private::ThreadList &old_thread_list, lldb_private::ThreadList &new_thread_list) override; + lldb_private::Status + DoGetMemoryRegionInfo(lldb::addr_t load_addr, + lldb_private::MemoryRegionInfo ®ion_info) override; + private: struct NT_FILE_Entry { lldb::addr_t start; diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index 82357cd117d7..fee2071a0780 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -2962,8 +2962,8 @@ lldb::addr_t ProcessGDBRemote::DoAllocateMemory(size_t size, return allocated_addr; } -Status ProcessGDBRemote::GetMemoryRegionInfo(addr_t load_addr, - MemoryRegionInfo ®ion_info) { +Status ProcessGDBRemote::DoGetMemoryRegionInfo(addr_t load_addr, + MemoryRegionInfo ®ion_info) { Status error(m_gdb_comm.GetMemoryRegionInfo(load_addr, region_info)); return error; diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h index bdf130e3ec11..dd907042608d 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h @@ -143,9 +143,6 @@ class ProcessGDBRemote : public Process, lldb::addr_t DoAllocateMemory(size_t size, uint32_t permissions, Status &error) override; - Status GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo ®ion_info) override; - Status DoDeallocateMemory(lldb::addr_t ptr) override; // Process STDIO @@ -415,6 +412,9 @@ class ProcessGDBRemote : public Process, Status DoWriteMemoryTags(lldb::addr_t addr, size_t len, int32_t type, const std::vector &tags) override; + Status DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo ®ion_info) override; + private: // For ProcessGDBRemote only std::string m_partial_profile_data; diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp index 37ee5466c5b9..162697bed7e9 100644 --- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp +++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp @@ -439,8 +439,8 @@ void ProcessMinidump::BuildMemoryRegions() { llvm::sort(*m_memory_regions); } -Status ProcessMinidump::GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo ®ion) { +Status ProcessMinidump::DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo ®ion) { BuildMemoryRegions(); region = MinidumpParser::GetMemoryRegionInfo(*m_memory_regions, load_addr); return Status(); diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.h b/lldb/source/Plugins/Process/minidump/ProcessMinidump.h index 3501d38a0f27..5360269199cd 100644 --- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.h +++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.h @@ -75,9 +75,6 @@ class ProcessMinidump : public PostMortemProcess { ArchSpec GetArchitecture(); - Status GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo &range_info) override; - Status GetMemoryRegions( lldb_private::MemoryRegionInfos ®ion_list) override; @@ -98,6 +95,9 @@ class ProcessMinidump : public PostMortemProcess { bool DoUpdateThreadList(ThreadList &old_thread_list, ThreadList &new_thread_list) override; + Status DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo &range_info) override; + void ReadModuleList(); lldb::ModuleSP GetOrCreateModule(lldb_private::UUID minidump_uuid, diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp index 5eb7cb0e6a5c..c231ea6b0d10 100644 --- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp +++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp @@ -245,8 +245,8 @@ ArchSpec ScriptedProcess::GetArchitecture() { return GetTarget().GetArchitecture(); } -Status ScriptedProcess::GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo ®ion) { +Status ScriptedProcess::DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo ®ion) { CheckInterpreterAndScriptObject(); Status error; diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.h b/lldb/source/Plugins/Process/scripted/ScriptedProcess.h index d56658a2e48a..c8355f35548a 100644 --- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.h +++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.h @@ -84,9 +84,6 @@ class ScriptedProcess : public Process { ArchSpec GetArchitecture(); - Status GetMemoryRegionInfo(lldb::addr_t load_addr, - MemoryRegionInfo &range_info) override; - Status GetMemoryRegions(lldb_private::MemoryRegionInfos ®ion_list) override; @@ -100,6 +97,9 @@ class ScriptedProcess : public Process { bool DoUpdateThreadList(ThreadList &old_thread_list, ThreadList &new_thread_list) override; + Status DoGetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo &range_info) override; + private: friend class ScriptedThread; diff --git a/lldb/source/Symbol/Symtab.cpp b/lldb/source/Symbol/Symtab.cpp index 97dc31bc9766..d148706003ad 100644 --- a/lldb/source/Symbol/Symtab.cpp +++ b/lldb/source/Symbol/Symtab.cpp @@ -328,8 +328,10 @@ void Symtab::InitNameIndexes() { const SymbolType type = symbol->GetType(); if (type == eSymbolTypeCode || type == eSymbolTypeResolver) { - if (mangled.DemangleWithRichManglingInfo(rmc, lldb_skip_name)) + if (mangled.GetRichManglingInfo(rmc, lldb_skip_name)) { RegisterMangledNameEntry(value, class_contexts, backlog, rmc); + continue; + } } } @@ -383,16 +385,13 @@ void Symtab::RegisterMangledNameEntry( std::vector> &backlog, RichManglingContext &rmc) { // Only register functions that have a base name. - rmc.ParseFunctionBaseName(); - llvm::StringRef base_name = rmc.GetBufferRef(); + llvm::StringRef base_name = rmc.ParseFunctionBaseName(); if (base_name.empty()) return; // The base name will be our entry's name. NameToIndexMap::Entry entry(ConstString(base_name), value); - - rmc.ParseFunctionDeclContextName(); - llvm::StringRef decl_context = rmc.GetBufferRef(); + llvm::StringRef decl_context = rmc.ParseFunctionDeclContextName(); // Register functions with no context. if (decl_context.empty()) { diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index 94f378886e50..6a306ab762da 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -5853,12 +5853,18 @@ Process::AdvanceAddressToNextBranchInstruction(Address default_stop_addr, return retval; } -Status -Process::GetMemoryRegions(lldb_private::MemoryRegionInfos ®ion_list) { +Status Process::GetMemoryRegionInfo(lldb::addr_t load_addr, + MemoryRegionInfo &range_info) { + if (const lldb::ABISP &abi = GetABI()) + load_addr = abi->FixDataAddress(load_addr); + return DoGetMemoryRegionInfo(load_addr, range_info); +} +Status Process::GetMemoryRegions(lldb_private::MemoryRegionInfos ®ion_list) { Status error; lldb::addr_t range_end = 0; + const lldb::ABISP &abi = GetABI(); region_list.clear(); do { @@ -5870,11 +5876,22 @@ Process::GetMemoryRegions(lldb_private::MemoryRegionInfos ®ion_list) { break; } + // We only check the end address, not start and end, because we assume that + // the start will not have non-address bits until the first unmappable + // region. We will have exited the loop by that point because the previous + // region, the last mappable region, will have non-address bits in its end + // address. range_end = region_info.GetRange().GetRangeEnd(); if (region_info.GetMapped() == MemoryRegionInfo::eYes) { region_list.push_back(std::move(region_info)); } - } while (range_end != LLDB_INVALID_ADDRESS); + } while ( + // For a process with no non-address bits, all address bits + // set means the end of memory. + range_end != LLDB_INVALID_ADDRESS && + // If we have non-address bits and some are set then the end + // is at or beyond the end of mappable memory. + !(abi && (abi->FixDataAddress(range_end) != range_end))); return error; } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 3861648a5feb..de8b0e9cfd10 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2851,6 +2851,14 @@ class TargetLoweringBase { return false; } + /// Return true if pulling a binary operation into a select with an identity + /// constant is profitable. This is the inverse of an IR transform. + /// Example: X + (Cond ? Y : 0) --> Cond ? (X + Y) : X + virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, + EVT VT) const { + return false; + } + /// Return true if it is beneficial to convert a load of a constant to /// just the constant itself. /// On some targets it might be more efficient to use a combination of diff --git a/llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h b/llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h new file mode 100644 index 000000000000..c5340b5f0460 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h @@ -0,0 +1,51 @@ +//===-- llvm/DebugInfo/Symbolize/DIFetcher.h --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares a DIFetcher abstraction for obtaining debug info from an +/// arbitrary outside source. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H +#define LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H + +#include +#include + +#include "llvm/ADT/ArrayRef.h" + +namespace llvm { +namespace symbolize { + +/// The DIFetcher interface provides arbitrary mechanisms for obtaining debug +/// info from an outside source. +class DIFetcher { +public: + virtual ~DIFetcher() = default; + virtual Optional + fetchBuildID(ArrayRef BuildID) const = 0; +}; + +/// LocalDIFetcher searches local cache directories for debug info. +class LocalDIFetcher : public DIFetcher { +public: + LocalDIFetcher(ArrayRef DebugFileDirectory) + : DebugFileDirectory(DebugFileDirectory){}; + virtual ~LocalDIFetcher() = default; + + Optional fetchBuildID(ArrayRef BuildID) const override; + +private: + const ArrayRef DebugFileDirectory; +}; + +} // end namespace symbolize +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H diff --git a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h index 4ec333422c4b..30e55c5d410a 100644 --- a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h +++ b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -13,6 +13,7 @@ #ifndef LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H #define LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H +#include "llvm/DebugInfo/Symbolize/DIFetcher.h" #include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ELFObjectFile.h" @@ -83,6 +84,10 @@ class LLVMSymbolizer { DemangleName(const std::string &Name, const SymbolizableModule *DbiModuleDescriptor); + void addDIFetcher(std::unique_ptr Fetcher) { + DIFetchers.push_back(std::move(Fetcher)); + } + private: // Bundles together object file with code/data and object file with // corresponding debug info. These objects can be the same. @@ -126,6 +131,12 @@ class LLVMSymbolizer { const ELFObjectFileBase *Obj, const std::string &ArchName); + bool findDebugBinary(const std::string &OrigPath, + const std::string &DebuglinkName, uint32_t CRCHash, + std::string &Result); + + bool findDebugBinary(const ArrayRef BuildID, std::string &Result); + /// Returns pair of pointers to object and debug object. Expected getOrCreateObjectPair(const std::string &Path, const std::string &ArchName); @@ -152,6 +163,8 @@ class LLVMSymbolizer { ObjectForUBPathAndArch; Options Opts; + + SmallVector> DIFetchers; }; } // end namespace symbolize diff --git a/llvm/include/llvm/Debuginfod/DIFetcher.h b/llvm/include/llvm/Debuginfod/DIFetcher.h new file mode 100644 index 000000000000..d398fd900051 --- /dev/null +++ b/llvm/include/llvm/Debuginfod/DIFetcher.h @@ -0,0 +1,34 @@ +//===- llvm/DebugInfod/DIFetcher.h - Debug info fetcher----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares a DIFetcher implementation for obtaining debug info from +/// debuginfod. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFOD_DIFETCHER_H +#define LLVM_DEBUGINFOD_DIFETCHER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/Symbolize/DIFetcher.h" + +namespace llvm { + +class DebuginfodDIFetcher : public symbolize::DIFetcher { +public: + virtual ~DebuginfodDIFetcher() = default; + + /// Fetches the given Build ID using debuginfod and returns a local path to + /// the resulting debug binary. + Optional fetchBuildID(ArrayRef BuildID) const override; +}; + +} // namespace llvm + +#endif // LLVM_DEBUGINFOD_DIFETCHER_H diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h index f3317049524f..80b2dfaec427 100644 --- a/llvm/include/llvm/Support/Compiler.h +++ b/llvm/include/llvm/Support/Compiler.h @@ -90,9 +90,11 @@ #define LLVM_MSC_PREREQ(version) (_MSC_VER >= (version)) // We require at least VS 2019. +#if !defined(LLVM_FORCE_USE_OLD_TOOLCHAIN) #if !LLVM_MSC_PREREQ(1920) #error LLVM requires at least VS 2019. #endif +#endif #else #define LLVM_MSC_PREREQ(version) 0 diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp index 0eb6100230bd..6af5f07d801a 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp +++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp @@ -148,6 +148,20 @@ static cl::opt EmulateOldLDV("emulate-old-livedebugvalues", cl::Hidden, cl::desc("Act like old LiveDebugValues did"), cl::init(false)); +// Limit for the maximum number of stack slots we should track, past which we +// will ignore any spills. InstrRefBasedLDV gathers detailed information on all +// stack slots which leads to high memory consumption, and in some scenarios +// (such as asan with very many locals) the working set of the function can be +// very large, causing many spills. In these scenarios, it is very unlikely that +// the developer has hundreds of variables live at the same time that they're +// carefully thinking about -- instead, they probably autogenerated the code. +// When this happens, gracefully stop tracking excess spill slots, rather than +// consuming all the developer's memory. +static cl::opt + StackWorkingSetLimit("livedebugvalues-max-stack-slots", cl::Hidden, + cl::desc("livedebugvalues-stack-ws-limit"), + cl::init(250)); + /// Tracker for converting machine value locations and variable values into /// variable locations (the output of LiveDebugValues), recorded as DBG_VALUEs /// specifying block live-in locations and transfers within blocks. @@ -757,9 +771,15 @@ void MLocTracker::writeRegMask(const MachineOperand *MO, unsigned CurBB, Masks.push_back(std::make_pair(MO, InstID)); } -SpillLocationNo MLocTracker::getOrTrackSpillLoc(SpillLoc L) { +Optional MLocTracker::getOrTrackSpillLoc(SpillLoc L) { SpillLocationNo SpillID(SpillLocs.idFor(L)); + if (SpillID.id() == 0) { + // If there is no location, and we have reached the limit of how many stack + // slots to track, then don't track this one. + if (SpillLocs.size() >= StackWorkingSetLimit) + return None; + // Spill location is untracked: create record for this one, and all // subregister slots too. SpillID = SpillLocationNo(SpillLocs.insert(L)); @@ -898,7 +918,7 @@ bool InstrRefBasedLDV::isCalleeSaved(LocIdx L) const { // void InstrRefBasedLDV::printVarLocInMBB(..) #endif -SpillLocationNo +Optional InstrRefBasedLDV::extractSpillBaseRegAndOffset(const MachineInstr &MI) { assert(MI.hasOneMemOperand() && "Spill instruction does not have exactly one memory operand?"); @@ -913,8 +933,11 @@ InstrRefBasedLDV::extractSpillBaseRegAndOffset(const MachineInstr &MI) { return MTracker->getOrTrackSpillLoc({Reg, Offset}); } -Optional InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr &MI) { - SpillLocationNo SpillLoc = extractSpillBaseRegAndOffset(MI); +Optional +InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr &MI) { + Optional SpillLoc = extractSpillBaseRegAndOffset(MI); + if (!SpillLoc) + return None; // Where in the stack slot is this value defined -- i.e., what size of value // is this? An important question, because it could be loaded into a register @@ -930,7 +953,7 @@ Optional InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr // occur, but the safe action is to indicate the variable is optimised out. return None; - unsigned SpillID = MTracker->getSpillIDWithIdx(SpillLoc, IdxIt->second); + unsigned SpillID = MTracker->getSpillIDWithIdx(*SpillLoc, IdxIt->second); return MTracker->getSpillMLoc(SpillID); } @@ -1006,7 +1029,7 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI, // Only handle this instruction when we are building the variable value // transfer function. - if (!VTracker) + if (!VTracker && !TTracker) return false; unsigned InstNo = MI.getOperand(0).getImm(); @@ -1162,7 +1185,8 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI, // for DBG_INSTR_REFs as DBG_VALUEs (just, the former can refer to values that // aren't immediately available). DbgValueProperties Properties(Expr, false); - VTracker->defVar(MI, Properties, NewID); + if (VTracker) + VTracker->defVar(MI, Properties, NewID); // If we're on the final pass through the function, decompose this INSTR_REF // into a plain DBG_VALUE. @@ -1251,7 +1275,12 @@ bool InstrRefBasedLDV::transferDebugPHI(MachineInstr &MI) { Register Base; StackOffset Offs = TFI->getFrameIndexReference(*MI.getMF(), FI, Base); SpillLoc SL = {Base, Offs}; - SpillLocationNo SpillNo = MTracker->getOrTrackSpillLoc(SL); + Optional SpillNo = MTracker->getOrTrackSpillLoc(SL); + + // We might be able to find a value, but have chosen not to, to avoid + // tracking too much stack information. + if (!SpillNo) + return true; // Problem: what value should we extract from the stack? LLVM does not // record what size the last store to the slot was, and it would become @@ -1263,7 +1292,7 @@ bool InstrRefBasedLDV::transferDebugPHI(MachineInstr &MI) { Optional Result = None; Optional SpillLoc = None; for (unsigned CS : CandidateSizes) { - unsigned SpillID = MTracker->getLocID(SpillNo, {CS, 0}); + unsigned SpillID = MTracker->getLocID(*SpillNo, {CS, 0}); SpillLoc = MTracker->getSpillMLoc(SpillID); ValueIDNum Val = MTracker->readMLoc(*SpillLoc); // If this value was defined in it's own position, then it was probably @@ -1280,7 +1309,7 @@ bool InstrRefBasedLDV::transferDebugPHI(MachineInstr &MI) { // "supposed" to be is more complex, and benefits a small number of // locations. if (!Result) { - unsigned SpillID = MTracker->getLocID(SpillNo, {64, 0}); + unsigned SpillID = MTracker->getLocID(*SpillNo, {64, 0}); SpillLoc = MTracker->getSpillMLoc(SpillID); Result = MTracker->readMLoc(*SpillLoc); } @@ -1357,11 +1386,12 @@ void InstrRefBasedLDV::transferRegisterDef(MachineInstr &MI) { // If this instruction writes to a spill slot, def that slot. if (hasFoldedStackStore(MI)) { - SpillLocationNo SpillNo = extractSpillBaseRegAndOffset(MI); - for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) { - unsigned SpillID = MTracker->getSpillIDWithIdx(SpillNo, I); - LocIdx L = MTracker->getSpillMLoc(SpillID); - MTracker->setMLoc(L, ValueIDNum(CurBB, CurInst, L)); + if (Optional SpillNo = extractSpillBaseRegAndOffset(MI)) { + for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) { + unsigned SpillID = MTracker->getSpillIDWithIdx(*SpillNo, I); + LocIdx L = MTracker->getSpillMLoc(SpillID); + MTracker->setMLoc(L, ValueIDNum(CurBB, CurInst, L)); + } } } @@ -1398,11 +1428,12 @@ void InstrRefBasedLDV::transferRegisterDef(MachineInstr &MI) { // Tell TTracker about any folded stack store. if (hasFoldedStackStore(MI)) { - SpillLocationNo SpillNo = extractSpillBaseRegAndOffset(MI); - for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) { - unsigned SpillID = MTracker->getSpillIDWithIdx(SpillNo, I); - LocIdx L = MTracker->getSpillMLoc(SpillID); - TTracker->clobberMloc(L, MI.getIterator(), true); + if (Optional SpillNo = extractSpillBaseRegAndOffset(MI)) { + for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) { + unsigned SpillID = MTracker->getSpillIDWithIdx(*SpillNo, I); + LocIdx L = MTracker->getSpillMLoc(SpillID); + TTracker->clobberMloc(L, MI.getIterator(), true); + } } } } @@ -1438,23 +1469,24 @@ void InstrRefBasedLDV::performCopy(Register SrcRegNum, Register DstRegNum) { } } -bool InstrRefBasedLDV::isSpillInstruction(const MachineInstr &MI, - MachineFunction *MF) { +Optional +InstrRefBasedLDV::isSpillInstruction(const MachineInstr &MI, + MachineFunction *MF) { // TODO: Handle multiple stores folded into one. if (!MI.hasOneMemOperand()) - return false; + return None; // Reject any memory operand that's aliased -- we can't guarantee its value. auto MMOI = MI.memoperands_begin(); const PseudoSourceValue *PVal = (*MMOI)->getPseudoValue(); if (PVal->isAliased(MFI)) - return false; + return None; if (!MI.getSpillSize(TII) && !MI.getFoldedSpillSize(TII)) - return false; // This is not a spill instruction, since no valid size was - // returned from either function. + return None; // This is not a spill instruction, since no valid size was + // returned from either function. - return true; + return extractSpillBaseRegAndOffset(MI); } bool InstrRefBasedLDV::isLocationSpill(const MachineInstr &MI, @@ -1511,13 +1543,11 @@ bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) { // First, if there are any DBG_VALUEs pointing at a spill slot that is // written to, terminate that variable location. The value in memory // will have changed. DbgEntityHistoryCalculator doesn't try to detect this. - if (isSpillInstruction(MI, MF)) { - SpillLocationNo Loc = extractSpillBaseRegAndOffset(MI); - + if (Optional Loc = isSpillInstruction(MI, MF)) { // Un-set this location and clobber, so that earlier locations don't // continue past this store. for (unsigned SlotIdx = 0; SlotIdx < MTracker->NumSlotIdxes; ++SlotIdx) { - unsigned SpillID = MTracker->getSpillIDWithIdx(Loc, SlotIdx); + unsigned SpillID = MTracker->getSpillIDWithIdx(*Loc, SlotIdx); Optional MLoc = MTracker->getSpillMLoc(SpillID); if (!MLoc) continue; @@ -1535,7 +1565,9 @@ bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) { // Try to recognise spill and restore instructions that may transfer a value. if (isLocationSpill(MI, MF, Reg)) { - SpillLocationNo Loc = extractSpillBaseRegAndOffset(MI); + // isLocationSpill returning true should guarantee we can extract a + // location. + SpillLocationNo Loc = *extractSpillBaseRegAndOffset(MI); auto DoTransfer = [&](Register SrcReg, unsigned SpillID) { auto ReadValue = MTracker->readReg(SrcReg); @@ -1562,10 +1594,9 @@ bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) { unsigned SpillID = MTracker->getLocID(Loc, {Size, 0}); DoTransfer(Reg, SpillID); } else { - Optional OptLoc = isRestoreInstruction(MI, MF, Reg); - if (!OptLoc) + Optional Loc = isRestoreInstruction(MI, MF, Reg); + if (!Loc) return false; - SpillLocationNo Loc = *OptLoc; // Assumption: we're reading from the base of the stack slot, not some // offset into it. It seems very unlikely LLVM would ever generate @@ -1592,13 +1623,13 @@ bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) { for (MCSubRegIterator SRI(Reg, TRI, false); SRI.isValid(); ++SRI) { unsigned Subreg = TRI->getSubRegIndex(Reg, *SRI); - unsigned SpillID = MTracker->getLocID(Loc, Subreg); + unsigned SpillID = MTracker->getLocID(*Loc, Subreg); DoTransfer(*SRI, SpillID); } // Directly look up this registers slot idx by size, and transfer. unsigned Size = TRI->getRegSizeInBits(Reg, *MRI); - unsigned SpillID = MTracker->getLocID(Loc, {Size, 0}); + unsigned SpillID = MTracker->getLocID(*Loc, {Size, 0}); DoTransfer(Reg, SpillID); } return true; @@ -2765,6 +2796,11 @@ void InstrRefBasedLDV::placePHIsForSingleVarDefinition( auto ValueIt = VLocs.Vars.find(Var); const DbgValue &Value = ValueIt->second; + // If it's an explicit assignment of "undef", that means there is no location + // anyway, anywhere. + if (Value.Kind == DbgValue::Undef) + return; + // Assign the variable value to entry to each dominated block that's in scope. // Skip the definition block -- it's assigned the variable value in the middle // of the block somewhere. @@ -2790,35 +2826,6 @@ void InstrRefBasedLDV::dump_mloc_transfer( } #endif -void InstrRefBasedLDV::emitLocations( - MachineFunction &MF, LiveInsT SavedLiveIns, ValueIDNum **MOutLocs, - ValueIDNum **MInLocs, DenseMap &AllVarsNumbering, - const TargetPassConfig &TPC) { - TTracker = new TransferTracker(TII, MTracker, MF, *TRI, CalleeSavedRegs, TPC); - unsigned NumLocs = MTracker->getNumLocs(); - - // For each block, load in the machine value locations and variable value - // live-ins, then step through each instruction in the block. New DBG_VALUEs - // to be inserted will be created along the way. - for (MachineBasicBlock &MBB : MF) { - unsigned bbnum = MBB.getNumber(); - MTracker->reset(); - MTracker->loadFromArray(MInLocs[bbnum], bbnum); - TTracker->loadInlocs(MBB, MInLocs[bbnum], SavedLiveIns[MBB.getNumber()], - NumLocs); - - CurBB = bbnum; - CurInst = 1; - for (auto &MI : MBB) { - process(MI, MOutLocs, MInLocs); - TTracker->checkInstForNewValues(CurInst, MI.getIterator()); - ++CurInst; - } - } - - emitTransfers(AllVarsNumbering); -} - void InstrRefBasedLDV::initialSetup(MachineFunction &MF) { // Build some useful data structures. @@ -2861,8 +2868,192 @@ void InstrRefBasedLDV::initialSetup(MachineFunction &MF) { #endif } +// Produce an "ejection map" for blocks, i.e., what's the highest-numbered +// lexical scope it's used in. When exploring in DFS order and we pass that +// scope, the block can be processed and any tracking information freed. +void InstrRefBasedLDV::makeDepthFirstEjectionMap( + SmallVectorImpl &EjectionMap, + const ScopeToDILocT &ScopeToDILocation, + ScopeToAssignBlocksT &ScopeToAssignBlocks) { + SmallPtrSet BlocksToExplore; + SmallVector, 4> WorkStack; + auto *TopScope = LS.getCurrentFunctionScope(); + + // Unlike lexical scope explorers, we explore in reverse order, to find the + // "last" lexical scope used for each block early. + WorkStack.push_back({TopScope, TopScope->getChildren().size() - 1}); + + while (!WorkStack.empty()) { + auto &ScopePosition = WorkStack.back(); + LexicalScope *WS = ScopePosition.first; + ssize_t ChildNum = ScopePosition.second--; + + const SmallVectorImpl &Children = WS->getChildren(); + if (ChildNum >= 0) { + // If ChildNum is positive, there are remaining children to explore. + // Push the child and its children-count onto the stack. + auto &ChildScope = Children[ChildNum]; + WorkStack.push_back( + std::make_pair(ChildScope, ChildScope->getChildren().size() - 1)); + } else { + WorkStack.pop_back(); + + // We've explored all children and any later blocks: examine all blocks + // in our scope. If they haven't yet had an ejection number set, then + // this scope will be the last to use that block. + auto DILocationIt = ScopeToDILocation.find(WS); + if (DILocationIt != ScopeToDILocation.end()) { + getBlocksForScope(DILocationIt->second, BlocksToExplore, + ScopeToAssignBlocks.find(WS)->second); + for (auto *MBB : BlocksToExplore) { + unsigned BBNum = MBB->getNumber(); + if (EjectionMap[BBNum] == 0) + EjectionMap[BBNum] = WS->getDFSOut(); + } + + BlocksToExplore.clear(); + } + } + } +} + +bool InstrRefBasedLDV::depthFirstVLocAndEmit( + unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation, + const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToAssignBlocks, + LiveInsT &Output, ValueIDNum **MOutLocs, ValueIDNum **MInLocs, + SmallVectorImpl &AllTheVLocs, MachineFunction &MF, + DenseMap &AllVarsNumbering, + const TargetPassConfig &TPC) { + TTracker = new TransferTracker(TII, MTracker, MF, *TRI, CalleeSavedRegs, TPC); + unsigned NumLocs = MTracker->getNumLocs(); + VTracker = nullptr; + + // No scopes? No variable locations. + if (!LS.getCurrentFunctionScope()) { + // FIXME: this is a sticking plaster to prevent a memory leak, these + // pointers will be automagically freed by being unique pointers, shortly. + for (unsigned int I = 0; I < MaxNumBlocks; ++I) { + delete[] MInLocs[I]; + delete[] MOutLocs[I]; + } + return false; + } + + // Build map from block number to the last scope that uses the block. + SmallVector EjectionMap; + EjectionMap.resize(MaxNumBlocks, 0); + makeDepthFirstEjectionMap(EjectionMap, ScopeToDILocation, + ScopeToAssignBlocks); + + // Helper lambda for ejecting a block -- if nothing is going to use the block, + // we can translate the variable location information into DBG_VALUEs and then + // free all of InstrRefBasedLDV's data structures. + auto EjectBlock = [&](MachineBasicBlock &MBB) -> void { + unsigned BBNum = MBB.getNumber(); + AllTheVLocs[BBNum].clear(); + + // Prime the transfer-tracker, and then step through all the block + // instructions, installing transfers. + MTracker->reset(); + MTracker->loadFromArray(MInLocs[BBNum], BBNum); + TTracker->loadInlocs(MBB, MInLocs[BBNum], Output[BBNum], NumLocs); + + CurBB = BBNum; + CurInst = 1; + for (auto &MI : MBB) { + process(MI, MOutLocs, MInLocs); + TTracker->checkInstForNewValues(CurInst, MI.getIterator()); + ++CurInst; + } + + // Free machine-location tables for this block. + delete[] MInLocs[BBNum]; + delete[] MOutLocs[BBNum]; + // Make ourselves brittle to use-after-free errors. + MInLocs[BBNum] = nullptr; + MOutLocs[BBNum] = nullptr; + // We don't need live-in variable values for this block either. + Output[BBNum].clear(); + AllTheVLocs[BBNum].clear(); + }; + + SmallPtrSet BlocksToExplore; + SmallVector, 4> WorkStack; + WorkStack.push_back({LS.getCurrentFunctionScope(), 0}); + unsigned HighestDFSIn = 0; + + // Proceed to explore in depth first order. + while (!WorkStack.empty()) { + auto &ScopePosition = WorkStack.back(); + LexicalScope *WS = ScopePosition.first; + ssize_t ChildNum = ScopePosition.second++; + + // We obesrve scopes with children twice here, once descending in, once + // ascending out of the scope nest. Use HighestDFSIn as a ratchet to ensure + // we don't process a scope twice. Additionally, ignore scopes that don't + // have a DILocation -- by proxy, this means we never tracked any variable + // assignments in that scope. + auto DILocIt = ScopeToDILocation.find(WS); + if (HighestDFSIn <= WS->getDFSIn() && DILocIt != ScopeToDILocation.end()) { + const DILocation *DILoc = DILocIt->second; + auto &VarsWeCareAbout = ScopeToVars.find(WS)->second; + auto &BlocksInScope = ScopeToAssignBlocks.find(WS)->second; + + buildVLocValueMap(DILoc, VarsWeCareAbout, BlocksInScope, Output, MOutLocs, + MInLocs, AllTheVLocs); + } + + HighestDFSIn = std::max(HighestDFSIn, WS->getDFSIn()); + + // Descend into any scope nests. + const SmallVectorImpl &Children = WS->getChildren(); + if (ChildNum < (ssize_t)Children.size()) { + // There are children to explore -- push onto stack and continue. + auto &ChildScope = Children[ChildNum]; + WorkStack.push_back(std::make_pair(ChildScope, 0)); + } else { + WorkStack.pop_back(); + + // We've explored a leaf, or have explored all the children of a scope. + // Try to eject any blocks where this is the last scope it's relevant to. + auto DILocationIt = ScopeToDILocation.find(WS); + if (DILocationIt == ScopeToDILocation.end()) + continue; + + getBlocksForScope(DILocationIt->second, BlocksToExplore, + ScopeToAssignBlocks.find(WS)->second); + for (auto *MBB : BlocksToExplore) + if (WS->getDFSOut() == EjectionMap[MBB->getNumber()]) + EjectBlock(const_cast(*MBB)); + + BlocksToExplore.clear(); + } + } + + // Some artificial blocks may not have been ejected, meaning they're not + // connected to an actual legitimate scope. This can technically happen + // with things like the entry block. In theory, we shouldn't need to do + // anything for such out-of-scope blocks, but for the sake of being similar + // to VarLocBasedLDV, eject these too. + for (auto *MBB : ArtificialBlocks) + if (MOutLocs[MBB->getNumber()]) + EjectBlock(*MBB); + + // Finally, there might have been gaps in the block numbering, from dead + // blocks being deleted or folded. In those scenarios, we might allocate a + // block-table that's never ejected, meaning we have to free it at the end. + for (unsigned int I = 0; I < MaxNumBlocks; ++I) { + if (MInLocs[I]) { + delete[] MInLocs[I]; + delete[] MOutLocs[I]; + } + } + + return emitTransfers(AllVarsNumbering); +} + bool InstrRefBasedLDV::emitTransfers( - DenseMap &AllVarsNumbering) { + DenseMap &AllVarsNumbering) { // Go through all the transfers recorded in the TransferTracker -- this is // both the live-ins to a block, and any movements of values that happen // in the middle. @@ -3050,31 +3241,22 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, << " has " << MaxNumBlocks << " basic blocks and " << VarAssignCount << " variable assignments, exceeding limits.\n"); - } else { - // Compute the extended ranges, iterating over scopes. There might be - // something to be said for ordering them by size/locality, but that's for - // the future. For each scope, solve the variable value problem, producing - // a map of variables to values in SavedLiveIns. - for (auto &P : ScopeToVars) { - buildVLocValueMap(ScopeToDILocation[P.first], P.second, - ScopeToAssignBlocks[P.first], SavedLiveIns, MOutLocs, MInLocs, - vlocs); + + // Perform memory cleanup that emitLocations would do otherwise. + for (int Idx = 0; Idx < MaxNumBlocks; ++Idx) { + delete[] MOutLocs[Idx]; + delete[] MInLocs[Idx]; } - - // Using the computed value locations and variable values for each block, - // create the DBG_VALUE instructions representing the extended variable - // locations. - emitLocations(MF, SavedLiveIns, MOutLocs, MInLocs, AllVarsNumbering, *TPC); - - // Did we actually make any changes? If we created any DBG_VALUEs, then yes. - Changed = TTracker->Transfers.size() != 0; + } else { + // Optionally, solve the variable value problem and emit to blocks by using + // a lexical-scope-depth search. It should be functionally identical to + // the "else" block of this condition. + Changed = depthFirstVLocAndEmit( + MaxNumBlocks, ScopeToDILocation, ScopeToVars, ScopeToAssignBlocks, + SavedLiveIns, MOutLocs, MInLocs, vlocs, MF, AllVarsNumbering, *TPC); } - // Common clean-up of memory. - for (int Idx = 0; Idx < MaxNumBlocks; ++Idx) { - delete[] MOutLocs[Idx]; - delete[] MInLocs[Idx]; - } + // Elements of these arrays will be deleted by emitLocations. delete[] MOutLocs; delete[] MInLocs; @@ -3092,6 +3274,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, DebugPHINumToValue.clear(); OverlapFragments.clear(); SeenFragments.clear(); + SeenDbgPHIs.clear(); return Changed; } @@ -3357,6 +3540,21 @@ Optional InstrRefBasedLDV::resolveDbgPHIs(MachineFunction &MF, ValueIDNum **MLiveIns, MachineInstr &Here, uint64_t InstrNum) { + // This function will be called twice per DBG_INSTR_REF, and might end up + // computing lots of SSA information: memoize it. + auto SeenDbgPHIIt = SeenDbgPHIs.find(&Here); + if (SeenDbgPHIIt != SeenDbgPHIs.end()) + return SeenDbgPHIIt->second; + + Optional Result = + resolveDbgPHIsImpl(MF, MLiveOuts, MLiveIns, Here, InstrNum); + SeenDbgPHIs.insert({&Here, Result}); + return Result; +} + +Optional InstrRefBasedLDV::resolveDbgPHIsImpl( + MachineFunction &MF, ValueIDNum **MLiveOuts, ValueIDNum **MLiveIns, + MachineInstr &Here, uint64_t InstrNum) { // Pick out records of DBG_PHI instructions that have been observed. If there // are none, then we cannot compute a value number. auto RangePair = std::equal_range(DebugPHINumToValue.begin(), diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h index e7383209c027..d778561db471 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h +++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h @@ -616,7 +616,9 @@ class MLocTracker { void writeRegMask(const MachineOperand *MO, unsigned CurBB, unsigned InstID); /// Find LocIdx for SpillLoc \p L, creating a new one if it's not tracked. - SpillLocationNo getOrTrackSpillLoc(SpillLoc L); + /// Returns None when in scenarios where a spill slot could be tracked, but + /// we would likely run into resource limitations. + Optional getOrTrackSpillLoc(SpillLoc L); // Get LocIdx of a spill ID. LocIdx getSpillMLoc(unsigned SpillID) { @@ -678,7 +680,7 @@ class VLocTracker { /// movement of values between locations inside of a block is handled at a /// much later stage, in the TransferTracker class. MapVector Vars; - DenseMap Scopes; + SmallDenseMap Scopes; MachineBasicBlock *MBB = nullptr; const OverlapMap &OverlappingFragments; DbgValueProperties EmptyProperties; @@ -747,6 +749,11 @@ class VLocTracker { Scopes[Overlapped] = Loc; } } + + void clear() { + Vars.clear(); + Scopes.clear(); + } }; // XXX XXX docs @@ -862,6 +869,12 @@ class InstrRefBasedLDV : public LDVImpl { OverlapMap OverlapFragments; VarToFragments SeenFragments; + /// Mapping of DBG_INSTR_REF instructions to their values, for those + /// DBG_INSTR_REFs that call resolveDbgPHIs. These variable references solve + /// a mini SSA problem caused by DBG_PHIs being cloned, this collection caches + /// the result. + DenseMap> SeenDbgPHIs; + /// True if we need to examine call instructions for stack clobbers. We /// normally assume that they don't clobber SP, but stack probes on Windows /// do. @@ -873,7 +886,8 @@ class InstrRefBasedLDV : public LDVImpl { StringRef StackProbeSymbolName; /// Tests whether this instruction is a spill to a stack slot. - bool isSpillInstruction(const MachineInstr &MI, MachineFunction *MF); + Optional isSpillInstruction(const MachineInstr &MI, + MachineFunction *MF); /// Decide if @MI is a spill instruction and return true if it is. We use 2 /// criteria to make this decision: @@ -891,7 +905,8 @@ class InstrRefBasedLDV : public LDVImpl { /// Given a spill instruction, extract the spill slot information, ensure it's /// tracked, and return the spill number. - SpillLocationNo extractSpillBaseRegAndOffset(const MachineInstr &MI); + Optional + extractSpillBaseRegAndOffset(const MachineInstr &MI); /// Observe a single instruction while stepping through a block. void process(MachineInstr &MI, ValueIDNum **MLiveOuts = nullptr, @@ -940,6 +955,12 @@ class InstrRefBasedLDV : public LDVImpl { ValueIDNum **MLiveIns, MachineInstr &Here, uint64_t InstrNum); + Optional resolveDbgPHIsImpl(MachineFunction &MF, + ValueIDNum **MLiveOuts, + ValueIDNum **MLiveIns, + MachineInstr &Here, + uint64_t InstrNum); + /// Step through the function, recording register definitions and movements /// in an MLocTracker. Convert the observations into a per-block transfer /// function in \p MLocTransfer, suitable for using with the machine value @@ -1050,18 +1071,6 @@ class InstrRefBasedLDV : public LDVImpl { const LiveIdxT &LiveOuts, ValueIDNum **MOutLocs, const SmallVectorImpl &BlockOrders); - /// Given the solutions to the two dataflow problems, machine value locations - /// in \p MInLocs and live-in variable values in \p SavedLiveIns, runs the - /// TransferTracker class over the function to produce live-in and transfer - /// DBG_VALUEs, then inserts them. Groups of DBG_VALUEs are inserted in the - /// order given by AllVarsNumbering -- this could be any stable order, but - /// right now "order of appearence in function, when explored in RPO", so - /// that we can compare explictly against VarLocBasedImpl. - void emitLocations(MachineFunction &MF, LiveInsT SavedLiveIns, - ValueIDNum **MOutLocs, ValueIDNum **MInLocs, - DenseMap &AllVarsNumbering, - const TargetPassConfig &TPC); - /// Take collections of DBG_VALUE instructions stored in TTracker, and /// install them into their output blocks. Preserves a stable order of /// DBG_VALUEs produced (which would otherwise cause nondeterminism) through @@ -1072,6 +1081,28 @@ class InstrRefBasedLDV : public LDVImpl { /// RPOT block ordering. void initialSetup(MachineFunction &MF); + /// Produce a map of the last lexical scope that uses a block, using the + /// scopes DFSOut number. Mapping is block-number to DFSOut. + /// \p EjectionMap Pre-allocated vector in which to install the built ma. + /// \p ScopeToDILocation Mapping of LexicalScopes to their DILocations. + /// \p AssignBlocks Map of blocks where assignments happen for a scope. + void makeDepthFirstEjectionMap(SmallVectorImpl &EjectionMap, + const ScopeToDILocT &ScopeToDILocation, + ScopeToAssignBlocksT &AssignBlocks); + + /// When determining per-block variable values and emitting to DBG_VALUEs, + /// this function explores by lexical scope depth. Doing so means that per + /// block information can be fully computed before exploration finishes, + /// allowing us to emit it and free data structures earlier than otherwise. + /// It's also good for locality. + bool depthFirstVLocAndEmit( + unsigned MaxNumBlocks, const ScopeToDILocT &ScopeToDILocation, + const ScopeToVarsT &ScopeToVars, ScopeToAssignBlocksT &ScopeToBlocks, + LiveInsT &Output, ValueIDNum **MOutLocs, ValueIDNum **MInLocs, + SmallVectorImpl &AllTheVLocs, MachineFunction &MF, + DenseMap &AllVarsNumbering, + const TargetPassConfig &TPC); + bool ExtendRanges(MachineFunction &MF, MachineDominatorTree *DomTree, TargetPassConfig *TPC, unsigned InputBBLimit, unsigned InputDbgValLimit) override; diff --git a/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp b/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp index 8f697611a82c..40770b15aa35 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp +++ b/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp @@ -123,6 +123,11 @@ bool LiveDebugValues::runOnMachineFunction(MachineFunction &MF) { } bool llvm::debuginfoShouldUseDebugInstrRef(const Triple &T) { + // Enable by default on x86_64, disable if explicitly turned off on cmdline. + if (T.getArch() == llvm::Triple::x86_64 && + ValueTrackingVariableLocations != cl::boolOrDefault::BOU_FALSE) + return true; + // Enable if explicitly requested on command line. return ValueTrackingVariableLocations == cl::boolOrDefault::BOU_TRUE; } diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 041d7e5b4a4a..ec297579090e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2101,10 +2101,80 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, VT.getTypeForEVT(*DAG.getContext()), AS); } +/// This inverts a canonicalization in IR that replaces a variable select arm +/// with an identity constant. Codegen improves if we re-use the variable +/// operand rather than load a constant. This can also be converted into a +/// masked vector operation if the target supports it. +static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG, + bool ShouldCommuteOperands) { + // Match a select as operand 1. The identity constant that we are looking for + // is only valid as operand 1 of a non-commutative binop. + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + if (ShouldCommuteOperands) + std::swap(N0, N1); + + // TODO: Should this apply to scalar select too? + if (!N1.hasOneUse() || N1.getOpcode() != ISD::VSELECT) + return SDValue(); + + unsigned Opcode = N->getOpcode(); + EVT VT = N->getValueType(0); + SDValue Cond = N1.getOperand(0); + SDValue TVal = N1.getOperand(1); + SDValue FVal = N1.getOperand(2); + + // TODO: The cases should match with IR's ConstantExpr::getBinOpIdentity(). + // TODO: Target-specific opcodes could be added. Ex: "isCommutativeBinOp()". + // TODO: With fast-math (NSZ), allow the opposite-sign form of zero? + auto isIdentityConstantForOpcode = [](unsigned Opcode, SDValue V) { + if (ConstantFPSDNode *C = isConstOrConstSplatFP(V)) { + switch (Opcode) { + case ISD::FADD: // X + -0.0 --> X + return C->isZero() && C->isNegative(); + case ISD::FSUB: // X - 0.0 --> X + return C->isZero() && !C->isNegative(); + case ISD::FMUL: // X * 1.0 --> X + case ISD::FDIV: // X / 1.0 --> X + return C->isExactlyValue(1.0); + } + } + return false; + }; + + // This transform increases uses of N0, so freeze it to be safe. + // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal) + if (isIdentityConstantForOpcode(Opcode, TVal)) { + SDValue F0 = DAG.getFreeze(N0); + SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags()); + return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO); + } + // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0 + if (isIdentityConstantForOpcode(Opcode, FVal)) { + SDValue F0 = DAG.getFreeze(N0); + SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags()); + return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0); + } + + return SDValue(); +} + SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) { assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && "Unexpected binary operator"); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + auto BinOpcode = BO->getOpcode(); + EVT VT = BO->getValueType(0); + if (TLI.shouldFoldSelectWithIdentityConstant(BinOpcode, VT)) { + if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, false)) + return Sel; + + if (TLI.isCommutativeBinOp(BO->getOpcode())) + if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, true)) + return Sel; + } + // Don't do this unless the old select is going away. We want to eliminate the // binary operator, not replace a binop with a select. // TODO: Handle ISD::SELECT_CC. @@ -2133,7 +2203,6 @@ SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) { // propagate non constant operands into select. I.e.: // and (select Cond, 0, -1), X --> select Cond, 0, X // or X, (select Cond, -1, 0) --> select Cond, -1, X - auto BinOpcode = BO->getOpcode(); bool CanFoldNonConst = (BinOpcode == ISD::AND || BinOpcode == ISD::OR) && (isNullOrNullSplat(CT) || isAllOnesOrAllOnesSplat(CT)) && @@ -2145,8 +2214,6 @@ SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) { !DAG.isConstantFPBuildVectorOrConstantFP(CBO)) return SDValue(); - EVT VT = BO->getValueType(0); - // We have a select-of-constants followed by a binary operator with a // constant. Eliminate the binop by pulling the constant math into the select. // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO diff --git a/llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp b/llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp new file mode 100644 index 000000000000..0493fcd3cbc5 --- /dev/null +++ b/llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp @@ -0,0 +1,58 @@ +//===-- lib/DebugInfo/Symbolize/DIFetcher.cpp -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines the implementation of the local debug info fetcher, which +/// searches cache directories. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/Symbolize/DIFetcher.h" + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +namespace llvm { +namespace symbolize { + +Optional +LocalDIFetcher::fetchBuildID(ArrayRef BuildID) const { + auto GetDebugPath = [&](StringRef Directory) { + SmallString<128> Path{Directory}; + sys::path::append(Path, ".build-id", + llvm::toHex(BuildID[0], /*LowerCase=*/true), + llvm::toHex(BuildID.slice(1), /*LowerCase=*/true)); + Path += ".debug"; + return Path; + }; + if (DebugFileDirectory.empty()) { + SmallString<128> Path = GetDebugPath( +#if defined(__NetBSD__) + // Try /usr/libdata/debug/.build-id/../... + "/usr/libdata/debug" +#else + // Try /usr/lib/debug/.build-id/../... + "/usr/lib/debug" +#endif + ); + if (llvm::sys::fs::exists(Path)) + return std::string(Path); + } else { + for (const auto &Directory : DebugFileDirectory) { + // Try /.build-id/../... + SmallString<128> Path = GetDebugPath(Directory); + if (llvm::sys::fs::exists(Path)) + return std::string(Path); + } + } + return None; +} + +} // namespace symbolize +} // namespace llvm diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp index 5ec79df17fed..f085cc63a8e3 100644 --- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp +++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp @@ -20,7 +20,7 @@ #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/PDB/PDB.h" #include "llvm/DebugInfo/PDB/PDBContext.h" -#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/DebugInfo/Symbolize/DIFetcher.h" #include "llvm/Demangle/Demangle.h" #include "llvm/Object/COFF.h" #include "llvm/Object/MachO.h" @@ -230,51 +230,6 @@ bool checkFileCRC(StringRef Path, uint32_t CRCHash) { return CRCHash == llvm::crc32(arrayRefFromStringRef(MB.get()->getBuffer())); } -bool findDebugBinary(const std::string &OrigPath, - const std::string &DebuglinkName, uint32_t CRCHash, - const std::string &FallbackDebugPath, - std::string &Result) { - SmallString<16> OrigDir(OrigPath); - llvm::sys::path::remove_filename(OrigDir); - SmallString<16> DebugPath = OrigDir; - // Try relative/path/to/original_binary/debuglink_name - llvm::sys::path::append(DebugPath, DebuglinkName); - if (checkFileCRC(DebugPath, CRCHash)) { - Result = std::string(DebugPath.str()); - return true; - } - // Try relative/path/to/original_binary/.debug/debuglink_name - DebugPath = OrigDir; - llvm::sys::path::append(DebugPath, ".debug", DebuglinkName); - if (checkFileCRC(DebugPath, CRCHash)) { - Result = std::string(DebugPath.str()); - return true; - } - // Make the path absolute so that lookups will go to - // "/usr/lib/debug/full/path/to/debug", not - // "/usr/lib/debug/to/debug" - llvm::sys::fs::make_absolute(OrigDir); - if (!FallbackDebugPath.empty()) { - // Try /absolute/path/to/original_binary/debuglink_name - DebugPath = FallbackDebugPath; - } else { -#if defined(__NetBSD__) - // Try /usr/libdata/debug/absolute/path/to/original_binary/debuglink_name - DebugPath = "/usr/libdata/debug"; -#else - // Try /usr/lib/debug/absolute/path/to/original_binary/debuglink_name - DebugPath = "/usr/lib/debug"; -#endif - } - llvm::sys::path::append(DebugPath, llvm::sys::path::relative_path(OrigDir), - DebuglinkName); - if (checkFileCRC(DebugPath, CRCHash)) { - Result = std::string(DebugPath.str()); - return true; - } - return false; -} - bool getGNUDebuglinkContents(const ObjectFile *Obj, std::string &DebugName, uint32_t &CRCHash) { if (!Obj) @@ -351,50 +306,6 @@ Optional> getBuildID(const ELFObjectFileBase *Obj) { return BuildID; } -bool findDebugBinary(const std::vector &DebugFileDirectory, - const ArrayRef BuildID, std::string &Result) { - auto getDebugPath = [&](StringRef Directory) { - SmallString<128> Path{Directory}; - sys::path::append(Path, ".build-id", - llvm::toHex(BuildID[0], /*LowerCase=*/true), - llvm::toHex(BuildID.slice(1), /*LowerCase=*/true)); - Path += ".debug"; - return Path; - }; - if (DebugFileDirectory.empty()) { - SmallString<128> Path = getDebugPath( -#if defined(__NetBSD__) - // Try /usr/libdata/debug/.build-id/../... - "/usr/libdata/debug" -#else - // Try /usr/lib/debug/.build-id/../... - "/usr/lib/debug" -#endif - ); - if (llvm::sys::fs::exists(Path)) { - Result = std::string(Path.str()); - return true; - } - } else { - for (const auto &Directory : DebugFileDirectory) { - // Try /.build-id/../... - SmallString<128> Path = getDebugPath(Directory); - if (llvm::sys::fs::exists(Path)) { - Result = std::string(Path.str()); - return true; - } - } - } - // Try debuginfod client cache and known servers. - Expected PathOrErr = getCachedOrDownloadDebuginfo(BuildID); - if (!PathOrErr) { - consumeError(PathOrErr.takeError()); - return false; - } - Result = *PathOrErr; - return true; -} - } // end anonymous namespace ObjectFile *LLVMSymbolizer::lookUpDsymFile(const std::string &ExePath, @@ -437,8 +348,7 @@ ObjectFile *LLVMSymbolizer::lookUpDebuglinkObject(const std::string &Path, std::string DebugBinaryPath; if (!getGNUDebuglinkContents(Obj, DebuglinkName, CRCHash)) return nullptr; - if (!findDebugBinary(Path, DebuglinkName, CRCHash, Opts.FallbackDebugPath, - DebugBinaryPath)) + if (!findDebugBinary(Path, DebuglinkName, CRCHash, DebugBinaryPath)) return nullptr; auto DbgObjOrErr = getOrCreateObject(DebugBinaryPath, ArchName); if (!DbgObjOrErr) { @@ -458,7 +368,7 @@ ObjectFile *LLVMSymbolizer::lookUpBuildIDObject(const std::string &Path, if (BuildID->size() < 2) return nullptr; std::string DebugBinaryPath; - if (!findDebugBinary(Opts.DebugFileDirectory, *BuildID, DebugBinaryPath)) + if (!findDebugBinary(*BuildID, DebugBinaryPath)) return nullptr; auto DbgObjOrErr = getOrCreateObject(DebugBinaryPath, ArchName); if (!DbgObjOrErr) { @@ -468,6 +378,71 @@ ObjectFile *LLVMSymbolizer::lookUpBuildIDObject(const std::string &Path, return DbgObjOrErr.get(); } +bool LLVMSymbolizer::findDebugBinary(const std::string &OrigPath, + const std::string &DebuglinkName, + uint32_t CRCHash, std::string &Result) { + SmallString<16> OrigDir(OrigPath); + llvm::sys::path::remove_filename(OrigDir); + SmallString<16> DebugPath = OrigDir; + // Try relative/path/to/original_binary/debuglink_name + llvm::sys::path::append(DebugPath, DebuglinkName); + if (checkFileCRC(DebugPath, CRCHash)) { + Result = std::string(DebugPath.str()); + return true; + } + // Try relative/path/to/original_binary/.debug/debuglink_name + DebugPath = OrigDir; + llvm::sys::path::append(DebugPath, ".debug", DebuglinkName); + if (checkFileCRC(DebugPath, CRCHash)) { + Result = std::string(DebugPath.str()); + return true; + } + // Make the path absolute so that lookups will go to + // "/usr/lib/debug/full/path/to/debug", not + // "/usr/lib/debug/to/debug" + llvm::sys::fs::make_absolute(OrigDir); + if (!Opts.FallbackDebugPath.empty()) { + // Try /absolute/path/to/original_binary/debuglink_name + DebugPath = Opts.FallbackDebugPath; + } else { +#if defined(__NetBSD__) + // Try /usr/libdata/debug/absolute/path/to/original_binary/debuglink_name + DebugPath = "/usr/libdata/debug"; +#else + // Try /usr/lib/debug/absolute/path/to/original_binary/debuglink_name + DebugPath = "/usr/lib/debug"; +#endif + } + llvm::sys::path::append(DebugPath, llvm::sys::path::relative_path(OrigDir), + DebuglinkName); + if (checkFileCRC(DebugPath, CRCHash)) { + Result = std::string(DebugPath.str()); + return true; + } + return false; +} + +bool LLVMSymbolizer::findDebugBinary(const ArrayRef BuildID, + std::string &Result) { + Optional Path; + Path = LocalDIFetcher(Opts.DebugFileDirectory).fetchBuildID(BuildID); + if (Path) { + Result = std::move(*Path); + return true; + } + + // Try caller-provided debug info fetchers. + for (const std::unique_ptr &Fetcher : DIFetchers) { + Path = Fetcher->fetchBuildID(BuildID); + if (Path) { + Result = std::move(*Path); + return true; + } + } + + return false; +} + Expected LLVMSymbolizer::getOrCreateObjectPair(const std::string &Path, const std::string &ArchName) { diff --git a/llvm/lib/Debuginfod/DIFetcher.cpp b/llvm/lib/Debuginfod/DIFetcher.cpp new file mode 100644 index 000000000000..f0c134654534 --- /dev/null +++ b/llvm/lib/Debuginfod/DIFetcher.cpp @@ -0,0 +1,28 @@ +//===- llvm/DebugInfod/DIFetcher.cpp - Debug info fetcher -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines a DIFetcher implementation for obtaining debug info +/// from debuginfod. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/DIFetcher.h" + +#include "llvm/Debuginfod/Debuginfod.h" + +using namespace llvm; + +Optional +DebuginfodDIFetcher::fetchBuildID(ArrayRef BuildID) const { + Expected PathOrErr = getCachedOrDownloadDebuginfo(BuildID); + if (PathOrErr) + return *PathOrErr; + consumeError(PathOrErr.takeError()); + return None; +} diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 2b3395b669b8..18de7dcd08f3 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -51,6 +51,11 @@ static const RISCVSupportedExtension SupportedExtensions[] = { {"zfhmin", RISCVExtensionVersion{1, 0}}, {"zfh", RISCVExtensionVersion{1, 0}}, + {"zfinx", RISCVExtensionVersion{1, 0}}, + {"zdinx", RISCVExtensionVersion{1, 0}}, + {"zhinxmin", RISCVExtensionVersion{1, 0}}, + {"zhinx", RISCVExtensionVersion{1, 0}}, + {"zba", RISCVExtensionVersion{1, 0}}, {"zbb", RISCVExtensionVersion{1, 0}}, {"zbc", RISCVExtensionVersion{1, 0}}, @@ -686,6 +691,8 @@ Error RISCVISAInfo::checkDependency() { bool HasE = Exts.count("e") != 0; bool HasD = Exts.count("d") != 0; bool HasF = Exts.count("f") != 0; + bool HasZfinx = Exts.count("zfinx") != 0; + bool HasZdinx = Exts.count("zdinx") != 0; bool HasZve32x = Exts.count("zve32x") != 0; bool HasZve32f = Exts.count("zve32f") != 0; bool HasZve64d = Exts.count("zve64d") != 0; @@ -706,17 +713,15 @@ Error RISCVISAInfo::checkDependency() { return createStringError(errc::invalid_argument, "d requires f extension to also be specified"); - // FIXME: Consider Zfinx in the future - if (HasZve32f && !HasF) + if (HasZve32f && !HasF && !HasZfinx) return createStringError( errc::invalid_argument, - "zve32f requires f extension to also be specified"); + "zve32f requires f or zfinx extension to also be specified"); - // FIXME: Consider Zdinx in the future - if (HasZve64d && !HasD) + if (HasZve64d && !HasD && !HasZdinx) return createStringError( errc::invalid_argument, - "zve64d requires d extension to also be specified"); + "zve64d requires d or zdinx extension to also be specified"); if (HasZvl && !HasVector) return createStringError( @@ -733,6 +738,9 @@ Error RISCVISAInfo::checkDependency() { static const char *ImpliedExtsV[] = {"zvl128b", "f", "d"}; static const char *ImpliedExtsZfhmin[] = {"f"}; static const char *ImpliedExtsZfh[] = {"f"}; +static const char *ImpliedExtsZdinx[] = {"zfinx"}; +static const char *ImpliedExtsZhinxmin[] = {"zfinx"}; +static const char *ImpliedExtsZhinx[] = {"zfinx"}; static const char *ImpliedExtsZve64d[] = {"zve64f"}; static const char *ImpliedExtsZve64f[] = {"zve64x", "zve32f"}; static const char *ImpliedExtsZve64x[] = {"zve32x", "zvl64b"}; @@ -767,8 +775,11 @@ struct ImpliedExtsEntry { // Note: The table needs to be sorted by name. static constexpr ImpliedExtsEntry ImpliedExts[] = { {{"v"}, {ImpliedExtsV}}, + {{"zdinx"}, {ImpliedExtsZdinx}}, {{"zfh"}, {ImpliedExtsZfh}}, {{"zfhmin"}, {ImpliedExtsZfhmin}}, + {{"zhinx"}, {ImpliedExtsZhinx}}, + {{"zhinxmin"}, {ImpliedExtsZhinxmin}}, {{"zk"}, {ImpliedExtsZk}}, {{"zkn"}, {ImpliedExtsZkn}}, {{"zks"}, {ImpliedExtsZks}}, diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 1d162610de9c..2397a6d320a2 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1679,60 +1679,61 @@ let Predicates = [HasSVEorStreamingSVE] in { defm FCVTZS_ZPmZ_DtoD : sve_fp_2op_p_zd< 0b1111110, "fcvtzs", ZPR64, ZPR64, null_frag, AArch64fcvtzs_mt, nxv2i64, nxv2i1, nxv2f64, ElementSizeD>; defm FCVTZU_ZPmZ_DtoD : sve_fp_2op_p_zd< 0b1111111, "fcvtzu", ZPR64, ZPR64, null_frag, AArch64fcvtzu_mt, nxv2i64, nxv2i1, nxv2f64, ElementSizeD>; - def : Pat<(nxv2f32 (AArch64fcvte_mt (nxv2i1 PPR:$Pg), (nxv2f16 ZPR:$Zs), (nxv2f32 ZPR:$Zd))), - (FCVT_ZPmZ_HtoS ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + //These patterns exist to improve the code quality of conversions on unpacked types. + def : Pat<(nxv2f32 (AArch64fcvte_mt (nxv2i1 (SVEAllActive):$Pg), (nxv2f16 ZPR:$Zs), (nxv2f32 ZPR:$Zd))), + (FCVT_ZPmZ_HtoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; // FP_ROUND has an additional 'precise' flag which indicates the type of rounding. // This is ignored by the pattern below where it is matched by (i64 timm0_1) - def : Pat<(nxv2f16 (AArch64fcvtr_mt (nxv2i1 PPR:$Pg), (nxv2f32 ZPR:$Zs), (i64 timm0_1), (nxv2f16 ZPR:$Zd))), - (FCVT_ZPmZ_StoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + def : Pat<(nxv2f16 (AArch64fcvtr_mt (nxv2i1 (SVEAllActive):$Pg), (nxv2f32 ZPR:$Zs), (i64 timm0_1), (nxv2f16 ZPR:$Zd))), + (FCVT_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - // Floating-point -> signed integer - def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 PPR:$Pg), + // Signed integer -> Floating-point + def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg), (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i16), (nxv2f16 ZPR:$Zd))), - (SCVTF_ZPmZ_HtoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (SCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv4f16 (AArch64scvtf_mt (nxv4i1 PPR:$Pg), + def : Pat<(nxv4f16 (AArch64scvtf_mt (nxv4i1 (SVEAllActive):$Pg), (sext_inreg (nxv4i32 ZPR:$Zs), nxv4i16), (nxv4f16 ZPR:$Zd))), - (SCVTF_ZPmZ_HtoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (SCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 PPR:$Pg), + def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg), (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (nxv2f16 ZPR:$Zd))), - (SCVTF_ZPmZ_StoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (SCVTF_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv2f32 (AArch64scvtf_mt (nxv2i1 PPR:$Pg), + def : Pat<(nxv2f32 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg), (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (nxv2f32 ZPR:$Zd))), - (SCVTF_ZPmZ_StoS ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (SCVTF_ZPmZ_StoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv2f64 (AArch64scvtf_mt (nxv2i1 PPR:$Pg), + def : Pat<(nxv2f64 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg), (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (nxv2f64 ZPR:$Zd))), - (SCVTF_ZPmZ_StoD ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (SCVTF_ZPmZ_StoD_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - // Floating-point -> unsigned integer - def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 PPR:$Pg), + // Unsigned integer -> Floating-point + def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg), (and (nxv2i64 ZPR:$Zs), (nxv2i64 (AArch64dup (i64 0xFFFF)))), (nxv2f16 ZPR:$Zd))), - (UCVTF_ZPmZ_HtoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (UCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 PPR:$Pg), + def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg), (and (nxv2i64 ZPR:$Zs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF)))), (nxv2f16 ZPR:$Zd))), - (UCVTF_ZPmZ_StoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (UCVTF_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv4f16 (AArch64ucvtf_mt (nxv4i1 PPR:$Pg), + def : Pat<(nxv4f16 (AArch64ucvtf_mt (nxv4i1 (SVEAllActive):$Pg), (and (nxv4i32 ZPR:$Zs), (nxv4i32 (AArch64dup (i32 0xFFFF)))), (nxv4f16 ZPR:$Zd))), - (UCVTF_ZPmZ_HtoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (UCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv2f32 (AArch64ucvtf_mt (nxv2i1 PPR:$Pg), + def : Pat<(nxv2f32 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg), (and (nxv2i64 ZPR:$Zs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF)))), (nxv2f32 ZPR:$Zd))), - (UCVTF_ZPmZ_StoS ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (UCVTF_ZPmZ_StoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; - def : Pat<(nxv2f64 (AArch64ucvtf_mt (nxv2i1 PPR:$Pg), + def : Pat<(nxv2f64 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg), (and (nxv2i64 ZPR:$Zs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF)))), (nxv2f64 ZPR:$Zd))), - (UCVTF_ZPmZ_StoD ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; + (UCVTF_ZPmZ_StoD_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>; defm FRINTN_ZPmZ : sve_fp_2op_p_zd_HSD<0b00000, "frintn", AArch64frintn_mt>; defm FRINTP_ZPmZ : sve_fp_2op_p_zd_HSD<0b00001, "frintp", AArch64frintp_mt>; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index 9d4bdbe5d053..37b2ac4d8759 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -370,6 +370,14 @@ class SVE_1_Op_Passthru_Round_Pat; +multiclass SVE_1_Op_PassthruUndef_Round_Pat{ + def : Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), (vtd undef))), + (inst (IMPLICIT_DEF), $Op1, $Op2)>; + def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (i64 timm0_1), vtd:$Op3)), + (inst $Op3, $Op1, $Op2)>; +} + class SVE_1_Op_Imm_OptLsl_Reverse_Pat : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))), @@ -2589,8 +2597,8 @@ multiclass sve_fp_2op_p_zd opc, string asm, SDPatternOperator int_op, SDPatternOperator ir_op, ValueType vt1, ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { - def NAME : sve_fp_2op_p_zd; - + def NAME : sve_fp_2op_p_zd, + SVEPseudo2Instr; // convert vt1 to a packed type for the intrinsic patterns defvar packedvt1 = !cond(!eq(!cast(vt1), "nxv2f16"): nxv8f16, !eq(!cast(vt1), "nxv4f16"): nxv8f16, @@ -2604,8 +2612,11 @@ multiclass sve_fp_2op_p_zd opc, string asm, 1 : vt3); def : SVE_3_Op_Pat(NAME)>; - def : SVE_1_Op_Passthru_Pat(NAME)>; + + def _UNDEF : PredOneOpPassthruPseudo(i_zprtype)>; + + defm : SVE_1_Op_PassthruUndef_Pat(NAME # _UNDEF)>; } multiclass sve_fp_2op_p_zdr opc, string asm, @@ -2614,7 +2625,8 @@ multiclass sve_fp_2op_p_zdr opc, string asm, SDPatternOperator int_op, SDPatternOperator ir_op, ValueType vt1, ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { - def NAME : sve_fp_2op_p_zd; + def NAME : sve_fp_2op_p_zd, + SVEPseudo2Instr; // convert vt1 to a packed type for the intrinsic patterns defvar packedvt1 = !cond(!eq(!cast(vt1), "nxv2f16"): nxv8f16, @@ -2623,8 +2635,11 @@ multiclass sve_fp_2op_p_zdr opc, string asm, 1 : vt1); def : SVE_3_Op_Pat(NAME)>; - def : SVE_1_Op_Passthru_Round_Pat(NAME)>; + + def _UNDEF : PredOneOpPassthruPseudo(i_zprtype)>; + + defm : SVE_1_Op_PassthruUndef_Round_Pat(NAME # _UNDEF)>; } multiclass sve_fp_2op_p_zd_HSD opc, string asm, SDPatternOperator op> { diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 01f36e6dcdd2..95319d1b0b74 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -170,6 +170,7 @@ class RISCVAsmParser : public MCTargetAsmParser { OperandMatchResultTy parseVTypeI(OperandVector &Operands); OperandMatchResultTy parseMaskReg(OperandVector &Operands); OperandMatchResultTy parseInsnDirectiveOpcode(OperandVector &Operands); + OperandMatchResultTy parseGPRAsFPR(OperandVector &Operands); bool parseOperand(OperandVector &Operands, StringRef Mnemonic); @@ -273,6 +274,8 @@ struct RISCVOperand : public MCParsedAsmOperand { bool IsRV64; + bool IsGPRAsFPR; + struct RegOp { MCRegister RegNum; }; @@ -343,6 +346,14 @@ struct RISCVOperand : public MCParsedAsmOperand { RISCVMCRegisterClasses[RISCV::GPRRegClassID].contains(Reg.RegNum); } + bool isGPRAsFPR() const { return isGPR() && IsGPRAsFPR; } + + bool isGPRF64AsFPR() const { return isGPR() && IsGPRAsFPR && IsRV64; } + + bool isGPRPF64AsFPR() const { + return isGPR() && IsGPRAsFPR && !IsRV64 && !((Reg.RegNum - RISCV::X0) & 1); + } + static bool evaluateConstantImm(const MCExpr *Expr, int64_t &Imm, RISCVMCExpr::VariantKind &VK) { if (auto *RE = dyn_cast(Expr)) { @@ -831,12 +842,14 @@ struct RISCVOperand : public MCParsedAsmOperand { } static std::unique_ptr createReg(unsigned RegNo, SMLoc S, - SMLoc E, bool IsRV64) { + SMLoc E, bool IsRV64, + bool IsGPRAsFPR = false) { auto Op = std::make_unique(KindTy::Register); Op->Reg.RegNum = RegNo; Op->StartLoc = S; Op->EndLoc = E; Op->IsRV64 = IsRV64; + Op->IsGPRAsFPR = IsGPRAsFPR; return Op; } @@ -1780,6 +1793,26 @@ OperandMatchResultTy RISCVAsmParser::parseMaskReg(OperandVector &Operands) { return MatchOperand_Success; } +OperandMatchResultTy RISCVAsmParser::parseGPRAsFPR(OperandVector &Operands) { + switch (getLexer().getKind()) { + default: + return MatchOperand_NoMatch; + case AsmToken::Identifier: + StringRef Name = getLexer().getTok().getIdentifier(); + MCRegister RegNo; + matchRegisterNameHelper(isRV32E(), RegNo, Name); + + if (RegNo == RISCV::NoRegister) + return MatchOperand_NoMatch; + SMLoc S = getLoc(); + SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); + getLexer().Lex(); + Operands.push_back(RISCVOperand::createReg( + RegNo, S, E, isRV64(), !getSTI().hasFeature(RISCV::FeatureStdExtF))); + } + return MatchOperand_Success; +} + OperandMatchResultTy RISCVAsmParser::parseMemOpBaseReg(OperandVector &Operands) { if (getLexer().isNot(AsmToken::LParen)) { diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index ff96b2b254ca..18947997dc58 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -161,6 +161,17 @@ static DecodeStatus DecodeGPRCRegisterClass(MCInst &Inst, uint64_t RegNo, return MCDisassembler::Success; } +static DecodeStatus DecodeGPRPF64RegisterClass(MCInst &Inst, uint64_t RegNo, + uint64_t Address, + const void *Decoder) { + if (RegNo >= 32 || RegNo & 1) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::X0 + RegNo; + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + static DecodeStatus DecodeVRRegisterClass(MCInst &Inst, uint64_t RegNo, uint64_t Address, const void *Decoder) { @@ -427,6 +438,27 @@ DecodeStatus RISCVDisassembler::getInstruction(MCInst &MI, uint64_t &Size, return MCDisassembler::Fail; } Insn = support::endian::read32le(Bytes.data()); + if (STI.getFeatureBits()[RISCV::FeatureStdExtZdinx] && + !STI.getFeatureBits()[RISCV::Feature64Bit]) { + LLVM_DEBUG(dbgs() << "Trying RV32Zdinx table (Double in Integer and" + "rv32)\n"); + Result = decodeInstruction(DecoderTableRV32Zdinx32, MI, Insn, Address, + this, STI); + if (Result != MCDisassembler::Fail) { + Size = 4; + return Result; + } + } + + if (STI.getFeatureBits()[RISCV::FeatureStdExtZfinx]) { + LLVM_DEBUG(dbgs() << "Trying RVZfinx table (Float in Integer):\n"); + Result = decodeInstruction(DecoderTableRVZfinx32, MI, Insn, Address, this, + STI); + if (Result != MCDisassembler::Fail) { + Size = 4; + return Result; + } + } LLVM_DEBUG(dbgs() << "Trying RISCV32 table :\n"); Result = decodeInstruction(DecoderTable32, MI, Insn, Address, this, STI); Size = 4; diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index e32a8fb010de..065e731ff6bc 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -63,6 +63,43 @@ def HasStdExtZfhOrZfhmin "'Zfh' (Half-Precision Floating-Point) or " "'Zfhmin' (Half-Precision Floating-Point Minimal)">; +def FeatureStdExtZfinx + : SubtargetFeature<"zfinx", "HasStdExtZfinx", "true", + "'Zfinx' (Float in Integer)">; +def HasStdExtZfinx : Predicate<"Subtarget->hasStdExtZfinx()">, + AssemblerPredicate<(all_of FeatureStdExtZfinx), + "'Zfinx' (Float in Integer)">; + +def FeatureStdExtZdinx + : SubtargetFeature<"zdinx", "HasStdExtZdinx", "true", + "'Zdinx' (Double in Integer)", + [FeatureStdExtZfinx]>; +def HasStdExtZdinx : Predicate<"Subtarget->hasStdExtZdinx()">, + AssemblerPredicate<(all_of FeatureStdExtZdinx), + "'Zdinx' (Double in Integer)">; + +def FeatureStdExtZhinxmin + : SubtargetFeature<"zhinxmin", "HasStdExtZhinxmin", "true", + "'Zhinxmin' (Half Float in Integer Minimal)", + [FeatureStdExtZfinx]>; +def HasStdExtZhinxmin : Predicate<"Subtarget->hasStdExtZhinxmin()">, + AssemblerPredicate<(all_of FeatureStdExtZhinxmin), + "'Zhinxmin' (Half Float in Integer Minimal)">; + +def FeatureStdExtZhinx + : SubtargetFeature<"zhinx", "HasStdExtZhinx", "true", + "'Zhinx' (Half Float in Integer)", + [FeatureStdExtZfinx]>; +def HasStdExtZhinx : Predicate<"Subtarget->hasStdExtZhinx()">, + AssemblerPredicate<(all_of FeatureStdExtZhinx), + "'Zhinx' (Half Float in Integer)">; + +def HasStdExtZhinxOrZhinxmin + : Predicate<"Subtarget->hasStdExtZhinx() || Subtarget->hasStdExtZhinxmin()">, + AssemblerPredicate<(any_of FeatureStdExtZhinx, FeatureStdExtZhinxmin), + "'Zhinx' (Half Float in Integer) or " + "'Zhinxmin' (Half Float in Integer Minimal)">; + def FeatureStdExtC : SubtargetFeature<"c", "HasStdExtC", "true", "'C' (Compressed Instructions)">; diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp index ad003404d793..f3cc7d3fb46f 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -1116,14 +1116,6 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters( return true; } -bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { - // Keep the conventional code flow when not optimizing. - if (MF.getFunction().hasOptNone()) - return false; - - return true; -} - bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const { MachineBasicBlock *TmpMBB = const_cast(&MBB); const MachineFunction *MF = MBB.getParent(); diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h index 1e94e34acf2f..bc3ace786272 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h @@ -65,8 +65,6 @@ class RISCVFrameLowering : public TargetFrameLowering { bool canUseAsPrologue(const MachineBasicBlock &MBB) const override; bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override; - bool enableShrinkWrapping(const MachineFunction &MF) const override; - bool isSupportedStackID(TargetStackID::Value ID) const override; TargetStackID::Value getStackIDForScalableVectors() const override; diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index 649eb57b325b..6c4d2682bcd8 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -334,6 +334,10 @@ class VSETVLIInfo { return false; } + bool operator!=(const VSETVLIInfo &Other) const { + return !(*this == Other); + } + // Calculate the VSETVLIInfo visible to a block assuming this and Other are // both predecessors. VSETVLIInfo intersect(const VSETVLIInfo &Other) const { @@ -999,12 +1003,6 @@ bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require, void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) { VSETVLIInfo CurInfo; - // BBLocalInfo tracks the VL/VTYPE state the same way BBInfo.Change was - // calculated in computeIncomingVLVTYPE. We need this to apply - // canSkipVSETVLIForLoadStore the same way computeIncomingVLVTYPE did. We - // can't include predecessor information in that decision to avoid disagreeing - // with the global analysis. - VSETVLIInfo BBLocalInfo; // Only be set if current VSETVLIInfo is from an explicit VSET(I)VLI. MachineInstr *PrevVSETVLIMI = nullptr; @@ -1020,7 +1018,6 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) { MI.getOperand(3).setIsDead(false); MI.getOperand(4).setIsDead(false); CurInfo = getInfoForVSETVLI(MI); - BBLocalInfo = getInfoForVSETVLI(MI); PrevVSETVLIMI = &MI; continue; } @@ -1050,22 +1047,12 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) { // use the predecessor information. assert(BlockInfo[MBB.getNumber()].Pred.isValid() && "Expected a valid predecessor state."); - // Don't use predecessor information if there was an earlier instruction - // in this block that allowed a vsetvli to be skipped for load/store. - if (!(BBLocalInfo.isValid() && - canSkipVSETVLIForLoadStore(MI, NewInfo, BBLocalInfo)) && - needVSETVLI(NewInfo, BlockInfo[MBB.getNumber()].Pred) && + if (needVSETVLI(NewInfo, BlockInfo[MBB.getNumber()].Pred) && needVSETVLIPHI(NewInfo, MBB)) { insertVSETVLI(MBB, MI, NewInfo, BlockInfo[MBB.getNumber()].Pred); CurInfo = NewInfo; - BBLocalInfo = NewInfo; } - - // We must update BBLocalInfo for every vector instruction. - if (!BBLocalInfo.isValid()) - BBLocalInfo = NewInfo; } else { - assert(BBLocalInfo.isValid()); // If this instruction isn't compatible with the previous VL/VTYPE // we need to insert a VSETVLI. // If this is a unit-stride or strided load/store, we may be able to use @@ -1101,7 +1088,6 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) { if (NeedInsertVSETVLI) insertVSETVLI(MBB, MI, NewInfo, CurInfo); CurInfo = NewInfo; - BBLocalInfo = NewInfo; } } PrevVSETVLIMI = nullptr; @@ -1112,9 +1098,19 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) { if (MI.isCall() || MI.isInlineAsm() || MI.modifiesRegister(RISCV::VL) || MI.modifiesRegister(RISCV::VTYPE)) { CurInfo = VSETVLIInfo::getUnknown(); - BBLocalInfo = VSETVLIInfo::getUnknown(); PrevVSETVLIMI = nullptr; } + + // If we reach the end of the block and our current info doesn't match the + // expected info, insert a vsetvli to correct. + if (MI.isTerminator()) { + const VSETVLIInfo &ExitInfo = BlockInfo[MBB.getNumber()].Exit; + if (CurInfo.isValid() && ExitInfo.isValid() && !ExitInfo.isUnknown() && + CurInfo != ExitInfo) { + insertVSETVLI(MBB, MI, ExitInfo, CurInfo); + CurInfo = ExitInfo; + } + } } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td index 2837b92da81f..4f5ec6aada61 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td @@ -25,6 +25,69 @@ def SDT_RISCVSplitF64 : SDTypeProfile<2, 1, [SDTCisVT<0, i32>, def RISCVBuildPairF64 : SDNode<"RISCVISD::BuildPairF64", SDT_RISCVBuildPairF64>; def RISCVSplitF64 : SDNode<"RISCVISD::SplitF64", SDT_RISCVSplitF64>; +//===----------------------------------------------------------------------===// +// Operand and SDNode transformation definitions. +//===----------------------------------------------------------------------===// + +// Zdinx + +def GPRPF64AsFPR : AsmOperandClass { + let Name = "GPRPF64AsFPR"; + let ParserMethod = "parseGPRAsFPR"; + let RenderMethod = "addRegOperands"; +} + +def GPRF64AsFPR : AsmOperandClass { + let Name = "GPRF64AsFPR"; + let ParserMethod = "parseGPRAsFPR"; + let RenderMethod = "addRegOperands"; +} + +def FPR64INX : RegisterOperand { + let ParserMatchClass = GPRF64AsFPR; + let DecoderMethod = "DecodeGPRRegisterClass"; +} + +def FPR64IN32X : RegisterOperand { + let ParserMatchClass = GPRPF64AsFPR; +} + +def DExt : ExtInfo<0, [HasStdExtD]>; +def D64Ext : ExtInfo<0, [HasStdExtD, IsRV64]>; +def ZdinxExt : ExtInfo<1, [HasStdExtZdinx, IsRV64]>; +def Zdinx32Ext : ExtInfo<2, [HasStdExtZdinx, IsRV32]>; + +def D : ExtInfo_r; +def D_INX : ExtInfo_r; +def D_IN32X : ExtInfo_r; + +def DD : ExtInfo_rr; +def DD_INX : ExtInfo_rr; +def DD_IN32X : ExtInfo_rr; +def DF : ExtInfo_rr; +def DF_INX : ExtInfo_rr; +def DF_IN32X : ExtInfo_rr; +def DX : ExtInfo_rr; +def DX_INX : ExtInfo_rr; +def DX_IN32X : ExtInfo_rr; +def DX_64 : ExtInfo_rr; +def FD : ExtInfo_rr; +def FD_INX : ExtInfo_rr; +def FD_IN32X : ExtInfo_rr; +def XD : ExtInfo_rr; +def XD_INX : ExtInfo_rr; +def XD_IN32X : ExtInfo_rr; +def XD_64 : ExtInfo_rr; + +defvar DINX = [D, D_INX, D_IN32X]; +defvar DDINX = [DD, DD_INX, DD_IN32X]; +defvar DXINX = [DX, DX_INX, DX_IN32X]; +defvar DFINX = [DF, DF_INX, DF_IN32X]; +defvar FDINX = [FD, FD_INX, FD_IN32X]; +defvar XDINX = [XD, XD_INX, XD_IN32X]; +defvar DXIN64X = [DX_64, DX_INX]; +defvar XDIN64X = [XD_64, XD_INX]; + //===----------------------------------------------------------------------===// // Instructions //===----------------------------------------------------------------------===// @@ -36,106 +99,104 @@ def FLD : FPLoad_r<0b011, "fld", FPR64, WriteFLD64>; // reflecting the order these fields are specified in the instruction // encoding. def FSD : FPStore_r<0b011, "fsd", FPR64, WriteFST64>; +} // Predicates = [HasStdExtD] let SchedRW = [WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64] in { -def FMADD_D : FPFMA_rrr_frm; -def FMSUB_D : FPFMA_rrr_frm; -def FNMSUB_D : FPFMA_rrr_frm; -def FNMADD_D : FPFMA_rrr_frm; +defm FMADD_D : FPFMA_rrr_frm_m; +defm FMSUB_D : FPFMA_rrr_frm_m; +defm FNMSUB_D : FPFMA_rrr_frm_m; +defm FNMADD_D : FPFMA_rrr_frm_m; } -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; -def FADD_D : FPALU_rr_frm<0b0000001, "fadd.d", FPR64>, - Sched<[WriteFALU64, ReadFALU64, ReadFALU64]>; -def FSUB_D : FPALU_rr_frm<0b0000101, "fsub.d", FPR64>, - Sched<[WriteFALU64, ReadFALU64, ReadFALU64]>; -def FMUL_D : FPALU_rr_frm<0b0001001, "fmul.d", FPR64>, - Sched<[WriteFMul64, ReadFMul64, ReadFMul64]>; -def FDIV_D : FPALU_rr_frm<0b0001101, "fdiv.d", FPR64>, - Sched<[WriteFDiv64, ReadFDiv64, ReadFDiv64]>; +let SchedRW = [WriteFALU64, ReadFALU64, ReadFALU64] in { +defm FADD_D : FPALU_rr_frm_m<0b0000001, "fadd.d", DINX>; +defm FSUB_D : FPALU_rr_frm_m<0b0000101, "fsub.d", DINX>; +} +let SchedRW = [WriteFMul64, ReadFMul64, ReadFMul64] in +defm FMUL_D : FPALU_rr_frm_m<0b0001001, "fmul.d", DINX>; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; +let SchedRW = [WriteFDiv64, ReadFDiv64, ReadFDiv64] in +defm FDIV_D : FPALU_rr_frm_m<0b0001101, "fdiv.d", DINX>; -def FSQRT_D : FPUnaryOp_r_frm<0b0101101, 0b00000, FPR64, FPR64, "fsqrt.d">, - Sched<[WriteFSqrt64, ReadFSqrt64]>; -def : FPUnaryOpDynFrmAlias; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; + +defm FSQRT_D : FPUnaryOp_r_frm_m<0b0101101, 0b00000, DDINX, "fsqrt.d">, + Sched<[WriteFSqrt64, ReadFSqrt64]>; +defm : FPUnaryOpDynFrmAlias_m; let SchedRW = [WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64], mayRaiseFPException = 0 in { -def FSGNJ_D : FPALU_rr<0b0010001, 0b000, "fsgnj.d", FPR64>; -def FSGNJN_D : FPALU_rr<0b0010001, 0b001, "fsgnjn.d", FPR64>; -def FSGNJX_D : FPALU_rr<0b0010001, 0b010, "fsgnjx.d", FPR64>; +defm FSGNJ_D : FPALU_rr_m<0b0010001, 0b000, "fsgnj.d", DINX>; +defm FSGNJN_D : FPALU_rr_m<0b0010001, 0b001, "fsgnjn.d", DINX>; +defm FSGNJX_D : FPALU_rr_m<0b0010001, 0b010, "fsgnjx.d", DINX>; } let SchedRW = [WriteFMinMax64, ReadFMinMax64, ReadFMinMax64] in { -def FMIN_D : FPALU_rr<0b0010101, 0b000, "fmin.d", FPR64>; -def FMAX_D : FPALU_rr<0b0010101, 0b001, "fmax.d", FPR64>; +defm FMIN_D : FPALU_rr_m<0b0010101, 0b000, "fmin.d", DINX>; +defm FMAX_D : FPALU_rr_m<0b0010101, 0b001, "fmax.d", DINX>; } -def FCVT_S_D : FPUnaryOp_r_frm<0b0100000, 0b00001, FPR32, FPR64, "fcvt.s.d">, - Sched<[WriteFCvtF64ToF32, ReadFCvtF64ToF32]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_S_D : FPUnaryOp_r_frm_m<0b0100000, 0b00001, FDINX, "fcvt.s.d">, + Sched<[WriteFCvtF64ToF32, ReadFCvtF64ToF32]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_D_S : FPUnaryOp_r<0b0100001, 0b00000, 0b000, FPR64, FPR32, "fcvt.d.s">, - Sched<[WriteFCvtF32ToF64, ReadFCvtF32ToF64]>; +defm FCVT_D_S : FPUnaryOp_r_m<0b0100001, 0b00000, 0b000, DFINX, "fcvt.d.s">, + Sched<[WriteFCvtF32ToF64, ReadFCvtF32ToF64]>; let SchedRW = [WriteFCmp64, ReadFCmp64, ReadFCmp64] in { -def FEQ_D : FPCmp_rr<0b1010001, 0b010, "feq.d", FPR64>; -def FLT_D : FPCmp_rr<0b1010001, 0b001, "flt.d", FPR64>; -def FLE_D : FPCmp_rr<0b1010001, 0b000, "fle.d", FPR64>; +defm FEQ_D : FPCmp_rr_m<0b1010001, 0b010, "feq.d", DINX>; +defm FLT_D : FPCmp_rr_m<0b1010001, 0b001, "flt.d", DINX>; +defm FLE_D : FPCmp_rr_m<0b1010001, 0b000, "fle.d", DINX>; } -let mayRaiseFPException = 0 in -def FCLASS_D : FPUnaryOp_r<0b1110001, 0b00000, 0b001, GPR, FPR64, "fclass.d">, - Sched<[WriteFClass64, ReadFClass64]>; +defm FCLASS_D : FPUnaryOp_r_m<0b1110001, 0b00000, 0b001, XDINX, "fclass.d">, + Sched<[WriteFClass64, ReadFClass64]>; -def FCVT_W_D : FPUnaryOp_r_frm<0b1100001, 0b00000, GPR, FPR64, "fcvt.w.d">, +defm FCVT_W_D : FPUnaryOp_r_frm_m<0b1100001, 0b00000, XDINX, "fcvt.w.d">, Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_WU_D : FPUnaryOp_r_frm<0b1100001, 0b00001, GPR, FPR64, "fcvt.wu.d">, - Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_WU_D : FPUnaryOp_r_frm_m<0b1100001, 0b00001, XDINX, "fcvt.wu.d">, + Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_D_W : FPUnaryOp_r<0b1101001, 0b00000, 0b000, FPR64, GPR, "fcvt.d.w">, - Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>; - -def FCVT_D_WU : FPUnaryOp_r<0b1101001, 0b00001, 0b000, FPR64, GPR, "fcvt.d.wu">, +defm FCVT_D_W : FPUnaryOp_r_m<0b1101001, 0b00000, 0b000, DXINX, "fcvt.d.w">, Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>; -} // Predicates = [HasStdExtD] -let Predicates = [HasStdExtD, IsRV64] in { -def FCVT_L_D : FPUnaryOp_r_frm<0b1100001, 0b00010, GPR, FPR64, "fcvt.l.d">, - Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_D_WU : FPUnaryOp_r_m<0b1101001, 0b00001, 0b000, DXINX, "fcvt.d.wu">, + Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>; -def FCVT_LU_D : FPUnaryOp_r_frm<0b1100001, 0b00011, GPR, FPR64, "fcvt.lu.d">, +defm FCVT_L_D : FPUnaryOp_r_frm_m<0b1100001, 0b00010, XDIN64X, "fcvt.l.d">, Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; -let mayRaiseFPException = 0 in +defm FCVT_LU_D : FPUnaryOp_r_frm_m<0b1100001, 0b00011, XDIN64X, "fcvt.lu.d">, + Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>; +defm : FPUnaryOpDynFrmAlias_m; + +let Predicates = [HasStdExtD, IsRV64], mayRaiseFPException = 0 in def FMV_X_D : FPUnaryOp_r<0b1110001, 0b00000, 0b000, GPR, FPR64, "fmv.x.d">, Sched<[WriteFMovF64ToI64, ReadFMovF64ToI64]>; -def FCVT_D_L : FPUnaryOp_r_frm<0b1101001, 0b00010, FPR64, GPR, "fcvt.d.l">, - Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>; -def : FPUnaryOpDynFrmAlias; - -def FCVT_D_LU : FPUnaryOp_r_frm<0b1101001, 0b00011, FPR64, GPR, "fcvt.d.lu">, +defm FCVT_D_L : FPUnaryOp_r_frm_m<0b1101001, 0b00010, DXIN64X, "fcvt.d.l">, Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; -let mayRaiseFPException = 0 in +defm FCVT_D_LU : FPUnaryOp_r_frm_m<0b1101001, 0b00011, DXIN64X, "fcvt.d.lu">, + Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>; +defm : FPUnaryOpDynFrmAlias_m; + +let Predicates = [HasStdExtD, IsRV64], mayRaiseFPException = 0 in def FMV_D_X : FPUnaryOp_r<0b1111001, 0b00000, 0b000, FPR64, GPR, "fmv.d.x">, Sched<[WriteFMovI64ToF64, ReadFMovI64ToF64]>; -} // Predicates = [HasStdExtD, IsRV64] //===----------------------------------------------------------------------===// // Assembler Pseudo Instructions (User-Level ISA, Version 2.2, Chapter 20) @@ -164,6 +225,26 @@ def PseudoQuietFLT_D : PseudoQuietFCMP; } } // Predicates = [HasStdExtD] +let Predicates = [HasStdExtZdinx, IsRV64] in { +def : InstAlias<"fabs.d $rd, $rs", (FSGNJX_D_INX FPR64INX:$rd, FPR64INX:$rs, FPR64INX:$rs)>; +def : InstAlias<"fneg.d $rd, $rs", (FSGNJN_D_INX FPR64INX:$rd, FPR64INX:$rs, FPR64INX:$rs)>; + +def : InstAlias<"fgt.d $rd, $rs, $rt", + (FLT_D_INX GPR:$rd, FPR64INX:$rt, FPR64INX:$rs), 0>; +def : InstAlias<"fge.d $rd, $rs, $rt", + (FLE_D_INX GPR:$rd, FPR64INX:$rt, FPR64INX:$rs), 0>; +} // Predicates = [HasStdExtZdinx, IsRV64] + +let Predicates = [HasStdExtZdinx, IsRV32] in { +def : InstAlias<"fabs.d $rd, $rs", (FSGNJX_D_IN32X FPR64IN32X:$rd, FPR64IN32X:$rs, FPR64IN32X:$rs)>; +def : InstAlias<"fneg.d $rd, $rs", (FSGNJN_D_IN32X FPR64IN32X:$rd, FPR64IN32X:$rs, FPR64IN32X:$rs)>; + +def : InstAlias<"fgt.d $rd, $rs, $rt", + (FLT_D_IN32X GPR:$rd, FPR64IN32X:$rt, FPR64IN32X:$rs), 0>; +def : InstAlias<"fge.d $rd, $rs, $rt", + (FLE_D_IN32X GPR:$rd, FPR64IN32X:$rt, FPR64IN32X:$rs), 0>; +} // Predicates = [HasStdExtZdinx, IsRV32] + //===----------------------------------------------------------------------===// // Pseudo-instructions and codegen patterns //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td index a8ac06ba8da3..4b45b47af451 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td @@ -57,6 +57,73 @@ def riscv_any_fcvt_wu_rv64 : PatFrags<(ops node:$src, node:$frm), // Operand and SDNode transformation definitions. //===----------------------------------------------------------------------===// +// Zfinx + +def GPRAsFPR : AsmOperandClass { + let Name = "GPRAsFPR"; + let ParserMethod = "parseGPRAsFPR"; + let RenderMethod = "addRegOperands"; +} + +def FPR32INX : RegisterOperand { + let ParserMatchClass = GPRAsFPR; + let DecoderMethod = "DecodeGPRRegisterClass"; +} + +// inx = 0 : f, d, zfh, zfhmin +// = 1 : zfinx, zdinx, zhinx, zhinxmin +// = 2 : zdinx_rv32 +class ExtInfo inx, list pres> { + string Suffix = !cond(!eq(inx, 0): "", + !eq(inx, 1): "_INX", + !eq(inx, 2): "_IN32X"); + list Predicates = pres; + string Space = !cond(!eq(inx, 0): "", + !eq(inx, 1): "RVZfinx", + !eq(inx, 2): "RV32Zdinx"); +} + +class ExtInfo_r { + string Suffix = ext.Suffix; + list Predicates = ext.Predicates; + string Space = ext.Space; + DAGOperand Reg = reg; +} + +class ExtInfo_rr { + string Suffix = ext.Suffix; + list Predicates = ext.Predicates; + string Space = ext.Space; + DAGOperand RdTy = rdty; + DAGOperand Rs1Ty = rs1ty; +} + +def FExt : ExtInfo<0, [HasStdExtF]>; +def F64Ext : ExtInfo<0, [HasStdExtF, IsRV64]>; +def ZfinxExt : ExtInfo<1, [HasStdExtZfinx]>; +def Zfinx64Ext : ExtInfo<1, [HasStdExtZfinx, IsRV64]>; + +def F : ExtInfo_r; +def F_INX : ExtInfo_r; + +def FF : ExtInfo_rr; +def FF_INX : ExtInfo_rr; +def FX : ExtInfo_rr; +def FX_INX : ExtInfo_rr; +def FX_64 : ExtInfo_rr; +def FX_INX_64 : ExtInfo_rr; +def XF : ExtInfo_rr; +def XF_64 : ExtInfo_rr; +def XF_INX : ExtInfo_rr; +def XF_INX_64 : ExtInfo_rr; + +defvar FINX = [F, F_INX]; +defvar FFINX = [FF, FF_INX]; +defvar FXINX = [FX, FX_INX]; +defvar XFINX = [XF, XF_INX]; +defvar XFIN64X = [XF_64, XF_INX_64]; +defvar FXIN64X = [FX_64, FX_INX_64]; + // Floating-point rounding mode def FRMArg : AsmOperandClass { @@ -94,62 +161,123 @@ class FPStore_r funct3, string opcodestr, RegisterClass rty, let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, UseNamedOperandTable = 1, hasPostISelHook = 1 in class FPFMA_rrr_frm funct2, string opcodestr, - RegisterClass rty> + DAGOperand rty> : RVInstR4Frm; +multiclass FPFMA_rrr_frm_m funct2, + string opcodestr, list Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates, DecoderNamespace = Ext.Space in + def Ext.Suffix : FPFMA_rrr_frm; +} + class FPFMADynFrmAlias + DAGOperand rty> : InstAlias; +multiclass FPFMADynFrmAlias_m Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates in + def : FPFMADynFrmAlias(Inst#Ext.Suffix), OpcodeStr, + Ext.Reg>; +} let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in class FPALU_rr funct7, bits<3> funct3, string opcodestr, - RegisterClass rty> + DAGOperand rty> : RVInstR; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, - UseNamedOperandTable = 1, hasPostISelHook = 1 in -class FPALU_rr_frm funct7, string opcodestr, RegisterClass rty> - : RVInstRFrm; - -class FPALUDynFrmAlias - : InstAlias; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in -class FPUnaryOp_r funct7, bits<5> rs2val, bits<3> funct3, - RegisterClass rdty, RegisterClass rs1ty, string opcodestr> - : RVInstR { - let rs2 = rs2val; +multiclass FPALU_rr_m funct7, bits<3> funct3, string opcodestr, + list Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates, DecoderNamespace = Ext.Space in + def Ext.Suffix : FPALU_rr; } let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, UseNamedOperandTable = 1, hasPostISelHook = 1 in -class FPUnaryOp_r_frm funct7, bits<5> rs2val, RegisterClass rdty, - RegisterClass rs1ty, string opcodestr> +class FPALU_rr_frm funct7, string opcodestr, DAGOperand rty> + : RVInstRFrm; + +multiclass FPALU_rr_frm_m funct7, string opcodestr, + list Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates, DecoderNamespace = Ext.Space in + def Ext.Suffix : FPALU_rr_frm; +} + +class FPALUDynFrmAlias + : InstAlias; +multiclass FPALUDynFrmAlias_m Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates in + def : FPALUDynFrmAlias(Inst#Ext.Suffix), OpcodeStr, + Ext.Reg>; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in +class FPUnaryOp_r funct7, bits<5> rs2val, bits<3> funct3, + DAGOperand rdty, DAGOperand rs1ty, string opcodestr> + : RVInstR { + let rs2 = rs2val; +} +multiclass FPUnaryOp_r_m funct7, bits<5> rs2val, bits<3> funct3, + list Exts, string opcodestr> { + foreach Ext = Exts in + let Predicates = Ext.Predicates, DecoderNamespace = Ext.Space in + def Ext.Suffix : FPUnaryOp_r; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, + UseNamedOperandTable = 1, hasPostISelHook = 1 in +class FPUnaryOp_r_frm funct7, bits<5> rs2val, DAGOperand rdty, + DAGOperand rs1ty, string opcodestr> : RVInstRFrm { let rs2 = rs2val; } +multiclass FPUnaryOp_r_frm_m funct7, bits<5> rs2val, + list Exts, string opcodestr> { + foreach Ext = Exts in + let Predicates = Ext.Predicates, DecoderNamespace = Ext.Space in + def Ext.Suffix : FPUnaryOp_r_frm; +} class FPUnaryOpDynFrmAlias + DAGOperand rdty, DAGOperand rs1ty> : InstAlias; +multiclass FPUnaryOpDynFrmAlias_m Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates in + def : FPUnaryOpDynFrmAlias(Inst#Ext.Suffix), + OpcodeStr, Ext.RdTy, Ext.Rs1Ty>; +} let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in class FPCmp_rr funct7, bits<3> funct3, string opcodestr, - RegisterClass rty> + DAGOperand rty> : RVInstR; +multiclass FPCmp_rr_m funct7, bits<3> funct3, string opcodestr, + list Exts> { + foreach Ext = Exts in + let Predicates = Ext.Predicates, DecoderNamespace = Ext.Space in + def Ext.Suffix : FPCmp_rr; +} //===----------------------------------------------------------------------===// // Instructions @@ -162,101 +290,100 @@ def FLW : FPLoad_r<0b010, "flw", FPR32, WriteFLD32>; // reflecting the order these fields are specified in the instruction // encoding. def FSW : FPStore_r<0b010, "fsw", FPR32, WriteFST32>; +} // Predicates = [HasStdExtF] let SchedRW = [WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32] in { -def FMADD_S : FPFMA_rrr_frm; -def FMSUB_S : FPFMA_rrr_frm; -def FNMSUB_S : FPFMA_rrr_frm; -def FNMADD_S : FPFMA_rrr_frm; +defm FMADD_S : FPFMA_rrr_frm_m; +defm FMSUB_S : FPFMA_rrr_frm_m; +defm FNMSUB_S : FPFMA_rrr_frm_m; +defm FNMADD_S : FPFMA_rrr_frm_m; } -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; -def FADD_S : FPALU_rr_frm<0b0000000, "fadd.s", FPR32>, - Sched<[WriteFALU32, ReadFALU32, ReadFALU32]>; -def FSUB_S : FPALU_rr_frm<0b0000100, "fsub.s", FPR32>, - Sched<[WriteFALU32, ReadFALU32, ReadFALU32]>; -def FMUL_S : FPALU_rr_frm<0b0001000, "fmul.s", FPR32>, - Sched<[WriteFMul32, ReadFMul32, ReadFMul32]>; -def FDIV_S : FPALU_rr_frm<0b0001100, "fdiv.s", FPR32>, - Sched<[WriteFDiv32, ReadFDiv32, ReadFDiv32]>; +let SchedRW = [WriteFALU32, ReadFALU32, ReadFALU32] in { +defm FADD_S : FPALU_rr_frm_m<0b0000000, "fadd.s", FINX>; +defm FSUB_S : FPALU_rr_frm_m<0b0000100, "fsub.s", FINX>; +} +let SchedRW = [WriteFMul32, ReadFMul32, ReadFMul32] in +defm FMUL_S : FPALU_rr_frm_m<0b0001000, "fmul.s", FINX>; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; +let SchedRW = [WriteFDiv32, ReadFDiv32, ReadFDiv32] in +defm FDIV_S : FPALU_rr_frm_m<0b0001100, "fdiv.s", FINX>; -def FSQRT_S : FPUnaryOp_r_frm<0b0101100, 0b00000, FPR32, FPR32, "fsqrt.s">, - Sched<[WriteFSqrt32, ReadFSqrt32]>; -def : FPUnaryOpDynFrmAlias; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; + +defm FSQRT_S : FPUnaryOp_r_frm_m<0b0101100, 0b00000, FFINX, "fsqrt.s">, + Sched<[WriteFSqrt32, ReadFSqrt32]>; +defm : FPUnaryOpDynFrmAlias_m; let SchedRW = [WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32], mayRaiseFPException = 0 in { -def FSGNJ_S : FPALU_rr<0b0010000, 0b000, "fsgnj.s", FPR32>; -def FSGNJN_S : FPALU_rr<0b0010000, 0b001, "fsgnjn.s", FPR32>; -def FSGNJX_S : FPALU_rr<0b0010000, 0b010, "fsgnjx.s", FPR32>; +defm FSGNJ_S : FPALU_rr_m<0b0010000, 0b000, "fsgnj.s", FINX>; +defm FSGNJN_S : FPALU_rr_m<0b0010000, 0b001, "fsgnjn.s", FINX>; +defm FSGNJX_S : FPALU_rr_m<0b0010000, 0b010, "fsgnjx.s", FINX>; } let SchedRW = [WriteFMinMax32, ReadFMinMax32, ReadFMinMax32] in { -def FMIN_S : FPALU_rr<0b0010100, 0b000, "fmin.s", FPR32>; -def FMAX_S : FPALU_rr<0b0010100, 0b001, "fmax.s", FPR32>; +defm FMIN_S : FPALU_rr_m<0b0010100, 0b000, "fmin.s", FINX>; +defm FMAX_S : FPALU_rr_m<0b0010100, 0b001, "fmax.s", FINX>; } -def FCVT_W_S : FPUnaryOp_r_frm<0b1100000, 0b00000, GPR, FPR32, "fcvt.w.s">, - Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>; -def : FPUnaryOpDynFrmAlias; - -def FCVT_WU_S : FPUnaryOp_r_frm<0b1100000, 0b00001, GPR, FPR32, "fcvt.wu.s">, +defm FCVT_W_S : FPUnaryOp_r_frm_m<0b1100000, 0b00000, XFINX, "fcvt.w.s">, Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; + +defm FCVT_WU_S : FPUnaryOp_r_frm_m<0b1100000, 0b00001, XFINX, "fcvt.wu.s">, + Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>; +defm : FPUnaryOpDynFrmAlias_m; let mayRaiseFPException = 0 in def FMV_X_W : FPUnaryOp_r<0b1110000, 0b00000, 0b000, GPR, FPR32, "fmv.x.w">, Sched<[WriteFMovF32ToI32, ReadFMovF32ToI32]>; let SchedRW = [WriteFCmp32, ReadFCmp32, ReadFCmp32] in { -def FEQ_S : FPCmp_rr<0b1010000, 0b010, "feq.s", FPR32>; -def FLT_S : FPCmp_rr<0b1010000, 0b001, "flt.s", FPR32>; -def FLE_S : FPCmp_rr<0b1010000, 0b000, "fle.s", FPR32>; +defm FEQ_S : FPCmp_rr_m<0b1010000, 0b010, "feq.s", FINX>; +defm FLT_S : FPCmp_rr_m<0b1010000, 0b001, "flt.s", FINX>; +defm FLE_S : FPCmp_rr_m<0b1010000, 0b000, "fle.s", FINX>; } let mayRaiseFPException = 0 in -def FCLASS_S : FPUnaryOp_r<0b1110000, 0b00000, 0b001, GPR, FPR32, "fclass.s">, - Sched<[WriteFClass32, ReadFClass32]>; +defm FCLASS_S : FPUnaryOp_r_m<0b1110000, 0b00000, 0b001, XFINX, "fclass.s">, + Sched<[WriteFClass32, ReadFClass32]>; -def FCVT_S_W : FPUnaryOp_r_frm<0b1101000, 0b00000, FPR32, GPR, "fcvt.s.w">, - Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>; -def : FPUnaryOpDynFrmAlias; - -def FCVT_S_WU : FPUnaryOp_r_frm<0b1101000, 0b00001, FPR32, GPR, "fcvt.s.wu">, +defm FCVT_S_W : FPUnaryOp_r_frm_m<0b1101000, 0b00000, FXINX, "fcvt.s.w">, Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; + +defm FCVT_S_WU : FPUnaryOp_r_frm_m<0b1101000, 0b00001, FXINX, "fcvt.s.wu">, + Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>; +defm : FPUnaryOpDynFrmAlias_m; let mayRaiseFPException = 0 in def FMV_W_X : FPUnaryOp_r<0b1111000, 0b00000, 0b000, FPR32, GPR, "fmv.w.x">, Sched<[WriteFMovI32ToF32, ReadFMovI32ToF32]>; -} // Predicates = [HasStdExtF] -let Predicates = [HasStdExtF, IsRV64] in { -def FCVT_L_S : FPUnaryOp_r_frm<0b1100000, 0b00010, GPR, FPR32, "fcvt.l.s">, - Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>; -def : FPUnaryOpDynFrmAlias; - -def FCVT_LU_S : FPUnaryOp_r_frm<0b1100000, 0b00011, GPR, FPR32, "fcvt.lu.s">, +defm FCVT_L_S : FPUnaryOp_r_frm_m<0b1100000, 0b00010, XFIN64X, "fcvt.l.s">, Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_S_L : FPUnaryOp_r_frm<0b1101000, 0b00010, FPR32, GPR, "fcvt.s.l">, - Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_LU_S : FPUnaryOp_r_frm_m<0b1100000, 0b00011, XFIN64X, "fcvt.lu.s">, + Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_S_LU : FPUnaryOp_r_frm<0b1101000, 0b00011, FPR32, GPR, "fcvt.s.lu">, +defm FCVT_S_L : FPUnaryOp_r_frm_m<0b1101000, 0b00010, FXIN64X, "fcvt.s.l">, Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>; -def : FPUnaryOpDynFrmAlias; -} // Predicates = [HasStdExtF, IsRV64] +defm : FPUnaryOpDynFrmAlias_m; + +defm FCVT_S_LU : FPUnaryOp_r_frm_m<0b1101000, 0b00011, FXIN64X, "fcvt.s.lu">, + Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>; +defm : FPUnaryOpDynFrmAlias_m; //===----------------------------------------------------------------------===// // Assembler Pseudo Instructions (User-Level ISA, Version 2.2, Chapter 20) @@ -315,6 +442,16 @@ def PseudoQuietFLT_S : PseudoQuietFCMP; } } // Predicates = [HasStdExtF] +let Predicates = [HasStdExtZfinx] in { +def : InstAlias<"fabs.s $rd, $rs", (FSGNJX_S_INX FPR32INX:$rd, FPR32INX:$rs, FPR32INX:$rs)>; +def : InstAlias<"fneg.s $rd, $rs", (FSGNJN_S_INX FPR32INX:$rd, FPR32INX:$rs, FPR32INX:$rs)>; + +def : InstAlias<"fgt.s $rd, $rs, $rt", + (FLT_S_INX GPR:$rd, FPR32INX:$rt, FPR32INX:$rs), 0>; +def : InstAlias<"fge.s $rd, $rs, $rt", + (FLE_S_INX GPR:$rd, FPR32INX:$rt, FPR32INX:$rs), 0>; +} // Predicates = [HasStdExtZfinx] + //===----------------------------------------------------------------------===// // Pseudo-instructions and codegen patterns //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td index a2753c132354..631525484bd9 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td @@ -25,6 +25,62 @@ def riscv_fmv_h_x def riscv_fmv_x_anyexth : SDNode<"RISCVISD::FMV_X_ANYEXTH", SDT_RISCVFMV_X_ANYEXTH>; +//===----------------------------------------------------------------------===// +// Operand and SDNode transformation definitions. +//===----------------------------------------------------------------------===// + +// Zhinxmin and Zhinx + +def FPR16INX : RegisterOperand { + let ParserMatchClass = GPRAsFPR; + let DecoderMethod = "DecodeGPRRegisterClass"; +} + +def ZfhExt : ExtInfo<0, [HasStdExtZfh]>; +def Zfh64Ext : ExtInfo<0, [HasStdExtZfh, IsRV64]>; +def ZfhminExt : ExtInfo<0, [HasStdExtZfhOrZfhmin]>; +def ZhinxExt : ExtInfo<1, [HasStdExtZhinx]>; +def ZhinxminExt : ExtInfo<1, [HasStdExtZhinxOrZhinxmin]>; +def Zhinx64Ext : ExtInfo<1, [HasStdExtZhinx, IsRV64]>; + +def ZfhminDExt : ExtInfo<0, [HasStdExtZfhOrZfhmin, HasStdExtD]>; +def ZhinxminZdinxExt : ExtInfo<1, [HasStdExtZhinxOrZhinxmin, HasStdExtZdinx]>; + +def H : ExtInfo_r; +def H_INX : ExtInfo_r; + +def HH : ExtInfo_rr; +def HH_INX : ExtInfo_rr; +def XH : ExtInfo_rr; +def XH_INX : ExtInfo_rr; +def HX : ExtInfo_rr; +def HX_INX : ExtInfo_rr; +def XH_64 : ExtInfo_rr; +def HX_64 : ExtInfo_rr; +def XH_INX_64 : ExtInfo_rr; +def HX_INX_64 : ExtInfo_rr; +def HFmin : ExtInfo_rr; +def HF_INXmin : ExtInfo_rr; +def HF_INX : ExtInfo_rr; +def FHmin : ExtInfo_rr; +def FH_INXmin : ExtInfo_rr; +def FH_INX : ExtInfo_rr; +def DHmin : ExtInfo_rr; +def DH_INXmin : ExtInfo_rr; +def HDmin : ExtInfo_rr; +def HD_INXmin : ExtInfo_rr; + +defvar HINX = [H, H_INX]; +defvar HHINX = [HH, HH_INX]; +defvar XHINX = [XH, XH_INX]; +defvar HXINX = [HX, HX_INX]; +defvar XHIN64X = [XH_64, XH_INX_64]; +defvar HXIN64X = [HX_64, HX_INX_64]; +defvar HFINXmin = [HFmin, HF_INXmin]; +defvar FHINXmin = [FHmin, FH_INXmin]; +defvar DHINXmin = [DHmin, DH_INXmin]; +defvar HDINXmin = [HDmin, HD_INXmin]; + //===----------------------------------------------------------------------===// // Instructions //===----------------------------------------------------------------------===// @@ -38,74 +94,73 @@ def FLH : FPLoad_r<0b001, "flh", FPR16, WriteFLD16>; def FSH : FPStore_r<0b001, "fsh", FPR16, WriteFST16>; } // Predicates = [HasStdExtZfhOrZfhmin] -let Predicates = [HasStdExtZfh] in { let SchedRW = [WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16] in { -def FMADD_H : FPFMA_rrr_frm; -def FMSUB_H : FPFMA_rrr_frm; -def FNMSUB_H : FPFMA_rrr_frm; -def FNMADD_H : FPFMA_rrr_frm; +defm FMADD_H : FPFMA_rrr_frm_m; +defm FMSUB_H : FPFMA_rrr_frm_m; +defm FNMSUB_H : FPFMA_rrr_frm_m; +defm FNMADD_H : FPFMA_rrr_frm_m; } -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; -def : FPFMADynFrmAlias; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; +defm : FPFMADynFrmAlias_m; -def FADD_H : FPALU_rr_frm<0b0000010, "fadd.h", FPR16>, - Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>; -def FSUB_H : FPALU_rr_frm<0b0000110, "fsub.h", FPR16>, - Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>; -def FMUL_H : FPALU_rr_frm<0b0001010, "fmul.h", FPR16>, - Sched<[WriteFMul16, ReadFMul16, ReadFMul16]>; -def FDIV_H : FPALU_rr_frm<0b0001110, "fdiv.h", FPR16>, - Sched<[WriteFDiv16, ReadFDiv16, ReadFDiv16]>; +let SchedRW = [WriteFALU16, ReadFALU16, ReadFALU16] in { +defm FADD_H : FPALU_rr_frm_m<0b0000010, "fadd.h", HINX>; +defm FSUB_H : FPALU_rr_frm_m<0b0000110, "fsub.h", HINX>; +} +let SchedRW = [WriteFMul16, ReadFMul16, ReadFMul16] in +defm FMUL_H : FPALU_rr_frm_m<0b0001010, "fmul.h", HINX>; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; -def : FPALUDynFrmAlias; +let SchedRW = [WriteFDiv16, ReadFDiv16, ReadFDiv16] in +defm FDIV_H : FPALU_rr_frm_m<0b0001110, "fdiv.h", HINX>; -def FSQRT_H : FPUnaryOp_r_frm<0b0101110, 0b00000, FPR16, FPR16, "fsqrt.h">, - Sched<[WriteFSqrt16, ReadFSqrt16]>; -def : FPUnaryOpDynFrmAlias; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; +defm : FPALUDynFrmAlias_m; + +defm FSQRT_H : FPUnaryOp_r_frm_m<0b0101110, 0b00000, HHINX, "fsqrt.h">, + Sched<[WriteFSqrt16, ReadFSqrt16]>; +defm : FPUnaryOpDynFrmAlias_m; let SchedRW = [WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16], mayRaiseFPException = 0 in { -def FSGNJ_H : FPALU_rr<0b0010010, 0b000, "fsgnj.h", FPR16>; -def FSGNJN_H : FPALU_rr<0b0010010, 0b001, "fsgnjn.h", FPR16>; -def FSGNJX_H : FPALU_rr<0b0010010, 0b010, "fsgnjx.h", FPR16>; +defm FSGNJ_H : FPALU_rr_m<0b0010010, 0b000, "fsgnj.h", HINX>; +defm FSGNJN_H : FPALU_rr_m<0b0010010, 0b001, "fsgnjn.h", HINX>; +defm FSGNJX_H : FPALU_rr_m<0b0010010, 0b010, "fsgnjx.h", HINX>; } let SchedRW = [WriteFMinMax16, ReadFMinMax16, ReadFMinMax16] in { -def FMIN_H : FPALU_rr<0b0010110, 0b000, "fmin.h", FPR16>; -def FMAX_H : FPALU_rr<0b0010110, 0b001, "fmax.h", FPR16>; +defm FMIN_H : FPALU_rr_m<0b0010110, 0b000, "fmin.h", HINX>; +defm FMAX_H : FPALU_rr_m<0b0010110, 0b001, "fmax.h", HINX>; } -def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, 0b00000, GPR, FPR16, "fcvt.w.h">, - Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>; -def : FPUnaryOpDynFrmAlias; - -def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, 0b00001, GPR, FPR16, "fcvt.wu.h">, +defm FCVT_W_H : FPUnaryOp_r_frm_m<0b1100010, 0b00000, XHINX, "fcvt.w.h">, Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, 0b00000, FPR16, GPR, "fcvt.h.w">, - Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_WU_H : FPUnaryOp_r_frm_m<0b1100010, 0b00001, XHINX, "fcvt.wu.h">, + Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, 0b00001, FPR16, GPR, "fcvt.h.wu">, +defm FCVT_H_W : FPUnaryOp_r_frm_m<0b1101010, 0b00000, HXINX, "fcvt.h.w">, Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>; -def : FPUnaryOpDynFrmAlias; -} // Predicates = [HasStdExtZfh] +defm : FPUnaryOpDynFrmAlias_m; -let Predicates = [HasStdExtZfhOrZfhmin] in { -def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, 0b00000, FPR16, FPR32, "fcvt.h.s">, - Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_H_WU : FPUnaryOp_r_frm_m<0b1101010, 0b00001, HXINX, "fcvt.h.wu">, + Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b00010, 0b000, FPR32, FPR16, "fcvt.s.h">, +defm FCVT_H_S : FPUnaryOp_r_frm_m<0b0100010, 0b00000, HFINXmin, "fcvt.h.s">, + Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]>; +defm : FPUnaryOpDynFrmAlias_m; + +defm FCVT_S_H : FPUnaryOp_r_m<0b0100000, 0b00010, 0b000, FHINXmin, "fcvt.s.h">, Sched<[WriteFCvtF16ToF32, ReadFCvtF16ToF32]>; +let Predicates = [HasStdExtZfhOrZfhmin] in { let mayRaiseFPException = 0 in def FMV_X_H : FPUnaryOp_r<0b1110010, 0b00000, 0b000, GPR, FPR16, "fmv.x.h">, Sched<[WriteFMovF16ToI16, ReadFMovF16ToI16]>; @@ -115,45 +170,38 @@ def FMV_H_X : FPUnaryOp_r<0b1111010, 0b00000, 0b000, FPR16, GPR, "fmv.h.x">, Sched<[WriteFMovI16ToF16, ReadFMovI16ToF16]>; } // Predicates = [HasStdExtZfhOrZfhmin] -let Predicates = [HasStdExtZfh] in { - let SchedRW = [WriteFCmp16, ReadFCmp16, ReadFCmp16] in { -def FEQ_H : FPCmp_rr<0b1010010, 0b010, "feq.h", FPR16>; -def FLT_H : FPCmp_rr<0b1010010, 0b001, "flt.h", FPR16>; -def FLE_H : FPCmp_rr<0b1010010, 0b000, "fle.h", FPR16>; +defm FEQ_H : FPCmp_rr_m<0b1010010, 0b010, "feq.h", HINX>; +defm FLT_H : FPCmp_rr_m<0b1010010, 0b001, "flt.h", HINX>; +defm FLE_H : FPCmp_rr_m<0b1010010, 0b000, "fle.h", HINX>; } let mayRaiseFPException = 0 in -def FCLASS_H : FPUnaryOp_r<0b1110010, 0b00000, 0b001, GPR, FPR16, "fclass.h">, - Sched<[WriteFClass16, ReadFClass16]>; -} // Predicates = [HasStdExtZfh] +defm FCLASS_H : FPUnaryOp_r_m<0b1110010, 0b00000, 0b001, XHINX, "fclass.h">, + Sched<[WriteFClass16, ReadFClass16]>; -let Predicates = [HasStdExtZfh, IsRV64] in { -def FCVT_L_H : FPUnaryOp_r_frm<0b1100010, 0b00010, GPR, FPR16, "fcvt.l.h">, - Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>; -def : FPUnaryOpDynFrmAlias; - -def FCVT_LU_H : FPUnaryOp_r_frm<0b1100010, 0b00011, GPR, FPR16, "fcvt.lu.h">, +defm FCVT_L_H : FPUnaryOp_r_frm_m<0b1100010, 0b00010, XHIN64X, "fcvt.l.h">, Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>; -def : FPUnaryOpDynFrmAlias; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, 0b00010, FPR16, GPR, "fcvt.h.l">, - Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_LU_H : FPUnaryOp_r_frm_m<0b1100010, 0b00011, XHIN64X, "fcvt.lu.h">, + Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, 0b00011, FPR16, GPR, "fcvt.h.lu">, +defm FCVT_H_L : FPUnaryOp_r_frm_m<0b1101010, 0b00010, HXIN64X, "fcvt.h.l">, Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>; -def : FPUnaryOpDynFrmAlias; -} // Predicates = [HasStdExtZfh, IsRV64] +defm : FPUnaryOpDynFrmAlias_m; -let Predicates = [HasStdExtZfhOrZfhmin, HasStdExtD] in { -def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, 0b00001, FPR16, FPR64, "fcvt.h.d">, - Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]>; -def : FPUnaryOpDynFrmAlias; +defm FCVT_H_LU : FPUnaryOp_r_frm_m<0b1101010, 0b00011, HXIN64X, "fcvt.h.lu">, + Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>; +defm : FPUnaryOpDynFrmAlias_m; -def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b00010, 0b000, FPR64, FPR16, "fcvt.d.h">, - Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]>; -} // Predicates = [HasStdExtZfhOrZfhmin, HasStdExtD] +defm FCVT_H_D : FPUnaryOp_r_frm_m<0b0100010, 0b00001, HDINXmin, "fcvt.h.d">, + Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]>; +defm : FPUnaryOpDynFrmAlias_m; + +defm FCVT_D_H : FPUnaryOp_r_m<0b0100001, 0b00010, 0b000, DHINXmin, "fcvt.d.h">, + Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]>; //===----------------------------------------------------------------------===// // Assembler Pseudo Instructions (User-Level ISA, Version 2.2, Chapter 20) @@ -186,6 +234,17 @@ def PseudoQuietFLT_H : PseudoQuietFCMP; } } // Predicates = [HasStdExtZfhOrZfhmin] +let Predicates = [HasStdExtZhinx] in { +def : InstAlias<"fmv.h $rd, $rs", (FSGNJ_H_INX FPR16INX:$rd, FPR16INX:$rs, FPR16INX:$rs)>; +def : InstAlias<"fabs.h $rd, $rs", (FSGNJX_H_INX FPR16INX:$rd, FPR16INX:$rs, FPR16INX:$rs)>; +def : InstAlias<"fneg.h $rd, $rs", (FSGNJN_H_INX FPR16INX:$rd, FPR16INX:$rs, FPR16INX:$rs)>; + +def : InstAlias<"fgt.h $rd, $rs, $rt", + (FLT_H_INX GPR:$rd, FPR16INX:$rt, FPR16INX:$rs), 0>; +def : InstAlias<"fge.h $rd, $rs, $rt", + (FLE_H_INX GPR:$rd, FPR16INX:$rt, FPR16INX:$rs), 0>; +} // Predicates = [HasStdExtZhinx] + //===----------------------------------------------------------------------===// // Pseudo-instructions and codegen patterns //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index 8c1c03b51c24..b06af3787b5d 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -66,6 +66,7 @@ def sub_vrm1_5 : ComposedSubRegIndex; def sub_vrm1_6 : ComposedSubRegIndex; def sub_vrm1_7 : ComposedSubRegIndex; +def sub_32_hi : SubRegIndex<32, 32>; } // Namespace = "RISCV" // Integer registers @@ -534,6 +535,35 @@ def VMV0 : RegisterClass<"RISCV", VMaskVTs, 64, (add V0)> { let Size = 64; } +let RegInfos = XLenRI in { +def GPRF16 : RegisterClass<"RISCV", [f16], 16, (add GPR)>; +def GPRF32 : RegisterClass<"RISCV", [f32], 32, (add GPR)>; +def GPRF64 : RegisterClass<"RISCV", [f64], 64, (add GPR)>; +} // RegInfos = XLenRI + +let RegAltNameIndices = [ABIRegAltName] in { + foreach Index = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, + 24, 26, 28, 30] in { + defvar Reg = !cast("X"#Index); + def X#Index#_PD : RISCVRegWithSubRegs("X"#Index), + !cast("X"#!add(Index, 1))], + Reg.AltNames> { + let SubRegIndices = [sub_32, sub_32_hi]; + } + } +} + +let RegInfos = RegInfoByHwMode<[RV64], [RegInfo<64, 64, 64>]> in +def GPRPF64 : RegisterClass<"RISCV", [f64], 64, (add + X10_PD, X12_PD, X14_PD, X16_PD, + X6_PD, + X28_PD, X30_PD, + X8_PD, + X18_PD, X20_PD, X22_PD, X24_PD, X26_PD, + X0_PD, X2_PD, X4_PD +)>; + // The register class is added for inline assembly for vector mask types. def VM : VReg()->hasPreallocatedCall() || MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() || MFI.hasStackMap() || MFI.hasPatchPoint() || - MFI.hasCopyImplyingStackAdjustment()); + (isWin64Prologue(MF) && MFI.hasCopyImplyingStackAdjustment())); } static unsigned getSUBriOpcode(bool IsLP64, int64_t Imm) { @@ -1289,6 +1289,9 @@ bool X86FrameLowering::has128ByteRedZone(const MachineFunction& MF) const { return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Attribute::NoRedZone); } +/// Return true if we need to use the restricted Windows x64 prologue and +/// epilogue code patterns that can be described with WinCFI (.seh_* +/// directives). bool X86FrameLowering::isWin64Prologue(const MachineFunction &MF) const { return MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); } diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 90753b5b4d33..a1c387574ebb 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -33418,6 +33418,20 @@ bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const { return !(VT1 == MVT::i32 && VT2 == MVT::i16); } +bool X86TargetLowering::shouldFoldSelectWithIdentityConstant(unsigned Opcode, + EVT VT) const { + // TODO: This is too general. There are cases where pre-AVX512 codegen would + // benefit. The transform may also be profitable for scalar code. + if (!Subtarget.hasAVX512()) + return false; + if (!Subtarget.hasVLX() && !VT.is512BitVector()) + return false; + if (!VT.isVector()) + return false; + + return true; +} + /// Targets can use this to indicate that they only support *some* /// VECTOR_SHUFFLE operations, those with specific masks. /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values @@ -43108,38 +43122,6 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, } } - // If this extract is from a loaded vector value and will be used as an - // integer, that requires a potentially expensive XMM -> GPR transfer. - // Additionally, if we can convert to a scalar integer load, that will likely - // be folded into a subsequent integer op. - // Note: Unlike the related fold for this in DAGCombiner, this is not limited - // to a single-use of the loaded vector. For the reasons above, we - // expect this to be profitable even if it creates an extra load. - bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) { - return Use->getOpcode() == ISD::STORE || - Use->getOpcode() == ISD::INSERT_VECTOR_ELT || - Use->getOpcode() == ISD::SCALAR_TO_VECTOR; - }); - auto *LoadVec = dyn_cast(InputVector); - if (LoadVec && CIdx && ISD::isNormalLoad(LoadVec) && VT.isInteger() && - SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() && - !LikelyUsedAsVector) { - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - SDValue NewPtr = - TLI.getVectorElementPointer(DAG, LoadVec->getBasePtr(), SrcVT, EltIdx); - unsigned PtrOff = VT.getSizeInBits() * CIdx->getZExtValue() / 8; - MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff); - Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff); - SDValue Load = - DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment, - LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo()); - SDValue Chain = Load.getValue(1); - SDValue From[] = {SDValue(N, 0), SDValue(LoadVec, 1)}; - SDValue To[] = {Load, Chain}; - DAG.ReplaceAllUsesOfValuesWith(From, To, 2); - return SDValue(N, 0); - } - return SDValue(); } diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 3f6d567d3f4d..50c7e2c319f6 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1288,6 +1288,9 @@ namespace llvm { /// from i32 to i8 but not from i32 to i16. bool isNarrowingProfitable(EVT VT1, EVT VT2) const override; + bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, + EVT VT) const override; + /// Given an intrinsic, checks if on the target the intrinsic will need to map /// to a MemIntrinsicNode (touches memory). If this is the case, it returns /// true and stores the intrinsic information into the IntrinsicInfo that was diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 2d88e329e093..4e4f768ed2cb 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -236,7 +236,8 @@ static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, } // Ensure the result has the requested type. - Ptr = IRB.CreateBitOrPointerCast(Ptr, ResTy, Ptr->getName() + ".cast"); + Ptr = IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, ResTy, + Ptr->getName() + ".cast"); LLVM_DEBUG(dbgs() << "Constructed pointer: " << *Ptr << "\n"); return Ptr; @@ -2691,40 +2692,38 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { return true; }; - auto InspectReturnInstForUB = - [&](Value &V, const SmallSetVector RetInsts) { - // Check if a return instruction always cause UB or not - // Note: It is guaranteed that the returned position of the anchor - // scope has noundef attribute when this is called. - // We also ensure the return position is not "assumed dead" - // because the returned value was then potentially simplified to - // `undef` in AAReturnedValues without removing the `noundef` - // attribute yet. + auto InspectReturnInstForUB = [&](Instruction &I) { + auto &RI = cast(I); + // Either we stopped and the appropriate action was taken, + // or we got back a simplified return value to continue. + Optional SimplifiedRetValue = + stopOnUndefOrAssumed(A, RI.getReturnValue(), &I); + if (!SimplifiedRetValue.hasValue() || !SimplifiedRetValue.getValue()) + return true; - // When the returned position has noundef attriubte, UB occur in the - // following cases. - // (1) Returned value is known to be undef. - // (2) The value is known to be a null pointer and the returned - // position has nonnull attribute (because the returned value is - // poison). - bool FoundUB = false; - if (isa(V)) { - FoundUB = true; - } else { - if (isa(V)) { - auto &NonNullAA = A.getAAFor( - *this, IRPosition::returned(*getAnchorScope()), - DepClassTy::NONE); - if (NonNullAA.isKnownNonNull()) - FoundUB = true; - } - } + // Check if a return instruction always cause UB or not + // Note: It is guaranteed that the returned position of the anchor + // scope has noundef attribute when this is called. + // We also ensure the return position is not "assumed dead" + // because the returned value was then potentially simplified to + // `undef` in AAReturnedValues without removing the `noundef` + // attribute yet. - if (FoundUB) - for (ReturnInst *RI : RetInsts) - KnownUBInsts.insert(RI); - return true; - }; + // When the returned position has noundef attriubte, UB occurs in the + // following cases. + // (1) Returned value is known to be undef. + // (2) The value is known to be a null pointer and the returned + // position has nonnull attribute (because the returned value is + // poison). + if (isa(*SimplifiedRetValue)) { + auto &NonNullAA = A.getAAFor( + *this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE); + if (NonNullAA.isKnownNonNull()) + KnownUBInsts.insert(&I); + } + + return true; + }; bool UsedAssumedInformation = false; A.checkForAllInstructions(InspectMemAccessInstForUB, *this, @@ -2747,8 +2746,9 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior { auto &RetPosNoUndefAA = A.getAAFor(*this, ReturnIRP, DepClassTy::NONE); if (RetPosNoUndefAA.isKnownNoUndef()) - A.checkForAllReturnedValuesAndReturnInsts(InspectReturnInstForUB, - *this); + A.checkForAllInstructions(InspectReturnInstForUB, *this, + {Instruction::Ret}, UsedAssumedInformation, + /* CheckBBLivenessOnly */ true); } } @@ -6749,8 +6749,8 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Type *PrivPtrType = PrivType->getPointerTo(); if (Base->getType() != PrivPtrType) - Base = BitCastInst::CreateBitOrPointerCast(Base, PrivPtrType, "", - ACS.getInstruction()); + Base = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( + Base, PrivPtrType, "", ACS.getInstruction()); // Traverse the type, build GEPs and loads. if (auto *PrivStructType = dyn_cast(PrivType)) { @@ -6817,14 +6817,16 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Function &ReplacementFn, Function::arg_iterator ArgIt) { BasicBlock &EntryBB = ReplacementFn.getEntryBlock(); Instruction *IP = &*EntryBB.getFirstInsertionPt(); - Instruction *AI = new AllocaInst(PrivatizableType.getValue(), 0, + const DataLayout &DL = IP->getModule()->getDataLayout(); + unsigned AS = DL.getAllocaAddrSpace(); + Instruction *AI = new AllocaInst(PrivatizableType.getValue(), AS, Arg->getName() + ".priv", IP); createInitialization(PrivatizableType.getValue(), *AI, ReplacementFn, ArgIt->getArgNo(), *IP); if (AI->getType() != Arg->getType()) - AI = - BitCastInst::CreateBitOrPointerCast(AI, Arg->getType(), "", IP); + AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( + AI, Arg->getType(), "", IP); Arg->replaceAllUsesWith(AI); for (CallInst *CI : TailCalls) diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index 2d765fb6ce6d..520b6ebf9e74 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -1458,7 +1458,6 @@ struct OpenMPOpt { case Intrinsic::nvvm_barrier0_and: case Intrinsic::nvvm_barrier0_or: case Intrinsic::nvvm_barrier0_popc: - case Intrinsic::amdgcn_s_barrier: return true; default: break; diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp index 8f5933b7bd71..ddc747a2ca29 100644 --- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp +++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp @@ -655,10 +655,13 @@ Value *InferAddressSpacesImpl::cloneInstructionWithNewAddressSpace( case Instruction::IntToPtr: { assert(isNoopPtrIntCastPair(cast(I), *DL, TTI)); Value *Src = cast(I->getOperand(0))->getOperand(0); - assert(Src->getType()->getPointerAddressSpace() == NewAddrSpace); - if (Src->getType() != NewPtrType) - return new BitCastInst(Src, NewPtrType); - return Src; + if (Src->getType() == NewPtrType) + return Src; + + // If we had a no-op inttoptr/ptrtoint pair, we may still have inferred a + // source address space from a generic pointer source need to insert a cast + // back. + return CastInst::CreatePointerBitCastOrAddrSpaceCast(Src, NewPtrType); } default: llvm_unreachable("Unexpected opcode"); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 3290439ecd07..21c16f07e237 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1701,6 +1701,11 @@ class LoopVectorizationCostModel { private: unsigned NumPredStores = 0; + /// Convenience function that returns the value of vscale_range iff + /// vscale_range.min == vscale_range.max or otherwise returns the value + /// returned by the corresponding TLI method. + Optional getVScaleForTuning() const; + /// \return An upper bound for the vectorization factors for both /// fixed and scalable vectorization, where the minimum-known number of /// elements is a power-of-2 larger than zero. If scalable vectorization is @@ -5600,6 +5605,18 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( return MaxVF; } +Optional LoopVectorizationCostModel::getVScaleForTuning() const { + if (TheFunction->hasFnAttribute(Attribute::VScaleRange)) { + auto Attr = TheFunction->getFnAttribute(Attribute::VScaleRange); + auto Min = Attr.getVScaleRangeMin(); + auto Max = Attr.getVScaleRangeMax(); + if (Max && Min == Max) + return Max; + } + + return TTI.getVScaleForTuning(); +} + bool LoopVectorizationCostModel::isMoreProfitable( const VectorizationFactor &A, const VectorizationFactor &B) const { InstructionCost CostA = A.Cost; @@ -5624,7 +5641,7 @@ bool LoopVectorizationCostModel::isMoreProfitable( // Improve estimate for the vector width if it is scalable. unsigned EstimatedWidthA = A.Width.getKnownMinValue(); unsigned EstimatedWidthB = B.Width.getKnownMinValue(); - if (Optional VScale = TTI.getVScaleForTuning()) { + if (Optional VScale = getVScaleForTuning()) { if (A.Width.isScalable()) EstimatedWidthA *= VScale.getValue(); if (B.Width.isScalable()) @@ -5673,7 +5690,7 @@ VectorizationFactor LoopVectorizationCostModel::selectVectorizationFactor( #ifndef NDEBUG unsigned AssumedMinimumVscale = 1; - if (Optional VScale = TTI.getVScaleForTuning()) + if (Optional VScale = getVScaleForTuning()) AssumedMinimumVscale = VScale.getValue(); unsigned Width = Candidate.Width.isScalable() @@ -5885,8 +5902,20 @@ LoopVectorizationCostModel::selectEpilogueVectorizationFactor( return Result; } + // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know + // the main loop handles 8 lanes per iteration. We could still benefit from + // vectorizing the epilogue loop with VF=4. + ElementCount EstimatedRuntimeVF = MainLoopVF; + if (MainLoopVF.isScalable()) { + EstimatedRuntimeVF = ElementCount::getFixed(MainLoopVF.getKnownMinValue()); + if (Optional VScale = getVScaleForTuning()) + EstimatedRuntimeVF *= VScale.getValue(); + } + for (auto &NextVF : ProfitableVFs) - if (ElementCount::isKnownLT(NextVF.Width, MainLoopVF) && + if (((!NextVF.Width.isScalable() && MainLoopVF.isScalable() && + ElementCount::isKnownLT(NextVF.Width, EstimatedRuntimeVF)) || + ElementCount::isKnownLT(NextVF.Width, MainLoopVF)) && (Result.Width.isScalar() || isMoreProfitable(NextVF, Result)) && LVP.hasPlanWithVF(NextVF.Width)) Result = NextVF; diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 15b349f53fd9..25bf69729c70 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -471,36 +471,17 @@ static bool isValidForAlternation(unsigned Opcode) { return true; } -static InstructionsState getSameOpcode(ArrayRef VL, - unsigned BaseIndex = 0); - -/// Checks if the provided operands of 2 cmp instructions are compatible, i.e. -/// compatible instructions or constants, or just some other regular values. -static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0, - Value *Op1) { - return (isConstant(BaseOp0) && isConstant(Op0)) || - (isConstant(BaseOp1) && isConstant(Op1)) || - (!isa(BaseOp0) && !isa(Op0) && - !isa(BaseOp1) && !isa(Op1)) || - getSameOpcode({BaseOp0, Op0}).getOpcode() || - getSameOpcode({BaseOp1, Op1}).getOpcode(); -} - /// \returns analysis of the Instructions in \p VL described in /// InstructionsState, the Opcode that we suppose the whole list /// could be vectorized even if its structure is diverse. static InstructionsState getSameOpcode(ArrayRef VL, - unsigned BaseIndex) { + unsigned BaseIndex = 0) { // Make sure these are all Instructions. if (llvm::any_of(VL, [](Value *V) { return !isa(V); })) return InstructionsState(VL[BaseIndex], nullptr, nullptr); bool IsCastOp = isa(VL[BaseIndex]); bool IsBinOp = isa(VL[BaseIndex]); - bool IsCmpOp = isa(VL[BaseIndex]); - CmpInst::Predicate BasePred = - IsCmpOp ? cast(VL[BaseIndex])->getPredicate() - : CmpInst::BAD_ICMP_PREDICATE; unsigned Opcode = cast(VL[BaseIndex])->getOpcode(); unsigned AltOpcode = Opcode; unsigned AltIndex = BaseIndex; @@ -533,57 +514,6 @@ static InstructionsState getSameOpcode(ArrayRef VL, continue; } } - } else if (IsCmpOp && isa(VL[Cnt])) { - auto *BaseInst = cast(VL[BaseIndex]); - auto *Inst = cast(VL[Cnt]); - Type *Ty0 = BaseInst->getOperand(0)->getType(); - Type *Ty1 = Inst->getOperand(0)->getType(); - if (Ty0 == Ty1) { - Value *BaseOp0 = BaseInst->getOperand(0); - Value *BaseOp1 = BaseInst->getOperand(1); - Value *Op0 = Inst->getOperand(0); - Value *Op1 = Inst->getOperand(1); - CmpInst::Predicate CurrentPred = - cast(VL[Cnt])->getPredicate(); - CmpInst::Predicate SwappedCurrentPred = - CmpInst::getSwappedPredicate(CurrentPred); - // Check for compatible operands. If the corresponding operands are not - // compatible - need to perform alternate vectorization. - if (InstOpcode == Opcode) { - if (BasePred == CurrentPred && - areCompatibleCmpOps(BaseOp0, BaseOp1, Op0, Op1)) - continue; - if (BasePred == SwappedCurrentPred && - areCompatibleCmpOps(BaseOp0, BaseOp1, Op1, Op0)) - continue; - if (E == 2 && - (BasePred == CurrentPred || BasePred == SwappedCurrentPred)) - continue; - auto *AltInst = cast(VL[AltIndex]); - CmpInst::Predicate AltPred = AltInst->getPredicate(); - Value *AltOp0 = AltInst->getOperand(0); - Value *AltOp1 = AltInst->getOperand(1); - // Check if operands are compatible with alternate operands. - if (AltPred == CurrentPred && - areCompatibleCmpOps(AltOp0, AltOp1, Op0, Op1)) - continue; - if (AltPred == SwappedCurrentPred && - areCompatibleCmpOps(AltOp0, AltOp1, Op1, Op0)) - continue; - } - if (BaseIndex == AltIndex) { - assert(isValidForAlternation(Opcode) && - isValidForAlternation(InstOpcode) && - "Cast isn't safe for alternation, logic needs to be updated!"); - AltIndex = Cnt; - continue; - } - auto *AltInst = cast(VL[AltIndex]); - CmpInst::Predicate AltPred = AltInst->getPredicate(); - if (BasePred == CurrentPred || BasePred == SwappedCurrentPred || - AltPred == CurrentPred || AltPred == SwappedCurrentPred) - continue; - } } else if (InstOpcode == Opcode || InstOpcode == AltOpcode) continue; return InstructionsState(VL[BaseIndex], nullptr, nullptr); @@ -4424,41 +4354,9 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n"); // Reorder operands if reordering would enable vectorization. - auto *CI = dyn_cast(VL0); - if (isa(VL0) || CI) { + if (isa(VL0)) { ValueList Left, Right; - if (!CI || all_of(VL, [](Value *V) { - return cast(V)->isCommutative(); - })) { - reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); - } else { - CmpInst::Predicate P0 = CI->getPredicate(); - CmpInst::Predicate AltP0 = cast(S.AltOp)->getPredicate(); - CmpInst::Predicate AltP0Swapped = CmpInst::getSwappedPredicate(AltP0); - Value *BaseOp0 = VL0->getOperand(0); - Value *BaseOp1 = VL0->getOperand(1); - // Collect operands - commute if it uses the swapped predicate or - // alternate operation. - for (Value *V : VL) { - auto *Cmp = cast(V); - Value *LHS = Cmp->getOperand(0); - Value *RHS = Cmp->getOperand(1); - CmpInst::Predicate CurrentPred = CI->getPredicate(); - CmpInst::Predicate CurrentPredSwapped = - CmpInst::getSwappedPredicate(CurrentPred); - if (P0 == AltP0 || P0 == AltP0Swapped) { - if ((P0 == CurrentPred && - !areCompatibleCmpOps(BaseOp0, BaseOp1, LHS, RHS)) || - (P0 == CurrentPredSwapped && - !areCompatibleCmpOps(BaseOp0, BaseOp1, RHS, LHS))) - std::swap(LHS, RHS); - } else if (!areCompatibleCmpOps(BaseOp0, BaseOp1, LHS, RHS)) { - std::swap(LHS, RHS); - } - Left.push_back(LHS); - Right.push_back(RHS); - } - } + reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); TE->setOperand(0, Left); TE->setOperand(1, Right); buildTree_rec(Left, Depth + 1, {TE, 0}); @@ -5390,8 +5288,7 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && - Instruction::isCast(E->getAltOpcode())) || - (isa(VL0) && isa(E->getAltOp()))) && + Instruction::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"); InstructionCost ScalarCost = 0; if (NeedToShuffleReuses) { @@ -5439,14 +5336,6 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind); VecCost += TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy, CostKind); - } else if (auto *CI0 = dyn_cast(VL0)) { - VecCost = TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, - Builder.getInt1Ty(), - CI0->getPredicate(), CostKind, VL0); - VecCost += TTI->getCmpSelInstrCost( - E->getOpcode(), ScalarTy, Builder.getInt1Ty(), - cast(E->getAltOp())->getPredicate(), CostKind, - E->getAltOp()); } else { Type *Src0SclTy = E->getMainOp()->getOperand(0)->getType(); Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType(); @@ -5463,29 +5352,6 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, E->Scalars, E->ReorderIndices, E->ReuseShuffleIndices, [E](Instruction *I) { assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"); - if (auto *CI0 = dyn_cast(E->getMainOp())) { - auto *AltCI0 = cast(E->getAltOp()); - auto *CI = cast(I); - CmpInst::Predicate P0 = CI0->getPredicate(); - CmpInst::Predicate AltP0 = AltCI0->getPredicate(); - CmpInst::Predicate AltP0Swapped = - CmpInst::getSwappedPredicate(AltP0); - CmpInst::Predicate CurrentPred = CI->getPredicate(); - CmpInst::Predicate CurrentPredSwapped = - CmpInst::getSwappedPredicate(CurrentPred); - if (P0 == AltP0 || P0 == AltP0Swapped) { - // Alternate cmps have same/swapped predicate as main cmps but - // different order of compatible operands. - return !( - (P0 == CurrentPred && - areCompatibleCmpOps(CI0->getOperand(0), CI0->getOperand(1), - I->getOperand(0), I->getOperand(1))) || - (P0 == CurrentPredSwapped && - areCompatibleCmpOps(CI0->getOperand(0), CI0->getOperand(1), - I->getOperand(1), I->getOperand(0)))); - } - return CurrentPred != P0 && CurrentPredSwapped != P0; - } return I->getOpcode() == E->getAltOpcode(); }, Mask); @@ -6968,12 +6834,11 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && - Instruction::isCast(E->getAltOpcode())) || - (isa(VL0) && isa(E->getAltOp()))) && + Instruction::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"); Value *LHS = nullptr, *RHS = nullptr; - if (Instruction::isBinaryOp(E->getOpcode()) || isa(VL0)) { + if (Instruction::isBinaryOp(E->getOpcode())) { setInsertPointAfterBundle(E); LHS = vectorizeTree(E->getOperand(0)); RHS = vectorizeTree(E->getOperand(1)); @@ -6993,15 +6858,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { static_cast(E->getOpcode()), LHS, RHS); V1 = Builder.CreateBinOp( static_cast(E->getAltOpcode()), LHS, RHS); - } else if (auto *CI0 = dyn_cast(VL0)) { - V0 = Builder.CreateCmp(CI0->getPredicate(), LHS, RHS); - auto *AltCI = cast(E->getAltOp()); - CmpInst::Predicate AltPred = AltCI->getPredicate(); - unsigned AltIdx = - std::distance(E->Scalars.begin(), find(E->Scalars, AltCI)); - if (AltCI->getOperand(0) != E->getOperand(0)[AltIdx]) - AltPred = CmpInst::getSwappedPredicate(AltPred); - V1 = Builder.CreateCmp(AltPred, LHS, RHS); } else { V0 = Builder.CreateCast( static_cast(E->getOpcode()), LHS, VecTy); @@ -7026,29 +6882,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { E->Scalars, E->ReorderIndices, E->ReuseShuffleIndices, [E](Instruction *I) { assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"); - if (auto *CI0 = dyn_cast(E->getMainOp())) { - auto *AltCI0 = cast(E->getAltOp()); - auto *CI = cast(I); - CmpInst::Predicate P0 = CI0->getPredicate(); - CmpInst::Predicate AltP0 = AltCI0->getPredicate(); - CmpInst::Predicate AltP0Swapped = - CmpInst::getSwappedPredicate(AltP0); - CmpInst::Predicate CurrentPred = CI->getPredicate(); - CmpInst::Predicate CurrentPredSwapped = - CmpInst::getSwappedPredicate(CurrentPred); - if (P0 == AltP0 || P0 == AltP0Swapped) { - // Alternate cmps have same/swapped predicate as main cmps but - // different order of compatible operands. - return !( - (P0 == CurrentPred && - areCompatibleCmpOps(CI0->getOperand(0), CI0->getOperand(1), - I->getOperand(0), I->getOperand(1))) || - (P0 == CurrentPredSwapped && - areCompatibleCmpOps(CI0->getOperand(0), CI0->getOperand(1), - I->getOperand(1), I->getOperand(0)))); - } - return CurrentPred != P0 && CurrentPredSwapped != P0; - } return I->getOpcode() == E->getAltOpcode(); }, Mask, &OpScalars, &AltScalars); diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp index 66a2e703129b..0315413ea0c3 100644 --- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp +++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp @@ -19,6 +19,7 @@ #include "llvm/Config/config.h" #include "llvm/DebugInfo/Symbolize/DIPrinter.h" #include "llvm/DebugInfo/Symbolize/Symbolize.h" +#include "llvm/Debuginfod/DIFetcher.h" #include "llvm/Debuginfod/HTTPClient.h" #include "llvm/Option/Arg.h" #include "llvm/Option/ArgList.h" @@ -262,8 +263,6 @@ static FunctionNameKind decideHowToPrintFunctions(const opt::InputArgList &Args, int main(int argc, char **argv) { InitLLVM X(argc, argv); - // The HTTPClient must be initialized for use by the debuginfod client. - HTTPClient::initialize(); sys::InitializeCOMRAII COM(sys::COMThreadingMode::MultiThreaded); bool IsAddr2Line = sys::path::stem(argv[0]).contains("addr2line"); @@ -330,6 +329,12 @@ int main(int argc, char **argv) { } LLVMSymbolizer Symbolizer(Opts); + + // Look up symbols using the debuginfod client. + Symbolizer.addDIFetcher(std::make_unique()); + // The HTTPClient must be initialized for use by the debuginfod client. + HTTPClient::initialize(); + std::unique_ptr Printer; if (Style == OutputStyle::GNU) Printer = std::make_unique(outs(), errs(), Config); diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp index 71e8b7fd10eb..d1f1b6790e2d 100644 --- a/openmp/runtime/src/kmp_affinity.cpp +++ b/openmp/runtime/src/kmp_affinity.cpp @@ -948,7 +948,7 @@ bool kmp_topology_t::filter_hw_subset() { bool using_core_effs = false; int hw_subset_depth = __kmp_hw_subset->get_depth(); kmp_hw_t specified[KMP_HW_LAST]; - int topology_levels[hw_subset_depth]; + int *topology_levels = (int *)KMP_ALLOCA(sizeof(int) * hw_subset_depth); KMP_ASSERT(hw_subset_depth > 0); KMP_FOREACH_HW_TYPE(i) { specified[i] = KMP_HW_UNKNOWN; } int core_level = get_level(KMP_HW_CORE);