From 77fc4c146f0870ffb09c1afb823ccbe742c5e6ff Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Sat, 25 Dec 2021 23:30:44 +0100 Subject: [PATCH] Vendor import of llvm-project main llvmorg-14-init-13186-g0c553cc1af2e. --- clang/include/clang/APINotes/Types.h | 10 +- clang/include/clang/AST/ASTContext.h | 21 +- clang/include/clang/AST/ASTDiagnostic.h | 6 + .../clang/AST/ASTImporterLookupTable.h | 4 + clang/include/clang/AST/AbstractBasicReader.h | 2 +- clang/include/clang/AST/DeclTemplate.h | 11 +- clang/include/clang/AST/Expr.h | 6 + clang/include/clang/AST/OpenMPClause.h | 41 + clang/include/clang/AST/PropertiesBase.td | 2 + clang/include/clang/AST/RecursiveASTVisitor.h | 23 +- clang/include/clang/AST/TextNodeDumper.h | 1 + clang/include/clang/AST/Type.h | 46 +- clang/include/clang/AST/TypeLoc.h | 22 +- clang/include/clang/AST/TypeProperties.td | 23 +- clang/include/clang/ASTMatchers/ASTMatchers.h | 56 +- .../clang/ASTMatchers/ASTMatchersInternal.h | 6 + .../Analysis/Analyses/ThreadSafetyCommon.h | 2 +- .../FlowSensitive/DataflowEnvironment.h | 11 +- .../Analysis/FlowSensitive/DataflowWorklist.h | 9 +- .../TypeErasedDataflowAnalysis.h | 18 + clang/include/clang/Basic/Attr.td | 8 + clang/include/clang/Basic/AttrDocs.td | 48 + clang/include/clang/Basic/Builtins.def | 4 +- .../Basic/BuiltinsAArch64NeonSVEBridge.def | 39 + .../Basic/BuiltinsAArch64NeonSVEBridge_cg.def | 39 + clang/include/clang/Basic/BuiltinsHexagon.def | 10 +- .../clang/Basic/BuiltinsHexagonDep.def | 147 + .../Basic/BuiltinsHexagonMapCustomDep.def | 192 - clang/include/clang/Basic/BuiltinsSVE.def | 1 + clang/include/clang/Basic/CodeGenOptions.def | 3 - clang/include/clang/Basic/CodeGenOptions.h | 5 +- clang/include/clang/Basic/Cuda.h | 5 +- .../clang/Basic/DiagnosticDriverKinds.td | 31 +- clang/include/clang/Basic/DiagnosticGroups.td | 15 +- .../clang/Basic/DiagnosticParseKinds.td | 9 + .../clang/Basic/DiagnosticSemaKinds.td | 71 +- clang/include/clang/Basic/Module.h | 4 + clang/include/clang/Basic/Specifiers.h | 2 +- clang/include/clang/Basic/TargetInfo.h | 8 +- clang/include/clang/Basic/TokenKinds.def | 1 + clang/include/clang/Basic/TypeNodes.td | 5 +- clang/include/clang/Driver/Driver.h | 15 + clang/include/clang/Driver/Job.h | 4 + clang/include/clang/Driver/Options.td | 52 +- clang/include/clang/Driver/Tool.h | 1 + clang/include/clang/Driver/ToolChain.h | 8 +- clang/include/clang/Format/Format.h | 1 + clang/include/clang/Lex/ModuleMap.h | 7 +- clang/include/clang/Parse/Parser.h | 4 + clang/include/clang/Sema/DeclSpec.h | 6 +- clang/include/clang/Sema/ParsedAttr.h | 1 + clang/include/clang/Sema/Sema.h | 27 +- .../clang/Serialization/TypeBitCodes.def | 5 +- .../StaticAnalyzer/Checkers/SValExplainer.h | 10 +- .../StaticAnalyzer/Core/AnalyzerOptions.def | 12 + .../Core/PathSensitive/CallDescription.h | 6 + .../PathSensitive/RangedConstraintManager.h | 27 + .../StaticAnalyzer/Core/PathSensitive/Store.h | 6 +- clang/include/clang/Testing/TestClangConfig.h | 2 +- .../DependencyScanningFilesystem.h | 192 +- .../clang/Tooling/Inclusions/IncludeStyle.h | 2 +- clang/include/clang/module.modulemap | 2 + clang/lib/ARCMigrate/ARCMT.cpp | 4 +- clang/lib/AST/ASTContext.cpp | 106 +- clang/lib/AST/ASTDiagnostic.cpp | 72 +- clang/lib/AST/ASTImporter.cpp | 51 +- clang/lib/AST/ASTImporterLookupTable.cpp | 5 + clang/lib/AST/ASTStructuralEquivalence.cpp | 18 +- clang/lib/AST/AttrImpl.cpp | 2 +- clang/lib/AST/Comment.cpp | 7 +- clang/lib/AST/CommentBriefParser.cpp | 15 +- clang/lib/AST/Decl.cpp | 12 +- clang/lib/AST/DeclarationName.cpp | 4 +- clang/lib/AST/Expr.cpp | 17 + clang/lib/AST/ExprConstant.cpp | 98 +- clang/lib/AST/ItaniumMangle.cpp | 37 +- clang/lib/AST/JSONNodeDumper.cpp | 17 +- clang/lib/AST/MicrosoftMangle.cpp | 10 +- clang/lib/AST/OpenMPClause.cpp | 8 +- clang/lib/AST/ParentMap.cpp | 3 +- clang/lib/AST/QualTypeNames.cpp | 7 + clang/lib/AST/StmtProfile.cpp | 2 + clang/lib/AST/TextNodeDumper.cpp | 4 + clang/lib/AST/Type.cpp | 62 +- clang/lib/AST/TypePrinter.cpp | 32 +- clang/lib/ASTMatchers/ASTMatchersInternal.cpp | 1 + clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp | 4 +- clang/lib/ASTMatchers/Dynamic/Marshallers.h | 1 - clang/lib/ASTMatchers/Dynamic/Parser.cpp | 2 +- clang/lib/ASTMatchers/Dynamic/Registry.cpp | 1 + clang/lib/Analysis/AnalysisDeclContext.cpp | 2 +- clang/lib/Analysis/CFG.cpp | 2 - .../TypeErasedDataflowAnalysis.cpp | 129 +- clang/lib/Analysis/ThreadSafety.cpp | 3 - clang/lib/Analysis/UninitializedValues.cpp | 17 +- clang/lib/Basic/Cuda.cpp | 1 + clang/lib/Basic/OpenMPKinds.cpp | 2 + clang/lib/Basic/SourceLocation.cpp | 4 +- clang/lib/Basic/Targets/AArch64.cpp | 11 +- clang/lib/Basic/Targets/AArch64.h | 2 +- clang/lib/Basic/Targets/AMDGPU.h | 2 +- clang/lib/Basic/Targets/ARC.h | 2 +- clang/lib/Basic/Targets/ARM.cpp | 13 +- clang/lib/Basic/Targets/ARM.h | 6 +- clang/lib/Basic/Targets/Hexagon.cpp | 9 +- clang/lib/Basic/Targets/Hexagon.h | 2 +- clang/lib/Basic/Targets/Lanai.h | 2 +- clang/lib/Basic/Targets/Mips.h | 2 +- clang/lib/Basic/Targets/NVPTX.cpp | 1 + clang/lib/Basic/Targets/NVPTX.h | 4 +- clang/lib/Basic/Targets/OSTargets.cpp | 73 +- clang/lib/Basic/Targets/OSTargets.h | 48 +- clang/lib/Basic/Targets/PNaCl.h | 2 +- clang/lib/Basic/Targets/PPC.cpp | 5 +- clang/lib/Basic/Targets/PPC.h | 2 +- clang/lib/Basic/Targets/RISCV.cpp | 3 + clang/lib/Basic/Targets/RISCV.h | 2 +- clang/lib/Basic/Targets/SPIR.h | 2 +- clang/lib/Basic/Targets/Sparc.h | 7 +- clang/lib/Basic/Targets/SystemZ.h | 2 +- clang/lib/Basic/Targets/WebAssembly.h | 2 +- clang/lib/Basic/Targets/X86.h | 9 +- clang/lib/Basic/Targets/XCore.h | 2 +- clang/lib/Basic/Version.cpp | 6 +- clang/lib/CodeGen/ABIInfo.h | 2 +- clang/lib/CodeGen/Address.h | 64 +- clang/lib/CodeGen/BackendUtil.cpp | 22 +- clang/lib/CodeGen/CGAtomic.cpp | 19 +- clang/lib/CodeGen/CGBlocks.cpp | 3 +- clang/lib/CodeGen/CGBuilder.h | 46 +- clang/lib/CodeGen/CGBuiltin.cpp | 199 +- clang/lib/CodeGen/CGCUDANV.cpp | 3 + clang/lib/CodeGen/CGCall.cpp | 57 +- clang/lib/CodeGen/CGCall.h | 3 +- clang/lib/CodeGen/CGClass.cpp | 50 +- clang/lib/CodeGen/CGCleanup.h | 11 +- clang/lib/CodeGen/CGCoroutine.cpp | 4 + clang/lib/CodeGen/CGDebugInfo.cpp | 19 +- clang/lib/CodeGen/CGDebugInfo.h | 2 +- clang/lib/CodeGen/CGDecl.cpp | 10 +- clang/lib/CodeGen/CGDeclCXX.cpp | 12 +- clang/lib/CodeGen/CGException.cpp | 8 +- clang/lib/CodeGen/CGExpr.cpp | 80 +- clang/lib/CodeGen/CGExprAgg.cpp | 11 +- clang/lib/CodeGen/CGExprCXX.cpp | 18 +- clang/lib/CodeGen/CGExprConstant.cpp | 7 +- clang/lib/CodeGen/CGExprScalar.cpp | 64 +- clang/lib/CodeGen/CGNonTrivialStruct.cpp | 33 +- clang/lib/CodeGen/CGObjC.cpp | 8 +- clang/lib/CodeGen/CGObjCGNU.cpp | 12 +- clang/lib/CodeGen/CGObjCMac.cpp | 5 +- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 106 +- clang/lib/CodeGen/CGOpenMPRuntime.h | 26 +- clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 39 +- clang/lib/CodeGen/CGOpenMPRuntimeGPU.h | 5 +- clang/lib/CodeGen/CGStmt.cpp | 2 +- clang/lib/CodeGen/CGStmtOpenMP.cpp | 76 +- clang/lib/CodeGen/CGValue.h | 76 +- clang/lib/CodeGen/CodeGenAction.cpp | 1 - clang/lib/CodeGen/CodeGenFunction.cpp | 71 +- clang/lib/CodeGen/CodeGenFunction.h | 34 +- clang/lib/CodeGen/CodeGenModule.cpp | 54 +- clang/lib/CodeGen/CodeGenModule.h | 3 + clang/lib/CodeGen/CodeGenTBAA.cpp | 4 +- clang/lib/CodeGen/CodeGenTypes.cpp | 12 +- clang/lib/CodeGen/ItaniumCXXABI.cpp | 11 +- clang/lib/CodeGen/MicrosoftCXXABI.cpp | 8 +- clang/lib/CodeGen/TargetInfo.cpp | 241 +- clang/lib/Driver/Driver.cpp | 170 +- clang/lib/Driver/Job.cpp | 2 + clang/lib/Driver/ToolChain.cpp | 12 +- clang/lib/Driver/ToolChains/AMDGPU.cpp | 3 +- clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp | 2 +- clang/lib/Driver/ToolChains/Arch/AArch64.cpp | 11 + clang/lib/Driver/ToolChains/Arch/ARM.cpp | 23 +- clang/lib/Driver/ToolChains/Arch/ARM.h | 3 +- clang/lib/Driver/ToolChains/Arch/RISCV.cpp | 22 +- clang/lib/Driver/ToolChains/Clang.cpp | 125 +- clang/lib/Driver/ToolChains/Clang.h | 7 +- clang/lib/Driver/ToolChains/CommonArgs.cpp | 2 +- clang/lib/Driver/ToolChains/Darwin.cpp | 56 +- clang/lib/Driver/ToolChains/Darwin.h | 2 +- clang/lib/Driver/ToolChains/FreeBSD.cpp | 8 +- .../Driver/ToolChains/{HIP.cpp => HIPAMD.cpp} | 198 +- .../lib/Driver/ToolChains/{HIP.h => HIPAMD.h} | 38 +- clang/lib/Driver/ToolChains/HIPSPV.cpp | 292 ++ clang/lib/Driver/ToolChains/HIPSPV.h | 103 + clang/lib/Driver/ToolChains/HIPUtility.cpp | 167 + clang/lib/Driver/ToolChains/HIPUtility.h | 35 + clang/lib/Driver/ToolChains/Hexagon.cpp | 130 +- clang/lib/Driver/ToolChains/Linux.cpp | 17 +- clang/lib/Driver/ToolChains/MSVC.cpp | 10 +- clang/lib/Driver/ToolChains/NetBSD.cpp | 23 +- clang/lib/Driver/ToolChains/SPIRV.cpp | 25 +- clang/lib/Driver/ToolChains/SPIRV.h | 33 + clang/lib/Driver/ToolChains/VEToolchain.cpp | 37 +- clang/lib/Driver/XRayArgs.cpp | 1 + clang/lib/Format/BreakableToken.cpp | 11 +- clang/lib/Format/ContinuationIndenter.cpp | 69 +- clang/lib/Format/Format.cpp | 24 +- clang/lib/Format/FormatToken.cpp | 4 + clang/lib/Format/FormatToken.h | 4 +- clang/lib/Format/FormatTokenLexer.cpp | 17 +- .../lib/Format/NamespaceEndCommentsFixer.cpp | 6 +- clang/lib/Format/TokenAnnotator.cpp | 193 +- clang/lib/Format/UnwrappedLineFormatter.cpp | 53 +- clang/lib/Format/UnwrappedLineParser.cpp | 192 +- clang/lib/Format/UnwrappedLineParser.h | 1 + clang/lib/Format/WhitespaceManager.cpp | 5 +- clang/lib/Frontend/CompilerInstance.cpp | 12 +- clang/lib/Frontend/CompilerInvocation.cpp | 15 +- clang/lib/Frontend/InitPreprocessor.cpp | 8 +- .../lib/Frontend/TestModuleFileExtension.cpp | 2 +- clang/lib/Headers/arm_neon_sve_bridge.h | 184 + clang/lib/Headers/hexagon_protos.h | 11 - clang/lib/Headers/hexagon_types.h | 32 - clang/lib/Headers/hvx_hexagon_protos.h | 1609 ++++-- clang/lib/Headers/opencl-c.h | 670 +-- clang/lib/Headers/unwind.h | 3 +- clang/lib/Lex/ModuleMap.cpp | 16 +- clang/lib/Lex/TokenLexer.cpp | 4 +- clang/lib/Parse/ParseCXXInlineMethods.cpp | 11 +- clang/lib/Parse/ParseDecl.cpp | 33 +- clang/lib/Parse/ParseExpr.cpp | 1 + clang/lib/Parse/ParseExprCXX.cpp | 6 +- clang/lib/Parse/ParseOpenMP.cpp | 1 + clang/lib/Parse/ParseTentative.cpp | 2 + clang/lib/Rewrite/HTMLRewrite.cpp | 2 +- clang/lib/Sema/CodeCompleteConsumer.cpp | 4 +- clang/lib/Sema/DeclSpec.cpp | 12 +- clang/lib/Sema/OpenCLBuiltins.td | 28 +- clang/lib/Sema/Sema.cpp | 4 +- clang/lib/Sema/SemaAttr.cpp | 2 +- clang/lib/Sema/SemaCUDA.cpp | 1 - clang/lib/Sema/SemaCXXScopeSpec.cpp | 12 +- clang/lib/Sema/SemaChecking.cpp | 252 +- clang/lib/Sema/SemaCodeComplete.cpp | 1 - clang/lib/Sema/SemaCoroutine.cpp | 244 +- clang/lib/Sema/SemaDecl.cpp | 73 +- clang/lib/Sema/SemaDeclAttr.cpp | 83 +- clang/lib/Sema/SemaDeclCXX.cpp | 230 +- clang/lib/Sema/SemaExpr.cpp | 31 +- clang/lib/Sema/SemaExprCXX.cpp | 2 +- clang/lib/Sema/SemaLookup.cpp | 2 +- clang/lib/Sema/SemaModule.cpp | 43 +- clang/lib/Sema/SemaOpenMP.cpp | 379 +- clang/lib/Sema/SemaStmtAsm.cpp | 10 +- clang/lib/Sema/SemaTemplate.cpp | 10 +- clang/lib/Sema/SemaTemplateDeduction.cpp | 16 +- clang/lib/Sema/SemaTemplateVariadic.cpp | 2 +- clang/lib/Sema/SemaType.cpp | 51 +- clang/lib/Sema/TreeTransform.h | 98 +- clang/lib/Serialization/ASTReader.cpp | 15 +- clang/lib/Serialization/ASTReaderDecl.cpp | 10 +- clang/lib/Serialization/ASTWriter.cpp | 12 +- .../Checkers/BuiltinFunctionChecker.cpp | 3 +- .../Checkers/ConversionChecker.cpp | 21 +- .../Checkers/FuchsiaHandleChecker.cpp | 10 +- clang/lib/StaticAnalyzer/Checkers/SmartPtr.h | 2 - clang/lib/StaticAnalyzer/Core/CallEvent.cpp | 9 +- clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 22 +- .../StaticAnalyzer/Core/HTMLDiagnostics.cpp | 8 +- clang/lib/StaticAnalyzer/Core/MemRegion.cpp | 2 +- .../Core/RangeConstraintManager.cpp | 243 +- clang/lib/StaticAnalyzer/Core/SValBuilder.cpp | 22 +- .../StaticAnalyzer/Core/SimpleSValBuilder.cpp | 44 +- clang/lib/StaticAnalyzer/Core/Store.cpp | 9 +- .../Frontend/AnalysisConsumer.cpp | 51 +- .../DependencyScanningFilesystem.cpp | 195 +- .../DependencyScanning/ModuleDepCollector.cpp | 14 +- clang/lib/Tooling/Syntax/Tokens.cpp | 2 +- clang/lib/Tooling/Syntax/Tree.cpp | 2 +- clang/tools/clang-format/ClangFormat.cpp | 2 +- clang/tools/driver/driver.cpp | 2 +- clang/utils/TableGen/MveEmitter.cpp | 9 +- compiler-rt/include/profile/InstrProfData.inc | 4 +- .../include/sanitizer/dfsan_interface.h | 11 + compiler-rt/lib/asan/asan_activation.cpp | 2 +- compiler-rt/lib/asan/asan_allocator.cpp | 29 +- compiler-rt/lib/asan/asan_debugging.cpp | 4 +- compiler-rt/lib/asan/asan_errors.cpp | 10 +- compiler-rt/lib/asan/asan_fake_stack.cpp | 13 +- compiler-rt/lib/asan/asan_flags.cpp | 6 +- compiler-rt/lib/asan/asan_globals.cpp | 7 +- compiler-rt/lib/asan/asan_interface.inc | 1 + compiler-rt/lib/asan/asan_linux.cpp | 2 +- compiler-rt/lib/asan/asan_mac.cpp | 2 +- compiler-rt/lib/asan/asan_mapping.h | 194 +- compiler-rt/lib/asan/asan_mapping_sparc64.h | 9 +- compiler-rt/lib/asan/asan_poisoning.cpp | 26 +- compiler-rt/lib/asan/asan_poisoning.h | 11 +- compiler-rt/lib/asan/asan_premap_shadow.cpp | 2 +- compiler-rt/lib/asan/asan_rtl.cpp | 41 +- compiler-rt/lib/asan/asan_rtl_x86_64.S | 146 + compiler-rt/lib/asan/asan_thread.cpp | 14 +- compiler-rt/lib/asan/asan_win.cpp | 2 +- compiler-rt/lib/builtins/cpu_model.c | 20 +- compiler-rt/lib/dfsan/dfsan.cpp | 85 +- compiler-rt/lib/dfsan/dfsan_allocator.cpp | 6 + compiler-rt/lib/dfsan/done_abilist.txt | 6 + compiler-rt/lib/hwasan/hwasan_allocator.cpp | 5 + .../lib/hwasan/hwasan_interceptors.cpp | 9 +- compiler-rt/lib/lsan/lsan.h | 23 +- compiler-rt/lib/lsan/lsan_allocator.cpp | 11 +- compiler-rt/lib/lsan/lsan_common.cpp | 480 +- compiler-rt/lib/lsan/lsan_common.h | 44 +- compiler-rt/lib/lsan/lsan_common_fuchsia.cpp | 6 +- compiler-rt/lib/lsan/lsan_common_linux.cpp | 5 +- compiler-rt/lib/lsan/lsan_common_mac.cpp | 5 +- compiler-rt/lib/lsan/lsan_interceptors.cpp | 7 + compiler-rt/lib/memprof/memprof_allocator.cpp | 15 +- compiler-rt/lib/memprof/memprof_allocator.h | 1 - compiler-rt/lib/memprof/memprof_rtl.cpp | 7 - compiler-rt/lib/msan/msan_allocator.cpp | 5 + compiler-rt/lib/msan/msan_interceptors.cpp | 3 + compiler-rt/lib/profile/InstrProfiling.c | 2 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 8 + .../lib/profile/InstrProfilingWriter.c | 21 +- .../sanitizer_common/sanitizer_allocator.cpp | 11 + .../sanitizer_common/sanitizer_allocator.h | 3 + .../lib/sanitizer_common/sanitizer_common.cpp | 8 + .../lib/sanitizer_common/sanitizer_common.h | 18 +- .../sanitizer_common_interceptors.inc | 8 +- .../sanitizer_common_interface_posix.inc | 2 + .../sanitizer_common_libcdep.cpp | 72 +- .../lib/sanitizer_common/sanitizer_flags.inc | 3 + .../sanitizer_common/sanitizer_fuchsia.cpp | 7 +- .../lib/sanitizer_common/sanitizer_linux.cpp | 6 + .../lib/sanitizer_common/sanitizer_linux.h | 3 + .../sanitizer_linux_libcdep.cpp | 26 + .../lib/sanitizer_common/sanitizer_lzw.h | 159 + .../lib/sanitizer_common/sanitizer_mac.cpp | 13 +- .../lib/sanitizer_common/sanitizer_platform.h | 320 +- .../sanitizer_stack_store.cpp | 241 +- .../sanitizer_common/sanitizer_stack_store.h | 25 +- .../sanitizer_common/sanitizer_stackdepot.cpp | 125 +- .../sanitizer_common/sanitizer_stackdepot.h | 1 + .../sanitizer_stacktrace_printer.cpp | 19 + .../sanitizer_stoptheworld_win.cpp | 175 + .../sanitizer_common/sanitizer_symbolizer.cpp | 20 +- .../sanitizer_common/sanitizer_symbolizer.h | 7 +- .../sanitizer_symbolizer_internal.h | 7 +- .../sanitizer_symbolizer_libcdep.cpp | 18 +- .../sanitizer_symbolizer_mac.cpp | 57 +- .../sanitizer_symbolizer_mac.h | 1 - .../sanitizer_symbolizer_markup.cpp | 4 +- .../sanitizer_symbolizer_posix_libcdep.cpp | 58 +- .../sanitizer_symbolizer_win.cpp | 2 +- .../sanitizer_thread_registry.cpp | 18 +- .../sanitizer_thread_registry.h | 7 + .../lib/sanitizer_common/sanitizer_win.cpp | 5 +- .../symbolizer/sanitizer_symbolize.cpp | 36 +- .../symbolizer/scripts/build_symbolizer.sh | 10 +- .../symbolizer/scripts/global_symbols.txt | 14 +- .../lib/sanitizer_common/weak_symbols.txt | 2 + compiler-rt/lib/tsan/go/tsan_go.cpp | 2 +- compiler-rt/lib/tsan/rtl-old/tsan.syms.extra | 31 + .../lib/tsan/{rtl => rtl-old}/tsan_clock.cpp | 0 .../lib/tsan/{rtl => rtl-old}/tsan_clock.h | 0 .../lib/tsan/rtl-old/tsan_debugging.cpp | 262 + compiler-rt/lib/tsan/rtl-old/tsan_defs.h | 236 + .../lib/tsan/rtl-old/tsan_dense_alloc.h | 156 + .../lib/tsan/rtl-old/tsan_dispatch_defs.h | 73 + .../lib/tsan/rtl-old/tsan_external.cpp | 126 + compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp | 316 ++ compiler-rt/lib/tsan/rtl-old/tsan_fd.h | 64 + compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp | 126 + compiler-rt/lib/tsan/rtl-old/tsan_flags.h | 34 + compiler-rt/lib/tsan/rtl-old/tsan_flags.inc | 84 + .../lib/tsan/rtl-old/tsan_ignoreset.cpp | 38 + compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h | 36 + compiler-rt/lib/tsan/rtl-old/tsan_ilist.h | 189 + .../lib/tsan/rtl-old/tsan_interceptors.h | 93 + .../rtl-old/tsan_interceptors_libdispatch.cpp | 814 +++ .../tsan/rtl-old/tsan_interceptors_mac.cpp | 521 ++ .../rtl-old/tsan_interceptors_mach_vm.cpp | 53 + .../tsan/rtl-old/tsan_interceptors_posix.cpp | 3015 +++++++++++ .../lib/tsan/rtl-old/tsan_interface.cpp | 106 + compiler-rt/lib/tsan/rtl-old/tsan_interface.h | 424 ++ .../lib/tsan/rtl-old/tsan_interface.inc | 182 + .../lib/tsan/rtl-old/tsan_interface_ann.cpp | 438 ++ .../lib/tsan/rtl-old/tsan_interface_ann.h | 32 + .../tsan/rtl-old/tsan_interface_atomic.cpp | 920 ++++ .../lib/tsan/rtl-old/tsan_interface_java.cpp | 258 + .../lib/tsan/rtl-old/tsan_interface_java.h | 99 + .../lib/tsan/rtl-old/tsan_malloc_mac.cpp | 71 + compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp | 250 + compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp | 436 ++ compiler-rt/lib/tsan/rtl-old/tsan_mman.h | 78 + .../lib/tsan/rtl-old/tsan_mutexset.cpp | 132 + compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h | 98 + .../lib/tsan/rtl-old/tsan_new_delete.cpp | 199 + compiler-rt/lib/tsan/rtl-old/tsan_platform.h | 988 ++++ .../lib/tsan/rtl-old/tsan_platform_linux.cpp | 545 ++ .../lib/tsan/rtl-old/tsan_platform_mac.cpp | 326 ++ .../lib/tsan/rtl-old/tsan_platform_posix.cpp | 147 + .../tsan/rtl-old/tsan_platform_windows.cpp | 36 + compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h | 96 + compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp | 26 + compiler-rt/lib/tsan/rtl-old/tsan_report.cpp | 479 ++ compiler-rt/lib/tsan/rtl-old/tsan_report.h | 127 + compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp | 811 +++ compiler-rt/lib/tsan/rtl-old/tsan_rtl.h | 796 +++ .../lib/tsan/rtl-old/tsan_rtl_aarch64.S | 245 + .../lib/tsan/rtl-old/tsan_rtl_access.cpp | 604 +++ compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S | 446 ++ .../lib/tsan/rtl-old/tsan_rtl_mips64.S | 214 + .../lib/tsan/rtl-old/tsan_rtl_mutex.cpp | 555 ++ compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S | 288 ++ .../lib/tsan/rtl-old/tsan_rtl_proc.cpp | 60 + .../lib/tsan/rtl-old/tsan_rtl_report.cpp | 984 ++++ compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S | 47 + .../lib/tsan/rtl-old/tsan_rtl_thread.cpp | 349 ++ compiler-rt/lib/tsan/rtl-old/tsan_shadow.h | 233 + .../lib/tsan/rtl-old/tsan_stack_trace.cpp | 57 + .../lib/tsan/rtl-old/tsan_stack_trace.h | 42 + .../lib/tsan/rtl-old/tsan_suppressions.cpp | 161 + .../lib/tsan/rtl-old/tsan_suppressions.h | 37 + .../lib/tsan/rtl-old/tsan_symbolize.cpp | 123 + compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h | 30 + compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp | 279 ++ compiler-rt/lib/tsan/rtl-old/tsan_sync.h | 153 + compiler-rt/lib/tsan/rtl-old/tsan_trace.h | 252 + .../tsan_update_shadow_word.inc | 0 .../lib/tsan/rtl-old/tsan_vector_clock.cpp | 126 + .../lib/tsan/rtl-old/tsan_vector_clock.h | 51 + compiler-rt/lib/tsan/rtl/tsan_debugging.cpp | 2 +- compiler-rt/lib/tsan/rtl/tsan_defs.h | 52 +- compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h | 9 + compiler-rt/lib/tsan/rtl/tsan_fd.cpp | 33 +- compiler-rt/lib/tsan/rtl/tsan_flags.cpp | 6 - compiler-rt/lib/tsan/rtl/tsan_flags.inc | 13 +- compiler-rt/lib/tsan/rtl/tsan_interceptors.h | 12 +- .../lib/tsan/rtl/tsan_interceptors_posix.cpp | 33 +- compiler-rt/lib/tsan/rtl/tsan_interface.cpp | 14 - compiler-rt/lib/tsan/rtl/tsan_interface.inc | 8 + .../lib/tsan/rtl/tsan_interface_atomic.cpp | 87 +- .../lib/tsan/rtl/tsan_interface_java.cpp | 4 +- compiler-rt/lib/tsan/rtl/tsan_mman.cpp | 40 +- compiler-rt/lib/tsan/rtl/tsan_mman.h | 2 + compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp | 54 +- compiler-rt/lib/tsan/rtl/tsan_mutexset.h | 11 +- compiler-rt/lib/tsan/rtl/tsan_platform.h | 285 +- .../lib/tsan/rtl/tsan_platform_linux.cpp | 48 +- .../lib/tsan/rtl/tsan_platform_mac.cpp | 9 +- .../lib/tsan/rtl/tsan_platform_posix.cpp | 18 +- .../lib/tsan/rtl/tsan_platform_windows.cpp | 3 - compiler-rt/lib/tsan/rtl/tsan_report.cpp | 26 +- compiler-rt/lib/tsan/rtl/tsan_report.h | 5 +- compiler-rt/lib/tsan/rtl/tsan_rtl.cpp | 669 ++- compiler-rt/lib/tsan/rtl/tsan_rtl.h | 337 +- compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp | 935 ++-- compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S | 236 - compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp | 646 +-- compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp | 1 - compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp | 365 +- compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 195 +- compiler-rt/lib/tsan/rtl/tsan_shadow.h | 305 +- compiler-rt/lib/tsan/rtl/tsan_sync.cpp | 82 +- compiler-rt/lib/tsan/rtl/tsan_sync.h | 47 +- compiler-rt/lib/tsan/rtl/tsan_trace.h | 73 +- compiler-rt/lib/xray/xray_allocator.h | 16 +- compiler-rt/lib/xray/xray_basic_logging.cpp | 4 +- compiler-rt/lib/xray/xray_hexagon.cpp | 168 + compiler-rt/lib/xray/xray_interface.cpp | 11 +- .../lib/xray/xray_trampoline_hexagon.S | 99 + compiler-rt/lib/xray/xray_tsc.h | 3 +- libcxx/CREDITS.TXT | 15 + libcxx/include/__availability | 7 + libcxx/include/__compare/strong_order.h | 6 +- libcxx/include/__compare/weak_order.h | 14 +- libcxx/include/__config | 45 +- libcxx/include/__debug | 2 +- libcxx/include/__filesystem/copy_options.h | 80 + libcxx/include/__filesystem/directory_entry.h | 504 ++ .../include/__filesystem/directory_iterator.h | 150 + .../include/__filesystem/directory_options.h | 78 + libcxx/include/__filesystem/file_status.h | 68 + libcxx/include/__filesystem/file_time_type.h | 27 + libcxx/include/__filesystem/file_type.h | 39 + .../include/__filesystem/filesystem_error.h | 99 + libcxx/include/__filesystem/operations.h | 599 +++ libcxx/include/__filesystem/path.h | 1018 ++++ libcxx/include/__filesystem/path_iterator.h | 132 + libcxx/include/__filesystem/perm_options.h | 73 + libcxx/include/__filesystem/perms.h | 91 + .../recursive_directory_iterator.h | 181 + libcxx/include/__filesystem/space_info.h | 35 + libcxx/include/__filesystem/u8path.h | 96 + libcxx/include/__format/format_arg.h | 17 +- libcxx/include/__format/format_context.h | 3 +- libcxx/include/__format/formatter_string.h | 1 - libcxx/include/__functional/bind.h | 2 +- libcxx/include/__functional/function.h | 34 +- libcxx/include/__iterator/advance.h | 1 + libcxx/include/__iterator/concepts.h | 4 - libcxx/include/__iterator/next.h | 3 +- libcxx/include/__iterator/prev.h | 1 + libcxx/include/__locale | 5 +- libcxx/include/__memory/allocator.h | 4 +- libcxx/include/__memory/compressed_pair.h | 35 +- libcxx/include/__memory/concepts.h | 66 + libcxx/include/__memory/construct_at.h | 43 +- .../ranges_uninitialized_algorithms.h | 212 + .../__memory/uninitialized_algorithms.h | 200 +- libcxx/include/__memory/voidify.h | 30 + libcxx/include/__mutex_base | 6 +- libcxx/include/__nullptr | 2 +- libcxx/include/__random/clamp_to_integral.h | 60 + .../include/__random/poisson_distribution.h | 1 + libcxx/include/__random/random_device.h | 8 +- libcxx/include/__random/seed_seq.h | 6 +- libcxx/include/__ranges/access.h | 141 +- libcxx/include/__ranges/all.h | 6 +- libcxx/include/__ranges/concepts.h | 4 - libcxx/include/__ranges/counted.h | 65 +- libcxx/include/__ranges/data.h | 8 +- libcxx/include/__ranges/empty.h | 16 +- libcxx/include/__ranges/size.h | 34 +- libcxx/include/__ranges/subrange.h | 4 +- libcxx/include/__ranges/transform_view.h | 16 +- libcxx/include/__string | 49 +- libcxx/include/__threading_support | 65 +- libcxx/include/__tuple | 2 +- .../__utility/{decay_copy.h => auto_cast.h} | 21 +- libcxx/include/__utility/rel_ops.h | 2 +- libcxx/include/__utility/transaction.h | 91 + libcxx/include/atomic | 19 +- libcxx/include/bit | 2 +- libcxx/include/charconv | 34 +- libcxx/include/chrono | 11 +- libcxx/include/cmath | 30 - libcxx/include/compare | 4 + libcxx/include/complex | 4 +- libcxx/include/exception | 2 +- libcxx/include/execution | 4 + libcxx/include/experimental/__memory | 4 + libcxx/include/experimental/iterator | 13 +- libcxx/include/ext/__hash | 2 +- libcxx/include/ext/hash_map | 2 +- libcxx/include/ext/hash_set | 6 +- libcxx/include/filesystem | 2808 +---------- libcxx/include/future | 10 +- libcxx/include/initializer_list | 2 +- libcxx/include/ios | 6 +- libcxx/include/istream | 6 +- libcxx/include/memory | 48 +- libcxx/include/module.modulemap | 314 +- libcxx/include/mutex | 31 +- libcxx/include/new | 2 +- libcxx/include/optional | 172 +- libcxx/include/ostream | 5 +- libcxx/include/random | 1 + libcxx/include/regex | 2 +- libcxx/include/stdexcept | 2 +- libcxx/include/string | 272 +- libcxx/include/string_view | 6 +- libcxx/include/system_error | 8 +- libcxx/include/thread | 7 +- libcxx/include/tuple | 4 +- libcxx/include/type_traits | 42 +- libcxx/include/typeinfo | 2 +- libcxx/include/utility | 6 + libcxx/include/version | 2 + libcxx/src/barrier.cpp | 6 +- libcxx/src/charconv.cpp | 70 +- libcxx/src/chrono.cpp | 6 +- libcxx/src/chrono_system_time_init.h | 2 + libcxx/src/experimental/memory_resource.cpp | 9 +- .../memory_resource_init_helper.h | 2 + libcxx/src/filesystem/directory_iterator.cpp | 3 +- libcxx/src/filesystem/filesystem_common.h | 1 + libcxx/src/format.cpp | 4 - libcxx/src/include/ryu/common.h | 107 + libcxx/src/include/ryu/d2fixed.h | 60 + libcxx/src/include/ryu/d2fixed_full_table.h | 4451 +++++++++++++++++ libcxx/src/include/ryu/d2s.h | 62 + libcxx/src/include/ryu/d2s_full_table.h | 368 ++ libcxx/src/include/ryu/d2s_intrinsics.h | 257 + libcxx/src/include/ryu/digit_table.h | 68 + libcxx/src/include/ryu/f2s.h | 55 + libcxx/src/include/ryu/ryu.h | 148 + libcxx/src/include/to_chars_floating_point.h | 1076 ++++ libcxx/src/ios.cpp | 2 +- libcxx/src/iostream.cpp | 9 +- libcxx/src/iostream_init.h | 2 + libcxx/src/random.cpp | 22 +- libcxx/src/ryu/README.txt | 11 + libcxx/src/ryu/d2fixed.cpp | 669 +++ libcxx/src/ryu/d2s.cpp | 782 +++ libcxx/src/ryu/f2s.cpp | 715 +++ libunwind/include/libunwind.h | 3 +- libunwind/include/unwind_arm_ehabi.h | 7 +- libunwind/src/DwarfInstructions.hpp | 14 + libunwind/src/Registers.hpp | 22 + libunwind/src/Unwind-EHABI.cpp | 47 + libunwind/src/UnwindCursor.hpp | 4 +- libunwind/src/UnwindRegistersRestore.S | 6 + libunwind/src/assembly.h | 13 +- lld/COFF/Driver.cpp | 2 +- lld/COFF/DriverUtils.cpp | 6 - lld/COFF/SymbolTable.cpp | 2 +- lld/ELF/AArch64ErrataFix.cpp | 5 +- lld/ELF/ARMErrataFix.cpp | 5 +- lld/ELF/Arch/AArch64.cpp | 4 +- lld/ELF/Arch/PPC.cpp | 5 +- lld/ELF/CallGraphSort.cpp | 4 +- lld/ELF/Config.h | 7 +- lld/ELF/DWARF.cpp | 3 +- lld/ELF/Driver.cpp | 129 +- lld/ELF/Driver.h | 2 +- lld/ELF/ICF.cpp | 16 + lld/ELF/InputFiles.cpp | 425 +- lld/ELF/InputFiles.h | 175 +- lld/ELF/InputSection.cpp | 119 +- lld/ELF/InputSection.h | 23 +- lld/ELF/LTO.cpp | 6 +- lld/ELF/LinkerScript.cpp | 49 +- lld/ELF/LinkerScript.h | 4 +- lld/ELF/MapFile.cpp | 15 +- lld/ELF/MarkLive.cpp | 85 +- lld/ELF/Options.td | 15 +- lld/ELF/OutputSections.cpp | 14 +- lld/ELF/OutputSections.h | 2 +- lld/ELF/Relocations.cpp | 465 +- lld/ELF/Relocations.h | 1 + lld/ELF/SymbolTable.cpp | 42 +- lld/ELF/SymbolTable.h | 19 +- lld/ELF/Symbols.cpp | 63 +- lld/ELF/Symbols.h | 53 +- lld/ELF/SyntheticSections.cpp | 163 +- lld/ELF/SyntheticSections.h | 84 +- lld/ELF/Thunks.cpp | 7 +- lld/ELF/Writer.cpp | 137 +- lld/MachO/ConcatOutputSection.cpp | 11 +- lld/MachO/InputFiles.cpp | 94 +- lld/MachO/InputFiles.h | 7 +- lld/MachO/InputSection.h | 9 +- lld/MachO/SymbolTable.cpp | 12 +- lld/MachO/SyntheticSections.cpp | 23 +- lld/MachO/Writer.cpp | 25 +- lld/docs/ELF/start-stop-gc.rst | 66 + lld/docs/ReleaseNotes.rst | 2 +- lld/docs/index.rst | 1 + lld/docs/ld.lld.1 | 10 + lld/include/lld/Common/Driver.h | 5 - lld/include/lld/Core/Reference.h | 3 +- .../lld/ReaderWriter/MachOLinkingContext.h | 505 -- lld/include/lld/ReaderWriter/YamlContext.h | 42 - lld/lib/Core/DefinedAtom.cpp | 81 - lld/lib/Core/Error.cpp | 93 - lld/lib/Core/File.cpp | 28 - lld/lib/Core/LinkingContext.cpp | 69 - lld/lib/Core/Reader.cpp | 113 - lld/lib/Core/Resolver.cpp | 496 -- lld/lib/Core/SymbolTable.cpp | 284 -- lld/lib/Core/Writer.cpp | 17 - lld/lib/Driver/DarwinLdDriver.cpp | 1229 ----- lld/lib/Driver/DarwinLdOptions.td | 250 - lld/lib/ReaderWriter/FileArchive.cpp | 227 - lld/lib/ReaderWriter/MachO/ArchHandler.cpp | 171 - lld/lib/ReaderWriter/MachO/ArchHandler.h | 322 -- .../ReaderWriter/MachO/ArchHandler_arm.cpp | 1522 ------ .../ReaderWriter/MachO/ArchHandler_arm64.cpp | 897 ---- .../ReaderWriter/MachO/ArchHandler_x86.cpp | 643 --- .../ReaderWriter/MachO/ArchHandler_x86_64.cpp | 899 ---- lld/lib/ReaderWriter/MachO/Atoms.h | 180 - .../ReaderWriter/MachO/CompactUnwindPass.cpp | 580 --- lld/lib/ReaderWriter/MachO/DebugInfo.h | 106 - lld/lib/ReaderWriter/MachO/ExecutableAtoms.h | 154 - lld/lib/ReaderWriter/MachO/File.h | 467 -- .../ReaderWriter/MachO/FlatNamespaceFile.h | 62 - lld/lib/ReaderWriter/MachO/GOTPass.cpp | 183 - lld/lib/ReaderWriter/MachO/LayoutPass.cpp | 490 -- lld/lib/ReaderWriter/MachO/LayoutPass.h | 118 - .../MachO/MachOLinkingContext.cpp | 1104 ---- .../ReaderWriter/MachO/MachONormalizedFile.h | 336 -- .../MachO/MachONormalizedFileBinaryReader.cpp | 614 --- .../MachO/MachONormalizedFileBinaryUtils.h | 213 - .../MachO/MachONormalizedFileBinaryWriter.cpp | 1560 ------ .../MachO/MachONormalizedFileFromAtoms.cpp | 1657 ------ .../MachO/MachONormalizedFileToAtoms.cpp | 1635 ------ .../MachO/MachONormalizedFileYAML.cpp | 840 ---- lld/lib/ReaderWriter/MachO/MachOPasses.h | 29 - lld/lib/ReaderWriter/MachO/ObjCPass.cpp | 131 - lld/lib/ReaderWriter/MachO/SectCreateFile.h | 101 - lld/lib/ReaderWriter/MachO/ShimPass.cpp | 128 - lld/lib/ReaderWriter/MachO/StubsPass.cpp | 377 -- lld/lib/ReaderWriter/MachO/TLVPass.cpp | 140 - lld/lib/ReaderWriter/MachO/WriterMachO.cpp | 70 - .../ReaderWriter/YAML/ReaderWriterYAML.cpp | 1403 ------ lld/tools/lld/lld.cpp | 15 +- lldb/bindings/interface/SBData.i | 4 + lldb/bindings/lua/lua-swigsafecast.swig | 31 +- lldb/bindings/lua/lua-typemaps.swig | 225 +- lldb/bindings/lua/lua-wrapper.swig | 155 +- lldb/bindings/lua/lua.swig | 1 + lldb/bindings/python/python-swigsafecast.swig | 96 +- lldb/bindings/python/python-typemaps.swig | 348 +- lldb/bindings/python/python-wrapper.swig | 1905 ++++--- lldb/include/lldb/API/SBData.h | 3 + lldb/include/lldb/API/SBStructuredData.h | 2 +- lldb/include/lldb/API/SBSymbolContext.h | 4 +- lldb/include/lldb/API/SBTypeSummary.h | 4 +- .../Breakpoint/BreakpointResolverScripted.h | 11 +- lldb/include/lldb/Core/DataFileCache.h | 216 + lldb/include/lldb/Core/Mangled.h | 41 +- lldb/include/lldb/Core/Module.h | 94 +- lldb/include/lldb/Core/ModuleList.h | 7 + lldb/include/lldb/Core/PluginManager.h | 2 + lldb/include/lldb/Core/StructuredDataImpl.h | 3 + lldb/include/lldb/Core/ValueObject.h | 2 +- lldb/include/lldb/Expression/UserExpression.h | 6 +- lldb/include/lldb/Host/Config.h.cmake | 2 + lldb/include/lldb/Host/FileSystem.h | 8 + .../lldb/Interpreter/CommandReturnObject.h | 8 +- .../lldb/Interpreter/ScriptInterpreter.h | 6 +- lldb/include/lldb/Symbol/ObjectFile.h | 31 +- lldb/include/lldb/Symbol/Symbol.h | 40 + lldb/include/lldb/Symbol/SymbolFile.h | 10 +- lldb/include/lldb/Symbol/Symtab.h | 81 +- lldb/include/lldb/Symbol/Type.h | 46 +- lldb/include/lldb/Target/LanguageRuntime.h | 2 +- lldb/include/lldb/Target/Target.h | 4 +- lldb/include/lldb/Target/ThreadPlanPython.h | 14 +- lldb/include/lldb/Utility/DataEncoder.h | 249 +- lldb/include/lldb/Utility/RangeMap.h | 1 - lldb/include/lldb/Utility/Reproducer.h | 7 - lldb/include/lldb/Version/Version.h | 23 + lldb/include/lldb/Version/Version.inc.in | 6 + lldb/include/lldb/lldb-forward.h | 5 + lldb/include/lldb/lldb-private.h | 6 - lldb/source/API/SBData.cpp | 19 + lldb/source/API/SBDebugger.cpp | 6 +- lldb/source/API/SBFrame.cpp | 6 +- lldb/source/API/SBReproducer.cpp | 97 +- lldb/source/API/SBStructuredData.cpp | 8 +- lldb/source/API/SBSymbolContext.cpp | 17 +- lldb/source/API/SBSymbolContextList.cpp | 5 +- lldb/source/API/SBThreadPlan.cpp | 6 +- lldb/source/API/SBTypeSummary.cpp | 21 +- lldb/source/API/SystemInitializerFull.cpp | 9 +- .../Breakpoint/BreakpointResolverScripted.cpp | 33 +- lldb/source/Commands/CommandObjectMemory.cpp | 2 +- .../Commands/CommandObjectReproducer.cpp | 15 +- lldb/source/Commands/CommandObjectVersion.cpp | 2 +- lldb/source/Core/CoreProperties.td | 20 + lldb/source/Core/DataFileCache.cpp | 307 ++ lldb/source/Core/IOHandlerCursesGUI.cpp | 15 +- lldb/source/Core/Mangled.cpp | 110 + lldb/source/Core/Module.cpp | 72 +- lldb/source/Core/ModuleList.cpp | 49 + lldb/source/Core/PluginManager.cpp | 6 + .../DataFormatters/CXXFunctionPointer.cpp | 31 +- lldb/source/DataFormatters/FormatManager.cpp | 9 +- lldb/source/Expression/DWARFExpression.cpp | 30 +- lldb/source/Expression/IRExecutionUnit.cpp | 2 - lldb/source/Expression/IRMemoryMap.cpp | 4 - lldb/source/Expression/REPL.cpp | 6 +- lldb/source/Expression/UserExpression.cpp | 11 +- lldb/source/Host/common/FileSystem.cpp | 8 + lldb/source/Host/common/ProcessLaunchInfo.cpp | 20 +- .../Host/posix/ProcessLauncherPosixFork.cpp | 1 - .../SystemInitializerCommon.cpp | 2 +- .../source/Interpreter/CommandInterpreter.cpp | 3 - .../Interpreter/CommandReturnObject.cpp | 8 +- .../Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp | 10 + .../Clang/ASTResultSynthesizer.cpp | 2 +- .../Clang/ClangASTImporter.cpp | 4 +- .../ExpressionParser/Clang/ClangASTSource.cpp | 2 - .../ExpressionParser/Clang/ClangASTSource.h | 2 +- .../Clang/ClangExpressionDeclMap.cpp | 2 - .../Clang/IRDynamicChecks.cpp | 2 +- .../ExpressionParser/Clang/IRForTarget.cpp | 2 +- .../Language/CPlusPlus/CPlusPlusLanguage.cpp | 36 +- .../Plugins/Language/CPlusPlus/Generic.h | 25 + .../Language/CPlusPlus/GenericOptional.cpp | 139 + .../Plugins/Language/CPlusPlus/LibCxx.h | 4 +- .../Language/CPlusPlus/LibCxxOptional.cpp | 84 - .../Plugins/Language/CPlusPlus/LibStdcpp.h | 4 + .../AppleObjCRuntime/AppleObjCDeclVendor.cpp | 3 - .../Plugins/Platform/POSIX/PlatformPOSIX.cpp | 2 + .../Platform/QemuUser/PlatformQemuUser.cpp | 90 +- .../Platform/QemuUser/PlatformQemuUser.h | 2 +- .../QemuUser/PlatformQemuUserProperties.td | 12 + .../gdb-server/PlatformRemoteGDBServer.cpp | 16 +- .../FreeBSDKernel/ProcessFreeBSDKernel.cpp | 218 + .../FreeBSDKernel/ProcessFreeBSDKernel.h | 51 + .../RegisterContextFreeBSDKernel_arm64.cpp | 110 + .../RegisterContextFreeBSDKernel_arm64.h | 41 + .../RegisterContextFreeBSDKernel_i386.cpp | 83 + .../RegisterContextFreeBSDKernel_i386.h | 41 + .../RegisterContextFreeBSDKernel_x86_64.cpp | 88 + .../RegisterContextFreeBSDKernel_x86_64.h | 41 + .../FreeBSDKernel/ThreadFreeBSDKernel.cpp | 85 + .../FreeBSDKernel/ThreadFreeBSDKernel.h | 36 + .../Process/elf-core/ProcessElfCore.cpp | 4 + .../GDBRemoteCommunicationClient.cpp | 36 + .../gdb-remote/GDBRemoteCommunicationClient.h | 6 + .../Process/gdb-remote/ProcessGDBRemote.cpp | 102 +- .../Process/scripted/ScriptedProcess.cpp | 5 + .../Process/scripted/ScriptedThread.cpp | 10 +- .../Plugins/ScriptInterpreter/Lua/Lua.cpp | 33 +- .../ScriptInterpreter/Lua/SWIGLuaBridge.h | 28 + .../Python/SWIGPythonBridge.h | 43 +- .../Python/ScriptInterpreterPython.cpp | 52 +- .../Python/ScriptInterpreterPython.h | 11 +- .../Python/ScriptInterpreterPythonImpl.h | 7 +- .../Python/ScriptedProcessPythonInterface.cpp | 6 +- .../Python/ScriptedThreadPythonInterface.cpp | 6 +- .../SymbolFile/DWARF/DWARFASTParserClang.cpp | 5 - .../Plugins/SymbolFile/DWARF/DWARFUnit.h | 2 +- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 11 +- .../SymbolFile/DWARF/SymbolFileDWARF.h | 12 +- .../SymbolFile/NativePDB/PdbAstBuilder.cpp | 133 +- .../SymbolFile/NativePDB/PdbAstBuilder.h | 6 + .../NativePDB/SymbolFileNativePDB.cpp | 2 +- .../NativePDB/UdtRecordCompleter.cpp | 30 +- .../SymbolFile/NativePDB/UdtRecordCompleter.h | 12 +- .../Plugins/SymbolFile/PDB/SymbolFilePDB.cpp | 3 +- .../ctf/CommandObjectThreadTraceExportCTF.cpp | 3 +- .../TypeSystem/Clang/TypeSystemClang.cpp | 19 +- lldb/source/Symbol/ObjectFile.cpp | 25 +- lldb/source/Symbol/Symbol.cpp | 129 + lldb/source/Symbol/SymbolFile.cpp | 12 +- lldb/source/Symbol/Symtab.cpp | 199 +- lldb/source/Symbol/Type.cpp | 2 +- lldb/source/Target/StackFrame.cpp | 24 +- lldb/source/Target/Target.cpp | 19 +- lldb/source/Target/Thread.cpp | 10 +- lldb/source/Target/ThreadPlanPython.cpp | 7 +- lldb/source/Target/ThreadPlanStack.cpp | 1 - lldb/source/Target/UnwindLLDB.cpp | 1 - lldb/source/Utility/DataEncoder.cpp | 157 +- lldb/source/Utility/FileSpec.cpp | 2 +- lldb/source/Utility/Reproducer.cpp | 27 +- lldb/source/{lldb.cpp => Version/Version.cpp} | 26 +- lldb/tools/driver/Driver.cpp | 49 - lldb/tools/driver/Options.td | 11 - lldb/tools/lldb-server/lldb-server.cpp | 2 +- llvm/include/llvm-c/Core.h | 86 +- llvm/include/llvm-c/Deprecated.h | 38 + llvm/include/llvm/ADT/GenericCycleImpl.h | 411 ++ llvm/include/llvm/ADT/GenericCycleInfo.h | 334 ++ llvm/include/llvm/ADT/GenericSSAContext.h | 74 + llvm/include/llvm/ADT/PointerUnion.h | 26 +- llvm/include/llvm/ADT/STLExtras.h | 55 + llvm/include/llvm/ADT/SmallVector.h | 34 +- llvm/include/llvm/ADT/StringRef.h | 4 +- llvm/include/llvm/ADT/Triple.h | 58 +- llvm/include/llvm/Analysis/CycleAnalysis.h | 77 + llvm/include/llvm/Analysis/IVDescriptors.h | 2 +- llvm/include/llvm/Analysis/InlineCost.h | 2 +- llvm/include/llvm/Analysis/MLModelRunner.h | 25 +- llvm/include/llvm/Analysis/MemoryBuiltins.h | 2 +- llvm/include/llvm/Analysis/MemoryLocation.h | 2 + .../llvm/Analysis/ModelUnderTrainingRunner.h | 59 + .../llvm/Analysis/NoInferenceModelRunner.h | 39 + .../llvm/Analysis/ReleaseModeModelRunner.h | 67 + .../llvm/Analysis/TargetTransformInfo.h | 67 +- .../llvm/Analysis/TargetTransformInfoImpl.h | 20 +- llvm/include/llvm/Analysis/Utils/TFUtils.h | 7 +- llvm/include/llvm/AsmParser/LLParser.h | 4 +- llvm/include/llvm/AsmParser/LLToken.h | 2 + llvm/include/llvm/BinaryFormat/ELF.h | 4 + llvm/include/llvm/Bitcode/LLVMBitCodes.h | 7 +- llvm/include/llvm/CodeGen/AsmPrinter.h | 5 + .../GlobalISel/LegalizationArtifactCombiner.h | 2 +- .../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 52 +- .../llvm/CodeGen/GlobalISel/LegalizerInfo.h | 38 +- .../llvm/CodeGen/GlobalISel/MIPatternMatch.h | 4 +- .../CodeGen/GlobalISel/MachineIRBuilder.h | 28 + llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 5 + llvm/include/llvm/CodeGen/MIRYamlMapping.h | 3 + .../llvm/CodeGen/MachineCycleAnalysis.h | 31 + llvm/include/llvm/CodeGen/MachineFunction.h | 21 +- llvm/include/llvm/CodeGen/MachineInstr.h | 20 +- .../llvm/CodeGen/MachinePassRegistry.def | 2 + llvm/include/llvm/CodeGen/MachineSSAContext.h | 58 + llvm/include/llvm/CodeGen/MachineSSAUpdater.h | 12 +- llvm/include/llvm/CodeGen/MachineScheduler.h | 5 + llvm/include/llvm/CodeGen/SelectionDAG.h | 4 +- llvm/include/llvm/CodeGen/StackProtector.h | 2 +- llvm/include/llvm/CodeGen/TargetInstrInfo.h | 6 +- llvm/include/llvm/CodeGen/TargetLowering.h | 32 +- .../llvm/CodeGen/VLIWMachineScheduler.h | 268 + .../llvm/DebugInfo/DWARF/DWARFContext.h | 5 + .../llvm/DebugInfo/DWARF/DWARFDebugLoc.h | 17 + .../llvm/DebugInfo/DWARF/DWARFFormValue.h | 31 +- llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h | 2 +- llvm/include/llvm/DebugInfo/MSF/MSFCommon.h | 20 + llvm/include/llvm/DebugInfo/MSF/MSFError.h | 5 +- .../llvm/DebugInfo/PDB/Native/PDBFile.h | 2 +- llvm/include/llvm/Debuginfod/Debuginfod.h | 71 + .../llvm/{Support => Debuginfod}/HTTPClient.h | 6 + llvm/include/llvm/ExecutionEngine/Orc/Core.h | 27 +- .../llvm/ExecutionEngine/Orc/ExecutionUtils.h | 16 +- llvm/include/llvm/ExecutionEngine/Orc/Layer.h | 31 +- .../llvm/ExecutionEngine/Orc/LazyReexports.h | 3 +- .../llvm/ExecutionEngine/Orc/Mangling.h | 5 - .../ExecutionEngine/Orc/ObjectFileInterface.h | 38 + llvm/include/llvm/Frontend/OpenMP/OMP.td | 4 +- .../llvm/Frontend/OpenMP/OMPIRBuilder.h | 62 +- .../include/llvm/Frontend/OpenMP/OMPKinds.def | 4 +- llvm/include/llvm/IR/Attributes.h | 27 +- llvm/include/llvm/IR/Attributes.td | 3 + llvm/include/llvm/IR/AttributesAMDGPU.td | 14 + llvm/include/llvm/IR/Constants.h | 35 + llvm/include/llvm/IR/DataLayout.h | 33 +- llvm/include/llvm/IR/Instructions.h | 9 + llvm/include/llvm/IR/IntrinsicInst.h | 4 +- llvm/include/llvm/IR/Intrinsics.td | 23 +- llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 8 +- llvm/include/llvm/IR/IntrinsicsARM.td | 3 +- llvm/include/llvm/IR/IntrinsicsHexagonDep.td | 1109 ++-- llvm/include/llvm/IR/IntrinsicsRISCV.td | 28 - llvm/include/llvm/IR/IntrinsicsWebAssembly.td | 32 + llvm/include/llvm/IR/Module.h | 11 + llvm/include/llvm/IR/ModuleSummaryIndex.h | 12 +- llvm/include/llvm/IR/SSAContext.h | 56 + llvm/include/llvm/IR/VPIntrinsics.def | 9 +- llvm/include/llvm/IR/Value.def | 1 + llvm/include/llvm/InitializePasses.h | 6 +- llvm/include/llvm/MC/MCAssembler.h | 16 + llvm/include/llvm/MC/MCObjectFileInfo.h | 19 + llvm/include/llvm/MC/MCObjectStreamer.h | 16 + .../llvm/MC/MCParser/MCParsedAsmOperand.h | 5 + llvm/include/llvm/MC/MCStreamer.h | 14 +- llvm/include/llvm/MC/MCTargetOptions.h | 3 +- llvm/include/llvm/Object/MachO.h | 7 + llvm/include/llvm/Option/ArgList.h | 6 + llvm/include/llvm/Passes/PassBuilder.h | 2 +- llvm/include/llvm/ProfileData/InstrProf.h | 7 +- .../llvm/ProfileData/InstrProfCorrelator.h | 170 + .../llvm/ProfileData/InstrProfData.inc | 4 +- .../llvm/ProfileData/InstrProfReader.h | 26 +- llvm/include/llvm/ProfileData/SampleProf.h | 67 +- .../llvm/ProfileData/SampleProfReader.h | 16 +- .../llvm/ProfileData/SampleProfWriter.h | 1 + llvm/include/llvm/Support/ARMEHABI.h | 4 + llvm/include/llvm/Support/Caching.h | 12 +- llvm/include/llvm/Support/Chrono.h | 12 +- llvm/include/llvm/Support/Compiler.h | 6 +- llvm/include/llvm/Support/GraphWriter.h | 7 +- llvm/include/llvm/Support/RISCVISAInfo.h | 3 + llvm/include/llvm/Support/ScopedPrinter.h | 627 ++- .../llvm/Support/SmallVectorMemoryBuffer.h | 28 +- llvm/include/llvm/Support/TargetParser.h | 4 +- llvm/include/llvm/Support/ThreadPool.h | 21 +- llvm/include/llvm/Support/ToolOutputFile.h | 5 +- llvm/include/llvm/Support/VirtualFileSystem.h | 2 + llvm/include/llvm/Target/TargetOptions.h | 5 + llvm/include/llvm/TextAPI/InterfaceFile.h | 2 + .../llvm/Transforms/IPO/ProfiledCallGraph.h | 3 +- .../Transforms/IPO/SampleContextTracker.h | 2 - .../llvm/Transforms/Scalar/FlattenCFG.h | 25 + .../llvm/Transforms/Utils/CodeLayout.h | 58 + .../Transforms/Utils/FunctionComparator.h | 1 + llvm/include/llvm/Transforms/Utils/Local.h | 8 + .../include/llvm/Transforms/Utils/LoopUtils.h | 9 +- .../Vectorize/LoopVectorizationLegality.h | 47 +- .../llvm/Transforms/Vectorize/LoopVectorize.h | 32 + llvm/include/llvm/module.modulemap | 1 + llvm/lib/Analysis/AliasAnalysis.cpp | 12 +- llvm/lib/Analysis/Analysis.cpp | 1 + llvm/lib/Analysis/BasicAliasAnalysis.cpp | 3 + llvm/lib/Analysis/CaptureTracking.cpp | 7 +- llvm/lib/Analysis/ConstantFolding.cpp | 21 +- llvm/lib/Analysis/CycleAnalysis.cpp | 77 + .../Analysis/DevelopmentModeInlineAdvisor.cpp | 131 +- llvm/lib/Analysis/IVDescriptors.cpp | 2 +- llvm/lib/Analysis/InlineAdvisor.cpp | 11 +- llvm/lib/Analysis/InstructionSimplify.cpp | 112 +- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 21 +- llvm/lib/Analysis/MLInlineAdvisor.cpp | 62 +- llvm/lib/Analysis/MemDerefPrinter.cpp | 8 +- llvm/lib/Analysis/MemoryBuiltins.cpp | 14 +- llvm/lib/Analysis/MemoryLocation.cpp | 81 +- .../lib/Analysis/ModelUnderTrainingRunner.cpp | 49 + llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 18 +- llvm/lib/Analysis/NoInferenceModelRunner.cpp | 33 + llvm/lib/Analysis/ReleaseModeModelRunner.cpp | 90 - llvm/lib/Analysis/ScalarEvolution.cpp | 34 +- llvm/lib/Analysis/TargetLibraryInfo.cpp | 5 +- llvm/lib/Analysis/TargetTransformInfo.cpp | 15 +- llvm/lib/Analysis/ValueTracking.cpp | 32 +- llvm/lib/AsmParser/LLLexer.cpp | 2 + llvm/lib/AsmParser/LLParser.cpp | 36 +- .../BinaryFormat/AMDGPUMetadataVerifier.cpp | 6 +- llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp | 25 +- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 35 +- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 8 +- llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 16 +- llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp | 9 +- llvm/lib/CodeGen/Analysis.cpp | 4 +- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 22 +- llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp | 29 +- llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h | 2 + .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp | 25 +- llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 10 +- .../CodeGen/AsmPrinter/DwarfExpression.cpp | 11 +- llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h | 19 +- llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp | 59 +- llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h | 3 + .../lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp | 39 +- .../CodeGen/AsmPrinter/PseudoProbePrinter.cpp | 3 +- llvm/lib/CodeGen/BranchFolding.cpp | 4 +- llvm/lib/CodeGen/CalcSpillWeights.cpp | 22 +- llvm/lib/CodeGen/CodeGen.cpp | 2 + llvm/lib/CodeGen/CodeGenPrepare.cpp | 11 +- llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp | 3 +- .../CodeGen/DeadMachineInstructionElim.cpp | 6 +- llvm/lib/CodeGen/EarlyIfConversion.cpp | 20 +- llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 36 +- llvm/lib/CodeGen/GlobalISel/Combiner.cpp | 2 +- .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 4 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 9 +- .../CodeGen/GlobalISel/InstructionSelect.cpp | 8 +- .../CodeGen/GlobalISel/LegalizerHelper.cpp | 1117 ++--- llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 5 +- .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 48 +- llvm/lib/CodeGen/GlobalISel/Utils.cpp | 39 +- llvm/lib/CodeGen/ImplicitNullChecks.cpp | 2 +- llvm/lib/CodeGen/InlineSpiller.cpp | 4 +- llvm/lib/CodeGen/InterferenceCache.cpp | 4 +- .../LiveDebugValues/InstrRefBasedImpl.cpp | 4 +- .../LiveDebugValues/VarLocBasedImpl.cpp | 16 +- llvm/lib/CodeGen/LiveDebugVariables.cpp | 11 +- llvm/lib/CodeGen/LiveDebugVariables.h | 5 + llvm/lib/CodeGen/LiveRangeEdit.cpp | 16 + llvm/lib/CodeGen/LiveVariables.cpp | 14 +- llvm/lib/CodeGen/LocalStackSlotAllocation.cpp | 6 +- llvm/lib/CodeGen/MIRParser/MIRParser.cpp | 18 + llvm/lib/CodeGen/MIRPrinter.cpp | 2 + llvm/lib/CodeGen/MachineBasicBlock.cpp | 31 +- llvm/lib/CodeGen/MachineBlockPlacement.cpp | 161 +- llvm/lib/CodeGen/MachineCombiner.cpp | 4 +- llvm/lib/CodeGen/MachineCopyPropagation.cpp | 28 +- llvm/lib/CodeGen/MachineCycleAnalysis.cpp | 113 + llvm/lib/CodeGen/MachineFunction.cpp | 32 +- llvm/lib/CodeGen/MachineInstr.cpp | 32 +- llvm/lib/CodeGen/MachinePipeliner.cpp | 73 +- llvm/lib/CodeGen/MachineSSAContext.cpp | 52 + llvm/lib/CodeGen/MachineSSAUpdater.cpp | 27 +- llvm/lib/CodeGen/MachineScheduler.cpp | 21 +- llvm/lib/CodeGen/MachineTraceMetrics.cpp | 25 +- llvm/lib/CodeGen/MachineVerifier.cpp | 88 +- llvm/lib/CodeGen/PHIElimination.cpp | 4 +- llvm/lib/CodeGen/PostRASchedulerList.cpp | 17 +- llvm/lib/CodeGen/PrologEpilogInserter.cpp | 14 +- llvm/lib/CodeGen/RDFGraph.cpp | 4 +- llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp | 121 + llvm/lib/CodeGen/RegAllocEvictionAdvisor.h | 210 + llvm/lib/CodeGen/RegAllocGreedy.cpp | 211 +- llvm/lib/CodeGen/RegAllocPBQP.cpp | 4 +- llvm/lib/CodeGen/RegAllocScore.cpp | 124 + llvm/lib/CodeGen/RegAllocScore.h | 80 + llvm/lib/CodeGen/RegisterClassInfo.cpp | 3 +- llvm/lib/CodeGen/RegisterCoalescer.cpp | 8 +- .../CodeGen/RemoveRedundantDebugValues.cpp | 17 +- llvm/lib/CodeGen/SafeStack.cpp | 18 +- llvm/lib/CodeGen/SafeStackLayout.cpp | 7 +- llvm/lib/CodeGen/SafeStackLayout.h | 12 +- llvm/lib/CodeGen/ScheduleDAG.cpp | 8 +- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 655 ++- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 8 +- .../SelectionDAG/LegalizeIntegerTypes.cpp | 16 +- .../CodeGen/SelectionDAG/LegalizeTypes.cpp | 10 +- .../SelectionDAG/LegalizeVectorOps.cpp | 99 +- .../SelectionDAG/ResourcePriorityQueue.cpp | 7 +- .../SelectionDAG/ScheduleDAGSDNodes.cpp | 30 +- .../CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp | 8 +- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 61 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 82 +- .../SelectionDAG/SelectionDAGBuilder.h | 5 +- .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 20 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 59 +- llvm/lib/CodeGen/ShadowStackGCLowering.cpp | 14 +- llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp | 10 +- llvm/lib/CodeGen/StackProtector.cpp | 21 +- llvm/lib/CodeGen/StackSlotColoring.cpp | 10 +- llvm/lib/CodeGen/TailDuplicator.cpp | 42 +- llvm/lib/CodeGen/TargetInstrInfo.cpp | 15 +- llvm/lib/CodeGen/TargetLoweringBase.cpp | 2 +- llvm/lib/CodeGen/TargetRegisterInfo.cpp | 4 +- llvm/lib/CodeGen/UnreachableBlockElim.cpp | 21 +- llvm/lib/CodeGen/VLIWMachineScheduler.cpp | 1009 ++++ llvm/lib/CodeGen/ValueTypes.cpp | 4 +- llvm/lib/CodeGen/WinEHPrepare.cpp | 15 +- llvm/lib/CodeGen/XRayInstrumentation.cpp | 1 + llvm/lib/DWARFLinker/DWARFLinker.cpp | 46 +- llvm/lib/DWARFLinker/DWARFStreamer.cpp | 4 +- llvm/lib/DebugInfo/DWARF/DWARFContext.cpp | 2 +- llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp | 15 +- llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp | 11 +- llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp | 6 +- llvm/lib/DebugInfo/DWARF/DWARFDie.cpp | 4 +- llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp | 4 +- llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp | 47 +- llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp | 12 +- llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp | 80 +- llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp | 4 +- llvm/lib/DebugInfo/MSF/MSFBuilder.cpp | 26 +- llvm/lib/DebugInfo/MSF/MSFError.cpp | 8 +- llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp | 2 +- llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp | 4 +- llvm/lib/DebugInfo/Symbolize/Symbolize.cpp | 10 +- llvm/lib/Debuginfod/Debuginfod.cpp | 183 + llvm/lib/Debuginfod/HTTPClient.cpp | 216 + llvm/lib/Demangle/DLangDemangle.cpp | 66 + .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 6 +- llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp | 4 +- llvm/lib/ExecutionEngine/MCJIT/MCJIT.h | 10 +- .../Orc/CompileOnDemandLayer.cpp | 11 +- llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp | 3 +- llvm/lib/ExecutionEngine/Orc/Core.cpp | 30 +- .../Orc/DebuggerSupportPlugin.cpp | 18 +- .../ExecutionEngine/Orc/ELFNixPlatform.cpp | 13 +- .../ExecutionEngine/Orc/ExecutionUtils.cpp | 44 +- .../ExecutionEngine/Orc/IndirectionUtils.cpp | 4 +- llvm/lib/ExecutionEngine/Orc/Layer.cpp | 57 +- .../lib/ExecutionEngine/Orc/LazyReexports.cpp | 6 +- .../lib/ExecutionEngine/Orc/MachOPlatform.cpp | 12 +- llvm/lib/ExecutionEngine/Orc/Mangling.cpp | 188 - .../Orc/ObjectFileInterface.cpp | 205 + .../Orc/ObjectLinkingLayer.cpp | 29 +- .../ExecutionEngine/Orc/OrcV2CBindings.cpp | 4 +- .../RuntimeDyld/RuntimeDyld.cpp | 22 +- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 72 +- llvm/lib/IR/AsmWriter.cpp | 46 +- llvm/lib/IR/AttributeImpl.h | 3 +- llvm/lib/IR/Attributes.cpp | 69 +- llvm/lib/IR/AutoUpgrade.cpp | 134 +- llvm/lib/IR/BasicBlock.cpp | 4 +- llvm/lib/IR/ConstantFold.cpp | 51 +- llvm/lib/IR/Constants.cpp | 57 +- llvm/lib/IR/Core.cpp | 25 +- llvm/lib/IR/DIBuilder.cpp | 8 +- llvm/lib/IR/DataLayout.cpp | 129 +- llvm/lib/IR/Function.cpp | 17 +- llvm/lib/IR/Globals.cpp | 4 +- llvm/lib/IR/InlineAsm.cpp | 6 +- llvm/lib/IR/Instruction.cpp | 11 +- llvm/lib/IR/Instructions.cpp | 48 +- llvm/lib/IR/IntrinsicInst.cpp | 14 +- llvm/lib/IR/LLVMContextImpl.h | 7 +- llvm/lib/IR/LegacyPassManager.cpp | 12 +- llvm/lib/IR/Module.cpp | 18 +- llvm/lib/IR/ModuleSummaryIndex.cpp | 16 +- llvm/lib/IR/Operator.cpp | 5 +- llvm/lib/IR/SSAContext.cpp | 47 + llvm/lib/IR/Value.cpp | 2 +- llvm/lib/IR/Verifier.cpp | 87 +- llvm/lib/LTO/LTO.cpp | 4 +- llvm/lib/LTO/LTOBackend.cpp | 3 +- llvm/lib/LTO/LTOCodeGenerator.cpp | 5 +- llvm/lib/LTO/ThinLTOCodeGenerator.cpp | 6 +- llvm/lib/LineEditor/LineEditor.cpp | 5 +- llvm/lib/Linker/IRMover.cpp | 4 +- llvm/lib/MC/MCAsmStreamer.cpp | 20 + llvm/lib/MC/MCAssembler.cpp | 3 + llvm/lib/MC/MCInstrAnalysis.cpp | 2 +- llvm/lib/MC/MCMachOStreamer.cpp | 15 +- llvm/lib/MC/MCNullStreamer.cpp | 3 + llvm/lib/MC/MCObjectStreamer.cpp | 25 + llvm/lib/MC/MCParser/AsmParser.cpp | 67 +- llvm/lib/MC/MCPseudoProbe.cpp | 4 +- llvm/lib/MC/MCStreamer.cpp | 72 +- llvm/lib/MC/MCWin64EH.cpp | 6 +- llvm/lib/MC/MachObjectWriter.cpp | 88 +- llvm/lib/MC/TargetRegistry.cpp | 8 +- llvm/lib/Object/ArchiveWriter.cpp | 2 +- llvm/lib/Object/ELF.cpp | 2 + llvm/lib/Object/MachOObjectFile.cpp | 46 + llvm/lib/Object/MachOUniversalWriter.cpp | 1 - llvm/lib/ObjectYAML/COFFEmitter.cpp | 12 +- llvm/lib/ObjectYAML/ELFYAML.cpp | 48 +- llvm/lib/ObjectYAML/XCOFFEmitter.cpp | 10 +- llvm/lib/ObjectYAML/YAML.cpp | 5 +- llvm/lib/Option/OptTable.cpp | 10 +- llvm/lib/Passes/PassBuilder.cpp | 2 + llvm/lib/Passes/PassBuilderPipelines.cpp | 80 +- llvm/lib/Passes/PassRegistry.def | 4 + llvm/lib/Passes/StandardInstrumentations.cpp | 140 +- llvm/lib/ProfileData/InstrProf.cpp | 25 +- llvm/lib/ProfileData/InstrProfCorrelator.cpp | 264 + llvm/lib/ProfileData/InstrProfReader.cpp | 125 +- llvm/lib/ProfileData/InstrProfWriter.cpp | 1 + .../lib/ProfileData/ProfileSummaryBuilder.cpp | 2 +- llvm/lib/ProfileData/SampleProf.cpp | 121 +- llvm/lib/ProfileData/SampleProfReader.cpp | 115 +- llvm/lib/ProfileData/SampleProfWriter.cpp | 77 +- llvm/lib/Support/AArch64TargetParser.cpp | 2 +- llvm/lib/Support/Caching.cpp | 13 +- llvm/lib/Support/CommandLine.cpp | 33 +- llvm/lib/Support/Compression.cpp | 8 +- llvm/lib/Support/ConvertUTFWrapper.cpp | 4 +- llvm/lib/Support/DAGDeltaAlgorithm.cpp | 68 +- llvm/lib/Support/DeltaAlgorithm.cpp | 5 +- llvm/lib/Support/HTTPClient.cpp | 97 - llvm/lib/Support/KnownBits.cpp | 18 +- llvm/lib/Support/MemoryBuffer.cpp | 9 +- llvm/lib/Support/NativeFormatting.cpp | 2 +- llvm/lib/Support/Path.cpp | 4 +- llvm/lib/Support/RISCVISAInfo.cpp | 249 +- llvm/lib/Support/ScopedPrinter.cpp | 10 + llvm/lib/Support/Signals.cpp | 6 +- llvm/lib/Support/SourceMgr.cpp | 3 +- llvm/lib/Support/Statistic.cpp | 17 +- llvm/lib/Support/TargetParser.cpp | 15 + llvm/lib/Support/ThreadPool.cpp | 24 +- llvm/lib/Support/Triple.cpp | 126 +- llvm/lib/Support/Unix/Path.inc | 13 - llvm/lib/Support/VirtualFileSystem.cpp | 6 + llvm/lib/Support/YAMLParser.cpp | 4 +- llvm/lib/TableGen/StringMatcher.cpp | 13 +- llvm/lib/Target/AArch64/AArch64.td | 3 + .../Target/AArch64/AArch64A53Fix835769.cpp | 8 +- .../AArch64/AArch64AdvSIMDScalarPass.cpp | 4 +- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 14 + llvm/lib/Target/AArch64/AArch64Combine.td | 10 +- llvm/lib/Target/AArch64/AArch64ExpandImm.cpp | 7 +- .../Target/AArch64/AArch64FalkorHWPFFix.cpp | 8 +- .../Target/AArch64/AArch64FrameLowering.cpp | 11 + .../Target/AArch64/AArch64ISelLowering.cpp | 222 +- llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 + .../lib/Target/AArch64/AArch64InstrFormats.td | 64 +- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 + llvm/lib/Target/AArch64/AArch64InstrInfo.td | 95 + .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 44 +- .../Target/AArch64/AArch64StackTagging.cpp | 17 +- llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 4 +- llvm/lib/Target/AArch64/AArch64Subtarget.h | 7 +- .../Target/AArch64/AArch64TargetMachine.cpp | 15 +- .../AArch64/AArch64TargetTransformInfo.cpp | 74 +- .../AArch64/AArch64TargetTransformInfo.h | 2 + .../AArch64/AsmParser/AArch64AsmParser.cpp | 7 +- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 5 + .../GISel/AArch64PostLegalizerCombiner.cpp | 38 + .../AArch64/GISel/AArch64RegisterBankInfo.cpp | 4 + .../MCTargetDesc/AArch64MCTargetDesc.cpp | 4 +- llvm/lib/Target/AArch64/SVEInstrFormats.td | 100 +- llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp | 14 +- .../Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp | 11 +- llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 26 +- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 33 +- llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 28 +- .../Target/AMDGPU/AMDGPUCombinerHelper.cpp | 1 - llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 2 + .../AMDGPU/AMDGPUHSAMetadataStreamer.cpp | 26 +- .../Target/AMDGPU/AMDGPUHSAMetadataStreamer.h | 12 +- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 9 + .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 106 +- .../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 116 +- .../AMDGPU/AMDGPUMachineCFGStructurizer.cpp | 8 +- .../Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp | 40 +- .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 2 +- .../Target/AMDGPU/AMDGPURegBankCombiner.cpp | 192 +- .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 33 +- .../AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp | 210 +- llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 21 +- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 8 - llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 1 - .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 85 - .../Target/AMDGPU/AMDILCFGStructurizer.cpp | 23 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 8 +- llvm/lib/Target/AMDGPU/BUFInstructions.td | 8 +- llvm/lib/Target/AMDGPU/FLATInstructions.td | 10 +- .../AMDGPU/MCA/AMDGPUCustomBehaviour.cpp | 3 +- .../AMDGPU/R600ControlFlowFinalizer.cpp | 21 +- llvm/lib/Target/AMDGPU/R600InstrInfo.cpp | 48 +- .../Target/AMDGPU/R600MachineScheduler.cpp | 6 +- .../R600OpenCLImageTypeLoweringPass.cpp | 4 +- .../AMDGPU/R600OptimizeVectorRegisters.cpp | 42 +- llvm/lib/Target/AMDGPU/R600Packetizer.cpp | 4 +- llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp | 6 +- llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 6 +- llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 12 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 73 +- llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 156 +- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 153 +- llvm/lib/Target/AMDGPU/SIInstrInfo.td | 8 +- llvm/lib/Target/AMDGPU/SIInstructions.td | 116 +- .../Target/AMDGPU/SIMachineFunctionInfo.cpp | 24 +- .../lib/Target/AMDGPU/SIMachineFunctionInfo.h | 7 + llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp | 53 +- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 59 +- .../Target/AMDGPU/SIShrinkInstructions.cpp | 11 +- llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp | 9 +- llvm/lib/Target/AMDGPU/SMInstructions.td | 21 +- llvm/lib/Target/AMDGPU/SOPInstructions.td | 106 +- .../Target/AMDGPU/Utils/AMDGPULDSUtils.cpp | 231 +- llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h | 31 - llvm/lib/Target/AMDGPU/VOPInstructions.td | 12 + llvm/lib/Target/ARM/A15SDOptimizer.cpp | 16 +- llvm/lib/Target/ARM/ARM.td | 5 + llvm/lib/Target/ARM/ARMAsmPrinter.cpp | 36 +- llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp | 217 +- llvm/lib/Target/ARM/ARMBaseInstrInfo.h | 27 +- llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp | 4 + llvm/lib/Target/ARM/ARMBranchTargets.cpp | 5 +- llvm/lib/Target/ARM/ARMCallingConv.cpp | 7 +- llvm/lib/Target/ARM/ARMConstantIslandPass.cpp | 48 +- llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp | 24 + llvm/lib/Target/ARM/ARMFrameLowering.cpp | 94 +- llvm/lib/Target/ARM/ARMISelLowering.cpp | 221 +- llvm/lib/Target/ARM/ARMISelLowering.h | 1 + llvm/lib/Target/ARM/ARMInstrMVE.td | 116 +- llvm/lib/Target/ARM/ARMInstrThumb2.td | 7 + llvm/lib/Target/ARM/ARMInstrVFP.td | 3 + llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 5 +- llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp | 16 +- llvm/lib/Target/ARM/ARMMachineFunctionInfo.h | 2 +- llvm/lib/Target/ARM/ARMRegisterInfo.td | 4 +- llvm/lib/Target/ARM/ARMSubtarget.h | 9 + .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 63 +- .../ARM/MCTargetDesc/ARMELFStreamer.cpp | 67 +- .../ARM/MCTargetDesc/ARMUnwindOpAsm.cpp | 5 +- .../ARM/MVETPAndVPTOptimisationsPass.cpp | 2 +- llvm/lib/Target/ARM/MVETailPredication.cpp | 10 +- llvm/lib/Target/ARM/Thumb1FrameLowering.cpp | 9 +- llvm/lib/Target/AVR/AVRFrameLowering.cpp | 4 +- llvm/lib/Target/AVR/AVRInstrInfo.cpp | 2 - llvm/lib/Target/BPF/BPFPreserveDIType.cpp | 9 +- .../Target/CSKY/AsmParser/CSKYAsmParser.cpp | 91 +- llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp | 13 + llvm/lib/Target/CSKY/CSKYAsmPrinter.h | 2 + llvm/lib/Target/CSKY/CSKYCallingConv.td | 2 +- llvm/lib/Target/CSKY/CSKYFrameLowering.cpp | 2 +- llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp | 89 +- llvm/lib/Target/CSKY/CSKYISelLowering.cpp | 40 + .../Target/CSKY/CSKYInstrFormats16Instr.td | 15 +- llvm/lib/Target/CSKY/CSKYInstrInfo.cpp | 288 ++ llvm/lib/Target/CSKY/CSKYInstrInfo.h | 25 + llvm/lib/Target/CSKY/CSKYInstrInfo.td | 241 +- llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td | 165 +- llvm/lib/Target/CSKY/CSKYMCInstLower.cpp | 2 +- llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp | 181 +- llvm/lib/Target/CSKY/CSKYRegisterInfo.h | 12 + llvm/lib/Target/CSKY/CSKYRegisterInfo.td | 5 + llvm/lib/Target/Hexagon/Hexagon.td | 55 +- llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp | 61 + llvm/lib/Target/Hexagon/HexagonAsmPrinter.h | 16 +- .../lib/Target/Hexagon/HexagonBitSimplify.cpp | 22 +- llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp | 12 +- llvm/lib/Target/Hexagon/HexagonDepArch.h | 34 +- llvm/lib/Target/Hexagon/HexagonDepArch.td | 2 + .../lib/Target/Hexagon/HexagonDepDecoders.inc | 1 + llvm/lib/Target/Hexagon/HexagonDepIICHVX.td | 1018 ++++ .../lib/Target/Hexagon/HexagonDepIICScalar.td | 768 +++ .../Target/Hexagon/HexagonDepInstrFormats.td | 14 + .../lib/Target/Hexagon/HexagonDepInstrInfo.td | 1253 ++++- .../Target/Hexagon/HexagonDepMapAsm2Intrin.td | 293 +- llvm/lib/Target/Hexagon/HexagonDepMappings.td | 1 - llvm/lib/Target/Hexagon/HexagonGenInsert.cpp | 4 +- .../Hexagon/HexagonHazardRecognizer.cpp | 4 +- .../lib/Target/Hexagon/HexagonInstrFormats.td | 7 +- llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp | 9 + llvm/lib/Target/Hexagon/HexagonInstrInfo.h | 2 + .../lib/Target/Hexagon/HexagonMCInstLower.cpp | 13 + .../Hexagon/HexagonMachineScheduler.cpp | 964 +--- .../Target/Hexagon/HexagonMachineScheduler.h | 253 +- llvm/lib/Target/Hexagon/HexagonPseudo.td | 11 + llvm/lib/Target/Hexagon/HexagonSchedule.td | 1 + llvm/lib/Target/Hexagon/HexagonScheduleV69.td | 40 + llvm/lib/Target/Hexagon/HexagonSubtarget.cpp | 72 +- llvm/lib/Target/Hexagon/HexagonSubtarget.h | 27 +- .../Target/Hexagon/HexagonTargetMachine.cpp | 5 +- .../Target/Hexagon/HexagonVLIWPacketizer.cpp | 9 +- .../Target/Hexagon/HexagonVectorCombine.cpp | 2 +- .../Hexagon/MCTargetDesc/HexagonBaseInfo.h | 5 +- .../Hexagon/MCTargetDesc/HexagonMCChecker.cpp | 38 +- .../Hexagon/MCTargetDesc/HexagonMCChecker.h | 1 + .../MCTargetDesc/HexagonMCInstrInfo.cpp | 18 +- .../Hexagon/MCTargetDesc/HexagonMCInstrInfo.h | 4 +- .../MCTargetDesc/HexagonMCTargetDesc.cpp | 32 +- llvm/lib/Target/M68k/M68kInstrControl.td | 16 +- .../lib/Target/MSP430/MSP430FrameLowering.cpp | 4 +- llvm/lib/Target/Mips/Mips16HardFloat.cpp | 6 +- llvm/lib/Target/Mips/MipsBranchExpansion.cpp | 52 +- llvm/lib/Target/Mips/MipsISelLowering.cpp | 2 +- llvm/lib/Target/Mips/MipsInstrInfo.cpp | 49 + llvm/lib/Target/Mips/MipsInstrInfo.h | 7 + llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 12 +- llvm/lib/Target/NVPTX/NVPTXPeephole.cpp | 6 +- .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 12 +- .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp | 29 + llvm/lib/Target/PowerPC/PPC.td | 4 + llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 1 - .../lib/Target/PowerPC/PPCBack2BackFusion.def | 1042 ++++ llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 14 +- llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 56 +- llvm/lib/Target/PowerPC/PPCInstrInfo.h | 4 +- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 1 + llvm/lib/Target/PowerPC/PPCInstrVSX.td | 17 + .../Target/PowerPC/PPCLoopInstrFormPrep.cpp | 6 +- llvm/lib/Target/PowerPC/PPCMacroFusion.def | 2 + llvm/lib/Target/PowerPC/PPCSubtarget.cpp | 1 + llvm/lib/Target/PowerPC/PPCSubtarget.h | 2 + .../Target/PowerPC/PPCTargetTransformInfo.cpp | 103 +- .../Target/PowerPC/PPCTargetTransformInfo.h | 16 +- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 124 +- .../RISCV/MCTargetDesc/RISCVBaseInfo.cpp | 5 + .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h | 10 + .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp | 4 +- llvm/lib/Target/RISCV/RISCV.td | 8 - llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 71 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 277 +- llvm/lib/Target/RISCV/RISCVISelLowering.h | 12 +- llvm/lib/Target/RISCV/RISCVInstrFormats.td | 70 +- llvm/lib/Target/RISCV/RISCVInstrFormatsV.td | 32 - llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 5 +- llvm/lib/Target/RISCV/RISCVInstrInfo.h | 5 + llvm/lib/Target/RISCV/RISCVInstrInfo.td | 52 +- llvm/lib/Target/RISCV/RISCVInstrInfoD.td | 247 +- llvm/lib/Target/RISCV/RISCVInstrInfoF.td | 297 +- llvm/lib/Target/RISCV/RISCVInstrInfoM.td | 13 - llvm/lib/Target/RISCV/RISCVInstrInfoV.td | 72 +- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 1229 +++-- llvm/lib/Target/RISCV/RISCVInstrInfoZb.td | 65 +- llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td | 262 +- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 1 - llvm/lib/Target/RISCV/RISCVRegisterInfo.td | 8 +- llvm/lib/Target/RISCV/RISCVSchedRocket.td | 3 +- llvm/lib/Target/RISCV/RISCVSchedSiFive7.td | 2 +- llvm/lib/Target/RISCV/RISCVSubtarget.h | 2 - .../Target/RISCV/RISCVTargetTransformInfo.cpp | 91 + .../Target/RISCV/RISCVTargetTransformInfo.h | 11 +- .../MCTargetDesc/SystemZMCAsmBackend.cpp | 32 +- .../MCTargetDesc/SystemZMCCodeEmitter.cpp | 10 +- llvm/lib/Target/SystemZ/SystemZCallingConv.td | 1 + .../Target/SystemZ/SystemZFrameLowering.cpp | 180 +- .../lib/Target/SystemZ/SystemZFrameLowering.h | 9 + .../Target/SystemZ/SystemZISelLowering.cpp | 165 +- llvm/lib/Target/SystemZ/SystemZISelLowering.h | 6 +- .../lib/Target/SystemZ/SystemZInstrFormats.td | 10 + llvm/lib/Target/SystemZ/SystemZInstrInfo.td | 6 + llvm/lib/Target/SystemZ/SystemZOperators.td | 6 + .../SystemZ/SystemZSelectionDAGInfo.cpp | 57 +- llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp | 35 + .../Target/VE/MCTargetDesc/VEAsmBackend.cpp | 3 + .../VE/MCTargetDesc/VEELFObjectWriter.cpp | 47 +- .../lib/Target/VE/MCTargetDesc/VEFixupKinds.h | 3 + .../VE/MCTargetDesc/VEMCCodeEmitter.cpp | 6 +- llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp | 11 +- llvm/lib/Target/VE/VEISelLowering.cpp | 2 +- llvm/lib/Target/VE/VVPInstrInfo.td | 35 + llvm/lib/Target/VE/VVPInstrPatternsVec.td | 185 +- llvm/lib/Target/VE/VVPNodes.def | 32 +- .../AsmParser/WebAssemblyAsmParser.cpp | 12 - .../AsmParser/WebAssemblyAsmTypeCheck.cpp | 13 +- .../AsmParser/WebAssemblyAsmTypeCheck.h | 2 +- .../Disassembler/WebAssemblyDisassembler.cpp | 22 - .../MCTargetDesc/WebAssemblyInstPrinter.cpp | 23 - .../MCTargetDesc/WebAssemblyInstPrinter.h | 2 - .../MCTargetDesc/WebAssemblyMCAsmInfo.cpp | 9 + .../MCTargetDesc/WebAssemblyMCCodeEmitter.cpp | 3 - .../MCTargetDesc/WebAssemblyMCTargetDesc.h | 2 - .../Utils/WebAssemblyTypeUtilities.cpp | 7 - .../Utils/WebAssemblyTypeUtilities.h | 8 - .../Utils/WebAssemblyUtilities.cpp | 25 + .../WebAssembly/Utils/WebAssemblyUtilities.h | 7 + .../WebAssembly/WebAssemblyAsmPrinter.cpp | 14 +- .../WebAssembly/WebAssemblyCFGStackify.cpp | 2 +- .../lib/Target/WebAssembly/WebAssemblyISD.def | 1 + .../WebAssembly/WebAssemblyISelLowering.cpp | 115 +- .../WebAssembly/WebAssemblyInstrInfo.td | 5 - .../Target/WebAssembly/WebAssemblyInstrRef.td | 19 +- .../WebAssembly/WebAssemblyInstrSIMD.td | 8 + .../WebAssembly/WebAssemblyInstrTable.td | 16 +- .../WebAssemblyLowerEmscriptenEHSjLj.cpp | 13 +- .../WebAssembly/WebAssemblyMCInstLower.cpp | 24 +- .../WebAssembly/WebAssemblyTargetMachine.cpp | 42 +- .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 34 +- llvm/lib/Target/X86/AsmParser/X86Operand.h | 6 + .../X86/MCTargetDesc/X86InstComments.cpp | 24 +- .../MCTargetDesc/X86WinCOFFTargetStreamer.cpp | 2 +- llvm/lib/Target/X86/X86AsmPrinter.cpp | 2 - llvm/lib/Target/X86/X86AsmPrinter.h | 19 +- llvm/lib/Target/X86/X86CmovConversion.cpp | 2 +- llvm/lib/Target/X86/X86ExpandPseudo.cpp | 26 +- llvm/lib/Target/X86/X86FastTileConfig.cpp | 6 +- llvm/lib/Target/X86/X86FixupBWInsts.cpp | 10 +- llvm/lib/Target/X86/X86FloatingPoint.cpp | 2 +- llvm/lib/Target/X86/X86FrameLowering.cpp | 23 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 487 +- llvm/lib/Target/X86/X86ISelLowering.h | 10 + .../Target/X86/X86IndirectBranchTracking.cpp | 4 +- llvm/lib/Target/X86/X86InstrAVX512.td | 138 +- llvm/lib/Target/X86/X86InstrCompiler.td | 6 +- llvm/lib/Target/X86/X86InstrFoldTables.cpp | 106 +- llvm/lib/Target/X86/X86InstrInfo.cpp | 107 +- llvm/lib/Target/X86/X86InstrInfo.h | 3 +- llvm/lib/Target/X86/X86InstrMMX.td | 40 +- llvm/lib/Target/X86/X86MCInstLower.cpp | 252 +- llvm/lib/Target/X86/X86RegisterInfo.td | 10 +- llvm/lib/Target/X86/X86SchedBroadwell.td | 26 +- llvm/lib/Target/X86/X86SchedHaswell.td | 32 +- llvm/lib/Target/X86/X86SchedIceLake.td | 167 +- llvm/lib/Target/X86/X86SchedSandyBridge.td | 4 +- llvm/lib/Target/X86/X86SchedSkylakeClient.td | 88 +- llvm/lib/Target/X86/X86SchedSkylakeServer.td | 98 +- llvm/lib/Target/X86/X86ScheduleAtom.td | 72 +- llvm/lib/Target/X86/X86ScheduleBdVer2.td | 36 +- llvm/lib/Target/X86/X86ScheduleBtVer2.td | 24 +- llvm/lib/Target/X86/X86ScheduleSLM.td | 8 +- llvm/lib/Target/X86/X86ScheduleZnver1.td | 18 +- llvm/lib/Target/X86/X86ScheduleZnver2.td | 18 +- llvm/lib/Target/X86/X86ScheduleZnver3.td | 14 +- llvm/lib/Target/X86/X86Subtarget.h | 3 +- llvm/lib/Target/X86/X86TargetMachine.cpp | 12 + .../lib/Target/X86/X86TargetTransformInfo.cpp | 91 +- llvm/lib/Target/X86/X86TargetTransformInfo.h | 5 +- .../TruncInstCombine.cpp | 6 +- llvm/lib/Transforms/CFGuard/CFGuard.cpp | 8 +- llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 33 +- llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 41 +- llvm/lib/Transforms/Coroutines/Coroutines.cpp | 1 - llvm/lib/Transforms/IPO/ArgumentPromotion.cpp | 10 +- llvm/lib/Transforms/IPO/Attributor.cpp | 6 +- .../Transforms/IPO/AttributorAttributes.cpp | 38 +- llvm/lib/Transforms/IPO/FunctionAttrs.cpp | 210 +- .../Transforms/IPO/FunctionSpecialization.cpp | 331 +- llvm/lib/Transforms/IPO/GlobalOpt.cpp | 11 +- llvm/lib/Transforms/IPO/HotColdSplitting.cpp | 2 +- llvm/lib/Transforms/IPO/Inliner.cpp | 2 + llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 7 +- llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 6 + .../Transforms/IPO/SampleContextTracker.cpp | 21 +- llvm/lib/Transforms/IPO/SampleProfile.cpp | 172 +- .../Transforms/IPO/ThinLTOBitcodeWriter.cpp | 3 +- .../lib/Transforms/IPO/WholeProgramDevirt.cpp | 104 +- .../InstCombine/InstCombineAndOrXor.cpp | 58 + .../InstCombine/InstCombineCalls.cpp | 6 + .../InstCombine/InstCombineCasts.cpp | 44 +- .../InstCombine/InstCombineInternal.h | 1 + .../InstCombineLoadStoreAlloca.cpp | 20 +- .../InstCombine/InstCombineMulDivRem.cpp | 18 + .../Transforms/InstCombine/InstCombinePHI.cpp | 7 +- .../InstCombine/InstCombineSelect.cpp | 7 +- .../InstCombineSimplifyDemanded.cpp | 6 - .../InstCombine/InstCombineVectorOps.cpp | 161 +- .../InstCombine/InstructionCombining.cpp | 39 +- .../Instrumentation/DataFlowSanitizer.cpp | 25 +- .../Instrumentation/InstrProfiling.cpp | 61 + .../Instrumentation/MemorySanitizer.cpp | 13 +- .../Instrumentation/PGOInstrumentation.cpp | 25 +- .../Transforms/Scalar/ConstantHoisting.cpp | 3 +- .../Transforms/Scalar/DFAJumpThreading.cpp | 58 +- .../Scalar/DeadStoreElimination.cpp | 193 +- llvm/lib/Transforms/Scalar/EarlyCSE.cpp | 12 +- llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp | 49 +- llvm/lib/Transforms/Scalar/LICM.cpp | 44 - .../Transforms/Scalar/LoopDataPrefetch.cpp | 4 +- .../Transforms/Scalar/LoopIdiomRecognize.cpp | 28 +- llvm/lib/Transforms/Scalar/LoopRerollPass.cpp | 12 +- .../Transforms/Scalar/LoopStrengthReduce.cpp | 2 +- llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 25 + llvm/lib/Transforms/Scalar/NewGVN.cpp | 35 +- .../Scalar/RewriteStatepointsForGC.cpp | 51 +- llvm/lib/Transforms/Scalar/SCCP.cpp | 3 +- llvm/lib/Transforms/Scalar/Scalar.cpp | 2 +- .../Scalar/SeparateConstOffsetFromGEP.cpp | 6 +- llvm/lib/Transforms/Utils/CodeLayout.cpp | 942 ++++ llvm/lib/Transforms/Utils/Debugify.cpp | 2 +- .../Transforms/Utils/FunctionComparator.cpp | 16 +- llvm/lib/Transforms/Utils/Local.cpp | 12 + llvm/lib/Transforms/Utils/LoopPeel.cpp | 31 +- llvm/lib/Transforms/Utils/LoopUtils.cpp | 113 +- llvm/lib/Transforms/Utils/MetaRenamer.cpp | 67 +- .../Utils/RelLookupTableConverter.cpp | 4 + .../Utils/SampleProfileInference.cpp | 385 +- .../Utils/ScalarEvolutionExpander.cpp | 6 +- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 2 +- .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 293 +- llvm/lib/Transforms/Utils/ValueMapper.cpp | 6 + .../Vectorize/LoopVectorizationLegality.cpp | 89 +- .../Vectorize/LoopVectorizationPlanner.h | 38 +- .../Transforms/Vectorize/LoopVectorize.cpp | 471 +- .../Transforms/Vectorize/SLPVectorizer.cpp | 633 ++- llvm/lib/Transforms/Vectorize/VPlan.cpp | 9 +- llvm/lib/Transforms/Vectorize/VPlan.h | 118 +- .../Transforms/Vectorize/VPlanPredicator.cpp | 6 +- llvm/lib/Transforms/Vectorize/VPlanSLP.cpp | 5 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 49 +- .../Transforms/Vectorize/VPlanTransforms.h | 21 +- .../Transforms/Vectorize/VPlanVerifier.cpp | 26 + .../Transforms/Vectorize/VectorCombine.cpp | 6 +- llvm/tools/llc/llc.cpp | 3 + llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp | 57 +- llvm/tools/llvm-lto2/llvm-lto2.cpp | 30 +- .../llvm-mca/Views/BottleneckAnalysis.cpp | 4 +- llvm/tools/llvm-mca/llvm-mca.cpp | 7 +- llvm/tools/llvm-objcopy/ELF/Object.cpp | 36 +- llvm/tools/llvm-objcopy/ELF/Object.h | 9 +- .../tools/llvm-objcopy/MachO/MachOObjcopy.cpp | 5 +- llvm/tools/llvm-objcopy/llvm-objcopy.cpp | 3 +- llvm/tools/llvm-objdump/llvm-objdump.cpp | 9 +- llvm/tools/llvm-pdbutil/PdbYaml.h | 2 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 64 +- llvm/tools/llvm-readobj/ARMEHABIPrinter.h | 10 +- llvm/tools/llvm-readobj/ELFDumper.cpp | 97 +- llvm/tools/llvm-readobj/ObjDumper.cpp | 13 + llvm/tools/llvm-readobj/ObjDumper.h | 4 + llvm/tools/llvm-readobj/Opts.td | 3 +- llvm/tools/llvm-readobj/llvm-readobj.cpp | 49 +- llvm/tools/llvm-readobj/llvm-readobj.h | 2 +- .../tools/llvm-symbolizer/llvm-symbolizer.cpp | 3 + llvm/tools/llvm-tapi-diff/llvm-tapi-diff.cpp | 34 +- llvm/utils/TableGen/AsmWriterEmitter.cpp | 2 - llvm/utils/TableGen/CodeEmitterGen.cpp | 2 +- llvm/utils/TableGen/CodeGenDAGPatterns.cpp | 22 +- llvm/utils/TableGen/CodeGenSchedule.cpp | 2 +- llvm/utils/TableGen/CodeGenTarget.cpp | 2 +- llvm/utils/TableGen/CodeGenTarget.h | 4 +- llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 2 - llvm/utils/TableGen/DAGISelMatcherGen.cpp | 2 +- llvm/utils/TableGen/FastISelEmitter.cpp | 1 - llvm/utils/TableGen/GICombinerEmitter.cpp | 2 +- llvm/utils/TableGen/GlobalISelEmitter.cpp | 4 +- llvm/utils/TableGen/IntrinsicEmitter.cpp | 10 +- llvm/utils/TableGen/PredicateExpander.cpp | 2 - llvm/utils/TableGen/SubtargetEmitter.cpp | 9 +- openmp/runtime/src/i18n/en_US.txt | 9 + openmp/runtime/src/include/omp_lib.h.var | 6 +- openmp/runtime/src/kmp.h | 14 + openmp/runtime/src/kmp_affinity.cpp | 463 +- openmp/runtime/src/kmp_affinity.h | 153 +- openmp/runtime/src/kmp_runtime.cpp | 17 +- openmp/runtime/src/kmp_settings.cpp | 130 +- 1630 files changed, 80158 insertions(+), 46865 deletions(-) create mode 100644 clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def create mode 100644 clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def rename clang/lib/Driver/ToolChains/{HIP.cpp => HIPAMD.cpp} (60%) rename clang/lib/Driver/ToolChains/{HIP.h => HIPAMD.h} (69%) create mode 100644 clang/lib/Driver/ToolChains/HIPSPV.cpp create mode 100644 clang/lib/Driver/ToolChains/HIPSPV.h create mode 100644 clang/lib/Driver/ToolChains/HIPUtility.cpp create mode 100644 clang/lib/Driver/ToolChains/HIPUtility.h create mode 100644 clang/lib/Headers/arm_neon_sve_bridge.h create mode 100644 compiler-rt/lib/asan/asan_rtl_x86_64.S create mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_lzw.h create mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan.syms.extra rename compiler-rt/lib/tsan/{rtl => rtl-old}/tsan_clock.cpp (100%) rename compiler-rt/lib/tsan/{rtl => rtl-old}/tsan_clock.h (100%) create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_defs.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_external.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_fd.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_flags.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_flags.inc create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_ilist.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface.inc create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_mman.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_platform.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_report.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_report.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_shadow.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_sync.h create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_trace.h rename compiler-rt/lib/tsan/{rtl => rtl-old}/tsan_update_shadow_word.inc (100%) create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp create mode 100644 compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h create mode 100644 compiler-rt/lib/xray/xray_hexagon.cpp create mode 100644 compiler-rt/lib/xray/xray_trampoline_hexagon.S create mode 100644 libcxx/include/__filesystem/copy_options.h create mode 100644 libcxx/include/__filesystem/directory_entry.h create mode 100644 libcxx/include/__filesystem/directory_iterator.h create mode 100644 libcxx/include/__filesystem/directory_options.h create mode 100644 libcxx/include/__filesystem/file_status.h create mode 100644 libcxx/include/__filesystem/file_time_type.h create mode 100644 libcxx/include/__filesystem/file_type.h create mode 100644 libcxx/include/__filesystem/filesystem_error.h create mode 100644 libcxx/include/__filesystem/operations.h create mode 100644 libcxx/include/__filesystem/path.h create mode 100644 libcxx/include/__filesystem/path_iterator.h create mode 100644 libcxx/include/__filesystem/perm_options.h create mode 100644 libcxx/include/__filesystem/perms.h create mode 100644 libcxx/include/__filesystem/recursive_directory_iterator.h create mode 100644 libcxx/include/__filesystem/space_info.h create mode 100644 libcxx/include/__filesystem/u8path.h create mode 100644 libcxx/include/__memory/concepts.h create mode 100644 libcxx/include/__memory/ranges_uninitialized_algorithms.h create mode 100644 libcxx/include/__memory/voidify.h create mode 100644 libcxx/include/__random/clamp_to_integral.h rename libcxx/include/__utility/{decay_copy.h => auto_cast.h} (52%) create mode 100644 libcxx/include/__utility/transaction.h create mode 100644 libcxx/src/chrono_system_time_init.h create mode 100644 libcxx/src/experimental/memory_resource_init_helper.h create mode 100644 libcxx/src/include/ryu/common.h create mode 100644 libcxx/src/include/ryu/d2fixed.h create mode 100644 libcxx/src/include/ryu/d2fixed_full_table.h create mode 100644 libcxx/src/include/ryu/d2s.h create mode 100644 libcxx/src/include/ryu/d2s_full_table.h create mode 100644 libcxx/src/include/ryu/d2s_intrinsics.h create mode 100644 libcxx/src/include/ryu/digit_table.h create mode 100644 libcxx/src/include/ryu/f2s.h create mode 100644 libcxx/src/include/ryu/ryu.h create mode 100644 libcxx/src/include/to_chars_floating_point.h create mode 100644 libcxx/src/iostream_init.h create mode 100644 libcxx/src/ryu/README.txt create mode 100644 libcxx/src/ryu/d2fixed.cpp create mode 100644 libcxx/src/ryu/d2s.cpp create mode 100644 libcxx/src/ryu/f2s.cpp create mode 100644 lld/docs/ELF/start-stop-gc.rst delete mode 100644 lld/include/lld/ReaderWriter/MachOLinkingContext.h delete mode 100644 lld/include/lld/ReaderWriter/YamlContext.h delete mode 100644 lld/lib/Core/DefinedAtom.cpp delete mode 100644 lld/lib/Core/Error.cpp delete mode 100644 lld/lib/Core/File.cpp delete mode 100644 lld/lib/Core/LinkingContext.cpp delete mode 100644 lld/lib/Core/Reader.cpp delete mode 100644 lld/lib/Core/Resolver.cpp delete mode 100644 lld/lib/Core/SymbolTable.cpp delete mode 100644 lld/lib/Core/Writer.cpp delete mode 100644 lld/lib/Driver/DarwinLdDriver.cpp delete mode 100644 lld/lib/Driver/DarwinLdOptions.td delete mode 100644 lld/lib/ReaderWriter/FileArchive.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/ArchHandler.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/ArchHandler.h delete mode 100644 lld/lib/ReaderWriter/MachO/ArchHandler_arm.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/ArchHandler_x86.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/ArchHandler_x86_64.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/Atoms.h delete mode 100644 lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/DebugInfo.h delete mode 100644 lld/lib/ReaderWriter/MachO/ExecutableAtoms.h delete mode 100644 lld/lib/ReaderWriter/MachO/File.h delete mode 100644 lld/lib/ReaderWriter/MachO/FlatNamespaceFile.h delete mode 100644 lld/lib/ReaderWriter/MachO/GOTPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/LayoutPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/LayoutPass.h delete mode 100644 lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFile.h delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryUtils.h delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/MachONormalizedFileYAML.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/MachOPasses.h delete mode 100644 lld/lib/ReaderWriter/MachO/ObjCPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/SectCreateFile.h delete mode 100644 lld/lib/ReaderWriter/MachO/ShimPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/StubsPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/TLVPass.cpp delete mode 100644 lld/lib/ReaderWriter/MachO/WriterMachO.cpp delete mode 100644 lld/lib/ReaderWriter/YAML/ReaderWriterYAML.cpp create mode 100644 lldb/include/lldb/Core/DataFileCache.h create mode 100644 lldb/include/lldb/Version/Version.h create mode 100644 lldb/include/lldb/Version/Version.inc.in create mode 100644 lldb/source/Core/DataFileCache.cpp create mode 100644 lldb/source/Plugins/Language/CPlusPlus/Generic.h create mode 100644 lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp delete mode 100644 lldb/source/Plugins/Language/CPlusPlus/LibCxxOptional.cpp create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.cpp create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.h create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.cpp create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.h create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.cpp create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.h create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.cpp create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.h create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.cpp create mode 100644 lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.h create mode 100644 lldb/source/Plugins/ScriptInterpreter/Lua/SWIGLuaBridge.h rename lldb/source/{lldb.cpp => Version/Version.cpp} (76%) create mode 100644 llvm/include/llvm-c/Deprecated.h create mode 100644 llvm/include/llvm/ADT/GenericCycleImpl.h create mode 100644 llvm/include/llvm/ADT/GenericCycleInfo.h create mode 100644 llvm/include/llvm/ADT/GenericSSAContext.h create mode 100644 llvm/include/llvm/Analysis/CycleAnalysis.h create mode 100644 llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h create mode 100644 llvm/include/llvm/Analysis/NoInferenceModelRunner.h create mode 100644 llvm/include/llvm/Analysis/ReleaseModeModelRunner.h create mode 100644 llvm/include/llvm/CodeGen/MachineCycleAnalysis.h create mode 100644 llvm/include/llvm/CodeGen/MachineSSAContext.h create mode 100644 llvm/include/llvm/CodeGen/VLIWMachineScheduler.h create mode 100644 llvm/include/llvm/Debuginfod/Debuginfod.h rename llvm/include/llvm/{Support => Debuginfod}/HTTPClient.h (97%) create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h create mode 100644 llvm/include/llvm/IR/AttributesAMDGPU.td create mode 100644 llvm/include/llvm/IR/SSAContext.h create mode 100644 llvm/include/llvm/ProfileData/InstrProfCorrelator.h create mode 100644 llvm/include/llvm/Transforms/Scalar/FlattenCFG.h create mode 100644 llvm/include/llvm/Transforms/Utils/CodeLayout.h create mode 100644 llvm/lib/Analysis/CycleAnalysis.cpp create mode 100644 llvm/lib/Analysis/ModelUnderTrainingRunner.cpp create mode 100644 llvm/lib/Analysis/NoInferenceModelRunner.cpp delete mode 100644 llvm/lib/Analysis/ReleaseModeModelRunner.cpp create mode 100644 llvm/lib/CodeGen/MachineCycleAnalysis.cpp create mode 100644 llvm/lib/CodeGen/MachineSSAContext.cpp create mode 100644 llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp create mode 100644 llvm/lib/CodeGen/RegAllocScore.cpp create mode 100644 llvm/lib/CodeGen/RegAllocScore.h create mode 100644 llvm/lib/CodeGen/VLIWMachineScheduler.cpp create mode 100644 llvm/lib/Debuginfod/Debuginfod.cpp create mode 100644 llvm/lib/Debuginfod/HTTPClient.cpp create mode 100644 llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp create mode 100644 llvm/lib/IR/SSAContext.cpp create mode 100644 llvm/lib/ProfileData/InstrProfCorrelator.cpp delete mode 100644 llvm/lib/Support/HTTPClient.cpp create mode 100644 llvm/lib/Target/Hexagon/HexagonScheduleV69.td create mode 100644 llvm/lib/Target/PowerPC/PPCBack2BackFusion.def create mode 100644 llvm/lib/Transforms/Utils/CodeLayout.cpp diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h index d9bf2f07291f..0d97e9ad8623 100644 --- a/clang/include/clang/APINotes/Types.h +++ b/clang/include/clang/APINotes/Types.h @@ -240,7 +240,7 @@ class ObjCContextInfo : public CommonTypeInfo { } void setSwiftImportAsNonGeneric(llvm::Optional Value) { SwiftImportAsNonGenericSpecified = Value.hasValue(); - SwiftImportAsNonGeneric = Value.hasValue() ? *Value : false; + SwiftImportAsNonGeneric = Value.getValueOr(false); } llvm::Optional getSwiftObjCMembers() const { @@ -249,7 +249,7 @@ class ObjCContextInfo : public CommonTypeInfo { } void setSwiftObjCMembers(llvm::Optional Value) { SwiftObjCMembersSpecified = Value.hasValue(); - SwiftObjCMembers = Value.hasValue() ? *Value : false; + SwiftObjCMembers = Value.getValueOr(false); } /// Strip off any information within the class information structure that is @@ -368,7 +368,7 @@ class ObjCPropertyInfo : public VariableInfo { } void setSwiftImportAsAccessors(llvm::Optional Value) { SwiftImportAsAccessorsSpecified = Value.hasValue(); - SwiftImportAsAccessors = Value.hasValue() ? *Value : false; + SwiftImportAsAccessors = Value.getValueOr(false); } friend bool operator==(const ObjCPropertyInfo &, const ObjCPropertyInfo &); @@ -433,7 +433,7 @@ class ParamInfo : public VariableInfo { } void setNoEscape(llvm::Optional Value) { NoEscapeSpecified = Value.hasValue(); - NoEscape = Value.hasValue() ? *Value : false; + NoEscape = Value.getValueOr(false); } llvm::Optional getRetainCountConvention() const { @@ -671,7 +671,7 @@ class TagInfo : public CommonTypeInfo { } void setFlagEnum(llvm::Optional Value) { HasFlagEnum = Value.hasValue(); - IsFlagEnum = Value.hasValue() ? *Value : false; + IsFlagEnum = Value.getValueOr(false); } TagInfo &operator|=(const TagInfo &RHS) { diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index d336342e4cda..63f2c948c79b 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -248,6 +248,7 @@ class ASTContext : public RefCountedBase { mutable llvm::ContextualFoldingSet TemplateSpecializationTypes; mutable llvm::FoldingSet ParenTypes; + mutable llvm::FoldingSet UsingTypes; mutable llvm::FoldingSet ElaboratedTypes; mutable llvm::FoldingSet DependentNameTypes; mutable llvm::ContextualFoldingSet { mutable llvm::FoldingSet AtomicTypes; llvm::FoldingSet AttributedTypes; mutable llvm::FoldingSet PipeTypes; - mutable llvm::FoldingSet ExtIntTypes; - mutable llvm::FoldingSet DependentExtIntTypes; + mutable llvm::FoldingSet BitIntTypes; + mutable llvm::FoldingSet DependentBitIntTypes; mutable llvm::FoldingSet QualifiedTemplateNames; mutable llvm::FoldingSet DependentTemplateNames; @@ -1350,13 +1351,13 @@ class ASTContext : public RefCountedBase { /// Return a write_only pipe type for the specified type. QualType getWritePipeType(QualType T) const; - /// Return an extended integer type with the specified signedness and bit + /// Return a bit-precise integer type with the specified signedness and bit /// count. - QualType getExtIntType(bool Unsigned, unsigned NumBits) const; + QualType getBitIntType(bool Unsigned, unsigned NumBits) const; - /// Return a dependent extended integer type with the specified signedness and - /// bit count. - QualType getDependentExtIntType(bool Unsigned, Expr *BitsExpr) const; + /// Return a dependent bit-precise integer type with the specified signedness + /// and bit count. + QualType getDependentBitIntType(bool Unsigned, Expr *BitsExpr) const; /// Gets the struct used to keep track of the extended descriptor for /// pointer to blocks. @@ -1555,6 +1556,9 @@ class ASTContext : public RefCountedBase { return getTypeDeclTypeSlow(Decl); } + QualType getUsingType(const UsingShadowDecl *Found, + QualType Underlying) const; + /// Return the unique reference to the type for the specified /// typedef-name decl. QualType getTypedefType(const TypedefNameDecl *Decl, @@ -1564,6 +1568,9 @@ class ASTContext : public RefCountedBase { QualType getEnumType(const EnumDecl *Decl) const; + QualType + getUnresolvedUsingType(const UnresolvedUsingTypenameDecl *Decl) const; + QualType getInjectedClassNameType(CXXRecordDecl *Decl, QualType TST) const; QualType getAttributedType(attr::Kind attrKind, diff --git a/clang/include/clang/AST/ASTDiagnostic.h b/clang/include/clang/AST/ASTDiagnostic.h index d6549e12d92a..4cd909751725 100644 --- a/clang/include/clang/AST/ASTDiagnostic.h +++ b/clang/include/clang/AST/ASTDiagnostic.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_AST_ASTDIAGNOSTIC_H #define LLVM_CLANG_AST_ASTDIAGNOSTIC_H +#include "clang/AST/Type.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticAST.h" @@ -31,6 +32,11 @@ namespace clang { SmallVectorImpl &Output, void *Cookie, ArrayRef QualTypeVals); + + /// Returns a desugared version of the QualType, and marks ShouldAKA as true + /// whenever we remove significant sugar from the type. + QualType desugarForDiagnostic(ASTContext &Context, QualType QT, + bool &ShouldAKA); } // end namespace clang #endif diff --git a/clang/include/clang/AST/ASTImporterLookupTable.h b/clang/include/clang/AST/ASTImporterLookupTable.h index 47dca2033839..918c2b9e350c 100644 --- a/clang/include/clang/AST/ASTImporterLookupTable.h +++ b/clang/include/clang/AST/ASTImporterLookupTable.h @@ -75,6 +75,10 @@ class ASTImporterLookupTable { // The function should be called when the old context is definitely different // from the new. void update(NamedDecl *ND, DeclContext *OldDC); + // Same as 'update' but allow if 'ND' is not in the table or the old context + // is the same as the new. + // FIXME: The old redeclaration context is not handled. + void updateForced(NamedDecl *ND, DeclContext *OldDC); using LookupResult = DeclList; LookupResult lookup(DeclContext *DC, DeclarationName Name) const; // Check if the `ND` is within the lookup table (with its current name) in diff --git a/clang/include/clang/AST/AbstractBasicReader.h b/clang/include/clang/AST/AbstractBasicReader.h index 5505d661b44e..442039044cfe 100644 --- a/clang/include/clang/AST/AbstractBasicReader.h +++ b/clang/include/clang/AST/AbstractBasicReader.h @@ -21,7 +21,7 @@ inline T makeNullableFromOptional(const Optional &value) { template inline T *makePointerFromOptional(Optional value) { - return (value ? *value : nullptr); + return value.getValueOr(nullptr); } // PropertyReader is a class concept that requires the following method: diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h index d33babef958e..f7a2e3146d06 100644 --- a/clang/include/clang/AST/DeclTemplate.h +++ b/clang/include/clang/AST/DeclTemplate.h @@ -1211,13 +1211,12 @@ class TemplateTypeParmDecl final : public TypeDecl, DefArgStorage DefaultArgument; TemplateTypeParmDecl(DeclContext *DC, SourceLocation KeyLoc, - SourceLocation IdLoc, IdentifierInfo *Id, - bool Typename, bool HasTypeConstraint, - Optional NumExpanded) + SourceLocation IdLoc, IdentifierInfo *Id, bool Typename, + bool HasTypeConstraint, Optional NumExpanded) : TypeDecl(TemplateTypeParm, DC, IdLoc, Id, KeyLoc), Typename(Typename), - HasTypeConstraint(HasTypeConstraint), TypeConstraintInitialized(false), - ExpandedParameterPack(NumExpanded), - NumExpanded(NumExpanded ? *NumExpanded : 0) {} + HasTypeConstraint(HasTypeConstraint), TypeConstraintInitialized(false), + ExpandedParameterPack(NumExpanded), + NumExpanded(NumExpanded.getValueOr(0)) {} public: static TemplateTypeParmDecl *Create(const ASTContext &C, DeclContext *DC, diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index 2c63406fba18..e2c36e12393f 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -572,6 +572,12 @@ class Expr : public ValueStmt { bool isConstantInitializer(ASTContext &Ctx, bool ForRef, const Expr **Culprit = nullptr) const; + /// If this expression is an unambiguous reference to a single declaration, + /// in the style of __builtin_function_start, return that declaration. Note + /// that this may return a non-static member function or field in C++ if this + /// expression is a member pointer constant. + const ValueDecl *getAsBuiltinConstantDeclRef(const ASTContext &Context) const; + /// EvalStatus is a struct with detailed info about an evaluation in progress. struct EvalStatus { /// Whether the evaluated expression has side effects. diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h index 565eb0c9cf99..3fd1b6d30080 100644 --- a/clang/include/clang/AST/OpenMPClause.h +++ b/clang/include/clang/AST/OpenMPClause.h @@ -2224,6 +2224,47 @@ class OMPCaptureClause : public OMPClause { } }; +/// This represents 'compare' clause in the '#pragma omp atomic' +/// directive. +/// +/// \code +/// #pragma omp atomic compare +/// \endcode +/// In this example directive '#pragma omp atomic' has 'compare' clause. +class OMPCompareClause final : public OMPClause { +public: + /// Build 'compare' clause. + /// + /// \param StartLoc Starting location of the clause. + /// \param EndLoc Ending location of the clause. + OMPCompareClause(SourceLocation StartLoc, SourceLocation EndLoc) + : OMPClause(llvm::omp::OMPC_compare, StartLoc, EndLoc) {} + + /// Build an empty clause. + OMPCompareClause() + : OMPClause(llvm::omp::OMPC_compare, SourceLocation(), SourceLocation()) { + } + + child_range children() { + return child_range(child_iterator(), child_iterator()); + } + + const_child_range children() const { + return const_child_range(const_child_iterator(), const_child_iterator()); + } + + child_range used_children() { + return child_range(child_iterator(), child_iterator()); + } + const_child_range used_children() const { + return const_child_range(const_child_iterator(), const_child_iterator()); + } + + static bool classof(const OMPClause *T) { + return T->getClauseKind() == llvm::omp::OMPC_compare; + } +}; + /// This represents 'seq_cst' clause in the '#pragma omp atomic' /// directive. /// diff --git a/clang/include/clang/AST/PropertiesBase.td b/clang/include/clang/AST/PropertiesBase.td index a087cb406b29..9282b24eb68c 100644 --- a/clang/include/clang/AST/PropertiesBase.td +++ b/clang/include/clang/AST/PropertiesBase.td @@ -107,6 +107,8 @@ def DeclRef : RefPropertyType<"Decl"> { let ConstWhenWriting = 1; } SubclassPropertyType<"TemplateTypeParmDecl", DeclRef>; def TemplateTemplateParmDeclRef : SubclassPropertyType<"TemplateTemplateParmDecl", DeclRef>; + def UsingShadowDeclRef : + SubclassPropertyType<"UsingShadowDecl", DeclRef>; def ValueDeclRef : SubclassPropertyType<"ValueDecl", DeclRef>; def ElaboratedTypeKeyword : EnumPropertyType; diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h index 9797eac53dde..f62dc36de556 100644 --- a/clang/include/clang/AST/RecursiveASTVisitor.h +++ b/clang/include/clang/AST/RecursiveASTVisitor.h @@ -981,6 +981,7 @@ DEF_TRAVERSE_TYPE(FunctionProtoType, { TRY_TO(TraverseStmt(NE)); }) +DEF_TRAVERSE_TYPE(UsingType, {}) DEF_TRAVERSE_TYPE(UnresolvedUsingType, {}) DEF_TRAVERSE_TYPE(TypedefType, {}) @@ -1072,8 +1073,8 @@ DEF_TRAVERSE_TYPE(AtomicType, { TRY_TO(TraverseType(T->getValueType())); }) DEF_TRAVERSE_TYPE(PipeType, { TRY_TO(TraverseType(T->getElementType())); }) -DEF_TRAVERSE_TYPE(ExtIntType, {}) -DEF_TRAVERSE_TYPE(DependentExtIntType, +DEF_TRAVERSE_TYPE(BitIntType, {}) +DEF_TRAVERSE_TYPE(DependentBitIntType, { TRY_TO(TraverseStmt(T->getNumBitsExpr())); }) #undef DEF_TRAVERSE_TYPE @@ -1252,6 +1253,7 @@ DEF_TRAVERSE_TYPELOC(FunctionProtoType, { TRY_TO(TraverseStmt(NE)); }) +DEF_TRAVERSE_TYPELOC(UsingType, {}) DEF_TRAVERSE_TYPELOC(UnresolvedUsingType, {}) DEF_TRAVERSE_TYPELOC(TypedefType, {}) @@ -1358,8 +1360,8 @@ DEF_TRAVERSE_TYPELOC(AtomicType, { TRY_TO(TraverseTypeLoc(TL.getValueLoc())); }) DEF_TRAVERSE_TYPELOC(PipeType, { TRY_TO(TraverseTypeLoc(TL.getValueLoc())); }) -DEF_TRAVERSE_TYPELOC(ExtIntType, {}) -DEF_TRAVERSE_TYPELOC(DependentExtIntType, { +DEF_TRAVERSE_TYPELOC(BitIntType, {}) +DEF_TRAVERSE_TYPELOC(DependentBitIntType, { TRY_TO(TraverseStmt(TL.getTypePtr()->getNumBitsExpr())); }) @@ -2095,7 +2097,13 @@ bool RecursiveASTVisitor::TraverseFunctionHelper(FunctionDecl *D) { } if (VisitBody) { - TRY_TO(TraverseStmt(D->getBody())); // Function body. + TRY_TO(TraverseStmt(D->getBody())); + // Body may contain using declarations whose shadows are parented to the + // FunctionDecl itself. + for (auto *Child : D->decls()) { + if (isa(Child)) + TRY_TO(TraverseDecl(Child)); + } } return true; } @@ -3226,6 +3234,11 @@ bool RecursiveASTVisitor::VisitOMPCaptureClause(OMPCaptureClause *) { return true; } +template +bool RecursiveASTVisitor::VisitOMPCompareClause(OMPCompareClause *) { + return true; +} + template bool RecursiveASTVisitor::VisitOMPSeqCstClause(OMPSeqCstClause *) { return true; diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h index 0eb0031de11f..41bbf2ec593a 100644 --- a/clang/include/clang/AST/TextNodeDumper.h +++ b/clang/include/clang/AST/TextNodeDumper.h @@ -311,6 +311,7 @@ class TextNodeDumper void VisitFunctionType(const FunctionType *T); void VisitFunctionProtoType(const FunctionProtoType *T); void VisitUnresolvedUsingType(const UnresolvedUsingType *T); + void VisitUsingType(const UsingType *T); void VisitTypedefType(const TypedefType *T); void VisitUnaryTransformType(const UnaryTransformType *T); void VisitTagType(const TagType *T); diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h index 4c89c297bf34..a69c0ae67d0a 100644 --- a/clang/include/clang/AST/Type.h +++ b/clang/include/clang/AST/Type.h @@ -129,6 +129,7 @@ class TemplateArgumentLoc; class TemplateTypeParmDecl; class TypedefNameDecl; class UnresolvedUsingTypenameDecl; +class UsingShadowDecl; using CanQualType = CanQual; @@ -2128,7 +2129,7 @@ class alignas(8) Type : public ExtQualsTypeCommonBase { bool isOCLExtOpaqueType() const; // Any OpenCL extension type bool isPipeType() const; // OpenCL pipe type - bool isExtIntType() const; // Extended Int Type + bool isBitIntType() const; // Bit-precise integer type bool isOpenCLSpecificType() const; // Any OpenCL specific type /// Determines if this type, which must satisfy @@ -4368,6 +4369,27 @@ class UnresolvedUsingType : public Type { } }; +class UsingType : public Type, public llvm::FoldingSetNode { + UsingShadowDecl *Found; + friend class ASTContext; // ASTContext creates these. + + UsingType(const UsingShadowDecl *Found, QualType Underlying, QualType Canon); + +public: + UsingShadowDecl *getFoundDecl() const { return Found; } + QualType getUnderlyingType() const; + + bool isSugared() const { return true; } + QualType desugar() const { return getUnderlyingType(); } + + void Profile(llvm::FoldingSetNodeID &ID) { Profile(ID, Found); } + static void Profile(llvm::FoldingSetNodeID &ID, + const UsingShadowDecl *Found) { + ID.AddPointer(Found); + } + static bool classof(const Type *T) { return T->getTypeClass() == Using; } +}; + class TypedefType : public Type { TypedefNameDecl *Decl; @@ -6307,13 +6329,13 @@ class PipeType : public Type, public llvm::FoldingSetNode { }; /// A fixed int type of a specified bitwidth. -class ExtIntType final : public Type, public llvm::FoldingSetNode { +class BitIntType final : public Type, public llvm::FoldingSetNode { friend class ASTContext; unsigned IsUnsigned : 1; unsigned NumBits : 24; protected: - ExtIntType(bool isUnsigned, unsigned NumBits); + BitIntType(bool isUnsigned, unsigned NumBits); public: bool isUnsigned() const { return IsUnsigned; } @@ -6333,16 +6355,16 @@ class ExtIntType final : public Type, public llvm::FoldingSetNode { ID.AddInteger(NumBits); } - static bool classof(const Type *T) { return T->getTypeClass() == ExtInt; } + static bool classof(const Type *T) { return T->getTypeClass() == BitInt; } }; -class DependentExtIntType final : public Type, public llvm::FoldingSetNode { +class DependentBitIntType final : public Type, public llvm::FoldingSetNode { friend class ASTContext; const ASTContext &Context; llvm::PointerIntPair ExprAndUnsigned; protected: - DependentExtIntType(const ASTContext &Context, bool IsUnsigned, + DependentBitIntType(const ASTContext &Context, bool IsUnsigned, Expr *NumBits); public: @@ -6360,7 +6382,7 @@ class DependentExtIntType final : public Type, public llvm::FoldingSetNode { bool IsUnsigned, Expr *NumBitsExpr); static bool classof(const Type *T) { - return T->getTypeClass() == DependentExtInt; + return T->getTypeClass() == DependentBitInt; } }; @@ -6891,8 +6913,8 @@ inline bool Type::isPipeType() const { return isa(CanonicalType); } -inline bool Type::isExtIntType() const { - return isa(CanonicalType); +inline bool Type::isBitIntType() const { + return isa(CanonicalType); } #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \ @@ -6998,7 +7020,7 @@ inline bool Type::isIntegerType() const { return IsEnumDeclComplete(ET->getDecl()) && !IsEnumDeclScoped(ET->getDecl()); } - return isExtIntType(); + return isBitIntType(); } inline bool Type::isFixedPointType() const { @@ -7056,7 +7078,7 @@ inline bool Type::isScalarType() const { isa(CanonicalType) || isa(CanonicalType) || isa(CanonicalType) || - isExtIntType(); + isBitIntType(); } inline bool Type::isIntegralOrEnumerationType() const { @@ -7069,7 +7091,7 @@ inline bool Type::isIntegralOrEnumerationType() const { if (const auto *ET = dyn_cast(CanonicalType)) return IsEnumDeclComplete(ET->getDecl()); - return isExtIntType(); + return isBitIntType(); } inline bool Type::isBooleanType() const { diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h index bb668c1980fe..7a036836e8c4 100644 --- a/clang/include/clang/AST/TypeLoc.h +++ b/clang/include/clang/AST/TypeLoc.h @@ -665,6 +665,16 @@ class BuiltinTypeLoc : public ConcreteTypeLoc { +public: + QualType getUnderlyingType() const { + return getTypePtr()->getUnderlyingType(); + } + UsingShadowDecl *getFoundDecl() const { return getTypePtr()->getFoundDecl(); } +}; + /// Wrapper for source info for typedefs. class TypedefTypeLoc : public InheritingConcreteTypeLoc(); } -class ExtIntTypeLoc final - : public InheritingConcreteTypeLoc {}; -class DependentExtIntTypeLoc final - : public InheritingConcreteTypeLoc {}; +class BitIntTypeLoc final + : public InheritingConcreteTypeLoc {}; +class DependentBitIntTypeLoc final + : public InheritingConcreteTypeLoc {}; } // namespace clang diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td index 438d5af5a2e2..8cbb67589b94 100644 --- a/clang/include/clang/AST/TypeProperties.td +++ b/clang/include/clang/AST/TypeProperties.td @@ -358,7 +358,20 @@ let Class = UnresolvedUsingType in { } def : Creator<[{ - return ctx.getTypeDeclType(cast(declaration)); + return ctx.getUnresolvedUsingType(cast(declaration)); + }]>; +} + +let Class = UsingType in { + def : Property<"foundDeclaration", UsingShadowDeclRef> { + let Read = [{ node->getFoundDecl() }]; + } + def : Property<"underlyingType", QualType> { + let Read = [{ node->getUnderlyingType() }]; + } + + def : Creator<[{ + return ctx.getUsingType(foundDeclaration, underlyingType); }]>; } @@ -882,7 +895,7 @@ let Class = PipeType in { }]>; } -let Class = ExtIntType in { +let Class = BitIntType in { def : Property<"isUnsigned", Bool> { let Read = [{ node->isUnsigned() }]; } @@ -891,11 +904,11 @@ let Class = ExtIntType in { } def : Creator<[{ - return ctx.getExtIntType(isUnsigned, numBits); + return ctx.getBitIntType(isUnsigned, numBits); }]>; } -let Class = DependentExtIntType in { +let Class = DependentBitIntType in { def : Property<"isUnsigned", Bool> { let Read = [{ node->isUnsigned() }]; } @@ -903,6 +916,6 @@ let Class = DependentExtIntType in { let Read = [{ node->getNumBitsExpr() }]; } def : Creator<[{ - return ctx.getDependentExtIntType(isUnsigned, numBitsExpr); + return ctx.getDependentBitIntType(isUnsigned, numBitsExpr); }]>; } diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h index 5221d05477d0..55cce324b436 100644 --- a/clang/include/clang/ASTMatchers/ASTMatchers.h +++ b/clang/include/clang/ASTMatchers/ASTMatchers.h @@ -4128,25 +4128,34 @@ AST_MATCHER_P(DeclRefExpr, to, internal::Matcher, InnerMatcher.matches(*DeclNode, Finder, Builder)); } -/// Matches a \c DeclRefExpr that refers to a declaration through a -/// specific using shadow declaration. +/// Matches if a node refers to a declaration through a specific +/// using shadow declaration. /// -/// Given +/// Examples: /// \code -/// namespace a { void f() {} } +/// namespace a { int f(); } /// using a::f; -/// void g() { -/// f(); // Matches this .. -/// a::f(); // .. but not this. -/// } +/// int x = f(); /// \endcode /// declRefExpr(throughUsingDecl(anything())) -/// matches \c f() -AST_MATCHER_P(DeclRefExpr, throughUsingDecl, - internal::Matcher, InnerMatcher) { +/// matches \c f +/// +/// \code +/// namespace a { class X{}; } +/// using a::X; +/// X x; +/// \code +/// typeLoc(loc(usingType(throughUsingDecl(anything())))) +/// matches \c X +/// +/// Usable as: Matcher, Matcher +AST_POLYMORPHIC_MATCHER_P(throughUsingDecl, + AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, + UsingType), + internal::Matcher, Inner) { const NamedDecl *FoundDecl = Node.getFoundDecl(); if (const UsingShadowDecl *UsingDecl = dyn_cast(FoundDecl)) - return InnerMatcher.matches(*UsingDecl, Finder, Builder); + return Inner.matches(*UsingDecl, Finder, Builder); return false; } @@ -4872,7 +4881,7 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParamType, } } - int ParamIndex = 0; + unsigned ParamIndex = 0; bool Matched = false; unsigned NumArgs = Node.getNumArgs(); if (FProto && FProto->isVariadic()) @@ -4886,7 +4895,7 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParamType, // This test is cheaper compared to the big matcher in the next if. // Therefore, please keep this order. - if (FProto) { + if (FProto && FProto->getNumParams() > ParamIndex) { QualType ParamType = FProto->getParamType(ParamIndex); if (ParamMatcher.matches(ParamType, Finder, &ParamMatches)) { Result.addMatch(ParamMatches); @@ -6843,7 +6852,7 @@ extern const AstTypeMatcher decltypeType; AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType, AST_POLYMORPHIC_SUPPORTED_TYPES(AutoType)); -/// Matches \c DecltypeType nodes to find out the underlying type. +/// Matches \c DecltypeType or \c UsingType nodes to find the underlying type. /// /// Given /// \code @@ -6853,9 +6862,10 @@ AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType, /// decltypeType(hasUnderlyingType(isInteger())) /// matches the type of "a" /// -/// Usable as: Matcher +/// Usable as: Matcher, Matcher AST_TYPE_TRAVERSE_MATCHER(hasUnderlyingType, getUnderlyingType, - AST_POLYMORPHIC_SUPPORTED_TYPES(DecltypeType)); + AST_POLYMORPHIC_SUPPORTED_TYPES(DecltypeType, + UsingType)); /// Matches \c FunctionType nodes. /// @@ -7183,6 +7193,18 @@ AST_MATCHER_P(ElaboratedType, namesType, internal::Matcher, return InnerMatcher.matches(Node.getNamedType(), Finder, Builder); } +/// Matches types specified through a using declaration. +/// +/// Given +/// \code +/// namespace a { struct S {}; } +/// using a::S; +/// S s; +/// \endcode +/// +/// \c usingType() matches the type of the variable declaration of \c s. +extern const AstTypeMatcher usingType; + /// Matches types that represent the result of substituting a type for a /// template type parameter. /// diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h index a77611001fb1..ab7a445dbcd4 100644 --- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h +++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h @@ -1090,6 +1090,12 @@ class HasDeclarationMatcher : public MatcherInterface { if (const auto *S = dyn_cast(&Node)) { return matchesSpecialized(S->desugar(), Finder, Builder); } + // Similarly types found via using declarations. + // These are *usually* meaningless sugar, and this matches the historical + // behavior prior to the introduction of UsingType. + if (const auto *S = dyn_cast(&Node)) { + return matchesSpecialized(S->desugar(), Finder, Builder); + } return false; } diff --git a/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h b/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h index 4a58fe870944..a0ae44131b45 100644 --- a/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h +++ b/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h @@ -155,7 +155,7 @@ class CFGWalker { return false; // Ignore anonymous functions. - if (!dyn_cast_or_null(AC.getDecl())) + if (!isa_and_nonnull(AC.getDecl())) return false; SortedGraph = AC.getAnalysis(); diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index 69a5c2e47b66..4a3c0239f8e1 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -15,11 +15,20 @@ #ifndef LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_DATAFLOWENVIRONMENT_H #define LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_DATAFLOWENVIRONMENT_H +#include "clang/Analysis/FlowSensitive/DataflowLattice.h" + namespace clang { namespace dataflow { /// Holds the state of the program (store and heap) at a given program point. -class Environment {}; +class Environment { +public: + bool operator==(const Environment &) const { return true; } + + LatticeJoinEffect join(const Environment &) { + return LatticeJoinEffect::Unchanged; + } +}; } // namespace dataflow } // namespace clang diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h b/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h index 90095735ad3d..52d84eb13c56 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h @@ -61,11 +61,12 @@ struct ReversePostOrderCompare { /// the same block multiple times at once. struct ForwardDataflowWorklist : DataflowWorklistBase { + ForwardDataflowWorklist(const CFG &Cfg, PostOrderCFGView *POV) + : DataflowWorklistBase(Cfg, POV, + ReversePostOrderCompare{POV->getComparator()}) {} + ForwardDataflowWorklist(const CFG &Cfg, AnalysisDeclContext &Ctx) - : DataflowWorklistBase( - Cfg, Ctx.getAnalysis(), - ReversePostOrderCompare{ - Ctx.getAnalysis()->getComparator()}) {} + : ForwardDataflowWorklist(Cfg, Ctx.getAnalysis()) {} void enqueueSuccessors(const CFGBlock *Block) { for (auto B : Block->succs()) diff --git a/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h b/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h index 9448b911f471..6193b9860d33 100644 --- a/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h +++ b/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h @@ -76,6 +76,24 @@ struct TypeErasedDataflowAnalysisState { Environment Env; }; +/// Transfers the state of a basic block by evaluating each of its statements in +/// the context of `Analysis` and the states of its predecessors that are +/// available in `BlockStates`. `HandleTransferredStmt` (if provided) will be +/// applied to each statement in the block, after it is evaluated. +/// +/// Requirements: +/// +/// All predecessors of `Block` except those with loop back edges must have +/// already been transferred. States in `BlockStates` that are set to +/// `llvm::None` represent basic blocks that are not evaluated yet. +TypeErasedDataflowAnalysisState transferBlock( + std::vector> &BlockStates, + const CFGBlock &Block, const Environment &InitEnv, + TypeErasedDataflowAnalysis &Analysis, + std::function + HandleTransferredStmt = nullptr); + /// Performs dataflow analysis and returns a mapping from basic block IDs to /// dataflow analysis states that model the respective basic blocks. Indices /// of the returned vector correspond to basic block IDs. diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index fab3f3edfb83..10c5c7f1b879 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -3865,6 +3865,14 @@ def ReleaseHandle : InheritableParamAttr { let Documentation = [ReleaseHandleDocs]; } +def DiagnoseAsBuiltin : InheritableAttr { + let Spellings = [Clang<"diagnose_as_builtin">]; + let Args = [DeclArgument, + VariadicUnsignedArgument<"ArgIndices">]; + let Subjects = SubjectList<[Function]>; + let Documentation = [DiagnoseAsBuiltinDocs]; +} + def Builtin : InheritableAttr { let Spellings = []; let Args = [UnsignedArgument<"ID">]; diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index 10cce4c2d689..8a7424a88c9f 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -187,6 +187,10 @@ primary use is for COFF object files which explicitly specify what interfaces are imported from external modules. See the dllimport_ documentation on MSDN for more information. +Note that a dllimport function may still be inlined, if its definition is +available and it doesn't reference any non-dllimport functions or global +variables. + .. _dllimport: https://msdn.microsoft.com/en-us/library/3y1sfaz2.aspx }]; } @@ -5980,6 +5984,50 @@ attribute requires a string literal argument to identify the handle being releas }]; } +def DiagnoseAsBuiltinDocs : Documentation { + let Category = DocCatFunction; + let Content = [{ +The ``diagnose_as_builtin` attribute indicates that Fortify diagnostics are to +be applied to the declared function as if it were the function specified by the +attribute. The builtin function whose diagnostics are to be mimicked should be +given. In addition, the order in which arguments should be applied must also +be given. + +For example, the attribute can be used as follows. + + .. code-block:: c + + __attribute__((diagnose_as_builtin(__builtin_memset, 3, 2, 1))) + void *mymemset(int n, int c, void *s) { + // ... + } + +This indicates that calls to ``mymemset`` should be diagnosed as if they were +calls to ``__builtin_memset``. The arguments ``3, 2, 1`` indicate by index the +order in which arguments of ``mymemset`` should be applied to +``__builtin_memset``. The third argument should be applied first, then the +second, and then the first. Thus (when Fortify warnings are enabled) the call +``mymemset(n, c, s)`` will diagnose overflows as if it were the call +``__builtin_memset(s, c, n)``. + +For variadic functions, the variadic arguments must come in the same order as +they would to the builtin function, after all normal arguments. For instance, +to diagnose a new function as if it were `sscanf`, we can use the attribute as +follows. + + .. code-block:: c + __attribute__((diagnose_as_builtin(sscanf, 1, 2))) + int mysscanf(const char *str, const char *format, ...) { + // ... + } + +Then the call `mysscanf("abc def", "%4s %4s", buf1, buf2)` will be diagnosed as +if it were the call `sscanf("abc def", "%4s %4s", buf1, buf2)`. + +This attribute cannot be applied to non-static member functions. +}]; +} + def ArmSveVectorBitsDocs : Documentation { let Category = DocCatType; let Content = [{ diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def index ad8b66aa490b..45d445163749 100644 --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -646,8 +646,10 @@ BUILTIN(__builtin_call_with_static_chain, "v.", "nt") BUILTIN(__builtin_elementwise_abs, "v.", "nct") BUILTIN(__builtin_elementwise_max, "v.", "nct") BUILTIN(__builtin_elementwise_min, "v.", "nct") +BUILTIN(__builtin_elementwise_ceil, "v.", "nct") BUILTIN(__builtin_reduce_max, "v.", "nct") BUILTIN(__builtin_reduce_min, "v.", "nct") +BUILTIN(__builtin_reduce_xor, "v.", "nct") BUILTIN(__builtin_matrix_transpose, "v.", "nFt") BUILTIN(__builtin_matrix_column_major_load, "v.", "nFt") @@ -1574,6 +1576,7 @@ BUILTIN(__builtin_smulll_overflow, "bSLLiCSLLiCSLLi*", "n") // Clang builtins (not available in GCC). BUILTIN(__builtin_addressof, "v*v&", "nct") +BUILTIN(__builtin_function_start, "v*v&", "nct") BUILTIN(__builtin_operator_new, "v*z", "tc") BUILTIN(__builtin_operator_delete, "vv*", "tn") BUILTIN(__builtin_char_memchr, "c*cC*iz", "n") @@ -1611,7 +1614,6 @@ LANGBUILTIN(__builtin_coro_alloc, "b", "n", COR_LANG) LANGBUILTIN(__builtin_coro_begin, "v*v*", "n", COR_LANG) LANGBUILTIN(__builtin_coro_end, "bv*Ib", "n", COR_LANG) LANGBUILTIN(__builtin_coro_suspend, "cIb", "n", COR_LANG) -LANGBUILTIN(__builtin_coro_param, "bv*v*", "n", COR_LANG) // OpenCL v2.0 s6.13.16, s9.17.3.5 - Pipe functions. // We need the generic prototype, since the packet type could be anything. diff --git a/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def new file mode 100644 index 000000000000..8e3229984d8b --- /dev/null +++ b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def @@ -0,0 +1,39 @@ +#ifdef GET_SVE_BUILTINS +BUILTIN(__builtin_sve_svget_neonq_s8, "V16Scq16Sc", "n") +BUILTIN(__builtin_sve_svget_neonq_s16, "V8sq8s", "n") +BUILTIN(__builtin_sve_svget_neonq_s32, "V4iq4i", "n") +BUILTIN(__builtin_sve_svget_neonq_s64, "V2Wiq2Wi", "n") +BUILTIN(__builtin_sve_svget_neonq_u8, "V16Ucq16Uc", "n") +BUILTIN(__builtin_sve_svget_neonq_u16, "V16Usq16Us", "n") +BUILTIN(__builtin_sve_svget_neonq_u32, "V4Uiq4Ui", "n") +BUILTIN(__builtin_sve_svget_neonq_u64, "V2UWiq2UWi", "n") +BUILTIN(__builtin_sve_svget_neonq_f16, "V8hq8h", "n") +BUILTIN(__builtin_sve_svget_neonq_f32, "V4fq4f", "n") +BUILTIN(__builtin_sve_svget_neonq_f64, "V2dq2d", "n") +BUILTIN(__builtin_sve_svget_neonq_bf16, "V8yq8y", "n") +BUILTIN(__builtin_sve_svset_neonq_s8, "q16Scq16ScV16Sc", "n") +BUILTIN(__builtin_sve_svset_neonq_s16, "q8sq8sV8s", "n") +BUILTIN(__builtin_sve_svset_neonq_s32, "q4iq4iV4i", "n") +BUILTIN(__builtin_sve_svset_neonq_s64, "q2Wiq2WiV2Wi", "n") +BUILTIN(__builtin_sve_svset_neonq_u8, "q16Ucq16UcV16Uc", "n") +BUILTIN(__builtin_sve_svset_neonq_u16, "q8Usq8UsV8s", "n") +BUILTIN(__builtin_sve_svset_neonq_u32, "q4Uiq4UiV4Ui", "n") +BUILTIN(__builtin_sve_svset_neonq_u64, "q2UWiq2UWiV2UWi", "n") +BUILTIN(__builtin_sve_svset_neonq_f16, "q8hq8hV8h", "n") +BUILTIN(__builtin_sve_svset_neonq_f32, "q4fq4fV4f", "n") +BUILTIN(__builtin_sve_svset_neonq_f64, "q2dq2dV2d", "n") +BUILTIN(__builtin_sve_svset_neonq_bf16, "q8yq8yV8y", "n") +BUILTIN(__builtin_sve_svdup_neonq_s8, "q16ScV16Sc", "n") +BUILTIN(__builtin_sve_svdup_neonq_s16, "q8sV8s", "n") +BUILTIN(__builtin_sve_svdup_neonq_s32, "q4iV4i", "n") +BUILTIN(__builtin_sve_svdup_neonq_s64, "q4iV4i", "n") +BUILTIN(__builtin_sve_svdup_neonq_u8, "q16UcV16Uc", "n") +BUILTIN(__builtin_sve_svdup_neonq_u16, "q8UsV8Us", "n") +BUILTIN(__builtin_sve_svdup_neonq_u32, "q4UiV4Ui", "n") +BUILTIN(__builtin_sve_svdup_neonq_u64, "q2UWiV2UWi", "n") +BUILTIN(__builtin_sve_svdup_neonq_f16, "q8hV8h", "n") +BUILTIN(__builtin_sve_svdup_neonq_f32, "q4fV4f", "n") +BUILTIN(__builtin_sve_svdup_neonq_f64, "q2dV2d", "n") +BUILTIN(__builtin_sve_svdup_neonq_bf16, "q8yV8y", "n") +#endif + diff --git a/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def new file mode 100644 index 000000000000..7717ba67b427 --- /dev/null +++ b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def @@ -0,0 +1,39 @@ +#ifdef GET_SVE_LLVM_INTRINSIC_MAP +SVEMAP2(svget_neonq_s8, SVETypeFlags::EltTyInt8), +SVEMAP2(svget_neonq_s16, SVETypeFlags::EltTyInt16), +SVEMAP2(svget_neonq_s32, SVETypeFlags::EltTyInt32), +SVEMAP2(svget_neonq_s64, SVETypeFlags::EltTyInt64), +SVEMAP2(svget_neonq_u8, SVETypeFlags::EltTyInt8), +SVEMAP2(svget_neonq_u16, SVETypeFlags::EltTyInt16), +SVEMAP2(svget_neonq_u32, SVETypeFlags::EltTyInt32), +SVEMAP2(svget_neonq_u64, SVETypeFlags::EltTyInt64), +SVEMAP2(svget_neonq_f16, SVETypeFlags::EltTyFloat16), +SVEMAP2(svget_neonq_f32, SVETypeFlags::EltTyFloat32), +SVEMAP2(svget_neonq_f64, SVETypeFlags::EltTyFloat64), +SVEMAP2(svget_neonq_bf16, SVETypeFlags::EltTyBFloat16), +SVEMAP2(svset_neonq_s8, SVETypeFlags::EltTyInt8), +SVEMAP2(svset_neonq_s16, SVETypeFlags::EltTyInt16), +SVEMAP2(svset_neonq_s32, SVETypeFlags::EltTyInt32), +SVEMAP2(svset_neonq_s64, SVETypeFlags::EltTyInt64), +SVEMAP2(svset_neonq_u8, SVETypeFlags::EltTyInt8), +SVEMAP2(svset_neonq_u16, SVETypeFlags::EltTyInt16), +SVEMAP2(svset_neonq_u32, SVETypeFlags::EltTyInt32), +SVEMAP2(svset_neonq_u64, SVETypeFlags::EltTyInt64), +SVEMAP2(svset_neonq_f16, SVETypeFlags::EltTyFloat16), +SVEMAP2(svset_neonq_f32, SVETypeFlags::EltTyFloat32), +SVEMAP2(svset_neonq_f64, SVETypeFlags::EltTyFloat64), +SVEMAP2(svset_neonq_bf16, SVETypeFlags::EltTyBFloat16), +SVEMAP2(svdup_neonq_s8, SVETypeFlags::EltTyInt8), +SVEMAP2(svdup_neonq_s16, SVETypeFlags::EltTyInt16), +SVEMAP2(svdup_neonq_s32, SVETypeFlags::EltTyInt32), +SVEMAP2(svdup_neonq_s64, SVETypeFlags::EltTyInt64), +SVEMAP2(svdup_neonq_u8, SVETypeFlags::EltTyInt8), +SVEMAP2(svdup_neonq_u16, SVETypeFlags::EltTyInt16), +SVEMAP2(svdup_neonq_u32, SVETypeFlags::EltTyInt32), +SVEMAP2(svdup_neonq_u64, SVETypeFlags::EltTyInt64), +SVEMAP2(svdup_neonq_f16, SVETypeFlags::EltTyFloat16), +SVEMAP2(svdup_neonq_f32, SVETypeFlags::EltTyFloat32), +SVEMAP2(svdup_neonq_f64, SVETypeFlags::EltTyFloat64), +SVEMAP2(svdup_neonq_bf16, SVETypeFlags::EltTyBFloat16), +#endif + diff --git a/clang/include/clang/Basic/BuiltinsHexagon.def b/clang/include/clang/Basic/BuiltinsHexagon.def index 0001bd556117..0f62c235bb62 100644 --- a/clang/include/clang/Basic/BuiltinsHexagon.def +++ b/clang/include/clang/Basic/BuiltinsHexagon.def @@ -17,8 +17,10 @@ # define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS) #endif +#pragma push_macro("V69") +#define V69 "v69" #pragma push_macro("V68") -#define V68 "v68" +#define V68 "v68|" V69 #pragma push_macro("V67") #define V67 "v67|" V68 #pragma push_macro("V66") @@ -34,8 +36,10 @@ #pragma push_macro("V5") #define V5 "v5|" V55 +#pragma push_macro("HVXV69") +#define HVXV69 "hvxv69" #pragma push_macro("HVXV68") -#define HVXV68 "hvxv68" +#define HVXV68 "hvxv68|" HVXV69 #pragma push_macro("HVXV67") #define HVXV67 "hvxv67|" HVXV68 #pragma push_macro("HVXV66") @@ -128,6 +132,7 @@ TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_acc_128B,"V64iV64iV32iLLi","", " #pragma pop_macro("HVXV66") #pragma pop_macro("HVXV67") #pragma pop_macro("HVXV68") +#pragma pop_macro("HVXV69") #pragma pop_macro("V5") #pragma pop_macro("V55") @@ -137,6 +142,7 @@ TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_acc_128B,"V64iV64iV32iLLi","", " #pragma pop_macro("V66") #pragma pop_macro("V67") #pragma pop_macro("V68") +#pragma pop_macro("V69") #undef BUILTIN #undef TARGET_BUILTIN diff --git a/clang/include/clang/Basic/BuiltinsHexagonDep.def b/clang/include/clang/Basic/BuiltinsHexagonDep.def index 152c9c4dd8ad..2eb4ca69c7bd 100644 --- a/clang/include/clang/Basic/BuiltinsHexagonDep.def +++ b/clang/include/clang/Basic/BuiltinsHexagonDep.def @@ -1739,3 +1739,150 @@ TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10, "V32iV32iV32iUIi", "", HVXV68) TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_128B, "V64iV64iV64iUIi", "", HVXV68) TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_vxx, "V32iV32iV32iV32iUIi", "", HVXV68) TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_vxx_128B, "V64iV64iV64iV64iUIi", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_hf, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_hf_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_sf, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_sf_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_mix, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_mix_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_mix, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_mix_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_hf, "V32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_hf_128B, "V64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_fp, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_fp_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf16, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf16_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf32, "V16iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf32_128B, "V32iV64i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_qf32, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_qf32_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_b_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_b_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_h_hf, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_h_hf_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_b, "V32iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_b_128B, "V64iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_h, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_h_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_ub, "V32iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_ub_128B, "V64iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_uh, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_uh_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_sf_hf, "V32iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_sf_hf_128B, "V64iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_ub_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_ub_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_uh_hf, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_uh_hf_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc, "V16iV16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc_128B, "V32iV32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_hf, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_hf_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_sf, "V16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_sf_128B, "V32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf, "V64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_128B, "V128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_and, "V64bV64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_and_128B, "V128bV128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_or, "V64bV64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_or_128B, "V128bV128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_xor, "V64bV64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_xor_128B, "V128bV128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf, "V64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_128B, "V128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_and, "V64bV64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_and_128B, "V128bV128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_or, "V64bV64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_or_128B, "V128bV128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_xor, "V64bV64bV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_xor_128B, "V128bV128bV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_acc, "V16iV16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_acc_128B, "V32iV32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_hf, "V32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_hf_128B, "V64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf, "V32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf_128B, "V64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_qf16, "V32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_qf16_128B, "V64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf, "V32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_128B, "V64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_acc, "V32iV32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_acc_128B, "V64iV64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_hf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_hf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_mix, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_mix_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_mix, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_mix_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_128B, "V32iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_hf, "V32iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_hf_128B, "V64iV32iV32i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_sf, "V16iV16iV16i", "", HVXV68) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_sf_128B, "V32iV32iV32i", "", HVXV68) + +// V69 HVX Instructions. + +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubrndsat, "V16iV32iV16i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubrndsat_128B, "V32iV64iV32i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubsat, "V16iV32iV16i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubsat_128B, "V32iV64iV32i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhrndsat, "V16iV32iV16i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhrndsat_128B, "V32iV64iV32i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhsat, "V16iV32iV16i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhsat_128B, "V32iV64iV32i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhvs, "V16iV16iV16i", "", HVXV69) +TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhvs_128B, "V32iV32iV32i", "", HVXV69) diff --git a/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def b/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def index 93f560fc5adc..389eadb21d01 100644 --- a/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def +++ b/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def @@ -8,199 +8,7 @@ // Automatically generated file, do not edit! //===----------------------------------------------------------------------===// -CUSTOM_BUILTIN_MAPPING(A2_add, 0) -CUSTOM_BUILTIN_MAPPING(A2_addi, 0) -CUSTOM_BUILTIN_MAPPING(A2_addp, 0) -CUSTOM_BUILTIN_MAPPING(A2_and, 0) -CUSTOM_BUILTIN_MAPPING(A2_andir, 0) -CUSTOM_BUILTIN_MAPPING(A2_neg, 0) -CUSTOM_BUILTIN_MAPPING(A2_not, 0) -CUSTOM_BUILTIN_MAPPING(A2_or, 0) -CUSTOM_BUILTIN_MAPPING(A2_orir, 0) -CUSTOM_BUILTIN_MAPPING(A2_sub, 0) -CUSTOM_BUILTIN_MAPPING(A2_subp, 0) -CUSTOM_BUILTIN_MAPPING(A2_subri, 0) -CUSTOM_BUILTIN_MAPPING(A2_sxtb, 0) -CUSTOM_BUILTIN_MAPPING(A2_sxth, 0) -CUSTOM_BUILTIN_MAPPING(A2_xor, 0) -CUSTOM_BUILTIN_MAPPING(A2_zxtb, 0) -CUSTOM_BUILTIN_MAPPING(A2_zxth, 0) -CUSTOM_BUILTIN_MAPPING(M2_dpmpyss_s0, 0) -CUSTOM_BUILTIN_MAPPING(M2_dpmpyuu_s0, 0) -CUSTOM_BUILTIN_MAPPING(M2_mpyi, 0) -CUSTOM_BUILTIN_MAPPING(M2_mpysmi, 0) -CUSTOM_BUILTIN_MAPPING(M2_mpyui, 0) -CUSTOM_BUILTIN_MAPPING(S2_asl_i_p, 0) -CUSTOM_BUILTIN_MAPPING(S2_asl_i_r, 0) -CUSTOM_BUILTIN_MAPPING(S2_asr_i_p, 0) -CUSTOM_BUILTIN_MAPPING(S2_asr_i_r, 0) -CUSTOM_BUILTIN_MAPPING(S2_lsr_i_p, 0) -CUSTOM_BUILTIN_MAPPING(S2_lsr_i_r, 0) -CUSTOM_BUILTIN_MAPPING(V6_pred_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_and_n, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_and_n_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_not, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_not_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_or_n, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_or_n_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_nqpred_ai, 64) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_nqpred_ai_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_nqpred_ai, 64) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_nqpred_ai_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_qpred_ai, 64) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_qpred_ai_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_qpred_ai, 64) -CUSTOM_BUILTIN_MAPPING(V6_vS32b_qpred_ai_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddbnq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddbnq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddbq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddbq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddhnq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddhnq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddhq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddhq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddwnq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddwnq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddwq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddwq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandqrt, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandqrt_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandqrt_acc, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandqrt_acc_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandvrt, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandvrt_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandvrt_acc, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandvrt_acc_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqb, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqb_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqb_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqb_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqb_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqb_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqb_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqb_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqh, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqh_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqh_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqh_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqh_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqh_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqh_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqh_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqw, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqw_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqw_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqw_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqw_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqw_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_veqw_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_veqw_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtb, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtb_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgth, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgth_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgth_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgth_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgth_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgth_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgth_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgth_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtub, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtub_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuh_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtuw_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtw, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_and, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_and_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_or, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_or_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_xor, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgtw_xor_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vmux, 64) -CUSTOM_BUILTIN_MAPPING(V6_vmux_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vsubbnq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vsubbnq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vsubbq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vsubbq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vsubhnq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vsubhnq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vsubhq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vsubhq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vsubwnq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vsubwnq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vsubwq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vsubwq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vswap, 64) -CUSTOM_BUILTIN_MAPPING(V6_vswap_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2v2, 64) -CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2v2_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_shuffeqh, 64) -CUSTOM_BUILTIN_MAPPING(V6_shuffeqh_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_shuffeqw, 64) -CUSTOM_BUILTIN_MAPPING(V6_shuffeqw_128B, 128) CUSTOM_BUILTIN_MAPPING(V6_vaddcarry, 64) CUSTOM_BUILTIN_MAPPING(V6_vaddcarry_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandnqrt, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandnqrt_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandnqrt_acc, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandnqrt_acc_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandvnqv, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandvnqv_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vandvqv, 64) -CUSTOM_BUILTIN_MAPPING(V6_vandvqv_128B, 128) CUSTOM_BUILTIN_MAPPING(V6_vsubcarry, 64) CUSTOM_BUILTIN_MAPPING(V6_vsubcarry_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgathermhq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgathermhq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgathermhwq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgathermhwq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vgathermwq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vgathermwq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vprefixqb, 64) -CUSTOM_BUILTIN_MAPPING(V6_vprefixqb_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vprefixqh, 64) -CUSTOM_BUILTIN_MAPPING(V6_vprefixqh_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vprefixqw, 64) -CUSTOM_BUILTIN_MAPPING(V6_vprefixqw_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vscattermhq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vscattermhq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vscattermhwq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vscattermhwq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vscattermwq, 64) -CUSTOM_BUILTIN_MAPPING(V6_vscattermwq_128B, 128) -CUSTOM_BUILTIN_MAPPING(V6_vaddcarrysat, 64) -CUSTOM_BUILTIN_MAPPING(V6_vaddcarrysat_128B, 128) diff --git a/clang/include/clang/Basic/BuiltinsSVE.def b/clang/include/clang/Basic/BuiltinsSVE.def index 2839ca992d98..c70525e967b1 100644 --- a/clang/include/clang/Basic/BuiltinsSVE.def +++ b/clang/include/clang/Basic/BuiltinsSVE.def @@ -15,6 +15,7 @@ #define GET_SVE_BUILTINS #include "clang/Basic/arm_sve_builtins.inc" +#include "clang/Basic/BuiltinsAArch64NeonSVEBridge.def" #undef GET_SVE_BUILTINS #undef BUILTIN diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def index 94b3003a9c33..723302f108e2 100644 --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -368,9 +368,6 @@ ENUM_CODEGENOPT(DebuggerTuning, llvm::DebuggerKind, 3, /// emitted. VALUE_CODEGENOPT(DwarfVersion, 3, 0) -/// Whether to use experimental new variable location tracking. -CODEGENOPT(ValueTrackingVariableLocations, 1, 0) - /// Whether we should emit CodeView debug information. It's possible to emit /// CodeView and DWARF into the same object. CODEGENOPT(EmitCodeView, 1, 0) diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index 664e4998b8de..33ec03a17136 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -227,6 +227,9 @@ class CodeGenOptions : public CodeGenOptionsBase { /// Output filename for the split debug info, not used in the skeleton CU. std::string SplitDwarfOutput; + /// Output filename used in the COFF debug information. + std::string ObjectFilenameForDebug; + /// The name of the relocation model to use. llvm::Reloc::Model RelocationModel; @@ -395,7 +398,7 @@ class CodeGenOptions : public CodeGenOptionsBase { /// Executable and command-line used to create a given CompilerInvocation. /// Most of the time this will be the full -cc1 command. const char *Argv0 = nullptr; - ArrayRef CommandLineArgs; + std::vector CommandLineArgs; /// The minimum hotness value a diagnostic needs in order to be included in /// optimization diagnostics. diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h index 8c08ab3f5d74..04967925e9a4 100644 --- a/clang/include/clang/Basic/Cuda.h +++ b/clang/include/clang/Basic/Cuda.h @@ -95,6 +95,8 @@ enum class CudaArch { GFX1033, GFX1034, GFX1035, + Generic, // A processor model named 'generic' if the target backend defines a + // public one. LAST, }; @@ -103,7 +105,8 @@ static inline bool IsNVIDIAGpuArch(CudaArch A) { } static inline bool IsAMDGpuArch(CudaArch A) { - return A >= CudaArch::GFX600 && A < CudaArch::LAST; + // Generic processor model is for testing only. + return A >= CudaArch::GFX600 && A < CudaArch::Generic; } const char *CudaArchToString(CudaArch A); diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 2f50918b527b..a7fd2f26478c 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -74,6 +74,14 @@ def err_drv_no_hip_runtime : Error< "cannot find HIP runtime; provide its path via '--rocm-path', or pass " "'-nogpuinc' to build without HIP runtime">; +def err_drv_no_hipspv_device_lib : Error< + "cannot find HIP device library%select{| for %1}0; provide its path via " + "'--hip-path' or '--hip-device-lib-path', or pass '-nogpulib' to build " + "without HIP device library">; +def err_drv_hipspv_no_hip_path : Error< + "'--hip-path' must be specified when offloading to " + "SPIR-V%select{| unless %1 is given}0.">; + def err_drv_undetermined_amdgpu_arch : Error< "cannot determine AMDGPU architecture: %0; consider passing it via " "'--march'">; @@ -108,6 +116,10 @@ def warn_drv_unsupported_option_for_target : Warning< "ignoring '%0' option as it is not currently supported for target '%1'">, InGroup; +def warn_drv_spirv_linking_multiple_inputs_unsupported: Warning< + "Linking multiple input files is not supported for SPIR-V yet">, + InGroup; + def err_drv_invalid_thread_model_for_target : Error< "invalid thread model '%0' in '%1' for this target">; def err_drv_invalid_linker_name : Error< @@ -431,11 +443,13 @@ def err_analyzer_checker_option_invalid_input : Error< def err_analyzer_checker_incompatible_analyzer_option : Error< "checker cannot be enabled with analyzer option '%0' == %1">; -def err_drv_invalid_hvx_length : Error< - "-mhvx-length is not supported without a -mhvx/-mhvx= flag">; -def warn_drv_vectorize_needs_hvx : Warning< - "auto-vectorization requires HVX, use -mhvx to enable it">, +def warn_drv_needs_hvx : Warning< + "%0 requires HVX, use -mhvx/-mhvx= to enable it">, InGroup; +def err_drv_needs_hvx : Error< + "%0 requires HVX, use -mhvx/-mhvx= to enable it">; +def err_drv_needs_hvx_version : Error< + "%0 is not supported on HVX %1">; def err_drv_module_header_wrong_kind : Error< "header file '%0' input type '%1' does not match type of prior input " @@ -551,7 +565,7 @@ def warn_drv_moutline_unsupported_opt : Warning< InGroup; def warn_drv_moutline_atomics_unsupported_opt : Warning< - "'%0' does not support '-moutline-atomics'; flag ignored">, + "'%0' does not support '-%1'; flag ignored">, InGroup; def warn_drv_darwin_sdk_invalid_settings : Warning< @@ -608,7 +622,14 @@ def err_cc1_round_trip_ok_then_fail : Error< "generated arguments parse failed in round-trip">; def err_cc1_round_trip_mismatch : Error< "generated arguments do not match in round-trip">; +def err_cc1_unbounded_vscale_min : Error< + "minimum vscale must be an unsigned integer greater than 0">; def err_drv_ssp_missing_offset_argument : Error< "'%0' is used without '-mstack-protector-guard-offset', and there is no default">; + +def err_drv_only_one_offload_target_supported_in : Error< + "Only one offload target is supported in %0.">; +def err_drv_invalid_or_unsupported_offload_target : Error< + "Invalid or unsupported offload target: '%0'.">; } diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 90df3a424406..c0642efaee4e 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -191,6 +191,7 @@ def DeprecatedVolatile : DiagGroup<"deprecated-volatile">; def DeprecatedWritableStr : DiagGroup<"deprecated-writable-strings", [CXX11CompatDeprecatedWritableStr]>; def DeprecatedPragma : DiagGroup<"deprecated-pragma">; +def DeprecatedType : DiagGroup<"deprecated-type">; // FIXME: Why is DeprecatedImplementations not in this group? def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion, DeprecatedArrayCompare, @@ -208,6 +209,7 @@ def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion, DeprecatedPragma, DeprecatedRegister, DeprecatedThisCapture, + DeprecatedType, DeprecatedVolatile, DeprecatedWritableStr]>, DiagCategory<"Deprecations">; @@ -1044,6 +1046,13 @@ def : DiagGroup<"unused-local-typedefs", [UnusedLocalTypedef]>; def NonGCC : DiagGroup<"non-gcc", [SignCompare, Conversion, LiteralRange]>; +def CXX14Attrs : DiagGroup<"c++14-attribute-extensions">; +def CXX17Attrs : DiagGroup<"c++17-attribute-extensions">; +def CXX20Attrs : DiagGroup<"c++20-attribute-extensions">; +def FutureAttrs : DiagGroup<"future-attribute-extensions", [CXX14Attrs, + CXX17Attrs, + CXX20Attrs]>; + // A warning group for warnings about using C++11 features as extensions in // earlier C++ versions. def CXX11 : DiagGroup<"c++11-extensions", [CXX11ExtraSemi, CXX11InlineNamespace, @@ -1051,15 +1060,15 @@ def CXX11 : DiagGroup<"c++11-extensions", [CXX11ExtraSemi, CXX11InlineNamespace, // A warning group for warnings about using C++14 features as extensions in // earlier C++ versions. -def CXX14 : DiagGroup<"c++14-extensions", [CXX14BinaryLiteral]>; +def CXX14 : DiagGroup<"c++14-extensions", [CXX14BinaryLiteral, CXX14Attrs]>; // A warning group for warnings about using C++17 features as extensions in // earlier C++ versions. -def CXX17 : DiagGroup<"c++17-extensions">; +def CXX17 : DiagGroup<"c++17-extensions", [CXX17Attrs]>; // A warning group for warnings about using C++20 features as extensions in // earlier C++ versions. -def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator]>; +def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>; // A warning group for warnings about using C++2b features as extensions in // earlier C++ versions. diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index 92e877074ad3..9dc036c03faa 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1503,6 +1503,15 @@ def warn_pragma_force_cuda_host_device_bad_arg : Warning< def err_pragma_cannot_end_force_cuda_host_device : Error< "force_cuda_host_device end pragma without matching " "force_cuda_host_device begin">; + +def warn_ext_int_deprecated : Warning< + "'_ExtInt' is deprecated; use '_BitInt' instead">, InGroup; +def ext_bit_int : Extension< + "'_BitInt' in %select{C17 and earlier|C++}0 is a Clang extension">, + InGroup>; +def warn_c17_compat_bit_int : Warning< + "'_BitInt' is incompatible with C standards before C2x">, + InGroup, DefaultIgnore; } // end of Parse Issue category. let CategoryName = "Modules Issue" in { diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index fb5bd53f7432..f2089bfda04d 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -838,6 +838,9 @@ def warn_fortify_scanf_overflow : Warning< "%2, but the corresponding specifier may require size %3">, InGroup; +def err_function_start_invalid_type: Error< + "argument must be a function">; + /// main() // static main() is not an error in C, just in C++. def warn_static_main : Warning<"'main' should not be declared static">, @@ -2956,6 +2959,17 @@ def err_attribute_invalid_argument : Error< def err_attribute_wrong_number_arguments : Error< "%0 attribute %plural{0:takes no arguments|1:takes one argument|" ":requires exactly %1 arguments}1">; +def err_attribute_wrong_number_arguments_for : Error < + "%0 attribute references function %1, which %plural{0:takes no arguments|1:takes one argument|" + ":takes exactly %2 arguments}2">; +def err_attribute_bounds_for_function : Error< + "%0 attribute references parameter %1, but the function %2 has only %3 parameters">; +def err_attribute_no_member_function : Error< + "%0 attribute cannot be applied to non-static member functions">; +def err_attribute_parameter_types : Error< + "%0 attribute parameter types do not match: parameter %1 of function %2 has type %3, " + "but parameter %4 of function %5 has type %6">; + def err_attribute_too_many_arguments : Error< "%0 attribute takes no more than %1 argument%s1">; def err_attribute_too_few_arguments : Error< @@ -3013,7 +3027,7 @@ def err_attribute_sizeless_type : Error< "%0 attribute cannot be applied to sizeless type %1">; def err_attribute_argument_n_type : Error< "%0 attribute requires parameter %1 to be %select{int or bool|an integer " - "constant|a string|an identifier|a constant expression}2">; + "constant|a string|an identifier|a constant expression|a builtin function}2">; def err_attribute_argument_type : Error< "%0 attribute requires %select{int or bool|an integer " "constant|a string|an identifier}1">; @@ -8337,8 +8351,8 @@ def err_atomic_exclusive_builtin_pointer_size : Error< " 1,2,4 or 8 byte type (%0 invalid)">; def err_atomic_builtin_ext_int_size : Error< "Atomic memory operand must have a power-of-two size">; -def err_atomic_builtin_ext_int_prohibit : Error< - "argument to atomic builtin of type '_ExtInt' is not supported">; +def err_atomic_builtin_bit_int_prohibit : Error< + "argument to atomic builtin of type '_BitInt' is not supported">; def err_atomic_op_needs_atomic : Error< "address argument to atomic operation must be a pointer to _Atomic " "type (%0 invalid)">; @@ -8374,8 +8388,8 @@ def err_overflow_builtin_must_be_int : Error< def err_overflow_builtin_must_be_ptr_int : Error< "result argument to overflow builtin must be a pointer " "to a non-const integer (%0 invalid)">; -def err_overflow_builtin_ext_int_max_size : Error< - "__builtin_mul_overflow does not support signed _ExtInt operands of more " +def err_overflow_builtin_bit_int_max_size : Error< + "__builtin_mul_overflow does not support 'signed _BitInt' operands of more " "than %0 bits">; def err_atomic_load_store_uses_lib : Error< @@ -8626,11 +8640,11 @@ def warn_unused_volatile : Warning< InGroup>; def ext_cxx14_attr : Extension< - "use of the %0 attribute is a C++14 extension">, InGroup; + "use of the %0 attribute is a C++14 extension">, InGroup; def ext_cxx17_attr : Extension< - "use of the %0 attribute is a C++17 extension">, InGroup; + "use of the %0 attribute is a C++17 extension">, InGroup; def ext_cxx20_attr : Extension< - "use of the %0 attribute is a C++20 extension">, InGroup; + "use of the %0 attribute is a C++20 extension">, InGroup; def warn_unused_comparison : Warning< "%select{equality|inequality|relational|three-way}0 comparison result unused">, @@ -9140,15 +9154,22 @@ def warn_cxx17_compat_defaulted_comparison : Warning< "before C++20">, InGroup, DefaultIgnore; def err_defaulted_comparison_template : Error< "comparison operator template cannot be defaulted">; -def err_defaulted_comparison_out_of_class : Error< - "%sub{select_defaulted_comparison_kind}0 can only be defaulted in a class " - "definition">; +def err_defaulted_comparison_num_args : Error< + "%select{non-member|member}0 %sub{select_defaulted_comparison_kind}1" + " comparison operator must have %select{2|1}0 parameters">; def err_defaulted_comparison_param : Error< "invalid parameter type for defaulted %sub{select_defaulted_comparison_kind}0" "; found %1, expected %2%select{| or %4}3">; +def err_defaulted_comparison_param_unknown : Error< + "invalid parameter type for non-member defaulted" + " %sub{select_defaulted_comparison_kind}0" + "; found %1, expected class or reference to a constant class">; def err_defaulted_comparison_param_mismatch : Error< "parameters for defaulted %sub{select_defaulted_comparison_kind}0 " "must have the same type%diff{ (found $ vs $)|}1,2">; +def err_defaulted_comparison_not_friend : Error< + "%sub{select_defaulted_comparison_kind}0 is not a friend of" + " %select{|incomplete class }1%2">; def err_defaulted_comparison_non_const : Error< "defaulted member %sub{select_defaulted_comparison_kind}0 must be " "const-qualified">; @@ -9811,6 +9832,8 @@ def err_ppc_builtin_only_on_arch : Error< "this builtin is only valid on POWER%0 or later CPUs">; def err_ppc_builtin_requires_vsx : Error< "this builtin requires VSX to be enabled">; +def err_ppc_builtin_requires_htm : Error< + "this builtin requires HTM to be enabled">; def err_ppc_builtin_requires_abi : Error< "this builtin requires ABI -mabi=%0">; def err_ppc_invalid_use_mma_type : Error< @@ -10486,7 +10509,7 @@ def err_omp_atomic_capture_not_compound_statement : Error< def note_omp_atomic_capture: Note< "%select{expected assignment expression|expected compound statement|expected exactly two expression statements|expected in right hand side of the first expression}0">; def err_omp_atomic_several_clauses : Error< - "directive '#pragma omp atomic' cannot contain more than one 'read', 'write', 'update' or 'capture' clause">; + "directive '#pragma omp atomic' cannot contain more than one 'read', 'write', 'update', 'capture', or 'compare' clause">; def err_omp_several_mem_order_clauses : Error< "directive '#pragma omp %0' cannot contain more than one %select{'seq_cst', 'relaxed', |}1'acq_rel', 'acquire' or 'release' clause">; def err_omp_atomic_incompatible_mem_order_clause : Error< @@ -11020,14 +11043,12 @@ def err_implied_coroutine_type_not_found : Error< "a coroutine; include if your version " "of libcxx is less than 14.0">; def warn_deprecated_coroutine_namespace : Warning< - "Please move from std::experimental::%0 to std::%0. " - "Support for std::experimental::%0 will be removed in LLVM 15.">, + "support for std::experimental::%0 will be removed in LLVM 15; " + "use std::%0 instead">, InGroup; -def err_mixed_use_std_and_experimental_namespace_for_coroutine : Error < - "Found mixed use of std namespace and std::experimental namespace for " - "coroutine, which is disallowed. The coroutine components in " - "std::experimental namespace is deprecated. Please use coroutine components " - "under std namespace.">; +def err_mixed_use_std_and_experimental_namespace_for_coroutine : Error< + "mixed use of std and std::experimental namespaces for " + "coroutine components">; def err_implicit_coroutine_std_nothrow_type_not_found : Error< "std::nothrow was not found; include before defining a coroutine which " "uses get_return_object_on_allocation_failure()">; @@ -11050,8 +11071,6 @@ def err_coroutine_type_missing_specialization : Error< "specialization %0">; def err_coroutine_promise_incompatible_return_functions : Error< "the coroutine promise type %0 declares both 'return_value' and 'return_void'">; -def err_coroutine_promise_requires_return_function : Error< - "the coroutine promise type %0 must declare either 'return_value' or 'return_void'">; def note_coroutine_promise_implicit_await_transform_required_here : Note< "call to 'await_transform' implicitly required by 'co_await' here">; def note_coroutine_promise_suspend_implicitly_required : Note< @@ -11355,8 +11374,10 @@ def err_builtin_launder_invalid_arg : Error< def err_builtin_invalid_arg_type: Error < "%ordinal0 argument must be a " "%select{vector, integer or floating point type|matrix|" - "pointer to a valid matrix element type|" - "signed integer or floating point type|vector type}1 (was %2)">; + "pointer to a valid matrix element type|" + "signed integer or floating point type|vector type|" + "floating point type|" + "vector of integers}1 (was %2)">; def err_builtin_matrix_disabled: Error< "matrix types extension is disabled. Pass -fenable-matrix to enable it">; @@ -11423,9 +11444,9 @@ def warn_sycl_kernel_return_type : Warning< "function template with 'sycl_kernel' attribute must have a 'void' return type">, InGroup; -def err_ext_int_bad_size : Error<"%select{signed|unsigned}0 _ExtInt must " +def err_bit_int_bad_size : Error<"%select{signed|unsigned}0 _BitInt must " "have a bit size of at least %select{2|1}0">; -def err_ext_int_max_size : Error<"%select{signed|unsigned}0 _ExtInt of bit " +def err_bit_int_max_size : Error<"%select{signed|unsigned}0 _BitInt of bit " "sizes greater than %1 not supported">; // errors of expect.with.probability diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 3476b05d2e92..de7857347bc2 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -153,6 +153,10 @@ class Module { return Kind == ModuleInterfaceUnit || Kind == PrivateModuleFragment; } + /// Does this Module scope describe a fragment of the global module within + /// some C++ module. + bool isGlobalModule() const { return Kind == GlobalModuleFragment; } + private: /// The submodules of this module, indexed by name. std::vector SubModules; diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h index 66cdba3f912e..82b3c1f3c274 100644 --- a/clang/include/clang/Basic/Specifiers.h +++ b/clang/include/clang/Basic/Specifiers.h @@ -59,7 +59,7 @@ namespace clang { TST_char32, // C++11 char32_t TST_int, TST_int128, - TST_extint, // Extended Int types. + TST_bitint, // Bit-precise integer types. TST_half, // OpenCL half, ARM NEON __fp16 TST_Float16, // C11 extension ISO/IEC TS 18661-3 TST_Accum, // ISO/IEC JTC1 SC22 WG14 N1169 Extension diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h index 3e1e09417c66..437feba85e23 100644 --- a/clang/include/clang/Basic/TargetInfo.h +++ b/clang/include/clang/Basic/TargetInfo.h @@ -582,9 +582,13 @@ class TargetInfo : public virtual TransferrableTargetInfo, return (getPointerWidth(0) >= 64) || getTargetOpts().ForceEnableInt128; } // FIXME - /// Determine whether the _ExtInt type is supported on this target. This + /// Determine whether the _BitInt type is supported on this target. This /// limitation is put into place for ABI reasons. - virtual bool hasExtIntType() const { + /// FIXME: _BitInt is a required type in C23, so there's not much utility in + /// asking whether the target supported it or not; I think this should be + /// removed once backends have been alerted to the type and have had the + /// chance to do implementation work if needed. + virtual bool hasBitIntType() const { return false; } diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 0dd5936aa3e6..e55244e1c3ac 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -300,6 +300,7 @@ KEYWORD(if , KEYALL) KEYWORD(inline , KEYC99|KEYCXX|KEYGNU) KEYWORD(int , KEYALL) KEYWORD(_ExtInt , KEYALL) +KEYWORD(_BitInt , KEYALL) KEYWORD(long , KEYALL) KEYWORD(register , KEYALL) KEYWORD(restrict , KEYC99) diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td index 011394c3ef45..b65e8ab521c0 100644 --- a/clang/include/clang/Basic/TypeNodes.td +++ b/clang/include/clang/Basic/TypeNodes.td @@ -75,6 +75,7 @@ def DependentSizedMatrixType : TypeNode, AlwaysDependent; def FunctionType : TypeNode; def FunctionProtoType : TypeNode; def FunctionNoProtoType : TypeNode; +def UsingType : TypeNode, NeverCanonical; def UnresolvedUsingType : TypeNode, AlwaysDependent; def ParenType : TypeNode, NeverCanonical; def TypedefType : TypeNode, NeverCanonical; @@ -107,5 +108,5 @@ def ObjCInterfaceType : TypeNode, LeafType; def ObjCObjectPointerType : TypeNode; def PipeType : TypeNode; def AtomicType : TypeNode; -def ExtIntType : TypeNode; -def DependentExtIntType : TypeNode, AlwaysDependent; +def BitIntType : TypeNode; +def DependentBitIntType : TypeNode, AlwaysDependent; diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h index 8b1f7091e701..9ae34a2eaf01 100644 --- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -595,6 +595,21 @@ class Driver { /// @} + /// Retrieves a ToolChain for a particular device \p Target triple + /// + /// \param[in] HostTC is the host ToolChain paired with the device + /// + /// \param[in] Action (e.g. OFK_Cuda/OFK_OpenMP/OFK_SYCL) is an Offloading + /// action that is optionally passed to a ToolChain (used by CUDA, to specify + /// if it's used in conjunction with OpenMP) + /// + /// Will cache ToolChains for the life of the driver object, and create them + /// on-demand. + const ToolChain &getOffloadingDeviceToolChain( + const llvm::opt::ArgList &Args, const llvm::Triple &Target, + const ToolChain &HostTC, + const Action::OffloadKind &TargetDeviceOffloadKind) const; + /// Get bitmasks for which option flags to include and exclude based on /// the driver mode. std::pair getIncludeExcludeOptionFlagMasks(bool IsClCompatMode) const; diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h index 8b287638a271..6e3b51f2a799 100644 --- a/clang/include/clang/Driver/Job.h +++ b/clang/include/clang/Driver/Job.h @@ -204,6 +204,10 @@ class Command { /// from the parent process will be used. virtual void setEnvironment(llvm::ArrayRef NewEnvironment); + void replaceArguments(llvm::opt::ArgStringList List) { + Arguments = std::move(List); + } + const char *getExecutable() const { return Executable; } const llvm::opt::ArgStringList &getArguments() const { return Arguments; } diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 4e6dd2050344..dc8bd831f2a2 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -985,6 +985,9 @@ defm hip_fp32_correctly_rounded_divide_sqrt : BoolFOption<"hip-fp32-correctly-ro BothFlags<[], " that single precision floating-point divide and sqrt used in " "the program source are correctly rounded (HIP device compilation only)">>, ShouldParseIf; +def hipspv_pass_plugin_EQ : Joined<["--"], "hipspv-pass-plugin=">, + Group, MetaVarName<"">, + HelpText<"path to a pass plugin for HIP to SPIR-V passes.">; defm gpu_allow_device_init : BoolFOption<"gpu-allow-device-init", LangOpts<"GPUAllowDeviceInit">, DefaultFalse, PosFlag, NegFlag, @@ -1132,6 +1135,13 @@ defm autolink : BoolFOption<"autolink", NegFlag, PosFlag>; +// In the future this option will be supported by other offloading +// languages and accept other values such as CPU/GPU architectures, +// offload kinds and target aliases. +def offload_EQ : CommaJoined<["--"], "offload=">, Flags<[NoXarchOption]>, + HelpText<"Specify comma-separated list of offloading target triples" + " (HIP only)">; + // C++ Coroutines TS defm coroutines_ts : BoolFOption<"coroutines-ts", LangOpts<"Coroutines">, Default, @@ -2392,7 +2402,8 @@ def fomit_frame_pointer : Flag<["-"], "fomit-frame-pointer">, Group; def fopenmp : Flag<["-"], "fopenmp">, Group, Flags<[CC1Option, NoArgumentUnused, FlangOption, FC1Option]>, HelpText<"Parse OpenMP pragmas and generate parallel code.">; def fno_openmp : Flag<["-"], "fno-openmp">, Group, Flags<[NoArgumentUnused]>; -def fopenmp_version_EQ : Joined<["-"], "fopenmp-version=">, Group, Flags<[CC1Option, NoArgumentUnused]>; +def fopenmp_version_EQ : Joined<["-"], "fopenmp-version=">, Group, Flags<[CC1Option, NoArgumentUnused]>, + HelpText<"Set OpenMP version (e.g. 45 for OpenMP 4.5, 50 for OpenMP 5.0). Default value is 50.">; defm openmp_extensions: BoolFOption<"openmp-extensions", LangOpts<"OpenMPExtensions">, DefaultTrue, PosFlag, Group, HelpText<"Enable debugging in the OpenMP offloading device RTL">; def fno_openmp_target_debug : Flag<["-"], "fno-openmp-target-debug">, Group, Flags<[NoArgumentUnused]>; def fopenmp_target_debug_EQ : Joined<["-"], "fopenmp-target-debug=">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; -def fopenmp_assume_teams_oversubscription : Flag<["-"], "fopenmp-assume-teams-oversubscription">, +def fopenmp_assume_teams_oversubscription : Flag<["-"], "fopenmp-assume-teams-oversubscription">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; -def fopenmp_assume_threads_oversubscription : Flag<["-"], "fopenmp-assume-threads-oversubscription">, +def fopenmp_assume_threads_oversubscription : Flag<["-"], "fopenmp-assume-threads-oversubscription">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; -def fno_openmp_assume_teams_oversubscription : Flag<["-"], "fno-openmp-assume-teams-oversubscription">, +def fno_openmp_assume_teams_oversubscription : Flag<["-"], "fno-openmp-assume-teams-oversubscription">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; -def fno_openmp_assume_threads_oversubscription : Flag<["-"], "fno-openmp-assume-threads-oversubscription">, +def fno_openmp_assume_threads_oversubscription : Flag<["-"], "fno-openmp-assume-threads-oversubscription">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; defm openmp_target_new_runtime: BoolFOption<"openmp-target-new-runtime", - LangOpts<"OpenMPTargetNewRuntime">, DefaultFalse, + LangOpts<"OpenMPTargetNewRuntime">, DefaultTrue, PosFlag, NegFlag>; defm openmp_optimistic_collapse : BoolFOption<"openmp-optimistic-collapse", @@ -3338,6 +3349,11 @@ def mno_fix_cortex_a53_835769 : Flag<["-"], "mno-fix-cortex-a53-835769">, def mmark_bti_property : Flag<["-"], "mmark-bti-property">, Group, HelpText<"Add .note.gnu.property with BTI to assembly files (AArch64 only)">; +def mno_bti_at_return_twice : Flag<["-"], "mno-bti-at-return-twice">, + Group, + HelpText<"Do not add a BTI instruction after a setjmp or other" + " return-twice construct (Arm only)">; + foreach i = {1-31} in def ffixed_x#i : Flag<["-"], "ffixed-x"#i>, Group, HelpText<"Reserve the x"#i#" register (AArch64/RISC-V only)">; @@ -3352,8 +3368,7 @@ def msve_vector_bits_EQ : Joined<["-"], "msve-vector-bits=">, Group, + HelpText<"Specify the vscale minimum. Defaults to \"1\". (AArch64 only)">, MarshallingInfoInt>; def mvscale_max_EQ : Joined<["-"], "mvscale-max=">, Group, Flags<[NoXarchOption,CC1Option]>, @@ -3759,6 +3774,8 @@ def nobuiltininc : Flag<["-"], "nobuiltininc">, Flags<[CC1Option, CoreOption]>, MarshallingInfoNegativeFlag>; def nogpuinc : Flag<["-"], "nogpuinc">, HelpText<"Do not add include paths for CUDA/HIP and" " do not include the default CUDA/HIP wrapper headers">; +def nohipwrapperinc : Flag<["-"], "nohipwrapperinc">, + HelpText<"Do not include the default HIP wrapper headers and include paths">; def : Flag<["-"], "nocudainc">, Alias; def nogpulib : Flag<["-"], "nogpulib">, HelpText<"Do not link device library for CUDA/HIP device compilation">; @@ -3785,6 +3802,11 @@ def o : JoinedOrSeparate<["-"], "o">, Flags<[NoXarchOption, RenderAsInput, CC1Option, CC1AsOption, FC1Option, FlangOption]>, HelpText<"Write output to ">, MetaVarName<"">, MarshallingInfoString>; +def object_file_name_EQ : Joined<["-"], "object-file-name=">, Flags<[CC1Option, CC1AsOption, CoreOption]>, + HelpText<"Set the output for debug infos">, MetaVarName<"">, + MarshallingInfoString>; +def object_file_name : Separate<["-"], "object-file-name">, Flags<[CC1Option, CC1AsOption, CoreOption]>, + Alias; def pagezero__size : JoinedOrSeparate<["-"], "pagezero_size">; def pass_exit_codes : Flag<["-", "--"], "pass-exit-codes">, Flags<[Unsupported]>; def pedantic_errors : Flag<["-", "--"], "pedantic-errors">, Group, Flags<[CC1Option]>, @@ -4138,6 +4160,8 @@ def mv67t : Flag<["-"], "mv67t">, Group, Alias, AliasArgs<["hexagonv67t"]>; def mv68 : Flag<["-"], "mv68">, Group, Alias, AliasArgs<["hexagonv68"]>; +def mv69 : Flag<["-"], "mv69">, Group, + Alias, AliasArgs<["hexagonv69"]>; def mhexagon_hvx : Flag<["-"], "mhvx">, Group, HelpText<"Enable Hexagon Vector eXtensions">; def mhexagon_hvx_EQ : Joined<["-"], "mhvx=">, @@ -4149,6 +4173,18 @@ def mno_hexagon_hvx : Flag<["-"], "mno-hvx">, def mhexagon_hvx_length_EQ : Joined<["-"], "mhvx-length=">, Group, HelpText<"Set Hexagon Vector Length">, Values<"64B,128B">; +def mhexagon_hvx_qfloat : Flag<["-"], "mhvx-qfloat">, + Group, + HelpText<"Enable Hexagon HVX QFloat instructions">; +def mno_hexagon_hvx_qfloat : Flag<["-"], "mno-hvx-qfloat">, + Group, + HelpText<"Disable Hexagon HVX QFloat instructions">; +def mhexagon_hvx_ieee_fp : Flag<["-"], "mhvx-ieee-fp">, + Group, + HelpText<"Enable Hexagon HVX IEEE floating-point">; +def mno_hexagon_hvx_ieee_fp : Flag<["-"], "mno-hvx-ieee-fp">, + Group, + HelpText<"Disable Hexagon HVX IEEE floating-point">; def ffixed_r19: Flag<["-"], "ffixed-r19">, HelpText<"Reserve register r19 (Hexagon only)">; def mmemops : Flag<["-"], "mmemops">, Group, diff --git a/clang/include/clang/Driver/Tool.h b/clang/include/clang/Driver/Tool.h index cc0a09fb2747..42cf99a4a970 100644 --- a/clang/include/clang/Driver/Tool.h +++ b/clang/include/clang/Driver/Tool.h @@ -52,6 +52,7 @@ class Tool { const ToolChain &getToolChain() const { return TheToolChain; } virtual bool hasIntegratedAssembler() const { return false; } + virtual bool hasIntegratedBackend() const { return true; } virtual bool canEmitIR() const { return false; } virtual bool hasIntegratedCPP() const = 0; virtual bool isLinkJob() const { return false; } diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h index dad861d586cb..4afc9bf36b5f 100644 --- a/clang/include/clang/Driver/ToolChain.h +++ b/clang/include/clang/Driver/ToolChain.h @@ -348,10 +348,7 @@ class ToolChain { /// is LLD. If it's set, it can be assumed that the linker is LLD built /// at the same revision as clang, and clang can make assumptions about /// LLD's supported flags, error output, etc. - /// If LinkerIsLLDDarwinNew is non-nullptr, it's set if the linker is - /// the new version in lld/MachO. - std::string GetLinkerPath(bool *LinkerIsLLD = nullptr, - bool *LinkerIsLLDDarwinNew = nullptr) const; + std::string GetLinkerPath(bool *LinkerIsLLD = nullptr) const; /// Returns the linker path for emitting a static library. std::string GetStaticLibToolPath() const; @@ -387,6 +384,9 @@ class ToolChain { /// Check if the toolchain should use the integrated assembler. virtual bool useIntegratedAs() const; + /// Check if the toolchain should use the integrated backend. + virtual bool useIntegratedBackend() const { return true; } + /// Check if the toolchain should use AsmParser to parse inlineAsm when /// integrated assembler is not default. virtual bool parseInlineAsmUsingAsmParser() const { return false; } diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index d38bc6e3f0e6..0f97e80d425e 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -2669,6 +2669,7 @@ struct FormatStyle { bool isCpp() const { return Language == LK_Cpp || Language == LK_ObjC; } bool isCSharp() const { return Language == LK_CSharp; } bool isJson() const { return Language == LK_Json; } + bool isJavaScript() const { return Language == LK_JavaScript; } /// Language, this format style is targeted at. /// \version 3.5 diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h index 41f85a1f572d..08c61a5dc560 100644 --- a/clang/include/clang/Lex/ModuleMap.h +++ b/clang/include/clang/Lex/ModuleMap.h @@ -538,8 +538,11 @@ class ModuleMap { /// /// We model the global module fragment as a submodule of the module /// interface unit. Unfortunately, we can't create the module interface - /// unit's Module until later, because we don't know what it will be called. - Module *createGlobalModuleFragmentForModuleUnit(SourceLocation Loc); + /// unit's Module until later, because we don't know what it will be called + /// usually. See C++20 [module.unit]/7.2 for the case we could know its + /// parent. + Module *createGlobalModuleFragmentForModuleUnit(SourceLocation Loc, + Module *Parent = nullptr); /// Create a global module fragment for a C++ module interface unit. Module *createPrivateModuleFragmentForInterfaceUnit(Module *Parent, diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index 92a703b42173..741a484390b2 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -2564,6 +2564,10 @@ class Parser : public CodeCompletionHandler { /// full validation of the syntactic structure of attributes. bool TrySkipAttributes(); + /// Diagnoses use of _ExtInt as being deprecated, and diagnoses use of + /// _BitInt as an extension when appropriate. + void DiagnoseBitIntUse(const Token &Tok); + public: TypeResult ParseTypeName(SourceRange *Range = nullptr, diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h index ed5be2da3acd..2704a9c1fc78 100644 --- a/clang/include/clang/Sema/DeclSpec.h +++ b/clang/include/clang/Sema/DeclSpec.h @@ -266,7 +266,7 @@ class DeclSpec { static const TST TST_char32 = clang::TST_char32; static const TST TST_int = clang::TST_int; static const TST TST_int128 = clang::TST_int128; - static const TST TST_extint = clang::TST_extint; + static const TST TST_bitint = clang::TST_bitint; static const TST TST_half = clang::TST_half; static const TST TST_BFloat16 = clang::TST_BFloat16; static const TST TST_float = clang::TST_float; @@ -404,7 +404,7 @@ class DeclSpec { T == TST_underlyingType || T == TST_atomic); } static bool isExprRep(TST T) { - return (T == TST_typeofExpr || T == TST_decltype || T == TST_extint); + return (T == TST_typeofExpr || T == TST_decltype || T == TST_bitint); } static bool isTemplateIdRep(TST T) { return (T == TST_auto || T == TST_decltype_auto); @@ -703,7 +703,7 @@ class DeclSpec { bool SetTypePipe(bool isPipe, SourceLocation Loc, const char *&PrevSpec, unsigned &DiagID, const PrintingPolicy &Policy); - bool SetExtIntType(SourceLocation KWLoc, Expr *BitWidth, + bool SetBitIntType(SourceLocation KWLoc, Expr *BitWidth, const char *&PrevSpec, unsigned &DiagID, const PrintingPolicy &Policy); bool SetTypeSpecSat(SourceLocation Loc, const char *&PrevSpec, diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h index ff2303c84bd2..6403179cb327 100644 --- a/clang/include/clang/Sema/ParsedAttr.h +++ b/clang/include/clang/Sema/ParsedAttr.h @@ -1097,6 +1097,7 @@ enum AttributeArgumentNType { AANT_ArgumentString, AANT_ArgumentIdentifier, AANT_ArgumentConstantExpr, + AANT_ArgumentBuiltinFunction, }; /// These constants match the enumerated choices of diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 1a82a9498d1d..79834554a50d 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -1324,12 +1324,15 @@ class Sema final { bool isImmediateFunctionContext() const { return Context == ExpressionEvaluationContext::ImmediateFunctionContext || - InImmediateFunctionContext; + (Context == ExpressionEvaluationContext::DiscardedStatement && + InImmediateFunctionContext); } bool isDiscardedStatementContext() const { return Context == ExpressionEvaluationContext::DiscardedStatement || - InDiscardedStatement; + (Context == + ExpressionEvaluationContext::ImmediateFunctionContext && + InDiscardedStatement); } }; @@ -2043,7 +2046,7 @@ class Sema final { SourceLocation Loc); QualType BuildWritePipeType(QualType T, SourceLocation Loc); - QualType BuildExtIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc); + QualType BuildBitIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc); TypeSourceInfo *GetTypeForDeclarator(Declarator &D, Scope *S); TypeSourceInfo *GetTypeForDeclaratorCast(Declarator &D, QualType FromTy); @@ -2219,6 +2222,17 @@ class Sema final { return ModuleScopes.empty() ? nullptr : ModuleScopes.back().Module; } + /// Helper function to judge if we are in module purview. + /// Return false if we are not in a module. + bool isCurrentModulePurview() const { + return getCurrentModule() ? getCurrentModule()->isModulePurview() : false; + } + + /// Enter the scope of the global module. + Module *PushGlobalModuleFragment(SourceLocation BeginLoc, bool IsImplicit); + /// Leave the scope of the global module. + void PopGlobalModuleFragment(); + VisibleModuleSet VisibleModules; public: @@ -11176,6 +11190,9 @@ class Sema final { /// Called on well-formed 'capture' clause. OMPClause *ActOnOpenMPCaptureClause(SourceLocation StartLoc, SourceLocation EndLoc); + /// Called on well-formed 'compare' clause. + OMPClause *ActOnOpenMPCompareClause(SourceLocation StartLoc, + SourceLocation EndLoc); /// Called on well-formed 'seq_cst' clause. OMPClause *ActOnOpenMPSeqCstClause(SourceLocation StartLoc, SourceLocation EndLoc); @@ -12769,8 +12786,8 @@ class Sema final { bool CheckPPCMMAType(QualType Type, SourceLocation TypeLoc); bool SemaBuiltinElementwiseMath(CallExpr *TheCall); - bool SemaBuiltinElementwiseMathOneArg(CallExpr *TheCall); - bool SemaBuiltinReduceMath(CallExpr *TheCall); + bool PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall); + bool PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall); // Matrix builtin handling. ExprResult SemaBuiltinMatrixTranspose(CallExpr *TheCall, diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def index e92e05810648..85571ea290f7 100644 --- a/clang/include/clang/Serialization/TypeBitCodes.def +++ b/clang/include/clang/Serialization/TypeBitCodes.def @@ -58,9 +58,10 @@ TYPE_BIT_CODE(DependentSizedExtVector, DEPENDENT_SIZED_EXT_VECTOR, 46) TYPE_BIT_CODE(DependentAddressSpace, DEPENDENT_ADDRESS_SPACE, 47) TYPE_BIT_CODE(DependentVector, DEPENDENT_SIZED_VECTOR, 48) TYPE_BIT_CODE(MacroQualified, MACRO_QUALIFIED, 49) -TYPE_BIT_CODE(ExtInt, EXT_INT, 50) -TYPE_BIT_CODE(DependentExtInt, DEPENDENT_EXT_INT, 51) +TYPE_BIT_CODE(BitInt, BIT_INT, 50) +TYPE_BIT_CODE(DependentBitInt, DEPENDENT_BIT_INT, 51) TYPE_BIT_CODE(ConstantMatrix, CONSTANT_MATRIX, 52) TYPE_BIT_CODE(DependentSizedMatrix, DEPENDENT_SIZE_MATRIX, 53) +TYPE_BIT_CODE(Using, USING, 54) #undef TYPE_BIT_CODE diff --git a/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h b/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h index 31a4ed50a723..48cd3395e936 100644 --- a/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h +++ b/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h @@ -32,7 +32,7 @@ class SValExplainer : public FullSValVisitor { std::string Str; llvm::raw_string_ostream OS(Str); S->printPretty(OS, nullptr, PrintingPolicy(ACtx.getLangOpts())); - return OS.str(); + return Str; } bool isThisObject(const SymbolicRegion *R) { @@ -69,7 +69,7 @@ class SValExplainer : public FullSValVisitor { std::string Str; llvm::raw_string_ostream OS(Str); OS << "concrete memory address '" << I << "'"; - return OS.str(); + return Str; } std::string VisitNonLocSymbolVal(nonloc::SymbolVal V) { @@ -82,7 +82,7 @@ class SValExplainer : public FullSValVisitor { llvm::raw_string_ostream OS(Str); OS << (I.isSigned() ? "signed " : "unsigned ") << I.getBitWidth() << "-bit integer '" << I << "'"; - return OS.str(); + return Str; } std::string VisitNonLocLazyCompoundVal(nonloc::LazyCompoundVal V) { @@ -123,7 +123,7 @@ class SValExplainer : public FullSValVisitor { OS << "(" << Visit(S->getLHS()) << ") " << std::string(BinaryOperator::getOpcodeStr(S->getOpcode())) << " " << S->getRHS(); - return OS.str(); + return Str; } // TODO: IntSymExpr doesn't appear in practice. @@ -177,7 +177,7 @@ class SValExplainer : public FullSValVisitor { else OS << "'" << Visit(R->getIndex()) << "'"; OS << " of " + Visit(R->getSuperRegion()); - return OS.str(); + return Str; } std::string VisitNonParamVarRegion(const NonParamVarRegion *R) { diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def index aab8e1284bf6..c0cade46d614 100644 --- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def +++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def @@ -336,6 +336,18 @@ ANALYZER_OPTION( "might be modeled by the analyzer to never return NULL.", false) +ANALYZER_OPTION( + bool, ShouldIgnoreBisonGeneratedFiles, "ignore-bison-generated-files", + "If enabled, any files containing the \"/* A Bison parser, made by\" " + "won't be analyzed.", + true) + +ANALYZER_OPTION( + bool, ShouldIgnoreFlexGeneratedFiles, "ignore-flex-generated-files", + "If enabled, any files containing the \"/* A lexical scanner generated by " + "flex\" won't be analyzed.", + true) + //===----------------------------------------------------------------------===// // Unsigned analyzer options. //===----------------------------------------------------------------------===// diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h index 396c9a4de440..cd972b7837d0 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h @@ -134,6 +134,9 @@ template class CallDescriptionMap { std::initializer_list> &&List) : LinearMap(List) {} + template + CallDescriptionMap(InputIt First, InputIt Last) : LinearMap(First, Last) {} + ~CallDescriptionMap() = default; // These maps are usually stored once per checker, so let's make sure @@ -141,6 +144,9 @@ template class CallDescriptionMap { CallDescriptionMap(const CallDescriptionMap &) = delete; CallDescriptionMap &operator=(const CallDescription &) = delete; + CallDescriptionMap(CallDescriptionMap &&) = default; + CallDescriptionMap &operator=(CallDescriptionMap &&) = default; + LLVM_NODISCARD const T *lookup(const CallEvent &Call) const { // Slow path: linear lookup. // TODO: Implement some sort of fast path. diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h index a80484610131..3a0bec9d04e5 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h @@ -140,6 +140,30 @@ class RangeSet { /// Complexity: O(N) /// where N = size(Original) RangeSet add(RangeSet Original, const llvm::APSInt &Point); + /// Create a new set which is a union of two given ranges. + /// Possible intersections are not checked here. + /// + /// Complexity: O(N + M) + /// where N = size(LHS), M = size(RHS) + RangeSet unite(RangeSet LHS, RangeSet RHS); + /// Create a new set by uniting given range set with the given range. + /// All intersections and adjacent ranges are handled here. + /// + /// Complexity: O(N) + /// where N = size(Original) + RangeSet unite(RangeSet Original, Range Element); + /// Create a new set by uniting given range set with the given point. + /// All intersections and adjacent ranges are handled here. + /// + /// Complexity: O(N) + /// where N = size(Original) + RangeSet unite(RangeSet Original, llvm::APSInt Point); + /// Create a new set by uniting given range set with the given range + /// between points. All intersections and adjacent ranges are handled here. + /// + /// Complexity: O(N) + /// where N = size(Original) + RangeSet unite(RangeSet Original, llvm::APSInt From, llvm::APSInt To); RangeSet getEmptySet() { return &EmptySet; } @@ -224,6 +248,9 @@ class RangeSet { ContainerType *construct(ContainerType &&From); RangeSet intersect(const ContainerType &LHS, const ContainerType &RHS); + /// NOTE: This function relies on the fact that all values in the + /// containers are persistent (created via BasicValueFactory::getValue). + ContainerType unite(const ContainerType &LHS, const ContainerType &RHS); // Many operations include producing new APSInt values and that's why // we need this factory. diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h index d2461705d128..bdf9662d5d99 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h @@ -172,9 +172,9 @@ class StoreManager { /// dynamic_cast. /// - We don't know (base is a symbolic region and we don't have /// enough info to determine if the cast will succeed at run time). - /// The function returns an SVal representing the derived class; it's - /// valid only if Failed flag is set to false. - SVal attemptDownCast(SVal Base, QualType DerivedPtrType, bool &Failed); + /// The function returns an optional with SVal representing the derived class + /// in case of a successful cast and `None` otherwise. + Optional evalBaseToDerived(SVal Base, QualType DerivedPtrType); const ElementRegion *GetElementZeroRegion(const SubRegion *R, QualType T); diff --git a/clang/include/clang/Testing/TestClangConfig.h b/clang/include/clang/Testing/TestClangConfig.h index 5d6be4f65d0a..92d5cc3cff99 100644 --- a/clang/include/clang/Testing/TestClangConfig.h +++ b/clang/include/clang/Testing/TestClangConfig.h @@ -73,7 +73,7 @@ struct TestClangConfig { std::string Result; llvm::raw_string_ostream OS(Result); OS << "{ Language=" << Language << ", Target=" << Target << " }"; - return OS.str(); + return Result; } friend std::ostream &operator<<(std::ostream &OS, diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index af02fa2e7e87..7d0b8f2138f9 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -26,57 +26,83 @@ namespace dependencies { /// the dependency scanning filesystem. /// /// It represents one of the following: -/// - an opened source file with minimized contents and a stat value. -/// - an opened source file with original contents and a stat value. -/// - a directory entry with its stat value. -/// - an error value to represent a file system error. -/// - a placeholder with an invalid stat indicating a not yet initialized entry. +/// - opened file with original contents and a stat value, +/// - opened file with original contents, minimized contents and a stat value, +/// - directory entry with its stat value, +/// - filesystem error, +/// - uninitialized entry with unknown status. class CachedFileSystemEntry { public: - /// Default constructor creates an entry with an invalid stat. - CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {} + /// Creates an uninitialized entry. + CachedFileSystemEntry() + : MaybeStat(llvm::vfs::Status()), MinimizedContentsAccess(nullptr) {} - CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {} + /// Initialize the cached file system entry. + void init(llvm::ErrorOr &&MaybeStatus, StringRef Filename, + llvm::vfs::FileSystem &FS); - /// Create an entry that represents an opened source file with minimized or - /// original contents. + /// Initialize the entry as file with minimized or original contents. /// /// The filesystem opens the file even for `stat` calls open to avoid the /// issues with stat + open of minimized files that might lead to a - /// mismatching size of the file. If file is not minimized, the full file is - /// read and copied into memory to ensure that it's not memory mapped to avoid - /// running out of file descriptors. - static CachedFileSystemEntry createFileEntry(StringRef Filename, - llvm::vfs::FileSystem &FS, - bool Minimize = true); + /// mismatching size of the file. + llvm::ErrorOr initFile(StringRef Filename, + llvm::vfs::FileSystem &FS); - /// Create an entry that represents a directory on the filesystem. - static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat); + /// Minimize contents of the file. + void minimizeFile(); - /// \returns True if the entry is valid. - bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); } + /// \returns True if the entry is initialized. + bool isInitialized() const { + return !MaybeStat || MaybeStat->isStatusKnown(); + } /// \returns True if the current entry points to a directory. bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); } - /// \returns The error or the file's contents. - llvm::ErrorOr getContents() const { + /// \returns The error or the file's original contents. + llvm::ErrorOr getOriginalContents() const { if (!MaybeStat) return MaybeStat.getError(); assert(!MaybeStat->isDirectory() && "not a file"); - assert(isValid() && "not initialized"); - return Contents.str(); + assert(isInitialized() && "not initialized"); + assert(OriginalContents && "not read"); + return OriginalContents->getBuffer(); + } + + /// \returns The error or the file's minimized contents. + llvm::ErrorOr getMinimizedContents() const { + if (!MaybeStat) + return MaybeStat.getError(); + assert(!MaybeStat->isDirectory() && "not a file"); + assert(isInitialized() && "not initialized"); + llvm::MemoryBuffer *Buffer = MinimizedContentsAccess.load(); + assert(Buffer && "not minimized"); + return Buffer->getBuffer(); + } + + /// \returns True if this entry represents a file that can be read. + bool isReadable() const { return MaybeStat && !MaybeStat->isDirectory(); } + + /// \returns True if this cached entry needs to be updated. + bool needsUpdate(bool ShouldBeMinimized) const { + return isReadable() && needsMinimization(ShouldBeMinimized); + } + + /// \returns True if the contents of this entry need to be minimized. + bool needsMinimization(bool ShouldBeMinimized) const { + return ShouldBeMinimized && !MinimizedContentsAccess.load(); } /// \returns The error or the status of the entry. llvm::ErrorOr getStatus() const { - assert(isValid() && "not initialized"); + assert(isInitialized() && "not initialized"); return MaybeStat; } /// \returns the name of the file. StringRef getName() const { - assert(isValid() && "not initialized"); + assert(isInitialized() && "not initialized"); return MaybeStat->getName(); } @@ -86,19 +112,16 @@ class CachedFileSystemEntry { return PPSkippedRangeMapping; } - CachedFileSystemEntry(CachedFileSystemEntry &&) = default; - CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default; - - CachedFileSystemEntry(const CachedFileSystemEntry &) = delete; - CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete; - private: llvm::ErrorOr MaybeStat; - // Store the contents in a small string to allow a - // move from the small string for the minimized contents. - // Note: small size of 1 allows us to store an empty string with an implicit - // null terminator without any allocations. - llvm::SmallString<1> Contents; + std::unique_ptr OriginalContents; + + /// Owning storage for the minimized file contents. + std::unique_ptr MinimizedContentsStorage; + /// Atomic view of the minimized file contents. + /// This prevents data races when multiple threads call `needsMinimization`. + std::atomic MinimizedContentsAccess; + PreprocessorSkippedRangeMapping PPSkippedRangeMapping; }; @@ -115,61 +138,70 @@ class DependencyScanningFilesystemSharedCache { CachedFileSystemEntry Value; }; + DependencyScanningFilesystemSharedCache(); + /// Returns a cache entry for the corresponding key. /// /// A new cache entry is created if the key is not in the cache. This is a /// thread safe call. - SharedFileSystemEntry &get(StringRef Key, bool Minimized); + SharedFileSystemEntry &get(StringRef Key); private: - class SingleCache { - public: - SingleCache(); - - SharedFileSystemEntry &get(StringRef Key); - - private: - struct CacheShard { - std::mutex CacheLock; - llvm::StringMap Cache; - }; - std::unique_ptr CacheShards; - unsigned NumShards; + struct CacheShard { + std::mutex CacheLock; + llvm::StringMap Cache; }; - - SingleCache CacheMinimized; - SingleCache CacheOriginal; + std::unique_ptr CacheShards; + unsigned NumShards; }; /// This class is a local cache, that caches the 'stat' and 'open' calls to the /// underlying real file system. It distinguishes between minimized and original /// files. class DependencyScanningFilesystemLocalCache { -private: - using SingleCache = - llvm::StringMap; - - SingleCache CacheMinimized; - SingleCache CacheOriginal; - - SingleCache &selectCache(bool Minimized) { - return Minimized ? CacheMinimized : CacheOriginal; - } + llvm::StringMap Cache; public: - void setCachedEntry(StringRef Filename, bool Minimized, - const CachedFileSystemEntry *Entry) { - SingleCache &Cache = selectCache(Minimized); - bool IsInserted = Cache.try_emplace(Filename, Entry).second; - (void)IsInserted; - assert(IsInserted && "local cache is updated more than once"); + const CachedFileSystemEntry *getCachedEntry(StringRef Filename) { + return Cache[Filename]; + } +}; + +/// Reference to a CachedFileSystemEntry. +/// If the underlying entry is an opened file, this wrapper returns the correct +/// contents (original or minimized) and ensures consistency with file size +/// reported by status. +class EntryRef { + /// For entry that is an opened file, this bit signifies whether its contents + /// are minimized. + bool Minimized; + + /// The underlying cached entry. + const CachedFileSystemEntry &Entry; + +public: + EntryRef(bool Minimized, const CachedFileSystemEntry &Entry) + : Minimized(Minimized), Entry(Entry) {} + + llvm::ErrorOr getStatus() const { + auto MaybeStat = Entry.getStatus(); + if (!MaybeStat || MaybeStat->isDirectory()) + return MaybeStat; + return llvm::vfs::Status::copyWithNewSize(*MaybeStat, + getContents()->size()); } - const CachedFileSystemEntry *getCachedEntry(StringRef Filename, - bool Minimized) { - SingleCache &Cache = selectCache(Minimized); - auto It = Cache.find(Filename); - return It == Cache.end() ? nullptr : It->getValue(); + bool isDirectory() const { return Entry.isDirectory(); } + + StringRef getName() const { return Entry.getName(); } + + llvm::ErrorOr getContents() const { + return Minimized ? Entry.getMinimizedContents() + : Entry.getOriginalContents(); + } + + const PreprocessorSkippedRangeMapping *getPPSkippedRangeMapping() const { + return Minimized ? &Entry.getPPSkippedRangeMapping() : nullptr; } }; @@ -204,19 +236,13 @@ class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem { /// Check whether the file should be minimized. bool shouldMinimize(StringRef Filename); - llvm::ErrorOr - getOrCreateFileSystemEntry(const StringRef Filename); - - /// Create a cached file system entry based on the initial status result. - CachedFileSystemEntry - createFileSystemEntry(llvm::ErrorOr &&MaybeStatus, - StringRef Filename, bool ShouldMinimize); + llvm::ErrorOr getOrCreateFileSystemEntry(StringRef Filename); /// The global cache shared between worker threads. DependencyScanningFilesystemSharedCache &SharedCache; /// The local cache is used by the worker thread to cache file system queries /// locally instead of querying the global cache every time. - DependencyScanningFilesystemLocalCache Cache; + DependencyScanningFilesystemLocalCache LocalCache; /// The optional mapping structure which records information about the /// excluded conditional directive skip mappings that are used by the /// currently active preprocessor. diff --git a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h index d54f8a402e2d..d5638642d017 100644 --- a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h +++ b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h @@ -106,7 +106,7 @@ struct IncludeStyle { /// Priority: 2 /// SortPriority: 2 /// CaseSensitive: true - /// - Regex: '^(<|"(gtest|gmock|isl|json)/)' + /// - Regex: '^((<|")(gtest|gmock|isl|json)/)' /// Priority: 3 /// - Regex: '<[[:alnum:].]+>' /// Priority: 4 diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap index e850a1cd4b9a..2b73cd5451b7 100644 --- a/clang/include/clang/module.modulemap +++ b/clang/include/clang/module.modulemap @@ -34,6 +34,8 @@ module Clang_Basic { textual header "Basic/AArch64SVEACLETypes.def" textual header "Basic/BuiltinsAArch64.def" textual header "Basic/BuiltinsAMDGPU.def" + textual header "Basic/BuiltinsAArch64NeonSVEBridge.def" + textual header "Basic/BuiltinsAArch64NeonSVEBridge_cg.def" textual header "Basic/BuiltinsARM.def" textual header "Basic/BuiltinsBPF.def" textual header "Basic/Builtins.def" diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp index 4851c434d765..68ee7c59270e 100644 --- a/clang/lib/ARCMigrate/ARCMT.cpp +++ b/clang/lib/ARCMigrate/ARCMT.cpp @@ -162,9 +162,7 @@ static bool HasARCRuntime(CompilerInvocation &origCI) { return triple.getOSMajorVersion() >= 11; if (triple.getOS() == llvm::Triple::MacOSX) { - unsigned Major, Minor, Micro; - triple.getOSVersion(Major, Minor, Micro); - return Major > 10 || (Major == 10 && Minor >= 7); + return triple.getOSVersion() >= VersionTuple(10, 7); } return false; diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index 2d85d72e5b8a..008b703d4c1a 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -2286,8 +2286,8 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const { Align = toBits(Layout.getAlignment()); break; } - case Type::ExtInt: { - const auto *EIT = cast(T); + case Type::BitInt: { + const auto *EIT = cast(T); Align = std::min(static_cast(std::max( getCharWidth(), llvm::PowerOf2Ceil(EIT->getNumBits()))), @@ -2349,6 +2349,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const { case Type::ObjCTypeParam: return getTypeInfo(cast(T)->desugar().getTypePtr()); + case Type::Using: + return getTypeInfo(cast(T)->desugar().getTypePtr()); + case Type::Typedef: { const TypedefNameDecl *Typedef = cast(T)->getDecl(); TypeInfo Info = getTypeInfo(Typedef->getUnderlyingType().getTypePtr()); @@ -3569,8 +3572,8 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const { case Type::Auto: case Type::DeducedTemplateSpecialization: case Type::PackExpansion: - case Type::ExtInt: - case Type::DependentExtInt: + case Type::BitInt: + case Type::DependentBitInt: llvm_unreachable("type should never be variably-modified"); // These types can be variably-modified but should never need to @@ -4482,34 +4485,34 @@ QualType ASTContext::getWritePipeType(QualType T) const { return getPipeType(T, false); } -QualType ASTContext::getExtIntType(bool IsUnsigned, unsigned NumBits) const { +QualType ASTContext::getBitIntType(bool IsUnsigned, unsigned NumBits) const { llvm::FoldingSetNodeID ID; - ExtIntType::Profile(ID, IsUnsigned, NumBits); + BitIntType::Profile(ID, IsUnsigned, NumBits); void *InsertPos = nullptr; - if (ExtIntType *EIT = ExtIntTypes.FindNodeOrInsertPos(ID, InsertPos)) + if (BitIntType *EIT = BitIntTypes.FindNodeOrInsertPos(ID, InsertPos)) return QualType(EIT, 0); - auto *New = new (*this, TypeAlignment) ExtIntType(IsUnsigned, NumBits); - ExtIntTypes.InsertNode(New, InsertPos); + auto *New = new (*this, TypeAlignment) BitIntType(IsUnsigned, NumBits); + BitIntTypes.InsertNode(New, InsertPos); Types.push_back(New); return QualType(New, 0); } -QualType ASTContext::getDependentExtIntType(bool IsUnsigned, +QualType ASTContext::getDependentBitIntType(bool IsUnsigned, Expr *NumBitsExpr) const { assert(NumBitsExpr->isInstantiationDependent() && "Only good for dependent"); llvm::FoldingSetNodeID ID; - DependentExtIntType::Profile(ID, *this, IsUnsigned, NumBitsExpr); + DependentBitIntType::Profile(ID, *this, IsUnsigned, NumBitsExpr); void *InsertPos = nullptr; - if (DependentExtIntType *Existing = - DependentExtIntTypes.FindNodeOrInsertPos(ID, InsertPos)) + if (DependentBitIntType *Existing = + DependentBitIntTypes.FindNodeOrInsertPos(ID, InsertPos)) return QualType(Existing, 0); auto *New = new (*this, TypeAlignment) - DependentExtIntType(*this, IsUnsigned, NumBitsExpr); - DependentExtIntTypes.InsertNode(New, InsertPos); + DependentBitIntType(*this, IsUnsigned, NumBitsExpr); + DependentBitIntTypes.InsertNode(New, InsertPos); Types.push_back(New); return QualType(New, 0); @@ -4568,9 +4571,7 @@ QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) const { assert(Enum->isFirstDecl() && "enum has previous declaration"); return getEnumType(Enum); } else if (const auto *Using = dyn_cast(Decl)) { - Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Using); - Decl->TypeForDecl = newType; - Types.push_back(newType); + return getUnresolvedUsingType(Using); } else llvm_unreachable("TypeDecl without a type?"); @@ -4593,6 +4594,27 @@ QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl, return QualType(newType, 0); } +QualType ASTContext::getUsingType(const UsingShadowDecl *Found, + QualType Underlying) const { + llvm::FoldingSetNodeID ID; + UsingType::Profile(ID, Found); + + void *InsertPos = nullptr; + UsingType *T = UsingTypes.FindNodeOrInsertPos(ID, InsertPos); + if (T) + return QualType(T, 0); + + assert(!Underlying.hasLocalQualifiers()); + assert(Underlying == getTypeDeclType(cast(Found->getTargetDecl()))); + QualType Canon = Underlying.getCanonicalType(); + + UsingType *NewType = + new (*this, TypeAlignment) UsingType(Found, Underlying, Canon); + Types.push_back(NewType); + UsingTypes.InsertNode(NewType, InsertPos); + return QualType(NewType, 0); +} + QualType ASTContext::getRecordType(const RecordDecl *Decl) const { if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0); @@ -4619,6 +4641,22 @@ QualType ASTContext::getEnumType(const EnumDecl *Decl) const { return QualType(newType, 0); } +QualType ASTContext::getUnresolvedUsingType( + const UnresolvedUsingTypenameDecl *Decl) const { + if (Decl->TypeForDecl) + return QualType(Decl->TypeForDecl, 0); + + if (const UnresolvedUsingTypenameDecl *CanonicalDecl = + Decl->getCanonicalDecl()) + if (CanonicalDecl->TypeForDecl) + return QualType(Decl->TypeForDecl = CanonicalDecl->TypeForDecl, 0); + + Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Decl); + Decl->TypeForDecl = newType; + Types.push_back(newType); + return QualType(newType, 0); +} + QualType ASTContext::getAttributedType(attr::Kind attrKind, QualType modifiedType, QualType equivalentType) { @@ -6444,7 +6482,7 @@ unsigned ASTContext::getIntegerRank(const Type *T) const { // Results in this 'losing' to any type of the same size, but winning if // larger. - if (const auto *EIT = dyn_cast(T)) + if (const auto *EIT = dyn_cast(T)) return 0 + (EIT->getNumBits() << 3); switch (cast(T)->getKind()) { @@ -7885,7 +7923,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S, return; case Type::Pipe: - case Type::ExtInt: + case Type::BitInt: #define ABSTRACT_TYPE(KIND, BASE) #define TYPE(KIND, BASE) #define DEPENDENT_TYPE(KIND, BASE) \ @@ -9234,7 +9272,7 @@ void getIntersectionOfProtocols(ASTContext &Context, // Remove any implied protocols from the list of inherited protocols. if (!ImpliedProtocols.empty()) { llvm::erase_if(IntersectionSet, [&](ObjCProtocolDecl *proto) -> bool { - return ImpliedProtocols.count(proto) > 0; + return ImpliedProtocols.contains(proto); }); } @@ -10101,12 +10139,12 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, assert(LHS != RHS && "Equivalent pipe types should have already been handled!"); return {}; - case Type::ExtInt: { - // Merge two ext-int types, while trying to preserve typedef info. - bool LHSUnsigned = LHS->castAs()->isUnsigned(); - bool RHSUnsigned = RHS->castAs()->isUnsigned(); - unsigned LHSBits = LHS->castAs()->getNumBits(); - unsigned RHSBits = RHS->castAs()->getNumBits(); + case Type::BitInt: { + // Merge two bit-precise int types, while trying to preserve typedef info. + bool LHSUnsigned = LHS->castAs()->isUnsigned(); + bool RHSUnsigned = RHS->castAs()->isUnsigned(); + unsigned LHSBits = LHS->castAs()->getNumBits(); + unsigned RHSBits = RHS->castAs()->getNumBits(); // Like unsigned/int, shouldn't have a type if they don't match. if (LHSUnsigned != RHSUnsigned) @@ -10256,7 +10294,7 @@ unsigned ASTContext::getIntWidth(QualType T) const { T = ET->getDecl()->getIntegerType(); if (T->isBooleanType()) return 1; - if(const auto *EIT = T->getAs()) + if (const auto *EIT = T->getAs()) return EIT->getNumBits(); // For builtin types, just use the standard type sizing method return (unsigned)getTypeSize(T); @@ -10271,9 +10309,9 @@ QualType ASTContext::getCorrespondingUnsignedType(QualType T) const { return getVectorType(getCorrespondingUnsignedType(VTy->getElementType()), VTy->getNumElements(), VTy->getVectorKind()); - // For _ExtInt, return an unsigned _ExtInt with same width. - if (const auto *EITy = T->getAs()) - return getExtIntType(/*IsUnsigned=*/true, EITy->getNumBits()); + // For _BitInt, return an unsigned _BitInt with same width. + if (const auto *EITy = T->getAs()) + return getBitIntType(/*IsUnsigned=*/true, EITy->getNumBits()); // For enums, get the underlying integer type of the enum, and let the general // integer type signchanging code handle it. @@ -10339,9 +10377,9 @@ QualType ASTContext::getCorrespondingSignedType(QualType T) const { return getVectorType(getCorrespondingSignedType(VTy->getElementType()), VTy->getNumElements(), VTy->getVectorKind()); - // For _ExtInt, return a signed _ExtInt with same width. - if (const auto *EITy = T->getAs()) - return getExtIntType(/*IsUnsigned=*/false, EITy->getNumBits()); + // For _BitInt, return a signed _BitInt with same width. + if (const auto *EITy = T->getAs()) + return getBitIntType(/*IsUnsigned=*/false, EITy->getNumBits()); // For enums, get the underlying integer type of the enum, and let the general // integer type signchanging code handle it. diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp index 7e435e8b35b8..724ede272fbf 100644 --- a/clang/lib/AST/ASTDiagnostic.cpp +++ b/clang/lib/AST/ASTDiagnostic.cpp @@ -26,7 +26,8 @@ using namespace clang; // Returns a desugared version of the QualType, and marks ShouldAKA as true // whenever we remove significant sugar from the type. -static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) { +QualType clang::desugarForDiagnostic(ASTContext &Context, QualType QT, + bool &ShouldAKA) { QualifierCollector QC; while (true) { @@ -37,6 +38,11 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) { QT = ET->desugar(); continue; } + // ... or a using type ... + if (const UsingType *UT = dyn_cast(Ty)) { + QT = UT->desugar(); + continue; + } // ... or a paren type ... if (const ParenType *PT = dyn_cast(Ty)) { QT = PT->desugar(); @@ -76,7 +82,7 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) { if (const FunctionType *FT = dyn_cast(Ty)) { bool DesugarReturn = false; QualType SugarRT = FT->getReturnType(); - QualType RT = Desugar(Context, SugarRT, DesugarReturn); + QualType RT = desugarForDiagnostic(Context, SugarRT, DesugarReturn); if (auto nullability = AttributedType::stripOuterNullability(SugarRT)) { RT = Context.getAttributedType( AttributedType::getNullabilityAttrKind(*nullability), RT, RT); @@ -87,7 +93,7 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) { const FunctionProtoType *FPT = dyn_cast(FT); if (FPT) { for (QualType SugarPT : FPT->param_types()) { - QualType PT = Desugar(Context, SugarPT, DesugarArgument); + QualType PT = desugarForDiagnostic(Context, SugarPT, DesugarArgument); if (auto nullability = AttributedType::stripOuterNullability(SugarPT)) { PT = Context.getAttributedType( @@ -115,7 +121,8 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) { for (unsigned I = 0, N = TST->getNumArgs(); I != N; ++I) { const TemplateArgument &Arg = TST->getArg(I); if (Arg.getKind() == TemplateArgument::Type) - Args.push_back(Desugar(Context, Arg.getAsType(), DesugarArgument)); + Args.push_back(desugarForDiagnostic(Context, Arg.getAsType(), + DesugarArgument)); else Args.push_back(Arg); } @@ -129,6 +136,29 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) { } } + if (const auto *AT = dyn_cast(Ty)) { + QualType ElementTy = + desugarForDiagnostic(Context, AT->getElementType(), ShouldAKA); + if (const auto *CAT = dyn_cast(AT)) + QT = Context.getConstantArrayType( + ElementTy, CAT->getSize(), CAT->getSizeExpr(), + CAT->getSizeModifier(), CAT->getIndexTypeCVRQualifiers()); + else if (const auto *VAT = dyn_cast(AT)) + QT = Context.getVariableArrayType( + ElementTy, VAT->getSizeExpr(), VAT->getSizeModifier(), + VAT->getIndexTypeCVRQualifiers(), VAT->getBracketsRange()); + else if (const auto *DSAT = dyn_cast(AT)) + QT = Context.getDependentSizedArrayType( + ElementTy, DSAT->getSizeExpr(), DSAT->getSizeModifier(), + DSAT->getIndexTypeCVRQualifiers(), DSAT->getBracketsRange()); + else if (const auto *IAT = dyn_cast(AT)) + QT = Context.getIncompleteArrayType(ElementTy, IAT->getSizeModifier(), + IAT->getIndexTypeCVRQualifiers()); + else + llvm_unreachable("Unhandled array type"); + break; + } + // Don't desugar magic Objective-C types. if (QualType(Ty,0) == Context.getObjCIdType() || QualType(Ty,0) == Context.getObjCClassType() || @@ -181,24 +211,25 @@ break; \ // If we have a pointer-like type, desugar the pointee as well. // FIXME: Handle other pointer-like types. if (const PointerType *Ty = QT->getAs()) { - QT = Context.getPointerType(Desugar(Context, Ty->getPointeeType(), - ShouldAKA)); + QT = Context.getPointerType( + desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA)); } else if (const auto *Ty = QT->getAs()) { - QT = Context.getObjCObjectPointerType(Desugar(Context, Ty->getPointeeType(), - ShouldAKA)); + QT = Context.getObjCObjectPointerType( + desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA)); } else if (const LValueReferenceType *Ty = QT->getAs()) { - QT = Context.getLValueReferenceType(Desugar(Context, Ty->getPointeeType(), - ShouldAKA)); + QT = Context.getLValueReferenceType( + desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA)); } else if (const RValueReferenceType *Ty = QT->getAs()) { - QT = Context.getRValueReferenceType(Desugar(Context, Ty->getPointeeType(), - ShouldAKA)); + QT = Context.getRValueReferenceType( + desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA)); } else if (const auto *Ty = QT->getAs()) { if (Ty->getBaseType().getTypePtr() != Ty && !ShouldAKA) { - QualType BaseType = Desugar(Context, Ty->getBaseType(), ShouldAKA); - QT = Context.getObjCObjectType(BaseType, Ty->getTypeArgsAsWritten(), - llvm::makeArrayRef(Ty->qual_begin(), - Ty->getNumProtocols()), - Ty->isKindOfTypeAsWritten()); + QualType BaseType = + desugarForDiagnostic(Context, Ty->getBaseType(), ShouldAKA); + QT = Context.getObjCObjectType( + BaseType, Ty->getTypeArgsAsWritten(), + llvm::makeArrayRef(Ty->qual_begin(), Ty->getNumProtocols()), + Ty->isKindOfTypeAsWritten()); } } @@ -251,7 +282,8 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty, continue; // Same canonical types std::string CompareS = CompareTy.getAsString(Context.getPrintingPolicy()); bool ShouldAKA = false; - QualType CompareDesugar = Desugar(Context, CompareTy, ShouldAKA); + QualType CompareDesugar = + desugarForDiagnostic(Context, CompareTy, ShouldAKA); std::string CompareDesugarStr = CompareDesugar.getAsString(Context.getPrintingPolicy()); if (CompareS != S && CompareDesugarStr != S) @@ -286,7 +318,7 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty, // sugar gives us something "significantly different". if (!Repeated) { bool ShouldAKA = false; - QualType DesugaredTy = Desugar(Context, Ty, ShouldAKA); + QualType DesugaredTy = desugarForDiagnostic(Context, Ty, ShouldAKA); if (ShouldAKA || ForceAKA) { if (DesugaredTy == Ty) { DesugaredTy = Ty.getCanonicalType(); @@ -308,7 +340,7 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty, OS << "'" << S << "' (vector of " << VTy->getNumElements() << " '" << VTy->getElementType().getAsString(Context.getPrintingPolicy()) << "' " << Values << ")"; - return OS.str(); + return DecoratedString; } } diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp index 710e40bbb4b7..7f78da10e0b3 100644 --- a/clang/lib/AST/ASTImporter.cpp +++ b/clang/lib/AST/ASTImporter.cpp @@ -354,6 +354,7 @@ namespace clang { ExpectedType VisitTypeOfExprType(const TypeOfExprType *T); // FIXME: DependentTypeOfExprType ExpectedType VisitTypeOfType(const TypeOfType *T); + ExpectedType VisitUsingType(const UsingType *T); ExpectedType VisitDecltypeType(const DecltypeType *T); ExpectedType VisitUnaryTransformType(const UnaryTransformType *T); ExpectedType VisitAutoType(const AutoType *T); @@ -1340,6 +1341,17 @@ ExpectedType ASTNodeImporter::VisitTypeOfType(const TypeOfType *T) { return Importer.getToContext().getTypeOfType(*ToUnderlyingTypeOrErr); } +ExpectedType ASTNodeImporter::VisitUsingType(const UsingType *T) { + Expected FoundOrErr = import(T->getFoundDecl()); + if (!FoundOrErr) + return FoundOrErr.takeError(); + Expected UnderlyingOrErr = import(T->getUnderlyingType()); + if (!UnderlyingOrErr) + return UnderlyingOrErr.takeError(); + + return Importer.getToContext().getUsingType(*FoundOrErr, *UnderlyingOrErr); +} + ExpectedType ASTNodeImporter::VisitDecltypeType(const DecltypeType *T) { // FIXME: Make sure that the "to" context supports C++0x! ExpectedExpr ToExprOrErr = import(T->getUnderlyingExpr()); @@ -6066,20 +6078,24 @@ ASTNodeImporter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) { if (Error Err = importInto(TemplatedFD, D->getTemplatedDecl())) return std::move(Err); - // Template parameters of the ClassTemplateDecl and FunctionTemplateDecl are - // shared, if the FunctionTemplateDecl is a deduction guide for the class. - // At import the ClassTemplateDecl object is always created first (FIXME: is - // this really true?) because the dependency, then the FunctionTemplateDecl. - // The DeclContext of the template parameters is changed when the - // FunctionTemplateDecl is created, but was set already when the class - // template was created. So here it is not the TU (default value) any more. - // FIXME: The DeclContext of the parameters is now set finally to the - // CXXDeductionGuideDecl object that was imported later. This may not be the - // same that is in the original AST, specially if there are multiple deduction - // guides. - DeclContext *OldParamDC = nullptr; - if (Params->size() > 0) - OldParamDC = Params->getParam(0)->getDeclContext(); + // At creation of the template the template parameters are "adopted" + // (DeclContext is changed). After this possible change the lookup table + // must be updated. + // At deduction guides the DeclContext of the template parameters may be + // different from what we would expect, it may be the class template, or a + // probably different CXXDeductionGuideDecl. This may come from the fact that + // the template parameter objects may be shared between deduction guides or + // the class template, and at creation of multiple FunctionTemplateDecl + // objects (for deduction guides) the same parameters are re-used. The + // "adoption" happens multiple times with different parent, even recursively + // for TemplateTemplateParmDecl. The same happens at import when the + // FunctionTemplateDecl objects are created, but in different order. + // In this way the DeclContext of these template parameters is not necessarily + // the same as in the "from" context. + SmallVector OldParamDC; + OldParamDC.reserve(Params->size()); + llvm::transform(*Params, std::back_inserter(OldParamDC), + [](NamedDecl *ND) { return ND->getDeclContext(); }); FunctionTemplateDecl *ToFunc; if (GetImportedOrCreateDecl(ToFunc, D, Importer.getToContext(), DC, Loc, Name, @@ -6091,7 +6107,12 @@ ASTNodeImporter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) { ToFunc->setAccess(D->getAccess()); ToFunc->setLexicalDeclContext(LexicalDC); LexicalDC->addDeclInternal(ToFunc); - updateLookupTableForTemplateParameters(*Params, OldParamDC); + + ASTImporterLookupTable *LT = Importer.SharedState->getLookupTable(); + if (LT && !OldParamDC.empty()) { + for (unsigned int I = 0; I < OldParamDC.size(); ++I) + LT->updateForced(Params->getParam(I), OldParamDC[I]); + } if (FoundByLookup) { auto *Recent = diff --git a/clang/lib/AST/ASTImporterLookupTable.cpp b/clang/lib/AST/ASTImporterLookupTable.cpp index ef42561c6f94..b7d17a5e92d0 100644 --- a/clang/lib/AST/ASTImporterLookupTable.cpp +++ b/clang/lib/AST/ASTImporterLookupTable.cpp @@ -140,6 +140,11 @@ void ASTImporterLookupTable::update(NamedDecl *ND, DeclContext *OldDC) { add(ND); } +void ASTImporterLookupTable::updateForced(NamedDecl *ND, DeclContext *OldDC) { + LookupTable[OldDC][ND->getDeclName()].remove(ND); + add(ND); +} + ASTImporterLookupTable::LookupResult ASTImporterLookupTable::lookup(DeclContext *DC, DeclarationName Name) const { auto DCI = LookupTable.find(DC->getPrimaryContext()); diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp index 7fd24e2aa9ad..0813a5204a5e 100644 --- a/clang/lib/AST/ASTStructuralEquivalence.cpp +++ b/clang/lib/AST/ASTStructuralEquivalence.cpp @@ -945,6 +945,12 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, return false; break; + case Type::Using: + if (!IsStructurallyEquivalent(Context, cast(T1)->getFoundDecl(), + cast(T2)->getFoundDecl())) + return false; + break; + case Type::Typedef: if (!IsStructurallyEquivalent(Context, cast(T1)->getDecl(), cast(T2)->getDecl())) @@ -1205,18 +1211,18 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, cast(T2)->getElementType())) return false; break; - case Type::ExtInt: { - const auto *Int1 = cast(T1); - const auto *Int2 = cast(T2); + case Type::BitInt: { + const auto *Int1 = cast(T1); + const auto *Int2 = cast(T2); if (Int1->isUnsigned() != Int2->isUnsigned() || Int1->getNumBits() != Int2->getNumBits()) return false; break; } - case Type::DependentExtInt: { - const auto *Int1 = cast(T1); - const auto *Int2 = cast(T2); + case Type::DependentBitInt: { + const auto *Int1 = cast(T1); + const auto *Int2 = cast(T2); if (Int1->isUnsigned() != Int2->isUnsigned() || !IsStructurallyEquivalent(Context, Int1->getNumBitsExpr(), diff --git a/clang/lib/AST/AttrImpl.cpp b/clang/lib/AST/AttrImpl.cpp index a3b46752c511..c2f13cf63830 100644 --- a/clang/lib/AST/AttrImpl.cpp +++ b/clang/lib/AST/AttrImpl.cpp @@ -60,7 +60,7 @@ std::string LoopHintAttr::getValueString(const PrintingPolicy &Policy) const { else OS << "disable"; OS << ")"; - return OS.str(); + return ValueName; } // Return a string suitable for identifying this attribute in diagnostics. diff --git a/clang/lib/AST/Comment.cpp b/clang/lib/AST/Comment.cpp index fae3640d5ff7..43820fc566e4 100644 --- a/clang/lib/AST/Comment.cpp +++ b/clang/lib/AST/Comment.cpp @@ -108,12 +108,7 @@ Comment::child_iterator Comment::child_end() const { } bool TextComment::isWhitespaceNoCache() const { - for (StringRef::const_iterator I = Text.begin(), E = Text.end(); - I != E; ++I) { - if (!clang::isWhitespace(*I)) - return false; - } - return true; + return llvm::all_of(Text, clang::isWhitespace); } bool ParagraphComment::isWhitespaceNoCache() const { diff --git a/clang/lib/AST/CommentBriefParser.cpp b/clang/lib/AST/CommentBriefParser.cpp index 2a5f7452b776..bf9e17993497 100644 --- a/clang/lib/AST/CommentBriefParser.cpp +++ b/clang/lib/AST/CommentBriefParser.cpp @@ -8,15 +8,12 @@ #include "clang/AST/CommentBriefParser.h" #include "clang/AST/CommentCommandTraits.h" +#include "clang/Basic/CharInfo.h" namespace clang { namespace comments { namespace { -inline bool isWhitespace(char C) { - return C == ' ' || C == '\n' || C == '\r' || - C == '\t' || C == '\f' || C == '\v'; -} /// Convert all whitespace into spaces, remove leading and trailing spaces, /// compress multiple spaces into one. @@ -26,12 +23,11 @@ void cleanupBrief(std::string &S) { for (std::string::iterator I = S.begin(), E = S.end(); I != E; ++I) { const char C = *I; - if (isWhitespace(C)) { + if (clang::isWhitespace(C)) { if (!PrevWasSpace) { *O++ = ' '; PrevWasSpace = true; } - continue; } else { *O++ = C; PrevWasSpace = false; @@ -44,12 +40,7 @@ void cleanupBrief(std::string &S) { } bool isWhitespace(StringRef Text) { - for (StringRef::const_iterator I = Text.begin(), E = Text.end(); - I != E; ++I) { - if (!isWhitespace(*I)) - return false; - } - return true; + return llvm::all_of(Text, clang::isWhitespace); } } // unnamed namespace diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 68dfef248f65..e63560f1b6fe 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -604,8 +604,14 @@ static LinkageInfo getExternalLinkageFor(const NamedDecl *D) { // - A name declared at namespace scope that does not have internal linkage // by the previous rules and that is introduced by a non-exported // declaration has module linkage. - if (isInModulePurview(D) && !isExportedFromModuleInterfaceUnit( - cast(D->getCanonicalDecl()))) + // + // [basic.namespace.general]/p2 + // A namespace is never attached to a named module and never has a name with + // module linkage. + if (isInModulePurview(D) && + !isExportedFromModuleInterfaceUnit( + cast(D->getCanonicalDecl())) && + !isa(D)) return LinkageInfo(ModuleLinkage, DefaultVisibility, false); return LinkageInfo::external(); @@ -1583,7 +1589,7 @@ std::string NamedDecl::getQualifiedNameAsString() const { std::string QualName; llvm::raw_string_ostream OS(QualName); printQualifiedName(OS, getASTContext().getPrintingPolicy()); - return OS.str(); + return QualName; } void NamedDecl::printQualifiedName(raw_ostream &OS) const { diff --git a/clang/lib/AST/DeclarationName.cpp b/clang/lib/AST/DeclarationName.cpp index 56cf4b457a48..b2232ddfced3 100644 --- a/clang/lib/AST/DeclarationName.cpp +++ b/clang/lib/AST/DeclarationName.cpp @@ -236,7 +236,7 @@ std::string DeclarationName::getAsString() const { std::string Result; llvm::raw_string_ostream OS(Result); OS << *this; - return OS.str(); + return Result; } void *DeclarationName::getFETokenInfoSlow() const { @@ -460,7 +460,7 @@ std::string DeclarationNameInfo::getAsString() const { std::string Result; llvm::raw_string_ostream OS(Result); OS << *this; - return OS.str(); + return Result; } raw_ostream &clang::operator<<(raw_ostream &OS, DeclarationNameInfo DNInfo) { diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index d3cb2ff3734c..2530beb89d17 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -202,6 +202,23 @@ bool Expr::isKnownToHaveBooleanValue(bool Semantic) const { return false; } +const ValueDecl * +Expr::getAsBuiltinConstantDeclRef(const ASTContext &Context) const { + Expr::EvalResult Eval; + + if (EvaluateAsConstantExpr(Eval, Context)) { + APValue &Value = Eval.Val; + + if (Value.isMemberPointer()) + return Value.getMemberPointerDecl(); + + if (Value.isLValue() && Value.getLValueOffset().isZero()) + return Value.getLValueBase().dyn_cast(); + } + + return nullptr; +} + // Amusing macro metaprogramming hack: check whether a class provides // a more specific implementation of getExprLoc(). // diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 99babd58b027..469339e8cd62 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -1954,11 +1954,12 @@ static bool EvaluateIgnoredValue(EvalInfo &Info, const Expr *E) { return true; } -/// Should this call expression be treated as a string literal? -static bool IsStringLiteralCall(const CallExpr *E) { +/// Should this call expression be treated as a constant? +static bool IsConstantCall(const CallExpr *E) { unsigned Builtin = E->getBuiltinCallee(); return (Builtin == Builtin::BI__builtin___CFStringMakeConstantString || - Builtin == Builtin::BI__builtin___NSStringMakeConstantString); + Builtin == Builtin::BI__builtin___NSStringMakeConstantString || + Builtin == Builtin::BI__builtin_function_start); } static bool IsGlobalLValue(APValue::LValueBase B) { @@ -2004,7 +2005,7 @@ static bool IsGlobalLValue(APValue::LValueBase B) { case Expr::ObjCBoxedExprClass: return cast(E)->isExpressibleAsConstantInitializer(); case Expr::CallExprClass: - return IsStringLiteralCall(cast(E)); + return IsConstantCall(cast(E)); // For GCC compatibility, &&label has static storage duration. case Expr::AddrLabelExprClass: return true; @@ -2931,6 +2932,11 @@ handleCompareOpForVectorHelper(const APTy &LHSValue, BinaryOperatorKind Opcode, break; } + // The boolean operations on these vector types use an instruction that + // results in a mask of '-1' for the 'truth' value. Ensure that we negate 1 + // to -1 to make sure that we produce the correct value. + Result.negate(); + return true; } @@ -8962,7 +8968,7 @@ bool PointerExprEvaluator::visitNonBuiltinCallExpr(const CallExpr *E) { } bool PointerExprEvaluator::VisitCallExpr(const CallExpr *E) { - if (IsStringLiteralCall(E)) + if (IsConstantCall(E)) return Success(E); if (unsigned BuiltinOp = E->getBuiltinCallee()) @@ -10179,7 +10185,8 @@ namespace { bool VisitInitListExpr(const InitListExpr *E); bool VisitUnaryImag(const UnaryOperator *E); bool VisitBinaryOperator(const BinaryOperator *E); - // FIXME: Missing: unary -, unary ~, conditional operator (for GNU + bool VisitUnaryOperator(const UnaryOperator *E); + // FIXME: Missing: conditional operator (for GNU // conditional select), shufflevector, ExtVectorElementExpr }; } // end anonymous namespace @@ -10364,6 +10371,83 @@ bool VectorExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { return Success(LHSValue, E); } +static llvm::Optional handleVectorUnaryOperator(ASTContext &Ctx, + QualType ResultTy, + UnaryOperatorKind Op, + APValue Elt) { + switch (Op) { + case UO_Plus: + // Nothing to do here. + return Elt; + case UO_Minus: + if (Elt.getKind() == APValue::Int) { + Elt.getInt().negate(); + } else { + assert(Elt.getKind() == APValue::Float && + "Vector can only be int or float type"); + Elt.getFloat().changeSign(); + } + return Elt; + case UO_Not: + // This is only valid for integral types anyway, so we don't have to handle + // float here. + assert(Elt.getKind() == APValue::Int && + "Vector operator ~ can only be int"); + Elt.getInt().flipAllBits(); + return Elt; + case UO_LNot: { + if (Elt.getKind() == APValue::Int) { + Elt.getInt() = !Elt.getInt(); + // operator ! on vectors returns -1 for 'truth', so negate it. + Elt.getInt().negate(); + return Elt; + } + assert(Elt.getKind() == APValue::Float && + "Vector can only be int or float type"); + // Float types result in an int of the same size, but -1 for true, or 0 for + // false. + APSInt EltResult{Ctx.getIntWidth(ResultTy), + ResultTy->isUnsignedIntegerType()}; + if (Elt.getFloat().isZero()) + EltResult.setAllBits(); + else + EltResult.clearAllBits(); + + return APValue{EltResult}; + } + default: + // FIXME: Implement the rest of the unary operators. + return llvm::None; + } +} + +bool VectorExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) { + Expr *SubExpr = E->getSubExpr(); + const auto *VD = SubExpr->getType()->castAs(); + // This result element type differs in the case of negating a floating point + // vector, since the result type is the a vector of the equivilant sized + // integer. + const QualType ResultEltTy = VD->getElementType(); + UnaryOperatorKind Op = E->getOpcode(); + + APValue SubExprValue; + if (!Evaluate(SubExprValue, Info, SubExpr)) + return false; + + assert(SubExprValue.getVectorLength() == VD->getNumElements() && + "Vector length doesn't match type?"); + + SmallVector ResultElements; + for (unsigned EltNum = 0; EltNum < VD->getNumElements(); ++EltNum) { + llvm::Optional Elt = handleVectorUnaryOperator( + Info.Ctx, ResultEltTy, Op, SubExprValue.getVectorElt(EltNum)); + if (!Elt) + return false; + ResultElements.push_back(*Elt); + } + return Success(APValue(ResultElements.data(), ResultElements.size()), E); +} + //===----------------------------------------------------------------------===// // Array Evaluation //===----------------------------------------------------------------------===// @@ -11077,7 +11161,7 @@ EvaluateBuiltinClassifyType(QualType T, const LangOptions &LangOpts) { case Type::ObjCInterface: case Type::ObjCObjectPointer: case Type::Pipe: - case Type::ExtInt: + case Type::BitInt: // GCC classifies vectors as None. We follow its lead and classify all // other types that don't fit into the regular classification the same way. return GCCTypeClass::None; diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp index 07579d04e275..7afc1250a36f 100644 --- a/clang/lib/AST/ItaniumMangle.cpp +++ b/clang/lib/AST/ItaniumMangle.cpp @@ -2263,8 +2263,8 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty, case Type::Atomic: case Type::Pipe: case Type::MacroQualified: - case Type::ExtInt: - case Type::DependentExtInt: + case Type::BitInt: + case Type::DependentBitInt: llvm_unreachable("type is illegal as a nested name specifier"); case Type::SubstTemplateTypeParmPack: @@ -2380,6 +2380,9 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty, break; } + case Type::Using: + return mangleUnresolvedTypeOrSimpleId(cast(Ty)->desugar(), + Prefix); case Type::Elaborated: return mangleUnresolvedTypeOrSimpleId( cast(Ty)->getNamedType(), Prefix); @@ -3967,26 +3970,20 @@ void CXXNameMangler::mangleType(const PipeType *T) { Out << "8ocl_pipe"; } -void CXXNameMangler::mangleType(const ExtIntType *T) { - Out << "U7_ExtInt"; - llvm::APSInt BW(32, true); - BW = T->getNumBits(); - TemplateArgument TA(Context.getASTContext(), BW, getASTContext().IntTy); - mangleTemplateArgs(TemplateName(), &TA, 1); - if (T->isUnsigned()) - Out << "j"; - else - Out << "i"; +void CXXNameMangler::mangleType(const BitIntType *T) { + // 5.1.5.2 Builtin types + // ::= DB _ + // ::= DU _ + Out << "D" << (T->isUnsigned() ? "U" : "B") << T->getNumBits() << "_"; } -void CXXNameMangler::mangleType(const DependentExtIntType *T) { - Out << "U7_ExtInt"; - TemplateArgument TA(T->getNumBitsExpr()); - mangleTemplateArgs(TemplateName(), &TA, 1); - if (T->isUnsigned()) - Out << "j"; - else - Out << "i"; +void CXXNameMangler::mangleType(const DependentBitIntType *T) { + // 5.1.5.2 Builtin types + // ::= DB _ + // ::= DU _ + Out << "D" << (T->isUnsigned() ? "U" : "B"); + mangleExpression(T->getNumBitsExpr()); + Out << "_"; } void CXXNameMangler::mangleIntegerLiteral(QualType T, diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp index 86879b8c3533..ae585def4d07 100644 --- a/clang/lib/AST/JSONNodeDumper.cpp +++ b/clang/lib/AST/JSONNodeDumper.cpp @@ -744,11 +744,18 @@ void JSONNodeDumper::VisitNamedDecl(const NamedDecl *ND) { JOS.attribute("name", ND->getNameAsString()); // FIXME: There are likely other contexts in which it makes no sense to ask // for a mangled name. - if (!isa(ND->getDeclContext())) { - std::string MangledName = ASTNameGen.getName(ND); - if (!MangledName.empty()) - JOS.attribute("mangledName", MangledName); - } + if (isa(ND->getDeclContext())) + return; + + // Mangled names are not meaningful for locals, and may not be well-defined + // in the case of VLAs. + auto *VD = dyn_cast(ND); + if (VD && VD->hasLocalStorage()) + return; + + std::string MangledName = ASTNameGen.getName(ND); + if (!MangledName.empty()) + JOS.attribute("mangledName", MangledName); } } diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp index 79a448a2435c..8802b6e500a6 100644 --- a/clang/lib/AST/MicrosoftMangle.cpp +++ b/clang/lib/AST/MicrosoftMangle.cpp @@ -3365,26 +3365,26 @@ void MicrosoftMangleContextImpl::mangleCXXName(GlobalDecl GD, return Mangler.mangle(GD); } -void MicrosoftCXXNameMangler::mangleType(const ExtIntType *T, Qualifiers, +void MicrosoftCXXNameMangler::mangleType(const BitIntType *T, Qualifiers, SourceRange Range) { llvm::SmallString<64> TemplateMangling; llvm::raw_svector_ostream Stream(TemplateMangling); MicrosoftCXXNameMangler Extra(Context, Stream); Stream << "?$"; if (T->isUnsigned()) - Extra.mangleSourceName("_UExtInt"); + Extra.mangleSourceName("_UBitInt"); else - Extra.mangleSourceName("_ExtInt"); + Extra.mangleSourceName("_BitInt"); Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumBits())); mangleArtificialTagType(TTK_Struct, TemplateMangling, {"__clang"}); } -void MicrosoftCXXNameMangler::mangleType(const DependentExtIntType *T, +void MicrosoftCXXNameMangler::mangleType(const DependentBitIntType *T, Qualifiers, SourceRange Range) { DiagnosticsEngine &Diags = Context.getDiags(); unsigned DiagID = Diags.getCustomDiagID( - DiagnosticsEngine::Error, "cannot mangle this DependentExtInt type yet"); + DiagnosticsEngine::Error, "cannot mangle this DependentBitInt type yet"); Diags.Report(Range.getBegin(), DiagID) << Range; } diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp index f721e56f7fdd..1bd049b88005 100644 --- a/clang/lib/AST/OpenMPClause.cpp +++ b/clang/lib/AST/OpenMPClause.cpp @@ -126,6 +126,7 @@ const OMPClauseWithPreInit *OMPClauseWithPreInit::get(const OMPClause *C) { case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -217,6 +218,7 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C) case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -1792,6 +1794,10 @@ void OMPClausePrinter::VisitOMPCaptureClause(OMPCaptureClause *) { OS << "capture"; } +void OMPClausePrinter::VisitOMPCompareClause(OMPCompareClause *) { + OS << "compare"; +} + void OMPClausePrinter::VisitOMPSeqCstClause(OMPSeqCstClause *) { OS << "seq_cst"; } @@ -2454,7 +2460,7 @@ std::string OMPTraitInfo::getMangledName() const { Property.RawString); } } - return OS.str(); + return MangledName; } OMPTraitInfo::OMPTraitInfo(StringRef MangledName) { diff --git a/clang/lib/AST/ParentMap.cpp b/clang/lib/AST/ParentMap.cpp index 2ff5c9d8aeb5..da21e573c320 100644 --- a/clang/lib/AST/ParentMap.cpp +++ b/clang/lib/AST/ParentMap.cpp @@ -133,8 +133,7 @@ void ParentMap::setParent(const Stmt *S, const Stmt *Parent) { Stmt* ParentMap::getParent(Stmt* S) const { MapTy* M = (MapTy*) Impl; - MapTy::iterator I = M->find(S); - return I == M->end() ? nullptr : I->second; + return M->lookup(S); } Stmt *ParentMap::getParentIgnoreParens(Stmt *S) const { diff --git a/clang/lib/AST/QualTypeNames.cpp b/clang/lib/AST/QualTypeNames.cpp index 673821078345..561757b1ba64 100644 --- a/clang/lib/AST/QualTypeNames.cpp +++ b/clang/lib/AST/QualTypeNames.cpp @@ -418,6 +418,13 @@ QualType getFullyQualifiedType(QualType QT, const ASTContext &Ctx, return QT; } + // We don't consider the alias introduced by `using a::X` as a new type. + // The qualified name is still a::X. + if (isa(QT.getTypePtr())) { + return getFullyQualifiedType(QT.getSingleStepDesugaredType(Ctx), Ctx, + WithGlobalNsPrefix); + } + // Remove the part of the type related to the type being a template // parameter (we won't report it as part of the 'type name' and it // is actually make the code below to be more complex (to handle diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp index 4339c249e027..09853e0f0e49 100644 --- a/clang/lib/AST/StmtProfile.cpp +++ b/clang/lib/AST/StmtProfile.cpp @@ -551,6 +551,8 @@ void OMPClauseProfiler::VisitOMPUpdateClause(const OMPUpdateClause *) {} void OMPClauseProfiler::VisitOMPCaptureClause(const OMPCaptureClause *) {} +void OMPClauseProfiler::VisitOMPCompareClause(const OMPCompareClause *) {} + void OMPClauseProfiler::VisitOMPSeqCstClause(const OMPSeqCstClause *) {} void OMPClauseProfiler::VisitOMPAcqRelClause(const OMPAcqRelClause *) {} diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp index b21e806e307c..67c934847c7f 100644 --- a/clang/lib/AST/TextNodeDumper.cpp +++ b/clang/lib/AST/TextNodeDumper.cpp @@ -1534,6 +1534,10 @@ void TextNodeDumper::VisitUnresolvedUsingType(const UnresolvedUsingType *T) { dumpDeclRef(T->getDecl()); } +void TextNodeDumper::VisitUsingType(const UsingType *T) { + dumpDeclRef(T->getFoundDecl()); +} + void TextNodeDumper::VisitTypedefType(const TypedefType *T) { dumpDeclRef(T->getDecl()); } diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index e0ac3f5b1351..c771fe264b0c 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -338,25 +338,25 @@ VectorType::VectorType(TypeClass tc, QualType vecType, unsigned nElements, VectorTypeBits.NumElements = nElements; } -ExtIntType::ExtIntType(bool IsUnsigned, unsigned NumBits) - : Type(ExtInt, QualType{}, TypeDependence::None), IsUnsigned(IsUnsigned), +BitIntType::BitIntType(bool IsUnsigned, unsigned NumBits) + : Type(BitInt, QualType{}, TypeDependence::None), IsUnsigned(IsUnsigned), NumBits(NumBits) {} -DependentExtIntType::DependentExtIntType(const ASTContext &Context, +DependentBitIntType::DependentBitIntType(const ASTContext &Context, bool IsUnsigned, Expr *NumBitsExpr) - : Type(DependentExtInt, QualType{}, + : Type(DependentBitInt, QualType{}, toTypeDependence(NumBitsExpr->getDependence())), Context(Context), ExprAndUnsigned(NumBitsExpr, IsUnsigned) {} -bool DependentExtIntType::isUnsigned() const { +bool DependentBitIntType::isUnsigned() const { return ExprAndUnsigned.getInt(); } -clang::Expr *DependentExtIntType::getNumBitsExpr() const { +clang::Expr *DependentBitIntType::getNumBitsExpr() const { return ExprAndUnsigned.getPointer(); } -void DependentExtIntType::Profile(llvm::FoldingSetNodeID &ID, +void DependentBitIntType::Profile(llvm::FoldingSetNodeID &ID, const ASTContext &Context, bool IsUnsigned, Expr *NumBitsExpr) { ID.AddBoolean(IsUnsigned); @@ -1932,7 +1932,7 @@ bool Type::isIntegralType(const ASTContext &Ctx) const { if (const auto *ET = dyn_cast(CanonicalType)) return ET->getDecl()->isComplete(); - return isExtIntType(); + return isBitIntType(); } bool Type::isIntegralOrUnscopedEnumerationType() const { @@ -1940,7 +1940,7 @@ bool Type::isIntegralOrUnscopedEnumerationType() const { return BT->getKind() >= BuiltinType::Bool && BT->getKind() <= BuiltinType::Int128; - if (isExtIntType()) + if (isBitIntType()) return true; return isUnscopedEnumerationType(); @@ -2023,7 +2023,9 @@ bool Type::isSignedIntegerType() const { return ET->getDecl()->getIntegerType()->isSignedIntegerType(); } - if (const ExtIntType *IT = dyn_cast(CanonicalType)) + if (const auto *IT = dyn_cast(CanonicalType)) + return IT->isSigned(); + if (const auto *IT = dyn_cast(CanonicalType)) return IT->isSigned(); return false; @@ -2040,9 +2042,10 @@ bool Type::isSignedIntegerOrEnumerationType() const { return ET->getDecl()->getIntegerType()->isSignedIntegerType(); } - if (const ExtIntType *IT = dyn_cast(CanonicalType)) + if (const auto *IT = dyn_cast(CanonicalType)) + return IT->isSigned(); + if (const auto *IT = dyn_cast(CanonicalType)) return IT->isSigned(); - return false; } @@ -2070,7 +2073,9 @@ bool Type::isUnsignedIntegerType() const { return ET->getDecl()->getIntegerType()->isUnsignedIntegerType(); } - if (const ExtIntType *IT = dyn_cast(CanonicalType)) + if (const auto *IT = dyn_cast(CanonicalType)) + return IT->isUnsigned(); + if (const auto *IT = dyn_cast(CanonicalType)) return IT->isUnsigned(); return false; @@ -2087,7 +2092,9 @@ bool Type::isUnsignedIntegerOrEnumerationType() const { return ET->getDecl()->getIntegerType()->isUnsignedIntegerType(); } - if (const ExtIntType *IT = dyn_cast(CanonicalType)) + if (const auto *IT = dyn_cast(CanonicalType)) + return IT->isUnsigned(); + if (const auto *IT = dyn_cast(CanonicalType)) return IT->isUnsigned(); return false; @@ -2129,7 +2136,7 @@ bool Type::isRealType() const { BT->getKind() <= BuiltinType::Ibm128; if (const auto *ET = dyn_cast(CanonicalType)) return ET->getDecl()->isComplete() && !ET->getDecl()->isScoped(); - return isExtIntType(); + return isBitIntType(); } bool Type::isArithmeticType() const { @@ -2145,7 +2152,7 @@ bool Type::isArithmeticType() const { // false for scoped enumerations since that will disable any // unwanted implicit conversions. return !ET->getDecl()->isScoped() && ET->getDecl()->isComplete(); - return isa(CanonicalType) || isExtIntType(); + return isa(CanonicalType) || isBitIntType(); } Type::ScalarTypeKind Type::getScalarTypeKind() const { @@ -2174,7 +2181,7 @@ Type::ScalarTypeKind Type::getScalarTypeKind() const { if (CT->getElementType()->isRealFloatingType()) return STK_FloatingComplex; return STK_IntegralComplex; - } else if (isExtIntType()) { + } else if (isBitIntType()) { return STK_Integral; } @@ -2381,7 +2388,7 @@ bool QualType::isCXX98PODType(const ASTContext &Context) const { case Type::MemberPointer: case Type::Vector: case Type::ExtVector: - case Type::ExtInt: + case Type::BitInt: return true; case Type::Enum: @@ -3400,6 +3407,17 @@ QualType TypedefType::desugar() const { return getDecl()->getUnderlyingType(); } +UsingType::UsingType(const UsingShadowDecl *Found, QualType Underlying, + QualType Canon) + : Type(Using, Canon, Underlying->getDependence()), + Found(const_cast(Found)) { + assert(Underlying == getUnderlyingType()); +} + +QualType UsingType::getUnderlyingType() const { + return QualType(cast(Found->getTargetDecl())->getTypeForDecl(), 0); +} + QualType MacroQualifiedType::desugar() const { return getUnderlyingType(); } QualType MacroQualifiedType::getModifiedType() const { @@ -3849,7 +3867,7 @@ static CachedProperties computeCachedProperties(const Type *T) { // here in error recovery. return CachedProperties(ExternalLinkage, false); - case Type::ExtInt: + case Type::BitInt: case Type::Builtin: // C++ [basic.link]p8: // A type is said to have linkage if and only if: @@ -3949,7 +3967,7 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) { assert(T->isInstantiationDependentType()); return LinkageInfo::external(); - case Type::ExtInt: + case Type::BitInt: case Type::Builtin: return LinkageInfo::external(); @@ -4169,8 +4187,8 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const { case Type::ObjCInterface: case Type::Atomic: case Type::Pipe: - case Type::ExtInt: - case Type::DependentExtInt: + case Type::BitInt: + case Type::DependentBitInt: return false; } llvm_unreachable("bad type kind!"); diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp index eca9af3e5f36..2a33a69f288d 100644 --- a/clang/lib/AST/TypePrinter.cpp +++ b/clang/lib/AST/TypePrinter.cpp @@ -212,6 +212,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T, case Type::Builtin: case Type::Complex: case Type::UnresolvedUsing: + case Type::Using: case Type::Typedef: case Type::TypeOfExpr: case Type::TypeOf: @@ -232,8 +233,8 @@ bool TypePrinter::canPrefixQualifiers(const Type *T, case Type::ObjCInterface: case Type::Atomic: case Type::Pipe: - case Type::ExtInt: - case Type::DependentExtInt: + case Type::BitInt: + case Type::DependentBitInt: CanPrefixQualifiers = true; break; @@ -1046,6 +1047,21 @@ void TypePrinter::printUnresolvedUsingBefore(const UnresolvedUsingType *T, void TypePrinter::printUnresolvedUsingAfter(const UnresolvedUsingType *T, raw_ostream &OS) {} +void TypePrinter::printUsingBefore(const UsingType *T, raw_ostream &OS) { + // After `namespace b { using a::X }`, is the type X within B a::X or b::X? + // + // - b::X is more formally correct given the UsingType model + // - b::X makes sense if "re-exporting" a symbol in a new namespace + // - a::X makes sense if "importing" a symbol for convenience + // + // The "importing" use seems much more common, so we print a::X. + // This could be a policy option, but the right choice seems to rest more + // with the intent of the code than the caller. + printTypeSpec(T->getFoundDecl()->getUnderlyingDecl(), OS); +} + +void TypePrinter::printUsingAfter(const UsingType *T, raw_ostream &OS) {} + void TypePrinter::printTypedefBefore(const TypedefType *T, raw_ostream &OS) { printTypeSpec(T->getDecl(), OS); } @@ -1200,26 +1216,26 @@ void TypePrinter::printPipeBefore(const PipeType *T, raw_ostream &OS) { void TypePrinter::printPipeAfter(const PipeType *T, raw_ostream &OS) {} -void TypePrinter::printExtIntBefore(const ExtIntType *T, raw_ostream &OS) { +void TypePrinter::printBitIntBefore(const BitIntType *T, raw_ostream &OS) { if (T->isUnsigned()) OS << "unsigned "; - OS << "_ExtInt(" << T->getNumBits() << ")"; + OS << "_BitInt(" << T->getNumBits() << ")"; spaceBeforePlaceHolder(OS); } -void TypePrinter::printExtIntAfter(const ExtIntType *T, raw_ostream &OS) {} +void TypePrinter::printBitIntAfter(const BitIntType *T, raw_ostream &OS) {} -void TypePrinter::printDependentExtIntBefore(const DependentExtIntType *T, +void TypePrinter::printDependentBitIntBefore(const DependentBitIntType *T, raw_ostream &OS) { if (T->isUnsigned()) OS << "unsigned "; - OS << "_ExtInt("; + OS << "_BitInt("; T->getNumBitsExpr()->printPretty(OS, nullptr, Policy); OS << ")"; spaceBeforePlaceHolder(OS); } -void TypePrinter::printDependentExtIntAfter(const DependentExtIntType *T, +void TypePrinter::printDependentBitIntAfter(const DependentBitIntType *T, raw_ostream &OS) {} /// Appends the given scope to the end of a string. diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp index 7680eb38283e..815058f32de4 100644 --- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp +++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp @@ -1059,6 +1059,7 @@ const AstTypeMatcher unaryTransformType; const AstTypeMatcher recordType; const AstTypeMatcher tagType; const AstTypeMatcher elaboratedType; +const AstTypeMatcher usingType; const AstTypeMatcher substTemplateTypeParmType; const AstTypeMatcher templateTypeParmType; const AstTypeMatcher injectedClassNameType; diff --git a/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp b/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp index ba2f49e6b623..41ab0ed70fda 100644 --- a/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp +++ b/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp @@ -204,7 +204,7 @@ std::string Diagnostics::toString() const { std::string S; llvm::raw_string_ostream OS(S); printToStream(OS); - return OS.str(); + return S; } void Diagnostics::printToStreamFull(llvm::raw_ostream &OS) const { @@ -223,7 +223,7 @@ std::string Diagnostics::toStringFull() const { std::string S; llvm::raw_string_ostream OS(S); printToStreamFull(OS); - return OS.str(); + return S; } } // namespace dynamic diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.h b/clang/lib/ASTMatchers/Dynamic/Marshallers.h index 783fb203c408..fa9d42247e24 100644 --- a/clang/lib/ASTMatchers/Dynamic/Marshallers.h +++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.h @@ -1035,7 +1035,6 @@ class MapAnyOfBuilderDescriptor : public MatcherDescriptor { void getArgKinds(ASTNodeKind ThisKind, unsigned, std::vector &ArgKinds) const override { ArgKinds.push_back(ArgKind::MakeNodeArg(ThisKind)); - return; } bool isConvertibleTo(ASTNodeKind Kind, unsigned *Specificity = nullptr, ASTNodeKind *LeastDerivedKind = nullptr) const override { diff --git a/clang/lib/ASTMatchers/Dynamic/Parser.cpp b/clang/lib/ASTMatchers/Dynamic/Parser.cpp index c6a77bb6c2e0..cab1476acf94 100644 --- a/clang/lib/ASTMatchers/Dynamic/Parser.cpp +++ b/clang/lib/ASTMatchers/Dynamic/Parser.cpp @@ -645,7 +645,7 @@ bool Parser::parseMatcherExpressionImpl(const TokenInfo &NameToken, Tokenizer->SkipNewlines(); { - ScopedContextEntry SCE(this, Ctor ? *Ctor : nullptr); + ScopedContextEntry SCE(this, Ctor.getValueOr(nullptr)); while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) { if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) { diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp index 878547923d27..4f3efdb0a663 100644 --- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp +++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp @@ -228,6 +228,7 @@ RegistryMaps::RegistryMaps() { REGISTER_MATCHER(eachOf); REGISTER_MATCHER(elaboratedType); REGISTER_MATCHER(elaboratedTypeLoc); + REGISTER_MATCHER(usingType); REGISTER_MATCHER(enumConstantDecl); REGISTER_MATCHER(enumDecl); REGISTER_MATCHER(enumType); diff --git a/clang/lib/Analysis/AnalysisDeclContext.cpp b/clang/lib/Analysis/AnalysisDeclContext.cpp index d8466ac34a3d..06f1f813aeed 100644 --- a/clang/lib/Analysis/AnalysisDeclContext.cpp +++ b/clang/lib/Analysis/AnalysisDeclContext.cpp @@ -387,7 +387,7 @@ std::string AnalysisDeclContext::getFunctionName(const Decl *D) { OS << ' ' << OMD->getSelector().getAsString() << ']'; } - return OS.str(); + return Str; } LocationContextManager &AnalysisDeclContext::getLocationContextManager() { diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp index abf65e3efce9..9ef3b5b6277a 100644 --- a/clang/lib/Analysis/CFG.cpp +++ b/clang/lib/Analysis/CFG.cpp @@ -1820,8 +1820,6 @@ void CFGBuilder::addScopesEnd(LocalScope::const_iterator B, for (VarDecl *VD : llvm::reverse(DeclsWithEndedScope)) appendScopeEnd(Block, VD, S); - - return; } /// addAutomaticObjDtors - Add to current block automatic objects destructors diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp index bb7eb9971068..413e8d14bf0a 100644 --- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp +++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp @@ -11,15 +11,82 @@ // //===----------------------------------------------------------------------===// +#include #include +#include "clang/Analysis/Analyses/PostOrderCFGView.h" #include "clang/Analysis/CFG.h" #include "clang/Analysis/FlowSensitive/DataflowEnvironment.h" +#include "clang/Analysis/FlowSensitive/DataflowWorklist.h" #include "clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h" +#include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" +#include "llvm/Support/raw_ostream.h" -using namespace clang; -using namespace dataflow; +namespace clang { +namespace dataflow { + +/// Computes the input state for a given basic block by joining the output +/// states of its predecessors. +/// +/// Requirements: +/// +/// All predecessors of `Block` except those with loop back edges must have +/// already been transferred. States in `BlockStates` that are set to +/// `llvm::None` represent basic blocks that are not evaluated yet. +static TypeErasedDataflowAnalysisState computeBlockInputState( + std::vector> &BlockStates, + const CFGBlock &Block, const Environment &InitEnv, + TypeErasedDataflowAnalysis &Analysis) { + // FIXME: Consider passing `Block` to `Analysis.typeErasedInitialElement()` + // to enable building analyses like computation of dominators that initialize + // the state of each basic block differently. + TypeErasedDataflowAnalysisState State = {Analysis.typeErasedInitialElement(), + InitEnv}; + for (const CFGBlock *Pred : Block.preds()) { + // Skip if the `Block` is unreachable or control flow cannot get past it. + if (!Pred || Pred->hasNoReturnElement()) + continue; + + // Skip if `Pred` was not evaluated yet. This could happen if `Pred` has a + // loop back edge to `Block`. + const llvm::Optional &MaybePredState = + BlockStates[Pred->getBlockID()]; + if (!MaybePredState.hasValue()) + continue; + + const TypeErasedDataflowAnalysisState &PredState = + MaybePredState.getValue(); + Analysis.joinTypeErased(State.Lattice, PredState.Lattice); + State.Env.join(PredState.Env); + } + return State; +} + +TypeErasedDataflowAnalysisState transferBlock( + std::vector> &BlockStates, + const CFGBlock &Block, const Environment &InitEnv, + TypeErasedDataflowAnalysis &Analysis, + std::function + HandleTransferredStmt) { + TypeErasedDataflowAnalysisState State = + computeBlockInputState(BlockStates, Block, InitEnv, Analysis); + for (const CFGElement &Element : Block) { + // FIXME: Evaluate other kinds of `CFGElement`. + const llvm::Optional Stmt = Element.getAs(); + if (!Stmt.hasValue()) + continue; + + // FIXME: Evaluate the statement contained in `Stmt`. + + State.Lattice = Analysis.transferTypeErased(Stmt.getValue().getStmt(), + State.Lattice, State.Env); + if (HandleTransferredStmt != nullptr) + HandleTransferredStmt(Stmt.getValue(), State); + } + return State; +} std::vector> runTypeErasedDataflowAnalysis(const CFG &Cfg, @@ -29,7 +96,59 @@ runTypeErasedDataflowAnalysis(const CFG &Cfg, // are specified in the header. This could be done by remembering // what options were used to build `Cfg` and asserting on them here. - // FIXME: Implement work list-based algorithm to compute the fixed - // point of `Analysis::transform` for every basic block in `Cfg`. - return {}; + PostOrderCFGView POV(&Cfg); + ForwardDataflowWorklist Worklist(Cfg, &POV); + + std::vector> BlockStates; + BlockStates.resize(Cfg.size(), llvm::None); + + // The entry basic block doesn't contain statements so it can be skipped. + const CFGBlock &Entry = Cfg.getEntry(); + BlockStates[Entry.getBlockID()] = {Analysis.typeErasedInitialElement(), + InitEnv}; + Worklist.enqueueSuccessors(&Entry); + + // Bugs in lattices and transfer functions can prevent the analysis from + // converging. To limit the damage (infinite loops) that these bugs can cause, + // limit the number of iterations. + // FIXME: Consider making the maximum number of iterations configurable. + // FIXME: Set up statistics (see llvm/ADT/Statistic.h) to count average number + // of iterations, number of functions that time out, etc. + unsigned Iterations = 0; + static constexpr unsigned MaxIterations = 1 << 16; + while (const CFGBlock *Block = Worklist.dequeue()) { + if (++Iterations > MaxIterations) { + llvm::errs() << "Maximum number of iterations reached, giving up.\n"; + break; + } + + const llvm::Optional &OldBlockState = + BlockStates[Block->getBlockID()]; + TypeErasedDataflowAnalysisState NewBlockState = + transferBlock(BlockStates, *Block, InitEnv, Analysis); + + if (OldBlockState.hasValue() && + Analysis.isEqualTypeErased(OldBlockState.getValue().Lattice, + NewBlockState.Lattice) && + OldBlockState->Env == NewBlockState.Env) { + // The state of `Block` didn't change after transfer so there's no need to + // revisit its successors. + continue; + } + + BlockStates[Block->getBlockID()] = std::move(NewBlockState); + + // Do not add unreachable successor blocks to `Worklist`. + if (Block->hasNoReturnElement()) + continue; + + Worklist.enqueueSuccessors(Block); + } + // FIXME: Consider evaluating unreachable basic blocks (those that have a + // state set to `llvm::None` at this point) to also analyze dead code. + + return BlockStates; } + +} // namespace dataflow +} // namespace clang diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp index b196ffa73cbf..9cc990bd35a3 100644 --- a/clang/lib/Analysis/ThreadSafety.cpp +++ b/clang/lib/Analysis/ThreadSafety.cpp @@ -418,7 +418,6 @@ class LocalVariableMap { private: Context::Factory ContextFactory; std::vector VarDefinitions; - std::vector CtxIndices; std::vector> SavedContexts; public: @@ -731,8 +730,6 @@ void LocalVariableMap::traverseCFG(CFG *CFGraph, std::vector &BlockInfo) { PostOrderCFGView::CFGBlockSet VisitedBlocks(CFGraph); - CtxIndices.resize(CFGraph->getNumBlockIDs()); - for (const auto *CurrBlock : *SortedGraph) { unsigned CurrBlockID = CurrBlock->getBlockID(); CFGBlockInfo *CurrBlockInfo = &BlockInfo[CurrBlockID]; diff --git a/clang/lib/Analysis/UninitializedValues.cpp b/clang/lib/Analysis/UninitializedValues.cpp index 67cd39728c35..a38ae34f4b81 100644 --- a/clang/lib/Analysis/UninitializedValues.cpp +++ b/clang/lib/Analysis/UninitializedValues.cpp @@ -591,8 +591,8 @@ class TransferFunctions : public StmtVisitor { if (AtPredExit == MayUninitialized) { // If the predecessor's terminator is an "asm goto" that initializes - // the variable, then it won't be counted as "initialized" on the - // non-fallthrough paths. + // the variable, then don't count it as "initialized" on the indirect + // paths. CFGTerminator term = Pred->getTerminator(); if (const auto *as = dyn_cast_or_null(term.getStmt())) { const CFGBlock *fallthrough = *Pred->succ_begin(); @@ -810,13 +810,22 @@ void TransferFunctions::VisitGCCAsmStmt(GCCAsmStmt *as) { if (!as->isAsmGoto()) return; - for (const Expr *o : as->outputs()) - if (const VarDecl *VD = findVar(o).getDecl()) + ASTContext &C = ac.getASTContext(); + for (const Expr *O : as->outputs()) { + const Expr *Ex = stripCasts(C, O); + + // Strip away any unary operators. Invalid l-values are reported by other + // semantic analysis passes. + while (const auto *UO = dyn_cast(Ex)) + Ex = stripCasts(C, UO->getSubExpr()); + + if (const VarDecl *VD = findVar(Ex).getDecl()) if (vals[VD] != Initialized) // If the variable isn't initialized by the time we get here, then we // mark it as potentially uninitialized for those cases where it's used // on an indirect path, where it's not guaranteed to be defined. vals[VD] = MayUninitialized; + } } void TransferFunctions::VisitObjCMessageExpr(ObjCMessageExpr *ME) { diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp index e82a3a705e70..2d75578b3de0 100644 --- a/clang/lib/Basic/Cuda.cpp +++ b/clang/lib/Basic/Cuda.cpp @@ -123,6 +123,7 @@ static const CudaArchToStringMap arch_names[] = { GFX(1033), // gfx1033 GFX(1034), // gfx1034 GFX(1035), // gfx1035 + {CudaArch::Generic, "generic", ""}, // clang-format on }; #undef SM diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp index 9e74e05bd863..1761c6d3d89b 100644 --- a/clang/lib/Basic/OpenMPKinds.cpp +++ b/clang/lib/Basic/OpenMPKinds.cpp @@ -163,6 +163,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str, case OMPC_read: case OMPC_write: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -428,6 +429,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind, case OMPC_read: case OMPC_write: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: diff --git a/clang/lib/Basic/SourceLocation.cpp b/clang/lib/Basic/SourceLocation.cpp index 6986fcd322f2..6e5e55fb09ce 100644 --- a/clang/lib/Basic/SourceLocation.cpp +++ b/clang/lib/Basic/SourceLocation.cpp @@ -90,7 +90,7 @@ SourceLocation::printToString(const SourceManager &SM) const { std::string S; llvm::raw_string_ostream OS(S); print(OS, SM); - return OS.str(); + return S; } LLVM_DUMP_METHOD void SourceLocation::dump(const SourceManager &SM) const { @@ -149,7 +149,7 @@ SourceRange::printToString(const SourceManager &SM) const { std::string S; llvm::raw_string_ostream OS(S); print(OS, SM); - return OS.str(); + return S; } //===----------------------------------------------------------------------===// diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp index 4d403ae1809d..4089a393b762 100644 --- a/clang/lib/Basic/Targets/AArch64.cpp +++ b/clang/lib/Basic/Targets/AArch64.cpp @@ -307,6 +307,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts, if (FPU & SveMode) Builder.defineMacro("__ARM_FEATURE_SVE", "1"); + if ((FPU & NeonMode) && (FPU & SveMode)) + Builder.defineMacro("__ARM_NEON_SVE_BRIDGE", "1"); + if (HasSVE2) Builder.defineMacro("__ARM_FEATURE_SVE2", "1"); @@ -474,10 +477,12 @@ ArrayRef AArch64TargetInfo::getTargetBuiltins() const { Optional> AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const { if (LangOpts.VScaleMin || LangOpts.VScaleMax) - return std::pair(LangOpts.VScaleMin, - LangOpts.VScaleMax); + return std::pair( + LangOpts.VScaleMin ? LangOpts.VScaleMin : 1, LangOpts.VScaleMax); + if (hasFeature("sve")) - return std::pair(0, 16); + return std::pair(1, 16); + return None; } diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h index dea59a9b015d..74745df3be8d 100644 --- a/clang/lib/Basic/Targets/AArch64.h +++ b/clang/lib/Basic/Targets/AArch64.h @@ -150,7 +150,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo { const char *getBFloat16Mangling() const override { return "u6__bf16"; }; bool hasInt128Type() const override; - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; class LLVM_LIBRARY_VISIBILITY AArch64leTargetInfo : public AArch64TargetInfo { diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h index 8b9d7ce79c16..974922191488 100644 --- a/clang/lib/Basic/Targets/AMDGPU.h +++ b/clang/lib/Basic/Targets/AMDGPU.h @@ -426,7 +426,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo { void setAuxTarget(const TargetInfo *Aux) override; - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } // Record offload arch features since they are needed for defining the // pre-defined macros. diff --git a/clang/lib/Basic/Targets/ARC.h b/clang/lib/Basic/Targets/ARC.h index 3c0c5f6df2f4..5411cd2cd869 100644 --- a/clang/lib/Basic/Targets/ARC.h +++ b/clang/lib/Basic/Targets/ARC.h @@ -66,7 +66,7 @@ class LLVM_LIBRARY_VISIBILITY ARCTargetInfo : public TargetInfo { return false; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } bool isCLZForZeroUndef() const override { return false; } }; diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp index f330780300f2..c619d6cde41d 100644 --- a/clang/lib/Basic/Targets/ARM.cpp +++ b/clang/lib/Basic/Targets/ARM.cpp @@ -465,6 +465,8 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector &Features, HWDiv = 0; DotProd = 0; HasMatMul = 0; + HasPAC = 0; + HasBTI = 0; HasFloat16 = true; ARMCDECoprocMask = 0; HasBFloat16 = false; @@ -547,6 +549,9 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector &Features, HasBFloat16 = true; } else if (Feature == "-fpregs") { FPRegsDisabled = true; + } else if (Feature == "+pacbti") { + HasPAC = 1; + HasBTI = 1; } } @@ -890,6 +895,12 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts, if (HasMatMul) Builder.defineMacro("__ARM_FEATURE_MATMUL_INT8", "1"); + if (HasPAC) + Builder.defineMacro("__ARM_FEATURE_PAUTH", "1"); + + if (HasBTI) + Builder.defineMacro("__ARM_FEATURE_BTI", "1"); + if (HasBFloat16) { Builder.defineMacro("__ARM_FEATURE_BF16", "1"); Builder.defineMacro("__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "1"); @@ -900,7 +911,7 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__ARM_FEATURE_BTI_DEFAULT", "1"); if (Opts.hasSignReturnAddress()) { - unsigned Value = Opts.isSignReturnAddressWithAKey() ? 1 : 2; + unsigned Value = 1; if (Opts.isSignReturnAddressScopeAll()) Value |= 1 << 2; Builder.defineMacro("__ARM_FEATURE_PAC_DEFAULT", Twine(Value)); diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h index 7d0011d134ea..40c658f3f40e 100644 --- a/clang/lib/Basic/Targets/ARM.h +++ b/clang/lib/Basic/Targets/ARM.h @@ -79,6 +79,8 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo { unsigned DotProd : 1; unsigned HasMatMul : 1; unsigned FPRegsDisabled : 1; + unsigned HasPAC : 1; + unsigned HasBTI : 1; enum { LDREX_B = (1 << 0), /// byte (8-bit) @@ -191,8 +193,8 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo { bool hasSjLjLowering() const override; - bool hasExtIntType() const override { return true; } - + bool hasBitIntType() const override { return true; } + const char *getBFloat16Mangling() const override { return "u6__bf16"; }; }; diff --git a/clang/lib/Basic/Targets/Hexagon.cpp b/clang/lib/Basic/Targets/Hexagon.cpp index 9c37dee7e89a..161369242926 100644 --- a/clang/lib/Basic/Targets/Hexagon.cpp +++ b/clang/lib/Basic/Targets/Hexagon.cpp @@ -68,6 +68,9 @@ void HexagonTargetInfo::getTargetDefines(const LangOptions &Opts, } else if (CPU == "hexagonv68") { Builder.defineMacro("__HEXAGON_V68__"); Builder.defineMacro("__HEXAGON_ARCH__", "68"); + } else if (CPU == "hexagonv69") { + Builder.defineMacro("__HEXAGON_V69__"); + Builder.defineMacro("__HEXAGON_ARCH__", "69"); } if (hasFeature("hvx-length64b")) { @@ -128,6 +131,10 @@ bool HexagonTargetInfo::handleTargetFeatures(std::vector &Features, else if (F == "+audio") HasAudio = true; } + if (CPU.compare("hexagonv68") >= 0) { + HasLegalHalfType = true; + HasFloat16 = true; + } return true; } @@ -214,7 +221,7 @@ static constexpr CPUSuffix Suffixes[] = { {{"hexagonv60"}, {"60"}}, {{"hexagonv62"}, {"62"}}, {{"hexagonv65"}, {"65"}}, {{"hexagonv66"}, {"66"}}, {{"hexagonv67"}, {"67"}}, {{"hexagonv67t"}, {"67t"}}, - {{"hexagonv68"}, {"68"}}, + {{"hexagonv68"}, {"68"}}, {{"hexagonv69"}, {"69"}}, }; const char *HexagonTargetInfo::getHexagonCPUSuffix(StringRef Name) { diff --git a/clang/lib/Basic/Targets/Hexagon.h b/clang/lib/Basic/Targets/Hexagon.h index d6c7da5f1e40..94441998f355 100644 --- a/clang/lib/Basic/Targets/Hexagon.h +++ b/clang/lib/Basic/Targets/Hexagon.h @@ -139,7 +139,7 @@ class LLVM_LIBRARY_VISIBILITY HexagonTargetInfo : public TargetInfo { return CPU.find('t') != std::string::npos; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; } // namespace targets } // namespace clang diff --git a/clang/lib/Basic/Targets/Lanai.h b/clang/lib/Basic/Targets/Lanai.h index 9af5427b81c4..56c6cced938a 100644 --- a/clang/lib/Basic/Targets/Lanai.h +++ b/clang/lib/Basic/Targets/Lanai.h @@ -87,7 +87,7 @@ class LLVM_LIBRARY_VISIBILITY LanaiTargetInfo : public TargetInfo { const char *getClobbers() const override { return ""; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; } // namespace targets } // namespace clang diff --git a/clang/lib/Basic/Targets/Mips.h b/clang/lib/Basic/Targets/Mips.h index b475c03889a1..b54d36e1c95f 100644 --- a/clang/lib/Basic/Targets/Mips.h +++ b/clang/lib/Basic/Targets/Mips.h @@ -406,7 +406,7 @@ class LLVM_LIBRARY_VISIBILITY MipsTargetInfo : public TargetInfo { unsigned getUnwindWordWidth() const override; bool validateTarget(DiagnosticsEngine &Diags) const override; - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; } // namespace targets } // namespace clang diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp index 3561b22677bc..75e82d819900 100644 --- a/clang/lib/Basic/Targets/NVPTX.cpp +++ b/clang/lib/Basic/Targets/NVPTX.cpp @@ -215,6 +215,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts, case CudaArch::GFX1033: case CudaArch::GFX1034: case CudaArch::GFX1035: + case CudaArch::Generic: case CudaArch::LAST: break; case CudaArch::UNUSED: diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h index ef751b8e1a8d..589f24f4bb03 100644 --- a/clang/lib/Basic/Targets/NVPTX.h +++ b/clang/lib/Basic/Targets/NVPTX.h @@ -121,7 +121,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXTargetInfo : public TargetInfo { void fillValidCPUList(SmallVectorImpl &Values) const override { for (int i = static_cast(CudaArch::SM_20); - i < static_cast(CudaArch::LAST); ++i) + i < static_cast(CudaArch::Generic); ++i) Values.emplace_back(CudaArchToString(static_cast(i))); } @@ -175,7 +175,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXTargetInfo : public TargetInfo { return CCCR_Warning; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; } // namespace targets } // namespace clang diff --git a/clang/lib/Basic/Targets/OSTargets.cpp b/clang/lib/Basic/Targets/OSTargets.cpp index 53748bf067cd..f8f12daaa072 100644 --- a/clang/lib/Basic/Targets/OSTargets.cpp +++ b/clang/lib/Basic/Targets/OSTargets.cpp @@ -48,12 +48,12 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts, Builder.defineMacro("_REENTRANT"); // Get the platform type and version number from the triple. - unsigned Maj, Min, Rev; + VersionTuple OsVersion; if (Triple.isMacOSX()) { - Triple.getMacOSXVersion(Maj, Min, Rev); + Triple.getMacOSXVersion(OsVersion); PlatformName = "macos"; } else { - Triple.getOSVersion(Maj, Min, Rev); + OsVersion = Triple.getOSVersion(); PlatformName = llvm::Triple::getOSTypeName(Triple.getOS()); if (PlatformName == "ios" && Triple.isMacCatalystEnvironment()) PlatformName = "maccatalyst"; @@ -63,29 +63,29 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts, // generating code for Win32 ABI. No need to emit // __ENVIRONMENT_XX_OS_VERSION_MIN_REQUIRED__. if (PlatformName == "win32") { - PlatformMinVersion = VersionTuple(Maj, Min, Rev); + PlatformMinVersion = OsVersion; return; } // Set the appropriate OS version define. if (Triple.isiOS()) { - assert(Maj < 100 && Min < 100 && Rev < 100 && "Invalid version!"); + assert(OsVersion < VersionTuple(100) && "Invalid version!"); char Str[7]; - if (Maj < 10) { - Str[0] = '0' + Maj; - Str[1] = '0' + (Min / 10); - Str[2] = '0' + (Min % 10); - Str[3] = '0' + (Rev / 10); - Str[4] = '0' + (Rev % 10); + if (OsVersion.getMajor() < 10) { + Str[0] = '0' + OsVersion.getMajor(); + Str[1] = '0' + (OsVersion.getMinor().getValueOr(0) / 10); + Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) % 10); + Str[3] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10); + Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10); Str[5] = '\0'; } else { // Handle versions >= 10. - Str[0] = '0' + (Maj / 10); - Str[1] = '0' + (Maj % 10); - Str[2] = '0' + (Min / 10); - Str[3] = '0' + (Min % 10); - Str[4] = '0' + (Rev / 10); - Str[5] = '0' + (Rev % 10); + Str[0] = '0' + (OsVersion.getMajor() / 10); + Str[1] = '0' + (OsVersion.getMajor() % 10); + Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) / 10); + Str[3] = '0' + (OsVersion.getMinor().getValueOr(0) % 10); + Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10); + Str[5] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10); Str[6] = '\0'; } if (Triple.isTvOS()) @@ -95,13 +95,13 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts, Str); } else if (Triple.isWatchOS()) { - assert(Maj < 10 && Min < 100 && Rev < 100 && "Invalid version!"); + assert(OsVersion < VersionTuple(10) && "Invalid version!"); char Str[6]; - Str[0] = '0' + Maj; - Str[1] = '0' + (Min / 10); - Str[2] = '0' + (Min % 10); - Str[3] = '0' + (Rev / 10); - Str[4] = '0' + (Rev % 10); + Str[0] = '0' + OsVersion.getMajor(); + Str[1] = '0' + (OsVersion.getMinor().getValueOr(0) / 10); + Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) % 10); + Str[3] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10); + Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10); Str[5] = '\0'; Builder.defineMacro("__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__", Str); } else if (Triple.isMacOSX()) { @@ -109,22 +109,22 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts, // define (because we only get a single digit for the minor and micro // revision numbers). So, we limit them to the maximum representable // version. - assert(Maj < 100 && Min < 100 && Rev < 100 && "Invalid version!"); + assert(OsVersion < VersionTuple(100) && "Invalid version!"); char Str[7]; - if (Maj < 10 || (Maj == 10 && Min < 10)) { - Str[0] = '0' + (Maj / 10); - Str[1] = '0' + (Maj % 10); - Str[2] = '0' + std::min(Min, 9U); - Str[3] = '0' + std::min(Rev, 9U); + if (OsVersion < VersionTuple(10, 10)) { + Str[0] = '0' + (OsVersion.getMajor() / 10); + Str[1] = '0' + (OsVersion.getMajor() % 10); + Str[2] = '0' + std::min(OsVersion.getMinor().getValueOr(0), 9U); + Str[3] = '0' + std::min(OsVersion.getSubminor().getValueOr(0), 9U); Str[4] = '\0'; } else { // Handle versions > 10.9. - Str[0] = '0' + (Maj / 10); - Str[1] = '0' + (Maj % 10); - Str[2] = '0' + (Min / 10); - Str[3] = '0' + (Min % 10); - Str[4] = '0' + (Rev / 10); - Str[5] = '0' + (Rev % 10); + Str[0] = '0' + (OsVersion.getMajor() / 10); + Str[1] = '0' + (OsVersion.getMajor() % 10); + Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) / 10); + Str[3] = '0' + (OsVersion.getMinor().getValueOr(0) % 10); + Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10); + Str[5] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10); Str[6] = '\0'; } Builder.defineMacro("__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__", Str); @@ -134,7 +134,7 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts, if (Triple.isOSDarwin()) Builder.defineMacro("__MACH__"); - PlatformMinVersion = VersionTuple(Maj, Min, Rev); + PlatformMinVersion = OsVersion; } static void addMinGWDefines(const llvm::Triple &Triple, const LangOptions &Opts, @@ -203,6 +203,7 @@ static void addVisualCDefines(const LangOptions &Opts, MacroBuilder &Builder) { } Builder.defineMacro("_INTEGRAL_MAX_BITS", "64"); + Builder.defineMacro("__STDC_NO_THREADS__"); // Starting with VS 2022 17.1, MSVC predefines the below macro to inform // users of the execution character set defined at compile time. diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h index 7fbe2cbc5653..3c1830d5f8e8 100644 --- a/clang/lib/Basic/Targets/OSTargets.h +++ b/clang/lib/Basic/Targets/OSTargets.h @@ -148,9 +148,7 @@ class LLVM_LIBRARY_VISIBILITY DarwinTargetInfo : public OSTargetInfo { return 64; } - unsigned Major, Minor, Micro; - T.getOSVersion(Major, Minor, Micro); - if (llvm::VersionTuple(Major, Minor, Micro) < MinVersion) + if (T.getOSVersion() < MinVersion) return 64; return OSTargetInfo::getExnObjectAlignment(); } @@ -294,7 +292,7 @@ class LLVM_LIBRARY_VISIBILITY HaikuTargetInfo : public OSTargetInfo { Builder.defineMacro("__HAIKU__"); Builder.defineMacro("__ELF__"); DefineStd(Builder, "unix", Opts); - if (this->HasFloat128) + if (this->HasFloat128) Builder.defineMacro("__FLOAT128__"); } @@ -376,10 +374,9 @@ class LLVM_LIBRARY_VISIBILITY LinuxTargetInfo : public OSTargetInfo { Builder.defineMacro("__ELF__"); if (Triple.isAndroid()) { Builder.defineMacro("__ANDROID__", "1"); - unsigned Maj, Min, Rev; - Triple.getEnvironmentVersion(Maj, Min, Rev); this->PlatformName = "android"; - this->PlatformMinVersion = VersionTuple(Maj, Min, Rev); + this->PlatformMinVersion = Triple.getEnvironmentVersion(); + const unsigned Maj = this->PlatformMinVersion.getMajor(); if (Maj) { Builder.defineMacro("__ANDROID_MIN_SDK_VERSION__", Twine(Maj)); // This historical but ambiguous name for the minSdkVersion macro. Keep @@ -693,23 +690,32 @@ class AIXTargetInfo : public OSTargetInfo { if (Opts.EnableAIXExtendedAltivecABI) Builder.defineMacro("__EXTABI__"); - unsigned Major, Minor, Micro; - Triple.getOSVersion(Major, Minor, Micro); + VersionTuple OsVersion = Triple.getOSVersion(); // Define AIX OS-Version Macros. // Includes logic for legacy versions of AIX; no specific intent to support. - std::pair OsVersion = {Major, Minor}; - if (OsVersion >= std::make_pair(3, 2)) Builder.defineMacro("_AIX32"); - if (OsVersion >= std::make_pair(4, 1)) Builder.defineMacro("_AIX41"); - if (OsVersion >= std::make_pair(4, 3)) Builder.defineMacro("_AIX43"); - if (OsVersion >= std::make_pair(5, 0)) Builder.defineMacro("_AIX50"); - if (OsVersion >= std::make_pair(5, 1)) Builder.defineMacro("_AIX51"); - if (OsVersion >= std::make_pair(5, 2)) Builder.defineMacro("_AIX52"); - if (OsVersion >= std::make_pair(5, 3)) Builder.defineMacro("_AIX53"); - if (OsVersion >= std::make_pair(6, 1)) Builder.defineMacro("_AIX61"); - if (OsVersion >= std::make_pair(7, 1)) Builder.defineMacro("_AIX71"); - if (OsVersion >= std::make_pair(7, 2)) Builder.defineMacro("_AIX72"); - if (OsVersion >= std::make_pair(7, 3)) Builder.defineMacro("_AIX73"); + if (OsVersion >= VersionTuple(3, 2)) + Builder.defineMacro("_AIX32"); + if (OsVersion >= VersionTuple(4, 1)) + Builder.defineMacro("_AIX41"); + if (OsVersion >= VersionTuple(4, 3)) + Builder.defineMacro("_AIX43"); + if (OsVersion >= VersionTuple(5, 0)) + Builder.defineMacro("_AIX50"); + if (OsVersion >= VersionTuple(5, 1)) + Builder.defineMacro("_AIX51"); + if (OsVersion >= VersionTuple(5, 2)) + Builder.defineMacro("_AIX52"); + if (OsVersion >= VersionTuple(5, 3)) + Builder.defineMacro("_AIX53"); + if (OsVersion >= VersionTuple(6, 1)) + Builder.defineMacro("_AIX61"); + if (OsVersion >= VersionTuple(7, 1)) + Builder.defineMacro("_AIX71"); + if (OsVersion >= VersionTuple(7, 2)) + Builder.defineMacro("_AIX72"); + if (OsVersion >= VersionTuple(7, 3)) + Builder.defineMacro("_AIX73"); // FIXME: Do not define _LONG_LONG when -fno-long-long is specified. Builder.defineMacro("_LONG_LONG"); diff --git a/clang/lib/Basic/Targets/PNaCl.h b/clang/lib/Basic/Targets/PNaCl.h index d5bfc369583f..b5cf73d73e95 100644 --- a/clang/lib/Basic/Targets/PNaCl.h +++ b/clang/lib/Basic/Targets/PNaCl.h @@ -69,7 +69,7 @@ class LLVM_LIBRARY_VISIBILITY PNaClTargetInfo : public TargetInfo { const char *getClobbers() const override { return ""; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; // We attempt to use PNaCl (le32) frontend and Mips32EL backend. diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp index c3c61ed443ca..7f7b44b658eb 100644 --- a/clang/lib/Basic/Targets/PPC.cpp +++ b/clang/lib/Basic/Targets/PPC.cpp @@ -623,14 +623,11 @@ void PPCTargetInfo::addP10SpecificFeatures( Features["pcrelative-memops"] = true; Features["prefix-instrs"] = true; Features["isa-v31-instructions"] = true; - return; } // Add features specific to the "Future" CPU. void PPCTargetInfo::addFutureSpecificFeatures( - llvm::StringMap &Features) const { - return; -} + llvm::StringMap &Features) const {} bool PPCTargetInfo::hasFeature(StringRef Feature) const { return llvm::StringSwitch(Feature) diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h index f19d3ebbc066..60701072ac4b 100644 --- a/clang/lib/Basic/Targets/PPC.h +++ b/clang/lib/Basic/Targets/PPC.h @@ -351,7 +351,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo { const char *getFloat128Mangling() const override { return "u9__ieee128"; } const char *getIbm128Mangling() const override { return "g"; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } bool isSPRegName(StringRef RegName) const override { return RegName.equals("r1") || RegName.equals("x1"); diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp index 93562dde2f54..770d37a1c1be 100644 --- a/clang/lib/Basic/Targets/RISCV.cpp +++ b/clang/lib/Basic/Targets/RISCV.cpp @@ -246,6 +246,9 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector &Features, ISAInfo = std::move(*ParseResult); } + if (ABI.empty()) + ABI = llvm::RISCV::computeDefaultABIFromArch(*ISAInfo).str(); + return true; } diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h index f7ffe9febcd0..5331ed4a50ae 100644 --- a/clang/lib/Basic/Targets/RISCV.h +++ b/clang/lib/Basic/Targets/RISCV.h @@ -95,7 +95,7 @@ class RISCVTargetInfo : public TargetInfo { bool handleTargetFeatures(std::vector &Features, DiagnosticsEngine &Diags) override; - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo { public: diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h index 8cf18b6c20f1..a40d4b3ca27e 100644 --- a/clang/lib/Basic/Targets/SPIR.h +++ b/clang/lib/Basic/Targets/SPIR.h @@ -162,7 +162,7 @@ class LLVM_LIBRARY_VISIBILITY BaseSPIRTargetInfo : public TargetInfo { supportAllOpenCLOpts(); } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } bool hasInt128Type() const override { return false; } }; diff --git a/clang/lib/Basic/Targets/Sparc.h b/clang/lib/Basic/Targets/Sparc.h index 22a1621fcb9f..177a117520da 100644 --- a/clang/lib/Basic/Targets/Sparc.h +++ b/clang/lib/Basic/Targets/Sparc.h @@ -48,8 +48,6 @@ class LLVM_LIBRARY_VISIBILITY SparcTargetInfo : public TargetInfo { bool hasFeature(StringRef Feature) const override; - bool hasSjLjLowering() const override { return true; } - ArrayRef getTargetBuiltins() const override { // FIXME: Implement! return None; @@ -178,8 +176,7 @@ class LLVM_LIBRARY_VISIBILITY SparcV8TargetInfo : public SparcTargetInfo { void getTargetDefines(const LangOptions &Opts, MacroBuilder &Builder) const override; - bool hasSjLjLowering() const override { return true; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; // SPARCV8el is the 32-bit little-endian mode selected by Triple::sparcel. @@ -232,7 +229,7 @@ class LLVM_LIBRARY_VISIBILITY SparcV9TargetInfo : public SparcTargetInfo { return getCPUGeneration(CPU) == CG_V9; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; } // namespace targets } // namespace clang diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h index d3e3ed50dd47..92cefeea5d26 100644 --- a/clang/lib/Basic/Targets/SystemZ.h +++ b/clang/lib/Basic/Targets/SystemZ.h @@ -170,7 +170,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo { const char *getLongDoubleMangling() const override { return "g"; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } int getEHDataRegisterNumber(unsigned RegNo) const override { return RegNo < 4 ? 6 + RegNo : -1; diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h index 16534d3ef99b..075486990558 100644 --- a/clang/lib/Basic/Targets/WebAssembly.h +++ b/clang/lib/Basic/Targets/WebAssembly.h @@ -137,7 +137,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo { } } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } bool hasProtectedVisibility() const override { return false; } diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h index b9b2ac79815b..c952b8c9a336 100644 --- a/clang/lib/Basic/Targets/X86.h +++ b/clang/lib/Basic/Targets/X86.h @@ -462,7 +462,7 @@ class LLVM_LIBRARY_VISIBILITY X86_32TargetInfo : public X86TargetInfo { ArrayRef getTargetBuiltins() const override; - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; class LLVM_LIBRARY_VISIBILITY NetBSDI386TargetInfo @@ -472,10 +472,9 @@ class LLVM_LIBRARY_VISIBILITY NetBSDI386TargetInfo : NetBSDTargetInfo(Triple, Opts) {} unsigned getFloatEvalMethod() const override { - unsigned Major, Minor, Micro; - getTriple().getOSVersion(Major, Minor, Micro); + VersionTuple OsVersion = getTriple().getOSVersion(); // New NetBSD uses the default rounding mode. - if (Major >= 7 || (Major == 6 && Minor == 99 && Micro >= 26) || Major == 0) + if (OsVersion >= VersionTuple(6, 99, 26) || OsVersion.getMajor() == 0) return X86_32TargetInfo::getFloatEvalMethod(); // NetBSD before 6.99.26 defaults to "double" rounding. return 1; @@ -769,7 +768,7 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo { ArrayRef getTargetBuiltins() const override; - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; // x86-64 Windows target diff --git a/clang/lib/Basic/Targets/XCore.h b/clang/lib/Basic/Targets/XCore.h index c33766751aa1..25f20581839d 100644 --- a/clang/lib/Basic/Targets/XCore.h +++ b/clang/lib/Basic/Targets/XCore.h @@ -76,7 +76,7 @@ class LLVM_LIBRARY_VISIBILITY XCoreTargetInfo : public TargetInfo { bool allowsLargerPreferedTypeAlignment() const override { return false; } - bool hasExtIntType() const override { return true; } + bool hasBitIntType() const override { return true; } }; } // namespace targets } // namespace clang diff --git a/clang/lib/Basic/Version.cpp b/clang/lib/Basic/Version.cpp index af3118b0f6da..e205da7adec1 100644 --- a/clang/lib/Basic/Version.cpp +++ b/clang/lib/Basic/Version.cpp @@ -82,7 +82,7 @@ std::string getClangFullRepositoryVersion() { OS << LLVMRepo << ' '; OS << LLVMRev << ')'; } - return OS.str(); + return buf; } std::string getClangFullVersion() { @@ -102,7 +102,7 @@ std::string getClangToolFullVersion(StringRef ToolName) { OS << " " << repo; } - return OS.str(); + return buf; } std::string getClangFullCPPVersion() { @@ -120,7 +120,7 @@ std::string getClangFullCPPVersion() { OS << " " << repo; } - return OS.str(); + return buf; } } // end namespace clang diff --git a/clang/lib/CodeGen/ABIInfo.h b/clang/lib/CodeGen/ABIInfo.h index 56f0dd4322d2..0d12183055e1 100644 --- a/clang/lib/CodeGen/ABIInfo.h +++ b/clang/lib/CodeGen/ABIInfo.h @@ -105,7 +105,7 @@ namespace swiftcall { uint64_t &Members) const; // Implement the Type::IsPromotableIntegerType for ABI specific needs. The - // only difference is that this considers _ExtInt as well. + // only difference is that this considers bit-precise integer types as well. bool isPromotableIntegerTypeForABI(QualType Ty) const; /// A convenience method to return an indirect ABIArgInfo with an diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h index 6a8e57f8db33..37c20291c0e8 100644 --- a/clang/lib/CodeGen/Address.h +++ b/clang/lib/CodeGen/Address.h @@ -23,15 +23,29 @@ namespace CodeGen { /// An aligned address. class Address { llvm::Value *Pointer; + llvm::Type *ElementType; CharUnits Alignment; + +protected: + Address(std::nullptr_t) : Pointer(nullptr), ElementType(nullptr) {} + public: - Address(llvm::Value *pointer, CharUnits alignment) - : Pointer(pointer), Alignment(alignment) { - assert((!alignment.isZero() || pointer == nullptr) && - "creating valid address with invalid alignment"); + Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment) + : Pointer(pointer), ElementType(elementType), Alignment(alignment) { + assert(pointer != nullptr && "Pointer cannot be null"); + assert(elementType != nullptr && "Element type cannot be null"); + assert(llvm::cast(pointer->getType()) + ->isOpaqueOrPointeeTypeMatches(elementType) && + "Incorrect pointer element type"); + assert(!alignment.isZero() && "Alignment cannot be zero"); } - static Address invalid() { return Address(nullptr, CharUnits()); } + // Deprecated: Use constructor with explicit element type instead. + Address(llvm::Value *Pointer, CharUnits Alignment) + : Address(Pointer, Pointer->getType()->getPointerElementType(), + Alignment) {} + + static Address invalid() { return Address(nullptr); } bool isValid() const { return Pointer != nullptr; } llvm::Value *getPointer() const { @@ -45,11 +59,9 @@ class Address { } /// Return the type of the values stored in this address. - /// - /// When IR pointer types lose their element type, we should simply - /// store it in Address instead for the convenience of writing code. llvm::Type *getElementType() const { - return getType()->getElementType(); + assert(isValid()); + return ElementType; } /// Return the address space that this address resides in. @@ -67,30 +79,42 @@ class Address { assert(isValid()); return Alignment; } + + /// Return address with different pointer, but same element type and + /// alignment. + Address withPointer(llvm::Value *NewPointer) const { + return Address(NewPointer, ElementType, Alignment); + } + + /// Return address with different alignment, but same pointer and element + /// type. + Address withAlignment(CharUnits NewAlignment) const { + return Address(Pointer, ElementType, NewAlignment); + } }; /// A specialization of Address that requires the address to be an /// LLVM Constant. class ConstantAddress : public Address { + ConstantAddress(std::nullptr_t) : Address(nullptr) {} + public: - ConstantAddress(llvm::Constant *pointer, CharUnits alignment) - : Address(pointer, alignment) {} + ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType, + CharUnits alignment) + : Address(pointer, elementType, alignment) {} static ConstantAddress invalid() { - return ConstantAddress(nullptr, CharUnits()); + return ConstantAddress(nullptr); } llvm::Constant *getPointer() const { return llvm::cast(Address::getPointer()); } - ConstantAddress getBitCast(llvm::Type *ty) const { - return ConstantAddress(llvm::ConstantExpr::getBitCast(getPointer(), ty), - getAlignment()); - } - - ConstantAddress getElementBitCast(llvm::Type *ty) const { - return getBitCast(ty->getPointerTo(getAddressSpace())); + ConstantAddress getElementBitCast(llvm::Type *ElemTy) const { + llvm::Constant *BitCast = llvm::ConstantExpr::getBitCast( + getPointer(), ElemTy->getPointerTo(getAddressSpace())); + return ConstantAddress(BitCast, ElemTy, getAlignment()); } static bool isaImpl(Address addr) { @@ -98,7 +122,7 @@ class ConstantAddress : public Address { } static ConstantAddress castImpl(Address addr) { return ConstantAddress(llvm::cast(addr.getPointer()), - addr.getAlignment()); + addr.getElementType(), addr.getAlignment()); } }; diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 510f3911939c..bacac0a20d4d 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -94,10 +94,16 @@ using namespace llvm; llvm::PassPluginLibraryInfo get##Ext##PluginInfo(); #include "llvm/Support/Extension.def" +namespace llvm { +extern cl::opt DebugInfoCorrelate; +} + namespace { // Default filename used for profile generation. -static constexpr StringLiteral DefaultProfileGenName = "default_%m.profraw"; +std::string getDefaultProfileGenName() { + return DebugInfoCorrelate ? "default_%p.proflite" : "default_%m.profraw"; +} class EmitAssemblyHelper { DiagnosticsEngine &Diags; @@ -597,8 +603,6 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection; Options.EmitCallSiteInfo = CodeGenOpts.EmitCallSiteInfo; Options.EnableAIXExtendedAltivecABI = CodeGenOpts.EnableAIXExtendedAltivecABI; - Options.ValueTrackingVariableLocations = - CodeGenOpts.ValueTrackingVariableLocations; Options.XRayOmitFunctionIndex = CodeGenOpts.XRayOmitFunctionIndex; Options.LoopAlignment = CodeGenOpts.LoopAlignment; @@ -640,6 +644,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Options.MCOptions.Argv0 = CodeGenOpts.Argv0; Options.MCOptions.CommandLineArgs = CodeGenOpts.CommandLineArgs; Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf; + Options.ObjectFilenameForDebug = CodeGenOpts.ObjectFilenameForDebug; return true; } @@ -886,7 +891,7 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, if (!CodeGenOpts.InstrProfileOutput.empty()) PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput; else - PMBuilder.PGOInstrGen = std::string(DefaultProfileGenName); + PMBuilder.PGOInstrGen = getDefaultProfileGenName(); } if (CodeGenOpts.hasProfileIRUse()) { PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath; @@ -1231,7 +1236,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline( if (CodeGenOpts.hasProfileIRInstr()) // -fprofile-generate. PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty() - ? std::string(DefaultProfileGenName) + ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput, "", "", PGOOptions::IRInstr, PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling); @@ -1269,13 +1274,13 @@ void EmitAssemblyHelper::RunOptimizationPipeline( "Cannot run CSProfileGen pass with ProfileGen or SampleUse " " pass"); PGOOpt->CSProfileGenFile = CodeGenOpts.InstrProfileOutput.empty() - ? std::string(DefaultProfileGenName) + ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput; PGOOpt->CSAction = PGOOptions::CSIRInstr; } else PGOOpt = PGOOptions("", CodeGenOpts.InstrProfileOutput.empty() - ? std::string(DefaultProfileGenName) + ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput, "", PGOOptions::NoAction, PGOOptions::CSIRInstr, CodeGenOpts.DebugInfoForProfiling); @@ -1577,7 +1582,8 @@ static void runThinLTOBackend( return; auto AddStream = [&](size_t Task) { - return std::make_unique(std::move(OS)); + return std::make_unique(std::move(OS), + CGOpts.ObjectFilenameForDebug); }; lto::Config Conf; if (CGOpts.SaveTempsFilePrefix != "") { diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index b68e6328acdf..e81c5ba5055c 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -1079,8 +1079,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { if (AS == LangAS::opencl_generic) return V; auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); - auto T = V->getType(); - auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); + auto T = llvm::cast(V->getType()); + auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS); return getTargetHooks().performAddrSpaceCast( *this, V, AS, LangAS::opencl_generic, DestType, false); @@ -1321,15 +1321,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { ResVal = Builder.CreateNot(ResVal); Builder.CreateStore( - ResVal, - Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); + ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType())); } if (RValTy->isVoidType()) return RValue::get(nullptr); return convertTempToRValue( - Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), + Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), RValTy, E->getExprLoc()); } @@ -1382,8 +1381,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { return RValue::get(nullptr); return convertTempToRValue( - Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( - Dest.getAddressSpace())), + Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), RValTy, E->getExprLoc()); } @@ -1455,17 +1453,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); return convertTempToRValue( - Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( - Dest.getAddressSpace())), + Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), RValTy, E->getExprLoc()); } Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { - unsigned addrspace = - cast(addr.getPointer()->getType())->getAddressSpace(); llvm::IntegerType *ty = llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); - return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); + return CGF.Builder.CreateElementBitCast(addr, ty); } Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp index 2da2014345d8..7bb6dbb8a8ac 100644 --- a/clang/lib/CodeGen/CGBlocks.cpp +++ b/clang/lib/CodeGen/CGBlocks.cpp @@ -2721,8 +2721,7 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { Address addr = emission.Addr; // That's an alloca of the byref structure type. - llvm::StructType *byrefType = cast( - cast(addr.getPointer()->getType())->getElementType()); + llvm::StructType *byrefType = cast(addr.getElementType()); unsigned nextHeaderIndex = 0; CharUnits nextHeaderOffset; diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h index 4fad44a105cd..7c9f41e84eaf 100644 --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -86,7 +86,8 @@ class CGBuilderTy : public CGBuilderBaseTy { llvm::LoadInst *CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name = "") { - assert(Addr->getType()->getPointerElementType() == Ty); + assert(llvm::cast(Addr->getType()) + ->isOpaqueOrPointeeTypeMatches(Ty)); return CreateAlignedLoad(Ty, Addr, Align.getAsAlign(), Name); } @@ -115,13 +116,15 @@ class CGBuilderTy : public CGBuilderBaseTy { /// Emit a load from an i1 flag variable. llvm::LoadInst *CreateFlagLoad(llvm::Value *Addr, const llvm::Twine &Name = "") { - assert(Addr->getType()->getPointerElementType() == getInt1Ty()); + assert(llvm::cast(Addr->getType()) + ->isOpaqueOrPointeeTypeMatches(getInt1Ty())); return CreateAlignedLoad(getInt1Ty(), Addr, CharUnits::One(), Name); } /// Emit a store to an i1 flag variable. llvm::StoreInst *CreateFlagStore(bool Value, llvm::Value *Addr) { - assert(Addr->getType()->getPointerElementType() == getInt1Ty()); + assert(llvm::cast(Addr->getType()) + ->isOpaqueOrPointeeTypeMatches(getInt1Ty())); return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One()); } @@ -165,8 +168,9 @@ class CGBuilderTy : public CGBuilderBaseTy { /// preserving information like the alignment and address space. Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name = "") { - auto PtrTy = Ty->getPointerTo(Addr.getAddressSpace()); - return CreateBitCast(Addr, PtrTy, Name); + auto *PtrTy = Ty->getPointerTo(Addr.getAddressSpace()); + return Address(CreateBitCast(Addr.getPointer(), PtrTy, Name), + Ty, Addr.getAlignment()); } using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; @@ -194,6 +198,7 @@ class CGBuilderTy : public CGBuilderBaseTy { return Address(CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), + ElTy->getElementType(Index), Addr.getAlignment().alignmentAtOffset(Offset)); } @@ -215,6 +220,7 @@ class CGBuilderTy : public CGBuilderBaseTy { return Address( CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), {getSize(CharUnits::Zero()), getSize(Index)}, Name), + ElTy->getElementType(), Addr.getAlignment().alignmentAtOffset(Index * EltSize)); } @@ -231,6 +237,7 @@ class CGBuilderTy : public CGBuilderBaseTy { return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), getSize(Index), Name), + ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize)); } @@ -247,15 +254,32 @@ class CGBuilderTy : public CGBuilderBaseTy { return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), getSize(Index), Name), + Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Index * EltSize)); } + /// Create GEP with single dynamic index. The address alignment is reduced + /// according to the element size. + using CGBuilderBaseTy::CreateGEP; + Address CreateGEP(Address Addr, llvm::Value *Index, + const llvm::Twine &Name = "") { + const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + CharUnits EltSize = + CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); + + return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, + Name), + Addr.getElementType(), + Addr.getAlignment().alignmentOfArrayElement(EltSize)); + } + /// Given a pointer to i8, adjust it by a given constant offset. Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), getSize(Offset), Name), + Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset)); } Address CreateConstByteGEP(Address Addr, CharUnits Offset, @@ -263,6 +287,7 @@ class CGBuilderTy : public CGBuilderBaseTy { assert(Addr.getElementType() == TypeCache.Int8Ty); return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), getSize(Offset), Name), + Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset)); } @@ -278,8 +303,9 @@ class CGBuilderTy : public CGBuilderBaseTy { /*isSigned=*/true); if (!GEP->accumulateConstantOffset(DL, Offset)) llvm_unreachable("offset of GEP with constants is always computable"); - return Address(GEP, Addr.getAlignment().alignmentAtOffset( - CharUnits::fromQuantity(Offset.getSExtValue()))); + return Address(GEP, GEP->getResultElementType(), + Addr.getAlignment().alignmentAtOffset( + CharUnits::fromQuantity(Offset.getSExtValue()))); } using CGBuilderBaseTy::CreateMemCpy; @@ -330,8 +356,14 @@ class CGBuilderTy : public CGBuilderBaseTy { return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(), Index, FieldIndex, DbgInfo), + ElTy->getElementType(Index), Addr.getAlignment().alignmentAtOffset(Offset)); } + + using CGBuilderBaseTy::CreateLaunderInvariantGroup; + Address CreateLaunderInvariantGroup(Address Addr) { + return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer())); + } }; } // end namespace CodeGen diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 5d6df59cc405..1982b40ff667 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -96,13 +96,33 @@ llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, StringRef Name; GlobalDecl D(FD); + // TODO: This list should be expanded or refactored after all GCC-compatible + // std libcall builtins are implemented. + static SmallDenseMap F128Builtins{ + {Builtin::BI__builtin_printf, "__printfieee128"}, + {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"}, + {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"}, + {Builtin::BI__builtin_sprintf, "__sprintfieee128"}, + {Builtin::BI__builtin_snprintf, "__snprintfieee128"}, + {Builtin::BI__builtin_fprintf, "__fprintfieee128"}, + {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"}, + }; + // If the builtin has been declared explicitly with an assembler label, // use the mangled name. This differs from the plain label on platforms // that prefix labels. if (FD->hasAttr()) Name = getMangledName(D); - else - Name = Context.BuiltinInfo.getName(BuiltinID) + 10; + else { + // TODO: This mutation should also be applied to other targets other than + // PPC, after backend supports IEEE 128-bit style libcalls. + if (getTriple().isPPC64() && + &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() && + F128Builtins.find(BuiltinID) != F128Builtins.end()) + Name = F128Builtins[BuiltinID]; + else + Name = Context.BuiltinInfo.getName(BuiltinID) + 10; + } llvm::FunctionType *Ty = cast(getTypes().ConvertType(FD->getType())); @@ -667,7 +687,7 @@ getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type) { assert(Type->isIntegerType() && "Given type is not an integer."); unsigned Width = Type->isBooleanType() ? 1 - : Type->isExtIntType() ? context.getIntWidth(Type) + : Type->isBitIntType() ? context.getIntWidth(Type) : context.getTypeInfo(Type).Width; bool Signed = Type->isSignedIntegerType(); return {Width, Signed}; @@ -1482,8 +1502,7 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, Value *ArgValue = EmitScalarExpr(E->getArg(1)); llvm::Type *ArgType = ArgValue->getType(); - llvm::Type *IndexType = - IndexAddress.getPointer()->getType()->getPointerElementType(); + llvm::Type *IndexType = IndexAddress.getElementType(); llvm::Type *ResultType = ConvertType(E->getType()); Value *ArgZero = llvm::Constant::getNullValue(ArgType); @@ -3113,6 +3132,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, "elt.abs"); return RValue::get(Result); } + + case Builtin::BI__builtin_elementwise_ceil: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + Value *Result = Builder.CreateUnaryIntrinsic(llvm::Intrinsic::ceil, Op0, + nullptr, "elt.ceil"); + return RValue::get(Result); + } + case Builtin::BI__builtin_elementwise_max: { Value *Op0 = EmitScalarExpr(E->getArg(0)); Value *Op1 = EmitScalarExpr(E->getArg(1)); @@ -3184,6 +3211,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return RValue::get(Result); } + case Builtin::BI__builtin_reduce_xor: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + Value *Result = Builder.CreateUnaryIntrinsic( + llvm::Intrinsic::vector_reduce_xor, Op0, nullptr, "rdx.xor"); + return RValue::get(Result); + } + case Builtin::BI__builtin_matrix_transpose: { const auto *MatrixTy = E->getArg(0)->getType()->getAs(); Value *MatValue = EmitScalarExpr(E->getArg(0)); @@ -4478,6 +4512,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, } case Builtin::BI__builtin_addressof: return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this)); + case Builtin::BI__builtin_function_start: + return RValue::get(CGM.GetFunctionStart( + E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext()))); case Builtin::BI__builtin_operator_new: return EmitBuiltinNewDeleteCall( E->getCallee()->getType()->castAs(), E, false); @@ -4674,8 +4711,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return EmitCoroutineIntrinsic(E, Intrinsic::coro_end); case Builtin::BI__builtin_coro_suspend: return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend); - case Builtin::BI__builtin_coro_param: - return EmitCoroutineIntrinsic(E, Intrinsic::coro_param); // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions case Builtin::BIread_pipe: @@ -5221,9 +5256,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Type *BPP = Int8PtrPtrTy; DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"), - DestAddr.getAlignment()); + Int8PtrTy, DestAddr.getAlignment()); SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"), - SrcAddr.getAlignment()); + Int8PtrTy, SrcAddr.getAlignment()); Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); return RValue::get(Builder.CreateStore(ArgPtr, DestAddr)); @@ -6385,6 +6420,7 @@ static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = { static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = { #define GET_SVE_LLVM_INTRINSIC_MAP #include "clang/Basic/arm_sve_builtin_cg.inc" +#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def" #undef GET_SVE_LLVM_INTRINSIC_MAP }; @@ -9308,6 +9344,54 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, VTy); return Builder.CreateCall(F, {V0, V1, Ops[1]}); } + + case SVE::BI__builtin_sve_svset_neonq_s8: + case SVE::BI__builtin_sve_svset_neonq_s16: + case SVE::BI__builtin_sve_svset_neonq_s32: + case SVE::BI__builtin_sve_svset_neonq_s64: + case SVE::BI__builtin_sve_svset_neonq_u8: + case SVE::BI__builtin_sve_svset_neonq_u16: + case SVE::BI__builtin_sve_svset_neonq_u32: + case SVE::BI__builtin_sve_svset_neonq_u64: + case SVE::BI__builtin_sve_svset_neonq_f16: + case SVE::BI__builtin_sve_svset_neonq_f32: + case SVE::BI__builtin_sve_svset_neonq_f64: + case SVE::BI__builtin_sve_svset_neonq_bf16: { + return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0)); + } + + case SVE::BI__builtin_sve_svget_neonq_s8: + case SVE::BI__builtin_sve_svget_neonq_s16: + case SVE::BI__builtin_sve_svget_neonq_s32: + case SVE::BI__builtin_sve_svget_neonq_s64: + case SVE::BI__builtin_sve_svget_neonq_u8: + case SVE::BI__builtin_sve_svget_neonq_u16: + case SVE::BI__builtin_sve_svget_neonq_u32: + case SVE::BI__builtin_sve_svget_neonq_u64: + case SVE::BI__builtin_sve_svget_neonq_f16: + case SVE::BI__builtin_sve_svget_neonq_f32: + case SVE::BI__builtin_sve_svget_neonq_f64: + case SVE::BI__builtin_sve_svget_neonq_bf16: { + return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0)); + } + + case SVE::BI__builtin_sve_svdup_neonq_s8: + case SVE::BI__builtin_sve_svdup_neonq_s16: + case SVE::BI__builtin_sve_svdup_neonq_s32: + case SVE::BI__builtin_sve_svdup_neonq_s64: + case SVE::BI__builtin_sve_svdup_neonq_u8: + case SVE::BI__builtin_sve_svdup_neonq_u16: + case SVE::BI__builtin_sve_svdup_neonq_u32: + case SVE::BI__builtin_sve_svdup_neonq_u64: + case SVE::BI__builtin_sve_svdup_neonq_f16: + case SVE::BI__builtin_sve_svdup_neonq_f32: + case SVE::BI__builtin_sve_svdup_neonq_f64: + case SVE::BI__builtin_sve_svdup_neonq_bf16: { + Value *Insert = Builder.CreateInsertVector(Ty, UndefValue::get(Ty), Ops[0], + Builder.getInt64(0)); + return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty}, + {Insert, Builder.getInt64(0)}); + } } /// Should not happen @@ -15331,7 +15415,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, // If the user wants the entire vector, just load the entire vector. if (NumBytes == 16) { Value *BC = Builder.CreateBitCast(Ops[0], ResTy->getPointerTo()); - Value *LD = Builder.CreateLoad(Address(BC, CharUnits::fromQuantity(1))); + Value *LD = + Builder.CreateLoad(Address(BC, ResTy, CharUnits::fromQuantity(1))); if (!IsLE) return LD; @@ -15392,8 +15477,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, RevMask.push_back(15 - Idx); StVec = Builder.CreateShuffleVector(Ops[2], Ops[2], RevMask); } - return Builder.CreateStore(StVec, - Address(BC, CharUnits::fromQuantity(1))); + return Builder.CreateStore( + StVec, Address(BC, Ops[2]->getType(), CharUnits::fromQuantity(1))); } auto *ConvTy = Int64Ty; unsigned NumElts = 0; @@ -15427,8 +15512,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Function *F = CGM.getIntrinsic(Intrinsic::bswap, ConvTy); Elt = Builder.CreateCall(F, Elt); } - return Builder.CreateStore(Elt, - Address(PtrBC, CharUnits::fromQuantity(1))); + return Builder.CreateStore( + Elt, Address(PtrBC, ConvTy, CharUnits::fromQuantity(1))); }; unsigned Stored = 0; unsigned RemainingBytes = NumBytes; @@ -16222,7 +16307,8 @@ Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) { auto *DstTy = CGF.Int16Ty->getPointerTo(GEP->getType()->getPointerAddressSpace()); auto *Cast = CGF.Builder.CreateBitCast(GEP, DstTy); - auto *LD = CGF.Builder.CreateLoad(Address(Cast, CharUnits::fromQuantity(2))); + auto *LD = CGF.Builder.CreateLoad( + Address(Cast, CGF.Int16Ty, CharUnits::fromQuantity(2))); llvm::MDBuilder MDHelper(CGF.getLLVMContext()); llvm::MDNode *RNode = MDHelper.createRange(APInt(16, 1), APInt(16, CGF.getTarget().getMaxOpenCLWorkGroupSize() + 1)); @@ -16242,7 +16328,8 @@ Value *EmitAMDGPUGridSize(CodeGenFunction &CGF, unsigned Index) { auto *DstTy = CGF.Int32Ty->getPointerTo(GEP->getType()->getPointerAddressSpace()); auto *Cast = CGF.Builder.CreateBitCast(GEP, DstTy); - auto *LD = CGF.Builder.CreateLoad(Address(Cast, CharUnits::fromQuantity(4))); + auto *LD = CGF.Builder.CreateLoad( + Address(Cast, CGF.Int32Ty, CharUnits::fromQuantity(4))); LD->setMetadata(llvm::LLVMContext::MD_invariant_load, llvm::MDNode::get(CGF.getLLVMContext(), None)); return LD; @@ -16314,8 +16401,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); - llvm::Type *RealFlagType - = FlagOutPtr.getPointer()->getType()->getPointerElementType(); + llvm::Type *RealFlagType = FlagOutPtr.getElementType(); llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); Builder.CreateStore(FlagExt, FlagOutPtr); @@ -16572,6 +16658,15 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Value *RayInverseDir = EmitScalarExpr(E->getArg(4)); llvm::Value *TextureDescr = EmitScalarExpr(E->getArg(5)); + // The builtins take these arguments as vec4 where the last element is + // ignored. The intrinsic takes them as vec3. + RayOrigin = Builder.CreateShuffleVector(RayOrigin, RayOrigin, + ArrayRef{0, 1, 2}); + RayDir = + Builder.CreateShuffleVector(RayDir, RayDir, ArrayRef{0, 1, 2}); + RayInverseDir = Builder.CreateShuffleVector(RayInverseDir, RayInverseDir, + ArrayRef{0, 1, 2}); + Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_image_bvh_intersect_ray, {NodePtr->getType(), RayDir->getType()}); return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir, @@ -17938,7 +18033,7 @@ RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) { if (getLangOpts().isSignedOverflowDefined()) Result = Builder.CreateGEP(Int8Ty, Base, Difference, "aligned_result"); else - Result = EmitCheckedInBoundsGEP(Base, Difference, + Result = EmitCheckedInBoundsGEP(Int8Ty, Base, Difference, /*SignedIndices=*/true, /*isSubtraction=*/!AlignUp, E->getExprLoc(), "aligned_result"); @@ -18501,6 +18596,7 @@ getIntrinsicForHexagonNonGCCBuiltin(unsigned BuiltinID) { CUSTOM_BUILTIN_MAPPING(S2_storerf_pcr, 0) CUSTOM_BUILTIN_MAPPING(S2_storeri_pcr, 0) CUSTOM_BUILTIN_MAPPING(S2_storerd_pcr, 0) + // Legacy builtins that take a vector in place of a vector predicate. CUSTOM_BUILTIN_MAPPING(V6_vmaskedstoreq, 64) CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorenq, 64) CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorentq, 64) @@ -18534,8 +18630,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) { // The base pointer is passed by address, so it needs to be loaded. Address A = EmitPointerWithAlignment(E->getArg(0)); - Address BP = Address( - Builder.CreateBitCast(A.getPointer(), Int8PtrPtrTy), A.getAlignment()); + Address BP = Address(Builder.CreateBitCast( + A.getPointer(), Int8PtrPtrTy), Int8PtrTy, A.getAlignment()); llvm::Value *Base = Builder.CreateLoad(BP); // The treatment of both loads and stores is the same: the arguments for // the builtin are the same as the arguments for the intrinsic. @@ -18579,7 +18675,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, // per call. Address DestAddr = EmitPointerWithAlignment(E->getArg(1)); DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), Int8PtrTy), - DestAddr.getAlignment()); + Int8Ty, DestAddr.getAlignment()); llvm::Value *DestAddress = DestAddr.getPointer(); // Operands are Base, Dest, Modifier. @@ -18626,8 +18722,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: { // Get the type from the 0-th argument. llvm::Type *VecType = ConvertType(E->getArg(0)->getType()); - Address PredAddr = Builder.CreateBitCast( - EmitPointerWithAlignment(E->getArg(2)), VecType->getPointerTo(0)); + Address PredAddr = Builder.CreateElementBitCast( + EmitPointerWithAlignment(E->getArg(2)), VecType); llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr)); llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn}); @@ -18638,6 +18734,27 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, return Builder.CreateExtractValue(Result, 0); } + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B: + case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: { + SmallVector Ops; + const Expr *PredOp = E->getArg(0); + // There will be an implicit cast to a boolean vector. Strip it. + if (auto *Cast = dyn_cast(PredOp)) { + if (Cast->getCastKind() == CK_BitCast) + PredOp = Cast->getSubExpr(); + Ops.push_back(V2Q(EmitScalarExpr(PredOp))); + } + for (int i = 1, e = E->getNumArgs(); i != e; ++i) + Ops.push_back(EmitScalarExpr(E->getArg(i))); + return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); + } + case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci: case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci: case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci: @@ -18674,40 +18791,6 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty); case Hexagon::BI__builtin_brev_ldd: return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty); - - default: { - if (ID == Intrinsic::not_intrinsic) - return nullptr; - - auto IsVectorPredTy = [](llvm::Type *T) { - return T->isVectorTy() && - cast(T)->getElementType()->isIntegerTy(1); - }; - - llvm::Function *IntrFn = CGM.getIntrinsic(ID); - llvm::FunctionType *IntrTy = IntrFn->getFunctionType(); - SmallVector Ops; - for (unsigned i = 0, e = IntrTy->getNumParams(); i != e; ++i) { - llvm::Type *T = IntrTy->getParamType(i); - const Expr *A = E->getArg(i); - if (IsVectorPredTy(T)) { - // There will be an implicit cast to a boolean vector. Strip it. - if (auto *Cast = dyn_cast(A)) { - if (Cast->getCastKind() == CK_BitCast) - A = Cast->getSubExpr(); - } - Ops.push_back(V2Q(EmitScalarExpr(A))); - } else { - Ops.push_back(EmitScalarExpr(A)); - } - } - - llvm::Value *Call = Builder.CreateCall(IntrFn, Ops); - if (IsVectorPredTy(IntrTy->getReturnType())) - Call = Q2V(Call); - - return Call; - } // default } // switch return nullptr; diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index a1b4431ca8c4..c4e3f7f54f4f 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -814,6 +814,9 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { Linkage, /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__hip_gpubin_handle"); + if (Linkage == llvm::GlobalValue::LinkOnceAnyLinkage) + GpuBinaryHandle->setComdat( + CGM.getModule().getOrInsertComdat(GpuBinaryHandle->getName())); GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign()); // Prevent the weak symbol in different shared libraries being merged. if (Linkage != llvm::GlobalValue::InternalLinkage) diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index d830a7e01709..d70f78fea6b4 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1261,8 +1261,7 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - Src = CGF.Builder.CreateBitCast(Src, - Ty->getPointerTo(Src.getAddressSpace())); + Src = CGF.Builder.CreateElementBitCast(Src, Ty); return CGF.Builder.CreateLoad(Src); } @@ -1832,11 +1831,6 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name, if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore) FuncAttrs.addAttribute("no-trapping-math", "true"); - // Strict (compliant) code is the default, so only add this attribute to - // indicate that we are trying to workaround a problem case. - if (!CodeGenOpts.StrictFloatCastOverflow) - FuncAttrs.addAttribute("strict-float-cast-overflow", "false"); - // TODO: Are these all needed? // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags. if (LangOpts.NoHonorInfs) @@ -1971,7 +1965,7 @@ static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types, // there's no internal padding (typeSizeEqualsStoreSize). return false; } - if (QTy->isExtIntType()) + if (QTy->isBitIntType()) return true; if (QTy->isReferenceType()) return true; @@ -2686,8 +2680,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); - Address ParamAddr = - Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign()); + Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), + ArgI.getIndirectAlign()); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we @@ -3475,12 +3469,19 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, case TEK_Aggregate: // Do nothing; aggregrates get evaluated directly into the destination. break; - case TEK_Scalar: - EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), - MakeNaturalAlignAddrLValue(&*AI, RetTy), - /*isInit*/ true); + case TEK_Scalar: { + LValueBaseInfo BaseInfo; + TBAAAccessInfo TBAAInfo; + CharUnits Alignment = + CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo); + Address ArgAddr(&*AI, ConvertType(RetTy), Alignment); + LValue ArgVal = + LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo); + EmitStoreOfScalar( + Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true); break; } + } break; } @@ -4134,8 +4135,7 @@ void CodeGenFunction::EmitCallArgs( } // If we still have any arguments, emit them using the type of the argument. - for (auto *A : llvm::make_range(std::next(ArgRange.begin(), ArgTypes.size()), - ArgRange.end())) + for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size())) ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType()); assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin())); @@ -4308,11 +4308,8 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, type->castAs()->getDecl()->isParamDestroyedInCallee()) { // If we're using inalloca, use the argument memory. Otherwise, use a // temporary. - AggValueSlot Slot; - if (args.isUsingInAlloca()) - Slot = createPlaceholderSlot(*this, type); - else - Slot = CreateAggTemp(type, "agg.tmp"); + AggValueSlot Slot = args.isUsingInAlloca() + ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp"); bool DestroyedInCallee = true, NeedsEHCleanup = true; if (const auto *RD = type->getAsCXXRecordDecl()) @@ -4651,13 +4648,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // // In other cases, we assert that the types match up (until pointers stop // having pointee types). - llvm::Type *TypeFromVal; if (Callee.isVirtual()) - TypeFromVal = Callee.getVirtualFunctionType(); - else - TypeFromVal = - Callee.getFunctionPointer()->getType()->getPointerElementType(); - assert(IRFuncTy == TypeFromVal); + assert(IRFuncTy == Callee.getVirtualFunctionType()); + else { + llvm::PointerType *PtrTy = + llvm::cast(Callee.getFunctionPointer()->getType()); + assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy)); + } } #endif @@ -4872,7 +4869,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, I->copyInto(*this, AI); } else { // Skip the extra memcpy call. - auto *T = V->getType()->getPointerElementType()->getPointerTo( + auto *T = llvm::PointerType::getWithSamePointeeType( + cast(V->getType()), CGM.getDataLayout().getAllocaAddrSpace()); IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast( *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T, @@ -4967,8 +4965,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Builder.CreateMemCpy(TempAlloca, Src, SrcSize); Src = TempAlloca; } else { - Src = Builder.CreateBitCast(Src, - STy->getPointerTo(Src.getAddressSpace())); + Src = Builder.CreateElementBitCast(Src, STy); } assert(NumIRArgs == STy->getNumElements()); diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index e3d9fec6d363..c8594068c3fc 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -115,7 +115,8 @@ class CGCallee { AbstractInfo = abstractInfo; assert(functionPtr && "configuring callee without function pointer"); assert(functionPtr->getType()->isPointerTy()); - assert(functionPtr->getType()->getPointerElementType()->isFunctionTy()); + assert(functionPtr->getType()->isOpaquePointerTy() || + functionPtr->getType()->getPointerElementType()->isFunctionTy()); } static CGCallee forBuiltin(unsigned builtinID, diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 0df64d4d5d26..8f99ff0d50ff 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -127,18 +127,18 @@ CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign, Address CodeGenFunction::LoadCXXThisAddress() { assert(CurFuncDecl && "loading 'this' without a func declaration?"); - assert(isa(CurFuncDecl)); + auto *MD = cast(CurFuncDecl); // Lazily compute CXXThisAlignment. if (CXXThisAlignment.isZero()) { // Just use the best known alignment for the parent. // TODO: if we're currently emitting a complete-object ctor/dtor, // we can always use the complete-object alignment. - auto RD = cast(CurFuncDecl)->getParent(); - CXXThisAlignment = CGM.getClassPointerAlignment(RD); + CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent()); } - return Address(LoadCXXThis(), CXXThisAlignment); + llvm::Type *Ty = ConvertType(MD->getThisType()->getPointeeType()); + return Address(LoadCXXThis(), Ty, CXXThisAlignment); } /// Emit the address of a field using a member data pointer. @@ -286,7 +286,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, } alignment = alignment.alignmentAtOffset(nonVirtualOffset); - return Address(ptr, alignment); + return Address(ptr, CGF.Int8Ty, alignment); } Address CodeGenFunction::GetAddressOfBaseClass( @@ -326,9 +326,9 @@ Address CodeGenFunction::GetAddressOfBaseClass( } // Get the base pointer type. + llvm::Type *BaseValueTy = ConvertType((PathEnd[-1])->getType()); llvm::Type *BasePtrTy = - ConvertType((PathEnd[-1])->getType()) - ->getPointerTo(Value.getType()->getPointerAddressSpace()); + BaseValueTy->getPointerTo(Value.getType()->getPointerAddressSpace()); QualType DerivedTy = getContext().getRecordType(Derived); CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived); @@ -342,7 +342,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks); } - return Builder.CreateBitCast(Value, BasePtrTy); + return Builder.CreateElementBitCast(Value, BaseValueTy); } llvm::BasicBlock *origBB = nullptr; @@ -379,7 +379,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( VirtualOffset, Derived, VBase); // Cast to the destination type. - Value = Builder.CreateBitCast(Value, BasePtrTy); + Value = Builder.CreateElementBitCast(Value, BaseValueTy); // Build a phi if we needed a null check. if (NullCheckValue) { @@ -406,16 +406,16 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, QualType DerivedTy = getContext().getCanonicalType(getContext().getTagDeclType(Derived)); - unsigned AddrSpace = - BaseAddr.getPointer()->getType()->getPointerAddressSpace(); - llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace); + unsigned AddrSpace = BaseAddr.getAddressSpace(); + llvm::Type *DerivedValueTy = ConvertType(DerivedTy); + llvm::Type *DerivedPtrTy = DerivedValueTy->getPointerTo(AddrSpace); llvm::Value *NonVirtualOffset = CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd); if (!NonVirtualOffset) { // No offset, we can just cast back. - return Builder.CreateBitCast(BaseAddr, DerivedPtrTy); + return Builder.CreateElementBitCast(BaseAddr, DerivedValueTy); } llvm::BasicBlock *CastNull = nullptr; @@ -453,7 +453,7 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, Value = PHI; } - return Address(Value, CGM.getClassPointerAlignment(Derived)); + return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived)); } llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, @@ -996,16 +996,8 @@ namespace { private: void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) { - llvm::PointerType *DPT = DestPtr.getType(); - llvm::Type *DBP = - llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace()); - DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP); - - llvm::PointerType *SPT = SrcPtr.getType(); - llvm::Type *SBP = - llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace()); - SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP); - + DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty); + SrcPtr = CGF.Builder.CreateElementBitCast(SrcPtr, CGF.Int8Ty); CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity()); } @@ -2068,8 +2060,8 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, if (SlotAS != ThisAS) { unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS); - llvm::Type *NewType = - ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS); + llvm::Type *NewType = llvm::PointerType::getWithSamePointeeType( + This.getType(), TargetThisAS); ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(), ThisAS, SlotAS, NewType); } @@ -2507,9 +2499,6 @@ void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) { // Apply the offsets. Address VTableField = LoadCXXThisAddress(); - unsigned ThisAddrSpace = - VTableField.getPointer()->getType()->getPointerAddressSpace(); - if (!NonVirtualOffset.isZero() || VirtualOffset) VTableField = ApplyNonVirtualAndVirtualOffset( *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass, @@ -2525,8 +2514,7 @@ void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) { ->getPointerTo(GlobalsAS); // vtable field is is derived from `this` pointer, therefore they should be in // the same addr space. Note that this might not be LLVM address space 0. - VTableField = Builder.CreateBitCast(VTableField, - VTablePtrTy->getPointerTo(ThisAddrSpace)); + VTableField = Builder.CreateElementBitCast(VTableField, VTablePtrTy); VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy); llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField); diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h index 1b54c0018d27..76f3a48f32f3 100644 --- a/clang/lib/CodeGen/CGCleanup.h +++ b/clang/lib/CodeGen/CGCleanup.h @@ -242,7 +242,7 @@ class alignas(8) EHCleanupScope : public EHScope { /// An optional i1 variable indicating whether this cleanup has been /// activated yet. - llvm::AllocaInst *ActiveFlag; + Address ActiveFlag; /// Extra information required for cleanups that have resolved /// branches through them. This has to be allocated on the side @@ -290,7 +290,8 @@ class alignas(8) EHCleanupScope : public EHScope { EHScopeStack::stable_iterator enclosingEH) : EHScope(EHScope::Cleanup, enclosingEH), EnclosingNormal(enclosingNormal), NormalBlock(nullptr), - ActiveFlag(nullptr), ExtInfo(nullptr), FixupDepth(fixupDepth) { + ActiveFlag(Address::invalid()), ExtInfo(nullptr), + FixupDepth(fixupDepth) { CleanupBits.IsNormalCleanup = isNormal; CleanupBits.IsEHCleanup = isEH; CleanupBits.IsActive = true; @@ -320,13 +321,13 @@ class alignas(8) EHCleanupScope : public EHScope { bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; } void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; } - bool hasActiveFlag() const { return ActiveFlag != nullptr; } + bool hasActiveFlag() const { return ActiveFlag.isValid(); } Address getActiveFlag() const { - return Address(ActiveFlag, CharUnits::One()); + return ActiveFlag; } void setActiveFlag(Address Var) { assert(Var.getAlignment().isOne()); - ActiveFlag = cast(Var.getPointer()); + ActiveFlag = Var; } void setTestFlagInNormalCleanup() { diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index ca071d3d2e80..2041d2a5b4c9 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -597,6 +597,10 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi}); CurCoro.Data->CoroBegin = CoroBegin; + // We need to emit `get_­return_­object` first. According to: + // [dcl.fct.def.coroutine]p7 + // The call to get_­return_­object is sequenced before the call to + // initial_­suspend and is invoked at most once. GetReturnObjectManager GroManager(*this, S); GroManager.EmitGroAlloca(); diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index af651e6f44b7..6e189a61dd20 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -768,7 +768,7 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { } // Element count = (VLENB / SEW) x LMUL - SmallVector Expr( + SmallVector Expr( // The DW_OP_bregx operation has two operands: a register which is // specified by an unsigned LEB128 number, followed by a signed LEB128 // offset. @@ -782,6 +782,8 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { Expr.push_back(llvm::dwarf::DW_OP_div); else Expr.push_back(llvm::dwarf::DW_OP_mul); + // Element max index = count - 1 + Expr.append({llvm::dwarf::DW_OP_constu, 1, llvm::dwarf::DW_OP_minus}); auto *LowerBound = llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned( @@ -884,9 +886,9 @@ llvm::DIType *CGDebugInfo::CreateType(const AutoType *Ty) { return DBuilder.createUnspecifiedType("auto"); } -llvm::DIType *CGDebugInfo::CreateType(const ExtIntType *Ty) { +llvm::DIType *CGDebugInfo::CreateType(const BitIntType *Ty) { - StringRef Name = Ty->isUnsigned() ? "unsigned _ExtInt" : "_ExtInt"; + StringRef Name = Ty->isUnsigned() ? "unsigned _BitInt" : "_BitInt"; llvm::dwarf::TypeKind Encoding = Ty->isUnsigned() ? llvm::dwarf::DW_ATE_unsigned : llvm::dwarf::DW_ATE_signed; @@ -3353,6 +3355,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { case Type::Elaborated: T = cast(T)->getNamedType(); break; + case Type::Using: + T = cast(T)->getUnderlyingType(); + break; case Type::Paren: T = cast(T)->getInnerType(); break; @@ -3531,8 +3536,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit, case Type::Atomic: return CreateType(cast(Ty), Unit); - case Type::ExtInt: - return CreateType(cast(Ty)); + case Type::BitInt: + return CreateType(cast(Ty)); case Type::Pipe: return CreateType(cast(Ty), Unit); @@ -3545,6 +3550,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit, case Type::Decayed: case Type::DeducedTemplateSpecialization: case Type::Elaborated: + case Type::Using: case Type::Paren: case Type::MacroQualified: case Type::SubstTemplateTypeParm: @@ -3633,6 +3639,9 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) { // Record exports it symbols to the containing structure. if (CXXRD->isAnonymousStructOrUnion()) Flags |= llvm::DINode::FlagExportSymbols; + + Flags |= getAccessFlag(CXXRD->getAccess(), + dyn_cast(CXXRD->getDeclContext())); } llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D); diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h index a7b72fa5f5a6..14ff0eeabd21 100644 --- a/clang/lib/CodeGen/CGDebugInfo.h +++ b/clang/lib/CodeGen/CGDebugInfo.h @@ -177,7 +177,7 @@ class CGDebugInfo { llvm::DIType *CreateType(const BuiltinType *Ty); llvm::DIType *CreateType(const ComplexType *Ty); llvm::DIType *CreateType(const AutoType *Ty); - llvm::DIType *CreateType(const ExtIntType *Ty); + llvm::DIType *CreateType(const BitIntType *Ty); llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg, TypeLoc TL = TypeLoc()); llvm::DIType *CreateQualifiedType(const FunctionProtoType *Ty, diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp index 941671c61482..e09279c1d455 100644 --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -405,7 +405,8 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D, // Store into LocalDeclMap before generating initializer to handle // circular references. - setAddrOfLocalVar(&D, Address(addr, alignment)); + llvm::Type *elemTy = ConvertTypeForMem(D.getType()); + setAddrOfLocalVar(&D, Address(addr, elemTy, alignment)); // We can't have a VLA here, but we can have a pointer to a VLA, // even though that doesn't really make any sense. @@ -458,8 +459,7 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D, // RAUW's the GV uses of this constant will be invalid. llvm::Constant *castedAddr = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(var, expectedType); - if (var != castedAddr) - LocalDeclMap.find(&D)->second = Address(castedAddr, alignment); + LocalDeclMap.find(&D)->second = Address(castedAddr, elemTy, alignment); CGM.setStaticLocalDeclAddress(&D, castedAddr); CGM.getSanitizerMetadata()->reportGlobalToASan(var, D); @@ -1146,7 +1146,7 @@ Address CodeGenModule::createUnnamedGlobalFrom(const VarDecl &D, CacheEntry->setAlignment(Align.getAsAlign()); } - return Address(CacheEntry, Align); + return Address(CacheEntry, CacheEntry->getValueType(), Align); } static Address createUnnamedGlobalForMemcpyFrom(CodeGenModule &CGM, @@ -1193,7 +1193,7 @@ static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D, bool valueAlreadyCorrect = constant->isNullValue() || isa(constant); if (!valueAlreadyCorrect) { - Loc = Builder.CreateBitCast(Loc, Ty->getPointerTo(Loc.getAddressSpace())); + Loc = Builder.CreateElementBitCast(Loc, Ty); emitStoresForInitAfterBZero(CGM, constant, Loc, isVolatile, Builder, IsAutoInit); } diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp index d22f9dc3b68c..3579761f1429 100644 --- a/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/clang/lib/CodeGen/CGDeclCXX.cpp @@ -172,7 +172,7 @@ void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) { } void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, - llvm::Constant *DeclPtr, + llvm::GlobalVariable *GV, bool PerformInit) { const Expr *Init = D.getInit(); @@ -194,14 +194,16 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, // "shared" address space qualifier, but the constructor of StructWithCtor // expects "this" in the "generic" address space. unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T); - unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace(); + unsigned ActualAddrSpace = GV->getAddressSpace(); + llvm::Constant *DeclPtr = GV; if (ActualAddrSpace != ExpectedAddrSpace) { - llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T); - llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace); + llvm::PointerType *PTy = llvm::PointerType::getWithSamePointeeType( + GV->getType(), ExpectedAddrSpace); DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy); } - ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D)); + ConstantAddress DeclAddr( + DeclPtr, GV->getValueType(), getContext().getDeclAlign(&D)); if (!T->isReferenceType()) { if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp index aff9c77d53c7..91ecbecc843f 100644 --- a/clang/lib/CodeGen/CGException.cpp +++ b/clang/lib/CodeGen/CGException.cpp @@ -400,8 +400,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { // __cxa_allocate_exception returns a void*; we need to cast this // to the appropriate type for the object. - llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo(); - Address typedAddr = Builder.CreateBitCast(addr, ty); + llvm::Type *ty = ConvertTypeForMem(e->getType()); + Address typedAddr = Builder.CreateElementBitCast(addr, ty); // FIXME: this isn't quite right! If there's a final unelided call // to a copy constructor, then according to [except.terminate]p1 we @@ -421,13 +421,13 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { Address CodeGenFunction::getExceptionSlot() { if (!ExceptionSlot) ExceptionSlot = CreateTempAlloca(Int8PtrTy, "exn.slot"); - return Address(ExceptionSlot, getPointerAlign()); + return Address(ExceptionSlot, Int8PtrTy, getPointerAlign()); } Address CodeGenFunction::getEHSelectorSlot() { if (!EHSelectorSlot) EHSelectorSlot = CreateTempAlloca(Int32Ty, "ehselector.slot"); - return Address(EHSelectorSlot, CharUnits::fromQuantity(4)); + return Address(EHSelectorSlot, Int32Ty, CharUnits::fromQuantity(4)); } llvm::Value *CodeGenFunction::getExceptionFromSlot() { diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 4332e74dbb24..34b4951a7f72 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -71,7 +71,7 @@ Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return Address(Alloca, Align); + return Address(Alloca, Ty, Align); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry @@ -101,7 +101,7 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return Address(V, Align); + return Address(V, Ty, Align); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -144,7 +144,7 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, /*ArraySize=*/nullptr, Alloca); if (Ty->isConstantMatrixType()) { - auto *ArrayTy = cast(Result.getType()->getElementType()); + auto *ArrayTy = cast(Result.getElementType()); auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), ArrayTy->getNumElements()); @@ -1099,7 +1099,7 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr = Address(Addr.getPointer(), Align); + Addr = Address(Addr.getPointer(), Addr.getElementType(), Align); } } @@ -1111,10 +1111,12 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, CodeGenFunction::CFITCK_UnrelatedCast, CE->getBeginLoc()); } - return CE->getCastKind() != CK_AddressSpaceConversion - ? Builder.CreateBitCast(Addr, ConvertType(E->getType())) - : Builder.CreateAddrSpaceCast(Addr, - ConvertType(E->getType())); + + if (CE->getCastKind() == CK_AddressSpaceConversion) + return Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType())); + + llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType()); + return Builder.CreateElementBitCast(Addr, ElemTy); } break; @@ -1160,7 +1162,8 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, // Otherwise, use the alignment of the type. CharUnits Align = CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); - return Address(EmitScalarExpr(E), Align); + llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType()); + return Address(EmitScalarExpr(E), ElemTy, Align); } llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) { @@ -1306,7 +1309,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { const ConstantExpr *CE = cast(E); if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) { QualType RetType = cast(CE->getSubExpr()->IgnoreImplicit()) - ->getCallReturnType(getContext()); + ->getCallReturnType(getContext()) + ->getPointeeType(); return MakeNaturalAlignAddrLValue(Result, RetType); } return EmitLValue(cast(E)->getSubExpr()); @@ -1342,10 +1346,11 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { if (LV.isSimple()) { // Defend against branches out of gnu statement expressions surrounded by // cleanups. - llvm::Value *V = LV.getPointer(*this); + Address Addr = LV.getAddress(*this); + llvm::Value *V = Addr.getPointer(); Scope.ForceCleanup({&V}); - return LValue::MakeAddr(Address(V, LV.getAlignment()), LV.getType(), - getContext(), LV.getBaseInfo(), LV.getTBAAInfo()); + return LValue::MakeAddr(Addr.withPointer(V), LV.getType(), getContext(), + LV.getBaseInfo(), LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1777,16 +1782,14 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { // MatrixType), if it points to a array (the memory type of MatrixType). static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, bool IsVector = true) { - auto *ArrayTy = dyn_cast( - cast(Addr.getPointer()->getType())->getElementType()); + auto *ArrayTy = dyn_cast(Addr.getElementType()); if (ArrayTy && IsVector) { auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), ArrayTy->getNumElements()); return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy)); } - auto *VectorTy = dyn_cast( - cast(Addr.getPointer()->getType())->getElementType()); + auto *VectorTy = dyn_cast(Addr.getElementType()); if (VectorTy && !IsVector) { auto *ArrayTy = llvm::ArrayType::get( VectorTy->getElementType(), @@ -2475,10 +2478,11 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal, Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); + QualType PointeeType = RefLVal.getType()->getPointeeType(); CharUnits Align = CGM.getNaturalTypeAlignment( - RefLVal.getType()->getPointeeType(), PointeeBaseInfo, PointeeTBAAInfo, + PointeeType, PointeeBaseInfo, PointeeTBAAInfo, /* forPointeeType= */ true); - return Address(Load, Align); + return Address(Load, ConvertTypeForMem(PointeeType), Align); } LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { @@ -2528,7 +2532,7 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType()); V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); CharUnits Alignment = CGF.getContext().getDeclAlign(VD); - Address Addr(V, Alignment); + Address Addr(V, RealVarTy, Alignment); // Emit reference to the private copy of the variable if it is an OpenMP // threadprivate variable. if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd && @@ -2610,7 +2614,7 @@ static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) { llvm::Value *Ptr = llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0)); - return LValue::MakeGlobalReg(Address(Ptr, Alignment), VD->getType()); + return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType()); } /// Determine whether we can emit a reference to \p VD from the current @@ -2706,7 +2710,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { /* BaseInfo= */ nullptr, /* TBAAInfo= */ nullptr, /* forPointeeType= */ true); - Addr = Address(Val, Alignment); + Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); } return MakeAddrLValue(Addr, T, AlignmentSource::Decl); } @@ -2783,9 +2787,10 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // Otherwise, it might be static local we haven't emitted yet for // some reason; most likely, because it's in an outer function. } else if (VD->isStaticLocal()) { - addr = Address(CGM.getOrCreateStaticVarDecl( - *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false)), - getContext().getDeclAlign(VD)); + llvm::Constant *var = CGM.getOrCreateStaticVarDecl( + *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false)); + addr = Address( + var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD)); // No other cases for now. } else { @@ -3586,7 +3591,7 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, SourceLocation loc, const llvm::Twine &name = "arrayidx") { if (inbounds) { - return CGF.EmitCheckedInBoundsGEP(ptr, indices, signedIndices, + return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices, CodeGenFunction::NotSubtraction, loc, name); } else { @@ -3698,7 +3703,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, idx, DbgInfo); } - return Address(eltPtr, eltAlign); + return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); } LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, @@ -4380,8 +4385,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, hasAnyVptr(FieldType, getContext())) // Because unions can easily skip invariant.barriers, we need to add // a barrier every time CXXRecord field with vptr is referenced. - addr = Address(Builder.CreateLaunderInvariantGroup(addr.getPointer()), - addr.getAlignment()); + addr = Builder.CreateLaunderInvariantGroup(addr); if (IsInPreservedAIRegion || (getDebugInfo() && rec->hasAttr())) { @@ -4539,10 +4543,10 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { // because it can't be used. if (auto *ThrowExpr = dyn_cast(live->IgnoreParens())) { EmitCXXThrowExpr(ThrowExpr); - llvm::Type *Ty = - llvm::PointerType::getUnqual(ConvertType(dead->getType())); + llvm::Type *ElemTy = ConvertType(dead->getType()); + llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy); return MakeAddrLValue( - Address(llvm::UndefValue::get(Ty), CharUnits::One()), + Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()), dead->getType()); } return EmitLValue(live); @@ -4584,11 +4588,13 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { EmitBlock(contBlock); if (lhs && rhs) { - llvm::PHINode *phi = - Builder.CreatePHI(lhs->getPointer(*this)->getType(), 2, "cond-lvalue"); - phi->addIncoming(lhs->getPointer(*this), lhsBlock); - phi->addIncoming(rhs->getPointer(*this), rhsBlock); - Address result(phi, std::min(lhs->getAlignment(), rhs->getAlignment())); + Address lhsAddr = lhs->getAddress(*this); + Address rhsAddr = rhs->getAddress(*this); + llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); + phi->addIncoming(lhsAddr.getPointer(), lhsBlock); + phi->addIncoming(rhsAddr.getPointer(), rhsBlock); + Address result(phi, lhsAddr.getElementType(), + std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); AlignmentSource alignSource = std::max(lhs->getBaseInfo().getAlignmentSource(), rhs->getBaseInfo().getAlignmentSource()); diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 5b56a587fa5f..3b996b89a1d7 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -301,7 +301,7 @@ void AggExprEmitter::withReturnValueSlot( if (!UseTemp) return; - assert(Dest.getPointer() != Src.getAggregatePointer()); + assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer()); EmitFinalDestCopy(E->getType(), Src); if (!RequiresDestruction && LifetimeStartInst) { @@ -493,7 +493,7 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); CharUnits elementAlign = DestPtr.getAlignment().alignmentOfArrayElement(elementSize); - llvm::Type *llvmElementType = begin->getType()->getPointerElementType(); + llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType); // Consider initializing the array by copying from a global. For this to be // more efficient than per-element initialization, the size of the elements @@ -513,7 +513,8 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, Emitter.finalize(GV); CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy); GV->setAlignment(Align.getAsAlign()); - EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GV, ArrayQTy, Align)); + Address GVAddr(GV, GV->getValueType(), Align); + EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, ArrayQTy)); return; } } @@ -565,8 +566,8 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); } - LValue elementLV = - CGF.MakeAddrLValue(Address(element, elementAlign), elementType); + LValue elementLV = CGF.MakeAddrLValue( + Address(element, llvmElementType, elementAlign), elementType); EmitInitializationToLValue(E->getInit(i), elementLV); } diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index cc838bf38c6c..0571c498c377 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -1052,13 +1052,8 @@ void CodeGenFunction::EmitNewArrayInitializer( InitListElements = cast(ILE->getType()->getAsArrayTypeUnsafe()) ->getSize().getZExtValue(); - CurPtr = - Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(), - CurPtr.getPointer(), - Builder.getSize(InitListElements), - "string.init.end"), - CurPtr.getAlignment().alignmentAtOffset(InitListElements * - ElementSize)); + CurPtr = Builder.CreateConstInBoundsGEP( + CurPtr, InitListElements, "string.init.end"); // Zero out the rest, if any remain. llvm::ConstantInt *ConstNum = dyn_cast(NumElements); @@ -1135,7 +1130,7 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Switch back to initializing one base element at a time. - CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr.getType()); + CurPtr = Builder.CreateElementBitCast(CurPtr, BeginPtr.getElementType()); } // If all elements have already been initialized, skip any further @@ -1594,7 +1589,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // In these cases, discard the computed alignment and use the // formal alignment of the allocated type. if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl) - allocation = Address(allocation.getPointer(), allocAlign); + allocation = allocation.withAlignment(allocAlign); // Set up allocatorArgs for the call to operator delete if it's not // the reserved global operator. @@ -1664,7 +1659,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { allocationAlign, getContext().toCharUnitsFromBits(AllocatorAlign)); } - allocation = Address(RV.getScalarVal(), allocationAlign); + allocation = Address(RV.getScalarVal(), Int8Ty, allocationAlign); } // Emit a null check on the allocation result if the allocation @@ -1725,8 +1720,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // of optimization level. if (CGM.getCodeGenOpts().StrictVTablePointers && allocator->isReservedGlobalPlacementOperator()) - result = Address(Builder.CreateLaunderInvariantGroup(result.getPointer()), - result.getAlignment()); + result = Builder.CreateLaunderInvariantGroup(result); // Emit sanitizer checks for pointer value now, so that in the case of an // array it was checked only once and not at each constructor call. We may diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index ff900ed077e6..cf1f2e0eab92 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -899,7 +899,7 @@ static ConstantAddress tryEmitGlobalCompoundLiteral(CodeGenModule &CGM, CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType()); if (llvm::GlobalVariable *Addr = CGM.getAddrOfConstantCompoundLiteralIfEmitted(E)) - return ConstantAddress(Addr, Align); + return ConstantAddress(Addr, Addr->getValueType(), Align); LangAS addressSpace = E->getType().getAddressSpace(); @@ -921,7 +921,7 @@ static ConstantAddress tryEmitGlobalCompoundLiteral(CodeGenModule &CGM, emitter.finalize(GV); GV->setAlignment(Align.getAsAlign()); CGM.setAddrOfConstantCompoundLiteral(E, GV); - return ConstantAddress(GV, Align); + return ConstantAddress(GV, GV->getValueType(), Align); } static llvm::Constant * @@ -1988,6 +1988,9 @@ ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) { ConstantLValue ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) { unsigned builtin = E->getBuiltinCallee(); + if (builtin == Builtin::BI__builtin_function_start) + return CGM.GetFunctionStart( + E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext())); if (builtin != Builtin::BI__builtin___CFStringMakeConstantString && builtin != Builtin::BI__builtin___NSStringMakeConstantString) return nullptr; diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index ae9434f96529..e32462eb635c 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -1240,7 +1240,18 @@ Value *ScalarExprEmitter::EmitScalarCast(Value *Src, QualType SrcType, if (isa(DstElementTy)) { assert(SrcElementTy->isFloatingPointTy() && "Unknown real conversion"); - if (DstElementType->isSignedIntegerOrEnumerationType()) + bool IsSigned = DstElementType->isSignedIntegerOrEnumerationType(); + + // If we can't recognize overflow as undefined behavior, assume that + // overflow saturates. This protects against normal optimizations if we are + // compiling with non-standard FP semantics. + if (!CGF.CGM.getCodeGenOpts().StrictFloatCastOverflow) { + llvm::Intrinsic::ID IID = + IsSigned ? llvm::Intrinsic::fptosi_sat : llvm::Intrinsic::fptoui_sat; + return Builder.CreateCall(CGF.CGM.getIntrinsic(IID, {DstTy, SrcTy}), Src); + } + + if (IsSigned) return Builder.CreateFPToSI(Src, DstTy, "conv"); return Builder.CreateFPToUI(Src, DstTy, "conv"); } @@ -2631,12 +2642,12 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, = CGF.getContext().getAsVariableArrayType(type)) { llvm::Value *numElts = CGF.getVLASize(vla).NumElts; if (!isInc) numElts = Builder.CreateNSWNeg(numElts, "vla.negsize"); + llvm::Type *elemTy = value->getType()->getPointerElementType(); if (CGF.getLangOpts().isSignedOverflowDefined()) - value = Builder.CreateGEP(value->getType()->getPointerElementType(), - value, numElts, "vla.inc"); + value = Builder.CreateGEP(elemTy, value, numElts, "vla.inc"); else value = CGF.EmitCheckedInBoundsGEP( - value, numElts, /*SignedIndices=*/false, isSubtraction, + elemTy, value, numElts, /*SignedIndices=*/false, isSubtraction, E->getExprLoc(), "vla.inc"); // Arithmetic on function pointers (!) is just +-1. @@ -2647,7 +2658,8 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, if (CGF.getLangOpts().isSignedOverflowDefined()) value = Builder.CreateGEP(CGF.Int8Ty, value, amt, "incdec.funcptr"); else - value = CGF.EmitCheckedInBoundsGEP(value, amt, /*SignedIndices=*/false, + value = CGF.EmitCheckedInBoundsGEP(CGF.Int8Ty, value, amt, + /*SignedIndices=*/false, isSubtraction, E->getExprLoc(), "incdec.funcptr"); value = Builder.CreateBitCast(value, input->getType()); @@ -2655,13 +2667,13 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, // For everything else, we can just do a simple increment. } else { llvm::Value *amt = Builder.getInt32(amount); + llvm::Type *elemTy = CGF.ConvertTypeForMem(type); if (CGF.getLangOpts().isSignedOverflowDefined()) - value = Builder.CreateGEP(value->getType()->getPointerElementType(), - value, amt, "incdec.ptr"); + value = Builder.CreateGEP(elemTy, value, amt, "incdec.ptr"); else - value = CGF.EmitCheckedInBoundsGEP(value, amt, /*SignedIndices=*/false, - isSubtraction, E->getExprLoc(), - "incdec.ptr"); + value = CGF.EmitCheckedInBoundsGEP( + elemTy, value, amt, /*SignedIndices=*/false, isSubtraction, + E->getExprLoc(), "incdec.ptr"); } // Vector increment/decrement. @@ -2771,9 +2783,9 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, if (CGF.getLangOpts().isSignedOverflowDefined()) value = Builder.CreateGEP(CGF.Int8Ty, value, sizeValue, "incdec.objptr"); else - value = CGF.EmitCheckedInBoundsGEP(value, sizeValue, - /*SignedIndices=*/false, isSubtraction, - E->getExprLoc(), "incdec.objptr"); + value = CGF.EmitCheckedInBoundsGEP( + CGF.Int8Ty, value, sizeValue, /*SignedIndices=*/false, isSubtraction, + E->getExprLoc(), "incdec.objptr"); value = Builder.CreateBitCast(value, input->getType()); } @@ -3508,16 +3520,15 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF, // GEP indexes are signed, and scaling an index isn't permitted to // signed-overflow, so we use the same semantics for our explicit // multiply. We suppress this if overflow is not undefined behavior. + llvm::Type *elemTy = pointer->getType()->getPointerElementType(); if (CGF.getLangOpts().isSignedOverflowDefined()) { index = CGF.Builder.CreateMul(index, numElements, "vla.index"); - pointer = CGF.Builder.CreateGEP( - pointer->getType()->getPointerElementType(), pointer, index, - "add.ptr"); + pointer = CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr"); } else { index = CGF.Builder.CreateNSWMul(index, numElements, "vla.index"); - pointer = - CGF.EmitCheckedInBoundsGEP(pointer, index, isSigned, isSubtraction, - op.E->getExprLoc(), "add.ptr"); + pointer = CGF.EmitCheckedInBoundsGEP( + elemTy, pointer, index, isSigned, isSubtraction, op.E->getExprLoc(), + "add.ptr"); } return pointer; } @@ -3531,12 +3542,13 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF, return CGF.Builder.CreateBitCast(result, pointer->getType()); } + llvm::Type *elemTy = CGF.ConvertTypeForMem(elementType); if (CGF.getLangOpts().isSignedOverflowDefined()) - return CGF.Builder.CreateGEP( - pointer->getType()->getPointerElementType(), pointer, index, "add.ptr"); + return CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr"); - return CGF.EmitCheckedInBoundsGEP(pointer, index, isSigned, isSubtraction, - op.E->getExprLoc(), "add.ptr"); + return CGF.EmitCheckedInBoundsGEP( + elemTy, pointer, index, isSigned, isSubtraction, op.E->getExprLoc(), + "add.ptr"); } // Construct an fmuladd intrinsic to represent a fused mul-add of MulOp and @@ -5057,12 +5069,12 @@ static GEPOffsetAndOverflow EmitGEPOffsetInBytes(Value *BasePtr, Value *GEPVal, } Value * -CodeGenFunction::EmitCheckedInBoundsGEP(Value *Ptr, ArrayRef IdxList, +CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr, + ArrayRef IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name) { llvm::Type *PtrTy = Ptr->getType(); - Value *GEPVal = Builder.CreateInBoundsGEP( - PtrTy->getPointerElementType(), Ptr, IdxList, Name); + Value *GEPVal = Builder.CreateInBoundsGEP(ElemTy, Ptr, IdxList, Name); // If the pointer overflow sanitizer isn't enabled, do nothing. if (!SanOpts.has(SanitizerKind::PointerOverflow)) diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp index ad505fc5a0d4..e3b0e069b830 100644 --- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp +++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp @@ -366,11 +366,11 @@ template struct GenFuncBase { llvm::ConstantInt::get(NumElts->getType(), BaseEltSize); llvm::Value *SizeInBytes = CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts); - Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy); + Address BC = CGF.Builder.CreateElementBitCast(DstAddr, CGF.CGM.Int8Ty); llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BC.getPointer(), SizeInBytes); - DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy, - "dstarray.end"); + DstArrayEnd = CGF.Builder.CreateBitCast( + DstArrayEnd, CGF.CGM.Int8PtrPtrTy, "dstarray.end"); llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock(); // Create the header block and insert the phi instructions. @@ -426,9 +426,9 @@ template struct GenFuncBase { assert(Addr.isValid() && "invalid address"); if (Offset.getQuantity() == 0) return Addr; - Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy); + Addr = CGF->Builder.CreateElementBitCast(Addr, CGF->CGM.Int8Ty); Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity()); - return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy); + return CGF->Builder.CreateElementBitCast(Addr, CGF->CGM.Int8PtrTy); } Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset, @@ -491,9 +491,8 @@ template struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { Alignments[I] = Addrs[I].getAlignment(); - Ptrs[I] = - CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy) - .getPointer(); + Ptrs[I] = CallerCGF.Builder.CreateElementBitCast( + Addrs[I], CallerCGF.CGM.Int8PtrTy).getPointer(); } if (llvm::Function *F = @@ -554,19 +553,21 @@ struct GenBinaryFunc : CopyStructVisitor, return; QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0); - llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo(); + llvm::Type *Ty = this->CGF->ConvertType(RT); Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset); LValue DstBase = this->CGF->MakeAddrLValue( - this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT); + this->CGF->Builder.CreateElementBitCast(DstAddr, Ty), FT); DstLV = this->CGF->EmitLValueForField(DstBase, FD); Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset); LValue SrcBase = this->CGF->MakeAddrLValue( - this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT); + this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty), FT); SrcLV = this->CGF->EmitLValueForField(SrcBase, FD); } else { - llvm::PointerType *Ty = this->CGF->ConvertTypeForMem(FT)->getPointerTo(); - Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty); - Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty); + llvm::Type *Ty = this->CGF->ConvertTypeForMem(FT); + Address DstAddr = + this->CGF->Builder.CreateElementBitCast(Addrs[DstIdx], Ty); + Address SrcAddr = + this->CGF->Builder.CreateElementBitCast(Addrs[SrcIdx], Ty); DstLV = this->CGF->MakeAddrLValue(DstAddr, FT); SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT); } @@ -817,7 +818,7 @@ void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF, void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) { GenDefaultInitialize Gen(getContext()); Address DstPtr = - Builder.CreateBitCast(Dst.getAddress(*this), CGM.Int8PtrPtrTy); + Builder.CreateElementBitCast(Dst.getAddress(*this), CGM.Int8PtrTy); Gen.setCGF(this); QualType QT = Dst.getType(); QT = Dst.isVolatile() ? QT.withVolatile() : QT; @@ -830,7 +831,7 @@ static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT, std::array Addrs) { auto SetArtificialLoc = ApplyDebugLocation::CreateArtificial(CGF); for (unsigned I = 0; I < N; ++I) - Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy); + Addrs[I] = CGF.Builder.CreateElementBitCast(Addrs[I], CGF.CGM.Int8PtrTy); QT = IsVolatile ? QT.withVolatile() : QT; Gen.callFunc(FuncName, QT, Addrs, CGF); } diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp index ac26f0d4232c..b5bcf157036d 100644 --- a/clang/lib/CodeGen/CGObjC.cpp +++ b/clang/lib/CodeGen/CGObjC.cpp @@ -3915,8 +3915,8 @@ static llvm::Value *emitIsPlatformVersionAtLeast(CodeGenFunction &CGF, Args.push_back( llvm::ConstantInt::get(CGM.Int32Ty, getBaseMachOPlatformID(TT))); Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Version.getMajor())); - Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Min ? *Min : 0)); - Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, SMin ? *SMin : 0)); + Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Min.getValueOr(0))); + Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, SMin.getValueOr(0))); }; assert(!Version.empty() && "unexpected empty version"); @@ -3952,8 +3952,8 @@ CodeGenFunction::EmitBuiltinAvailable(const VersionTuple &Version) { Optional Min = Version.getMinor(), SMin = Version.getSubminor(); llvm::Value *Args[] = { llvm::ConstantInt::get(CGM.Int32Ty, Version.getMajor()), - llvm::ConstantInt::get(CGM.Int32Ty, Min ? *Min : 0), - llvm::ConstantInt::get(CGM.Int32Ty, SMin ? *SMin : 0), + llvm::ConstantInt::get(CGM.Int32Ty, Min.getValueOr(0)), + llvm::ConstantInt::get(CGM.Int32Ty, SMin.getValueOr(0)) }; llvm::Value *CallRes = diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp index e016644150b4..b2bf60d2c0fc 100644 --- a/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/clang/lib/CodeGen/CGObjCGNU.cpp @@ -978,7 +978,9 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { // Look for an existing one llvm::StringMap::iterator old = ObjCStrings.find(Str); if (old != ObjCStrings.end()) - return ConstantAddress(old->getValue(), Align); + return ConstantAddress( + old->getValue(), old->getValue()->getType()->getPointerElementType(), + Align); bool isNonASCII = SL->containsNonAscii(); @@ -1000,7 +1002,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { auto *ObjCStr = llvm::ConstantExpr::getIntToPtr( llvm::ConstantInt::get(Int64Ty, str), IdTy); ObjCStrings[Str] = ObjCStr; - return ConstantAddress(ObjCStr, Align); + return ConstantAddress(ObjCStr, IdTy->getPointerElementType(), Align); } StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass; @@ -1114,7 +1116,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { llvm::Constant *ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStrGV, IdTy); ObjCStrings[Str] = ObjCStr; ConstantStrings.push_back(ObjCStr); - return ConstantAddress(ObjCStr, Align); + return ConstantAddress(ObjCStr, IdTy->getPointerElementType(), Align); } void PushProperty(ConstantArrayBuilder &PropertiesArray, @@ -2476,7 +2478,7 @@ ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) { // Look for an existing one llvm::StringMap::iterator old = ObjCStrings.find(Str); if (old != ObjCStrings.end()) - return ConstantAddress(old->getValue(), Align); + return ConstantAddress(old->getValue(), Int8Ty, Align); StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass; @@ -2503,7 +2505,7 @@ ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) { ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty); ObjCStrings[Str] = ObjCStr; ConstantStrings.push_back(ObjCStr); - return ConstantAddress(ObjCStr, Align); + return ConstantAddress(ObjCStr, Int8Ty, Align); } ///Generates a message send where the super is the receiver. This is a message diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp index 5b925359ac25..425d1a793439 100644 --- a/clang/lib/CodeGen/CGObjCMac.cpp +++ b/clang/lib/CodeGen/CGObjCMac.cpp @@ -1983,7 +1983,8 @@ CGObjCCommonMac::GenerateConstantNSString(const StringLiteral *Literal) { GetConstantStringEntry(NSConstantStringMap, Literal, StringLength); if (auto *C = Entry.second) - return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); + return ConstantAddress( + C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment())); // If we don't already have it, get _NSConstantStringClassReference. llvm::Constant *Class = getNSConstantStringClassRef(); @@ -2036,7 +2037,7 @@ CGObjCCommonMac::GenerateConstantNSString(const StringLiteral *Literal) { : NSStringSection); Entry.second = GV; - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); } enum { diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 75709b3c7e78..e35c15421520 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -15,6 +15,7 @@ #include "CGCleanup.h" #include "CGRecordLayout.h" #include "CodeGenFunction.h" +#include "TargetInfo.h" #include "clang/AST/APValue.h" #include "clang/AST/Attr.h" #include "clang/AST/Decl.h" @@ -687,8 +688,6 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, // Drill down to the base element type on both arrays. const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); - DestAddr = - CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); if (DRD) SrcAddr = CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); @@ -775,7 +774,7 @@ LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, } void ReductionCodeGen::emitAggregateInitialization( - CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, + CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, const OMPDeclareReductionDecl *DRD) { // Emit VarDecl with copy init for arrays. // Get the address of the original variable captured in current @@ -788,7 +787,7 @@ void ReductionCodeGen::emitAggregateInitialization( EmitDeclareReductionInit, EmitDeclareReductionInit ? ClausesData[N].ReductionOp : PrivateVD->getInit(), - DRD, SharedLVal.getAddress(CGF)); + DRD, SharedAddr); } ReductionCodeGen::ReductionCodeGen(ArrayRef Shareds, @@ -882,7 +881,7 @@ void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, } void ReductionCodeGen::emitInitialization( - CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, + CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref DefaultInit) { assert(SharedAddresses.size() > N && "No variable was generated"); const auto *PrivateVD = @@ -892,21 +891,15 @@ void ReductionCodeGen::emitInitialization( QualType PrivateType = PrivateVD->getType(); PrivateAddr = CGF.Builder.CreateElementBitCast( PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); - QualType SharedType = SharedAddresses[N].first.getType(); - SharedLVal = CGF.MakeAddrLValue( - CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF), - CGF.ConvertTypeForMem(SharedType)), - SharedType, SharedAddresses[N].first.getBaseInfo(), - CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType)); if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { if (DRD && DRD->getInitializer()) (void)DefaultInit(CGF); - emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD); + emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD); } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { (void)DefaultInit(CGF); + QualType SharedType = SharedAddresses[N].first.getType(); emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, - PrivateAddr, SharedLVal.getAddress(CGF), - SharedLVal.getType()); + PrivateAddr, SharedAddr, SharedType); } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && !CGF.isTrivialInitializer(PrivateVD->getInit())) { CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, @@ -2016,12 +2009,13 @@ Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, StringRef Name) { std::string Suffix = getName({"artificial", ""}); llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); - llvm::Value *GAddr = + llvm::GlobalVariable *GAddr = getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix)); if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS && CGM.getTarget().isTLSSupported()) { - cast(GAddr)->setThreadLocal(/*Val=*/true); - return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType)); + GAddr->setThreadLocal(/*Val=*/true); + return Address(GAddr, GAddr->getValueType(), + CGM.getContext().getTypeAlignInChars(VarType)); } std::string CacheSuffix = getName({"cache", ""}); llvm::Value *Args[] = { @@ -2084,7 +2078,8 @@ void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef CapturedVars, - const Expr *IfCond) { + const Expr *IfCond, + llvm::Value *NumThreads) { if (!CGF.HaveInsertPoint()) return; llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); @@ -2175,7 +2170,7 @@ Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, return ThreadIDTemp; } -llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable( +llvm::GlobalVariable *CGOpenMPRuntime::getOrCreateInternalVariable( llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) { SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); @@ -2183,7 +2178,7 @@ llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable( StringRef RuntimeName = Out.str(); auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first; if (Elem.second) { - assert(Elem.second->getType()->getPointerElementType() == Ty && + assert(Elem.second->getType()->isOpaqueOrPointeeTypeMatches(Ty) && "OMP internal variable has different type than requested"); return &*Elem.second; } @@ -4498,10 +4493,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, std::tie(Addr, Size) = getPointerAndSize(CGF, E); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); LValue Base = CGF.MakeAddrLValue( - Address(CGF.Builder.CreateGEP(AffinitiesArray.getElementType(), - AffinitiesArray.getPointer(), Idx), - AffinitiesArray.getAlignment()), - KmpTaskAffinityInfoTy); + CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); // affs[i].base_addr = &; LValue BaseAddrLVal = CGF.EmitLValueForField( Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); @@ -4665,12 +4657,10 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy)); Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); - llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.getPointer(), - llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); + Address DepObjAddr = CGF.Builder.CreateGEP( + Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( - Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy, - Base.getBaseInfo(), Base.getTBAAInfo()); + DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); // NumDeps = deps[i].base_addr; LValue BaseAddrLVal = CGF.EmitLValueForField( NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr)); @@ -4706,10 +4696,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue &PosLVal = *Pos.get(); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); Base = CGF.MakeAddrLValue( - Address(CGF.Builder.CreateGEP(DependenciesArray.getElementType(), - DependenciesArray.getPointer(), Idx), - DependenciesArray.getAlignment()), - KmpDependInfoTy); + CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); } // deps[i].base_addr = &; LValue BaseAddrLVal = CGF.EmitLValueForField( @@ -4766,12 +4753,10 @@ emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, Base.getAddress(CGF), KmpDependInfoPtrT); Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); - llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.getPointer(), - llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); + Address DepObjAddr = CGF.Builder.CreateGEP( + Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( - Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy, - Base.getBaseInfo(), Base.getTBAAInfo()); + DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); // NumDeps = deps[i].base_addr; LValue BaseAddrLVal = CGF.EmitLValueForField( NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr)); @@ -4827,12 +4812,10 @@ static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, Base.getTBAAInfo()); // Get number of elements in a single depobj. - llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.getPointer(), - llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); + Address DepObjAddr = CGF.Builder.CreateGEP( + Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( - Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy, - Base.getBaseInfo(), Base.getTBAAInfo()); + DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); // NumDeps = deps[i].base_addr; LValue BaseAddrLVal = CGF.EmitLValueForField( NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr)); @@ -4844,10 +4827,7 @@ static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, ElSize, CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - Address DepAddr = - Address(CGF.Builder.CreateGEP(DependenciesArray.getElementType(), - DependenciesArray.getPointer(), Pos), - DependenciesArray.getAlignment()); + Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); // Increase pos. @@ -5929,25 +5909,20 @@ static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM, CGM.getContext().getSizeType(), Loc); } RCG.emitAggregateType(CGF, N, Size); - LValue OrigLVal; + Address OrigAddr = Address::invalid(); // If initializer uses initializer from declare reduction construct, emit a // pointer to the address of the original reduction item (reuired by reduction // initializer) if (RCG.usesReductionInitializer(N)) { Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig); - SharedAddr = CGF.EmitLoadOfPointer( + OrigAddr = CGF.EmitLoadOfPointer( SharedAddr, CGM.getContext().VoidPtrTy.castAs()->getTypePtr()); - OrigLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy); - } else { - OrigLVal = CGF.MakeNaturalAlignAddrLValue( - llvm::ConstantPointerNull::get(CGM.VoidPtrTy), - CGM.getContext().VoidPtrTy); } // Emit the initializer: // %0 = bitcast void* %arg to * // store , * %0 - RCG.emitInitialization(CGF, N, PrivateAddr, OrigLVal, + RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr, [](CodeGenFunction &) { return false; }); CGF.FinishFunction(); return Fn; @@ -6122,7 +6097,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0), llvm::ConstantInt::get(CGM.SizeTy, Cnt)}; llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( - TaskRedInput.getPointer(), Idxs, + TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs, /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, ".rd_input.gep."); LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); @@ -6620,6 +6595,8 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper( OutlinedFn->addFnAttr("omp_target_thread_limit", std::to_string(DefaultValThreads)); } + + CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM); } /// Checks if the expression is constant or does not have non-trivial function @@ -12680,12 +12657,11 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, // Last value of the lastprivate conditional. // decltype(priv_a) last_a; - llvm::Constant *Last = getOrCreateInternalVariable( + llvm::GlobalVariable *Last = getOrCreateInternalVariable( CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); - cast(Last)->setAlignment( - LVal.getAlignment().getAsAlign()); - LValue LastLVal = - CGF.MakeAddrLValue(Last, LVal.getType(), LVal.getAlignment()); + Last->setAlignment(LVal.getAlignment().getAsAlign()); + LValue LastLVal = CGF.MakeAddrLValue( + Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType()); // Global loop counter. Required to handle inner parallel-for regions. // iv @@ -12812,7 +12788,7 @@ void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional( const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back()); for (const auto &Pair : It->DeclToUniqueName) { const auto *VD = cast(Pair.first->getCanonicalDecl()); - if (!CS->capturesVariable(VD) || IgnoredDecls.count(VD) > 0) + if (!CS->capturesVariable(VD) || IgnoredDecls.contains(VD)) continue; auto I = LPCI->getSecond().find(Pair.first); assert(I != LPCI->getSecond().end() && @@ -12858,7 +12834,8 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( if (!GV) return; LValue LPLVal = CGF.MakeAddrLValue( - GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment()); + Address(GV, GV->getValueType(), PrivLVal.getAlignment()), + PrivLVal.getType().getNonReferenceType()); llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); CGF.EmitStoreOfScalar(Res, PrivLVal); } @@ -12887,7 +12864,8 @@ void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef CapturedVars, - const Expr *IfCond) { + const Expr *IfCond, + llvm::Value *NumThreads) { llvm_unreachable("Not supported in SIMD-only mode"); } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h index 527a23a8af6a..b83ec78696d1 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -162,10 +162,10 @@ class ReductionCodeGen { /// Performs aggregate initialization. /// \param N Number of reduction item in the common list. /// \param PrivateAddr Address of the corresponding private item. - /// \param SharedLVal Address of the original shared variable. + /// \param SharedAddr Address of the original shared variable. /// \param DRD Declare reduction construct used for reduction item. void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N, - Address PrivateAddr, LValue SharedLVal, + Address PrivateAddr, Address SharedAddr, const OMPDeclareReductionDecl *DRD); public: @@ -187,10 +187,10 @@ class ReductionCodeGen { /// \param PrivateAddr Address of the corresponding private item. /// \param DefaultInit Default initialization sequence that should be /// performed if no reduction specific initialization is found. - /// \param SharedLVal Address of the original shared variable. + /// \param SharedAddr Address of the original shared variable. void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, - LValue SharedLVal, + Address SharedAddr, llvm::function_ref DefaultInit); /// Returns true if the private copy requires cleanups. bool needCleanups(unsigned N); @@ -471,8 +471,8 @@ class CGOpenMPRuntime { /// + ".var" for "omp critical" directives; 2) /// + ".cache." for cache for threadprivate /// variables. - llvm::StringMap, llvm::BumpPtrAllocator> - InternalVars; + llvm::StringMap, + llvm::BumpPtrAllocator> InternalVars; /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); llvm::Type *KmpRoutineEntryPtrTy = nullptr; QualType KmpRoutineEntryPtrQTy; @@ -829,9 +829,9 @@ class CGOpenMPRuntime { /// \param Ty Type of the global variable. If it is exist already the type /// must be the same. /// \param Name Name of the variable. - llvm::Constant *getOrCreateInternalVariable(llvm::Type *Ty, - const llvm::Twine &Name, - unsigned AddressSpace = 0); + llvm::GlobalVariable *getOrCreateInternalVariable(llvm::Type *Ty, + const llvm::Twine &Name, + unsigned AddressSpace = 0); /// Set of threadprivate variables with the generated initializer. llvm::StringSet<> ThreadPrivateWithDefinition; @@ -1015,11 +1015,13 @@ class CGOpenMPRuntime { /// variables used in \a OutlinedFn function. /// \param IfCond Condition in the associated 'if' clause, if it was /// specified, nullptr otherwise. + /// \param NumThreads The value corresponding to the num_threads clause, if + /// any, or nullptr. /// virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef CapturedVars, - const Expr *IfCond); + const Expr *IfCond, llvm::Value *NumThreads); /// Emits a critical region. /// \param CriticalName Name of the critical region. @@ -1991,11 +1993,13 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime { /// variables used in \a OutlinedFn function. /// \param IfCond Condition in the associated 'if' clause, if it was /// specified, nullptr otherwise. + /// \param NumThreads The value corresponding to the num_threads clause, if + /// any, or nullptr. /// void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef CapturedVars, - const Expr *IfCond) override; + const Expr *IfCond, llvm::Value *NumThreads) override; /// Emits a critical region. /// \param CriticalName Name of the critical region. diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index dcb224f33156..866454ddeaed 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -1221,11 +1221,7 @@ void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) { - // Do nothing in case of SPMD mode and L0 parallel. - if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD) - return; - - CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc); + // Nothing to do. } void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, @@ -1510,13 +1506,16 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef CapturedVars, - const Expr *IfCond) { + const Expr *IfCond, + llvm::Value *NumThreads) { if (!CGF.HaveInsertPoint()) return; - auto &&ParallelGen = [this, Loc, OutlinedFn, CapturedVars, - IfCond](CodeGenFunction &CGF, PrePostActionTy &Action) { + auto &&ParallelGen = [this, Loc, OutlinedFn, CapturedVars, IfCond, + NumThreads](CodeGenFunction &CGF, + PrePostActionTy &Action) { CGBuilderTy &Bld = CGF.Builder; + llvm::Value *NumThreadsVal = NumThreads; llvm::Function *WFn = WrapperFunctionsMap[OutlinedFn]; llvm::Value *ID = llvm::ConstantPointerNull::get(CGM.Int8PtrTy); if (WFn) @@ -1556,13 +1555,18 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, else IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); - assert(IfCondVal && "Expected a value"); + if (!NumThreadsVal) + NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1); + else + NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), + + assert(IfCondVal && "Expected a value"); llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); llvm::Value *Args[] = { RTLoc, getThreadID(CGF, Loc), IfCondVal, - llvm::ConstantInt::get(CGF.Int32Ty, -1), + NumThreadsVal, llvm::ConstantInt::get(CGF.Int32Ty, -1), FnPtr, ID, @@ -2186,11 +2190,8 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, // elemptr = ((CopyType*)(elemptrptr)) + I Address ElemPtr = Address(ElemPtrPtr, Align); ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType); - if (NumIters > 1) { - ElemPtr = Address(Bld.CreateGEP(ElemPtr.getElementType(), - ElemPtr.getPointer(), Cnt), - ElemPtr.getAlignment()); - } + if (NumIters > 1) + ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); // Get pointer to location in transfer medium. // MediumPtr = &medium[warp_id] @@ -2256,11 +2257,8 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc); Address TargetElemPtr = Address(TargetElemPtrVal, Align); TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType); - if (NumIters > 1) { - TargetElemPtr = Address(Bld.CreateGEP(TargetElemPtr.getElementType(), - TargetElemPtr.getPointer(), Cnt), - TargetElemPtr.getAlignment()); - } + if (NumIters > 1) + TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); // *TargetElemPtr = SrcMediumVal; llvm::Value *SrcMediumValue = @@ -3899,6 +3897,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective( case CudaArch::GFX1033: case CudaArch::GFX1034: case CudaArch::GFX1035: + case CudaArch::Generic: case CudaArch::UNUSED: case CudaArch::UNKNOWN: break; diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h index ac51264d7685..1d30c5061743 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h @@ -257,10 +257,13 @@ class CGOpenMPRuntimeGPU : public CGOpenMPRuntime { /// variables used in \a OutlinedFn function. /// \param IfCond Condition in the associated 'if' clause, if it was /// specified, nullptr otherwise. + /// \param NumThreads The value corresponding to the num_threads clause, if + /// any, + /// or nullptr. void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef CapturedVars, - const Expr *IfCond) override; + const Expr *IfCond, llvm::Value *NumThreads) override; /// Emit an implicit/explicit barrier for OpenMP threads. /// \param Kind Directive for which this implicit barrier call must be diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index d399ff919cc3..ef0068cd3b0c 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -2454,7 +2454,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo(); if (RetAI.isDirect() || RetAI.isExtend()) { // Make a fake lvalue for the return value slot. - LValue ReturnSlot = MakeAddrLValue(ReturnValue, FnRetTy); + LValue ReturnSlot = MakeAddrLValueWithoutTBAA(ReturnValue, FnRetTy); CGM.getTargetCodeGenInfo().addReturnRegisterOutputs( *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes, ResultRegDests, AsmString, S.getNumOutputs()); diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp index f6853a22cd36..4c11f7d67534 100644 --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -24,10 +24,13 @@ #include "clang/AST/StmtVisitor.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/PrettyStackTrace.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/Frontend/OpenMP/OMPConstants.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" #include "llvm/Support/AtomicOrdering.h" using namespace clang; using namespace CodeGen; @@ -375,8 +378,7 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), Ctx.getPointerType(DstType), Loc); Address TmpAddr = - CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) - .getAddress(CGF); + CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); return TmpAddr; } @@ -1245,7 +1247,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit( RedCG.emitAggregateType(*this, Count); AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(), - RedCG.getSharedLValue(Count), + RedCG.getSharedLValue(Count).getAddress(*this), [&Emission](CodeGenFunction &CGF) { CGF.EmitAutoVarInit(Emission); return true; @@ -1557,14 +1559,14 @@ static void emitCommonOMPParallelDirective( OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters) { const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); + llvm::Value *NumThreads = nullptr; llvm::Function *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); if (const auto *NumThreadsClause = S.getSingleClause()) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); - llvm::Value *NumThreads = - CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), - /*IgnoreResultAssign=*/true); + NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), + /*IgnoreResultAssign=*/true); CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( CGF, NumThreads, NumThreadsClause->getBeginLoc()); } @@ -1591,7 +1593,7 @@ static void emitCommonOMPParallelDirective( CodeGenBoundParameters(CGF, S, CapturedVars); CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, - CapturedVars, IfCond); + CapturedVars, IfCond, NumThreads); } static bool isAllocatableDecl(const VarDecl *VD) { @@ -1972,7 +1974,7 @@ CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) { // Pop the \p Depth loops requested by the call from that stack and restore // the previous context. - OMPLoopNestStack.set_size(OMPLoopNestStack.size() - Depth); + OMPLoopNestStack.pop_back_n(Depth); ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth; return Result; @@ -4299,10 +4301,10 @@ class CheckVarsEscapingUntiedTaskDeclContext final PrivateDecls.push_back(VD); } } - void VisitOMPExecutableDirective(const OMPExecutableDirective *) { return; } - void VisitCapturedStmt(const CapturedStmt *) { return; } - void VisitLambdaExpr(const LambdaExpr *) { return; } - void VisitBlockExpr(const BlockExpr *) { return; } + void VisitOMPExecutableDirective(const OMPExecutableDirective *) {} + void VisitCapturedStmt(const CapturedStmt *) {} + void VisitLambdaExpr(const LambdaExpr *) {} + void VisitBlockExpr(const BlockExpr *) {} void VisitStmt(const Stmt *S) { if (!S) return; @@ -4431,6 +4433,53 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( UntiedLocalVars; // Set proper addresses for generated private copies. OMPPrivateScope Scope(CGF); + // Generate debug info for variables present in shared clause. + if (auto *DI = CGF.getDebugInfo()) { + llvm::SmallDenseMap CaptureFields = + CGF.CapturedStmtInfo->getCaptureFields(); + llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue(); + if (CaptureFields.size() && ContextValue) { + unsigned CharWidth = CGF.getContext().getCharWidth(); + // The shared variables are packed together as members of structure. + // So the address of each shared variable can be computed by adding + // offset of it (within record) to the base address of record. For each + // shared variable, debug intrinsic llvm.dbg.declare is generated with + // appropriate expressions (DIExpression). + // Ex: + // %12 = load %struct.anon*, %struct.anon** %__context.addr.i + // call void @llvm.dbg.declare(metadata %struct.anon* %12, + // metadata !svar1, + // metadata !DIExpression(DW_OP_deref)) + // call void @llvm.dbg.declare(metadata %struct.anon* %12, + // metadata !svar2, + // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref)) + for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) { + const VarDecl *SharedVar = It->first; + RecordDecl *CaptureRecord = It->second->getParent(); + const ASTRecordLayout &Layout = + CGF.getContext().getASTRecordLayout(CaptureRecord); + unsigned Offset = + Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth; + (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue, + CGF.Builder, false); + llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back(); + // Get the call dbg.declare instruction we just created and update + // its DIExpression to add offset to base address. + if (auto DDI = dyn_cast(&Last)) { + SmallVector Ops; + // Add offset to the base address if non zero. + if (Offset) { + Ops.push_back(llvm::dwarf::DW_OP_plus_uconst); + Ops.push_back(Offset); + } + Ops.push_back(llvm::dwarf::DW_OP_deref); + auto &Ctx = DDI->getContext(); + llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops); + Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr)); + } + } + } + } llvm::SmallVector, 16> FirstprivatePtrs; if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() || !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) { @@ -5918,6 +5967,9 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, IsXLHSInRHSPart, Loc); break; + case OMPC_compare: + // Do nothing here as we already emit an error. + break; case OMPC_if: case OMPC_final: case OMPC_num_threads: diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h index 4b39a0520833..f01eece042f8 100644 --- a/clang/lib/CodeGen/CGValue.h +++ b/clang/lib/CodeGen/CGValue.h @@ -47,6 +47,8 @@ class RValue { llvm::PointerIntPair V1; // Stores second value and volatility. llvm::PointerIntPair V2; + // Stores element type for aggregate values. + llvm::Type *ElementType; public: bool isScalar() const { return V1.getInt() == Scalar; } @@ -71,7 +73,8 @@ class RValue { Address getAggregateAddress() const { assert(isAggregate() && "Not an aggregate!"); auto align = reinterpret_cast(V2.getPointer()) >> AggAlignShift; - return Address(V1.getPointer(), CharUnits::fromQuantity(align)); + return Address( + V1.getPointer(), ElementType, CharUnits::fromQuantity(align)); } llvm::Value *getAggregatePointer() const { assert(isAggregate() && "Not an aggregate!"); @@ -108,6 +111,7 @@ class RValue { RValue ER; ER.V1.setPointer(addr.getPointer()); ER.V1.setInt(Aggregate); + ER.ElementType = addr.getElementType(); auto align = static_cast(addr.getAlignment().getQuantity()); ER.V2.setPointer(reinterpret_cast(align << AggAlignShift)); @@ -175,6 +179,7 @@ class LValue { } LVType; llvm::Value *V; + llvm::Type *ElementType; union { // Index into a vector subscript: V[i] @@ -230,6 +235,13 @@ class LValue { LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { assert((!Alignment.isZero() || Type->isIncompleteType()) && "initializing l-value with zero alignment!"); + if (isGlobalReg()) + assert(ElementType == nullptr && "Global reg does not store elem type"); + else + assert(llvm::cast(V->getType()) + ->isOpaqueOrPointeeTypeMatches(ElementType) && + "Pointer element type mismatch"); + this->Type = Type; this->Quals = Quals; const unsigned MaxAlign = 1U << 31; @@ -327,17 +339,18 @@ class LValue { return V; } Address getAddress(CodeGenFunction &CGF) const { - return Address(getPointer(CGF), getAlignment()); + return Address(getPointer(CGF), ElementType, getAlignment()); } void setAddress(Address address) { assert(isSimple()); V = address.getPointer(); + ElementType = address.getElementType(); Alignment = address.getAlignment().getQuantity(); } // vector elt lvalue Address getVectorAddress() const { - return Address(getVectorPointer(), getAlignment()); + return Address(getVectorPointer(), ElementType, getAlignment()); } llvm::Value *getVectorPointer() const { assert(isVectorElt()); @@ -349,7 +362,7 @@ class LValue { } Address getMatrixAddress() const { - return Address(getMatrixPointer(), getAlignment()); + return Address(getMatrixPointer(), ElementType, getAlignment()); } llvm::Value *getMatrixPointer() const { assert(isMatrixElt()); @@ -362,7 +375,7 @@ class LValue { // extended vector elements. Address getExtVectorAddress() const { - return Address(getExtVectorPointer(), getAlignment()); + return Address(getExtVectorPointer(), ElementType, getAlignment()); } llvm::Value *getExtVectorPointer() const { assert(isExtVectorElt()); @@ -375,7 +388,7 @@ class LValue { // bitfield lvalue Address getBitFieldAddress() const { - return Address(getBitFieldPointer(), getAlignment()); + return Address(getBitFieldPointer(), ElementType, getAlignment()); } llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } const CGBitFieldInfo &getBitFieldInfo() const { @@ -395,6 +408,7 @@ class LValue { R.LVType = Simple; assert(address.getPointer()->getType()->isPointerTy()); R.V = address.getPointer(); + R.ElementType = address.getElementType(); R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo); return R; } @@ -405,6 +419,7 @@ class LValue { LValue R; R.LVType = VectorElt; R.V = vecAddress.getPointer(); + R.ElementType = vecAddress.getElementType(); R.VectorIdx = Idx; R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), BaseInfo, TBAAInfo); @@ -417,6 +432,7 @@ class LValue { LValue R; R.LVType = ExtVectorElt; R.V = vecAddress.getPointer(); + R.ElementType = vecAddress.getElementType(); R.VectorElts = Elts; R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), BaseInfo, TBAAInfo); @@ -435,17 +451,20 @@ class LValue { LValue R; R.LVType = BitField; R.V = Addr.getPointer(); + R.ElementType = Addr.getElementType(); R.BitFieldInfo = &Info; R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo, TBAAInfo); return R; } - static LValue MakeGlobalReg(Address Reg, QualType type) { + static LValue MakeGlobalReg(llvm::Value *V, CharUnits alignment, + QualType type) { LValue R; R.LVType = GlobalReg; - R.V = Reg.getPointer(); - R.Initialize(type, type.getQualifiers(), Reg.getAlignment(), + R.V = V; + R.ElementType = nullptr; + R.Initialize(type, type.getQualifiers(), alignment, LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); return R; } @@ -456,6 +475,7 @@ class LValue { LValue R; R.LVType = MatrixElt; R.V = matAddress.getPointer(); + R.ElementType = matAddress.getElementType(); R.VectorIdx = Idx; R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(), BaseInfo, TBAAInfo); @@ -470,13 +490,11 @@ class LValue { /// An aggregate value slot. class AggValueSlot { /// The address. - llvm::Value *Addr; + Address Addr; // Qualifiers Qualifiers Quals; - unsigned Alignment; - /// DestructedFlag - This is set to true if some external code is /// responsible for setting up a destructor for the slot. Otherwise /// the code which constructs it should push the appropriate cleanup. @@ -520,6 +538,14 @@ class AggValueSlot { /// them. bool SanitizerCheckedFlag : 1; + AggValueSlot(Address Addr, Qualifiers Quals, bool DestructedFlag, + bool ObjCGCFlag, bool ZeroedFlag, bool AliasedFlag, + bool OverlapFlag, bool SanitizerCheckedFlag) + : Addr(Addr), Quals(Quals), DestructedFlag(DestructedFlag), + ObjCGCFlag(ObjCGCFlag), ZeroedFlag(ZeroedFlag), + AliasedFlag(AliasedFlag), OverlapFlag(OverlapFlag), + SanitizerCheckedFlag(SanitizerCheckedFlag) {} + public: enum IsAliased_t { IsNotAliased, IsAliased }; enum IsDestructed_t { IsNotDestructed, IsDestructed }; @@ -553,22 +579,8 @@ class AggValueSlot { Overlap_t mayOverlap, IsZeroed_t isZeroed = IsNotZeroed, IsSanitizerChecked_t isChecked = IsNotSanitizerChecked) { - AggValueSlot AV; - if (addr.isValid()) { - AV.Addr = addr.getPointer(); - AV.Alignment = addr.getAlignment().getQuantity(); - } else { - AV.Addr = nullptr; - AV.Alignment = 0; - } - AV.Quals = quals; - AV.DestructedFlag = isDestructed; - AV.ObjCGCFlag = needsGC; - AV.ZeroedFlag = isZeroed; - AV.AliasedFlag = isAliased; - AV.OverlapFlag = mayOverlap; - AV.SanitizerCheckedFlag = isChecked; - return AV; + return AggValueSlot(addr, quals, isDestructed, needsGC, isZeroed, isAliased, + mayOverlap, isChecked); } static AggValueSlot @@ -609,19 +621,19 @@ class AggValueSlot { } llvm::Value *getPointer() const { - return Addr; + return Addr.getPointer(); } Address getAddress() const { - return Address(Addr, getAlignment()); + return Addr; } bool isIgnored() const { - return Addr == nullptr; + return !Addr.isValid(); } CharUnits getAlignment() const { - return CharUnits::fromQuantity(Alignment); + return Addr.getAlignment(); } IsAliased_t isPotentiallyAliased() const { diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp index 52c54d3c7a72..b72b16cf2b5f 100644 --- a/clang/lib/CodeGen/CodeGenAction.cpp +++ b/clang/lib/CodeGen/CodeGenAction.cpp @@ -571,7 +571,6 @@ void BackendConsumer::SrcMgrDiagHandler(const llvm::DiagnosticInfoSrcMgr &DI) { // If Loc is invalid, we still need to report the issue, it just gets no // location info. Diags.Report(Loc, DiagID).AddString(Message); - return; } bool diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index d87cf2d49720..e6adec6948af 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -188,8 +188,8 @@ LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo); - return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo, - TBAAInfo); + Address Addr(V, ConvertTypeForMem(T), Alignment); + return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo); } /// Given a value of type T* that may not be to a complete object, @@ -200,7 +200,8 @@ CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { TBAAAccessInfo TBAAInfo; CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, /* forPointeeType= */ true); - return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo); + Address Addr(V, ConvertTypeForMem(T), Align); + return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); } @@ -243,7 +244,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) { case Type::Enum: case Type::ObjCObjectPointer: case Type::Pipe: - case Type::ExtInt: + case Type::BitInt: return TEK_Scalar; // Complexes. @@ -1070,7 +1071,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign()); + ReturnValue = Address(&*AI, ConvertType(RetTy), + CurFnInfo->getReturnInfo().getIndirectAlign()); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { ReturnValuePointer = CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr"); @@ -1298,47 +1300,44 @@ QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD, void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, const CGFunctionInfo &FnInfo) { + assert(Fn && "generating code for null Function"); const FunctionDecl *FD = cast(GD.getDecl()); CurGD = GD; FunctionArgList Args; QualType ResTy = BuildFunctionArgList(GD, Args); - // When generating code for a builtin with an inline declaration, use a - // mangled name to hold the actual body, while keeping an external definition - // in case the function pointer is referenced somewhere. - if (Fn) { - if (FD->isInlineBuiltinDeclaration()) { - std::string FDInlineName = (Fn->getName() + ".inline").str(); - llvm::Module *M = Fn->getParent(); - llvm::Function *Clone = M->getFunction(FDInlineName); - if (!Clone) { - Clone = llvm::Function::Create(Fn->getFunctionType(), - llvm::GlobalValue::InternalLinkage, - Fn->getAddressSpace(), FDInlineName, M); - Clone->addFnAttr(llvm::Attribute::AlwaysInline); - } - Fn->setLinkage(llvm::GlobalValue::ExternalLinkage); - Fn = Clone; + if (FD->isInlineBuiltinDeclaration()) { + // When generating code for a builtin with an inline declaration, use a + // mangled name to hold the actual body, while keeping an external + // definition in case the function pointer is referenced somewhere. + std::string FDInlineName = (Fn->getName() + ".inline").str(); + llvm::Module *M = Fn->getParent(); + llvm::Function *Clone = M->getFunction(FDInlineName); + if (!Clone) { + Clone = llvm::Function::Create(Fn->getFunctionType(), + llvm::GlobalValue::InternalLinkage, + Fn->getAddressSpace(), FDInlineName, M); + Clone->addFnAttr(llvm::Attribute::AlwaysInline); } - + Fn->setLinkage(llvm::GlobalValue::ExternalLinkage); + Fn = Clone; + } else { // Detect the unusual situation where an inline version is shadowed by a // non-inline version. In that case we should pick the external one // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way // to detect that situation before we reach codegen, so do some late // replacement. - else { - for (const FunctionDecl *PD = FD->getPreviousDecl(); PD; - PD = PD->getPreviousDecl()) { - if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) { - std::string FDInlineName = (Fn->getName() + ".inline").str(); - llvm::Module *M = Fn->getParent(); - if (llvm::Function *Clone = M->getFunction(FDInlineName)) { - Clone->replaceAllUsesWith(Fn); - Clone->eraseFromParent(); - } - break; + for (const FunctionDecl *PD = FD->getPreviousDecl(); PD; + PD = PD->getPreviousDecl()) { + if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) { + std::string FDInlineName = (Fn->getName() + ".inline").str(); + llvm::Module *M = Fn->getParent(); + if (llvm::Function *Clone = M->getFunction(FDInlineName)) { + Clone->replaceAllUsesWith(Fn); + Clone->eraseFromParent(); } + break; } } } @@ -1347,8 +1346,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, if (FD->hasAttr()) { // Clear non-distinct debug info that was possibly attached to the function // due to an earlier declaration without the nodebug attribute - if (Fn) - Fn->setSubprogram(nullptr); + Fn->setSubprogram(nullptr); // Disable debug info indefinitely for this function DebugInfo = nullptr; } @@ -2202,12 +2200,13 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { case Type::Record: case Type::Enum: case Type::Elaborated: + case Type::Using: case Type::TemplateSpecialization: case Type::ObjCTypeParam: case Type::ObjCObject: case Type::ObjCInterface: case Type::ObjCObjectPointer: - case Type::ExtInt: + case Type::BitInt: llvm_unreachable("type class is never variably-modified!"); case Type::Adjusted: diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index ff5b6634da1c..f76ce8a6400d 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -459,6 +459,11 @@ class CodeGenFunction : public CodeGenTypeCache { /// Get the name of the capture helper. virtual StringRef getHelperName() const { return "__captured_stmt"; } + /// Get the CaptureFields + llvm::SmallDenseMap getCaptureFields() { + return CaptureFields; + } + private: /// The kind of captured statement being generated. CapturedRegionKind Kind; @@ -2494,14 +2499,16 @@ class CodeGenFunction : public CodeGenTypeCache { LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source = AlignmentSource::Type) { - return LValue::MakeAddr(Address(V, Alignment), T, getContext(), - LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); + Address Addr(V, ConvertTypeForMem(T), Alignment); + return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), + CGM.getTBAAAccessInfo(T)); } - LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, - LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - return LValue::MakeAddr(Address(V, Alignment), T, getContext(), - BaseInfo, TBAAInfo); + LValue + MakeAddrLValueWithoutTBAA(Address Addr, QualType T, + AlignmentSource Source = AlignmentSource::Type) { + return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), + TBAAAccessInfo()); } LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T); @@ -3128,15 +3135,18 @@ class CodeGenFunction : public CodeGenTypeCache { class ParamValue { llvm::Value *Value; + llvm::Type *ElementType; unsigned Alignment; - ParamValue(llvm::Value *V, unsigned A) : Value(V), Alignment(A) {} + ParamValue(llvm::Value *V, llvm::Type *T, unsigned A) + : Value(V), ElementType(T), Alignment(A) {} public: static ParamValue forDirect(llvm::Value *value) { - return ParamValue(value, 0); + return ParamValue(value, nullptr, 0); } static ParamValue forIndirect(Address addr) { assert(!addr.getAlignment().isZero()); - return ParamValue(addr.getPointer(), addr.getAlignment().getQuantity()); + return ParamValue(addr.getPointer(), addr.getElementType(), + addr.getAlignment().getQuantity()); } bool isIndirect() const { return Alignment != 0; } @@ -3149,7 +3159,7 @@ class CodeGenFunction : public CodeGenTypeCache { Address getIndirectAddress() const { assert(isIndirect()); - return Address(Value, CharUnits::fromQuantity(Alignment)); + return Address(Value, ElementType, CharUnits::fromQuantity(Alignment)); } }; @@ -4405,7 +4415,7 @@ class CodeGenFunction : public CodeGenTypeCache { /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++ /// variable with global storage. - void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr, + void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV, bool PerformInit); llvm::Function *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor, @@ -4556,7 +4566,7 @@ class CodeGenFunction : public CodeGenTypeCache { /// \p SignedIndices indicates whether any of the GEP indices are signed. /// \p IsSubtraction indicates whether the expression used to form the GEP /// is a subtraction. - llvm::Value *EmitCheckedInBoundsGEP(llvm::Value *Ptr, + llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef IdxList, bool SignedIndices, bool IsSubtraction, diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 9ba1a5c25e81..36b7ce87336c 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -2832,7 +2832,7 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) { // Look for an existing global. if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name)) - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); ConstantEmitter Emitter(*this); llvm::Constant *Init; @@ -2866,15 +2866,15 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) { GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); setDSOLocal(GV); - llvm::Constant *Addr = GV; if (!V.isAbsent()) { Emitter.finalize(GV); - } else { - llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType()); - Addr = llvm::ConstantExpr::getBitCast( - GV, Ty->getPointerTo(GV->getAddressSpace())); + return ConstantAddress(GV, GV->getValueType(), Alignment); } - return ConstantAddress(Addr, Alignment); + + llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType()); + llvm::Constant *Addr = llvm::ConstantExpr::getBitCast( + GV, Ty->getPointerTo(GV->getAddressSpace())); + return ConstantAddress(Addr, Ty, Alignment); } ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject( @@ -2883,7 +2883,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject( CharUnits Alignment = getNaturalTypeAlignment(TPO->getType()); if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name)) - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); ConstantEmitter Emitter(*this); llvm::Constant *Init = Emitter.emitForInitializer( @@ -2901,7 +2901,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject( GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); Emitter.finalize(GV); - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); } ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { @@ -2916,7 +2916,7 @@ ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { if (Entry) { unsigned AS = getContext().getTargetAddressSpace(VD->getType()); auto Ptr = llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS)); - return ConstantAddress(Ptr, Alignment); + return ConstantAddress(Ptr, DeclTy, Alignment); } llvm::Constant *Aliasee; @@ -2932,7 +2932,7 @@ ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { F->setLinkage(llvm::Function::ExternalWeakLinkage); WeakRefReferences.insert(F); - return ConstantAddress(Aliasee, Alignment); + return ConstantAddress(Aliasee, DeclTy, Alignment); } void CodeGenModule::EmitGlobal(GlobalDecl GD) { @@ -3886,6 +3886,14 @@ llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD, return F; } +llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) { + llvm::GlobalValue *F = + cast(GetAddrOfFunction(Decl)->stripPointerCasts()); + + return llvm::ConstantExpr::getBitCast(llvm::NoCFIValue::get(F), + llvm::Type::getInt8PtrTy(VMContext)); +} + static const FunctionDecl * GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) { TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl(); @@ -5228,7 +5236,8 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { StringLength); if (auto *C = Entry.second) - return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); + return ConstantAddress( + C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment())); llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); llvm::Constant *Zeros[] = { Zero, Zero }; @@ -5409,7 +5418,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { } Entry.second = GV; - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); } bool CodeGenModule::getExpressionLocationsEnabled() const { @@ -5527,7 +5536,7 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, if (uint64_t(Alignment.getQuantity()) > GV->getAlignment()) GV->setAlignment(Alignment.getAsAlign()); return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } } @@ -5557,7 +5566,7 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, QualType()); return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } /// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant @@ -5590,7 +5599,7 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString( if (uint64_t(Alignment.getQuantity()) > GV->getAlignment()) GV->setAlignment(Alignment.getAsAlign()); return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } } @@ -5604,7 +5613,7 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString( *Entry = GV; return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( @@ -5634,7 +5643,9 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( getModule(), Type, false, llvm::GlobalVariable::InternalLinkage, nullptr); } - return ConstantAddress(InsertResult.first->second, Align); + return ConstantAddress( + InsertResult.first->second, + InsertResult.first->second->getType()->getPointerElementType(), Align); } // FIXME: If an externally-visible declaration extends multiple temporaries, @@ -5725,7 +5736,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( } Entry = CV; - return ConstantAddress(CV, Align); + return ConstantAddress(CV, Type, Align); } /// EmitObjCPropertyImplementations - Emit information for synthesized @@ -6398,6 +6409,11 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { llvm::Metadata * CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, StringRef Suffix) { + if (auto *FnType = T->getAs()) + T = getContext().getFunctionType( + FnType->getReturnType(), FnType->getParamTypes(), + FnType->getExtProtoInfo().withExceptionSpec(EST_None)); + llvm::Metadata *&InternalId = Map[T.getCanonicalType()]; if (InternalId) return InternalId; diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h index e1c7f486d334..f1565511f98a 100644 --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -881,6 +881,9 @@ class CodeGenModule : public CodeGenTypeCache { ForDefinition_t IsForDefinition = NotForDefinition); + // Return the function body address of the given function. + llvm::Constant *GetFunctionStart(const ValueDecl *Decl); + /// Get the address of the RTTI descriptor for the given type. llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false); diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp index f4ebe6885675..95763d8e18b7 100644 --- a/clang/lib/CodeGen/CodeGenTBAA.cpp +++ b/clang/lib/CodeGen/CodeGenTBAA.cpp @@ -209,12 +209,12 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) { return createScalarTypeNode(OutName, getChar(), Size); } - if (const auto *EIT = dyn_cast(Ty)) { + if (const auto *EIT = dyn_cast(Ty)) { SmallString<256> OutName; llvm::raw_svector_ostream Out(OutName); // Don't specify signed/unsigned since integer types can alias despite sign // differences. - Out << "_ExtInt(" << EIT->getNumBits() << ')'; + Out << "_BitInt(" << EIT->getNumBits() << ')'; return createScalarTypeNode(OutName, getChar(), Size); } diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp index fb05475a4e8c..77721510dfd0 100644 --- a/clang/lib/CodeGen/CodeGenTypes.cpp +++ b/clang/lib/CodeGen/CodeGenTypes.cpp @@ -97,10 +97,10 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T, bool ForBitField) { llvm::Type *R = ConvertType(T); - // If this is a bool type, or an ExtIntType in a bitfield representation, - // map this integer to the target-specified size. - if ((ForBitField && T->isExtIntType()) || - (!T->isExtIntType() && R->isIntegerTy(1))) + // If this is a bool type, or a bit-precise integer type in a bitfield + // representation, map this integer to the target-specified size. + if ((ForBitField && T->isBitIntType()) || + (!T->isBitIntType() && R->isIntegerTy(1))) return llvm::IntegerType::get(getLLVMContext(), (unsigned)Context.getTypeSize(T)); @@ -786,8 +786,8 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { ResultType = CGM.getOpenCLRuntime().getPipeType(cast(Ty)); break; } - case Type::ExtInt: { - const auto &EIT = cast(Ty); + case Type::BitInt: { + const auto &EIT = cast(Ty); ResultType = llvm::Type::getIntNTy(getLLVMContext(), EIT->getNumBits()); break; } diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index 04163aeaddc5..1a15b09c7b2b 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -1345,7 +1345,8 @@ void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception"); CharUnits ExnAlign = CGF.getContext().getExnObjectAlignment(); - CGF.EmitAnyExprToExn(E->getSubExpr(), Address(ExceptionPtr, ExnAlign)); + CGF.EmitAnyExprToExn( + E->getSubExpr(), Address(ExceptionPtr, CGM.Int8Ty, ExnAlign)); // Now throw the exception. llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, @@ -2465,7 +2466,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, CGM.setStaticLocalDeclGuardAddress(&D, guard); } - Address guardAddr = Address(guard, guardAlignment); + Address guardAddr = Address(guard, guard->getValueType(), guardAlignment); // Test whether the variable has completed initialization. // @@ -2880,7 +2881,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( Guard->setAlignment(GuardAlign.getAsAlign()); CodeGenFunction(CGM).GenerateCXXGlobalInitFunc( - InitFunc, OrderedInits, ConstantAddress(Guard, GuardAlign)); + InitFunc, OrderedInits, ConstantAddress(Guard, CGM.Int8Ty, GuardAlign)); // On Darwin platforms, use CXX_FAST_TLS calling convention. if (CGM.getTarget().getTriple().isOSDarwin()) { InitFunc->setCallingConv(llvm::CallingConv::CXX_FAST_TLS); @@ -3529,7 +3530,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) { llvm_unreachable("Pipe types shouldn't get here"); case Type::Builtin: - case Type::ExtInt: + case Type::BitInt: // GCC treats vector and complex types as fundamental types. case Type::Vector: case Type::ExtVector: @@ -3802,7 +3803,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo( case Type::Pipe: break; - case Type::ExtInt: + case Type::BitInt: break; case Type::ConstantArray: diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 0fd5a0ffe06c..5971a7709304 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -917,7 +917,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, std::tuple MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy) { - Value = CGF.Builder.CreateBitCast(Value, CGF.Int8PtrTy); + Value = CGF.Builder.CreateElementBitCast(Value, CGF.Int8Ty); const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); const ASTContext &Context = getContext(); @@ -2408,14 +2408,14 @@ static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) { StringRef VarName("_Init_thread_epoch"); CharUnits Align = CGM.getIntAlign(); if (auto *GV = CGM.getModule().getNamedGlobal(VarName)) - return ConstantAddress(GV, Align); + return ConstantAddress(GV, GV->getValueType(), Align); auto *GV = new llvm::GlobalVariable( CGM.getModule(), CGM.IntTy, /*isConstant=*/false, llvm::GlobalVariable::ExternalLinkage, /*Initializer=*/nullptr, VarName, /*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel); GV->setAlignment(Align.getAsAlign()); - return ConstantAddress(GV, Align); + return ConstantAddress(GV, GV->getValueType(), Align); } static llvm::FunctionCallee getInitThreadHeaderFn(CodeGenModule &CGM) { @@ -2567,7 +2567,7 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, GI->Guard = GuardVar; } - ConstantAddress GuardAddr(GuardVar, GuardAlign); + ConstantAddress GuardAddr(GuardVar, GuardTy, GuardAlign); assert(GuardVar->getLinkage() == GV->getLinkage() && "static local from the same function had different linkage"); diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp index 36e0319c8ab9..85089cdb2200 100644 --- a/clang/lib/CodeGen/TargetInfo.cpp +++ b/clang/lib/CodeGen/TargetInfo.cpp @@ -104,7 +104,7 @@ bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const { if (Ty->isPromotableIntegerType()) return true; - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy)) return true; @@ -431,7 +431,7 @@ static Address emitMergePHI(CodeGenFunction &CGF, PHI->addIncoming(Addr1.getPointer(), Block1); PHI->addIncoming(Addr2.getPointer(), Block2); CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment()); - return Address(PHI, Align); + return Address(PHI, Addr1.getElementType(), Align); } TargetCodeGenInfo::~TargetCodeGenInfo() = default; @@ -762,7 +762,7 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { Ty = EnumTy->getDecl()->getIntegerType(); ASTContext &Context = getContext(); - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > Context.getTypeSize(Context.getTargetInfo().hasInt128Type() ? Context.Int128Ty @@ -784,7 +784,7 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const { if (const EnumType *EnumTy = RetTy->getAs()) RetTy = EnumTy->getDecl()->getIntegerType(); - if (const auto *EIT = RetTy->getAs()) + if (const auto *EIT = RetTy->getAs()) if (EIT->getNumBits() > getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type() ? getContext().Int128Ty @@ -1008,8 +1008,9 @@ ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const { } else if (Ty->isFloatingType()) { // Floating-point types don't go inreg. return ABIArgInfo::getDirect(); - } else if (const auto *EIT = Ty->getAs()) { - // Treat extended integers as integers if <=64, otherwise pass indirectly. + } else if (const auto *EIT = Ty->getAs()) { + // Treat bit-precise integers as integers if <= 64, otherwise pass + // indirectly. if (EIT->getNumBits() > 64) return getNaturalAlignIndirect(Ty); return ABIArgInfo::getDirect(); @@ -1027,8 +1028,8 @@ ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const { if (isAggregateTypeForABI(RetTy)) return getNaturalAlignIndirect(RetTy); - // Treat extended integers as integers if <=64, otherwise pass indirectly. - if (const auto *EIT = RetTy->getAs()) { + // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly. + if (const auto *EIT = RetTy->getAs()) { if (EIT->getNumBits() > 64) return getNaturalAlignIndirect(RetTy); return ABIArgInfo::getDirect(); @@ -1590,7 +1591,7 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, if (const EnumType *EnumTy = RetTy->getAs()) RetTy = EnumTy->getDecl()->getIntegerType(); - if (const auto *EIT = RetTy->getAs()) + if (const auto *EIT = RetTy->getAs()) if (EIT->getNumBits() > 64) return getIndirectReturnResult(RetTy, State); @@ -1926,7 +1927,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, return ABIArgInfo::getExtend(Ty); } - if (const auto * EIT = Ty->getAs()) { + if (const auto *EIT = Ty->getAs()) { if (EIT->getNumBits() <= 64) { if (InReg) return ABIArgInfo::getDirectInReg(); @@ -3009,7 +3010,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, return; } - if (const auto *EITy = Ty->getAs()) { + if (const auto *EITy = Ty->getAs()) { if (EITy->getNumBits() <= 64) Current = Integer; else if (EITy->getNumBits() <= 128) @@ -3200,7 +3201,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const { if (const EnumType *EnumTy = Ty->getAs()) Ty = EnumTy->getDecl()->getIntegerType(); - if (Ty->isExtIntType()) + if (Ty->isBitIntType()) return getNaturalAlignIndirect(Ty); return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) @@ -3237,7 +3238,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty, // but this code would be much safer if we could mark the argument with // 'onstack'. See PR12193. if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) && - !Ty->isExtIntType()) { + !Ty->isBitIntType()) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs()) Ty = EnumTy->getDecl()->getIntegerType(); @@ -4033,7 +4034,7 @@ static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF, CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p); // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type. - return Address(Res, Align); + return Address(Res, LTy, Align); } Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, @@ -4146,7 +4147,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy); } else if (neededInt) { RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset), - CharUnits::fromQuantity(8)); + CGF.Int8Ty, CharUnits::fromQuantity(8)); RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy); // Copy to a temporary if necessary to ensure the appropriate alignment. @@ -4164,7 +4165,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, } else if (neededSSE == 1) { RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset), - CharUnits::fromQuantity(16)); + CGF.Int8Ty, CharUnits::fromQuantity(16)); RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy); } else { assert(neededSSE == 2 && "Invalid number of needed registers!"); @@ -4176,7 +4177,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // all the SSE registers to the RSA. Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset), - CharUnits::fromQuantity(16)); + CGF.Int8Ty, CharUnits::fromQuantity(16)); Address RegAddrHi = CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo, CharUnits::fromQuantity(16)); @@ -4357,12 +4358,12 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, } } - if (Ty->isExtIntType()) { + if (Ty->isBitIntType()) { // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." - // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes - // anyway as long is it fits in them, so we don't have to check the power of - // 2. + // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4, + // or 8 bytes anyway as long is it fits in them, so we don't have to check + // the power of 2. if (Width <= 64) return ABIArgInfo::getDirect(); return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); @@ -5069,7 +5070,7 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const { break; } - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() < 64) return true; @@ -5083,13 +5084,16 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const { if (const ComplexType *CTy = Ty->getAs()) Ty = CTy->getElementType(); + auto FloatUsesVector = [this](QualType Ty){ + return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics( + Ty) == &llvm::APFloat::IEEEquad(); + }; + // Only vector types of size 16 bytes need alignment (larger types are // passed via reference, smaller types are not aligned). if (Ty->isVectorType()) { return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8); - } else if (Ty->isRealFloatingType() && - &getContext().getFloatTypeSemantics(Ty) == - &llvm::APFloat::IEEEquad()) { + } else if (FloatUsesVector(Ty)) { // According to ABI document section 'Optional Save Areas': If extended // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION // format are supported, map them to a single quadword, quadword aligned. @@ -5116,7 +5120,9 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const { // With special case aggregates, only vector base types need alignment. if (AlignAsType) { - return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8); + bool UsesVector = AlignAsType->isVectorType() || + FloatUsesVector(QualType(AlignAsType, 0)); + return CharUnits::fromQuantity(UsesVector ? 16 : 8); } // Otherwise, we only need alignment for any aggregate type that @@ -5289,7 +5295,7 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { } } - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > 128) return getNaturalAlignIndirect(Ty, /*ByVal=*/true); @@ -5365,7 +5371,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { } } - if (const auto *EIT = RetTy->getAs()) + if (const auto *EIT = RetTy->getAs()) if (EIT->getNumBits() > 128) return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); @@ -5717,7 +5723,7 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic, if (const EnumType *EnumTy = Ty->getAs()) Ty = EnumTy->getDecl()->getIntegerType(); - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > 128) return getNaturalAlignIndirect(Ty); @@ -5819,7 +5825,7 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy, if (const EnumType *EnumTy = RetTy->getAs()) RetTy = EnumTy->getDecl()->getIntegerType(); - if (const auto *EIT = RetTy->getAs()) + if (const auto *EIT = RetTy->getAs()) if (EIT->getNumBits() > 128) return getNaturalAlignIndirect(RetTy); @@ -6561,7 +6567,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, Ty = EnumTy->getDecl()->getIntegerType(); } - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > 64) return getNaturalAlignIndirect(Ty, /*ByVal=*/true); @@ -6763,7 +6769,7 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic, if (const EnumType *EnumTy = RetTy->getAs()) RetTy = EnumTy->getDecl()->getIntegerType(); - if (const auto *EIT = RetTy->getAs()) + if (const auto *EIT = RetTy->getAs()) if (EIT->getNumBits() > 64) return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); @@ -7100,7 +7106,7 @@ bool NVPTXABIInfo::isUnsupportedType(QualType T) const { (T->isFloat128Type() || (T->isRealFloatingType() && Context.getTypeSize(T) == 128))) return true; - if (const auto *EIT = T->getAs()) + if (const auto *EIT = T->getAs()) return EIT->getNumBits() > (Context.getTargetInfo().hasInt128Type() ? 128U : 64U); if (!Context.getTargetInfo().hasInt128Type() && T->isIntegerType() && @@ -7177,7 +7183,7 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const { return getNaturalAlignIndirect(Ty, /* byval */ true); } - if (const auto *EIT = Ty->getAs()) { + if (const auto *EIT = Ty->getAs()) { if ((EIT->getNumBits() > 128) || (!getContext().getTargetInfo().hasInt128Type() && EIT->getNumBits() > 64)) @@ -7391,7 +7397,7 @@ bool SystemZABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const { if (ABIInfo::isPromotableIntegerTypeForABI(Ty)) return true; - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() < 64) return true; @@ -7994,7 +8000,7 @@ MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const { Ty = EnumTy->getDecl()->getIntegerType(); // Make sure we pass indirectly things that are too large. - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > 128 || (EIT->getNumBits() > 64 && !getContext().getTargetInfo().hasInt128Type())) @@ -8085,7 +8091,7 @@ ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const { RetTy = EnumTy->getDecl()->getIntegerType(); // Make sure we pass indirectly things that are too large. - if (const auto *EIT = RetTy->getAs()) + if (const auto *EIT = RetTy->getAs()) if (EIT->getNumBits() > 128 || (EIT->getNumBits() > 64 && !getContext().getTargetInfo().hasInt128Type())) @@ -8460,7 +8466,7 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty, if (Size <= 64) HexagonAdjustRegsLeft(Size, RegsLeft); - if (Size > 64 && Ty->isExtIntType()) + if (Size > 64 && Ty->isBitIntType()) return getNaturalAlignIndirect(Ty, /*ByVal=*/true); return isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty) @@ -8516,7 +8522,7 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const { if (const EnumType *EnumTy = RetTy->getAs()) RetTy = EnumTy->getDecl()->getIntegerType(); - if (Size > 64 && RetTy->isExtIntType()) + if (Size > 64 && RetTy->isBitIntType()) return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy) @@ -8887,7 +8893,7 @@ ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty, bool InReg = shouldUseInReg(Ty, State); // Don't pass >64 bit integers in registers. - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > 64) return getIndirectResult(Ty, /*ByVal=*/true, State); @@ -9161,6 +9167,10 @@ class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo { public: AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(std::make_unique(CGT)) {} + + void setFunctionDeclAttributes(const FunctionDecl *FD, llvm::Function *F, + CodeGenModule &CGM) const; + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; unsigned getOpenCLKernelCallingConv() const override; @@ -9200,36 +9210,13 @@ static bool requiresAMDGPUProtectedVisibility(const Decl *D, cast(D)->getType()->isCUDADeviceBuiltinTextureType())); } -void AMDGPUTargetCodeGenInfo::setTargetAttributes( - const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { - if (requiresAMDGPUProtectedVisibility(D, GV)) { - GV->setVisibility(llvm::GlobalValue::ProtectedVisibility); - GV->setDSOLocal(true); - } - - if (GV->isDeclaration()) - return; - const FunctionDecl *FD = dyn_cast_or_null(D); - if (!FD) - return; - - llvm::Function *F = cast(GV); - - const auto *ReqdWGS = M.getLangOpts().OpenCL ? - FD->getAttr() : nullptr; - - - const bool IsOpenCLKernel = M.getLangOpts().OpenCL && - FD->hasAttr(); - const bool IsHIPKernel = M.getLangOpts().HIP && - FD->hasAttr(); - if ((IsOpenCLKernel || IsHIPKernel) && - (M.getTriple().getOS() == llvm::Triple::AMDHSA)) - F->addFnAttr("amdgpu-implicitarg-num-bytes", "56"); - - if (IsHIPKernel) - F->addFnAttr("uniform-work-group-size", "true"); - +void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes( + const FunctionDecl *FD, llvm::Function *F, CodeGenModule &M) const { + const auto *ReqdWGS = + M.getLangOpts().OpenCL ? FD->getAttr() : nullptr; + const bool IsOpenCLKernel = + M.getLangOpts().OpenCL && FD->hasAttr(); + const bool IsHIPKernel = M.getLangOpts().HIP && FD->hasAttr(); const auto *FlatWGS = FD->getAttr(); if (ReqdWGS || FlatWGS) { @@ -9297,6 +9284,38 @@ void AMDGPUTargetCodeGenInfo::setTargetAttributes( if (NumVGPR != 0) F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR)); } +} + +void AMDGPUTargetCodeGenInfo::setTargetAttributes( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { + if (requiresAMDGPUProtectedVisibility(D, GV)) { + GV->setVisibility(llvm::GlobalValue::ProtectedVisibility); + GV->setDSOLocal(true); + } + + if (GV->isDeclaration()) + return; + + llvm::Function *F = dyn_cast(GV); + if (!F) + return; + + const FunctionDecl *FD = dyn_cast_or_null(D); + if (FD) + setFunctionDeclAttributes(FD, F, M); + + const bool IsOpenCLKernel = + M.getLangOpts().OpenCL && FD && FD->hasAttr(); + const bool IsHIPKernel = + M.getLangOpts().HIP && FD && FD->hasAttr(); + + const bool IsOpenMP = M.getLangOpts().OpenMP && !FD; + if ((IsOpenCLKernel || IsHIPKernel || IsOpenMP) && + (M.getTriple().getOS() == llvm::Triple::AMDHSA)) + F->addFnAttr("amdgpu-implicitarg-num-bytes", "56"); + + if (IsHIPKernel) + F->addFnAttr("uniform-work-group-size", "true"); if (M.getContext().getTargetInfo().allowAMDGPUUnsafeFPAtomics()) F->addFnAttr("amdgpu-unsafe-fp-atomics", "true"); @@ -9343,7 +9362,9 @@ AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM, if (AddrSpace != LangAS::Default) return AddrSpace; - if (CGM.isTypeConstant(D->getType(), false)) { + // Only promote to address space 4 if VarDecl has constant initialization. + if (CGM.isTypeConstant(D->getType(), false) && + D->hasConstantInitialization()) { if (auto ConstAS = CGM.getTarget().getConstantAddressSpace()) return ConstAS.getValue(); } @@ -9606,7 +9627,7 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { if (Size < 64 && Ty->isIntegerType()) return ABIArgInfo::getExtend(Ty); - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() < 64) return ABIArgInfo::getExtend(Ty); @@ -9860,7 +9881,7 @@ ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty, ABIArgInfo::getDirect(Result, 0, nullptr, false); } - if (const auto *EIT = Ty->getAs()) + if (const auto *EIT = Ty->getAs()) if (EIT->getNumBits() > 64) return getIndirectByValue(Ty); @@ -10209,12 +10230,23 @@ class CommonSPIRABIInfo : public DefaultABIInfo { private: void setCCs(); }; + +class SPIRVABIInfo : public CommonSPIRABIInfo { +public: + SPIRVABIInfo(CodeGenTypes &CGT) : CommonSPIRABIInfo(CGT) {} + void computeInfo(CGFunctionInfo &FI) const override; + +private: + ABIArgInfo classifyKernelArgumentType(QualType Ty) const; +}; } // end anonymous namespace namespace { class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo { public: CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) : TargetCodeGenInfo(std::make_unique(CGT)) {} + CommonSPIRTargetCodeGenInfo(std::unique_ptr ABIInfo) + : TargetCodeGenInfo(std::move(ABIInfo)) {} LangAS getASTAllocaAddressSpace() const override { return getLangASFromTargetAS( @@ -10223,18 +10255,60 @@ class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo { unsigned getOpenCLKernelCallingConv() const override; }; - +class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo { +public: + SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) + : CommonSPIRTargetCodeGenInfo(std::make_unique(CGT)) {} + void setCUDAKernelCallingConvention(const FunctionType *&FT) const override; +}; } // End anonymous namespace. + void CommonSPIRABIInfo::setCCs() { assert(getRuntimeCC() == llvm::CallingConv::C); RuntimeCC = llvm::CallingConv::SPIR_FUNC; } +ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const { + if (getContext().getLangOpts().HIP) { + // Coerce pointer arguments with default address space to CrossWorkGroup + // pointers for HIPSPV. When the language mode is HIP, the SPIRTargetInfo + // maps cuda_device to SPIR-V's CrossWorkGroup address space. + llvm::Type *LTy = CGT.ConvertType(Ty); + auto DefaultAS = getContext().getTargetAddressSpace(LangAS::Default); + auto GlobalAS = getContext().getTargetAddressSpace(LangAS::cuda_device); + if (LTy->isPointerTy() && LTy->getPointerAddressSpace() == DefaultAS) { + LTy = llvm::PointerType::get( + cast(LTy)->getElementType(), GlobalAS); + return ABIArgInfo::getDirect(LTy, 0, nullptr, false); + } + } + return classifyArgumentType(Ty); +} + +void SPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const { + // The logic is same as in DefaultABIInfo with an exception on the kernel + // arguments handling. + llvm::CallingConv::ID CC = FI.getCallingConvention(); + + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + + for (auto &I : FI.arguments()) { + if (CC == llvm::CallingConv::SPIR_KERNEL) { + I.info = classifyKernelArgumentType(I.type); + } else { + I.info = classifyArgumentType(I.type); + } + } +} + namespace clang { namespace CodeGen { void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) { - DefaultABIInfo SPIRABI(CGM.getTypes()); - SPIRABI.computeInfo(FI); + if (CGM.getTarget().getTriple().isSPIRV()) + SPIRVABIInfo(CGM.getTypes()).computeInfo(FI); + else + CommonSPIRABIInfo(CGM.getTypes()).computeInfo(FI); } } } @@ -10243,6 +10317,16 @@ unsigned CommonSPIRTargetCodeGenInfo::getOpenCLKernelCallingConv() const { return llvm::CallingConv::SPIR_KERNEL; } +void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention( + const FunctionType *&FT) const { + // Convert HIP kernels to SPIR-V kernels. + if (getABIInfo().getContext().getLangOpts().HIP) { + FT = getABIInfo().getContext().adjustFunctionType( + FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel)); + return; + } +} + static bool appendType(SmallStringEnc &Enc, QualType QType, const CodeGen::CodeGenModule &CGM, TypeStringCache &TSC); @@ -10943,7 +11027,7 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed, return extendType(Ty); } - if (const auto *EIT = Ty->getAs()) { + if (const auto *EIT = Ty->getAs()) { if (EIT->getNumBits() < XLen && !MustUseStack) return extendType(Ty); if (EIT->getNumBits() > 128 || @@ -11308,9 +11392,10 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return SetCGInfo(new ARCTargetCodeGenInfo(Types)); case llvm::Triple::spir: case llvm::Triple::spir64: + return SetCGInfo(new CommonSPIRTargetCodeGenInfo(Types)); case llvm::Triple::spirv32: case llvm::Triple::spirv64: - return SetCGInfo(new CommonSPIRTargetCodeGenInfo(Types)); + return SetCGInfo(new SPIRVTargetCodeGenInfo(Types)); case llvm::Triple::ve: return SetCGInfo(new VETargetCodeGenInfo(Types)); } diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index d501bd026219..3b551ea94cc2 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -23,7 +23,8 @@ #include "ToolChains/FreeBSD.h" #include "ToolChains/Fuchsia.h" #include "ToolChains/Gnu.h" -#include "ToolChains/HIP.h" +#include "ToolChains/HIPAMD.h" +#include "ToolChains/HIPSPV.h" #include "ToolChains/Haiku.h" #include "ToolChains/Hexagon.h" #include "ToolChains/Hurd.h" @@ -42,6 +43,7 @@ #include "ToolChains/PPCLinux.h" #include "ToolChains/PS4CPU.h" #include "ToolChains/RISCVToolchain.h" +#include "ToolChains/SPIRV.h" #include "ToolChains/Solaris.h" #include "ToolChains/TCE.h" #include "ToolChains/VEToolchain.h" @@ -99,8 +101,39 @@ using namespace clang::driver; using namespace clang; using namespace llvm::opt; -static llvm::Triple getHIPOffloadTargetTriple() { - static const llvm::Triple T("amdgcn-amd-amdhsa"); +static llvm::Optional +getHIPOffloadTargetTriple(const Driver &D, const ArgList &Args) { + if (Args.hasArg(options::OPT_offload_EQ)) { + auto HIPOffloadTargets = Args.getAllArgValues(options::OPT_offload_EQ); + + // HIP compilation flow does not support multiple targets for now. We need + // the HIPActionBuilder (and possibly the CudaActionBuilder{,Base}too) to + // support multiple tool chains first. + switch (HIPOffloadTargets.size()) { + default: + D.Diag(diag::err_drv_only_one_offload_target_supported_in) << "HIP"; + return llvm::None; + case 0: + D.Diag(diag::err_drv_invalid_or_unsupported_offload_target) << ""; + return llvm::None; + case 1: + break; + } + llvm::Triple TT(HIPOffloadTargets[0]); + if (TT.getArch() == llvm::Triple::amdgcn && + TT.getVendor() == llvm::Triple::AMD && + TT.getOS() == llvm::Triple::AMDHSA) + return TT; + if (TT.getArch() == llvm::Triple::spirv64 && + TT.getVendor() == llvm::Triple::UnknownVendor && + TT.getOS() == llvm::Triple::UnknownOS) + return TT; + D.Diag(diag::err_drv_invalid_or_unsupported_offload_target) + << HIPOffloadTargets[0]; + return llvm::None; + } + + static const llvm::Triple T("amdgcn-amd-amdhsa"); // Default HIP triple. return T; } @@ -694,17 +727,14 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C, return; } const ToolChain *HostTC = C.getSingleOffloadToolChain(); - const llvm::Triple &HostTriple = HostTC->getTriple(); auto OFK = Action::OFK_HIP; - llvm::Triple HIPTriple = getHIPOffloadTargetTriple(); - // Use the HIP and host triples as the key into the ToolChains map, - // because the device toolchain we create depends on both. - auto &HIPTC = ToolChains[HIPTriple.str() + "/" + HostTriple.str()]; - if (!HIPTC) { - HIPTC = std::make_unique( - *this, HIPTriple, *HostTC, C.getInputArgs()); - } - C.addOffloadDeviceToolChain(HIPTC.get(), OFK); + auto HIPTriple = getHIPOffloadTargetTriple(*this, C.getInputArgs()); + if (!HIPTriple) + return; + auto *HIPTC = &getOffloadingDeviceToolChain(C.getInputArgs(), *HIPTriple, + *HostTC, OFK); + assert(HIPTC && "Could not create offloading device tool chain."); + C.addOffloadDeviceToolChain(HIPTC, OFK); } // @@ -1545,7 +1575,7 @@ int Driver::ExecuteCompilation( if (Diags.hasErrorOccurred()) return 1; - // Set up response file names for each command, if necessary + // Set up response file names for each command, if necessary. for (auto &Job : C.getJobs()) setUpResponseFiles(C, Job); @@ -2040,11 +2070,7 @@ static bool ContainsCompileOrAssembleAction(const Action *A) { isa(A)) return true; - for (const Action *Input : A->inputs()) - if (ContainsCompileOrAssembleAction(Input)) - return true; - - return false; + return llvm::any_of(A->inputs(), ContainsCompileOrAssembleAction); } void Driver::BuildUniversalActions(Compilation &C, const ToolChain &TC, @@ -2733,6 +2759,14 @@ class OffloadingActionBuilder final { } } + // --offload and --offload-arch options are mutually exclusive. + if (Args.hasArgNoClaim(options::OPT_offload_EQ) && + Args.hasArgNoClaim(options::OPT_offload_arch_EQ, + options::OPT_no_offload_arch_EQ)) { + C.getDriver().Diag(diag::err_opt_not_valid_with_opt) << "--offload-arch" + << "--offload"; + } + // Collect all cuda_gpu_arch parameters, removing duplicates. std::set GpuArchs; bool Error = false; @@ -2775,8 +2809,12 @@ class OffloadingActionBuilder final { // Default to sm_20 which is the lowest common denominator for // supported GPUs. sm_20 code should work correctly, if // suboptimally, on all newer GPUs. - if (GpuArchList.empty()) - GpuArchList.push_back(DefaultCudaArch); + if (GpuArchList.empty()) { + if (ToolChains.front()->getTriple().isSPIRV()) + GpuArchList.push_back(CudaArch::Generic); + else + GpuArchList.push_back(DefaultCudaArch); + } return Error; } @@ -2937,8 +2975,11 @@ class OffloadingActionBuilder final { StringRef getCanonicalOffloadArch(StringRef IdStr) override { llvm::StringMap Features; - auto ArchStr = - parseTargetID(getHIPOffloadTargetTriple(), IdStr, &Features); + // getHIPOffloadTargetTriple() is known to return valid value as it has + // been called successfully in the CreateOffloadingDeviceToolChains(). + auto ArchStr = parseTargetID( + *getHIPOffloadTargetTriple(C.getDriver(), C.getInputArgs()), IdStr, + &Features); if (!ArchStr) { C.getDriver().Diag(clang::diag::err_drv_bad_target_id) << IdStr; C.setContainsError(); @@ -2992,9 +3033,19 @@ class OffloadingActionBuilder final { // When LTO is not enabled, we follow the conventional // compiler phases, including backend and assemble phases. ActionList AL; - auto BackendAction = C.getDriver().ConstructPhaseAction( - C, Args, phases::Backend, CudaDeviceActions[I], - AssociatedOffloadKind); + Action *BackendAction = nullptr; + if (ToolChains.front()->getTriple().isSPIRV()) { + // Emit LLVM bitcode for SPIR-V targets. SPIR-V device tool chain + // (HIPSPVToolChain) runs post-link LLVM IR passes. + types::ID Output = Args.hasArg(options::OPT_S) + ? types::TY_LLVM_IR + : types::TY_LLVM_BC; + BackendAction = + C.MakeAction(CudaDeviceActions[I], Output); + } else + BackendAction = C.getDriver().ConstructPhaseAction( + C, Args, phases::Backend, CudaDeviceActions[I], + AssociatedOffloadKind); auto AssembleAction = C.getDriver().ConstructPhaseAction( C, Args, phases::Assemble, BackendAction, AssociatedOffloadKind); @@ -3724,6 +3775,14 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args, } } + // FIXME: Linking separate translation units for SPIR-V is not supported yet. + // It can be done either by LLVM IR linking before conversion of the final + // linked module to SPIR-V or external SPIR-V linkers can be used e.g. + // spirv-link. + if (C.getDefaultToolChain().getTriple().isSPIRV() && Inputs.size() > 1) { + Diag(clang::diag::warn_drv_spirv_linking_multiple_inputs_unsupported); + } + handleArguments(C, Args, Inputs, Actions); // Builder to be used to build offloading actions. @@ -3763,8 +3822,15 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args, // Queue linker inputs. if (Phase == phases::Link) { assert(Phase == PL.back() && "linking must be final compilation step."); - LinkerInputs.push_back(Current); - Current = nullptr; + // Compilation phases are setup per language, however for SPIR-V the + // final linking phase is meaningless since the compilation phase + // produces the final binary. + // FIXME: OpenCL - we could strip linking phase out from OpenCL + // compilation phases if we could verify it is not needed by any target. + if (!C.getDefaultToolChain().getTriple().isSPIRV()) { + LinkerInputs.push_back(Current); + Current = nullptr; + } break; } @@ -4337,6 +4403,12 @@ class ToolSelector final { if (!T) return nullptr; + // Can't collapse if we don't have codegen support unless we are + // emitting LLVM IR. + bool OutputIsLLVM = types::isLLVMIR(ActionInfo[0].JA->getType()); + if (!T->hasIntegratedBackend() && !(OutputIsLLVM && T->canEmitIR())) + return nullptr; + // When using -fembed-bitcode, it is required to have the same tool (clang) // for both CompilerJA and BackendJA. Otherwise, combine two stages. if (EmbedBitcode) { @@ -4406,6 +4478,12 @@ class ToolSelector final { if (!T) return nullptr; + // Can't collapse if we don't have codegen support unless we are + // emitting LLVM IR. + bool OutputIsLLVM = types::isLLVMIR(ActionInfo[0].JA->getType()); + if (!T->hasIntegratedBackend() && !(OutputIsLLVM && T->canEmitIR())) + return nullptr; + if (T->canEmitIR() && ((SaveTemps && !InputIsBitcode) || EmbedBitcode)) return nullptr; @@ -5429,6 +5507,10 @@ const ToolChain &Driver::getToolChain(const ArgList &Args, case llvm::Triple::ve: TC = std::make_unique(*this, Target, Args); break; + case llvm::Triple::spirv32: + case llvm::Triple::spirv64: + TC = std::make_unique(*this, Target, Args); + break; default: if (Target.getVendor() == llvm::Triple::Myriad) TC = std::make_unique(*this, Target, @@ -5453,6 +5535,38 @@ const ToolChain &Driver::getToolChain(const ArgList &Args, return *TC; } +const ToolChain &Driver::getOffloadingDeviceToolChain( + const ArgList &Args, const llvm::Triple &Target, const ToolChain &HostTC, + const Action::OffloadKind &TargetDeviceOffloadKind) const { + // Use device / host triples as the key into the ToolChains map because the + // device ToolChain we create depends on both. + auto &TC = ToolChains[Target.str() + "/" + HostTC.getTriple().str()]; + if (!TC) { + // Categorized by offload kind > arch rather than OS > arch like + // the normal getToolChain call, as it seems a reasonable way to categorize + // things. + switch (TargetDeviceOffloadKind) { + case Action::OFK_HIP: { + if (Target.getArch() == llvm::Triple::amdgcn && + Target.getVendor() == llvm::Triple::AMD && + Target.getOS() == llvm::Triple::AMDHSA) + TC = std::make_unique(*this, Target, + HostTC, Args); + else if (Target.getArch() == llvm::Triple::spirv64 && + Target.getVendor() == llvm::Triple::UnknownVendor && + Target.getOS() == llvm::Triple::UnknownOS) + TC = std::make_unique(*this, Target, + HostTC, Args); + break; + } + default: + break; + } + } + + return *TC; +} + bool Driver::ShouldUseClangCompiler(const JobAction &JA) const { // Say "no" if there is not exactly one input of a type clang understands. if (JA.size() != 1 || diff --git a/clang/lib/Driver/Job.cpp b/clang/lib/Driver/Job.cpp index 5b87106b6565..f63763effaff 100644 --- a/clang/lib/Driver/Job.cpp +++ b/clang/lib/Driver/Job.cpp @@ -388,6 +388,8 @@ int CC1Command::Execute(ArrayRef> Redirects, Argv.push_back(getExecutable()); Argv.append(getArguments().begin(), getArguments().end()); Argv.push_back(nullptr); + Argv.pop_back(); // The terminating null element shall not be part of the + // slice (main() behavior). // This flag simply indicates that the program couldn't start, which isn't // applicable here. diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp index 6588cdf9fecd..50c89aaadc18 100644 --- a/clang/lib/Driver/ToolChain.cpp +++ b/clang/lib/Driver/ToolChain.cpp @@ -260,7 +260,7 @@ bool ToolChain::IsUnwindTablesDefault(const ArgList &Args) const { Tool *ToolChain::getClang() const { if (!Clang) - Clang.reset(new tools::Clang(*this)); + Clang.reset(new tools::Clang(*this, useIntegratedBackend())); return Clang.get(); } @@ -541,12 +541,9 @@ std::string ToolChain::GetProgramPath(const char *Name) const { return D.GetProgramPath(Name, *this); } -std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD, - bool *LinkerIsLLDDarwinNew) const { +std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD) const { if (LinkerIsLLD) *LinkerIsLLD = false; - if (LinkerIsLLDDarwinNew) - *LinkerIsLLDDarwinNew = false; // Get -fuse-ld= first to prevent -Wunused-command-line-argument. -fuse-ld= is // considered as the linker flavor, e.g. "bfd", "gold", or "lld". @@ -599,11 +596,8 @@ std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD, std::string LinkerPath(GetProgramPath(LinkerName.c_str())); if (llvm::sys::fs::can_execute(LinkerPath)) { - // FIXME: Remove LinkerIsLLDDarwinNew once there's only one MachO lld. if (LinkerIsLLD) - *LinkerIsLLD = UseLinker == "lld" || UseLinker == "lld.darwinold"; - if (LinkerIsLLDDarwinNew) - *LinkerIsLLDDarwinNew = UseLinker == "lld"; + *LinkerIsLLD = UseLinker == "lld"; return LinkerPath; } } diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp index b5eaf1adca6b..43ce33750eba 100644 --- a/clang/lib/Driver/ToolChains/AMDGPU.cpp +++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp @@ -478,7 +478,8 @@ void RocmInstallationDetector::print(raw_ostream &OS) const { void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs, ArgStringList &CC1Args) const { - bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5); + bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5) && + !DriverArgs.hasArg(options::OPT_nohipwrapperinc); if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) { // HIP header includes standard library wrapper headers under clang diff --git a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp index 863e2c597d53..f282f04b7931 100644 --- a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp +++ b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp @@ -267,7 +267,7 @@ void AMDGPUOpenMPToolChain::addClangTargetOptions( std::string BitcodeSuffix; if (DriverArgs.hasFlag(options::OPT_fopenmp_target_new_runtime, - options::OPT_fno_openmp_target_new_runtime, false)) + options::OPT_fno_openmp_target_new_runtime, true)) BitcodeSuffix = "new-amdgpu-" + GPUArch; else BitcodeSuffix = "amdgcn-" + GPUArch; diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp index abc32f22d2a1..be13d6d583ce 100644 --- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp +++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp @@ -568,4 +568,15 @@ void aarch64::getAArch64TargetFeatures(const Driver &D, if (Args.hasArg(options::OPT_mno_neg_immediates)) Features.push_back("+no-neg-immediates"); + + if (Arg *A = Args.getLastArg(options::OPT_mfix_cortex_a53_835769, + options::OPT_mno_fix_cortex_a53_835769)) { + if (A->getOption().matches(options::OPT_mfix_cortex_a53_835769)) + Features.push_back("+fix-cortex-a53-835769"); + else + Features.push_back("-fix-cortex-a53-835769"); + } else if (Triple.isAndroid()) { + // Enabled A53 errata (835769) workaround by default on android + Features.push_back("+fix-cortex-a53-835769"); + } } diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp index 21c091e1a0ba..4013cf230026 100644 --- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp +++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp @@ -147,19 +147,27 @@ bool arm::useAAPCSForMachO(const llvm::Triple &T) { T.getOS() == llvm::Triple::UnknownOS || isARMMProfile(T); } +// We follow GCC and support when the backend has support for the MRC/MCR +// instructions that are used to set the hard thread pointer ("CP15 C13 +// Thread id"). +bool arm::isHardTPSupported(const llvm::Triple &Triple) { + int Ver = getARMSubArchVersionNumber(Triple); + llvm::ARM::ArchKind AK = llvm::ARM::parseArch(Triple.getArchName()); + return Triple.isARM() || AK == llvm::ARM::ArchKind::ARMV6T2 || + (Ver >= 7 && AK != llvm::ARM::ArchKind::ARMV8MBaseline); +} + // Select mode for reading thread pointer (-mtp=soft/cp15). arm::ReadTPMode arm::getReadTPMode(const Driver &D, const ArgList &Args, - const llvm::Triple &Triple) { + const llvm::Triple &Triple, bool ForAS) { if (Arg *A = Args.getLastArg(options::OPT_mtp_mode_EQ)) { arm::ReadTPMode ThreadPointer = llvm::StringSwitch(A->getValue()) .Case("cp15", ReadTPMode::Cp15) .Case("soft", ReadTPMode::Soft) .Default(ReadTPMode::Invalid); - if (ThreadPointer == ReadTPMode::Cp15 && - getARMSubArchVersionNumber(Triple) < 7 && - llvm::ARM::parseArch(Triple.getArchName()) != - llvm::ARM::ArchKind::ARMV6T2) { + if (ThreadPointer == ReadTPMode::Cp15 && !isHardTPSupported(Triple) && + !ForAS) { D.Diag(diag::err_target_unsupported_tp_hard) << Triple.getArchName(); return ReadTPMode::Invalid; } @@ -430,7 +438,6 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple, bool KernelOrKext = Args.hasArg(options::OPT_mkernel, options::OPT_fapple_kext); arm::FloatABI ABI = arm::getARMFloatABI(D, Triple, Args); - arm::ReadTPMode ThreadPointer = arm::getReadTPMode(D, Args, Triple); llvm::Optional> WaCPU, WaFPU, WaHDiv, WaArch; @@ -482,7 +489,7 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple, } } - if (ThreadPointer == arm::ReadTPMode::Cp15) + if (getReadTPMode(D, Args, Triple, ForAS) == ReadTPMode::Cp15) Features.push_back("+read-tp-hard"); const Arg *ArchArg = Args.getLastArg(options::OPT_march_EQ); @@ -869,6 +876,8 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple, } } + if (Args.getLastArg(options::OPT_mno_bti_at_return_twice)) + Features.push_back("+no-bti-at-return-twice"); } std::string arm::getARMArch(StringRef Arch, const llvm::Triple &Triple) { diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.h b/clang/lib/Driver/ToolChains/Arch/ARM.h index b6fd68fbb9c6..881b63bd36b9 100644 --- a/clang/lib/Driver/ToolChains/Arch/ARM.h +++ b/clang/lib/Driver/ToolChains/Arch/ARM.h @@ -53,8 +53,9 @@ FloatABI getARMFloatABI(const Driver &D, const llvm::Triple &Triple, const llvm::opt::ArgList &Args); void setFloatABIInTriple(const Driver &D, const llvm::opt::ArgList &Args, llvm::Triple &triple); +bool isHardTPSupported(const llvm::Triple &Triple); ReadTPMode getReadTPMode(const Driver &D, const llvm::opt::ArgList &Args, - const llvm::Triple &Triple); + const llvm::Triple &Triple, bool ForAS); void setArchNameInTriple(const Driver &D, const llvm::opt::ArgList &Args, types::ID InputType, llvm::Triple &Triple); diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp index 323f588c8269..7ad8ca69bed5 100644 --- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp +++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp @@ -194,27 +194,11 @@ StringRef riscv::getRISCVABI(const ArgList &Args, const llvm::Triple &Triple) { auto ParseResult = llvm::RISCVISAInfo::parseArchString( Arch, /* EnableExperimentalExtension */ true); - if (!ParseResult) { + if (!ParseResult) // Ignore parsing error, just go 3rd step. consumeError(ParseResult.takeError()); - } else { - auto &ISAInfo = *ParseResult; - bool HasD = ISAInfo->hasExtension("d"); - unsigned XLen = ISAInfo->getXLen(); - if (XLen == 32) { - bool HasE = ISAInfo->hasExtension("e"); - if (HasD) - return "ilp32d"; - if (HasE) - return "ilp32e"; - return "ilp32"; - } else if (XLen == 64) { - if (HasD) - return "lp64d"; - return "lp64"; - } - llvm_unreachable("unhandled XLen"); - } + else + return llvm::RISCV::computeDefaultABIFromArch(**ParseResult); // 3. Choose a default based on the triple // diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index c5aaa067c4f5..65347a38490e 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -625,8 +625,9 @@ getFramePointerKind(const ArgList &Args, const llvm::Triple &Triple) { } /// Add a CC1 option to specify the debug compilation directory. -static void addDebugCompDirArg(const ArgList &Args, ArgStringList &CmdArgs, - const llvm::vfs::FileSystem &VFS) { +static const char *addDebugCompDirArg(const ArgList &Args, + ArgStringList &CmdArgs, + const llvm::vfs::FileSystem &VFS) { if (Arg *A = Args.getLastArg(options::OPT_ffile_compilation_dir_EQ, options::OPT_fdebug_compilation_dir_EQ)) { if (A->getOption().matches(options::OPT_ffile_compilation_dir_EQ)) @@ -638,6 +639,31 @@ static void addDebugCompDirArg(const ArgList &Args, ArgStringList &CmdArgs, VFS.getCurrentWorkingDirectory()) { CmdArgs.push_back(Args.MakeArgString("-fdebug-compilation-dir=" + *CWD)); } + StringRef Path(CmdArgs.back()); + return Path.substr(Path.find('=') + 1).data(); +} + +static void addDebugObjectName(const ArgList &Args, ArgStringList &CmdArgs, + const char *DebugCompilationDir, + const char *OutputFileName) { + // No need to generate a value for -object-file-name if it was provided. + for (auto *Arg : Args.filtered(options::OPT_Xclang)) + if (StringRef(Arg->getValue()).startswith("-object-file-name")) + return; + + if (Args.hasArg(options::OPT_object_file_name_EQ)) + return; + + SmallString<128> ObjFileNameForDebug(OutputFileName); + if (ObjFileNameForDebug != "-" && + !llvm::sys::path::is_absolute(ObjFileNameForDebug) && + (!DebugCompilationDir || + llvm::sys::path::is_absolute(DebugCompilationDir))) { + // Make the path absolute in the debug infos like MSVC does. + llvm::sys::fs::make_absolute(ObjFileNameForDebug); + } + CmdArgs.push_back( + Args.MakeArgString(Twine("-object-file-name=") + ObjFileNameForDebug)); } /// Add a CC1 and CC1AS option to specify the debug file path prefix map. @@ -976,11 +1002,7 @@ static bool ContainsCompileAction(const Action *A) { if (isa(A) || isa(A)) return true; - for (const auto &AI : A->inputs()) - if (ContainsCompileAction(AI)) - return true; - - return false; + return llvm::any_of(A->inputs(), ContainsCompileAction); } /// Check if -relax-all should be passed to the internal assembler. @@ -1806,19 +1828,6 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args, RenderAArch64ABI(Triple, Args, CmdArgs); - if (Arg *A = Args.getLastArg(options::OPT_mfix_cortex_a53_835769, - options::OPT_mno_fix_cortex_a53_835769)) { - CmdArgs.push_back("-mllvm"); - if (A->getOption().matches(options::OPT_mfix_cortex_a53_835769)) - CmdArgs.push_back("-aarch64-fix-cortex-a53-835769=1"); - else - CmdArgs.push_back("-aarch64-fix-cortex-a53-835769=0"); - } else if (Triple.isAndroid()) { - // Enabled A53 errata (835769) workaround by default on android - CmdArgs.push_back("-mllvm"); - CmdArgs.push_back("-aarch64-fix-cortex-a53-835769=1"); - } - // Forward the -mglobal-merge option for explicit control over the pass. if (Arg *A = Args.getLastArg(options::OPT_mglobal_merge, options::OPT_mno_global_merge)) { @@ -5666,7 +5675,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back("-fno-autolink"); // Add in -fdebug-compilation-dir if necessary. - addDebugCompDirArg(Args, CmdArgs, D.getVFS()); + const char *DebugCompilationDir = + addDebugCompDirArg(Args, CmdArgs, D.getVFS()); addDebugPrefixMapArg(D, Args, CmdArgs); @@ -5904,7 +5914,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, // runtime. if (Args.hasFlag(options::OPT_fopenmp_target_new_runtime, options::OPT_fno_openmp_target_new_runtime, - /*Default=*/!getToolChain().getTriple().isAMDGCN())) + /*Default=*/true)) CmdArgs.push_back("-fopenmp-target-new-runtime"); // When in OpenMP offloading mode, enable debugging on the device. @@ -6997,18 +7007,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, if (Arg *A = Args.getLastArg(options::OPT_moutline_atomics, options::OPT_mno_outline_atomics)) { - if (A->getOption().matches(options::OPT_moutline_atomics)) { - // Option -moutline-atomics supported for AArch64 target only. - if (!Triple.isAArch64()) { - D.Diag(diag::warn_drv_moutline_atomics_unsupported_opt) - << Triple.getArchName(); - } else { + // Option -moutline-atomics supported for AArch64 target only. + if (!Triple.isAArch64()) { + D.Diag(diag::warn_drv_moutline_atomics_unsupported_opt) + << Triple.getArchName() << A->getOption().getName(); + } else { + if (A->getOption().matches(options::OPT_moutline_atomics)) { CmdArgs.push_back("-target-feature"); CmdArgs.push_back("+outline-atomics"); + } else { + CmdArgs.push_back("-target-feature"); + CmdArgs.push_back("-outline-atomics"); } - } else { - CmdArgs.push_back("-target-feature"); - CmdArgs.push_back("-outline-atomics"); } } else if (Triple.isAArch64() && getToolChain().IsAArch64OutlineAtomicsDefault(Args)) { @@ -7038,6 +7048,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(Args.MakeArgString(Str)); } + // Add the output path to the object file for CodeView debug infos. + if (EmitCodeView && Output.isFilename()) + addDebugObjectName(Args, CmdArgs, DebugCompilationDir, + Output.getFilename()); + // Add the "-o out -x type src.c" flags last. This is done primarily to make // the -cc1 command easier to edit when reproducing compiler crashes. if (Output.getType() == types::TY_Dependencies) { @@ -7111,11 +7126,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, Args.ClaimAllArgs(options::OPT_emit_llvm); } -Clang::Clang(const ToolChain &TC) +Clang::Clang(const ToolChain &TC, bool HasIntegratedBackend) // CAUTION! The first constructor argument ("clang") is not arbitrary, // as it is for other tools. Some operations on a Tool actually test // whether that tool is Clang based on the Tool's Name as a string. - : Tool("clang", "clang frontend", TC) {} + : Tool("clang", "clang frontend", TC), HasBackend(HasIntegratedBackend) {} Clang::~Clang() {} @@ -7655,11 +7670,14 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA, Args.AddAllArgs(CmdArgs, options::OPT_I_Group); // Determine the original source input. - const Action *SourceAction = &JA; - while (SourceAction->getKind() != Action::InputClass) { - assert(!SourceAction->getInputs().empty() && "unexpected root action!"); - SourceAction = SourceAction->getInputs()[0]; - } + auto FindSource = [](const Action *S) -> const Action * { + while (S->getKind() != Action::InputClass) { + assert(!S->getInputs().empty() && "unexpected root action!"); + S = S->getInputs()[0]; + } + return S; + }; + const Action *SourceAction = FindSource(&JA); // Forward -g and handle debug info related flags, assuming we are dealing // with an actual assembly file. @@ -7678,6 +7696,10 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA, codegenoptions::DebugInfoKind DebugInfoKind = codegenoptions::NoDebugInfo; + // Add the -fdebug-compilation-dir flag if needed. + const char *DebugCompilationDir = + addDebugCompDirArg(Args, CmdArgs, C.getDriver().getVFS()); + if (SourceAction->getType() == types::TY_Asm || SourceAction->getType() == types::TY_PP_Asm) { // You might think that it would be ok to set DebugInfoKind outside of @@ -7686,8 +7708,6 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA, // and it's not clear whether that test is just overly restrictive. DebugInfoKind = (WantDebug ? codegenoptions::DebugInfoConstructor : codegenoptions::NoDebugInfo); - // Add the -fdebug-compilation-dir flag if needed. - addDebugCompDirArg(Args, CmdArgs, C.getDriver().getVFS()); addDebugPrefixMapArg(getToolChain().getDriver(), Args, CmdArgs); @@ -7798,6 +7818,29 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA, Args.AddAllArgs(CmdArgs, options::OPT_mllvm); + if (DebugInfoKind > codegenoptions::NoDebugInfo && Output.isFilename()) + addDebugObjectName(Args, CmdArgs, DebugCompilationDir, + Output.getFilename()); + + // Fixup any previous commands that use -object-file-name because when we + // generated them, the final .obj name wasn't yet known. + for (Command &J : C.getJobs()) { + if (SourceAction != FindSource(&J.getSource())) + continue; + auto &JArgs = J.getArguments(); + for (unsigned I = 0; I < JArgs.size(); ++I) { + if (StringRef(JArgs[I]).startswith("-object-file-name=") && + Output.isFilename()) { + ArgStringList NewArgs(JArgs.begin(), JArgs.begin() + I); + addDebugObjectName(Args, NewArgs, DebugCompilationDir, + Output.getFilename()); + NewArgs.append(JArgs.begin() + I + 1, JArgs.end()); + J.replaceArguments(NewArgs); + break; + } + } + } + assert(Output.isFilename() && "Unexpected lipo output."); CmdArgs.push_back("-o"); CmdArgs.push_back(Output.getFilename()); @@ -7878,7 +7921,7 @@ void OffloadBundler::ConstructJob(Compilation &C, const JobAction &JA, Triples += '-'; Triples += CurTC->getTriple().normalize(); if ((CurKind == Action::OFK_HIP || CurKind == Action::OFK_Cuda) && - CurDep->getOffloadingArch()) { + !StringRef(CurDep->getOffloadingArch()).empty()) { Triples += '-'; Triples += CurDep->getOffloadingArch(); } diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h index d4b4988b4a8c..00e0490e069b 100644 --- a/clang/lib/Driver/ToolChains/Clang.h +++ b/clang/lib/Driver/ToolChains/Clang.h @@ -26,6 +26,10 @@ namespace tools { /// Clang compiler tool. class LLVM_LIBRARY_VISIBILITY Clang : public Tool { + // Indicates whether this instance has integrated backend using + // internal LLVM infrastructure. + bool HasBackend; + public: static const char *getBaseInputName(const llvm::opt::ArgList &Args, const InputInfo &Input); @@ -99,11 +103,12 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { const InputInfo &Input, const llvm::opt::ArgList &Args) const; public: - Clang(const ToolChain &TC); + Clang(const ToolChain &TC, bool HasIntegratedBackend = true); ~Clang() override; bool hasGoodDiagnostics() const override { return true; } bool hasIntegratedAssembler() const override { return true; } + bool hasIntegratedBackend() const override { return HasBackend; } bool hasIntegratedCPP() const override { return true; } bool canEmitIR() const override { return true; } diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 630baf9d6ae6..407f81a2ae09 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -15,7 +15,7 @@ #include "Arch/SystemZ.h" #include "Arch/VE.h" #include "Arch/X86.h" -#include "HIP.h" +#include "HIPAMD.h" #include "Hexagon.h" #include "clang/Basic/CharInfo.h" #include "clang/Basic/LangOptions.h" diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp index 06d3edc70e45..f7da3f187814 100644 --- a/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/clang/lib/Driver/ToolChains/Darwin.cpp @@ -209,8 +209,7 @@ static bool shouldLinkerNotDedup(bool IsLinkerOnlyAction, const ArgList &Args) { void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args, ArgStringList &CmdArgs, const InputInfoList &Inputs, - unsigned Version[5], bool LinkerIsLLD, - bool LinkerIsLLDDarwinNew) const { + unsigned Version[5], bool LinkerIsLLD) const { const Driver &D = getToolChain().getDriver(); const toolchains::MachO &MachOTC = getMachOToolChain(); @@ -343,7 +342,7 @@ void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args, Args.AddAllArgs(CmdArgs, options::OPT_init); // Add the deployment target. - if (Version[0] >= 520 || LinkerIsLLDDarwinNew) + if (Version[0] >= 520 || LinkerIsLLD) MachOTC.addPlatformVersionArgs(Args, CmdArgs); else MachOTC.addMinVersionArgs(Args, CmdArgs); @@ -556,14 +555,13 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA, << A->getAsString(Args); } - bool LinkerIsLLD, LinkerIsLLDDarwinNew; - const char *Exec = Args.MakeArgString( - getToolChain().GetLinkerPath(&LinkerIsLLD, &LinkerIsLLDDarwinNew)); + bool LinkerIsLLD; + const char *Exec = + Args.MakeArgString(getToolChain().GetLinkerPath(&LinkerIsLLD)); // I'm not sure why this particular decomposition exists in gcc, but // we follow suite for ease of comparison. - AddLinkArgs(C, Args, CmdArgs, Inputs, Version, LinkerIsLLD, - LinkerIsLLDDarwinNew); + AddLinkArgs(C, Args, CmdArgs, Inputs, Version, LinkerIsLLD); if (willEmitRemarks(Args) && checkRemarksOptions(getToolChain().getDriver(), Args, @@ -715,7 +713,7 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA, } ResponseFileSupport ResponseSupport; - if (Version[0] >= 705 || LinkerIsLLDDarwinNew) { + if (Version[0] >= 705 || LinkerIsLLD) { ResponseSupport = ResponseFileSupport::AtFileUTF8(); } else { // For older versions of the linker, use the legacy filelist method instead. @@ -1412,8 +1410,8 @@ static std::string getSystemOrSDKMacOSVersion(StringRef MacOSSDKVersion) { llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); if (!SystemTriple.isMacOSX()) return std::string(MacOSSDKVersion); - SystemTriple.getMacOSXVersion(Major, Minor, Micro); - VersionTuple SystemVersion(Major, Minor, Micro); + VersionTuple SystemVersion; + SystemTriple.getMacOSXVersion(SystemVersion); bool HadExtra; if (!Driver::GetReleaseVersion(MacOSSDKVersion, Major, Minor, Micro, HadExtra)) @@ -1554,12 +1552,10 @@ struct DarwinPlatform { const Optional &SDKInfo) { DarwinPlatform Result(TargetArg, getPlatformFromOS(TT.getOS()), OSVersion, A); - unsigned Major, Minor, Micro; - TT.getOSVersion(Major, Minor, Micro); - if (Major == 0) + VersionTuple OsVersion = TT.getOSVersion(); + if (OsVersion.getMajor() == 0) Result.HasOSVersion = false; - Result.setEnvironment(TT.getEnvironment(), - VersionTuple(Major, Minor, Micro), SDKInfo); + Result.setEnvironment(TT.getEnvironment(), OsVersion, SDKInfo); return Result; } static DarwinPlatform @@ -1805,7 +1801,7 @@ inferDeploymentTargetFromSDK(DerivedArgList &Args, std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple, const Driver &TheDriver) { - unsigned Major, Minor, Micro; + VersionTuple OsVersion; llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); switch (OS) { case llvm::Triple::Darwin: @@ -1814,24 +1810,22 @@ std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple, // macos, use the host triple to infer OS version. if (Triple.isMacOSX() && SystemTriple.isMacOSX() && !Triple.getOSMajorVersion()) - SystemTriple.getMacOSXVersion(Major, Minor, Micro); - else if (!Triple.getMacOSXVersion(Major, Minor, Micro)) + SystemTriple.getMacOSXVersion(OsVersion); + else if (!Triple.getMacOSXVersion(OsVersion)) TheDriver.Diag(diag::err_drv_invalid_darwin_version) << Triple.getOSName(); break; case llvm::Triple::IOS: if (Triple.isMacCatalystEnvironment() && !Triple.getOSMajorVersion()) { - Major = 13; - Minor = 1; - Micro = 0; + OsVersion = VersionTuple(13, 1); } else - Triple.getiOSVersion(Major, Minor, Micro); + OsVersion = Triple.getiOSVersion(); break; case llvm::Triple::TvOS: - Triple.getOSVersion(Major, Minor, Micro); + OsVersion = Triple.getOSVersion(); break; case llvm::Triple::WatchOS: - Triple.getWatchOSVersion(Major, Minor, Micro); + OsVersion = Triple.getWatchOSVersion(); break; default: llvm_unreachable("Unexpected OS type"); @@ -1839,7 +1833,9 @@ std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple, } std::string OSVersion; - llvm::raw_string_ostream(OSVersion) << Major << '.' << Minor << '.' << Micro; + llvm::raw_string_ostream(OSVersion) + << OsVersion.getMajor() << '.' << OsVersion.getMinor().getValueOr(0) + << '.' << OsVersion.getSubminor().getValueOr(0); return OSVersion; } @@ -1909,15 +1905,13 @@ getDeploymentTargetFromMTargetOSArg(DerivedArgList &Args, return None; } - unsigned Major, Minor, Micro; - TT.getOSVersion(Major, Minor, Micro); - if (!Major) { + VersionTuple Version = TT.getOSVersion(); + if (!Version.getMajor()) { TheDriver.Diag(diag::err_drv_invalid_version_number) << A->getAsString(Args); return None; } - return DarwinPlatform::createFromMTargetOS(TT.getOS(), - VersionTuple(Major, Minor, Micro), + return DarwinPlatform::createFromMTargetOS(TT.getOS(), Version, TT.getEnvironment(), A, SDKInfo); } diff --git a/clang/lib/Driver/ToolChains/Darwin.h b/clang/lib/Driver/ToolChains/Darwin.h index a307cd317ac3..5e23047a5512 100644 --- a/clang/lib/Driver/ToolChains/Darwin.h +++ b/clang/lib/Driver/ToolChains/Darwin.h @@ -64,7 +64,7 @@ class LLVM_LIBRARY_VISIBILITY Linker : public MachOTool { void AddLinkArgs(Compilation &C, const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs, const InputInfoList &Inputs, unsigned Version[5], - bool LinkerIsLLD, bool LinkerIsLLDDarwinNew) const; + bool LinkerIsLLD) const; public: Linker(const ToolChain &TC) : MachOTool("darwin::Linker", "linker", TC) {} diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp index d08ea282f6df..de635f5816cf 100644 --- a/clang/lib/Driver/ToolChains/FreeBSD.cpp +++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp @@ -293,8 +293,8 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, addLinkerCompressDebugSectionsOption(ToolChain, Args, CmdArgs); AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA); - bool Profiling = Args.hasArg(options::OPT_pg) && - ToolChain.getTriple().getOSMajorVersion() < 14; + unsigned Major = ToolChain.getTriple().getOSMajorVersion(); + bool Profiling = Args.hasArg(options::OPT_pg) && Major != 0 && Major < 14; if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) { // Use the static OpenMP runtime with -static-openmp bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && @@ -419,8 +419,8 @@ void FreeBSD::addLibStdCxxIncludePaths( void FreeBSD::AddCXXStdlibLibArgs(const ArgList &Args, ArgStringList &CmdArgs) const { CXXStdlibType Type = GetCXXStdlibType(Args); - bool Profiling = - Args.hasArg(options::OPT_pg) && getTriple().getOSMajorVersion() < 14; + unsigned Major = getTriple().getOSMajorVersion(); + bool Profiling = Args.hasArg(options::OPT_pg) && Major != 0 && Major < 14; switch (Type) { case ToolChain::CST_Libcxx: diff --git a/clang/lib/Driver/ToolChains/HIP.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp similarity index 60% rename from clang/lib/Driver/ToolChains/HIP.cpp rename to clang/lib/Driver/ToolChains/HIPAMD.cpp index 07af1a0457c7..6d553791b394 100644 --- a/clang/lib/Driver/ToolChains/HIP.cpp +++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp @@ -1,4 +1,4 @@ -//===--- HIP.cpp - HIP Tool and ToolChain Implementations -------*- C++ -*-===// +//===--- HIPAMD.cpp - HIP Tool and ToolChain Implementations ----*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,9 +6,10 @@ // //===----------------------------------------------------------------------===// -#include "HIP.h" +#include "HIPAMD.h" #include "AMDGPU.h" #include "CommonArgs.h" +#include "HIPUtility.h" #include "clang/Basic/Cuda.h" #include "clang/Basic/TargetID.h" #include "clang/Driver/Compilation.h" @@ -34,10 +35,6 @@ using namespace llvm::opt; #define NULL_FILE "/dev/null" #endif -namespace { -const unsigned HIPCodeObjectAlign = 4096; -} // namespace - static bool shouldSkipSanitizeOption(const ToolChain &TC, const llvm::opt::ArgList &DriverArgs, StringRef TargetID, @@ -76,9 +73,9 @@ static bool shouldSkipSanitizeOption(const ToolChain &TC, } void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA, - const InputInfoList &Inputs, - const InputInfo &Output, - const llvm::opt::ArgList &Args) const { + const InputInfoList &Inputs, + const InputInfo &Output, + const llvm::opt::ArgList &Args) const { // Construct lld command. // The output from ld.lld is an HSA code object file. ArgStringList LldArgs{"-flavor", "gnu", "--no-undefined", "-shared", @@ -129,142 +126,28 @@ void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA, Lld, LldArgs, Inputs, Output)); } -// Construct a clang-offload-bundler command to bundle code objects for -// different GPU's into a HIP fat binary. -void AMDGCN::constructHIPFatbinCommand(Compilation &C, const JobAction &JA, - StringRef OutputFileName, const InputInfoList &Inputs, - const llvm::opt::ArgList &Args, const Tool& T) { - // Construct clang-offload-bundler command to bundle object files for - // for different GPU archs. - ArgStringList BundlerArgs; - BundlerArgs.push_back(Args.MakeArgString("-type=o")); - BundlerArgs.push_back( - Args.MakeArgString("-bundle-align=" + Twine(HIPCodeObjectAlign))); - - // ToDo: Remove the dummy host binary entry which is required by - // clang-offload-bundler. - std::string BundlerTargetArg = "-targets=host-x86_64-unknown-linux"; - std::string BundlerInputArg = "-inputs=" NULL_FILE; - - // For code object version 2 and 3, the offload kind in bundle ID is 'hip' - // for backward compatibility. For code object version 4 and greater, the - // offload kind in bundle ID is 'hipv4'. - std::string OffloadKind = "hip"; - if (getAMDGPUCodeObjectVersion(C.getDriver(), Args) >= 4) - OffloadKind = OffloadKind + "v4"; - for (const auto &II : Inputs) { - const auto* A = II.getAction(); - BundlerTargetArg = BundlerTargetArg + "," + OffloadKind + - "-amdgcn-amd-amdhsa--" + - StringRef(A->getOffloadingArch()).str(); - BundlerInputArg = BundlerInputArg + "," + II.getFilename(); - } - BundlerArgs.push_back(Args.MakeArgString(BundlerTargetArg)); - BundlerArgs.push_back(Args.MakeArgString(BundlerInputArg)); - - std::string Output = std::string(OutputFileName); - auto BundlerOutputArg = - Args.MakeArgString(std::string("-outputs=").append(Output)); - BundlerArgs.push_back(BundlerOutputArg); - - const char *Bundler = Args.MakeArgString( - T.getToolChain().GetProgramPath("clang-offload-bundler")); - C.addCommand(std::make_unique( - JA, T, ResponseFileSupport::None(), Bundler, BundlerArgs, Inputs, - InputInfo(&JA, Args.MakeArgString(Output)))); -} - -/// Add Generated HIP Object File which has device images embedded into the -/// host to the argument list for linking. Using MC directives, embed the -/// device code and also define symbols required by the code generation so that -/// the image can be retrieved at runtime. -void AMDGCN::Linker::constructGenerateObjFileFromHIPFatBinary( - Compilation &C, const InputInfo &Output, - const InputInfoList &Inputs, const ArgList &Args, - const JobAction &JA) const { - const ToolChain &TC = getToolChain(); - std::string Name = - std::string(llvm::sys::path::stem(Output.getFilename())); - - // Create Temp Object File Generator, - // Offload Bundled file and Bundled Object file. - // Keep them if save-temps is enabled. - const char *McinFile; - const char *BundleFile; - if (C.getDriver().isSaveTempsEnabled()) { - McinFile = C.getArgs().MakeArgString(Name + ".mcin"); - BundleFile = C.getArgs().MakeArgString(Name + ".hipfb"); - } else { - auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin"); - McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin)); - auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb"); - BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb)); - } - constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, *this); - - // Create a buffer to write the contents of the temp obj generator. - std::string ObjBuffer; - llvm::raw_string_ostream ObjStream(ObjBuffer); - - // Add MC directives to embed target binaries. We ensure that each - // section and image is 16-byte aligned. This is not mandatory, but - // increases the likelihood of data to be aligned with a cache block - // in several main host machines. - ObjStream << "# HIP Object Generator\n"; - ObjStream << "# *** Automatically generated by Clang ***\n"; - ObjStream << " .protected __hip_fatbin\n"; - ObjStream << " .type __hip_fatbin,@object\n"; - ObjStream << " .section .hip_fatbin,\"a\",@progbits\n"; - ObjStream << " .globl __hip_fatbin\n"; - ObjStream << " .p2align " << llvm::Log2(llvm::Align(HIPCodeObjectAlign)) - << "\n"; - ObjStream << "__hip_fatbin:\n"; - ObjStream << " .incbin \"" << BundleFile << "\"\n"; - ObjStream.flush(); - - // Dump the contents of the temp object file gen if the user requested that. - // We support this option to enable testing of behavior with -###. - if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script)) - llvm::errs() << ObjBuffer; - - // Open script file and write the contents. - std::error_code EC; - llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None); - - if (EC) { - C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message(); - return; - } - - Objf << ObjBuffer; - - ArgStringList McArgs{"-o", Output.getFilename(), - McinFile, "--filetype=obj"}; - const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc")); - C.addCommand(std::make_unique(JA, *this, ResponseFileSupport::None(), - Mc, McArgs, Inputs, Output)); -} - // For amdgcn the inputs of the linker job are device bitcode and output is // object file. It calls llvm-link, opt, llc, then lld steps. void AMDGCN::Linker::ConstructJob(Compilation &C, const JobAction &JA, - const InputInfo &Output, - const InputInfoList &Inputs, - const ArgList &Args, - const char *LinkingOutput) const { + const InputInfo &Output, + const InputInfoList &Inputs, + const ArgList &Args, + const char *LinkingOutput) const { if (Inputs.size() > 0 && Inputs[0].getType() == types::TY_Image && JA.getType() == types::TY_Object) - return constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs, Args, JA); + return HIP::constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs, + Args, JA, *this); if (JA.getType() == types::TY_HIP_FATBIN) - return constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs, Args, *this); + return HIP::constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs, + Args, *this); return constructLldCommand(C, JA, Inputs, Output, Args); } -HIPToolChain::HIPToolChain(const Driver &D, const llvm::Triple &Triple, - const ToolChain &HostTC, const ArgList &Args) +HIPAMDToolChain::HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple, + const ToolChain &HostTC, const ArgList &Args) : ROCMToolChain(D, Triple, Args), HostTC(HostTC) { // Lookup binaries into the driver directory, this is used to // discover the clang-offload-bundler executable. @@ -279,9 +162,8 @@ HIPToolChain::HIPToolChain(const Driver &D, const llvm::Triple &Triple, } } -void HIPToolChain::addClangTargetOptions( - const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args, +void HIPAMDToolChain::addClangTargetOptions( + const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args, Action::OffloadKind DeviceOffloadingKind) const { HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind); @@ -324,9 +206,9 @@ void HIPToolChain::addClangTargetOptions( } llvm::opt::DerivedArgList * -HIPToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args, - StringRef BoundArch, - Action::OffloadKind DeviceOffloadKind) const { +HIPAMDToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args, + StringRef BoundArch, + Action::OffloadKind DeviceOffloadKind) const { DerivedArgList *DAL = HostTC.TranslateArgs(Args, BoundArch, DeviceOffloadKind); if (!DAL) @@ -349,44 +231,44 @@ HIPToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args, return DAL; } -Tool *HIPToolChain::buildLinker() const { +Tool *HIPAMDToolChain::buildLinker() const { assert(getTriple().getArch() == llvm::Triple::amdgcn); return new tools::AMDGCN::Linker(*this); } -void HIPToolChain::addClangWarningOptions(ArgStringList &CC1Args) const { +void HIPAMDToolChain::addClangWarningOptions(ArgStringList &CC1Args) const { HostTC.addClangWarningOptions(CC1Args); } ToolChain::CXXStdlibType -HIPToolChain::GetCXXStdlibType(const ArgList &Args) const { +HIPAMDToolChain::GetCXXStdlibType(const ArgList &Args) const { return HostTC.GetCXXStdlibType(Args); } -void HIPToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { +void HIPAMDToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, + ArgStringList &CC1Args) const { HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args); } -void HIPToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args, - ArgStringList &CC1Args) const { +void HIPAMDToolChain::AddClangCXXStdlibIncludeArgs( + const ArgList &Args, ArgStringList &CC1Args) const { HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args); } -void HIPToolChain::AddIAMCUIncludeArgs(const ArgList &Args, - ArgStringList &CC1Args) const { +void HIPAMDToolChain::AddIAMCUIncludeArgs(const ArgList &Args, + ArgStringList &CC1Args) const { HostTC.AddIAMCUIncludeArgs(Args, CC1Args); } -void HIPToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs, - ArgStringList &CC1Args) const { +void HIPAMDToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs, + ArgStringList &CC1Args) const { RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args); } -SanitizerMask HIPToolChain::getSupportedSanitizers() const { - // The HIPToolChain only supports sanitizers in the sense that it allows +SanitizerMask HIPAMDToolChain::getSupportedSanitizers() const { + // The HIPAMDToolChain only supports sanitizers in the sense that it allows // sanitizer arguments on the command line if they are supported by the host - // toolchain. The HIPToolChain will actually ignore any command line + // toolchain. The HIPAMDToolChain will actually ignore any command line // arguments for any of these "supported" sanitizers. That means that no // sanitization of device code is actually supported at this time. // @@ -396,13 +278,13 @@ SanitizerMask HIPToolChain::getSupportedSanitizers() const { return HostTC.getSupportedSanitizers(); } -VersionTuple HIPToolChain::computeMSVCVersion(const Driver *D, - const ArgList &Args) const { +VersionTuple HIPAMDToolChain::computeMSVCVersion(const Driver *D, + const ArgList &Args) const { return HostTC.computeMSVCVersion(D, Args); } llvm::SmallVector -HIPToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const { +HIPAMDToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const { llvm::SmallVector BCLibs; if (DriverArgs.hasArg(options::OPT_nogpulib)) return {}; @@ -476,11 +358,11 @@ HIPToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const { return BCLibs; } -void HIPToolChain::checkTargetID(const llvm::opt::ArgList &DriverArgs) const { +void HIPAMDToolChain::checkTargetID( + const llvm::opt::ArgList &DriverArgs) const { auto PTID = getParsedTargetID(DriverArgs); if (PTID.OptionalTargetID && !PTID.OptionalGPUArch) { getDriver().Diag(clang::diag::err_drv_bad_target_id) << PTID.OptionalTargetID.getValue(); } - return; } diff --git a/clang/lib/Driver/ToolChains/HIP.h b/clang/lib/Driver/ToolChains/HIPAMD.h similarity index 69% rename from clang/lib/Driver/ToolChains/HIP.h rename to clang/lib/Driver/ToolChains/HIPAMD.h index 60b3d69b3f52..cc472a595db9 100644 --- a/clang/lib/Driver/ToolChains/HIP.h +++ b/clang/lib/Driver/ToolChains/HIPAMD.h @@ -1,4 +1,4 @@ -//===--- HIP.h - HIP ToolChain Implementations ------------------*- C++ -*-===// +//===--- HIPAMD.h - HIP ToolChain Implementations ---------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIP_H -#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIP_H +#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPAMD_H +#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPAMD_H -#include "clang/Driver/ToolChain.h" -#include "clang/Driver/Tool.h" #include "AMDGPU.h" +#include "clang/Driver/Tool.h" +#include "clang/Driver/ToolChain.h" namespace clang { namespace driver { @@ -19,11 +19,6 @@ namespace driver { namespace tools { namespace AMDGCN { - // Construct command for creating HIP fatbin. - void constructHIPFatbinCommand(Compilation &C, const JobAction &JA, - StringRef OutputFileName, const InputInfoList &Inputs, - const llvm::opt::ArgList &TCArgs, const Tool& T); - // Runs llvm-link/opt/llc/lld, which links multiple LLVM bitcode, together with // device library, then compiles it to ISA in a shared object. class LLVM_LIBRARY_VISIBILITY Linker : public Tool { @@ -38,17 +33,9 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool { const char *LinkingOutput) const override; private: - void constructLldCommand(Compilation &C, const JobAction &JA, const InputInfoList &Inputs, const InputInfo &Output, const llvm::opt::ArgList &Args) const; - - // Construct command for creating Object from HIP fatbin. - void constructGenerateObjFileFromHIPFatBinary(Compilation &C, - const InputInfo &Output, - const InputInfoList &Inputs, - const llvm::opt::ArgList &Args, - const JobAction &JA) const; }; } // end namespace AMDGCN @@ -56,10 +43,10 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool { namespace toolchains { -class LLVM_LIBRARY_VISIBILITY HIPToolChain final : public ROCMToolChain { +class LLVM_LIBRARY_VISIBILITY HIPAMDToolChain final : public ROCMToolChain { public: - HIPToolChain(const Driver &D, const llvm::Triple &Triple, - const ToolChain &HostTC, const llvm::opt::ArgList &Args); + HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple, + const ToolChain &HostTC, const llvm::opt::ArgList &Args); const llvm::Triple *getAuxTriple() const override { return &HostTC.getTriple(); @@ -68,9 +55,10 @@ class LLVM_LIBRARY_VISIBILITY HIPToolChain final : public ROCMToolChain { llvm::opt::DerivedArgList * TranslateArgs(const llvm::opt::DerivedArgList &Args, StringRef BoundArch, Action::OffloadKind DeviceOffloadKind) const override; - void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, - llvm::opt::ArgStringList &CC1Args, - Action::OffloadKind DeviceOffloadKind) const override; + void + addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; void addClangWarningOptions(llvm::opt::ArgStringList &CC1Args) const override; CXXStdlibType GetCXXStdlibType(const llvm::opt::ArgList &Args) const override; void @@ -105,4 +93,4 @@ class LLVM_LIBRARY_VISIBILITY HIPToolChain final : public ROCMToolChain { } // end namespace driver } // end namespace clang -#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIP_H +#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPAMD_H diff --git a/clang/lib/Driver/ToolChains/HIPSPV.cpp b/clang/lib/Driver/ToolChains/HIPSPV.cpp new file mode 100644 index 000000000000..d68c87e9b3e7 --- /dev/null +++ b/clang/lib/Driver/ToolChains/HIPSPV.cpp @@ -0,0 +1,292 @@ +//===--- HIPSPV.cpp - HIPSPV ToolChain Implementation -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "HIPSPV.h" +#include "CommonArgs.h" +#include "HIPUtility.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/InputInfo.h" +#include "clang/Driver/Options.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +using namespace clang::driver; +using namespace clang::driver::toolchains; +using namespace clang::driver::tools; +using namespace clang; +using namespace llvm::opt; + +// Convenience function for creating temporary file for both modes of +// isSaveTempsEnabled(). +static const char *getTempFile(Compilation &C, StringRef Prefix, + StringRef Extension) { + if (C.getDriver().isSaveTempsEnabled()) { + return C.getArgs().MakeArgString(Prefix + "." + Extension); + } + auto TmpFile = C.getDriver().GetTemporaryPath(Prefix, Extension); + return C.addTempFile(C.getArgs().MakeArgString(TmpFile)); +} + +// Locates HIP pass plugin. +static std::string findPassPlugin(const Driver &D, + const llvm::opt::ArgList &Args) { + StringRef Path = Args.getLastArgValue(options::OPT_hipspv_pass_plugin_EQ); + if (!Path.empty()) { + if (llvm::sys::fs::exists(Path)) + return Path.str(); + D.Diag(diag::err_drv_no_such_file) << Path; + } + + StringRef hipPath = Args.getLastArgValue(options::OPT_hip_path_EQ); + if (!hipPath.empty()) { + SmallString<128> PluginPath(hipPath); + llvm::sys::path::append(PluginPath, "lib", "libLLVMHipSpvPasses.so"); + if (llvm::sys::fs::exists(PluginPath)) + return PluginPath.str().str(); + PluginPath.assign(hipPath); + llvm::sys::path::append(PluginPath, "lib", "llvm", + "libLLVMHipSpvPasses.so"); + if (llvm::sys::fs::exists(PluginPath)) + return PluginPath.str().str(); + } + + return std::string(); +} + +void HIPSPV::Linker::constructLinkAndEmitSpirvCommand( + Compilation &C, const JobAction &JA, const InputInfoList &Inputs, + const InputInfo &Output, const llvm::opt::ArgList &Args) const { + + assert(!Inputs.empty() && "Must have at least one input."); + std::string Name = std::string(llvm::sys::path::stem(Output.getFilename())); + const char *TempFile = getTempFile(C, Name + "-link", "bc"); + + // Link LLVM bitcode. + ArgStringList LinkArgs{}; + for (auto Input : Inputs) + LinkArgs.push_back(Input.getFilename()); + LinkArgs.append({"-o", TempFile}); + const char *LlvmLink = + Args.MakeArgString(getToolChain().GetProgramPath("llvm-link")); + C.addCommand(std::make_unique(JA, *this, ResponseFileSupport::None(), + LlvmLink, LinkArgs, Inputs, Output)); + + // Post-link HIP lowering. + + // Run LLVM IR passes to lower/expand/emulate HIP code that does not translate + // to SPIR-V (E.g. dynamic shared memory). + auto PassPluginPath = findPassPlugin(C.getDriver(), Args); + if (!PassPluginPath.empty()) { + const char *PassPathCStr = C.getArgs().MakeArgString(PassPluginPath); + const char *OptOutput = getTempFile(C, Name + "-lower", "bc"); + ArgStringList OptArgs{TempFile, "-load-pass-plugin", + PassPathCStr, "-passes=hip-post-link-passes", + "-o", OptOutput}; + const char *Opt = Args.MakeArgString(getToolChain().GetProgramPath("opt")); + C.addCommand(std::make_unique( + JA, *this, ResponseFileSupport::None(), Opt, OptArgs, Inputs, Output)); + TempFile = OptOutput; + } + + // Emit SPIR-V binary. + + llvm::opt::ArgStringList TrArgs{"--spirv-max-version=1.1", + "--spirv-ext=+all"}; + InputInfo TrInput = InputInfo(types::TY_LLVM_BC, TempFile, ""); + SPIRV::constructTranslateCommand(C, *this, JA, Output, TrInput, TrArgs); +} + +void HIPSPV::Linker::ConstructJob(Compilation &C, const JobAction &JA, + const InputInfo &Output, + const InputInfoList &Inputs, + const ArgList &Args, + const char *LinkingOutput) const { + if (Inputs.size() > 0 && Inputs[0].getType() == types::TY_Image && + JA.getType() == types::TY_Object) + return HIP::constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs, + Args, JA, *this); + + if (JA.getType() == types::TY_HIP_FATBIN) + return HIP::constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs, + Args, *this); + + constructLinkAndEmitSpirvCommand(C, JA, Inputs, Output, Args); +} + +HIPSPVToolChain::HIPSPVToolChain(const Driver &D, const llvm::Triple &Triple, + const ToolChain &HostTC, const ArgList &Args) + : ToolChain(D, Triple, Args), HostTC(HostTC) { + // Lookup binaries into the driver directory, this is used to + // discover the clang-offload-bundler executable. + getProgramPaths().push_back(getDriver().Dir); +} + +void HIPSPVToolChain::addClangTargetOptions( + const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadingKind) const { + HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind); + + assert(DeviceOffloadingKind == Action::OFK_HIP && + "Only HIP offloading kinds are supported for GPUs."); + + CC1Args.append( + {"-fcuda-is-device", "-fcuda-allow-variadic-functions", + // A crude workaround for llvm-spirv which does not handle the + // autovectorized code well (vector reductions, non-i{8,16,32,64} types). + // TODO: Allow autovectorization when SPIR-V backend arrives. + "-mllvm", "-vectorize-loops=false", "-mllvm", "-vectorize-slp=false"}); + + if (DriverArgs.hasFlag(options::OPT_fcuda_approx_transcendentals, + options::OPT_fno_cuda_approx_transcendentals, false)) + CC1Args.push_back("-fcuda-approx-transcendentals"); + + // Default to "hidden" visibility, as object level linking will not be + // supported for the foreseeable future. + if (!DriverArgs.hasArg(options::OPT_fvisibility_EQ, + options::OPT_fvisibility_ms_compat)) + CC1Args.append( + {"-fvisibility", "hidden", "-fapply-global-visibility-to-externs"}); + + llvm::for_each(getHIPDeviceLibs(DriverArgs), + [&](const BitCodeLibraryInfo &BCFile) { + CC1Args.append({"-mlink-builtin-bitcode", + DriverArgs.MakeArgString(BCFile.Path)}); + }); +} + +Tool *HIPSPVToolChain::buildLinker() const { + assert(getTriple().getArch() == llvm::Triple::spirv64); + return new tools::HIPSPV::Linker(*this); +} + +void HIPSPVToolChain::addClangWarningOptions(ArgStringList &CC1Args) const { + HostTC.addClangWarningOptions(CC1Args); +} + +ToolChain::CXXStdlibType +HIPSPVToolChain::GetCXXStdlibType(const ArgList &Args) const { + return HostTC.GetCXXStdlibType(Args); +} + +void HIPSPVToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, + ArgStringList &CC1Args) const { + HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args); +} + +void HIPSPVToolChain::AddClangCXXStdlibIncludeArgs( + const ArgList &Args, ArgStringList &CC1Args) const { + HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args); +} + +void HIPSPVToolChain::AddIAMCUIncludeArgs(const ArgList &Args, + ArgStringList &CC1Args) const { + HostTC.AddIAMCUIncludeArgs(Args, CC1Args); +} + +void HIPSPVToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs, + ArgStringList &CC1Args) const { + if (DriverArgs.hasArg(options::OPT_nogpuinc)) + return; + + StringRef hipPath = DriverArgs.getLastArgValue(options::OPT_hip_path_EQ); + if (hipPath.empty()) { + getDriver().Diag(diag::err_drv_hipspv_no_hip_path) << 1 << "'-nogpuinc'"; + return; + } + SmallString<128> P(hipPath); + llvm::sys::path::append(P, "include"); + CC1Args.append({"-isystem", DriverArgs.MakeArgString(P)}); +} + +llvm::SmallVector +HIPSPVToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const { + llvm::SmallVector BCLibs; + if (DriverArgs.hasArg(options::OPT_nogpulib)) + return {}; + + ArgStringList LibraryPaths; + // Find device libraries in --hip-device-lib-path and HIP_DEVICE_LIB_PATH. + auto HipDeviceLibPathArgs = DriverArgs.getAllArgValues( + // --hip-device-lib-path is alias to this option. + clang::driver::options::OPT_rocm_device_lib_path_EQ); + for (auto Path : HipDeviceLibPathArgs) + LibraryPaths.push_back(DriverArgs.MakeArgString(Path)); + + StringRef HipPath = DriverArgs.getLastArgValue(options::OPT_hip_path_EQ); + if (!HipPath.empty()) { + SmallString<128> Path(HipPath); + llvm::sys::path::append(Path, "lib", "hip-device-lib"); + LibraryPaths.push_back(DriverArgs.MakeArgString(Path)); + } + + addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH"); + + // Maintain compatability with --hip-device-lib. + auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ); + if (!BCLibArgs.empty()) { + llvm::for_each(BCLibArgs, [&](StringRef BCName) { + StringRef FullName; + for (std::string LibraryPath : LibraryPaths) { + SmallString<128> Path(LibraryPath); + llvm::sys::path::append(Path, BCName); + FullName = Path; + if (llvm::sys::fs::exists(FullName)) { + BCLibs.emplace_back(FullName.str()); + return; + } + } + getDriver().Diag(diag::err_drv_no_such_file) << BCName; + }); + } else { + // Search device library named as 'hipspv-.bc'. + auto TT = getTriple().normalize(); + std::string BCName = "hipspv-" + TT + ".bc"; + for (auto *LibPath : LibraryPaths) { + SmallString<128> Path(LibPath); + llvm::sys::path::append(Path, BCName); + if (llvm::sys::fs::exists(Path)) { + BCLibs.emplace_back(Path.str().str()); + return BCLibs; + } + } + getDriver().Diag(diag::err_drv_no_hipspv_device_lib) + << 1 << ("'" + TT + "' target"); + return {}; + } + + return BCLibs; +} + +SanitizerMask HIPSPVToolChain::getSupportedSanitizers() const { + // The HIPSPVToolChain only supports sanitizers in the sense that it allows + // sanitizer arguments on the command line if they are supported by the host + // toolchain. The HIPSPVToolChain will actually ignore any command line + // arguments for any of these "supported" sanitizers. That means that no + // sanitization of device code is actually supported at this time. + // + // This behavior is necessary because the host and device toolchains + // invocations often share the command line, so the device toolchain must + // tolerate flags meant only for the host toolchain. + return HostTC.getSupportedSanitizers(); +} + +VersionTuple HIPSPVToolChain::computeMSVCVersion(const Driver *D, + const ArgList &Args) const { + return HostTC.computeMSVCVersion(D, Args); +} + +void HIPSPVToolChain::adjustDebugInfoKind( + codegenoptions::DebugInfoKind &DebugInfoKind, + const llvm::opt::ArgList &Args) const { + // Debug info generation is disabled for SPIRV-LLVM-Translator + // which currently aborts on the presence of DW_OP_LLVM_convert. + // TODO: Enable debug info when the SPIR-V backend arrives. + DebugInfoKind = codegenoptions::NoDebugInfo; +} diff --git a/clang/lib/Driver/ToolChains/HIPSPV.h b/clang/lib/Driver/ToolChains/HIPSPV.h new file mode 100644 index 000000000000..79520f77c742 --- /dev/null +++ b/clang/lib/Driver/ToolChains/HIPSPV.h @@ -0,0 +1,103 @@ +//===--- HIPSPV.h - HIP ToolChain Implementations ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPSPV_H +#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPSPV_H + +#include "SPIRV.h" +#include "clang/Driver/Tool.h" +#include "clang/Driver/ToolChain.h" + +namespace clang { +namespace driver { +namespace tools { +namespace HIPSPV { + +// Runs llvm-link/opt/llc/lld, which links multiple LLVM bitcode, together with +// device library, then compiles it to SPIR-V in a shared object. +class LLVM_LIBRARY_VISIBILITY Linker : public Tool { +public: + Linker(const ToolChain &TC) : Tool("HIPSPV::Linker", "hipspv-link", TC) {} + + bool hasIntegratedCPP() const override { return false; } + + void ConstructJob(Compilation &C, const JobAction &JA, + const InputInfo &Output, const InputInfoList &Inputs, + const llvm::opt::ArgList &TCArgs, + const char *LinkingOutput) const override; + +private: + void constructLinkAndEmitSpirvCommand(Compilation &C, const JobAction &JA, + const InputInfoList &Inputs, + const InputInfo &Output, + const llvm::opt::ArgList &Args) const; +}; + +} // namespace HIPSPV +} // namespace tools + +namespace toolchains { + +class LLVM_LIBRARY_VISIBILITY HIPSPVToolChain final : public ToolChain { +public: + HIPSPVToolChain(const Driver &D, const llvm::Triple &Triple, + const ToolChain &HostTC, const llvm::opt::ArgList &Args); + + const llvm::Triple *getAuxTriple() const override { + return &HostTC.getTriple(); + } + + void + addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, + llvm::opt::ArgStringList &CC1Args, + Action::OffloadKind DeviceOffloadKind) const override; + void addClangWarningOptions(llvm::opt::ArgStringList &CC1Args) const override; + CXXStdlibType GetCXXStdlibType(const llvm::opt::ArgList &Args) const override; + void + AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs, + llvm::opt::ArgStringList &CC1Args) const override; + void AddClangCXXStdlibIncludeArgs( + const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CC1Args) const override; + void AddIAMCUIncludeArgs(const llvm::opt::ArgList &DriverArgs, + llvm::opt::ArgStringList &CC1Args) const override; + void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs, + llvm::opt::ArgStringList &CC1Args) const override; + llvm::SmallVector + getHIPDeviceLibs(const llvm::opt::ArgList &Args) const override; + + SanitizerMask getSupportedSanitizers() const override; + + VersionTuple + computeMSVCVersion(const Driver *D, + const llvm::opt::ArgList &Args) const override; + + void adjustDebugInfoKind(codegenoptions::DebugInfoKind &DebugInfoKind, + const llvm::opt::ArgList &Args) const override; + bool IsIntegratedAssemblerDefault() const override { return true; } + bool IsMathErrnoDefault() const override { return false; } + bool useIntegratedAs() const override { return true; } + bool isCrossCompiling() const override { return true; } + bool isPICDefault() const override { return false; } + bool isPIEDefault(const llvm::opt::ArgList &Args) const override { + return false; + } + bool isPICDefaultForced() const override { return false; } + bool SupportsProfiling() const override { return false; } + + const ToolChain &HostTC; + +protected: + Tool *buildLinker() const override; +}; + +} // end namespace toolchains +} // end namespace driver +} // end namespace clang + +#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPSPV_H diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp new file mode 100644 index 000000000000..1b04a20bacbf --- /dev/null +++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp @@ -0,0 +1,167 @@ +//===--- HIPUtility.cpp - Common HIP Tool Chain Utilities -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "HIPUtility.h" +#include "CommonArgs.h" +#include "clang/Driver/Compilation.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" +#include "llvm/Support/Path.h" + +using namespace clang::driver; +using namespace clang::driver::tools; +using namespace llvm::opt; + +#if defined(_WIN32) || defined(_WIN64) +#define NULL_FILE "nul" +#else +#define NULL_FILE "/dev/null" +#endif + +namespace { +const unsigned HIPCodeObjectAlign = 4096; +} // namespace + +// Constructs a triple string for clang offload bundler. +static std::string normalizeForBundler(const llvm::Triple &T, + bool HasTargetID) { + return HasTargetID ? (T.getArchName() + "-" + T.getVendorName() + "-" + + T.getOSName() + "-" + T.getEnvironmentName()) + .str() + : T.normalize(); +} + +// Construct a clang-offload-bundler command to bundle code objects for +// different devices into a HIP fat binary. +void HIP::constructHIPFatbinCommand(Compilation &C, const JobAction &JA, + llvm::StringRef OutputFileName, + const InputInfoList &Inputs, + const llvm::opt::ArgList &Args, + const Tool &T) { + // Construct clang-offload-bundler command to bundle object files for + // for different GPU archs. + ArgStringList BundlerArgs; + BundlerArgs.push_back(Args.MakeArgString("-type=o")); + BundlerArgs.push_back( + Args.MakeArgString("-bundle-align=" + Twine(HIPCodeObjectAlign))); + + // ToDo: Remove the dummy host binary entry which is required by + // clang-offload-bundler. + std::string BundlerTargetArg = "-targets=host-x86_64-unknown-linux"; + std::string BundlerInputArg = "-inputs=" NULL_FILE; + + // AMDGCN: + // For code object version 2 and 3, the offload kind in bundle ID is 'hip' + // for backward compatibility. For code object version 4 and greater, the + // offload kind in bundle ID is 'hipv4'. + std::string OffloadKind = "hip"; + auto &TT = T.getToolChain().getTriple(); + if (TT.isAMDGCN() && getAMDGPUCodeObjectVersion(C.getDriver(), Args) >= 4) + OffloadKind = OffloadKind + "v4"; + for (const auto &II : Inputs) { + const auto *A = II.getAction(); + auto ArchStr = llvm::StringRef(A->getOffloadingArch()); + BundlerTargetArg += + "," + OffloadKind + "-" + normalizeForBundler(TT, !ArchStr.empty()); + if (!ArchStr.empty()) + BundlerTargetArg += "-" + ArchStr.str(); + BundlerInputArg = BundlerInputArg + "," + II.getFilename(); + } + BundlerArgs.push_back(Args.MakeArgString(BundlerTargetArg)); + BundlerArgs.push_back(Args.MakeArgString(BundlerInputArg)); + + std::string Output = std::string(OutputFileName); + auto *BundlerOutputArg = + Args.MakeArgString(std::string("-outputs=").append(Output)); + BundlerArgs.push_back(BundlerOutputArg); + + const char *Bundler = Args.MakeArgString( + T.getToolChain().GetProgramPath("clang-offload-bundler")); + C.addCommand(std::make_unique( + JA, T, ResponseFileSupport::None(), Bundler, BundlerArgs, Inputs, + InputInfo(&JA, Args.MakeArgString(Output)))); +} + +/// Add Generated HIP Object File which has device images embedded into the +/// host to the argument list for linking. Using MC directives, embed the +/// device code and also define symbols required by the code generation so that +/// the image can be retrieved at runtime. +void HIP::constructGenerateObjFileFromHIPFatBinary( + Compilation &C, const InputInfo &Output, const InputInfoList &Inputs, + const ArgList &Args, const JobAction &JA, const Tool &T) { + const ToolChain &TC = T.getToolChain(); + std::string Name = std::string(llvm::sys::path::stem(Output.getFilename())); + + // Create Temp Object File Generator, + // Offload Bundled file and Bundled Object file. + // Keep them if save-temps is enabled. + const char *McinFile; + const char *BundleFile; + if (C.getDriver().isSaveTempsEnabled()) { + McinFile = C.getArgs().MakeArgString(Name + ".mcin"); + BundleFile = C.getArgs().MakeArgString(Name + ".hipfb"); + } else { + auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin"); + McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin)); + auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb"); + BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb)); + } + HIP::constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, T); + + // Create a buffer to write the contents of the temp obj generator. + std::string ObjBuffer; + llvm::raw_string_ostream ObjStream(ObjBuffer); + + auto HostTriple = + C.getSingleOffloadToolChain()->getTriple(); + + // Add MC directives to embed target binaries. We ensure that each + // section and image is 16-byte aligned. This is not mandatory, but + // increases the likelihood of data to be aligned with a cache block + // in several main host machines. + ObjStream << "# HIP Object Generator\n"; + ObjStream << "# *** Automatically generated by Clang ***\n"; + if (HostTriple.isWindowsMSVCEnvironment()) { + ObjStream << " .section .hip_fatbin, \"dw\"\n"; + } else { + ObjStream << " .protected __hip_fatbin\n"; + ObjStream << " .type __hip_fatbin,@object\n"; + ObjStream << " .section .hip_fatbin,\"a\",@progbits\n"; + } + ObjStream << " .globl __hip_fatbin\n"; + ObjStream << " .p2align " << llvm::Log2(llvm::Align(HIPCodeObjectAlign)) + << "\n"; + ObjStream << "__hip_fatbin:\n"; + ObjStream << " .incbin "; + llvm::sys::printArg(ObjStream, BundleFile, /*Quote=*/true); + ObjStream << "\n"; + ObjStream.flush(); + + // Dump the contents of the temp object file gen if the user requested that. + // We support this option to enable testing of behavior with -###. + if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script)) + llvm::errs() << ObjBuffer; + + // Open script file and write the contents. + std::error_code EC; + llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None); + + if (EC) { + C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message(); + return; + } + + Objf << ObjBuffer; + + ArgStringList McArgs{"-triple", Args.MakeArgString(HostTriple.normalize()), + "-o", Output.getFilename(), + McinFile, "--filetype=obj"}; + const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc")); + C.addCommand(std::make_unique(JA, T, ResponseFileSupport::None(), Mc, + McArgs, Inputs, Output)); +} diff --git a/clang/lib/Driver/ToolChains/HIPUtility.h b/clang/lib/Driver/ToolChains/HIPUtility.h new file mode 100644 index 000000000000..29e5a922024a --- /dev/null +++ b/clang/lib/Driver/ToolChains/HIPUtility.h @@ -0,0 +1,35 @@ +//===--- HIPUtility.h - Common HIP Tool Chain Utilities ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPUTILITY_H +#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPUTILITY_H + +#include "clang/Driver/Tool.h" + +namespace clang { +namespace driver { +namespace tools { +namespace HIP { + +// Construct command for creating HIP fatbin. +void constructHIPFatbinCommand(Compilation &C, const JobAction &JA, + StringRef OutputFileName, + const InputInfoList &Inputs, + const llvm::opt::ArgList &TCArgs, const Tool &T); + +// Construct command for creating Object from HIP fatbin. +void constructGenerateObjFileFromHIPFatBinary( + Compilation &C, const InputInfo &Output, const InputInfoList &Inputs, + const llvm::opt::ArgList &Args, const JobAction &JA, const Tool &T); + +} // namespace HIP +} // namespace tools +} // namespace driver +} // namespace clang + +#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPUTILITY_H diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp index 18270818d158..ba3040636604 100644 --- a/clang/lib/Driver/ToolChains/Hexagon.cpp +++ b/clang/lib/Driver/ToolChains/Hexagon.cpp @@ -26,8 +26,8 @@ using namespace clang; using namespace llvm::opt; // Default hvx-length for various versions. -static StringRef getDefaultHvxLength(StringRef Cpu) { - return llvm::StringSwitch(Cpu) +static StringRef getDefaultHvxLength(StringRef HvxVer) { + return llvm::StringSwitch(HvxVer) .Case("v60", "64b") .Case("v62", "64b") .Case("v65", "64b") @@ -51,42 +51,107 @@ static void handleHVXTargetFeatures(const Driver &D, const ArgList &Args, // Handle HVX warnings. handleHVXWarnings(D, Args); - // Add the +hvx* features based on commandline flags. - StringRef HVXFeature, HVXLength; + auto makeFeature = [&Args](Twine T, bool Enable) -> StringRef { + const std::string &S = T.str(); + StringRef Opt(S); + if (Opt.endswith("=")) + Opt = Opt.drop_back(1); + if (Opt.startswith("mno-")) + Opt = Opt.drop_front(4); + else if (Opt.startswith("m")) + Opt = Opt.drop_front(1); + return Args.MakeArgString(Twine(Enable ? "+" : "-") + Twine(Opt)); + }; - // Handle -mhvx, -mhvx=, -mno-hvx. - if (Arg *A = Args.getLastArg(options::OPT_mno_hexagon_hvx, - options::OPT_mhexagon_hvx, - options::OPT_mhexagon_hvx_EQ)) { - if (A->getOption().matches(options::OPT_mno_hexagon_hvx)) - return; - if (A->getOption().matches(options::OPT_mhexagon_hvx_EQ)) { - HasHVX = true; - HVXFeature = Cpu = A->getValue(); - HVXFeature = Args.MakeArgString(llvm::Twine("+hvx") + HVXFeature.lower()); - } else if (A->getOption().matches(options::OPT_mhexagon_hvx)) { - HasHVX = true; - HVXFeature = Args.MakeArgString(llvm::Twine("+hvx") + Cpu); + auto withMinus = [](StringRef S) -> std::string { + return "-" + S.str(); + }; + + // Drop tiny core suffix for HVX version. + std::string HvxVer = + (Cpu.back() == 'T' || Cpu.back() == 't' ? Cpu.drop_back(1) : Cpu).str(); + HasHVX = false; + + // Handle -mhvx, -mhvx=, -mno-hvx. If both present, -mhvx= wins over -mhvx. + auto argOrNull = [&Args](auto FlagOn, auto FlagOff) -> Arg* { + if (Arg *A = Args.getLastArg(FlagOn, FlagOff)) { + if (A->getOption().matches(FlagOn)) + return A; } - Features.push_back(HVXFeature); + return nullptr; + }; + + Arg *HvxBareA = + argOrNull(options::OPT_mhexagon_hvx, options::OPT_mno_hexagon_hvx); + Arg *HvxVerA = + argOrNull(options::OPT_mhexagon_hvx_EQ, options::OPT_mno_hexagon_hvx); + + if (Arg *A = HvxVerA ? HvxVerA : HvxBareA) { + if (A->getOption().matches(options::OPT_mhexagon_hvx_EQ)) + HvxVer = StringRef(A->getValue()).lower(); // lower produces std:string + HasHVX = true; + Features.push_back(makeFeature(Twine("hvx") + HvxVer, true)); + } else if (Arg *A = Args.getLastArg(options::OPT_mno_hexagon_hvx)) { + // If there was an explicit -mno-hvx, add -hvx to target features. + Features.push_back(makeFeature(A->getOption().getName(), false)); } + StringRef HvxLen = getDefaultHvxLength(HvxVer); + // Handle -mhvx-length=. if (Arg *A = Args.getLastArg(options::OPT_mhexagon_hvx_length_EQ)) { // These flags are valid only if HVX in enabled. if (!HasHVX) - D.Diag(diag::err_drv_invalid_hvx_length); + D.Diag(diag::err_drv_needs_hvx) << withMinus(A->getOption().getName()); else if (A->getOption().matches(options::OPT_mhexagon_hvx_length_EQ)) - HVXLength = A->getValue(); + HvxLen = A->getValue(); } - // Default hvx-length based on Cpu. - else if (HasHVX) - HVXLength = getDefaultHvxLength(Cpu); - if (!HVXLength.empty()) { - HVXFeature = - Args.MakeArgString(llvm::Twine("+hvx-length") + HVXLength.lower()); - Features.push_back(HVXFeature); + if (HasHVX) { + StringRef L = makeFeature(Twine("hvx-length") + HvxLen.lower(), true); + Features.push_back(L); + } + + unsigned HvxVerNum; + // getAsInteger returns 'true' on error. + if (StringRef(HvxVer).drop_front(1).getAsInteger(10, HvxVerNum)) + HvxVerNum = 0; + + // Handle HVX floating point flags. + auto checkFlagHvxVersion = [&](auto FlagOn, auto FlagOff, + unsigned MinVerNum) -> Optional { + // Return an Optional: + // - None indicates a verification failure, or that the flag was not + // present in Args. + // - Otherwise the returned value is that name of the feature to add + // to Features. + Arg *A = Args.getLastArg(FlagOn, FlagOff); + if (!A) + return None; + + StringRef OptName = A->getOption().getName(); + if (A->getOption().matches(FlagOff)) + return makeFeature(OptName, false); + + if (!HasHVX) { + D.Diag(diag::err_drv_needs_hvx) << withMinus(OptName); + return None; + } + if (HvxVerNum < MinVerNum) { + D.Diag(diag::err_drv_needs_hvx_version) + << withMinus(OptName) << ("v" + std::to_string(HvxVerNum)); + return None; + } + return makeFeature(OptName, true); + }; + + if (auto F = checkFlagHvxVersion(options::OPT_mhexagon_hvx_qfloat, + options::OPT_mno_hexagon_hvx_qfloat, 68)) { + Features.push_back(*F); + } + if (auto F = checkFlagHvxVersion(options::OPT_mhexagon_hvx_ieee_fp, + options::OPT_mno_hexagon_hvx_ieee_fp, 68)) { + Features.push_back(*F); } } @@ -117,7 +182,7 @@ void hexagon::getHexagonTargetFeatures(const Driver &D, const ArgList &Args, handleHVXTargetFeatures(D, Args, Features, Cpu, HasHVX); if (HexagonToolChain::isAutoHVXEnabled(Args) && !HasHVX) - D.Diag(diag::warn_drv_vectorize_needs_hvx); + D.Diag(diag::warn_drv_needs_hvx) << "auto-vectorization"; } // Hexagon tools start. @@ -156,6 +221,12 @@ void hexagon::Assembler::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back("-fsyntax-only"); } + if (Arg *A = Args.getLastArg(options::OPT_mhexagon_hvx_ieee_fp, + options::OPT_mno_hexagon_hvx_ieee_fp)) { + if (A->getOption().matches(options::OPT_mhexagon_hvx_ieee_fp)) + CmdArgs.push_back("-mhvx-ieee-fp"); + } + if (auto G = toolchains::HexagonToolChain::getSmallDataThreshold(Args)) { CmdArgs.push_back(Args.MakeArgString("-gpsize=" + Twine(G.getValue()))); } @@ -226,6 +297,7 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA, StringRef CpuVer = toolchains::HexagonToolChain::GetTargetCPUVersion(Args); bool NeedsSanitizerDeps = addSanitizerRuntimes(HTC, Args, CmdArgs); + bool NeedsXRayDeps = addXRayRuntime(HTC, Args, CmdArgs); //---------------------------------------------------------------------------- // Silence warnings for various options @@ -297,6 +369,8 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA, CmdArgs.push_back("-lunwind"); } + if (NeedsXRayDeps) + linkXRayRuntimeDeps(HTC, CmdArgs); CmdArgs.push_back("-lclang_rt.builtins-hexagon"); CmdArgs.push_back("-lc"); diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp index 198774506e5e..e413640abad3 100644 --- a/clang/lib/Driver/ToolChains/Linux.cpp +++ b/clang/lib/Driver/ToolChains/Linux.cpp @@ -277,14 +277,11 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args) // Android sysroots contain a library directory for each supported OS // version as well as some unversioned libraries in the usual multiarch // directory. - unsigned Major; - unsigned Minor; - unsigned Micro; - Triple.getEnvironmentVersion(Major, Minor, Micro); - addPathIfExists(D, - SysRoot + "/usr/lib/" + MultiarchTriple + "/" + - llvm::to_string(Major), - Paths); + addPathIfExists( + D, + SysRoot + "/usr/lib/" + MultiarchTriple + "/" + + llvm::to_string(Triple.getEnvironmentVersion().getMajor()), + Paths); } addPathIfExists(D, SysRoot + "/usr/lib/" + MultiarchTriple, Paths); @@ -666,8 +663,8 @@ void Linux::AddIAMCUIncludeArgs(const ArgList &DriverArgs, } bool Linux::isPIEDefault(const llvm::opt::ArgList &Args) const { - return getTriple().isAndroid() || getTriple().isMusl() || - getSanitizerArgs(Args).requiresPIE(); + return CLANG_DEFAULT_PIE_ON_LINUX || getTriple().isAndroid() || + getTriple().isMusl() || getSanitizerArgs(Args).requiresPIE(); } bool Linux::IsAArch64OutlineAtomicsDefault(const ArgList &Args) const { diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp index 792b0a51fea0..66e9d8ab525a 100644 --- a/clang/lib/Driver/ToolChains/MSVC.cpp +++ b/clang/lib/Driver/ToolChains/MSVC.cpp @@ -1194,14 +1194,6 @@ bool MSVCToolChain::getUniversalCRTLibraryPath(const ArgList &Args, return true; } -static VersionTuple getMSVCVersionFromTriple(const llvm::Triple &Triple) { - unsigned Major, Minor, Micro; - Triple.getEnvironmentVersion(Major, Minor, Micro); - if (Major || Minor || Micro) - return VersionTuple(Major, Minor, Micro); - return VersionTuple(); -} - static VersionTuple getMSVCVersionFromExe(const std::string &BinDir) { VersionTuple Version; #ifdef _WIN32 @@ -1374,7 +1366,7 @@ VersionTuple MSVCToolChain::computeMSVCVersion(const Driver *D, bool IsWindowsMSVC = getTriple().isWindowsMSVCEnvironment(); VersionTuple MSVT = ToolChain::computeMSVCVersion(D, Args); if (MSVT.empty()) - MSVT = getMSVCVersionFromTriple(getTriple()); + MSVT = getTriple().getEnvironmentVersion(); if (MSVT.empty() && IsWindowsMSVC) MSVT = getMSVCVersionFromExe(getSubDirectoryPath(SubDirectoryType::Bin)); if (MSVT.empty() && diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp index 7571398b7cc6..37b1fc5215ff 100644 --- a/clang/lib/Driver/ToolChains/NetBSD.cpp +++ b/clang/lib/Driver/ToolChains/NetBSD.cpp @@ -270,10 +270,9 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(Args.MakeArgString(ToolChain.getCompilerRTPath())); } - unsigned Major, Minor, Micro; - Triple.getOSVersion(Major, Minor, Micro); + VersionTuple OsVersion = Triple.getOSVersion(); bool useLibgcc = true; - if (Major >= 7 || Major == 0) { + if (OsVersion >= VersionTuple(7) || OsVersion.getMajor() == 0) { switch (ToolChain.getArch()) { case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: @@ -409,9 +408,8 @@ Tool *NetBSD::buildAssembler() const { Tool *NetBSD::buildLinker() const { return new tools::netbsd::Linker(*this); } ToolChain::CXXStdlibType NetBSD::GetDefaultCXXStdlibType() const { - unsigned Major, Minor, Micro; - getTriple().getOSVersion(Major, Minor, Micro); - if (Major >= 7 || Major == 0) { + VersionTuple OsVersion = getTriple().getOSVersion(); + if (OsVersion >= VersionTuple(7) || OsVersion.getMajor() == 0) { switch (getArch()) { case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: @@ -505,14 +503,13 @@ void NetBSD::addClangTargetOptions(const ArgList &DriverArgs, if (SanArgs.hasAnySanitizer()) CC1Args.push_back("-D_REENTRANT"); - unsigned Major, Minor, Micro; - getTriple().getOSVersion(Major, Minor, Micro); + VersionTuple OsVersion = getTriple().getOSVersion(); bool UseInitArrayDefault = - Major >= 9 || Major == 0 || - getTriple().getArch() == llvm::Triple::aarch64 || - getTriple().getArch() == llvm::Triple::aarch64_be || - getTriple().getArch() == llvm::Triple::arm || - getTriple().getArch() == llvm::Triple::armeb; + OsVersion >= VersionTuple(9) || OsVersion.getMajor() == 0 || + getTriple().getArch() == llvm::Triple::aarch64 || + getTriple().getArch() == llvm::Triple::aarch64_be || + getTriple().getArch() == llvm::Triple::arm || + getTriple().getArch() == llvm::Triple::armeb; if (!DriverArgs.hasFlag(options::OPT_fuse_init_array, options::OPT_fno_use_init_array, UseInitArrayDefault)) diff --git a/clang/lib/Driver/ToolChains/SPIRV.cpp b/clang/lib/Driver/ToolChains/SPIRV.cpp index 16e72d3c733f..50d03e79bbb0 100644 --- a/clang/lib/Driver/ToolChains/SPIRV.cpp +++ b/clang/lib/Driver/ToolChains/SPIRV.cpp @@ -13,6 +13,7 @@ #include "clang/Driver/Options.h" using namespace clang::driver; +using namespace clang::driver::toolchains; using namespace clang::driver::tools; using namespace llvm::opt; @@ -27,7 +28,7 @@ void SPIRV::constructTranslateCommand(Compilation &C, const Tool &T, if (Input.getType() == types::TY_PP_Asm) CmdArgs.push_back("-to-binary"); if (Output.getType() == types::TY_PP_Asm) - CmdArgs.push_back("-spirv-text"); + CmdArgs.push_back("--spirv-tools-dis"); CmdArgs.append({"-o", Output.getFilename()}); @@ -47,3 +48,25 @@ void SPIRV::Translator::ConstructJob(Compilation &C, const JobAction &JA, llvm_unreachable("Invalid number of input files."); constructTranslateCommand(C, *this, JA, Output, Inputs[0], {}); } + +clang::driver::Tool *SPIRVToolChain::getTranslator() const { + if (!Translator) + Translator = std::make_unique(*this); + return Translator.get(); +} + +clang::driver::Tool *SPIRVToolChain::SelectTool(const JobAction &JA) const { + Action::ActionClass AC = JA.getKind(); + return SPIRVToolChain::getTool(AC); +} + +clang::driver::Tool *SPIRVToolChain::getTool(Action::ActionClass AC) const { + switch (AC) { + default: + break; + case Action::BackendJobClass: + case Action::AssembleJobClass: + return SPIRVToolChain::getTranslator(); + } + return ToolChain::getTool(AC); +} diff --git a/clang/lib/Driver/ToolChains/SPIRV.h b/clang/lib/Driver/ToolChains/SPIRV.h index 35d0446bd8b8..229f7018e3b5 100644 --- a/clang/lib/Driver/ToolChains/SPIRV.h +++ b/clang/lib/Driver/ToolChains/SPIRV.h @@ -41,6 +41,39 @@ class LLVM_LIBRARY_VISIBILITY Translator : public Tool { } // namespace SPIRV } // namespace tools + +namespace toolchains { + +class LLVM_LIBRARY_VISIBILITY SPIRVToolChain final : public ToolChain { + mutable std::unique_ptr Translator; + +public: + SPIRVToolChain(const Driver &D, const llvm::Triple &Triple, + const llvm::opt::ArgList &Args) + : ToolChain(D, Triple, Args) {} + + bool useIntegratedAs() const override { return true; } + bool useIntegratedBackend() const override { return false; } + + bool IsMathErrnoDefault() const override { return false; } + bool isCrossCompiling() const override { return true; } + bool isPICDefault() const override { return false; } + bool isPIEDefault(const llvm::opt::ArgList &Args) const override { + return false; + } + bool isPICDefaultForced() const override { return false; } + bool SupportsProfiling() const override { return false; } + + clang::driver::Tool *SelectTool(const JobAction &JA) const override; + +protected: + clang::driver::Tool *getTool(Action::ActionClass AC) const override; + +private: + clang::driver::Tool *getTranslator() const; +}; + +} // namespace toolchains } // namespace driver } // namespace clang #endif diff --git a/clang/lib/Driver/ToolChains/VEToolchain.cpp b/clang/lib/Driver/ToolChains/VEToolchain.cpp index 1fcc52684baa..4cdeec7f9d8a 100644 --- a/clang/lib/Driver/ToolChains/VEToolchain.cpp +++ b/clang/lib/Driver/ToolChains/VEToolchain.cpp @@ -28,17 +28,27 @@ VEToolChain::VEToolChain(const Driver &D, const llvm::Triple &Triple, getProgramPaths().push_back("/opt/nec/ve/bin"); // ProgramPaths are found via 'PATH' environment variable. - // default file paths are: - // ${RESOURCEDIR}/lib/linux/ve (== getArchSpecificLibPath) - // /lib/../lib64 - // /usr/lib/../lib64 - // ${BINPATH}/../lib - // /lib - // /usr/lib - // - // These are OK for host, but no go for VE. So, defines them all - // from scratch here. + // Default library paths are following: + // ${RESOURCEDIR}/lib/ve-unknown-linux-gnu, + // These are OK. + + // Default file paths are following: + // ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPath) + // /lib/../lib64, + // /usr/lib/../lib64, + // ${BINPATH}/../lib, + // /lib, + // /usr/lib, + // These are OK for host, but no go for VE. + + // Define file paths from scratch here. getFilePaths().clear(); + + // Add library directories: + // ${BINPATH}/../lib/ve-unknown-linux-gnu, (== getStdlibPath) + // ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPath) + // ${SYSROOT}/opt/nec/ve/lib, + getFilePaths().push_back(getStdlibPath()); getFilePaths().push_back(getArchSpecificLibPath()); getFilePaths().push_back(computeSysRoot() + "/opt/nec/ve/lib"); } @@ -115,9 +125,10 @@ void VEToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs, ArrayRef DirVec(Dirs); addSystemIncludes(DriverArgs, CC1Args, DirVec); } else { - SmallString<128> P(getDriver().ResourceDir); - llvm::sys::path::append(P, "include/c++/v1"); - addSystemInclude(DriverArgs, CC1Args, P); + // Add following paths for multiple target installation. + // ${INSTALLDIR}/include/ve-unknown-linux-gnu/c++/v1, + // ${INSTALLDIR}/include/c++/v1, + addLibCxxIncludePaths(DriverArgs, CC1Args); } } diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp index b44509ad3b88..63b575178bd1 100644 --- a/clang/lib/Driver/XRayArgs.cpp +++ b/clang/lib/Driver/XRayArgs.cpp @@ -40,6 +40,7 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) { case llvm::Triple::x86_64: case llvm::Triple::arm: case llvm::Triple::aarch64: + case llvm::Triple::hexagon: case llvm::Triple::ppc64le: case llvm::Triple::mips: case llvm::Triple::mipsel: diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp index 968b35bfda23..5d03c9811e1b 100644 --- a/clang/lib/Format/BreakableToken.cpp +++ b/clang/lib/Format/BreakableToken.cpp @@ -91,7 +91,7 @@ getCommentSplit(StringRef Text, unsigned ContentStartColumn, // In JavaScript, some @tags can be followed by {, and machinery that parses // these comments will fail to understand the comment if followed by a line // break. So avoid ever breaking before a {. - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { StringRef::size_type SpaceOffset = Text.find_first_of(Blanks, MaxSplitBytes); if (SpaceOffset != StringRef::npos && SpaceOffset + 1 < Text.size() && @@ -127,8 +127,7 @@ getCommentSplit(StringRef Text, unsigned ContentStartColumn, } // Avoid ever breaking before a @tag or a { in JavaScript. - if (Style.Language == FormatStyle::LK_JavaScript && - SpaceOffset + 1 < Text.size() && + if (Style.isJavaScript() && SpaceOffset + 1 < Text.size() && (Text[SpaceOffset + 1] == '{' || Text[SpaceOffset + 1] == '@')) { SpaceOffset = Text.find_last_of(Blanks, SpaceOffset); continue; @@ -460,8 +459,7 @@ BreakableBlockComment::BreakableBlockComment( IndentAtLineBreak = std::max(IndentAtLineBreak, Decoration.size()); // Detect a multiline jsdoc comment and set DelimitersOnNewline in that case. - if (Style.Language == FormatStyle::LK_JavaScript || - Style.Language == FormatStyle::LK_Java) { + if (Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) { if ((Lines[0] == "*" || Lines[0].startswith("* ")) && Lines.size() > 1) { // This is a multiline jsdoc comment. DelimitersOnNewline = true; @@ -580,8 +578,7 @@ const llvm::StringSet<> }; unsigned BreakableBlockComment::getContentIndent(unsigned LineIndex) const { - if (Style.Language != FormatStyle::LK_Java && - Style.Language != FormatStyle::LK_JavaScript) + if (Style.Language != FormatStyle::LK_Java && !Style.isJavaScript()) return 0; // The content at LineIndex 0 of a comment like: // /** line 0 */ diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp index 5073f5105d05..4225d6b67b0e 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp @@ -422,8 +422,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { // ... // }.bind(...)); // FIXME: We should find a more generic solution to this problem. - !(State.Column <= NewLineColumn && - Style.Language == FormatStyle::LK_JavaScript) && + !(State.Column <= NewLineColumn && Style.isJavaScript()) && !(Previous.closesScopeAfterBlock() && State.Column <= NewLineColumn)) return true; @@ -493,14 +492,14 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { return true; } - // Break after the closing parenthesis of TypeScript decorators before - // functions, getters and setters. - static const llvm::StringSet<> BreakBeforeDecoratedTokens = {"get", "set", - "function"}; - if (Style.Language == FormatStyle::LK_JavaScript && - BreakBeforeDecoratedTokens.contains(Current.TokenText) && - Previous.is(tok::r_paren) && Previous.is(TT_JavaAnnotation)) { - return true; + if (Style.isJavaScript() && Previous.is(tok::r_paren) && + Previous.is(TT_JavaAnnotation)) { + // Break after the closing parenthesis of TypeScript decorators before + // functions, getters and setters. + static const llvm::StringSet<> BreakBeforeDecoratedTokens = {"get", "set", + "function"}; + if (BreakBeforeDecoratedTokens.contains(Current.TokenText)) + return true; } // If the return type spans multiple lines, wrap before the function name. @@ -510,7 +509,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { Style.AlwaysBreakAfterReturnType != FormatStyle::RTBS_None) && // Don't always break between a JavaScript `function` and the function // name. - Style.Language != FormatStyle::LK_JavaScript) || + !Style.isJavaScript()) || (Current.is(tok::kw_operator) && !Previous.is(tok::coloncolon))) && !Previous.is(tok::kw_template) && State.Stack.back().BreakBeforeParameter) return true; @@ -827,9 +826,8 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State, // is common and should be formatted like a free-standing function. The same // goes for wrapping before the lambda return type arrow. if (!Current.is(TT_LambdaArrow) && - (Style.Language != FormatStyle::LK_JavaScript || - Current.NestingLevel != 0 || !PreviousNonComment || - !PreviousNonComment->is(tok::equal) || + (!Style.isJavaScript() || Current.NestingLevel != 0 || + !PreviousNonComment || !PreviousNonComment->is(tok::equal) || !Current.isOneOf(Keywords.kw_async, Keywords.kw_function))) State.Stack.back().NestedBlockIndent = State.Column; @@ -1337,6 +1335,9 @@ unsigned ContinuationIndenter::moveStateToNextToken(LineState &State, void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, bool Newline) { const FormatToken &Current = *State.NextToken; + if (Current.FakeLParens.empty()) + return; + const FormatToken *Previous = Current.getPreviousNonComment(); // Don't add extra indentation for the first fake parenthesis after @@ -1348,10 +1349,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, (Previous->getPrecedence() == prec::Assignment && Style.AlignOperands != FormatStyle::OAS_DontAlign) || Previous->is(TT_ObjCMethodExpr))); - for (SmallVectorImpl::const_reverse_iterator - I = Current.FakeLParens.rbegin(), - E = Current.FakeLParens.rend(); - I != E; ++I) { + for (const auto &PrecedenceLevel : llvm::reverse(Current.FakeLParens)) { ParenState NewParenState = State.Stack.back(); NewParenState.Tok = nullptr; NewParenState.ContainsLineBreak = false; @@ -1363,7 +1361,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, NewParenState.NoLineBreak || State.Stack.back().NoLineBreakInOperand; // Don't propagate AvoidBinPacking into subexpressions of arg/param lists. - if (*I > prec::Comma) + if (PrecedenceLevel > prec::Comma) NewParenState.AvoidBinPacking = false; // Indent from 'LastSpace' unless these are fake parentheses encapsulating @@ -1371,11 +1369,11 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, // brackets is disabled. if (!Current.isTrailingComment() && (Style.AlignOperands != FormatStyle::OAS_DontAlign || - *I < prec::Assignment) && + PrecedenceLevel < prec::Assignment) && (!Previous || Previous->isNot(tok::kw_return) || - (Style.Language != FormatStyle::LK_Java && *I > 0)) && + (Style.Language != FormatStyle::LK_Java && PrecedenceLevel > 0)) && (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign || - *I != prec::Comma || Current.NestingLevel == 0)) { + PrecedenceLevel != prec::Comma || Current.NestingLevel == 0)) { NewParenState.Indent = std::max(std::max(State.Column, NewParenState.Indent), State.Stack.back().LastSpace); @@ -1384,7 +1382,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, if (Previous && (Previous->getPrecedence() == prec::Assignment || Previous->is(tok::kw_return) || - (*I == prec::Conditional && Previous->is(tok::question) && + (PrecedenceLevel == prec::Conditional && Previous->is(tok::question) && Previous->is(TT_ConditionalExpr))) && !Newline) { // If BreakBeforeBinaryOperators is set, un-indent a bit to account for @@ -1402,9 +1400,9 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, // ParameterToInnerFunction)); // OuterFunction(SomeObject.InnerFunctionCall( // break // ParameterToInnerFunction)); - if (*I > prec::Unknown) + if (PrecedenceLevel > prec::Unknown) NewParenState.LastSpace = std::max(NewParenState.LastSpace, State.Column); - if (*I != prec::Conditional && !Current.is(TT_UnaryOperator) && + if (PrecedenceLevel != prec::Conditional && !Current.is(TT_UnaryOperator) && Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign) NewParenState.StartOfFunctionCall = State.Column; @@ -1413,17 +1411,18 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, // an assignment (i.e. *I <= prec::Assignment) as those have different // indentation rules. Indent other expression, unless the indentation needs // to be skipped. - if (*I == prec::Conditional && Previous && Previous->is(tok::colon) && - Previous->is(TT_ConditionalExpr) && I == Current.FakeLParens.rbegin() && + if (PrecedenceLevel == prec::Conditional && Previous && + Previous->is(tok::colon) && Previous->is(TT_ConditionalExpr) && + &PrecedenceLevel == &Current.FakeLParens.back() && !State.Stack.back().IsWrappedConditional) { NewParenState.IsChainedConditional = true; NewParenState.UnindentOperator = State.Stack.back().UnindentOperator; - } else if (*I == prec::Conditional || - (!SkipFirstExtraIndent && *I > prec::Assignment && + } else if (PrecedenceLevel == prec::Conditional || + (!SkipFirstExtraIndent && PrecedenceLevel > prec::Assignment && !Current.isTrailingComment())) { NewParenState.Indent += Style.ContinuationIndentWidth; } - if ((Previous && !Previous->opensScope()) || *I != prec::Comma) + if ((Previous && !Previous->opensScope()) || PrecedenceLevel != prec::Comma) NewParenState.BreakBeforeParameter = false; State.Stack.push_back(NewParenState); SkipFirstExtraIndent = false; @@ -1518,7 +1517,7 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State, AvoidBinPacking = (State.Stack.back().IsCSharpGenericTypeConstraint) || - (Style.Language == FormatStyle::LK_JavaScript && EndsInComma) || + (Style.isJavaScript() && EndsInComma) || (State.Line->MustBeDeclaration && !BinPackDeclaration) || (!State.Line->MustBeDeclaration && !Style.BinPackArguments) || (Style.ExperimentalAutoDetectBinPacking && @@ -1547,7 +1546,7 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State, } } - if (Style.Language == FormatStyle::LK_JavaScript && EndsInComma) + if (Style.isJavaScript() && EndsInComma) BreakBeforeParameter = true; } // Generally inherit NoLineBreak from the current scope to nested scope. @@ -1924,9 +1923,9 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current, // FIXME: String literal breaking is currently disabled for C#, Java, Json // and JavaScript, as it requires strings to be merged using "+" which we // don't support. - if (Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp() || - Style.isJson() || !Style.BreakStringLiterals || !AllowBreak) + if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() || + Style.isCSharp() || Style.isJson() || !Style.BreakStringLiterals || + !AllowBreak) return nullptr; // Don't break string literals inside preprocessor directives (except for diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 17de1075aeaa..be01daa38929 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -2586,12 +2586,31 @@ tooling::Replacements sortCppIncludes(const FormatStyle &Style, StringRef Code, bool MainIncludeFound = false; bool FormattingOff = false; + llvm::Regex RawStringRegex( + "R\"(([\\[A-Za-z0-9_{}#<>%:;.?*+/^&\\$|~!=,'\\-]|])*)\\("); + SmallVector RawStringMatches; + std::string RawStringTermination = ")\""; + for (;;) { auto Pos = Code.find('\n', SearchFrom); StringRef Line = Code.substr(Prev, (Pos != StringRef::npos ? Pos : Code.size()) - Prev); StringRef Trimmed = Line.trim(); + + // #includes inside raw string literals need to be ignored. + // or we will sort the contents of the string. + // Skip past until we think we are at the rawstring literal close. + if (RawStringRegex.match(Trimmed, &RawStringMatches)) { + std::string CharSequence = RawStringMatches[1].str(); + RawStringTermination = ")" + CharSequence + "\""; + FormattingOff = true; + } + + if (Trimmed.contains(RawStringTermination)) { + FormattingOff = false; + } + if (Trimmed == "// clang-format off" || Trimmed == "/* clang-format off */") FormattingOff = true; else if (Trimmed == "// clang-format on" || @@ -3031,8 +3050,7 @@ reformat(const FormatStyle &Style, StringRef Code, }); } - if (Style.Language == FormatStyle::LK_JavaScript && - Style.JavaScriptQuotes != FormatStyle::JSQS_Leave) + if (Style.isJavaScript() && Style.JavaScriptQuotes != FormatStyle::JSQS_Leave) Passes.emplace_back([&](const Environment &Env) { return JavaScriptRequoter(Env, Expanded).process(); }); @@ -3041,7 +3059,7 @@ reformat(const FormatStyle &Style, StringRef Code, return Formatter(Env, Expanded, Status).process(); }); - if (Style.Language == FormatStyle::LK_JavaScript && + if (Style.isJavaScript() && Style.InsertTrailingCommas == FormatStyle::TCS_Wrapped) Passes.emplace_back([&](const Environment &Env) { return TrailingCommaInserter(Env, Expanded).process(); diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp index 6768f041135c..57f8a5a45cbb 100644 --- a/clang/lib/Format/FormatToken.cpp +++ b/clang/lib/Format/FormatToken.cpp @@ -70,6 +70,10 @@ bool FormatToken::isSimpleTypeSpecifier() const { } } +bool FormatToken::isTypeOrIdentifier() const { + return isSimpleTypeSpecifier() || Tok.isOneOf(tok::kw_auto, tok::identifier); +} + TokenRole::~TokenRole() {} void TokenRole::precomputeFormattingInfos(const FormatToken *Token) {} diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 1a2858018fde..d410ede32240 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -521,7 +521,9 @@ struct FormatToken { } /// Determine whether the token is a simple-type-specifier. - bool isSimpleTypeSpecifier() const; + LLVM_NODISCARD bool isSimpleTypeSpecifier() const; + + LLVM_NODISCARD bool isTypeOrIdentifier() const; bool isObjCAccessSpecifier() const { return is(tok::at) && Next && diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp index 64fbd2d5d45b..7736a7042f86 100644 --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -78,7 +78,7 @@ ArrayRef FormatTokenLexer::lex() { assert(FirstInLineIndex == 0); do { Tokens.push_back(getNextToken()); - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { tryParseJSRegexLiteral(); handleTemplateStrings(); } @@ -107,7 +107,7 @@ void FormatTokenLexer::tryMergePreviousTokens() { if (Style.isCpp() && tryTransformTryUsageForC()) return; - if (Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) { + if (Style.isJavaScript() || Style.isCSharp()) { static const tok::TokenKind NullishCoalescingOperator[] = {tok::question, tok::question}; static const tok::TokenKind NullPropagatingOperator[] = {tok::question, @@ -152,7 +152,7 @@ void FormatTokenLexer::tryMergePreviousTokens() { if (tryMergeNSStringLiteral()) return; - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { static const tok::TokenKind JSIdentity[] = {tok::equalequal, tok::equal}; static const tok::TokenKind JSNotIdentity[] = {tok::exclaimequal, tok::equal}; @@ -920,8 +920,7 @@ FormatToken *FormatTokenLexer::getNextToken() { // finds comments that contain a backslash followed by a line break, truncates // the comment token at the backslash, and resets the lexer to restart behind // the backslash. - if ((Style.Language == FormatStyle::LK_JavaScript || - Style.Language == FormatStyle::LK_Java) && + if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) && FormatTok->is(tok::comment) && FormatTok->TokenText.startswith("//")) { size_t BackslashPos = FormatTok->TokenText.find('\\'); while (BackslashPos != StringRef::npos) { @@ -982,7 +981,7 @@ FormatToken *FormatTokenLexer::getNextToken() { tok::kw_operator)) { FormatTok->Tok.setKind(tok::identifier); FormatTok->Tok.setIdentifierInfo(nullptr); - } else if (Style.Language == FormatStyle::LK_JavaScript && + } else if (Style.isJavaScript() && FormatTok->isOneOf(tok::kw_struct, tok::kw_union, tok::kw_operator)) { FormatTok->Tok.setKind(tok::identifier); @@ -1060,14 +1059,12 @@ void FormatTokenLexer::readRawToken(FormatToken &Tok) { if (!Tok.TokenText.empty() && Tok.TokenText[0] == '"') { Tok.Tok.setKind(tok::string_literal); Tok.IsUnterminatedLiteral = true; - } else if (Style.Language == FormatStyle::LK_JavaScript && - Tok.TokenText == "''") { + } else if (Style.isJavaScript() && Tok.TokenText == "''") { Tok.Tok.setKind(tok::string_literal); } } - if ((Style.Language == FormatStyle::LK_JavaScript || - Style.Language == FormatStyle::LK_Proto || + if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Proto || Style.Language == FormatStyle::LK_TextProto) && Tok.is(tok::char_constant)) { Tok.Tok.setKind(tok::string_literal); diff --git a/clang/lib/Format/NamespaceEndCommentsFixer.cpp b/clang/lib/Format/NamespaceEndCommentsFixer.cpp index def551f863cd..38ab5b9df76d 100644 --- a/clang/lib/Format/NamespaceEndCommentsFixer.cpp +++ b/clang/lib/Format/NamespaceEndCommentsFixer.cpp @@ -180,9 +180,13 @@ getNamespaceToken(const AnnotatedLine *Line, if (NamespaceTok->is(tok::l_brace)) { // "namespace" keyword can be on the line preceding '{', e.g. in styles // where BraceWrapping.AfterNamespace is true. - if (StartLineIndex > 0) + if (StartLineIndex > 0) { NamespaceTok = AnnotatedLines[StartLineIndex - 1]->First; + if (AnnotatedLines[StartLineIndex - 1]->endsWith(tok::semi)) + return nullptr; + } } + return NamespaceTok->getNamespaceToken(); } diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index a94d8cdc3b04..505a7250572b 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -242,7 +242,7 @@ class AnnotatingParser { bool OperatorCalledAsMemberFunction = Prev->Previous && Prev->Previous->isOneOf(tok::period, tok::arrow); Contexts.back().IsExpression = OperatorCalledAsMemberFunction; - } else if (Style.Language == FormatStyle::LK_JavaScript && + } else if (Style.isJavaScript() && (Line.startsWith(Keywords.kw_type, tok::identifier) || Line.startsWith(tok::kw_export, Keywords.kw_type, tok::identifier))) { @@ -256,13 +256,13 @@ class AnnotatingParser { Left->Previous->is(TT_BinaryOperator))) { // static_assert, if and while usually contain expressions. Contexts.back().IsExpression = true; - } else if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous && + } else if (Style.isJavaScript() && Left->Previous && (Left->Previous->is(Keywords.kw_function) || (Left->Previous->endsSequence(tok::identifier, Keywords.kw_function)))) { // function(...) or function f(...) Contexts.back().IsExpression = false; - } else if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous && + } else if (Style.isJavaScript() && Left->Previous && Left->Previous->is(TT_JsTypeColon)) { // let x: (SomeType); Contexts.back().IsExpression = false; @@ -582,7 +582,7 @@ class AnnotatingParser { Left->setType(TT_InlineASMSymbolicNameLSquare); } else if (IsCpp11AttributeSpecifier) { Left->setType(TT_AttributeSquare); - } else if (Style.Language == FormatStyle::LK_JavaScript && Parent && + } else if (Style.isJavaScript() && Parent && Contexts.back().ContextKind == tok::l_brace && Parent->isOneOf(tok::l_brace, tok::comma)) { Left->setType(TT_JsComputedPropertyName); @@ -646,8 +646,7 @@ class AnnotatingParser { ScopedContextCreator ContextCreator(*this, tok::l_square, BindingIncrease); Contexts.back().IsExpression = true; - if (Style.Language == FormatStyle::LK_JavaScript && Parent && - Parent->is(TT_JsTypeColon)) + if (Style.isJavaScript() && Parent && Parent->is(TT_JsTypeColon)) Contexts.back().IsExpression = false; Contexts.back().ColonIsObjCMethodExpr = StartsObjCMethodExpr; @@ -774,7 +773,7 @@ class AnnotatingParser { Contexts.back().ColonIsDictLiteral = true; if (Left->is(BK_BracedInit)) Contexts.back().IsExpression = true; - if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous && + if (Style.isJavaScript() && Left->Previous && Left->Previous->is(TT_JsTypeColon)) Contexts.back().IsExpression = false; @@ -808,12 +807,11 @@ class AnnotatingParser { Previous->is(tok::string_literal)) Previous->setType(TT_SelectorName); } - if (CurrentToken->is(tok::colon) || - Style.Language == FormatStyle::LK_JavaScript) + if (CurrentToken->is(tok::colon) || Style.isJavaScript()) Left->setType(TT_DictLiteral); } if (CurrentToken->is(tok::comma)) { - if (Style.Language == FormatStyle::LK_JavaScript) + if (Style.isJavaScript()) Left->setType(TT_DictLiteral); ++CommaCount; } @@ -879,7 +877,7 @@ class AnnotatingParser { if (!Tok->Previous) return false; // Colons from ?: are handled in parseConditional(). - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { if (Contexts.back().ColonIsForRangeExpr || // colon in for loop (Contexts.size() == 1 && // switch/case labels !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) || @@ -979,8 +977,7 @@ class AnnotatingParser { case tok::amp: // | and & in declarations/type expressions represent union and // intersection types, respectively. - if (Style.Language == FormatStyle::LK_JavaScript && - !Contexts.back().IsExpression) + if (Style.isJavaScript() && !Contexts.back().IsExpression) Tok->setType(TT_JsTypeOperator); break; case tok::kw_if: @@ -995,7 +992,7 @@ class AnnotatingParser { } break; case tok::kw_for: - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { // x.for and {for: ...} if ((Tok->Previous && Tok->Previous->is(tok::period)) || (Tok->Next && Tok->Next->is(tok::colon))) @@ -1106,7 +1103,7 @@ class AnnotatingParser { CurrentToken->Previous->setType(TT_OverloadedOperator); break; case tok::question: - if (Style.Language == FormatStyle::LK_JavaScript && Tok->Next && + if (Style.isJavaScript() && Tok->Next && Tok->Next->isOneOf(tok::semi, tok::comma, tok::colon, tok::r_paren, tok::r_brace)) { // Question marks before semicolons, colons, etc. indicate optional @@ -1119,7 +1116,7 @@ class AnnotatingParser { // Declarations cannot be conditional expressions, this can only be part // of a type declaration. if (Line.MustBeDeclaration && !Contexts.back().IsExpression && - Style.Language == FormatStyle::LK_JavaScript) + Style.isJavaScript()) break; if (Style.isCSharp()) { // `Type?)`, `Type?>`, `Type? name;` and `Type? name =` can only be @@ -1252,7 +1249,7 @@ class AnnotatingParser { if (!CurrentToken) return Type; - if (Style.Language == FormatStyle::LK_JavaScript && IsFirstToken) { + if (Style.isJavaScript() && IsFirstToken) { // JavaScript files can contain shebang lines of the form: // #!/usr/bin/env node // Treat these like C++ #include directives. @@ -1354,14 +1351,13 @@ class AnnotatingParser { bool ImportStatement = false; // import {...} from '...'; - if (Style.Language == FormatStyle::LK_JavaScript && - CurrentToken->is(Keywords.kw_import)) + if (Style.isJavaScript() && CurrentToken->is(Keywords.kw_import)) ImportStatement = true; while (CurrentToken) { if (CurrentToken->is(tok::kw_virtual)) KeywordVirtualFound = true; - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { // export {...} from '...'; // An export followed by "from 'some string';" is a re-export from // another module identified by a URI and is treated as a @@ -1504,7 +1500,7 @@ class AnnotatingParser { !Line.First->isOneOf(tok::kw_template, tok::kw_using, tok::kw_return) && // Type aliases use `type X = ...;` in TypeScript and can be exported // using `export type ...`. - !(Style.Language == FormatStyle::LK_JavaScript && + !(Style.isJavaScript() && (Line.startsWith(Keywords.kw_type, tok::identifier) || Line.startsWith(tok::kw_export, Keywords.kw_type, tok::identifier))) && @@ -1633,11 +1629,11 @@ class AnnotatingParser { // The token type is already known. return; - if ((Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) && + if ((Style.isJavaScript() || Style.isCSharp()) && Current.is(tok::exclaim)) { if (Current.Previous) { bool IsIdentifier = - Style.Language == FormatStyle::LK_JavaScript + Style.isJavaScript() ? Keywords.IsJavaScriptIdentifier( *Current.Previous, /* AcceptIdentifierName= */ true) : Current.Previous->is(tok::identifier); @@ -1679,7 +1675,7 @@ class AnnotatingParser { Current.setType(TT_LambdaArrow); } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration && Current.NestingLevel == 0 && - !Current.Previous->is(tok::kw_operator)) { + !Current.Previous->isOneOf(tok::kw_operator, tok::identifier)) { // not auto operator->() -> xxx; Current.setType(TT_TrailingReturnArrow); } else if (Current.is(tok::arrow) && Current.Previous && @@ -1704,8 +1700,8 @@ class AnnotatingParser { } else if (Current.isOneOf(tok::exclaim, tok::tilde)) { Current.setType(TT_UnaryOperator); } else if (Current.is(tok::question)) { - if (Style.Language == FormatStyle::LK_JavaScript && - Line.MustBeDeclaration && !Contexts.back().IsExpression) { + if (Style.isJavaScript() && Line.MustBeDeclaration && + !Contexts.back().IsExpression) { // In JavaScript, `interface X { foo?(): bar; }` is an optional method // on the interface, not a ternary expression. Current.setType(TT_JsTypeOptionalQuestion); @@ -1748,8 +1744,7 @@ class AnnotatingParser { Current.setType(TT_FunctionAnnotationRParen); } } - } else if (Current.is(tok::at) && Current.Next && - Style.Language != FormatStyle::LK_JavaScript && + } else if (Current.is(tok::at) && Current.Next && !Style.isJavaScript() && Style.Language != FormatStyle::LK_Java) { // In Java & JavaScript, "@..." is a decorator or annotation. In ObjC, it // marks declarations and properties that need special formatting. @@ -1796,7 +1791,7 @@ class AnnotatingParser { // function declaration have been found. Current.setType(TT_TrailingAnnotation); } else if ((Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript) && + Style.isJavaScript()) && Current.Previous) { if (Current.Previous->is(tok::at) && Current.isNot(Keywords.kw_interface)) { @@ -1826,14 +1821,16 @@ class AnnotatingParser { if (Tok.Previous->isOneOf(TT_LeadingJavaAnnotation, Keywords.kw_instanceof, Keywords.kw_as)) return false; - if (Style.Language == FormatStyle::LK_JavaScript && - Tok.Previous->is(Keywords.kw_in)) + if (Style.isJavaScript() && Tok.Previous->is(Keywords.kw_in)) return false; // Skip "const" as it does not have an influence on whether this is a name. FormatToken *PreviousNotConst = Tok.getPreviousNonComment(); - while (PreviousNotConst && PreviousNotConst->is(tok::kw_const)) - PreviousNotConst = PreviousNotConst->getPreviousNonComment(); + + // For javascript const can be like "let" or "var" + if (!Style.isJavaScript()) + while (PreviousNotConst && PreviousNotConst->is(tok::kw_const)) + PreviousNotConst = PreviousNotConst->getPreviousNonComment(); if (!PreviousNotConst) return false; @@ -1852,10 +1849,24 @@ class AnnotatingParser { PreviousNotConst->is(TT_TypeDeclarationParen)) return true; - return (!IsPPKeyword && - PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto)) || - PreviousNotConst->is(TT_PointerOrReference) || - PreviousNotConst->isSimpleTypeSpecifier(); + // If is a preprocess keyword like #define. + if (IsPPKeyword) + return false; + + // int a or auto a. + if (PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto)) + return true; + + // *a or &a or &&a. + if (PreviousNotConst->is(TT_PointerOrReference)) + return true; + + // MyClass a; + if (PreviousNotConst->isSimpleTypeSpecifier()) + return true; + + // const a = in JavaScript. + return (Style.isJavaScript() && PreviousNotConst->is(tok::kw_const)); } /// Determine whether ')' is ending a cast. @@ -2006,7 +2017,7 @@ class AnnotatingParser { /// Return the type of the given token assuming it is * or &. TokenType determineStarAmpUsage(const FormatToken &Tok, bool IsExpression, bool InTemplateArgument) { - if (Style.Language == FormatStyle::LK_JavaScript) + if (Style.isJavaScript()) return TT_BinaryOperator; // && in C# must be a binary operator. @@ -2034,9 +2045,8 @@ class AnnotatingParser { tok::comma, tok::semi, tok::kw_return, tok::colon, tok::kw_co_return, tok::kw_co_await, tok::kw_co_yield, tok::equal, tok::kw_delete, - tok::kw_sizeof, tok::kw_throw) || - PrevToken->isOneOf(TT_BinaryOperator, TT_ConditionalExpr, - TT_UnaryOperator, TT_CastRParen)) + tok::kw_sizeof, tok::kw_throw, TT_BinaryOperator, + TT_ConditionalExpr, TT_UnaryOperator, TT_CastRParen)) return TT_UnaryOperator; if (NextToken->is(tok::l_square) && NextToken->isNot(TT_LambdaLSquare)) @@ -2174,8 +2184,8 @@ class ExpressionParser { int CurrentPrecedence = getCurrentPrecedence(); - if (Current && Current->is(TT_SelectorName) && - Precedence == CurrentPrecedence) { + if (Precedence == CurrentPrecedence && Current && + Current->is(TT_SelectorName)) { if (LatestOperator) addFakeParenthesis(Start, prec::Level(Precedence)); Start = Current; @@ -2251,19 +2261,17 @@ class ExpressionParser { return 0; if (Current->is(TT_RangeBasedForLoopColon)) return prec::Comma; - if ((Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript) && + if ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) && Current->is(Keywords.kw_instanceof)) return prec::Relational; - if (Style.Language == FormatStyle::LK_JavaScript && + if (Style.isJavaScript() && Current->isOneOf(Keywords.kw_in, Keywords.kw_as)) return prec::Relational; if (Current->is(TT_BinaryOperator) || Current->is(tok::comma)) return Current->getPrecedence(); if (Current->isOneOf(tok::period, tok::arrow)) return PrecedenceArrowAndPeriod; - if ((Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript) && + if ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) && Current->isOneOf(Keywords.kw_extends, Keywords.kw_implements, Keywords.kw_throws)) return 0; @@ -2374,11 +2382,9 @@ static unsigned maxNestingDepth(const AnnotatedLine &Line) { } void TokenAnnotator::annotate(AnnotatedLine &Line) { - for (SmallVectorImpl::iterator I = Line.Children.begin(), - E = Line.Children.end(); - I != E; ++I) { - annotate(**I); - } + for (auto &Child : Line.Children) + annotate(*Child); + AnnotatingParser Parser(Style, Line, Keywords); Line.Type = Parser.parseLine(); @@ -2734,7 +2740,7 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line, return 2; if (Left.is(tok::comma) && Left.NestingLevel == 0) return 3; - } else if (Style.Language == FormatStyle::LK_JavaScript) { + } else if (Style.isJavaScript()) { if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma)) return 100; if (Left.is(TT_JsTypeColon)) @@ -3024,8 +3030,14 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both) && (Left.is(TT_AttributeParen) || Left.canBePointerOrReferenceQualifier())) return true; + if (Left.Tok.isLiteral()) + return true; + // for (auto a = 0, b = 0; const auto & c : {1, 2, 3}) + if (Left.isTypeOrIdentifier() && Right.Next && Right.Next->Next && + Right.Next->Next->is(TT_RangeBasedForLoopColon)) + return getTokenPointerOrReferenceAlignment(Right) != + FormatStyle::PAS_Left; return ( - Left.Tok.isLiteral() || (!Left.isOneOf(TT_PointerOrReference, tok::l_paren) && (getTokenPointerOrReferenceAlignment(Right) != FormatStyle::PAS_Left || (Line.IsMultiVariableDeclStmt && @@ -3044,18 +3056,32 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both) && Right.canBePointerOrReferenceQualifier()) return true; - return Right.Tok.isLiteral() || Right.is(TT_BlockComment) || - (Right.isOneOf(Keywords.kw_override, Keywords.kw_final) && - !Right.is(TT_StartOfName)) || - (Right.is(tok::l_brace) && Right.is(BK_Block)) || - (!Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare, - tok::l_paren) && - (getTokenPointerOrReferenceAlignment(Left) != - FormatStyle::PAS_Right && - !Line.IsMultiVariableDeclStmt) && - Left.Previous && - !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon, - tok::l_square)); + // & 1 + if (Right.Tok.isLiteral()) + return true; + // & /* comment + if (Right.is(TT_BlockComment)) + return true; + // foo() -> const Bar * override/final + if (Right.isOneOf(Keywords.kw_override, Keywords.kw_final) && + !Right.is(TT_StartOfName)) + return true; + // & { + if (Right.is(tok::l_brace) && Right.is(BK_Block)) + return true; + // for (auto a = 0, b = 0; const auto& c : {1, 2, 3}) + if (Left.Previous && Left.Previous->isTypeOrIdentifier() && Right.Next && + Right.Next->is(TT_RangeBasedForLoopColon)) + return getTokenPointerOrReferenceAlignment(Left) != + FormatStyle::PAS_Right; + return !Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare, + tok::l_paren) && + (getTokenPointerOrReferenceAlignment(Left) != + FormatStyle::PAS_Right && + !Line.IsMultiVariableDeclStmt) && + Left.Previous && + !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon, + tok::l_square); } // Ensure right pointer alignment with ellipsis e.g. int *...P if (Left.is(tok::ellipsis) && Left.Previous && @@ -3146,8 +3172,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return Style.Cpp11BracedListStyle ? Style.SpacesInParentheses : true; if (Left.is(TT_BlockComment)) // No whitespace in x(/*foo=*/1), except for JavaScript. - return Style.Language == FormatStyle::LK_JavaScript || - !Left.TokenText.endswith("=*/"); + return Style.isJavaScript() || !Left.TokenText.endswith("=*/"); // Space between template and attribute. // e.g. template [[nodiscard]] ... @@ -3225,7 +3250,13 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return false; if (Left.is(tok::period) || Right.is(tok::period)) return false; - if (Right.is(tok::hash) && Left.is(tok::identifier) && Left.TokenText == "L") + // u#str, U#str, L#str, u8#str + // uR#str, UR#str, LR#str, u8R#str + if (Right.is(tok::hash) && Left.is(tok::identifier) && + (Left.TokenText == "L" || Left.TokenText == "u" || + Left.TokenText == "U" || Left.TokenText == "u8" || + Left.TokenText == "LR" || Left.TokenText == "uR" || + Left.TokenText == "UR" || Left.TokenText == "u8R")) return false; if (Left.is(TT_TemplateCloser) && Left.MatchingParen && Left.MatchingParen->Previous && @@ -3396,7 +3427,7 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, Keywords.kw_async, Keywords.kw_unsafe) && Right.is(tok::l_paren)) return true; - } else if (Style.Language == FormatStyle::LK_JavaScript) { + } else if (Style.isJavaScript()) { if (Left.is(TT_FatArrow)) return true; // for await ( ... @@ -3694,11 +3725,18 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, return true; if (Style.isCSharp()) { + if (Left.is(TT_FatArrow) && Right.is(tok::l_brace) && + Style.BraceWrapping.AfterFunction) + return true; if (Right.is(TT_CSharpNamedArgumentColon) || Left.is(TT_CSharpNamedArgumentColon)) return false; if (Right.is(TT_CSharpGenericTypeConstraint)) return true; + if (Right.Next && Right.Next->is(TT_FatArrow) && + (Right.is(tok::numeric_constant) || + (Right.is(tok::identifier) && Right.TokenText == "_"))) + return true; // Break after C# [...] and before public/protected/private/internal. if (Left.is(TT_AttributeSquare) && Left.is(tok::r_square) && @@ -3710,7 +3748,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, Left.is(tok::r_square) && Right.is(tok::l_square)) return true; - } else if (Style.Language == FormatStyle::LK_JavaScript) { + } else if (Style.isJavaScript()) { // FIXME: This might apply to other languages and token kinds. if (Right.is(tok::string_literal) && Left.is(tok::plus) && Left.Previous && Left.Previous->is(tok::string_literal)) @@ -3800,15 +3838,13 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, if (Style.JavaScriptWrapImports || Line.Type != LT_ImportStatement) { const FormatToken *BeforeClosingBrace = nullptr; if ((Left.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) || - (Style.Language == FormatStyle::LK_JavaScript && - Left.is(tok::l_paren))) && + (Style.isJavaScript() && Left.is(tok::l_paren))) && Left.isNot(BK_Block) && Left.MatchingParen) BeforeClosingBrace = Left.MatchingParen->Previous; else if (Right.MatchingParen && (Right.MatchingParen->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) || - (Style.Language == FormatStyle::LK_JavaScript && - Right.MatchingParen->is(tok::l_paren)))) + (Style.isJavaScript() && Right.MatchingParen->is(tok::l_paren)))) BeforeClosingBrace = &Left; if (BeforeClosingBrace && (BeforeClosingBrace->is(tok::comma) || BeforeClosingBrace->isTrailingComment())) @@ -3927,8 +3963,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, } // Put multiple Java annotation on a new line. - if ((Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript) && + if ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) && Left.is(TT_LeadingJavaAnnotation) && Right.isNot(TT_LeadingJavaAnnotation) && Right.isNot(tok::l_paren) && (Line.Last->is(tok::l_brace) || Style.BreakAfterJavaFieldAnnotations)) @@ -4071,7 +4106,7 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, if (Right.isOneOf(Keywords.kw_throws, Keywords.kw_extends, Keywords.kw_implements)) return true; - } else if (Style.Language == FormatStyle::LK_JavaScript) { + } else if (Style.isJavaScript()) { const FormatToken *NonComment = Right.getPreviousNonComment(); if (NonComment && NonComment->isOneOf( diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp index d099cfee9dea..f652a4e7088f 100644 --- a/clang/lib/Format/UnwrappedLineFormatter.cpp +++ b/clang/lib/Format/UnwrappedLineFormatter.cpp @@ -62,7 +62,7 @@ class LevelIndentTracker { Indent = Line.Level * IndentWidth + AdditionalIndent; } else { IndentForLevel.resize(Line.Level + 1); - Indent = getIndent(IndentForLevel, Line.Level); + Indent = getIndent(Line.Level); } if (static_cast(Indent) + Offset >= 0) Indent += Offset; @@ -97,8 +97,8 @@ class LevelIndentTracker { /// For example, 'public:' labels in classes are offset by 1 or 2 /// characters to the left from their level. int getIndentOffset(const FormatToken &RootToken) { - if (Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) + if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() || + Style.isCSharp()) return 0; if (RootToken.isAccessSpecifier(false) || RootToken.isObjCAccessSpecifier() || @@ -118,12 +118,12 @@ class LevelIndentTracker { /// \p IndentForLevel must contain the indent for the level \c l /// at \p IndentForLevel[l], or a value < 0 if the indent for /// that level is unknown. - unsigned getIndent(ArrayRef IndentForLevel, unsigned Level) { + unsigned getIndent(unsigned Level) const { if (IndentForLevel[Level] != -1) return IndentForLevel[Level]; if (Level == 0) return 0; - return getIndent(IndentForLevel, Level - 1) + Style.IndentWidth; + return getIndent(Level - 1) + Style.IndentWidth; } const FormatStyle &Style; @@ -393,11 +393,24 @@ class LineJoiner { // Try to merge a block with left brace wrapped that wasn't yet covered if (TheLine->Last->is(tok::l_brace)) { - return !Style.BraceWrapping.AfterFunction || - (I[1]->First->is(tok::r_brace) && - !Style.BraceWrapping.SplitEmptyRecord) - ? tryMergeSimpleBlock(I, E, Limit) - : 0; + const FormatToken *Tok = TheLine->First; + bool ShouldMerge = false; + if (Tok->is(tok::kw_typedef)) { + Tok = Tok->getNextNonComment(); + assert(Tok); + } + if (Tok->isOneOf(tok::kw_class, tok::kw_struct)) { + ShouldMerge = !Style.BraceWrapping.AfterClass || + (I[1]->First->is(tok::r_brace) && + !Style.BraceWrapping.SplitEmptyRecord); + } else if (Tok->is(tok::kw_enum)) { + ShouldMerge = Style.AllowShortEnumsOnASingleLine; + } else { + ShouldMerge = !Style.BraceWrapping.AfterFunction || + (I[1]->First->is(tok::r_brace) && + !Style.BraceWrapping.SplitEmptyFunction); + } + return ShouldMerge ? tryMergeSimpleBlock(I, E, Limit) : 0; } // Try to merge a function block with left brace wrapped if (I[1]->First->is(TT_FunctionLBrace) && @@ -584,6 +597,9 @@ class LineJoiner { Keywords.kw___except)) { if (Style.AllowShortBlocksOnASingleLine == FormatStyle::SBS_Never) return 0; + if (Style.AllowShortBlocksOnASingleLine == FormatStyle::SBS_Empty && + !I[1]->First->is(tok::r_brace)) + return 0; // Don't merge when we can't except the case when // the control statement block is empty if (!Style.AllowShortIfStatementsOnASingleLine && @@ -1015,9 +1031,9 @@ class OptimizingLineFormatter : public LineFormatter { QueueType Queue; // Insert start element into queue. - StateNode *Node = + StateNode *RootNode = new (Allocator.Allocate()) StateNode(InitialState, false, nullptr); - Queue.push(QueueItem(OrderedPenalty(0, Count), Node)); + Queue.push(QueueItem(OrderedPenalty(0, Count), RootNode)); ++Count; unsigned Penalty = 0; @@ -1044,9 +1060,9 @@ class OptimizingLineFormatter : public LineFormatter { FormatDecision LastFormat = Node->State.NextToken->getDecision(); if (LastFormat == FD_Unformatted || LastFormat == FD_Continue) - addNextStateToQueue(Penalty, Node, /*NewLine=*/false, &Count, &Queue); + addNextStateToQueue(Penalty, Node, /*NewLine=*/false, Count, Queue); if (LastFormat == FD_Unformatted || LastFormat == FD_Break) - addNextStateToQueue(Penalty, Node, /*NewLine=*/true, &Count, &Queue); + addNextStateToQueue(Penalty, Node, /*NewLine=*/true, Count, Queue); } if (Queue.empty()) { @@ -1072,7 +1088,7 @@ class OptimizingLineFormatter : public LineFormatter { /// Assume the current state is \p PreviousNode and has been reached with a /// penalty of \p Penalty. Insert a line break if \p NewLine is \c true. void addNextStateToQueue(unsigned Penalty, StateNode *PreviousNode, - bool NewLine, unsigned *Count, QueueType *Queue) { + bool NewLine, unsigned &Count, QueueType &Queue) { if (NewLine && !Indenter->canBreak(PreviousNode->State)) return; if (!NewLine && Indenter->mustBreak(PreviousNode->State)) @@ -1085,8 +1101,8 @@ class OptimizingLineFormatter : public LineFormatter { Penalty += Indenter->addTokenToState(Node->State, NewLine, true); - Queue->push(QueueItem(OrderedPenalty(Penalty, *Count), Node)); - ++(*Count); + Queue.push(QueueItem(OrderedPenalty(Penalty, Count), Node)); + ++Count; } /// Applies the best formatting by reconstructing the path in the @@ -1184,8 +1200,7 @@ unsigned UnwrappedLineFormatter::format( bool FitsIntoOneLine = TheLine.Last->TotalLength + Indent <= ColumnLimit || (TheLine.Type == LT_ImportStatement && - (Style.Language != FormatStyle::LK_JavaScript || - !Style.JavaScriptWrapImports)) || + (!Style.isJavaScript() || !Style.JavaScriptWrapImports)) || (Style.isCSharp() && TheLine.InPPDirective); // don't split #regions in C# if (Style.ColumnLimit == 0) diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index 5b9fe267aae6..b6e55aab708f 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -366,8 +366,7 @@ void UnwrappedLineParser::parse() { void UnwrappedLineParser::parseFile() { // The top-level context in a file always has declarations, except for pre- // processor directives and JavaScript files. - bool MustBeDeclaration = - !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript; + bool MustBeDeclaration = !Line->InPPDirective && !Style.isJavaScript(); ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack, MustBeDeclaration); if (Style.Language == FormatStyle::LK_TextProto) @@ -478,8 +477,7 @@ void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) { LLVM_FALLTHROUGH; } case tok::kw_case: - if (Style.Language == FormatStyle::LK_JavaScript && - Line->MustBeDeclaration) { + if (Style.isJavaScript() && Line->MustBeDeclaration) { // A 'case: string' style field declaration. parseStructuralElement(); break; @@ -528,7 +526,7 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) { switch (Tok->Tok.getKind()) { case tok::l_brace: - if (Style.Language == FormatStyle::LK_JavaScript && PrevTok) { + if (Style.isJavaScript() && PrevTok) { if (PrevTok->isOneOf(tok::colon, tok::less)) // A ':' indicates this code is in a type, or a braced list // following a label in an object literal ({a: {b: 1}}). @@ -581,7 +579,7 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) { // FIXME: Some of these do not apply to JS, e.g. "} {" can never be a // braced list in JS. ProbablyBracedList = - (Style.Language == FormatStyle::LK_JavaScript && + (Style.isJavaScript() && NextTok->isOneOf(Keywords.kw_of, Keywords.kw_in, Keywords.kw_as)) || (Style.isCpp() && NextTok->is(tok::l_paren)) || @@ -791,7 +789,7 @@ void UnwrappedLineParser::parseChildBlock() { FormatTok->setBlockKind(BK_Block); nextToken(); { - bool SkipIndent = (Style.Language == FormatStyle::LK_JavaScript && + bool SkipIndent = (Style.isJavaScript() && (isGoogScope(*Line) || isIIFE(*Line, Keywords))); ScopedLineState LineState(*this); ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack, @@ -1140,7 +1138,6 @@ void UnwrappedLineParser::parseModuleImport() { } addUnwrappedLine(); - return; } // readTokenWithJavaScriptASI reads the next token and terminates the current @@ -1222,39 +1219,39 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { case tok::kw_public: case tok::kw_protected: case tok::kw_private: - if (Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) + if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() || + Style.isCSharp()) nextToken(); else parseAccessSpecifier(); return; case tok::kw_if: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // field/method declaration. break; parseIfThenElse(); return; case tok::kw_for: case tok::kw_while: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // field/method declaration. break; parseForOrWhileLoop(); return; case tok::kw_do: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // field/method declaration. break; parseDoWhile(); return; case tok::kw_switch: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // 'switch: string' field declaration. break; parseSwitch(); return; case tok::kw_default: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // 'default: string' field declaration. break; nextToken(); @@ -1265,14 +1262,14 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { // e.g. "default void f() {}" in a Java interface. break; case tok::kw_case: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // 'case: string' field declaration. break; parseCaseLabel(); return; case tok::kw_try: case tok::kw___try: - if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration) + if (Style.isJavaScript() && Line->MustBeDeclaration) // field/method declaration. break; parseTryCatch(); @@ -1282,24 +1279,25 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { if (FormatTok->Tok.is(tok::string_literal)) { nextToken(); if (FormatTok->Tok.is(tok::l_brace)) { - if (!Style.IndentExternBlock) { - if (Style.BraceWrapping.AfterExternBlock) { - addUnwrappedLine(); - } - unsigned AddLevels = Style.BraceWrapping.AfterExternBlock ? 1u : 0u; - parseBlock(/*MustBeDeclaration=*/true, AddLevels); - } else { - unsigned AddLevels = - Style.IndentExternBlock == FormatStyle::IEBS_Indent ? 1u : 0u; - parseBlock(/*MustBeDeclaration=*/true, AddLevels); - } + if (Style.BraceWrapping.AfterExternBlock) + addUnwrappedLine(); + // Either we indent or for backwards compatibility we follow the + // AfterExternBlock style. + unsigned AddLevels = + (Style.IndentExternBlock == FormatStyle::IEBS_Indent) || + (Style.BraceWrapping.AfterExternBlock && + Style.IndentExternBlock == + FormatStyle::IEBS_AfterExternBlock) + ? 1u + : 0u; + parseBlock(/*MustBeDeclaration=*/true, AddLevels); addUnwrappedLine(); return; } } break; case tok::kw_export: - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { parseJavaScriptEs6ImportExport(); return; } @@ -1325,7 +1323,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { return; } if (FormatTok->is(Keywords.kw_import)) { - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { parseJavaScriptEs6ImportExport(); return; } @@ -1437,10 +1435,10 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { break; case tok::kw_concept: parseConcept(); - break; + return; case tok::kw_requires: parseRequires(); - break; + return; case tok::kw_enum: // Ignore if this is part of "template is(tok::less)) { @@ -1479,7 +1477,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { if (Style.Language == FormatStyle::LK_Java && FormatTok && FormatTok->is(tok::kw_class)) nextToken(); - if (Style.Language == FormatStyle::LK_JavaScript && FormatTok && + if (Style.isJavaScript() && FormatTok && FormatTok->Tok.getIdentifierInfo()) // JavaScript only has pseudo keywords, all keywords are allowed to // appear in "IdentifierName" positions. See http://es5.github.io/#x7.6 @@ -1536,8 +1534,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { // element continues. break; case tok::kw_try: - if (Style.Language == FormatStyle::LK_JavaScript && - Line->MustBeDeclaration) { + if (Style.isJavaScript() && Line->MustBeDeclaration) { // field/method declaration. nextToken(); break; @@ -1564,17 +1561,15 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { // expressions (functions that are not on their own line) must not create // a new unwrapped line, so they are special cased below. size_t TokenCount = Line->Tokens.size(); - if (Style.Language == FormatStyle::LK_JavaScript && - FormatTok->is(Keywords.kw_function) && + if (Style.isJavaScript() && FormatTok->is(Keywords.kw_function) && (TokenCount > 1 || (TokenCount == 1 && !Line->Tokens.front().Tok->is( Keywords.kw_async)))) { tryToParseJSFunction(); break; } - if ((Style.Language == FormatStyle::LK_JavaScript || - Style.Language == FormatStyle::LK_Java) && + if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) && FormatTok->is(Keywords.kw_interface)) { - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { // In JavaScript/TypeScript, "interface" can be used as a standalone // identifier, e.g. in `var interface = 1;`. If "interface" is // followed by another identifier, it is very like to be an actual @@ -1610,7 +1605,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { // JS doesn't have macros, and within classes colons indicate fields, not // labels. - if (Style.Language == FormatStyle::LK_JavaScript) + if (Style.isJavaScript()) break; TokenCount = Line->Tokens.size(); @@ -1641,19 +1636,9 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) { break; } case tok::equal: - // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType - // TT_FatArrow. They always start an expression or a child block if - // followed by a curly brace. - if (FormatTok->is(TT_FatArrow)) { - nextToken(); - if (FormatTok->is(tok::l_brace)) { - // C# may break after => if the next character is a newline. - if (Style.isCSharp() && Style.BraceWrapping.AfterFunction == true) { - // calling `addUnwrappedLine()` here causes odd parsing errors. - FormatTok->MustBreakBefore = true; - } - parseChildBlock(); - } + if ((Style.isJavaScript() || Style.isCSharp()) && + FormatTok->is(TT_FatArrow)) { + tryToParseChildBlock(); break; } @@ -1729,7 +1714,7 @@ bool UnwrappedLineParser::tryToParsePropertyAccessor() { // Try to parse the property accessor: // https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/properties Tokens->setPosition(StoredPosition); - if (!IsTrivialPropertyAccessor && Style.BraceWrapping.AfterFunction == true) + if (!IsTrivialPropertyAccessor && Style.BraceWrapping.AfterFunction) addUnwrappedLine(); nextToken(); do { @@ -1944,6 +1929,19 @@ bool UnwrappedLineParser::tryToParseBracedList() { return true; } +bool UnwrappedLineParser::tryToParseChildBlock() { + assert(Style.isJavaScript() || Style.isCSharp()); + assert(FormatTok->is(TT_FatArrow)); + // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType TT_FatArrow. + // They always start an expression or a child block if followed by a curly + // brace. + nextToken(); + if (FormatTok->isNot(tok::l_brace)) + return false; + parseChildBlock(); + return true; +} + bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, bool IsEnum, tok::TokenKind ClosingBraceKind) { @@ -1952,38 +1950,15 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, // FIXME: Once we have an expression parser in the UnwrappedLineParser, // replace this by using parseAssignmentExpression() inside. do { - if (Style.isCSharp()) { - // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType - // TT_FatArrow. They always start an expression or a child block if - // followed by a curly brace. - if (FormatTok->is(TT_FatArrow)) { - nextToken(); - if (FormatTok->is(tok::l_brace)) { - // C# may break after => if the next character is a newline. - if (Style.isCSharp() && Style.BraceWrapping.AfterFunction == true) { - // calling `addUnwrappedLine()` here causes odd parsing errors. - FormatTok->MustBreakBefore = true; - } - parseChildBlock(); - continue; - } - } - } - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isCSharp() && FormatTok->is(TT_FatArrow) && + tryToParseChildBlock()) + continue; + if (Style.isJavaScript()) { if (FormatTok->is(Keywords.kw_function) || FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) { tryToParseJSFunction(); continue; } - if (FormatTok->is(TT_FatArrow)) { - nextToken(); - // Fat arrows can be followed by simple expressions or by child blocks - // in curly braces. - if (FormatTok->is(tok::l_brace)) { - parseChildBlock(); - continue; - } - } if (FormatTok->is(tok::l_brace)) { // Could be a method inside of a braced list `{a() { return 1; }}`. if (tryToParseBracedList()) @@ -1998,12 +1973,6 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, return !HasError; } switch (FormatTok->Tok.getKind()) { - case tok::caret: - nextToken(); - if (FormatTok->is(tok::l_brace)) { - parseChildBlock(); - } - break; case tok::l_square: if (Style.isCSharp()) parseSquare(); @@ -2014,7 +1983,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, parseParens(); // JavaScript can just have free standing methods and getters/setters in // object literals. Detect them by a "{" following ")". - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { if (FormatTok->is(tok::l_brace)) parseChildBlock(); break; @@ -2041,7 +2010,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons, // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be // used for error recovery if we have otherwise determined that this is // a braced list. - if (Style.Language == FormatStyle::LK_JavaScript) { + if (Style.isJavaScript()) { nextToken(); break; } @@ -2095,18 +2064,18 @@ void UnwrappedLineParser::parseParens() { break; case tok::equal: if (Style.isCSharp() && FormatTok->is(TT_FatArrow)) - parseStructuralElement(); + tryToParseChildBlock(); else nextToken(); break; case tok::kw_class: - if (Style.Language == FormatStyle::LK_JavaScript) + if (Style.isJavaScript()) parseRecord(/*ParseAsExpr=*/true); else nextToken(); break; case tok::identifier: - if (Style.Language == FormatStyle::LK_JavaScript && + if (Style.isJavaScript() && (FormatTok->is(Keywords.kw_function) || FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function))) tryToParseJSFunction(); @@ -2160,15 +2129,22 @@ void UnwrappedLineParser::parseSquare(bool LambdaIntroducer) { } void UnwrappedLineParser::parseIfThenElse() { + auto HandleAttributes = [this]() { + // Handle AttributeMacro, e.g. `if (x) UNLIKELY`. + if (FormatTok->is(TT_AttributeMacro)) + nextToken(); + // Handle [[likely]] / [[unlikely]] attributes. + if (FormatTok->is(tok::l_square) && tryToParseSimpleAttribute()) + parseSquare(); + }; + assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected"); nextToken(); if (FormatTok->Tok.isOneOf(tok::kw_constexpr, tok::identifier)) nextToken(); if (FormatTok->Tok.is(tok::l_paren)) parseParens(); - // handle [[likely]] / [[unlikely]] - if (FormatTok->is(tok::l_square) && tryToParseSimpleAttribute()) - parseSquare(); + HandleAttributes(); bool NeedsUnwrappedLine = false; if (FormatTok->Tok.is(tok::l_brace)) { CompoundStatementIndenter Indenter(this, Style, Line->Level); @@ -2185,9 +2161,7 @@ void UnwrappedLineParser::parseIfThenElse() { } if (FormatTok->Tok.is(tok::kw_else)) { nextToken(); - // handle [[likely]] / [[unlikely]] - if (FormatTok->Tok.is(tok::l_square) && tryToParseSimpleAttribute()) - parseSquare(); + HandleAttributes(); if (FormatTok->Tok.is(tok::l_brace)) { CompoundStatementIndenter Indenter(this, Style, Line->Level); parseBlock(); @@ -2272,8 +2246,7 @@ void UnwrappedLineParser::parseTryCatch() { nextToken(); if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except, tok::kw___finally) || - ((Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript) && + ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) && FormatTok->is(Keywords.kw_finally)) || (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) || FormatTok->Tok.isObjCAtKeyword(tok::objc_finally)))) @@ -2396,8 +2369,7 @@ void UnwrappedLineParser::parseForOrWhileLoop() { "'for', 'while' or foreach macro expected"); nextToken(); // JS' for await ( ... - if (Style.Language == FormatStyle::LK_JavaScript && - FormatTok->is(Keywords.kw_await)) + if (Style.isJavaScript() && FormatTok->is(Keywords.kw_await)) nextToken(); if (Style.isCpp() && FormatTok->is(tok::kw_co_await)) nextToken(); @@ -2643,8 +2615,7 @@ bool UnwrappedLineParser::parseEnum() { // In TypeScript, "enum" can also be used as property name, e.g. in interface // declarations. An "enum" keyword followed by a colon would be a syntax // error and thus assume it is just an identifier. - if (Style.Language == FormatStyle::LK_JavaScript && - FormatTok->isOneOf(tok::colon, tok::question)) + if (Style.isJavaScript() && FormatTok->isOneOf(tok::colon, tok::question)) return false; // In protobuf, "enum" can be used as a field name. @@ -2716,8 +2687,8 @@ bool UnwrappedLineParser::parseStructLike() { // record declaration or definition can start a structural element. parseRecord(); // This does not apply to Java, JavaScript and C#. - if (Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) { + if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() || + Style.isCSharp()) { if (FormatTok->is(tok::semi)) nextToken(); addUnwrappedLine(); @@ -2846,10 +2817,9 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) { while (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::hashhash, tok::kw___attribute, tok::kw___declspec, tok::kw_alignas, tok::l_square, tok::r_square) || - ((Style.Language == FormatStyle::LK_Java || - Style.Language == FormatStyle::LK_JavaScript) && + ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) && FormatTok->isOneOf(tok::period, tok::comma))) { - if (Style.Language == FormatStyle::LK_JavaScript && + if (Style.isJavaScript() && FormatTok->isOneOf(Keywords.kw_extends, Keywords.kw_implements)) { // JavaScript/TypeScript supports inline object types in // extends/implements positions: @@ -3323,7 +3293,7 @@ void UnwrappedLineParser::nextToken(int LevelDifference) { flushComments(isOnNewLine(*FormatTok)); pushToken(FormatTok); FormatToken *Previous = FormatTok; - if (Style.Language != FormatStyle::LK_JavaScript) + if (!Style.isJavaScript()) readToken(LevelDifference); else readTokenWithJavaScriptASI(); diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h index b4c082654597..0c79723d50fc 100644 --- a/clang/lib/Format/UnwrappedLineParser.h +++ b/clang/lib/Format/UnwrappedLineParser.h @@ -138,6 +138,7 @@ class UnwrappedLineParser { // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/where-generic-type-constraint void parseCSharpGenericTypeConstraint(); bool tryToParseLambda(); + bool tryToParseChildBlock(); bool tryToParseLambdaIntroducer(); bool tryToParsePropertyAccessor(); void tryToParseJSFunction(); diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index fae8a1c3fdc6..96a66da0f82b 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -1282,9 +1282,12 @@ void WhitespaceManager::generateChanges() { C.EscapedNewlineColumn); else appendNewlineText(ReplacementText, C.NewlinesBefore); + // FIXME: This assert should hold if we computed the column correctly. + // assert((int)C.StartOfTokenColumn >= C.Spaces); appendIndentText( ReplacementText, C.Tok->IndentLevel, std::max(0, C.Spaces), - C.StartOfTokenColumn - std::max(0, C.Spaces), C.IsAligned); + std::max((int)C.StartOfTokenColumn, C.Spaces) - std::max(0, C.Spaces), + C.IsAligned); ReplacementText.append(C.CurrentLinePrefix); storeReplacement(C.OriginalWhitespaceRange, ReplacementText); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 1432607204bd..31e7ea3d243d 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -1154,12 +1154,12 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, // Remove any macro definitions that are explicitly ignored by the module. // They aren't supposed to affect how the module is built anyway. HeaderSearchOptions &HSOpts = Invocation->getHeaderSearchOpts(); - llvm::erase_if( - PPOpts.Macros, [&HSOpts](const std::pair &def) { - StringRef MacroDef = def.first; - return HSOpts.ModulesIgnoreMacros.count( - llvm::CachedHashString(MacroDef.split('=').first)) > 0; - }); + llvm::erase_if(PPOpts.Macros, + [&HSOpts](const std::pair &def) { + StringRef MacroDef = def.first; + return HSOpts.ModulesIgnoreMacros.contains( + llvm::CachedHashString(MacroDef.split('=').first)); + }); // If the original compiler invocation had -fmodule-name, pass it through. Invocation->getLangOpts()->ModuleName = diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index c104a6f40e20..b71addd84bfd 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -770,9 +770,7 @@ static void parseAnalyzerConfigs(AnalyzerOptions &AnOpts, static void getAllNoBuiltinFuncValues(ArgList &Args, std::vector &Funcs) { std::vector Values = Args.getAllArgValues(OPT_fno_builtin_); - auto BuiltinEnd = llvm::partition(Values, [](const std::string FuncName) { - return Builtin::Context::isBuiltinFunc(FuncName); - }); + auto BuiltinEnd = llvm::partition(Values, Builtin::Context::isBuiltinFunc); Funcs.insert(Funcs.end(), Values.begin(), BuiltinEnd); } @@ -1285,7 +1283,7 @@ static std::string serializeXRayInstrumentationBundle(const XRayInstrSet &S) { std::string Buffer; llvm::raw_string_ostream OS(Buffer); llvm::interleave(BundleParts, OS, [&OS](StringRef Part) { OS << Part; }, ","); - return OS.str(); + return Buffer; } // Set the profile kind using fprofile-instrument-use-path. @@ -4123,6 +4121,13 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, {std::string(Split.first), std::string(Split.second)}); } + // Error if -mvscale-min is unbounded. + if (Arg *A = Args.getLastArg(options::OPT_mvscale_min_EQ)) { + unsigned VScaleMin; + if (StringRef(A->getValue()).getAsInteger(10, VScaleMin) || VScaleMin == 0) + Diags.Report(diag::err_cc1_unbounded_vscale_min); + } + return Diags.getNumErrors() == NumErrorsBefore; } @@ -4513,7 +4518,7 @@ bool CompilerInvocation::CreateFromArgsImpl( // Store the command-line for using in the CodeView backend. Res.getCodeGenOpts().Argv0 = Argv0; - Res.getCodeGenOpts().CommandLineArgs = CommandLineArgs; + append_range(Res.getCodeGenOpts().CommandLineArgs, CommandLineArgs); FixupInvocation(Res, Diags, Args, DashX); diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp index 0c153446142e..629f99110661 100644 --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -500,8 +500,12 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI, // Not "standard" per se, but available even with the -undef flag. if (LangOpts.AsmPreprocessor) Builder.defineMacro("__ASSEMBLER__"); - if (LangOpts.CUDA && !LangOpts.HIP) - Builder.defineMacro("__CUDA__"); + if (LangOpts.CUDA) { + if (LangOpts.GPURelocatableDeviceCode) + Builder.defineMacro("__CLANG_RDC__"); + if (!LangOpts.HIP) + Builder.defineMacro("__CUDA__"); + } if (LangOpts.HIP) { Builder.defineMacro("__HIP__"); Builder.defineMacro("__HIPCC__"); diff --git a/clang/lib/Frontend/TestModuleFileExtension.cpp b/clang/lib/Frontend/TestModuleFileExtension.cpp index ea737e6891bf..2d5145d0c54c 100644 --- a/clang/lib/Frontend/TestModuleFileExtension.cpp +++ b/clang/lib/Frontend/TestModuleFileExtension.cpp @@ -133,5 +133,5 @@ std::string TestModuleFileExtension::str() const { llvm::raw_string_ostream OS(Buffer); OS << BlockName << ":" << MajorVersion << ":" << MinorVersion << ":" << Hashed << ":" << UserInfo; - return OS.str(); + return Buffer; } diff --git a/clang/lib/Headers/arm_neon_sve_bridge.h b/clang/lib/Headers/arm_neon_sve_bridge.h new file mode 100644 index 000000000000..17699d8d11dd --- /dev/null +++ b/clang/lib/Headers/arm_neon_sve_bridge.h @@ -0,0 +1,184 @@ +/*===---- arm_neon_sve_bridge.h - ARM NEON SVE Bridge intrinsics -----------=== + * + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ + +#ifndef __ARM_NEON_SVE_BRIDGE_H +#define __ARM_NEON_SVE_BRIDGE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Function attributes */ +#define __ai static __inline__ __attribute__((__always_inline__, __nodebug__)) +#define __aio \ + static __inline__ \ + __attribute__((__always_inline__, __nodebug__, __overloadable__)) + +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s8))) +svint8_t svset_neonq(svint8_t, int8x16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s16))) +svint16_t svset_neonq(svint16_t, int16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s32))) +svint32_t svset_neonq(svint32_t, int32x4_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s64))) +svint64_t svset_neonq(svint64_t, int64x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u8))) +svuint8_t svset_neonq(svuint8_t, uint8x16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u16))) +svuint16_t svset_neonq(svuint16_t, uint16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u32))) +svuint32_t svset_neonq(svuint32_t, uint32x4_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u64))) +svuint64_t svset_neonq(svuint64_t, uint64x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f16))) +svfloat16_t svset_neonq(svfloat16_t, float16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f32))) +svfloat32_t svset_neonq(svfloat32_t, float32x4_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f64))) +svfloat64_t svset_neonq(svfloat64_t, float64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s8))) +svint8_t svset_neonq_s8(svint8_t, int8x16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s16))) +svint16_t svset_neonq_s16(svint16_t, int16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s32))) +svint32_t svset_neonq_s32(svint32_t, int32x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s64))) +svint64_t svset_neonq_s64(svint64_t, int64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u8))) +svuint8_t svset_neonq_u8(svuint8_t, uint8x16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u16))) +svuint16_t svset_neonq_u16(svuint16_t, uint16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u32))) +svuint32_t svset_neonq_u32(svuint32_t, uint32x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u64))) +svuint64_t svset_neonq_u64(svuint64_t, uint64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f16))) +svfloat16_t svset_neonq_f16(svfloat16_t, float16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f32))) +svfloat32_t svset_neonq_f32(svfloat32_t, float32x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f64))) +svfloat64_t svset_neonq_f64(svfloat64_t, float64x2_t); + +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s8))) +int8x16_t svget_neonq(svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s16))) +int16x8_t svget_neonq(svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s32))) +int32x4_t svget_neonq(svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s64))) +int64x2_t svget_neonq(svint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u8))) +uint8x16_t svget_neonq(svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u16))) +uint16x8_t svget_neonq(svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u32))) +uint32x4_t svget_neonq(svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u64))) +uint64x2_t svget_neonq(svuint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f16))) +float16x8_t svget_neonq(svfloat16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f32))) +float32x4_t svget_neonq(svfloat32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f64))) +float64x2_t svget_neonq(svfloat64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s8))) +int8x16_t svget_neonq_s8(svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s16))) +int16x8_t svget_neonq_s16(svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s32))) +int32x4_t svget_neonq_s32(svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s64))) +int64x2_t svget_neonq_s64(svint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u8))) +uint8x16_t svget_neonq_u8(svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u16))) +uint16x8_t svget_neonq_u16(svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u32))) +uint32x4_t svget_neonq_u32(svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u64))) +uint64x2_t svget_neonq_u64(svuint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f16))) +float16x8_t svget_neonq_f16(svfloat16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f32))) +float32x4_t svget_neonq_f32(svfloat32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f64))) +float64x2_t svget_neonq_f64(svfloat64_t); + +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s8))) +svint8_t svdup_neonq(int8x16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s16))) +svint16_t svdup_neonq(int16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s32))) +svint32_t svdup_neonq(int32x4_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s64))) +svint64_t svdup_neonq(int64x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u8))) +svuint8_t svdup_neonq(uint8x16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u16))) +svuint16_t svdup_neonq(uint16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u32))) +svuint32_t svdup_neonq(uint32x4_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u64))) +svuint64_t svdup_neonq(uint64x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f16))) +svfloat16_t svdup_neonq(float16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f32))) +svfloat32_t svdup_neonq(float32x4_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f64))) +svfloat64_t svdup_neonq(float64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s8))) +svint8_t svdup_neonq_s8(int8x16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s16))) +svint16_t svdup_neonq_s16(int16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s32))) +svint32_t svdup_neonq_s32(int32x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s64))) +svint64_t svdup_neonq_s64(int64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u8))) +svuint8_t svdup_neonq_u8(uint8x16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u16))) +svuint16_t svdup_neonq_u16(uint16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u32))) +svuint32_t svdup_neonq_u32(uint32x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u64))) +svuint64_t svdup_neonq_u64(uint64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f16))) +svfloat16_t svdup_neonq_f16(float16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f32))) +svfloat32_t svdup_neonq_f32(float32x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f64))) +svfloat64_t svdup_neonq_f64(float64x2_t); + +#if defined(__ARM_FEATURE_SVE_BF16) +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_bf16))) +svbfloat16_t svset_neonq(svbfloat16_t, bfloat16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_bf16))) +svbfloat16_t svset_neonq_bf16(svbfloat16_t, bfloat16x8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_bf16))) +bfloat16x8_t svget_neonq(svbfloat16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_bf16))) +bfloat16x8_t svget_neonq_bf16(svbfloat16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_bf16))) +svbfloat16_t svdup_neonq(bfloat16x8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_bf16))) +svbfloat16_t svdup_neonq_bf16(bfloat16x8_t); +#endif // defined(__ARM_FEATURE_SVE_BF16) + +#undef __ai +#undef __aio + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif //__ARM_NEON_SVE_BRIDGE_H diff --git a/clang/lib/Headers/hexagon_protos.h b/clang/lib/Headers/hexagon_protos.h index cdffd93bb859..2642f3c8428d 100644 --- a/clang/lib/Headers/hexagon_protos.h +++ b/clang/lib/Headers/hexagon_protos.h @@ -8003,17 +8003,6 @@ #define Q6_P_vtrunohb_PP __builtin_HEXAGON_S6_vtrunohb_ppp #endif /* __HEXAGON_ARCH___ >= 62 */ -#if __HEXAGON_ARCH__ >= 62 -/* ========================================================================== - Assembly Syntax: Vd32=vmem(Rt32):nt - C Intrinsic Prototype: HVX_Vector Q6_V_vmem_R_nt(Word32 Rt) - Instruction Type: MAPPING - Execution Slots: SLOT0123 - ========================================================================== */ - -#define Q6_V_vmem_R_nt __builtin_HEXAGON_V6_ldntnt0 -#endif /* __HEXAGON_ARCH___ >= 62 */ - #if __HEXAGON_ARCH__ >= 65 /* ========================================================================== Assembly Syntax: Pd4=!any8(vcmpb.eq(Rss32,Rtt32)) diff --git a/clang/lib/Headers/hexagon_types.h b/clang/lib/Headers/hexagon_types.h index 6958809418d8..029727cc4817 100644 --- a/clang/lib/Headers/hexagon_types.h +++ b/clang/lib/Headers/hexagon_types.h @@ -1177,37 +1177,6 @@ class HEXAGON_Vect32C { #endif /* __cplusplus */ -// V65 Silver types -#if __Q6S_ARCH__ >= 65 - // Silver vector types are 128 bytes, and pairs are 256. The vector predicate - // types are 16 bytes and 32 bytes for pairs. - typedef long HEXAGON_VecPred128 __attribute__((__vector_size__(16))) - __attribute__((aligned(128))); - - typedef long HEXAGON_VecPred256 __attribute__((__vector_size__(32))) - __attribute__((aligned(128))); - - typedef long HEXAGON_Vect1024 __attribute__((__vector_size__(128))) - __attribute__((aligned(128))); - - typedef long HEXAGON_Vect2048 __attribute__((__vector_size__(256))) - __attribute__((aligned(256))); - - typedef long HEXAGON_UVect1024 __attribute__((__vector_size__(128))) - __attribute__((aligned(4))); - - typedef long HEXAGON_UVect2048 __attribute__((__vector_size__(256))) - __attribute__((aligned(4))); - - #define Q6S_VectorPredPair HEXAGON_VecPred256 - #define Q6S_VectorPred HEXAGON_VecPred128 - #define Q6S_Vector HEXAGON_Vect1024 - #define Q6S_VectorPair HEXAGON_Vect2048 - #define Q6S_UVector HEXAGON_UVect1024 - #define Q6S_UVectorPair HEXAGON_UVect2048 - -#else /* __Q6S_ARCH__ >= 65 */ - // V65 Vector types #if __HVX_ARCH__ >= 65 #if defined __HVX__ && (__HVX_LENGTH__ == 128) @@ -1256,7 +1225,6 @@ class HEXAGON_Vect32C { #endif /* defined __HVX__ && (__HVX_LENGTH__ == 64) */ #endif /* defined __HVX__ && (__HVX_LENGTH__ == 128) */ #endif /* __HVX_ARCH__ >= 65 */ -#endif /* __Q6S_ARCH__ >= 65 */ /* Predicates */ diff --git a/clang/lib/Headers/hvx_hexagon_protos.h b/clang/lib/Headers/hvx_hexagon_protos.h index 41ce7a6b93e9..7e3679a38b2c 100644 --- a/clang/lib/Headers/hvx_hexagon_protos.h +++ b/clang/lib/Headers/hvx_hexagon_protos.h @@ -9,7 +9,6 @@ //===----------------------------------------------------------------------===// - #ifndef _HVX_HEXAGON_PROTOS_H_ #define _HVX_HEXAGON_PROTOS_H_ 1 @@ -28,7 +27,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_R_vextract_VR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_extractw) +#define Q6_R_vextract_VR(Vu,Rs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_extractw)(Vu,Rs) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -39,7 +38,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_hi_W __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_hi) +#define Q6_V_hi_W(Vss) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_hi)(Vss) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -50,7 +49,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_lo_W __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lo) +#define Q6_V_lo_W(Vss) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lo)(Vss) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -61,7 +60,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_V_vsplat_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatw) +#define Q6_V_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatw)(Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -72,7 +71,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_and_QQ __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and) +#define Q6_Q_and_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -83,7 +82,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_and_QQn __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and_n) +#define Q6_Q_and_QQn(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and_n)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -94,7 +93,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_not_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_not) +#define Q6_Q_not_Q(Qs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_not)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -105,7 +104,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_or_QQ __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or) +#define Q6_Q_or_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -116,7 +115,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_or_QQn __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or_n) +#define Q6_Q_or_QQn(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or_n)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -127,7 +126,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vsetq_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2) +#define Q6_Q_vsetq_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2)(Rt)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -138,7 +137,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_xor_QQ __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_xor) +#define Q6_Q_xor_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -149,7 +148,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vmem_QnRIV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nqpred_ai) +#define Q6_vmem_QnRIV(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nqpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -160,7 +159,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vmem_QnRIV_nt __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai) +#define Q6_vmem_QnRIV_nt(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -171,7 +170,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vmem_QRIV_nt __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai) +#define Q6_vmem_QRIV_nt(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -182,7 +181,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vmem_QRIV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_qpred_ai) +#define Q6_vmem_QRIV(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_qpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -193,7 +192,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuh_vabsdiff_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffh) +#define Q6_Vuh_vabsdiff_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -204,7 +203,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vub_vabsdiff_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffub) +#define Q6_Vub_vabsdiff_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -215,7 +214,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuh_vabsdiff_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffuh) +#define Q6_Vuh_vabsdiff_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -226,7 +225,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vabsdiff_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffw) +#define Q6_Vuw_vabsdiff_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -237,7 +236,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vabs_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh) +#define Q6_Vh_vabs_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -248,7 +247,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vabs_Vh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh_sat) +#define Q6_Vh_vabs_Vh_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh_sat)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -259,7 +258,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vabs_Vw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw) +#define Q6_Vw_vabs_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -270,7 +269,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vabs_Vw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw_sat) +#define Q6_Vw_vabs_Vw_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw_sat)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -281,7 +280,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vadd_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb) +#define Q6_Vb_vadd_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -292,7 +291,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wb_vadd_WbWb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb_dv) +#define Q6_Wb_vadd_WbWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -303,7 +302,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_condacc_QnVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbnq) +#define Q6_Vb_condacc_QnVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -314,7 +313,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_condacc_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbq) +#define Q6_Vb_condacc_QVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -325,7 +324,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vadd_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh) +#define Q6_Vh_vadd_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -336,7 +335,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vadd_WhWh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh_dv) +#define Q6_Wh_vadd_WhWh(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -347,7 +346,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_condacc_QnVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhnq) +#define Q6_Vh_condacc_QnVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -358,7 +357,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_condacc_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhq) +#define Q6_Vh_condacc_QVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -369,7 +368,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vadd_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat) +#define Q6_Vh_vadd_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -380,7 +379,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vadd_WhWh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat_dv) +#define Q6_Wh_vadd_WhWh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -391,7 +390,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vadd_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw) +#define Q6_Ww_vadd_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -402,7 +401,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vadd_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh) +#define Q6_Wh_vadd_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -413,7 +412,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vadd_VubVub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat) +#define Q6_Vub_vadd_VubVub_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -424,7 +423,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wub_vadd_WubWub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat_dv) +#define Q6_Wub_vadd_WubWub_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -435,7 +434,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vadd_VuhVuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat) +#define Q6_Vuh_vadd_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -446,7 +445,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuh_vadd_WuhWuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat_dv) +#define Q6_Wuh_vadd_WuhWuh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -457,7 +456,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vadd_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw) +#define Q6_Ww_vadd_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -468,7 +467,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vadd_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw) +#define Q6_Vw_vadd_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -479,7 +478,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vadd_WwWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw_dv) +#define Q6_Ww_vadd_WwWw(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -490,7 +489,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_condacc_QnVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwnq) +#define Q6_Vw_condacc_QnVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -501,7 +500,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_condacc_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwq) +#define Q6_Vw_condacc_QVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -512,7 +511,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vadd_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat) +#define Q6_Vw_vadd_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -523,7 +522,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vadd_WwWw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat_dv) +#define Q6_Ww_vadd_WwWw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -534,7 +533,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_valign_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignb) +#define Q6_V_valign_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignb)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -545,7 +544,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_valign_VVI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignbi) +#define Q6_V_valign_VVI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignbi)(Vu,Vv,Iu3) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -556,7 +555,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vand_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vand) +#define Q6_V_vand_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vand)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -567,7 +566,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_V_vand_QR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) +#define Q6_V_vand_QR(Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -578,7 +577,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_V_vandor_VQR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt_acc) +#define Q6_V_vandor_VQR(Vx,Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt_acc)(Vx,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -589,7 +588,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Q_vand_VR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt) +#define Q6_Q_vand_VR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)(Vu,Rt)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -600,7 +599,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Q_vandor_QVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt_acc) +#define Q6_Q_vandor_QVR(Qx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt_acc)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Rt)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -611,7 +610,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasl_VhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh) +#define Q6_Vh_vasl_VhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -622,7 +621,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasl_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslhv) +#define Q6_Vh_vasl_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslhv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -633,7 +632,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vasl_VwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw) +#define Q6_Vw_vasl_VwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -644,7 +643,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vaslacc_VwVwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw_acc) +#define Q6_Vw_vaslacc_VwVwR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -655,7 +654,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vasl_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslwv) +#define Q6_Vw_vasl_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslwv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -666,7 +665,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasr_VhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh) +#define Q6_Vh_vasr_VhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -677,7 +676,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vasr_VhVhR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbrndsat) +#define Q6_Vb_vasr_VhVhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbrndsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -688,7 +687,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vasr_VhVhR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubrndsat) +#define Q6_Vub_vasr_VhVhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubrndsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -699,7 +698,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vasr_VhVhR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubsat) +#define Q6_Vub_vasr_VhVhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -710,7 +709,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasr_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhv) +#define Q6_Vh_vasr_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -721,7 +720,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vasr_VwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw) +#define Q6_Vw_vasr_VwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -732,7 +731,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vasracc_VwVwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw_acc) +#define Q6_Vw_vasracc_VwVwR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -743,7 +742,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasr_VwVwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwh) +#define Q6_Vh_vasr_VwVwR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwh)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -754,7 +753,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasr_VwVwR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhrndsat) +#define Q6_Vh_vasr_VwVwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhrndsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -765,7 +764,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasr_VwVwR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhsat) +#define Q6_Vh_vasr_VwVwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -776,7 +775,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vasr_VwVwR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhsat) +#define Q6_Vuh_vasr_VwVwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -787,7 +786,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vasr_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwv) +#define Q6_Vw_vasr_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -798,7 +797,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_equals_V __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign) +#define Q6_V_equals_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -809,7 +808,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_W_equals_W __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassignp) +#define Q6_W_equals_W(Vuu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassignp)(Vuu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -820,7 +819,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vavg_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgh) +#define Q6_Vh_vavg_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -831,7 +830,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vavg_VhVh_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavghrnd) +#define Q6_Vh_vavg_VhVh_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavghrnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -842,7 +841,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vavg_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgub) +#define Q6_Vub_vavg_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -853,7 +852,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vavg_VubVub_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgubrnd) +#define Q6_Vub_vavg_VubVub_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgubrnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -864,7 +863,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vavg_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguh) +#define Q6_Vuh_vavg_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -875,7 +874,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vavg_VuhVuh_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguhrnd) +#define Q6_Vuh_vavg_VuhVuh_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguhrnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -886,7 +885,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vavg_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgw) +#define Q6_Vw_vavg_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -897,7 +896,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vavg_VwVw_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgwrnd) +#define Q6_Vw_vavg_VwVw_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgwrnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -908,7 +907,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vcl0_Vuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0h) +#define Q6_Vuh_vcl0_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0h)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -919,7 +918,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vcl0_Vuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0w) +#define Q6_Vuw_vcl0_Vuw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0w)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -930,7 +929,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_W_vcombine_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcombine) +#define Q6_W_vcombine_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcombine)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -941,7 +940,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vzero __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vd0) +#define Q6_V_vzero() __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vd0)() #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -952,7 +951,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vdeal_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb) +#define Q6_Vb_vdeal_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -963,7 +962,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vdeale_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb4w) +#define Q6_Vb_vdeale_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb4w)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -974,7 +973,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vdeal_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealh) +#define Q6_Vh_vdeal_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -985,7 +984,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_W_vdeal_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealvdd) +#define Q6_W_vdeal_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealvdd)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -996,7 +995,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vdelta_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdelta) +#define Q6_V_vdelta_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdelta)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1007,7 +1006,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vdmpy_VubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus) +#define Q6_Vh_vdmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1018,7 +1017,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vdmpyacc_VhVubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_acc) +#define Q6_Vh_vdmpyacc_VhVubRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1029,7 +1028,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vdmpy_WubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv) +#define Q6_Wh_vdmpy_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1040,7 +1039,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vdmpyacc_WhWubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv_acc) +#define Q6_Wh_vdmpyacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1051,7 +1050,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpy_VhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb) +#define Q6_Vw_vdmpy_VhRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1062,7 +1061,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpyacc_VwVhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_acc) +#define Q6_Vw_vdmpyacc_VwVhRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1073,7 +1072,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vdmpy_WhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv) +#define Q6_Ww_vdmpy_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1084,7 +1083,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vdmpyacc_WwWhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv_acc) +#define Q6_Ww_vdmpyacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1095,7 +1094,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpy_WhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat) +#define Q6_Vw_vdmpy_WhRh_sat(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1106,29 +1105,29 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpyacc_VwWhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat_acc) +#define Q6_Vw_vdmpyacc_VwWhRh_sat(Vx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat_acc)(Vx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Rt32.h):sat C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRh_sat(HVX_Vector Vu, Word32 Rt) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpy_VhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat) +#define Q6_Vw_vdmpy_VhRh_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vx32.w+=vdmpy(Vu32.h,Rt32.h):sat C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRh_sat(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpyacc_VwVhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat_acc) +#define Q6_Vw_vdmpyacc_VwVhRh_sat(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1139,7 +1138,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpy_WhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat) +#define Q6_Vw_vdmpy_WhRuh_sat(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1150,40 +1149,40 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpyacc_VwWhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat_acc) +#define Q6_Vw_vdmpyacc_VwWhRuh_sat(Vx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat_acc)(Vx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Rt32.uh):sat C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRuh_sat(HVX_Vector Vu, Word32 Rt) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpy_VhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat) +#define Q6_Vw_vdmpy_VhRuh_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vx32.w+=vdmpy(Vu32.h,Rt32.uh):sat C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRuh_sat(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpyacc_VwVhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat_acc) +#define Q6_Vw_vdmpyacc_VwVhRuh_sat(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Vv32.h):sat C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpy_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat) +#define Q6_Vw_vdmpy_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1194,7 +1193,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vdmpyacc_VwVhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat_acc) +#define Q6_Vw_vdmpyacc_VwVhVh_sat(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1205,7 +1204,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vdsad_WuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh) +#define Q6_Wuw_vdsad_WuhRuh(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1216,7 +1215,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vdsadacc_WuwWuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh_acc) +#define Q6_Wuw_vdsadacc_WuwWuhRuh(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1227,7 +1226,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eq_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb) +#define Q6_Q_vcmp_eq_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1238,7 +1237,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqand_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_and) +#define Q6_Q_vcmp_eqand_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1249,7 +1248,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqor_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_or) +#define Q6_Q_vcmp_eqor_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1260,7 +1259,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqxacc_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_xor) +#define Q6_Q_vcmp_eqxacc_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1271,7 +1270,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eq_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh) +#define Q6_Q_vcmp_eq_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1282,7 +1281,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqand_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_and) +#define Q6_Q_vcmp_eqand_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1293,7 +1292,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqor_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_or) +#define Q6_Q_vcmp_eqor_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1304,7 +1303,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqxacc_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_xor) +#define Q6_Q_vcmp_eqxacc_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1315,7 +1314,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eq_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw) +#define Q6_Q_vcmp_eq_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1326,7 +1325,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqand_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_and) +#define Q6_Q_vcmp_eqand_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1337,7 +1336,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqor_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_or) +#define Q6_Q_vcmp_eqor_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1348,7 +1347,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_eqxacc_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_xor) +#define Q6_Q_vcmp_eqxacc_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1359,7 +1358,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gt_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb) +#define Q6_Q_vcmp_gt_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1370,7 +1369,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtand_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_and) +#define Q6_Q_vcmp_gtand_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1381,7 +1380,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtor_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_or) +#define Q6_Q_vcmp_gtor_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1392,7 +1391,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtxacc_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_xor) +#define Q6_Q_vcmp_gtxacc_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1403,7 +1402,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gt_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth) +#define Q6_Q_vcmp_gt_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1414,7 +1413,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtand_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_and) +#define Q6_Q_vcmp_gtand_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1425,7 +1424,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtor_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_or) +#define Q6_Q_vcmp_gtor_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1436,7 +1435,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtxacc_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_xor) +#define Q6_Q_vcmp_gtxacc_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1447,7 +1446,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gt_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub) +#define Q6_Q_vcmp_gt_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1458,7 +1457,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtand_QVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_and) +#define Q6_Q_vcmp_gtand_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1469,7 +1468,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtor_QVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_or) +#define Q6_Q_vcmp_gtor_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1480,7 +1479,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtxacc_QVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_xor) +#define Q6_Q_vcmp_gtxacc_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1491,7 +1490,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gt_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh) +#define Q6_Q_vcmp_gt_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1502,7 +1501,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtand_QVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_and) +#define Q6_Q_vcmp_gtand_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1513,7 +1512,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtor_QVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_or) +#define Q6_Q_vcmp_gtor_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1524,7 +1523,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtxacc_QVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_xor) +#define Q6_Q_vcmp_gtxacc_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1535,7 +1534,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gt_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw) +#define Q6_Q_vcmp_gt_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1546,7 +1545,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtand_QVuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_and) +#define Q6_Q_vcmp_gtand_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1557,7 +1556,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtor_QVuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_or) +#define Q6_Q_vcmp_gtor_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1568,7 +1567,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtxacc_QVuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_xor) +#define Q6_Q_vcmp_gtxacc_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1579,7 +1578,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gt_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw) +#define Q6_Q_vcmp_gt_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw)(Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1590,7 +1589,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtand_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_and) +#define Q6_Q_vcmp_gtand_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1601,7 +1600,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtor_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_or) +#define Q6_Q_vcmp_gtor_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1612,7 +1611,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vcmp_gtxacc_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_xor) +#define Q6_Q_vcmp_gtxacc_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1623,7 +1622,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vinsert_VwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vinsertwr) +#define Q6_Vw_vinsert_VwR(Vx,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vinsertwr)(Vx,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1634,7 +1633,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vlalign_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignb) +#define Q6_V_vlalign_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignb)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1645,7 +1644,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vlalign_VVI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignbi) +#define Q6_V_vlalign_VVI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignbi)(Vu,Vv,Iu3) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1656,7 +1655,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vlsr_VuhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrh) +#define Q6_Vuh_vlsr_VuhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrh)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1667,7 +1666,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vlsr_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrhv) +#define Q6_Vh_vlsr_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrhv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1678,7 +1677,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vlsr_VuwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrw) +#define Q6_Vuw_vlsr_VuwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrw)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1689,7 +1688,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vlsr_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrwv) +#define Q6_Vw_vlsr_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrwv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1700,7 +1699,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vlut32_VbVbR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb) +#define Q6_Vb_vlut32_VbVbR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1711,7 +1710,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vlut32or_VbVbVbR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracc) +#define Q6_Vb_vlut32or_VbVbVbR(Vx,Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracc)(Vx,Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1722,7 +1721,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vlut16_VbVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh) +#define Q6_Wh_vlut16_VbVhR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1733,7 +1732,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vlut16or_WhVbVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracc) +#define Q6_Wh_vlut16or_WhVbVhR(Vxx,Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracc)(Vxx,Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1744,7 +1743,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vmax_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxh) +#define Q6_Vh_vmax_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1755,7 +1754,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vmax_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxub) +#define Q6_Vub_vmax_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1766,7 +1765,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vmax_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxuh) +#define Q6_Vuh_vmax_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1777,7 +1776,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vmax_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxw) +#define Q6_Vw_vmax_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1788,7 +1787,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vmin_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminh) +#define Q6_Vh_vmin_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1799,7 +1798,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vmin_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminub) +#define Q6_Vub_vmin_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1810,7 +1809,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vmin_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminuh) +#define Q6_Vuh_vmin_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1821,7 +1820,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vmin_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminw) +#define Q6_Vw_vmin_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1832,7 +1831,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpa_WubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus) +#define Q6_Wh_vmpa_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1843,7 +1842,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpaacc_WhWubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus_acc) +#define Q6_Wh_vmpaacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1854,7 +1853,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpa_WubWb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabusv) +#define Q6_Wh_vmpa_WubWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabusv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1865,7 +1864,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpa_WubWub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuuv) +#define Q6_Wh_vmpa_WubWub(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuuv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1876,7 +1875,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpa_WhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb) +#define Q6_Ww_vmpa_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1887,7 +1886,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpaacc_WwWhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb_acc) +#define Q6_Ww_vmpaacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1898,7 +1897,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpy_VubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus) +#define Q6_Wh_vmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1909,7 +1908,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpyacc_WhVubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus_acc) +#define Q6_Wh_vmpyacc_WhVubRb(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus_acc)(Vxx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1920,7 +1919,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpy_VubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv) +#define Q6_Wh_vmpy_VubVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1931,7 +1930,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpyacc_WhVubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv_acc) +#define Q6_Wh_vmpyacc_WhVubVb(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1942,7 +1941,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpy_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv) +#define Q6_Wh_vmpy_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1953,7 +1952,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpyacc_WhVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv_acc) +#define Q6_Wh_vmpyacc_WhVbVb(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1964,7 +1963,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpye_VwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh) +#define Q6_Vw_vmpye_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1975,7 +1974,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpy_VhRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh) +#define Q6_Ww_vmpy_VhRh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -1986,29 +1985,29 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpyacc_WwVhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsat_acc) +#define Q6_Ww_vmpyacc_WwVhRh_sat(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsat_acc)(Vxx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vd32.h=vmpy(Vu32.h,Rt32.h):<<1:rnd:sat C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhRh_s1_rnd_sat(HVX_Vector Vu, Word32 Rt) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpy_VhRh_s1_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsrs) +#define Q6_Vh_vmpy_VhRh_s1_rnd_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsrs)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vd32.h=vmpy(Vu32.h,Rt32.h):<<1:sat C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhRh_s1_sat(HVX_Vector Vu, Word32 Rt) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpy_VhRh_s1_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhss) +#define Q6_Vh_vmpy_VhRh_s1_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhss)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2019,7 +2018,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpy_VhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus) +#define Q6_Ww_vmpy_VhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2030,7 +2029,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpyacc_WwVhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus_acc) +#define Q6_Ww_vmpyacc_WwVhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2041,7 +2040,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpy_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv) +#define Q6_Ww_vmpy_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2052,18 +2051,18 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpyacc_WwVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv_acc) +#define Q6_Ww_vmpyacc_WwVhVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vd32.h=vmpy(Vu32.h,Vv32.h):<<1:rnd:sat C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhVh_s1_rnd_sat(HVX_Vector Vu, HVX_Vector Vv) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpy_VhVh_s1_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhvsrs) +#define Q6_Vh_vmpy_VhVh_s1_rnd_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhvsrs)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2074,7 +2073,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyieo_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyieoh) +#define Q6_Vw_vmpyieo_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyieoh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2085,7 +2084,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyieacc_VwVwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewh_acc) +#define Q6_Vw_vmpyieacc_VwVwVh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewh_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2096,7 +2095,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyie_VwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh) +#define Q6_Vw_vmpyie_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2107,7 +2106,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyieacc_VwVwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh_acc) +#define Q6_Vw_vmpyieacc_VwVwVuh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2118,7 +2117,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpyi_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih) +#define Q6_Vh_vmpyi_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2129,7 +2128,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpyiacc_VhVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih_acc) +#define Q6_Vh_vmpyiacc_VhVhVh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2140,7 +2139,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpyi_VhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb) +#define Q6_Vh_vmpyi_VhRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2151,7 +2150,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vmpyiacc_VhVhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb_acc) +#define Q6_Vh_vmpyiacc_VhVhRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2162,7 +2161,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyio_VwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiowh) +#define Q6_Vw_vmpyio_VwVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiowh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2173,7 +2172,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyi_VwRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb) +#define Q6_Vw_vmpyi_VwRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2184,7 +2183,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyiacc_VwVwRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb_acc) +#define Q6_Vw_vmpyiacc_VwVwRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2195,7 +2194,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyi_VwRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh) +#define Q6_Vw_vmpyi_VwRh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2206,7 +2205,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyiacc_VwVwRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh_acc) +#define Q6_Vw_vmpyiacc_VwVwRh(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2217,7 +2216,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyo_VwVh_s1_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh) +#define Q6_Vw_vmpyo_VwVh_s1_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2228,7 +2227,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyo_VwVh_s1_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd) +#define Q6_Vw_vmpyo_VwVh_s1_rnd_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2239,7 +2238,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyoacc_VwVwVh_s1_rnd_sat_shift __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc) +#define Q6_Vw_vmpyoacc_VwVwVh_s1_rnd_sat_shift(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2250,7 +2249,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyoacc_VwVwVh_s1_sat_shift __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_sacc) +#define Q6_Vw_vmpyoacc_VwVwVh_s1_sat_shift(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_sacc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2261,7 +2260,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuh_vmpy_VubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub) +#define Q6_Wuh_vmpy_VubRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2272,7 +2271,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuh_vmpyacc_WuhVubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub_acc) +#define Q6_Wuh_vmpyacc_WuhVubRub(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub_acc)(Vxx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2283,7 +2282,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuh_vmpy_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv) +#define Q6_Wuh_vmpy_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2294,7 +2293,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuh_vmpyacc_WuhVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv_acc) +#define Q6_Wuh_vmpyacc_WuhVubVub(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2305,7 +2304,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vmpy_VuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh) +#define Q6_Wuw_vmpy_VuhRuh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2316,7 +2315,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vmpyacc_WuwVuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh_acc) +#define Q6_Wuw_vmpyacc_WuwVuhRuh(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh_acc)(Vxx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2327,7 +2326,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vmpy_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv) +#define Q6_Wuw_vmpy_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2338,7 +2337,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vmpyacc_WuwVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv_acc) +#define Q6_Wuw_vmpyacc_WuwVuhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2349,7 +2348,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vmux_QVV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmux) +#define Q6_V_vmux_QVV(Qt,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmux)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1),Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2360,7 +2359,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vnavg_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgh) +#define Q6_Vh_vnavg_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2371,7 +2370,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vnavg_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgub) +#define Q6_Vb_vnavg_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2382,7 +2381,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vnavg_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgw) +#define Q6_Vw_vnavg_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2393,7 +2392,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vnormamt_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamth) +#define Q6_Vh_vnormamt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamth)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2404,7 +2403,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vnormamt_Vw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamtw) +#define Q6_Vw_vnormamt_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamtw)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2415,7 +2414,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vnot_V __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnot) +#define Q6_V_vnot_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnot)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2426,7 +2425,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vor_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vor) +#define Q6_V_vor_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vor)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2437,7 +2436,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vpacke_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeb) +#define Q6_Vb_vpacke_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2448,7 +2447,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vpacke_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeh) +#define Q6_Vh_vpacke_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2459,7 +2458,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vpack_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhb_sat) +#define Q6_Vb_vpack_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhb_sat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2470,7 +2469,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vpack_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhub_sat) +#define Q6_Vub_vpack_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhub_sat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2481,7 +2480,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vpacko_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackob) +#define Q6_Vb_vpacko_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackob)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2492,7 +2491,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vpacko_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackoh) +#define Q6_Vh_vpacko_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackoh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2503,7 +2502,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vpack_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwh_sat) +#define Q6_Vh_vpack_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwh_sat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2514,7 +2513,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vpack_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwuh_sat) +#define Q6_Vuh_vpack_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwuh_sat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2525,7 +2524,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vpopcount_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpopcounth) +#define Q6_Vh_vpopcount_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpopcounth)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2536,7 +2535,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vrdelta_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrdelta) +#define Q6_V_vrdelta_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrdelta)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2547,7 +2546,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vrmpy_VubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus) +#define Q6_Vw_vrmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2558,7 +2557,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vrmpyacc_VwVubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus_acc) +#define Q6_Vw_vrmpyacc_VwVubRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2569,7 +2568,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vrmpy_WubRbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi) +#define Q6_Ww_vrmpy_WubRbI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi)(Vuu,Rt,Iu1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2580,7 +2579,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vrmpyacc_WwWubRbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi_acc) +#define Q6_Ww_vrmpyacc_WwWubRbI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi_acc)(Vxx,Vuu,Rt,Iu1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2591,18 +2590,18 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vrmpy_VubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv) +#define Q6_Vw_vrmpy_VubVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vx32.w+=vrmpy(Vu32.ub,Vv32.b) C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVubVb(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vrmpyacc_VwVubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv_acc) +#define Q6_Vw_vrmpyacc_VwVubVb(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2613,18 +2612,18 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vrmpy_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv) +#define Q6_Vw_vrmpy_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vx32.w+=vrmpy(Vu32.b,Vv32.b) C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVbVb(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vrmpyacc_VwVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv_acc) +#define Q6_Vw_vrmpyacc_VwVbVb(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2635,7 +2634,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vrmpy_VubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub) +#define Q6_Vuw_vrmpy_VubRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2646,7 +2645,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vrmpyacc_VuwVubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub_acc) +#define Q6_Vuw_vrmpyacc_VuwVubRub(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2657,7 +2656,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vrmpy_WubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi) +#define Q6_Wuw_vrmpy_WubRubI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi)(Vuu,Rt,Iu1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2668,7 +2667,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vrmpyacc_WuwWubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi_acc) +#define Q6_Wuw_vrmpyacc_WuwWubRubI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi_acc)(Vxx,Vuu,Rt,Iu1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2679,18 +2678,18 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vrmpy_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv) +#define Q6_Vuw_vrmpy_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 /* ========================================================================== Assembly Syntax: Vx32.uw+=vrmpy(Vu32.ub,Vv32.ub) C Intrinsic Prototype: HVX_Vector Q6_Vuw_vrmpyacc_VuwVubVub(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv) - Instruction Type: CVI_VX_DV + Instruction Type: CVI_VX Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vrmpyacc_VuwVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv_acc) +#define Q6_Vuw_vrmpyacc_VuwVubVub(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv_acc)(Vx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2701,7 +2700,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vror_VR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vror) +#define Q6_V_vror_VR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vror)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2712,7 +2711,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vround_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhb) +#define Q6_Vb_vround_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2723,7 +2722,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vround_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhub) +#define Q6_Vub_vround_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2734,7 +2733,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vround_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwh) +#define Q6_Vh_vround_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2745,7 +2744,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vround_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwuh) +#define Q6_Vuh_vround_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2756,7 +2755,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vrsad_WubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi) +#define Q6_Wuw_vrsad_WubRubI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi)(Vuu,Rt,Iu1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2767,7 +2766,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wuw_vrsadacc_WuwWubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi_acc) +#define Q6_Wuw_vrsadacc_WuwWubRubI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi_acc)(Vxx,Vuu,Rt,Iu1) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2778,7 +2777,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vsat_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsathub) +#define Q6_Vub_vsat_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsathub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2789,7 +2788,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vsat_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatwh) +#define Q6_Vh_vsat_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatwh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2800,7 +2799,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vsxt_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsb) +#define Q6_Wh_vsxt_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsb)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2811,7 +2810,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vsxt_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsh) +#define Q6_Ww_vsxt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2822,7 +2821,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vshuffe_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufeh) +#define Q6_Vh_vshuffe_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufeh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2833,7 +2832,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vshuff_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffb) +#define Q6_Vb_vshuff_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffb)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2844,7 +2843,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vshuffe_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffeb) +#define Q6_Vb_vshuffe_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffeb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2855,7 +2854,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vshuff_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffh) +#define Q6_Vh_vshuff_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2866,7 +2865,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vshuffo_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffob) +#define Q6_Vb_vshuffo_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffob)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2877,7 +2876,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_W_vshuff_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffvdd) +#define Q6_W_vshuff_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffvdd)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2888,7 +2887,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wb_vshuffoe_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeb) +#define Q6_Wb_vshuffoe_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2899,7 +2898,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vshuffoe_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeh) +#define Q6_Wh_vshuffoe_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2910,7 +2909,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vshuffo_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoh) +#define Q6_Vh_vshuffo_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2921,7 +2920,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vsub_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb) +#define Q6_Vb_vsub_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2932,7 +2931,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wb_vsub_WbWb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb_dv) +#define Q6_Wb_vsub_WbWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2943,7 +2942,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_condnac_QnVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbnq) +#define Q6_Vb_condnac_QnVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2954,7 +2953,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_condnac_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbq) +#define Q6_Vb_condnac_QVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2965,7 +2964,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vsub_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh) +#define Q6_Vh_vsub_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2976,7 +2975,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vsub_WhWh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh_dv) +#define Q6_Wh_vsub_WhWh(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2987,7 +2986,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_condnac_QnVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhnq) +#define Q6_Vh_condnac_QnVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -2998,7 +2997,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_condnac_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhq) +#define Q6_Vh_condnac_QVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3009,7 +3008,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vsub_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat) +#define Q6_Vh_vsub_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3020,7 +3019,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vsub_WhWh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat_dv) +#define Q6_Wh_vsub_WhWh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3031,7 +3030,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vsub_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhw) +#define Q6_Ww_vsub_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3042,7 +3041,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vsub_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububh) +#define Q6_Wh_vsub_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3053,7 +3052,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vsub_VubVub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat) +#define Q6_Vub_vsub_VubVub_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3064,7 +3063,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wub_vsub_WubWub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat_dv) +#define Q6_Wub_vsub_WubWub_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3075,7 +3074,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vsub_VuhVuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat) +#define Q6_Vuh_vsub_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3086,7 +3085,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuh_vsub_WuhWuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat_dv) +#define Q6_Wuh_vsub_WuhWuh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3097,7 +3096,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vsub_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhw) +#define Q6_Ww_vsub_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3108,7 +3107,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vsub_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw) +#define Q6_Vw_vsub_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3119,7 +3118,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vsub_WwWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw_dv) +#define Q6_Ww_vsub_WwWw(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3130,7 +3129,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_condnac_QnVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwnq) +#define Q6_Vw_condnac_QnVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3141,7 +3140,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_condnac_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwq) +#define Q6_Vw_condnac_QVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3152,7 +3151,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vsub_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat) +#define Q6_Vw_vsub_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3163,7 +3162,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vsub_WwWw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat_dv) +#define Q6_Ww_vsub_WwWw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3174,7 +3173,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_W_vswap_QVV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vswap) +#define Q6_W_vswap_QVV(Qt,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vswap)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1),Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3185,7 +3184,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vtmpy_WbRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb) +#define Q6_Wh_vtmpy_WbRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3196,7 +3195,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vtmpyacc_WhWbRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb_acc) +#define Q6_Wh_vtmpyacc_WhWbRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3207,7 +3206,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vtmpy_WubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus) +#define Q6_Wh_vtmpy_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3218,7 +3217,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vtmpyacc_WhWubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus_acc) +#define Q6_Wh_vtmpyacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3229,7 +3228,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vtmpy_WhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb) +#define Q6_Ww_vtmpy_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3240,7 +3239,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vtmpyacc_WwWhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb_acc) +#define Q6_Ww_vtmpyacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3251,7 +3250,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vunpack_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackb) +#define Q6_Wh_vunpack_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackb)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3262,7 +3261,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vunpack_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackh) +#define Q6_Ww_vunpack_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3273,7 +3272,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vunpackoor_WhVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackob) +#define Q6_Wh_vunpackoor_WhVb(Vxx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackob)(Vxx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3284,7 +3283,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vunpackoor_WwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackoh) +#define Q6_Ww_vunpackoor_WwVh(Vxx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackoh)(Vxx,Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3295,7 +3294,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuh_vunpack_Vub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackub) +#define Q6_Wuh_vunpack_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackub)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3306,7 +3305,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuw_vunpack_Vuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackuh) +#define Q6_Wuw_vunpack_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackuh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3317,7 +3316,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vxor_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vxor) +#define Q6_V_vxor_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vxor)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3328,7 +3327,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuh_vzxt_Vub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzb) +#define Q6_Wuh_vzxt_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzb)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 60 @@ -3339,7 +3338,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuw_vzxt_Vuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzh) +#define Q6_Wuw_vzxt_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzh)(Vu) #endif /* __HEXAGON_ARCH___ >= 60 */ #if __HVX_ARCH__ >= 62 @@ -3350,7 +3349,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vb_vsplat_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatb) +#define Q6_Vb_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatb)(Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3361,7 +3360,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vh_vsplat_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplath) +#define Q6_Vh_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplath)(Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3372,7 +3371,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Q_vsetq2_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2v2) +#define Q6_Q_vsetq2_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2v2)(Rt)),-1) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3383,7 +3382,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Qb_vshuffe_QhQh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqh) +#define Q6_Qb_vshuffe_QhQh(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqh)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3394,7 +3393,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Qh_vshuffe_QwQw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqw) +#define Q6_Qh_vshuffe_QwQw(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqw)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3405,7 +3404,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vadd_VbVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat) +#define Q6_Vb_vadd_VbVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3416,7 +3415,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wb_vadd_WbWb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat_dv) +#define Q6_Wb_vadd_WbWb_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3427,7 +3426,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vadd_VwVwQ_carry __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarry) +#define Q6_Vw_vadd_VwVwQ_carry(Vu,Vv,Qx) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarry)(Vu,Vv,Qx) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3438,7 +3437,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vadd_vclb_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbh) +#define Q6_Vh_vadd_vclb_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3449,7 +3448,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vadd_vclb_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbw) +#define Q6_Vw_vadd_vclb_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3460,7 +3459,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vaddacc_WwVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw_acc) +#define Q6_Ww_vaddacc_WwVhVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3471,7 +3470,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vaddacc_WhVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh_acc) +#define Q6_Wh_vaddacc_WhVubVub(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3482,7 +3481,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vadd_VubVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddububb_sat) +#define Q6_Vub_vadd_VubVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddububb_sat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3493,7 +3492,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vaddacc_WwVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw_acc) +#define Q6_Ww_vaddacc_WwVuhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3504,7 +3503,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vadd_VuwVuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat) +#define Q6_Vuw_vadd_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3515,7 +3514,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuw_vadd_WuwWuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat_dv) +#define Q6_Wuw_vadd_WuwWuw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3526,7 +3525,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_V_vand_QnR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt) +#define Q6_V_vand_QnR(Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3537,7 +3536,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_V_vandor_VQnR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt_acc) +#define Q6_V_vandor_VQnR(Vx,Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt_acc)(Vx,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3548,7 +3547,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vand_QnV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvnqv) +#define Q6_V_vand_QnV(Qv,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvnqv)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vu) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3559,7 +3558,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_V_vand_QV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvqv) +#define Q6_V_vand_QV(Qv,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvqv)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vu) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3570,7 +3569,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vasr_VhVhR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbsat) +#define Q6_Vb_vasr_VhVhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3581,7 +3580,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vasr_VuwVuwR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhrndsat) +#define Q6_Vuh_vasr_VuwVuwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhrndsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3592,7 +3591,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vasr_VwVwR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhrndsat) +#define Q6_Vuh_vasr_VwVwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhrndsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3603,7 +3602,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vlsr_VubR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrb) +#define Q6_Vub_vlsr_VubR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrb)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3614,7 +3613,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vlut32_VbVbR_nomatch __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_nm) +#define Q6_Vb_vlut32_VbVbR_nomatch(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_nm)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3625,7 +3624,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vlut32or_VbVbVbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracci) +#define Q6_Vb_vlut32or_VbVbVbI(Vx,Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracci)(Vx,Vu,Vv,Iu3) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3636,7 +3635,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vlut32_VbVbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvbi) +#define Q6_Vb_vlut32_VbVbI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvbi)(Vu,Vv,Iu3) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3647,7 +3646,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vlut16_VbVhR_nomatch __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_nm) +#define Q6_Wh_vlut16_VbVhR_nomatch(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_nm)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3658,7 +3657,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vlut16or_WhVbVhI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracci) +#define Q6_Wh_vlut16or_WhVbVhI(Vxx,Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracci)(Vxx,Vu,Vv,Iu3) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3669,7 +3668,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wh_vlut16_VbVhI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwhi) +#define Q6_Wh_vlut16_VbVhI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwhi)(Vu,Vv,Iu3) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3680,7 +3679,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vmax_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxb) +#define Q6_Vb_vmax_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3691,7 +3690,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vmin_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminb) +#define Q6_Vb_vmin_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3702,7 +3701,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpa_WuhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb) +#define Q6_Ww_vmpa_WuhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3713,7 +3712,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpaacc_WwWuhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb_acc) +#define Q6_Ww_vmpaacc_WwWuhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3724,7 +3723,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_W_vmpye_VwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh_64) +#define Q6_W_vmpye_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh_64)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3735,7 +3734,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyi_VwRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub) +#define Q6_Vw_vmpyi_VwRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3746,7 +3745,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vw_vmpyiacc_VwVwRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub_acc) +#define Q6_Vw_vmpyiacc_VwVwRub(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3757,7 +3756,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_W_vmpyoacc_WVwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_64_acc) +#define Q6_W_vmpyoacc_WVwVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_64_acc)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3768,7 +3767,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vround_VuhVuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduhub) +#define Q6_Vub_vround_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduhub)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3779,7 +3778,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vround_VuwVuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduwuh) +#define Q6_Vuh_vround_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduwuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3790,7 +3789,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vsat_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatuwuh) +#define Q6_Vuh_vsat_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatuwuh)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3801,7 +3800,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vsub_VbVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat) +#define Q6_Vb_vsub_VbVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3812,7 +3811,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wb_vsub_WbWb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat_dv) +#define Q6_Wb_vsub_WbWb_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3823,7 +3822,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vsub_VwVwQ_carry __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubcarry) +#define Q6_Vw_vsub_VwVwQ_carry(Vu,Vv,Qx) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubcarry)(Vu,Vv,Qx) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3834,7 +3833,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vsub_VubVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubububb_sat) +#define Q6_Vub_vsub_VubVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubububb_sat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3845,7 +3844,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vsub_VuwVuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat) +#define Q6_Vuw_vsub_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 62 @@ -3856,7 +3855,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Wuw_vsub_WuwWuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat_dv) +#define Q6_Wuw_vsub_WuwWuw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat_dv)(Vuu,Vvv) #endif /* __HEXAGON_ARCH___ >= 62 */ #if __HVX_ARCH__ >= 65 @@ -3867,7 +3866,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vabs_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb) +#define Q6_Vb_vabs_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb)(Vu) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3878,7 +3877,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vabs_Vb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb_sat) +#define Q6_Vb_vabs_Vb_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb_sat)(Vu) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3889,7 +3888,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vaslacc_VhVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh_acc) +#define Q6_Vh_vaslacc_VhVhR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3900,7 +3899,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_vasracc_VhVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh_acc) +#define Q6_Vh_vasracc_VhVhR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3911,7 +3910,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vasr_VuhVuhR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubrndsat) +#define Q6_Vub_vasr_VuhVuhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubrndsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3922,7 +3921,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vub_vasr_VuhVuhR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubsat) +#define Q6_Vub_vasr_VuhVuhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3933,7 +3932,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuh_vasr_VuwVuwR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhsat) +#define Q6_Vuh_vasr_VuwVuwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhsat)(Vu,Vv,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3944,7 +3943,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vavg_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgb) +#define Q6_Vb_vavg_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3955,7 +3954,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vavg_VbVb_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgbrnd) +#define Q6_Vb_vavg_VbVb_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgbrnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3966,7 +3965,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vavg_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguw) +#define Q6_Vuw_vavg_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3977,7 +3976,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vavg_VuwVuw_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguwrnd) +#define Q6_Vuw_vavg_VuwVuw_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguwrnd)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3988,7 +3987,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_W_vzero __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdd0) +#define Q6_W_vzero() __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdd0)() #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -3999,7 +3998,7 @@ Execution Slots: SLOT01 ========================================================================== */ -#define Q6_vgather_ARMVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermh) +#define Q6_vgather_ARMVh(Rs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermh)(Rs,Rt,Mu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4010,7 +4009,7 @@ Execution Slots: SLOT01 ========================================================================== */ -#define Q6_vgather_AQRMVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhq) +#define Q6_vgather_AQRMVh(Rs,Qs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4021,7 +4020,7 @@ Execution Slots: SLOT01 ========================================================================== */ -#define Q6_vgather_ARMWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhw) +#define Q6_vgather_ARMWw(Rs,Rt,Mu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhw)(Rs,Rt,Mu,Vvv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4032,7 +4031,7 @@ Execution Slots: SLOT01 ========================================================================== */ -#define Q6_vgather_AQRMWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhwq) +#define Q6_vgather_AQRMWw(Rs,Qs,Rt,Mu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhwq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vvv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4043,7 +4042,7 @@ Execution Slots: SLOT01 ========================================================================== */ -#define Q6_vgather_ARMVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermw) +#define Q6_vgather_ARMVw(Rs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermw)(Rs,Rt,Mu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4054,7 +4053,7 @@ Execution Slots: SLOT01 ========================================================================== */ -#define Q6_vgather_AQRMVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermwq) +#define Q6_vgather_AQRMVw(Rs,Qs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermwq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4065,7 +4064,7 @@ Execution Slots: SLOT2 ========================================================================== */ -#define Q6_Vh_vlut4_VuhPh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlut4) +#define Q6_Vh_vlut4_VuhPh(Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlut4)(Vu,Rtt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4076,7 +4075,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpa_WubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu) +#define Q6_Wh_vmpa_WubRub(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu)(Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4087,7 +4086,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Wh_vmpaacc_WhWubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu_acc) +#define Q6_Wh_vmpaacc_WhWubRub(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu_acc)(Vxx,Vuu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4098,7 +4097,7 @@ Execution Slots: SLOT2 ========================================================================== */ -#define Q6_Vh_vmpa_VhVhVhPh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahhsat) +#define Q6_Vh_vmpa_VhVhVhPh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahhsat)(Vx,Vu,Rtt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4109,7 +4108,7 @@ Execution Slots: SLOT2 ========================================================================== */ -#define Q6_Vh_vmpa_VhVhVuhPuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhuhsat) +#define Q6_Vh_vmpa_VhVhVuhPuh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhuhsat)(Vx,Vu,Rtt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4120,7 +4119,7 @@ Execution Slots: SLOT2 ========================================================================== */ -#define Q6_Vh_vmps_VhVhVuhPuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpsuhuhsat) +#define Q6_Vh_vmps_VhVhVuhPuh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpsuhuhsat)(Vx,Vu,Rtt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4131,7 +4130,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_vmpyacc_WwVhRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh_acc) +#define Q6_Ww_vmpyacc_WwVhRh(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh_acc)(Vxx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4142,7 +4141,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vmpye_VuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe) +#define Q6_Vuw_vmpye_VuhRuh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe)(Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4153,7 +4152,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Vuw_vmpyeacc_VuwVuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe_acc) +#define Q6_Vuw_vmpyeacc_VuwVuhRuh(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe_acc)(Vx,Vu,Rt) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4164,7 +4163,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_vnavg_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgb) +#define Q6_Vb_vnavg_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgb)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4175,7 +4174,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vb_prefixsum_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqb) +#define Q6_Vb_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqb)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1)) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4186,7 +4185,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vh_prefixsum_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqh) +#define Q6_Vh_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqh)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1)) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4197,7 +4196,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_prefixsum_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqw) +#define Q6_Vw_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqw)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1)) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4208,7 +4207,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatter_RMVhV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh) +#define Q6_vscatter_RMVhV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh)(Rt,Mu,Vv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4219,7 +4218,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatteracc_RMVhV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh_add) +#define Q6_vscatteracc_RMVhV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh_add)(Rt,Mu,Vv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4230,7 +4229,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatter_QRMVhV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhq) +#define Q6_vscatter_QRMVhV(Qs,Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4241,7 +4240,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatter_RMWwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw) +#define Q6_vscatter_RMWwV(Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw)(Rt,Mu,Vvv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4252,7 +4251,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatteracc_RMWwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw_add) +#define Q6_vscatteracc_RMWwV(Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw_add)(Rt,Mu,Vvv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4263,7 +4262,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatter_QRMWwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhwq) +#define Q6_vscatter_QRMWwV(Qs,Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vvv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4274,7 +4273,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatter_RMVwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw) +#define Q6_vscatter_RMVwV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw)(Rt,Mu,Vv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4285,7 +4284,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatteracc_RMVwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw_add) +#define Q6_vscatteracc_RMVwV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw_add)(Rt,Mu,Vv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 65 @@ -4296,7 +4295,7 @@ Execution Slots: SLOT0 ========================================================================== */ -#define Q6_vscatter_QRMVwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermwq) +#define Q6_vscatter_QRMVwV(Qs,Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv,Vw) #endif /* __HEXAGON_ARCH___ >= 65 */ #if __HVX_ARCH__ >= 66 @@ -4307,7 +4306,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vadd_VwVwQ_carry_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarrysat) +#define Q6_Vw_vadd_VwVwQ_carry_sat(Vu,Vv,Qs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarrysat)(Vu,Vv,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1)) #endif /* __HEXAGON_ARCH___ >= 66 */ #if __HVX_ARCH__ >= 66 @@ -4318,7 +4317,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Ww_vasrinto_WwVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasr_into) +#define Q6_Ww_vasrinto_WwVwVw(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasr_into)(Vxx,Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 66 */ #if __HVX_ARCH__ >= 66 @@ -4329,7 +4328,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vuw_vrotr_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrotr) +#define Q6_Vuw_vrotr_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrotr)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 66 */ #if __HVX_ARCH__ >= 66 @@ -4340,7 +4339,7 @@ Execution Slots: SLOT0123 ========================================================================== */ -#define Q6_Vw_vsatdw_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatdw) +#define Q6_Vw_vsatdw_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatdw)(Vu,Vv) #endif /* __HEXAGON_ARCH___ >= 66 */ #if __HVX_ARCH__ >= 68 @@ -4351,7 +4350,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_v6mpy_WubWbI_h __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10) +#define Q6_Ww_v6mpy_WubWbI_h(Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10)(Vuu,Vvv,Iu2) #endif /* __HEXAGON_ARCH___ >= 68 */ #if __HVX_ARCH__ >= 68 @@ -4362,7 +4361,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_v6mpyacc_WwWubWbI_h __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10_vxx) +#define Q6_Ww_v6mpyacc_WwWubWbI_h(Vxx,Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10_vxx)(Vxx,Vuu,Vvv,Iu2) #endif /* __HEXAGON_ARCH___ >= 68 */ #if __HVX_ARCH__ >= 68 @@ -4373,7 +4372,7 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_v6mpy_WubWbI_v __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10) +#define Q6_Ww_v6mpy_WubWbI_v(Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10)(Vuu,Vvv,Iu2) #endif /* __HEXAGON_ARCH___ >= 68 */ #if __HVX_ARCH__ >= 68 @@ -4384,9 +4383,801 @@ Execution Slots: SLOT23 ========================================================================== */ -#define Q6_Ww_v6mpyacc_WwWubWbI_v __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10_vxx) +#define Q6_Ww_v6mpyacc_WwWubWbI_v(Vxx,Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10_vxx)(Vxx,Vuu,Vvv,Iu2) #endif /* __HEXAGON_ARCH___ >= 68 */ +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vabs(Vu32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vabs_Vhf(HVX_Vector Vu) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vabs_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_hf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vabs(Vu32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vabs_Vsf(HVX_Vector Vu) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vabs_Vsf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_sf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vadd(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf16_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vadd(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vadd(Vu32.qf16,Vv32.qf16) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vadd_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf16_vadd_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf16)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vadd(Vu32.qf16,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vadd_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf16_vadd_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf16_mix)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vadd(Vu32.qf32,Vv32.qf32) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vadd_Vqf32Vqf32(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf32_vadd_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf32)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vadd(Vu32.qf32,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vadd_Vqf32Vsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf32_vadd_Vqf32Vsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf32_mix)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vadd(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vadd_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf32_vadd_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.sf=vadd(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wsf_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vadd(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vadd_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vadd_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.w=vfmv(Vu32.w) + C Intrinsic Prototype: HVX_Vector Q6_Vw_vfmv_Vw(HVX_Vector Vu) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vw_vfmv_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign_fp)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=Vu32.qf16 + C Intrinsic Prototype: HVX_Vector Q6_Vhf_equals_Vqf16(HVX_Vector Vu) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vhf_equals_Vqf16(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_qf16)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=Vuu32.qf32 + C Intrinsic Prototype: HVX_Vector Q6_Vhf_equals_Wqf32(HVX_VectorPair Vuu) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vhf_equals_Wqf32(Vuu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_qf32)(Vuu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=Vu32.qf32 + C Intrinsic Prototype: HVX_Vector Q6_Vsf_equals_Vqf32(HVX_Vector Vu) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vsf_equals_Vqf32(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_sf_qf32)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.b=vcvt(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vb_vcvt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_b_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.h=vcvt(Vu32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vh_vcvt_Vhf(HVX_Vector Vu) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vh_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_h_hf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.hf=vcvt(Vu32.b) + C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_Vb(HVX_Vector Vu) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Whf_vcvt_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_b)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vcvt(Vu32.h) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_Vh(HVX_Vector Vu) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vcvt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_h)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vcvt(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vcvt_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.hf=vcvt(Vu32.ub) + C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_Vub(HVX_Vector Vu) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Whf_vcvt_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_ub)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vcvt(Vu32.uh) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_Vuh(HVX_Vector Vu) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vcvt_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_uh)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.sf=vcvt(Vu32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vcvt_Vhf(HVX_Vector Vu) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wsf_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_sf_hf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.ub=vcvt(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vub_vcvt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_ub_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.uh=vcvt(Vu32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vuh_vcvt_Vhf(HVX_Vector Vu) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vuh_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_uh_hf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vdmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vdmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vdmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpy_sf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vx32.sf+=vdmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vdmpyacc_VsfVhfVhf(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vdmpyacc_VsfVhfVhf(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc)(Vx,Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vfmax(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfmax_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vfmax_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vfmax(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfmax_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vfmax_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vfmin(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfmin_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vfmin_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vfmin(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfmin_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vfmin_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vfneg(Vu32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfneg_Vhf(HVX_Vector Vu) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vfneg_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_hf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vfneg(Vu32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfneg_Vsf(HVX_Vector Vu) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vfneg_Vsf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_sf)(Vu) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qd4=vcmp.gt(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf)(Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qx4&=vcmp.gt(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gtand_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qx4|=vcmp.gt(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gtor_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qx4^=vcmp.gt(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gtxacc_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qd4=vcmp.gt(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gt_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf)(Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qx4&=vcmp.gt(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gtand_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qx4|=vcmp.gt(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gtor_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Qx4^=vcmp.gt(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Q_vcmp_gtxacc_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vmax(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmax_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vhf_vmax_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vmax(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmax_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vsf_vmax_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vmin(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmin_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vhf_vmin_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vmin(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmin_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vsf_vmin_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vx32.hf+=vmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmpyacc_VhfVhfVhf(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vmpyacc_VhfVhfVhf(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_hf_acc)(Vx,Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vmpy(Vu32.qf16,Vv32.qf16) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vqf16_vmpy_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vqf16_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vmpy(Vu32.qf16,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vqf16_vmpy_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vmpy(Vu32.qf32,Vv32.qf32) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_Vqf32Vqf32(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vqf32_vmpy_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.qf32=vmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wqf32_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wqf32_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.qf32=vmpy(Vu32.qf16,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wqf32_vmpy_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wqf32_vmpy_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.qf32=vmpy(Vu32.qf16,Vv32.qf16) + C Intrinsic Prototype: HVX_VectorPair Q6_Wqf32_vmpy_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wqf32_vmpy_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_qf16)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vmpy(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vqf32_vmpy_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.sf=vmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wsf_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vxx32.sf+=vmpy(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVhfVhf(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wsf_vmpyacc_WsfVhfVhf(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_hf_acc)(Vxx,Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vmpy(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmpy_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vmpy_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vsub(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf16_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.hf=vsub(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vhf_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vhf_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vsub(Vu32.qf16,Vv32.qf16) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf16_vsub_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf16)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf16=vsub(Vu32.qf16,Vv32.hf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf16_vsub_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf16_mix)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vsub(Vu32.qf32,Vv32.qf32) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_Vqf32Vqf32(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf32_vsub_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf32)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vsub(Vu32.qf32,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_Vqf32Vsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf32_vsub_Vqf32Vsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf32_mix)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.qf32=vsub(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vqf32_vsub_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vdd32.sf=vsub(Vu32.hf,Vv32.hf) + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Wsf_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_hf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 68 +/* ========================================================================== + Assembly Syntax: Vd32.sf=vsub(Vu32.sf,Vv32.sf) + C Intrinsic Prototype: HVX_Vector Q6_Vsf_vsub_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vsf_vsub_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_sf)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 68 */ + +#if __HVX_ARCH__ >= 69 +/* ========================================================================== + Assembly Syntax: Vd32.ub=vasr(Vuu32.uh,Vv32.ub):rnd:sat + C Intrinsic Prototype: HVX_Vector Q6_Vub_vasr_WuhVub_rnd_sat(HVX_VectorPair Vuu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vub_vasr_WuhVub_rnd_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvuhubrndsat)(Vuu,Vv) +#endif /* __HEXAGON_ARCH___ >= 69 */ + +#if __HVX_ARCH__ >= 69 +/* ========================================================================== + Assembly Syntax: Vd32.ub=vasr(Vuu32.uh,Vv32.ub):sat + C Intrinsic Prototype: HVX_Vector Q6_Vub_vasr_WuhVub_sat(HVX_VectorPair Vuu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vub_vasr_WuhVub_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvuhubsat)(Vuu,Vv) +#endif /* __HEXAGON_ARCH___ >= 69 */ + +#if __HVX_ARCH__ >= 69 +/* ========================================================================== + Assembly Syntax: Vd32.uh=vasr(Vuu32.w,Vv32.uh):rnd:sat + C Intrinsic Prototype: HVX_Vector Q6_Vuh_vasr_WwVuh_rnd_sat(HVX_VectorPair Vuu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vuh_vasr_WwVuh_rnd_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvwuhrndsat)(Vuu,Vv) +#endif /* __HEXAGON_ARCH___ >= 69 */ + +#if __HVX_ARCH__ >= 69 +/* ========================================================================== + Assembly Syntax: Vd32.uh=vasr(Vuu32.w,Vv32.uh):sat + C Intrinsic Prototype: HVX_Vector Q6_Vuh_vasr_WwVuh_sat(HVX_VectorPair Vuu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 + ========================================================================== */ + +#define Q6_Vuh_vasr_WwVuh_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvwuhsat)(Vuu,Vv) +#endif /* __HEXAGON_ARCH___ >= 69 */ + +#if __HVX_ARCH__ >= 69 +/* ========================================================================== + Assembly Syntax: Vd32.uh=vmpy(Vu32.uh,Vv32.uh):>>16 + C Intrinsic Prototype: HVX_Vector Q6_Vuh_vmpy_VuhVuh_rs16(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 + ========================================================================== */ + +#define Q6_Vuh_vmpy_VuhVuh_rs16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhvs)(Vu,Vv) +#endif /* __HEXAGON_ARCH___ >= 69 */ + #endif /* __HVX__ */ #endif diff --git a/clang/lib/Headers/opencl-c.h b/clang/lib/Headers/opencl-c.h index 32af848a94c4..77a7a8b9bb3a 100644 --- a/clang/lib/Headers/opencl-c.h +++ b/clang/lib/Headers/opencl-c.h @@ -11190,305 +11190,305 @@ half16 __ovld __cnfn select(half16 a, half16 b, ushort16 c); * 64-bit aligned if gentype is long, ulong, double. */ -char2 __ovld vload2(size_t offset, const __constant char *p); -uchar2 __ovld vload2(size_t offset, const __constant uchar *p); -short2 __ovld vload2(size_t offset, const __constant short *p); -ushort2 __ovld vload2(size_t offset, const __constant ushort *p); -int2 __ovld vload2(size_t offset, const __constant int *p); -uint2 __ovld vload2(size_t offset, const __constant uint *p); -long2 __ovld vload2(size_t offset, const __constant long *p); -ulong2 __ovld vload2(size_t offset, const __constant ulong *p); -float2 __ovld vload2(size_t offset, const __constant float *p); -char3 __ovld vload3(size_t offset, const __constant char *p); -uchar3 __ovld vload3(size_t offset, const __constant uchar *p); -short3 __ovld vload3(size_t offset, const __constant short *p); -ushort3 __ovld vload3(size_t offset, const __constant ushort *p); -int3 __ovld vload3(size_t offset, const __constant int *p); -uint3 __ovld vload3(size_t offset, const __constant uint *p); -long3 __ovld vload3(size_t offset, const __constant long *p); -ulong3 __ovld vload3(size_t offset, const __constant ulong *p); -float3 __ovld vload3(size_t offset, const __constant float *p); -char4 __ovld vload4(size_t offset, const __constant char *p); -uchar4 __ovld vload4(size_t offset, const __constant uchar *p); -short4 __ovld vload4(size_t offset, const __constant short *p); -ushort4 __ovld vload4(size_t offset, const __constant ushort *p); -int4 __ovld vload4(size_t offset, const __constant int *p); -uint4 __ovld vload4(size_t offset, const __constant uint *p); -long4 __ovld vload4(size_t offset, const __constant long *p); -ulong4 __ovld vload4(size_t offset, const __constant ulong *p); -float4 __ovld vload4(size_t offset, const __constant float *p); -char8 __ovld vload8(size_t offset, const __constant char *p); -uchar8 __ovld vload8(size_t offset, const __constant uchar *p); -short8 __ovld vload8(size_t offset, const __constant short *p); -ushort8 __ovld vload8(size_t offset, const __constant ushort *p); -int8 __ovld vload8(size_t offset, const __constant int *p); -uint8 __ovld vload8(size_t offset, const __constant uint *p); -long8 __ovld vload8(size_t offset, const __constant long *p); -ulong8 __ovld vload8(size_t offset, const __constant ulong *p); -float8 __ovld vload8(size_t offset, const __constant float *p); -char16 __ovld vload16(size_t offset, const __constant char *p); -uchar16 __ovld vload16(size_t offset, const __constant uchar *p); -short16 __ovld vload16(size_t offset, const __constant short *p); -ushort16 __ovld vload16(size_t offset, const __constant ushort *p); -int16 __ovld vload16(size_t offset, const __constant int *p); -uint16 __ovld vload16(size_t offset, const __constant uint *p); -long16 __ovld vload16(size_t offset, const __constant long *p); -ulong16 __ovld vload16(size_t offset, const __constant ulong *p); -float16 __ovld vload16(size_t offset, const __constant float *p); +char2 __ovld __purefn vload2(size_t offset, const __constant char *p); +uchar2 __ovld __purefn vload2(size_t offset, const __constant uchar *p); +short2 __ovld __purefn vload2(size_t offset, const __constant short *p); +ushort2 __ovld __purefn vload2(size_t offset, const __constant ushort *p); +int2 __ovld __purefn vload2(size_t offset, const __constant int *p); +uint2 __ovld __purefn vload2(size_t offset, const __constant uint *p); +long2 __ovld __purefn vload2(size_t offset, const __constant long *p); +ulong2 __ovld __purefn vload2(size_t offset, const __constant ulong *p); +float2 __ovld __purefn vload2(size_t offset, const __constant float *p); +char3 __ovld __purefn vload3(size_t offset, const __constant char *p); +uchar3 __ovld __purefn vload3(size_t offset, const __constant uchar *p); +short3 __ovld __purefn vload3(size_t offset, const __constant short *p); +ushort3 __ovld __purefn vload3(size_t offset, const __constant ushort *p); +int3 __ovld __purefn vload3(size_t offset, const __constant int *p); +uint3 __ovld __purefn vload3(size_t offset, const __constant uint *p); +long3 __ovld __purefn vload3(size_t offset, const __constant long *p); +ulong3 __ovld __purefn vload3(size_t offset, const __constant ulong *p); +float3 __ovld __purefn vload3(size_t offset, const __constant float *p); +char4 __ovld __purefn vload4(size_t offset, const __constant char *p); +uchar4 __ovld __purefn vload4(size_t offset, const __constant uchar *p); +short4 __ovld __purefn vload4(size_t offset, const __constant short *p); +ushort4 __ovld __purefn vload4(size_t offset, const __constant ushort *p); +int4 __ovld __purefn vload4(size_t offset, const __constant int *p); +uint4 __ovld __purefn vload4(size_t offset, const __constant uint *p); +long4 __ovld __purefn vload4(size_t offset, const __constant long *p); +ulong4 __ovld __purefn vload4(size_t offset, const __constant ulong *p); +float4 __ovld __purefn vload4(size_t offset, const __constant float *p); +char8 __ovld __purefn vload8(size_t offset, const __constant char *p); +uchar8 __ovld __purefn vload8(size_t offset, const __constant uchar *p); +short8 __ovld __purefn vload8(size_t offset, const __constant short *p); +ushort8 __ovld __purefn vload8(size_t offset, const __constant ushort *p); +int8 __ovld __purefn vload8(size_t offset, const __constant int *p); +uint8 __ovld __purefn vload8(size_t offset, const __constant uint *p); +long8 __ovld __purefn vload8(size_t offset, const __constant long *p); +ulong8 __ovld __purefn vload8(size_t offset, const __constant ulong *p); +float8 __ovld __purefn vload8(size_t offset, const __constant float *p); +char16 __ovld __purefn vload16(size_t offset, const __constant char *p); +uchar16 __ovld __purefn vload16(size_t offset, const __constant uchar *p); +short16 __ovld __purefn vload16(size_t offset, const __constant short *p); +ushort16 __ovld __purefn vload16(size_t offset, const __constant ushort *p); +int16 __ovld __purefn vload16(size_t offset, const __constant int *p); +uint16 __ovld __purefn vload16(size_t offset, const __constant uint *p); +long16 __ovld __purefn vload16(size_t offset, const __constant long *p); +ulong16 __ovld __purefn vload16(size_t offset, const __constant ulong *p); +float16 __ovld __purefn vload16(size_t offset, const __constant float *p); #ifdef cl_khr_fp64 -double2 __ovld vload2(size_t offset, const __constant double *p); -double3 __ovld vload3(size_t offset, const __constant double *p); -double4 __ovld vload4(size_t offset, const __constant double *p); -double8 __ovld vload8(size_t offset, const __constant double *p); -double16 __ovld vload16(size_t offset, const __constant double *p); +double2 __ovld __purefn vload2(size_t offset, const __constant double *p); +double3 __ovld __purefn vload3(size_t offset, const __constant double *p); +double4 __ovld __purefn vload4(size_t offset, const __constant double *p); +double8 __ovld __purefn vload8(size_t offset, const __constant double *p); +double16 __ovld __purefn vload16(size_t offset, const __constant double *p); #endif //cl_khr_fp64 #ifdef cl_khr_fp16 -half __ovld vload(size_t offset, const __constant half *p); -half2 __ovld vload2(size_t offset, const __constant half *p); -half3 __ovld vload3(size_t offset, const __constant half *p); -half4 __ovld vload4(size_t offset, const __constant half *p); -half8 __ovld vload8(size_t offset, const __constant half *p); -half16 __ovld vload16(size_t offset, const __constant half *p); +half __ovld __purefn vload(size_t offset, const __constant half *p); +half2 __ovld __purefn vload2(size_t offset, const __constant half *p); +half3 __ovld __purefn vload3(size_t offset, const __constant half *p); +half4 __ovld __purefn vload4(size_t offset, const __constant half *p); +half8 __ovld __purefn vload8(size_t offset, const __constant half *p); +half16 __ovld __purefn vload16(size_t offset, const __constant half *p); #endif //cl_khr_fp16 #if defined(__opencl_c_generic_address_space) -char2 __ovld vload2(size_t offset, const char *p); -uchar2 __ovld vload2(size_t offset, const uchar *p); -short2 __ovld vload2(size_t offset, const short *p); -ushort2 __ovld vload2(size_t offset, const ushort *p); -int2 __ovld vload2(size_t offset, const int *p); -uint2 __ovld vload2(size_t offset, const uint *p); -long2 __ovld vload2(size_t offset, const long *p); -ulong2 __ovld vload2(size_t offset, const ulong *p); -float2 __ovld vload2(size_t offset, const float *p); -char3 __ovld vload3(size_t offset, const char *p); -uchar3 __ovld vload3(size_t offset, const uchar *p); -short3 __ovld vload3(size_t offset, const short *p); -ushort3 __ovld vload3(size_t offset, const ushort *p); -int3 __ovld vload3(size_t offset, const int *p); -uint3 __ovld vload3(size_t offset, const uint *p); -long3 __ovld vload3(size_t offset, const long *p); -ulong3 __ovld vload3(size_t offset, const ulong *p); -float3 __ovld vload3(size_t offset, const float *p); -char4 __ovld vload4(size_t offset, const char *p); -uchar4 __ovld vload4(size_t offset, const uchar *p); -short4 __ovld vload4(size_t offset, const short *p); -ushort4 __ovld vload4(size_t offset, const ushort *p); -int4 __ovld vload4(size_t offset, const int *p); -uint4 __ovld vload4(size_t offset, const uint *p); -long4 __ovld vload4(size_t offset, const long *p); -ulong4 __ovld vload4(size_t offset, const ulong *p); -float4 __ovld vload4(size_t offset, const float *p); -char8 __ovld vload8(size_t offset, const char *p); -uchar8 __ovld vload8(size_t offset, const uchar *p); -short8 __ovld vload8(size_t offset, const short *p); -ushort8 __ovld vload8(size_t offset, const ushort *p); -int8 __ovld vload8(size_t offset, const int *p); -uint8 __ovld vload8(size_t offset, const uint *p); -long8 __ovld vload8(size_t offset, const long *p); -ulong8 __ovld vload8(size_t offset, const ulong *p); -float8 __ovld vload8(size_t offset, const float *p); -char16 __ovld vload16(size_t offset, const char *p); -uchar16 __ovld vload16(size_t offset, const uchar *p); -short16 __ovld vload16(size_t offset, const short *p); -ushort16 __ovld vload16(size_t offset, const ushort *p); -int16 __ovld vload16(size_t offset, const int *p); -uint16 __ovld vload16(size_t offset, const uint *p); -long16 __ovld vload16(size_t offset, const long *p); -ulong16 __ovld vload16(size_t offset, const ulong *p); -float16 __ovld vload16(size_t offset, const float *p); +char2 __ovld __purefn vload2(size_t offset, const char *p); +uchar2 __ovld __purefn vload2(size_t offset, const uchar *p); +short2 __ovld __purefn vload2(size_t offset, const short *p); +ushort2 __ovld __purefn vload2(size_t offset, const ushort *p); +int2 __ovld __purefn vload2(size_t offset, const int *p); +uint2 __ovld __purefn vload2(size_t offset, const uint *p); +long2 __ovld __purefn vload2(size_t offset, const long *p); +ulong2 __ovld __purefn vload2(size_t offset, const ulong *p); +float2 __ovld __purefn vload2(size_t offset, const float *p); +char3 __ovld __purefn vload3(size_t offset, const char *p); +uchar3 __ovld __purefn vload3(size_t offset, const uchar *p); +short3 __ovld __purefn vload3(size_t offset, const short *p); +ushort3 __ovld __purefn vload3(size_t offset, const ushort *p); +int3 __ovld __purefn vload3(size_t offset, const int *p); +uint3 __ovld __purefn vload3(size_t offset, const uint *p); +long3 __ovld __purefn vload3(size_t offset, const long *p); +ulong3 __ovld __purefn vload3(size_t offset, const ulong *p); +float3 __ovld __purefn vload3(size_t offset, const float *p); +char4 __ovld __purefn vload4(size_t offset, const char *p); +uchar4 __ovld __purefn vload4(size_t offset, const uchar *p); +short4 __ovld __purefn vload4(size_t offset, const short *p); +ushort4 __ovld __purefn vload4(size_t offset, const ushort *p); +int4 __ovld __purefn vload4(size_t offset, const int *p); +uint4 __ovld __purefn vload4(size_t offset, const uint *p); +long4 __ovld __purefn vload4(size_t offset, const long *p); +ulong4 __ovld __purefn vload4(size_t offset, const ulong *p); +float4 __ovld __purefn vload4(size_t offset, const float *p); +char8 __ovld __purefn vload8(size_t offset, const char *p); +uchar8 __ovld __purefn vload8(size_t offset, const uchar *p); +short8 __ovld __purefn vload8(size_t offset, const short *p); +ushort8 __ovld __purefn vload8(size_t offset, const ushort *p); +int8 __ovld __purefn vload8(size_t offset, const int *p); +uint8 __ovld __purefn vload8(size_t offset, const uint *p); +long8 __ovld __purefn vload8(size_t offset, const long *p); +ulong8 __ovld __purefn vload8(size_t offset, const ulong *p); +float8 __ovld __purefn vload8(size_t offset, const float *p); +char16 __ovld __purefn vload16(size_t offset, const char *p); +uchar16 __ovld __purefn vload16(size_t offset, const uchar *p); +short16 __ovld __purefn vload16(size_t offset, const short *p); +ushort16 __ovld __purefn vload16(size_t offset, const ushort *p); +int16 __ovld __purefn vload16(size_t offset, const int *p); +uint16 __ovld __purefn vload16(size_t offset, const uint *p); +long16 __ovld __purefn vload16(size_t offset, const long *p); +ulong16 __ovld __purefn vload16(size_t offset, const ulong *p); +float16 __ovld __purefn vload16(size_t offset, const float *p); #ifdef cl_khr_fp64 -double2 __ovld vload2(size_t offset, const double *p); -double3 __ovld vload3(size_t offset, const double *p); -double4 __ovld vload4(size_t offset, const double *p); -double8 __ovld vload8(size_t offset, const double *p); -double16 __ovld vload16(size_t offset, const double *p); +double2 __ovld __purefn vload2(size_t offset, const double *p); +double3 __ovld __purefn vload3(size_t offset, const double *p); +double4 __ovld __purefn vload4(size_t offset, const double *p); +double8 __ovld __purefn vload8(size_t offset, const double *p); +double16 __ovld __purefn vload16(size_t offset, const double *p); #endif //cl_khr_fp64 #ifdef cl_khr_fp16 -half __ovld vload(size_t offset, const half *p); -half2 __ovld vload2(size_t offset, const half *p); -half3 __ovld vload3(size_t offset, const half *p); -half4 __ovld vload4(size_t offset, const half *p); -half8 __ovld vload8(size_t offset, const half *p); -half16 __ovld vload16(size_t offset, const half *p); +half __ovld __purefn vload(size_t offset, const half *p); +half2 __ovld __purefn vload2(size_t offset, const half *p); +half3 __ovld __purefn vload3(size_t offset, const half *p); +half4 __ovld __purefn vload4(size_t offset, const half *p); +half8 __ovld __purefn vload8(size_t offset, const half *p); +half16 __ovld __purefn vload16(size_t offset, const half *p); #endif //cl_khr_fp16 #else -char2 __ovld vload2(size_t offset, const __global char *p); -uchar2 __ovld vload2(size_t offset, const __global uchar *p); -short2 __ovld vload2(size_t offset, const __global short *p); -ushort2 __ovld vload2(size_t offset, const __global ushort *p); -int2 __ovld vload2(size_t offset, const __global int *p); -uint2 __ovld vload2(size_t offset, const __global uint *p); -long2 __ovld vload2(size_t offset, const __global long *p); -ulong2 __ovld vload2(size_t offset, const __global ulong *p); -float2 __ovld vload2(size_t offset, const __global float *p); -char3 __ovld vload3(size_t offset, const __global char *p); -uchar3 __ovld vload3(size_t offset, const __global uchar *p); -short3 __ovld vload3(size_t offset, const __global short *p); -ushort3 __ovld vload3(size_t offset, const __global ushort *p); -int3 __ovld vload3(size_t offset, const __global int *p); -uint3 __ovld vload3(size_t offset, const __global uint *p); -long3 __ovld vload3(size_t offset, const __global long *p); -ulong3 __ovld vload3(size_t offset, const __global ulong *p); -float3 __ovld vload3(size_t offset, const __global float *p); -char4 __ovld vload4(size_t offset, const __global char *p); -uchar4 __ovld vload4(size_t offset, const __global uchar *p); -short4 __ovld vload4(size_t offset, const __global short *p); -ushort4 __ovld vload4(size_t offset, const __global ushort *p); -int4 __ovld vload4(size_t offset, const __global int *p); -uint4 __ovld vload4(size_t offset, const __global uint *p); -long4 __ovld vload4(size_t offset, const __global long *p); -ulong4 __ovld vload4(size_t offset, const __global ulong *p); -float4 __ovld vload4(size_t offset, const __global float *p); -char8 __ovld vload8(size_t offset, const __global char *p); -uchar8 __ovld vload8(size_t offset, const __global uchar *p); -short8 __ovld vload8(size_t offset, const __global short *p); -ushort8 __ovld vload8(size_t offset, const __global ushort *p); -int8 __ovld vload8(size_t offset, const __global int *p); -uint8 __ovld vload8(size_t offset, const __global uint *p); -long8 __ovld vload8(size_t offset, const __global long *p); -ulong8 __ovld vload8(size_t offset, const __global ulong *p); -float8 __ovld vload8(size_t offset, const __global float *p); -char16 __ovld vload16(size_t offset, const __global char *p); -uchar16 __ovld vload16(size_t offset, const __global uchar *p); -short16 __ovld vload16(size_t offset, const __global short *p); -ushort16 __ovld vload16(size_t offset, const __global ushort *p); -int16 __ovld vload16(size_t offset, const __global int *p); -uint16 __ovld vload16(size_t offset, const __global uint *p); -long16 __ovld vload16(size_t offset, const __global long *p); -ulong16 __ovld vload16(size_t offset, const __global ulong *p); -float16 __ovld vload16(size_t offset, const __global float *p); -char2 __ovld vload2(size_t offset, const __local char *p); -uchar2 __ovld vload2(size_t offset, const __local uchar *p); -short2 __ovld vload2(size_t offset, const __local short *p); -ushort2 __ovld vload2(size_t offset, const __local ushort *p); -int2 __ovld vload2(size_t offset, const __local int *p); -uint2 __ovld vload2(size_t offset, const __local uint *p); -long2 __ovld vload2(size_t offset, const __local long *p); -ulong2 __ovld vload2(size_t offset, const __local ulong *p); -float2 __ovld vload2(size_t offset, const __local float *p); -char3 __ovld vload3(size_t offset, const __local char *p); -uchar3 __ovld vload3(size_t offset, const __local uchar *p); -short3 __ovld vload3(size_t offset, const __local short *p); -ushort3 __ovld vload3(size_t offset, const __local ushort *p); -int3 __ovld vload3(size_t offset, const __local int *p); -uint3 __ovld vload3(size_t offset, const __local uint *p); -long3 __ovld vload3(size_t offset, const __local long *p); -ulong3 __ovld vload3(size_t offset, const __local ulong *p); -float3 __ovld vload3(size_t offset, const __local float *p); -char4 __ovld vload4(size_t offset, const __local char *p); -uchar4 __ovld vload4(size_t offset, const __local uchar *p); -short4 __ovld vload4(size_t offset, const __local short *p); -ushort4 __ovld vload4(size_t offset, const __local ushort *p); -int4 __ovld vload4(size_t offset, const __local int *p); -uint4 __ovld vload4(size_t offset, const __local uint *p); -long4 __ovld vload4(size_t offset, const __local long *p); -ulong4 __ovld vload4(size_t offset, const __local ulong *p); -float4 __ovld vload4(size_t offset, const __local float *p); -char8 __ovld vload8(size_t offset, const __local char *p); -uchar8 __ovld vload8(size_t offset, const __local uchar *p); -short8 __ovld vload8(size_t offset, const __local short *p); -ushort8 __ovld vload8(size_t offset, const __local ushort *p); -int8 __ovld vload8(size_t offset, const __local int *p); -uint8 __ovld vload8(size_t offset, const __local uint *p); -long8 __ovld vload8(size_t offset, const __local long *p); -ulong8 __ovld vload8(size_t offset, const __local ulong *p); -float8 __ovld vload8(size_t offset, const __local float *p); -char16 __ovld vload16(size_t offset, const __local char *p); -uchar16 __ovld vload16(size_t offset, const __local uchar *p); -short16 __ovld vload16(size_t offset, const __local short *p); -ushort16 __ovld vload16(size_t offset, const __local ushort *p); -int16 __ovld vload16(size_t offset, const __local int *p); -uint16 __ovld vload16(size_t offset, const __local uint *p); -long16 __ovld vload16(size_t offset, const __local long *p); -ulong16 __ovld vload16(size_t offset, const __local ulong *p); -float16 __ovld vload16(size_t offset, const __local float *p); -char2 __ovld vload2(size_t offset, const __private char *p); -uchar2 __ovld vload2(size_t offset, const __private uchar *p); -short2 __ovld vload2(size_t offset, const __private short *p); -ushort2 __ovld vload2(size_t offset, const __private ushort *p); -int2 __ovld vload2(size_t offset, const __private int *p); -uint2 __ovld vload2(size_t offset, const __private uint *p); -long2 __ovld vload2(size_t offset, const __private long *p); -ulong2 __ovld vload2(size_t offset, const __private ulong *p); -float2 __ovld vload2(size_t offset, const __private float *p); -char3 __ovld vload3(size_t offset, const __private char *p); -uchar3 __ovld vload3(size_t offset, const __private uchar *p); -short3 __ovld vload3(size_t offset, const __private short *p); -ushort3 __ovld vload3(size_t offset, const __private ushort *p); -int3 __ovld vload3(size_t offset, const __private int *p); -uint3 __ovld vload3(size_t offset, const __private uint *p); -long3 __ovld vload3(size_t offset, const __private long *p); -ulong3 __ovld vload3(size_t offset, const __private ulong *p); -float3 __ovld vload3(size_t offset, const __private float *p); -char4 __ovld vload4(size_t offset, const __private char *p); -uchar4 __ovld vload4(size_t offset, const __private uchar *p); -short4 __ovld vload4(size_t offset, const __private short *p); -ushort4 __ovld vload4(size_t offset, const __private ushort *p); -int4 __ovld vload4(size_t offset, const __private int *p); -uint4 __ovld vload4(size_t offset, const __private uint *p); -long4 __ovld vload4(size_t offset, const __private long *p); -ulong4 __ovld vload4(size_t offset, const __private ulong *p); -float4 __ovld vload4(size_t offset, const __private float *p); -char8 __ovld vload8(size_t offset, const __private char *p); -uchar8 __ovld vload8(size_t offset, const __private uchar *p); -short8 __ovld vload8(size_t offset, const __private short *p); -ushort8 __ovld vload8(size_t offset, const __private ushort *p); -int8 __ovld vload8(size_t offset, const __private int *p); -uint8 __ovld vload8(size_t offset, const __private uint *p); -long8 __ovld vload8(size_t offset, const __private long *p); -ulong8 __ovld vload8(size_t offset, const __private ulong *p); -float8 __ovld vload8(size_t offset, const __private float *p); -char16 __ovld vload16(size_t offset, const __private char *p); -uchar16 __ovld vload16(size_t offset, const __private uchar *p); -short16 __ovld vload16(size_t offset, const __private short *p); -ushort16 __ovld vload16(size_t offset, const __private ushort *p); -int16 __ovld vload16(size_t offset, const __private int *p); -uint16 __ovld vload16(size_t offset, const __private uint *p); -long16 __ovld vload16(size_t offset, const __private long *p); -ulong16 __ovld vload16(size_t offset, const __private ulong *p); -float16 __ovld vload16(size_t offset, const __private float *p); +char2 __ovld __purefn vload2(size_t offset, const __global char *p); +uchar2 __ovld __purefn vload2(size_t offset, const __global uchar *p); +short2 __ovld __purefn vload2(size_t offset, const __global short *p); +ushort2 __ovld __purefn vload2(size_t offset, const __global ushort *p); +int2 __ovld __purefn vload2(size_t offset, const __global int *p); +uint2 __ovld __purefn vload2(size_t offset, const __global uint *p); +long2 __ovld __purefn vload2(size_t offset, const __global long *p); +ulong2 __ovld __purefn vload2(size_t offset, const __global ulong *p); +float2 __ovld __purefn vload2(size_t offset, const __global float *p); +char3 __ovld __purefn vload3(size_t offset, const __global char *p); +uchar3 __ovld __purefn vload3(size_t offset, const __global uchar *p); +short3 __ovld __purefn vload3(size_t offset, const __global short *p); +ushort3 __ovld __purefn vload3(size_t offset, const __global ushort *p); +int3 __ovld __purefn vload3(size_t offset, const __global int *p); +uint3 __ovld __purefn vload3(size_t offset, const __global uint *p); +long3 __ovld __purefn vload3(size_t offset, const __global long *p); +ulong3 __ovld __purefn vload3(size_t offset, const __global ulong *p); +float3 __ovld __purefn vload3(size_t offset, const __global float *p); +char4 __ovld __purefn vload4(size_t offset, const __global char *p); +uchar4 __ovld __purefn vload4(size_t offset, const __global uchar *p); +short4 __ovld __purefn vload4(size_t offset, const __global short *p); +ushort4 __ovld __purefn vload4(size_t offset, const __global ushort *p); +int4 __ovld __purefn vload4(size_t offset, const __global int *p); +uint4 __ovld __purefn vload4(size_t offset, const __global uint *p); +long4 __ovld __purefn vload4(size_t offset, const __global long *p); +ulong4 __ovld __purefn vload4(size_t offset, const __global ulong *p); +float4 __ovld __purefn vload4(size_t offset, const __global float *p); +char8 __ovld __purefn vload8(size_t offset, const __global char *p); +uchar8 __ovld __purefn vload8(size_t offset, const __global uchar *p); +short8 __ovld __purefn vload8(size_t offset, const __global short *p); +ushort8 __ovld __purefn vload8(size_t offset, const __global ushort *p); +int8 __ovld __purefn vload8(size_t offset, const __global int *p); +uint8 __ovld __purefn vload8(size_t offset, const __global uint *p); +long8 __ovld __purefn vload8(size_t offset, const __global long *p); +ulong8 __ovld __purefn vload8(size_t offset, const __global ulong *p); +float8 __ovld __purefn vload8(size_t offset, const __global float *p); +char16 __ovld __purefn vload16(size_t offset, const __global char *p); +uchar16 __ovld __purefn vload16(size_t offset, const __global uchar *p); +short16 __ovld __purefn vload16(size_t offset, const __global short *p); +ushort16 __ovld __purefn vload16(size_t offset, const __global ushort *p); +int16 __ovld __purefn vload16(size_t offset, const __global int *p); +uint16 __ovld __purefn vload16(size_t offset, const __global uint *p); +long16 __ovld __purefn vload16(size_t offset, const __global long *p); +ulong16 __ovld __purefn vload16(size_t offset, const __global ulong *p); +float16 __ovld __purefn vload16(size_t offset, const __global float *p); +char2 __ovld __purefn vload2(size_t offset, const __local char *p); +uchar2 __ovld __purefn vload2(size_t offset, const __local uchar *p); +short2 __ovld __purefn vload2(size_t offset, const __local short *p); +ushort2 __ovld __purefn vload2(size_t offset, const __local ushort *p); +int2 __ovld __purefn vload2(size_t offset, const __local int *p); +uint2 __ovld __purefn vload2(size_t offset, const __local uint *p); +long2 __ovld __purefn vload2(size_t offset, const __local long *p); +ulong2 __ovld __purefn vload2(size_t offset, const __local ulong *p); +float2 __ovld __purefn vload2(size_t offset, const __local float *p); +char3 __ovld __purefn vload3(size_t offset, const __local char *p); +uchar3 __ovld __purefn vload3(size_t offset, const __local uchar *p); +short3 __ovld __purefn vload3(size_t offset, const __local short *p); +ushort3 __ovld __purefn vload3(size_t offset, const __local ushort *p); +int3 __ovld __purefn vload3(size_t offset, const __local int *p); +uint3 __ovld __purefn vload3(size_t offset, const __local uint *p); +long3 __ovld __purefn vload3(size_t offset, const __local long *p); +ulong3 __ovld __purefn vload3(size_t offset, const __local ulong *p); +float3 __ovld __purefn vload3(size_t offset, const __local float *p); +char4 __ovld __purefn vload4(size_t offset, const __local char *p); +uchar4 __ovld __purefn vload4(size_t offset, const __local uchar *p); +short4 __ovld __purefn vload4(size_t offset, const __local short *p); +ushort4 __ovld __purefn vload4(size_t offset, const __local ushort *p); +int4 __ovld __purefn vload4(size_t offset, const __local int *p); +uint4 __ovld __purefn vload4(size_t offset, const __local uint *p); +long4 __ovld __purefn vload4(size_t offset, const __local long *p); +ulong4 __ovld __purefn vload4(size_t offset, const __local ulong *p); +float4 __ovld __purefn vload4(size_t offset, const __local float *p); +char8 __ovld __purefn vload8(size_t offset, const __local char *p); +uchar8 __ovld __purefn vload8(size_t offset, const __local uchar *p); +short8 __ovld __purefn vload8(size_t offset, const __local short *p); +ushort8 __ovld __purefn vload8(size_t offset, const __local ushort *p); +int8 __ovld __purefn vload8(size_t offset, const __local int *p); +uint8 __ovld __purefn vload8(size_t offset, const __local uint *p); +long8 __ovld __purefn vload8(size_t offset, const __local long *p); +ulong8 __ovld __purefn vload8(size_t offset, const __local ulong *p); +float8 __ovld __purefn vload8(size_t offset, const __local float *p); +char16 __ovld __purefn vload16(size_t offset, const __local char *p); +uchar16 __ovld __purefn vload16(size_t offset, const __local uchar *p); +short16 __ovld __purefn vload16(size_t offset, const __local short *p); +ushort16 __ovld __purefn vload16(size_t offset, const __local ushort *p); +int16 __ovld __purefn vload16(size_t offset, const __local int *p); +uint16 __ovld __purefn vload16(size_t offset, const __local uint *p); +long16 __ovld __purefn vload16(size_t offset, const __local long *p); +ulong16 __ovld __purefn vload16(size_t offset, const __local ulong *p); +float16 __ovld __purefn vload16(size_t offset, const __local float *p); +char2 __ovld __purefn vload2(size_t offset, const __private char *p); +uchar2 __ovld __purefn vload2(size_t offset, const __private uchar *p); +short2 __ovld __purefn vload2(size_t offset, const __private short *p); +ushort2 __ovld __purefn vload2(size_t offset, const __private ushort *p); +int2 __ovld __purefn vload2(size_t offset, const __private int *p); +uint2 __ovld __purefn vload2(size_t offset, const __private uint *p); +long2 __ovld __purefn vload2(size_t offset, const __private long *p); +ulong2 __ovld __purefn vload2(size_t offset, const __private ulong *p); +float2 __ovld __purefn vload2(size_t offset, const __private float *p); +char3 __ovld __purefn vload3(size_t offset, const __private char *p); +uchar3 __ovld __purefn vload3(size_t offset, const __private uchar *p); +short3 __ovld __purefn vload3(size_t offset, const __private short *p); +ushort3 __ovld __purefn vload3(size_t offset, const __private ushort *p); +int3 __ovld __purefn vload3(size_t offset, const __private int *p); +uint3 __ovld __purefn vload3(size_t offset, const __private uint *p); +long3 __ovld __purefn vload3(size_t offset, const __private long *p); +ulong3 __ovld __purefn vload3(size_t offset, const __private ulong *p); +float3 __ovld __purefn vload3(size_t offset, const __private float *p); +char4 __ovld __purefn vload4(size_t offset, const __private char *p); +uchar4 __ovld __purefn vload4(size_t offset, const __private uchar *p); +short4 __ovld __purefn vload4(size_t offset, const __private short *p); +ushort4 __ovld __purefn vload4(size_t offset, const __private ushort *p); +int4 __ovld __purefn vload4(size_t offset, const __private int *p); +uint4 __ovld __purefn vload4(size_t offset, const __private uint *p); +long4 __ovld __purefn vload4(size_t offset, const __private long *p); +ulong4 __ovld __purefn vload4(size_t offset, const __private ulong *p); +float4 __ovld __purefn vload4(size_t offset, const __private float *p); +char8 __ovld __purefn vload8(size_t offset, const __private char *p); +uchar8 __ovld __purefn vload8(size_t offset, const __private uchar *p); +short8 __ovld __purefn vload8(size_t offset, const __private short *p); +ushort8 __ovld __purefn vload8(size_t offset, const __private ushort *p); +int8 __ovld __purefn vload8(size_t offset, const __private int *p); +uint8 __ovld __purefn vload8(size_t offset, const __private uint *p); +long8 __ovld __purefn vload8(size_t offset, const __private long *p); +ulong8 __ovld __purefn vload8(size_t offset, const __private ulong *p); +float8 __ovld __purefn vload8(size_t offset, const __private float *p); +char16 __ovld __purefn vload16(size_t offset, const __private char *p); +uchar16 __ovld __purefn vload16(size_t offset, const __private uchar *p); +short16 __ovld __purefn vload16(size_t offset, const __private short *p); +ushort16 __ovld __purefn vload16(size_t offset, const __private ushort *p); +int16 __ovld __purefn vload16(size_t offset, const __private int *p); +uint16 __ovld __purefn vload16(size_t offset, const __private uint *p); +long16 __ovld __purefn vload16(size_t offset, const __private long *p); +ulong16 __ovld __purefn vload16(size_t offset, const __private ulong *p); +float16 __ovld __purefn vload16(size_t offset, const __private float *p); #ifdef cl_khr_fp64 -double2 __ovld vload2(size_t offset, const __global double *p); -double3 __ovld vload3(size_t offset, const __global double *p); -double4 __ovld vload4(size_t offset, const __global double *p); -double8 __ovld vload8(size_t offset, const __global double *p); -double16 __ovld vload16(size_t offset, const __global double *p); -double2 __ovld vload2(size_t offset, const __local double *p); -double3 __ovld vload3(size_t offset, const __local double *p); -double4 __ovld vload4(size_t offset, const __local double *p); -double8 __ovld vload8(size_t offset, const __local double *p); -double16 __ovld vload16(size_t offset, const __local double *p); -double2 __ovld vload2(size_t offset, const __private double *p); -double3 __ovld vload3(size_t offset, const __private double *p); -double4 __ovld vload4(size_t offset, const __private double *p); -double8 __ovld vload8(size_t offset, const __private double *p); -double16 __ovld vload16(size_t offset, const __private double *p); +double2 __ovld __purefn vload2(size_t offset, const __global double *p); +double3 __ovld __purefn vload3(size_t offset, const __global double *p); +double4 __ovld __purefn vload4(size_t offset, const __global double *p); +double8 __ovld __purefn vload8(size_t offset, const __global double *p); +double16 __ovld __purefn vload16(size_t offset, const __global double *p); +double2 __ovld __purefn vload2(size_t offset, const __local double *p); +double3 __ovld __purefn vload3(size_t offset, const __local double *p); +double4 __ovld __purefn vload4(size_t offset, const __local double *p); +double8 __ovld __purefn vload8(size_t offset, const __local double *p); +double16 __ovld __purefn vload16(size_t offset, const __local double *p); +double2 __ovld __purefn vload2(size_t offset, const __private double *p); +double3 __ovld __purefn vload3(size_t offset, const __private double *p); +double4 __ovld __purefn vload4(size_t offset, const __private double *p); +double8 __ovld __purefn vload8(size_t offset, const __private double *p); +double16 __ovld __purefn vload16(size_t offset, const __private double *p); #endif //cl_khr_fp64 #ifdef cl_khr_fp16 -half __ovld vload(size_t offset, const __global half *p); -half2 __ovld vload2(size_t offset, const __global half *p); -half3 __ovld vload3(size_t offset, const __global half *p); -half4 __ovld vload4(size_t offset, const __global half *p); -half8 __ovld vload8(size_t offset, const __global half *p); -half16 __ovld vload16(size_t offset, const __global half *p); -half __ovld vload(size_t offset, const __local half *p); -half2 __ovld vload2(size_t offset, const __local half *p); -half3 __ovld vload3(size_t offset, const __local half *p); -half4 __ovld vload4(size_t offset, const __local half *p); -half8 __ovld vload8(size_t offset, const __local half *p); -half16 __ovld vload16(size_t offset, const __local half *p); -half __ovld vload(size_t offset, const __private half *p); -half2 __ovld vload2(size_t offset, const __private half *p); -half3 __ovld vload3(size_t offset, const __private half *p); -half4 __ovld vload4(size_t offset, const __private half *p); -half8 __ovld vload8(size_t offset, const __private half *p); -half16 __ovld vload16(size_t offset, const __private half *p); +half __ovld __purefn vload(size_t offset, const __global half *p); +half2 __ovld __purefn vload2(size_t offset, const __global half *p); +half3 __ovld __purefn vload3(size_t offset, const __global half *p); +half4 __ovld __purefn vload4(size_t offset, const __global half *p); +half8 __ovld __purefn vload8(size_t offset, const __global half *p); +half16 __ovld __purefn vload16(size_t offset, const __global half *p); +half __ovld __purefn vload(size_t offset, const __local half *p); +half2 __ovld __purefn vload2(size_t offset, const __local half *p); +half3 __ovld __purefn vload3(size_t offset, const __local half *p); +half4 __ovld __purefn vload4(size_t offset, const __local half *p); +half8 __ovld __purefn vload8(size_t offset, const __local half *p); +half16 __ovld __purefn vload16(size_t offset, const __local half *p); +half __ovld __purefn vload(size_t offset, const __private half *p); +half2 __ovld __purefn vload2(size_t offset, const __private half *p); +half3 __ovld __purefn vload3(size_t offset, const __private half *p); +half4 __ovld __purefn vload4(size_t offset, const __private half *p); +half8 __ovld __purefn vload8(size_t offset, const __private half *p); +half16 __ovld __purefn vload16(size_t offset, const __private half *p); #endif //cl_khr_fp16 #endif //defined(__opencl_c_generic_address_space) @@ -11736,13 +11736,13 @@ void __ovld vstore16(half16 data, size_t offset, __private half *p); * The read address computed as (p + offset) * must be 16-bit aligned. */ -float __ovld vload_half(size_t offset, const __constant half *p); +float __ovld __purefn vload_half(size_t offset, const __constant half *p); #if defined(__opencl_c_generic_address_space) -float __ovld vload_half(size_t offset, const half *p); +float __ovld __purefn vload_half(size_t offset, const half *p); #else -float __ovld vload_half(size_t offset, const __global half *p); -float __ovld vload_half(size_t offset, const __local half *p); -float __ovld vload_half(size_t offset, const __private half *p); +float __ovld __purefn vload_half(size_t offset, const __global half *p); +float __ovld __purefn vload_half(size_t offset, const __local half *p); +float __ovld __purefn vload_half(size_t offset, const __private half *p); #endif //defined(__opencl_c_generic_address_space) /** @@ -11753,33 +11753,33 @@ float __ovld vload_half(size_t offset, const __private half *p); * value is returned. The read address computed * as (p + (offset * n)) must be 16-bit aligned. */ -float2 __ovld vload_half2(size_t offset, const __constant half *p); -float3 __ovld vload_half3(size_t offset, const __constant half *p); -float4 __ovld vload_half4(size_t offset, const __constant half *p); -float8 __ovld vload_half8(size_t offset, const __constant half *p); -float16 __ovld vload_half16(size_t offset, const __constant half *p); +float2 __ovld __purefn vload_half2(size_t offset, const __constant half *p); +float3 __ovld __purefn vload_half3(size_t offset, const __constant half *p); +float4 __ovld __purefn vload_half4(size_t offset, const __constant half *p); +float8 __ovld __purefn vload_half8(size_t offset, const __constant half *p); +float16 __ovld __purefn vload_half16(size_t offset, const __constant half *p); #if defined(__opencl_c_generic_address_space) -float2 __ovld vload_half2(size_t offset, const half *p); -float3 __ovld vload_half3(size_t offset, const half *p); -float4 __ovld vload_half4(size_t offset, const half *p); -float8 __ovld vload_half8(size_t offset, const half *p); -float16 __ovld vload_half16(size_t offset, const half *p); +float2 __ovld __purefn vload_half2(size_t offset, const half *p); +float3 __ovld __purefn vload_half3(size_t offset, const half *p); +float4 __ovld __purefn vload_half4(size_t offset, const half *p); +float8 __ovld __purefn vload_half8(size_t offset, const half *p); +float16 __ovld __purefn vload_half16(size_t offset, const half *p); #else -float2 __ovld vload_half2(size_t offset, const __global half *p); -float3 __ovld vload_half3(size_t offset, const __global half *p); -float4 __ovld vload_half4(size_t offset, const __global half *p); -float8 __ovld vload_half8(size_t offset, const __global half *p); -float16 __ovld vload_half16(size_t offset, const __global half *p); -float2 __ovld vload_half2(size_t offset, const __local half *p); -float3 __ovld vload_half3(size_t offset, const __local half *p); -float4 __ovld vload_half4(size_t offset, const __local half *p); -float8 __ovld vload_half8(size_t offset, const __local half *p); -float16 __ovld vload_half16(size_t offset, const __local half *p); -float2 __ovld vload_half2(size_t offset, const __private half *p); -float3 __ovld vload_half3(size_t offset, const __private half *p); -float4 __ovld vload_half4(size_t offset, const __private half *p); -float8 __ovld vload_half8(size_t offset, const __private half *p); -float16 __ovld vload_half16(size_t offset, const __private half *p); +float2 __ovld __purefn vload_half2(size_t offset, const __global half *p); +float3 __ovld __purefn vload_half3(size_t offset, const __global half *p); +float4 __ovld __purefn vload_half4(size_t offset, const __global half *p); +float8 __ovld __purefn vload_half8(size_t offset, const __global half *p); +float16 __ovld __purefn vload_half16(size_t offset, const __global half *p); +float2 __ovld __purefn vload_half2(size_t offset, const __local half *p); +float3 __ovld __purefn vload_half3(size_t offset, const __local half *p); +float4 __ovld __purefn vload_half4(size_t offset, const __local half *p); +float8 __ovld __purefn vload_half8(size_t offset, const __local half *p); +float16 __ovld __purefn vload_half16(size_t offset, const __local half *p); +float2 __ovld __purefn vload_half2(size_t offset, const __private half *p); +float3 __ovld __purefn vload_half3(size_t offset, const __private half *p); +float4 __ovld __purefn vload_half4(size_t offset, const __private half *p); +float8 __ovld __purefn vload_half8(size_t offset, const __private half *p); +float16 __ovld __purefn vload_half16(size_t offset, const __private half *p); #endif //defined(__opencl_c_generic_address_space) /** @@ -12073,33 +12073,33 @@ void __ovld vstore_half16_rtn(double16 data, size_t offset, __private half *p); * The address computed as (p + (offset * 4)) * must be aligned to sizeof (half) * 4 bytes. */ -float2 __ovld vloada_half2(size_t offset, const __constant half *p); -float3 __ovld vloada_half3(size_t offset, const __constant half *p); -float4 __ovld vloada_half4(size_t offset, const __constant half *p); -float8 __ovld vloada_half8(size_t offset, const __constant half *p); -float16 __ovld vloada_half16(size_t offset, const __constant half *p); +float2 __ovld __purefn vloada_half2(size_t offset, const __constant half *p); +float3 __ovld __purefn vloada_half3(size_t offset, const __constant half *p); +float4 __ovld __purefn vloada_half4(size_t offset, const __constant half *p); +float8 __ovld __purefn vloada_half8(size_t offset, const __constant half *p); +float16 __ovld __purefn vloada_half16(size_t offset, const __constant half *p); #if defined(__opencl_c_generic_address_space) -float2 __ovld vloada_half2(size_t offset, const half *p); -float3 __ovld vloada_half3(size_t offset, const half *p); -float4 __ovld vloada_half4(size_t offset, const half *p); -float8 __ovld vloada_half8(size_t offset, const half *p); -float16 __ovld vloada_half16(size_t offset, const half *p); +float2 __ovld __purefn vloada_half2(size_t offset, const half *p); +float3 __ovld __purefn vloada_half3(size_t offset, const half *p); +float4 __ovld __purefn vloada_half4(size_t offset, const half *p); +float8 __ovld __purefn vloada_half8(size_t offset, const half *p); +float16 __ovld __purefn vloada_half16(size_t offset, const half *p); #else -float2 __ovld vloada_half2(size_t offset, const __global half *p); -float3 __ovld vloada_half3(size_t offset, const __global half *p); -float4 __ovld vloada_half4(size_t offset, const __global half *p); -float8 __ovld vloada_half8(size_t offset, const __global half *p); -float16 __ovld vloada_half16(size_t offset, const __global half *p); -float2 __ovld vloada_half2(size_t offset, const __local half *p); -float3 __ovld vloada_half3(size_t offset, const __local half *p); -float4 __ovld vloada_half4(size_t offset, const __local half *p); -float8 __ovld vloada_half8(size_t offset, const __local half *p); -float16 __ovld vloada_half16(size_t offset, const __local half *p); -float2 __ovld vloada_half2(size_t offset, const __private half *p); -float3 __ovld vloada_half3(size_t offset, const __private half *p); -float4 __ovld vloada_half4(size_t offset, const __private half *p); -float8 __ovld vloada_half8(size_t offset, const __private half *p); -float16 __ovld vloada_half16(size_t offset, const __private half *p); +float2 __ovld __purefn vloada_half2(size_t offset, const __global half *p); +float3 __ovld __purefn vloada_half3(size_t offset, const __global half *p); +float4 __ovld __purefn vloada_half4(size_t offset, const __global half *p); +float8 __ovld __purefn vloada_half8(size_t offset, const __global half *p); +float16 __ovld __purefn vloada_half16(size_t offset, const __global half *p); +float2 __ovld __purefn vloada_half2(size_t offset, const __local half *p); +float3 __ovld __purefn vloada_half3(size_t offset, const __local half *p); +float4 __ovld __purefn vloada_half4(size_t offset, const __local half *p); +float8 __ovld __purefn vloada_half8(size_t offset, const __local half *p); +float16 __ovld __purefn vloada_half16(size_t offset, const __local half *p); +float2 __ovld __purefn vloada_half2(size_t offset, const __private half *p); +float3 __ovld __purefn vloada_half3(size_t offset, const __private half *p); +float4 __ovld __purefn vloada_half4(size_t offset, const __private half *p); +float8 __ovld __purefn vloada_half8(size_t offset, const __private half *p); +float16 __ovld __purefn vloada_half16(size_t offset, const __private half *p); #endif //defined(__opencl_c_generic_address_space) /** diff --git a/clang/lib/Headers/unwind.h b/clang/lib/Headers/unwind.h index 029524b7bc84..6e069798f02d 100644 --- a/clang/lib/Headers/unwind.h +++ b/clang/lib/Headers/unwind.h @@ -172,7 +172,8 @@ typedef enum { _UVRSC_CORE = 0, /* integer register */ _UVRSC_VFP = 1, /* vfp */ _UVRSC_WMMXD = 3, /* Intel WMMX data register */ - _UVRSC_WMMXC = 4 /* Intel WMMX control register */ + _UVRSC_WMMXC = 4, /* Intel WMMX control register */ + _UVRSC_PSEUDO = 5 /* Special purpose pseudo register */ } _Unwind_VRS_RegClass; typedef enum { diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 9fa170410da3..0b136aeb580f 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -832,12 +832,16 @@ std::pair ModuleMap::findOrCreateModule(StringRef Name, return std::make_pair(Result, true); } -Module *ModuleMap::createGlobalModuleFragmentForModuleUnit(SourceLocation Loc) { - PendingSubmodules.emplace_back( - new Module("", Loc, nullptr, /*IsFramework*/ false, - /*IsExplicit*/ true, NumCreatedModules++)); - PendingSubmodules.back()->Kind = Module::GlobalModuleFragment; - return PendingSubmodules.back().get(); +Module *ModuleMap::createGlobalModuleFragmentForModuleUnit(SourceLocation Loc, + Module *Parent) { + auto *Result = new Module("", Loc, Parent, /*IsFramework*/ false, + /*IsExplicit*/ true, NumCreatedModules++); + Result->Kind = Module::GlobalModuleFragment; + // If the created module isn't owned by a parent, send it to PendingSubmodules + // to wait for its parent. + if (!Result->Parent) + PendingSubmodules.emplace_back(Result); + return Result; } Module * diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp index 41e7f3f1dccb..e71a65f031e4 100644 --- a/clang/lib/Lex/TokenLexer.cpp +++ b/clang/lib/Lex/TokenLexer.cpp @@ -472,11 +472,9 @@ void TokenLexer::ExpandFunctionArguments() { // If the '##' came from expanding an argument, turn it into 'unknown' // to avoid pasting. - for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult, - ResultToks.end())) { + for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult)) if (Tok.is(tok::hashhash)) Tok.setKind(tok::unknown); - } if(ExpandLocStart.isValid()) { updateLocForMacroArgTokens(CurTok.getLocation(), diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp index 116724a0d50b..19cddc69ebfc 100644 --- a/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -452,13 +452,14 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) { CXXMethodDecl *Method; if (FunctionTemplateDecl *FunTmpl = dyn_cast(LM.Method)) - Method = cast(FunTmpl->getTemplatedDecl()); + Method = dyn_cast(FunTmpl->getTemplatedDecl()); else - Method = cast(LM.Method); + Method = dyn_cast(LM.Method); - Sema::CXXThisScopeRAII ThisScope(Actions, Method->getParent(), - Method->getMethodQualifiers(), - getLangOpts().CPlusPlus11); + Sema::CXXThisScopeRAII ThisScope( + Actions, Method ? Method->getParent() : nullptr, + Method ? Method->getMethodQualifiers() : Qualifiers{}, + Method && getLangOpts().CPlusPlus11); // Parse the exception-specification. SourceRange SpecificationRange; diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 1bdeccc4cbf5..0c1f88bc51d1 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -2891,7 +2891,8 @@ void Parser::ParseAlignmentSpecifier(ParsedAttributes &Attrs, } ExprResult Parser::ParseExtIntegerArgument() { - assert(Tok.is(tok::kw__ExtInt) && "Not an extended int type"); + assert(Tok.isOneOf(tok::kw__ExtInt, tok::kw__BitInt) && + "Not an extended int type"); ConsumeToken(); BalancedDelimiterTracker T(*this, tok::l_paren); @@ -3882,11 +3883,13 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, isInvalid = DS.SetTypeSpecType(DeclSpec::TST_int, Loc, PrevSpec, DiagID, Policy); break; - case tok::kw__ExtInt: { + case tok::kw__ExtInt: + case tok::kw__BitInt: { + DiagnoseBitIntUse(Tok); ExprResult ER = ParseExtIntegerArgument(); if (ER.isInvalid()) continue; - isInvalid = DS.SetExtIntType(Loc, ER.get(), PrevSpec, DiagID, Policy); + isInvalid = DS.SetBitIntType(Loc, ER.get(), PrevSpec, DiagID, Policy); ConsumedEnd = PrevTokLocation; break; } @@ -5015,6 +5018,7 @@ bool Parser::isKnownToBeTypeSpecifier(const Token &Tok) const { case tok::kw_char32_t: case tok::kw_int: case tok::kw__ExtInt: + case tok::kw__BitInt: case tok::kw___bf16: case tok::kw_half: case tok::kw_float: @@ -5097,6 +5101,7 @@ bool Parser::isTypeSpecifierQualifier() { case tok::kw_char32_t: case tok::kw_int: case tok::kw__ExtInt: + case tok::kw__BitInt: case tok::kw_half: case tok::kw___bf16: case tok::kw_float: @@ -5268,6 +5273,7 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) { case tok::kw_int: case tok::kw__ExtInt: + case tok::kw__BitInt: case tok::kw_half: case tok::kw___bf16: case tok::kw_float: @@ -7476,3 +7482,24 @@ bool Parser::TryAltiVecTokenOutOfLine(DeclSpec &DS, SourceLocation Loc, } return false; } + +void Parser::DiagnoseBitIntUse(const Token &Tok) { + // If the token is for _ExtInt, diagnose it as being deprecated. Otherwise, + // the token is about _BitInt and gets (potentially) diagnosed as use of an + // extension. + assert(Tok.isOneOf(tok::kw__ExtInt, tok::kw__BitInt) && + "expected either an _ExtInt or _BitInt token!"); + + SourceLocation Loc = Tok.getLocation(); + if (Tok.is(tok::kw__ExtInt)) { + Diag(Loc, diag::warn_ext_int_deprecated) + << FixItHint::CreateReplacement(Loc, "_BitInt"); + } else { + // In C2x mode, diagnose that the use is not compatible with pre-C2x modes. + // Otherwise, diagnose that the use is a Clang extension. + if (getLangOpts().C2x) + Diag(Loc, diag::warn_c17_compat_bit_int); + else + Diag(Loc, diag::ext_bit_int) << getLangOpts().CPlusPlus; + } +} diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index 2c8b4f9f441f..09a3842f5809 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -1513,6 +1513,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, case tok::kw___int64: case tok::kw___int128: case tok::kw__ExtInt: + case tok::kw__BitInt: case tok::kw_signed: case tok::kw_unsigned: case tok::kw_half: diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 4e5c0ac6c1c1..76c510ddd36c 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -2191,12 +2191,14 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) { return; } - case tok::kw__ExtInt: { + case tok::kw__ExtInt: + case tok::kw__BitInt: { + DiagnoseBitIntUse(Tok); ExprResult ER = ParseExtIntegerArgument(); if (ER.isInvalid()) DS.SetTypeSpecError(); else - DS.SetExtIntType(Loc, ER.get(), PrevSpec, DiagID, Policy); + DS.SetBitIntType(Loc, ER.get(), PrevSpec, DiagID, Policy); // Do this here because we have already consumed the close paren. DS.SetRangeEnd(PrevTokLocation); diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp index 613ad742c93f..300b022d83b9 100644 --- a/clang/lib/Parse/ParseOpenMP.cpp +++ b/clang/lib/Parse/ParseOpenMP.cpp @@ -3192,6 +3192,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind, case OMPC_read: case OMPC_write: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp index be3823ecda01..35c9036fb27e 100644 --- a/clang/lib/Parse/ParseTentative.cpp +++ b/clang/lib/Parse/ParseTentative.cpp @@ -1690,6 +1690,7 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult, case tok::kw__Atomic: return TPResult::True; + case tok::kw__BitInt: case tok::kw__ExtInt: { if (NextToken().isNot(tok::l_paren)) return TPResult::Error; @@ -1741,6 +1742,7 @@ bool Parser::isCXXDeclarationSpecifierAType() { case tok::kw_short: case tok::kw_int: case tok::kw__ExtInt: + case tok::kw__BitInt: case tok::kw_long: case tok::kw___int64: case tok::kw___int128: diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp index 371557a624c9..e9b678b69594 100644 --- a/clang/lib/Rewrite/HTMLRewrite.cpp +++ b/clang/lib/Rewrite/HTMLRewrite.cpp @@ -203,7 +203,7 @@ std::string html::EscapeText(StringRef s, bool EscapeSpaces, bool ReplaceTabs) { } } - return os.str(); + return Str; } static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo, diff --git a/clang/lib/Sema/CodeCompleteConsumer.cpp b/clang/lib/Sema/CodeCompleteConsumer.cpp index 6c47cedfccf3..0a2ca54e244a 100644 --- a/clang/lib/Sema/CodeCompleteConsumer.cpp +++ b/clang/lib/Sema/CodeCompleteConsumer.cpp @@ -335,7 +335,7 @@ std::string CodeCompletionString::getAsString() const { break; } } - return OS.str(); + return Result; } const char *CodeCompletionString::getTypedText() const { @@ -640,7 +640,7 @@ static std::string getOverloadAsString(const CodeCompletionString &CCS) { break; } } - return OS.str(); + return Result; } void PrintingCodeCompleteConsumer::ProcessOverloadCandidates( diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp index 4405f29f3d99..d4dc790c008a 100644 --- a/clang/lib/Sema/DeclSpec.cpp +++ b/clang/lib/Sema/DeclSpec.cpp @@ -365,7 +365,7 @@ bool Declarator::isDeclarationOfFunction() const { case TST_half: case TST_int: case TST_int128: - case TST_extint: + case TST_bitint: case TST_struct: case TST_interface: case TST_union: @@ -551,7 +551,7 @@ const char *DeclSpec::getSpecifierName(DeclSpec::TST T, case DeclSpec::TST_char32: return "char32_t"; case DeclSpec::TST_int: return "int"; case DeclSpec::TST_int128: return "__int128"; - case DeclSpec::TST_extint: return "_ExtInt"; + case DeclSpec::TST_bitint: return "_BitInt"; case DeclSpec::TST_half: return "half"; case DeclSpec::TST_float: return "float"; case DeclSpec::TST_double: return "double"; @@ -932,7 +932,7 @@ bool DeclSpec::SetTypeSpecError() { return false; } -bool DeclSpec::SetExtIntType(SourceLocation KWLoc, Expr *BitsExpr, +bool DeclSpec::SetBitIntType(SourceLocation KWLoc, Expr *BitsExpr, const char *&PrevSpec, unsigned &DiagID, const PrintingPolicy &Policy) { assert(BitsExpr && "no expression provided!"); @@ -945,7 +945,7 @@ bool DeclSpec::SetExtIntType(SourceLocation KWLoc, Expr *BitsExpr, return true; } - TypeSpecType = TST_extint; + TypeSpecType = TST_bitint; ExprRep = BitsExpr; TSTLoc = KWLoc; TSTNameLoc = KWLoc; @@ -1252,7 +1252,7 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) { TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int. else if (TypeSpecType != TST_int && TypeSpecType != TST_int128 && TypeSpecType != TST_char && TypeSpecType != TST_wchar && - !IsFixedPointType && TypeSpecType != TST_extint) { + !IsFixedPointType && TypeSpecType != TST_bitint) { S.Diag(TSSLoc, diag::err_invalid_sign_spec) << getSpecifierName((TST)TypeSpecType, Policy); // signed double -> double. @@ -1302,7 +1302,7 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) { " double"); TypeSpecType = TST_double; // _Complex -> _Complex double. } else if (TypeSpecType == TST_int || TypeSpecType == TST_char || - TypeSpecType == TST_extint) { + TypeSpecType == TST_bitint) { // Note that this intentionally doesn't include _Complex _Bool. if (!S.getLangOpts().CPlusPlus) S.Diag(TSTLoc, diag::ext_integer_complex); diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td index 8cf7ec58eff5..38debc5aa9fc 100644 --- a/clang/lib/Sema/OpenCLBuiltins.td +++ b/clang/lib/Sema/OpenCLBuiltins.td @@ -806,17 +806,17 @@ multiclass VloadVstore addrspaces, bit defStores> { foreach AS = addrspaces in { foreach VSize = [2, 3, 4, 8, 16] in { foreach name = ["vload" # VSize] in { - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; - def : Builtin, Size, PointerType, AS>]>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; } if defStores then { foreach name = ["vstore" # VSize] in { @@ -848,10 +848,10 @@ defm : VloadVstore<[ConstantAS], 0>; multiclass VloadVstoreHalf addrspaces, bit defStores> { foreach AS = addrspaces in { - def : Builtin<"vload_half", [Float, Size, PointerType, AS>]>; + def : Builtin<"vload_half", [Float, Size, PointerType, AS>], Attr.Pure>; foreach VSize = [2, 3, 4, 8, 16] in { foreach name = ["vload_half" # VSize, "vloada_half" # VSize] in { - def : Builtin, Size, PointerType, AS>]>; + def : Builtin, Size, PointerType, AS>], Attr.Pure>; } } if defStores then { @@ -877,7 +877,7 @@ let MaxVersion = CL20 in { let MinVersion = CL20 in { defm : VloadVstoreHalf<[GenericAS], 1>; } -// vload with constant address space is available regardless of version. +// vload_half and vloada_half with constant address space are available regardless of version. defm : VloadVstoreHalf<[ConstantAS], 0>; // OpenCL v3.0 s6.15.8 - Synchronization Functions. diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index a2b8f475aa8c..734ed0f62ec6 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -1881,8 +1881,8 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) { if (Ty->isDependentType()) return; - if (Ty->isExtIntType()) { - if (!Context.getTargetInfo().hasExtIntType()) { + if (Ty->isBitIntType()) { + if (!Context.getTargetInfo().hasBitIntType()) { PartialDiagnostic PD = PDiag(diag::err_target_unsupported_type); if (D) PD << D; diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp index 100f8e36a9b8..b69492768848 100644 --- a/clang/lib/Sema/SemaAttr.cpp +++ b/clang/lib/Sema/SemaAttr.cpp @@ -792,7 +792,7 @@ attrMatcherRuleListToString(ArrayRef Rules) { OS << (I.index() == Rules.size() - 1 ? ", and " : ", "); OS << "'" << attr::getSubjectMatchRuleSpelling(I.value()) << "'"; } - return OS.str(); + return Result; } } // end anonymous namespace diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp index 840b3daae63c..59601c5ce79d 100644 --- a/clang/lib/Sema/SemaCUDA.cpp +++ b/clang/lib/Sema/SemaCUDA.cpp @@ -886,7 +886,6 @@ void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee, diag::warn_maybe_capture_bad_target_this_ptr, Callee, *this); } - return; } void Sema::CUDASetLambdaAttrs(CXXMethodDecl *Method) { diff --git a/clang/lib/Sema/SemaCXXScopeSpec.cpp b/clang/lib/Sema/SemaCXXScopeSpec.cpp index c4826b5a6e8f..8cecf6c6ab4f 100644 --- a/clang/lib/Sema/SemaCXXScopeSpec.cpp +++ b/clang/lib/Sema/SemaCXXScopeSpec.cpp @@ -736,8 +736,15 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo, QualType T = Context.getTypeDeclType(cast(SD->getUnderlyingDecl())); + + if (T->isEnumeralType()) + Diag(IdInfo.IdentifierLoc, diag::warn_cxx98_compat_enum_nested_name_spec); + TypeLocBuilder TLB; - if (isa(T)) { + if (const auto *USD = dyn_cast(SD)) { + T = Context.getUsingType(USD, T); + TLB.pushTypeSpec(T).setNameLoc(IdInfo.IdentifierLoc); + } else if (isa(T)) { InjectedClassNameTypeLoc InjectedTL = TLB.push(T); InjectedTL.setNameLoc(IdInfo.IdentifierLoc); @@ -770,9 +777,6 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo, llvm_unreachable("Unhandled TypeDecl node in nested-name-specifier"); } - if (T->isEnumeralType()) - Diag(IdInfo.IdentifierLoc, diag::warn_cxx98_compat_enum_nested_name_spec); - SS.Extend(Context, SourceLocation(), TLB.getTypeLocInContext(Context, T), IdInfo.CCLoc); return false; diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 33e2b3b5027d..4e83fa1fffca 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -195,6 +195,29 @@ static bool SemaBuiltinAddressof(Sema &S, CallExpr *TheCall) { return false; } +/// Check that the argument to __builtin_function_start is a function. +static bool SemaBuiltinFunctionStart(Sema &S, CallExpr *TheCall) { + if (checkArgCount(S, TheCall, 1)) + return true; + + ExprResult Arg = S.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0)); + if (Arg.isInvalid()) + return true; + + TheCall->setArg(0, Arg.get()); + const FunctionDecl *FD = dyn_cast_or_null( + Arg.get()->getAsBuiltinConstantDeclRef(S.getASTContext())); + + if (!FD) { + S.Diag(TheCall->getBeginLoc(), diag::err_function_start_invalid_type) + << TheCall->getSourceRange(); + return true; + } + + return !S.checkAddressOfFunctionIsAvailable(FD, /*Complain=*/true, + TheCall->getBeginLoc()); +} + /// Check the number of arguments and set the result type to /// the argument type. static bool SemaBuiltinPreserveAI(Sema &S, CallExpr *TheCall) { @@ -325,17 +348,17 @@ static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall, } } - // Disallow signed ExtIntType args larger than 128 bits to mul function until - // we improve backend support. + // Disallow signed bit-precise integer args larger than 128 bits to mul + // function until we improve backend support. if (BuiltinID == Builtin::BI__builtin_mul_overflow) { for (unsigned I = 0; I < 3; ++I) { const auto Arg = TheCall->getArg(I); // Third argument will be a pointer. auto Ty = I < 2 ? Arg->getType() : Arg->getType()->getPointeeType(); - if (Ty->isExtIntType() && Ty->isSignedIntegerType() && + if (Ty->isBitIntType() && Ty->isSignedIntegerType() && S.getASTContext().getIntWidth(Ty) > 128) return S.Diag(Arg->getBeginLoc(), - diag::err_overflow_builtin_ext_int_max_size) + diag::err_overflow_builtin_bit_int_max_size) << 128; } } @@ -446,14 +469,14 @@ class ScanfDiagnosticFormatHandler break; } - auto OptionalFW = FS.getFieldWidth(); - if (OptionalFW.getHowSpecified() != + analyze_format_string::OptionalAmount FW = FS.getFieldWidth(); + if (FW.getHowSpecified() != analyze_format_string::OptionalAmount::HowSpecified::Constant) return true; - unsigned SourceSize = OptionalFW.getConstantAmount() + NulByte; + unsigned SourceSize = FW.getConstantAmount() + NulByte; - auto DestSizeAPS = ComputeSizeArgument(FS.getArgIndex()); + Optional DestSizeAPS = ComputeSizeArgument(FS.getArgIndex()); if (!DestSizeAPS) return true; @@ -652,20 +675,53 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD, isConstantEvaluated()) return; - unsigned BuiltinID = FD->getBuiltinID(/*ConsiderWrappers=*/true); + bool UseDABAttr = false; + const FunctionDecl *UseDecl = FD; + + const auto *DABAttr = FD->getAttr(); + if (DABAttr) { + UseDecl = DABAttr->getFunction(); + assert(UseDecl && "Missing FunctionDecl in DiagnoseAsBuiltin attribute!"); + UseDABAttr = true; + } + + unsigned BuiltinID = UseDecl->getBuiltinID(/*ConsiderWrappers=*/true); + if (!BuiltinID) return; const TargetInfo &TI = getASTContext().getTargetInfo(); unsigned SizeTypeWidth = TI.getTypeWidth(TI.getSizeType()); + auto TranslateIndex = [&](unsigned Index) -> Optional { + // If we refer to a diagnose_as_builtin attribute, we need to change the + // argument index to refer to the arguments of the called function. Unless + // the index is out of bounds, which presumably means it's a variadic + // function. + if (!UseDABAttr) + return Index; + unsigned DABIndices = DABAttr->argIndices_size(); + unsigned NewIndex = Index < DABIndices + ? DABAttr->argIndices_begin()[Index] + : Index - DABIndices + FD->getNumParams(); + if (NewIndex >= TheCall->getNumArgs()) + return llvm::None; + return NewIndex; + }; + auto ComputeExplicitObjectSizeArgument = [&](unsigned Index) -> Optional { + Optional IndexOptional = TranslateIndex(Index); + if (!IndexOptional) + return llvm::None; + unsigned NewIndex = IndexOptional.getValue(); Expr::EvalResult Result; - Expr *SizeArg = TheCall->getArg(Index); + Expr *SizeArg = TheCall->getArg(NewIndex); if (!SizeArg->EvaluateAsInt(Result, getASTContext())) return llvm::None; - return Result.Val.getInt(); + llvm::APSInt Integer = Result.Val.getInt(); + Integer.setIsUnsigned(true); + return Integer; }; auto ComputeSizeArgument = [&](unsigned Index) -> Optional { @@ -680,7 +736,12 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD, BOSType = POS->getType(); } - const Expr *ObjArg = TheCall->getArg(Index); + Optional IndexOptional = TranslateIndex(Index); + if (!IndexOptional) + return llvm::None; + unsigned NewIndex = IndexOptional.getValue(); + + const Expr *ObjArg = TheCall->getArg(NewIndex); uint64_t Result; if (!ObjArg->tryEvaluateObjectSize(Result, getASTContext(), BOSType)) return llvm::None; @@ -690,7 +751,12 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD, }; auto ComputeStrLenArgument = [&](unsigned Index) -> Optional { - Expr *ObjArg = TheCall->getArg(Index); + Optional IndexOptional = TranslateIndex(Index); + if (!IndexOptional) + return llvm::None; + unsigned NewIndex = IndexOptional.getValue(); + + const Expr *ObjArg = TheCall->getArg(NewIndex); uint64_t Result; if (!ObjArg->tryEvaluateStrLen(Result, getASTContext())) return llvm::None; @@ -898,7 +964,8 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD, } if (!SourceSize || !DestinationSize || - SourceSize.getValue().ule(DestinationSize.getValue())) + llvm::APSInt::compareValues(SourceSize.getValue(), + DestinationSize.getValue()) <= 0) return; StringRef FunctionName = GetFunctionName(); @@ -1874,6 +1941,10 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, if (SemaBuiltinAddressof(*this, TheCall)) return ExprError(); break; + case Builtin::BI__builtin_function_start: + if (SemaBuiltinFunctionStart(*this, TheCall)) + return ExprError(); + break; case Builtin::BI__builtin_is_aligned: case Builtin::BI__builtin_align_up: case Builtin::BI__builtin_align_down: @@ -2098,20 +2169,85 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, break; } - case Builtin::BI__builtin_elementwise_abs: - if (SemaBuiltinElementwiseMathOneArg(TheCall)) + // __builtin_elementwise_abs restricts the element type to signed integers or + // floating point types only. + case Builtin::BI__builtin_elementwise_abs: { + if (PrepareBuiltinElementwiseMathOneArgCall(TheCall)) return ExprError(); + + QualType ArgTy = TheCall->getArg(0)->getType(); + QualType EltTy = ArgTy; + + if (auto *VecTy = EltTy->getAs()) + EltTy = VecTy->getElementType(); + if (EltTy->isUnsignedIntegerType()) { + Diag(TheCall->getArg(0)->getBeginLoc(), + diag::err_builtin_invalid_arg_type) + << 1 << /* signed integer or float ty*/ 3 << ArgTy; + return ExprError(); + } break; + } + + // __builtin_elementwise_ceil restricts the element type to floating point + // types only. + case Builtin::BI__builtin_elementwise_ceil: { + if (PrepareBuiltinElementwiseMathOneArgCall(TheCall)) + return ExprError(); + + QualType ArgTy = TheCall->getArg(0)->getType(); + QualType EltTy = ArgTy; + + if (auto *VecTy = EltTy->getAs()) + EltTy = VecTy->getElementType(); + if (!EltTy->isFloatingType()) { + Diag(TheCall->getArg(0)->getBeginLoc(), + diag::err_builtin_invalid_arg_type) + << 1 << /* float ty*/ 5 << ArgTy; + + return ExprError(); + } + break; + } + case Builtin::BI__builtin_elementwise_min: case Builtin::BI__builtin_elementwise_max: if (SemaBuiltinElementwiseMath(TheCall)) return ExprError(); break; case Builtin::BI__builtin_reduce_max: - case Builtin::BI__builtin_reduce_min: - if (SemaBuiltinReduceMath(TheCall)) + case Builtin::BI__builtin_reduce_min: { + if (PrepareBuiltinReduceMathOneArgCall(TheCall)) return ExprError(); + + const Expr *Arg = TheCall->getArg(0); + const auto *TyA = Arg->getType()->getAs(); + if (!TyA) { + Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type) + << 1 << /* vector ty*/ 4 << Arg->getType(); + return ExprError(); + } + + TheCall->setType(TyA->getElementType()); break; + } + + // __builtin_reduce_xor supports vector of integers only. + case Builtin::BI__builtin_reduce_xor: { + if (PrepareBuiltinReduceMathOneArgCall(TheCall)) + return ExprError(); + + const Expr *Arg = TheCall->getArg(0); + const auto *TyA = Arg->getType()->getAs(); + if (!TyA || !TyA->getElementType()->isIntegerType()) { + Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type) + << 1 << /* vector of integers */ 6 << Arg->getType(); + return ExprError(); + } + TheCall->setType(TyA->getElementType()); + break; + } + case Builtin::BI__builtin_matrix_transpose: return SemaBuiltinMatrixTranspose(TheCall, TheCallResult); @@ -3496,14 +3632,43 @@ bool Sema::CheckPPCBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, case PPC::BI__builtin_altivec_dss: return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3); case PPC::BI__builtin_tbegin: - case PPC::BI__builtin_tend: i = 0; l = 0; u = 1; break; - case PPC::BI__builtin_tsr: i = 0; l = 0; u = 7; break; + case PPC::BI__builtin_tend: + return SemaBuiltinConstantArgRange(TheCall, 0, 0, 1) || + SemaFeatureCheck(*this, TheCall, "htm", + diag::err_ppc_builtin_requires_htm); + case PPC::BI__builtin_tsr: + return SemaBuiltinConstantArgRange(TheCall, 0, 0, 7) || + SemaFeatureCheck(*this, TheCall, "htm", + diag::err_ppc_builtin_requires_htm); case PPC::BI__builtin_tabortwc: - case PPC::BI__builtin_tabortdc: i = 0; l = 0; u = 31; break; + case PPC::BI__builtin_tabortdc: + return SemaBuiltinConstantArgRange(TheCall, 0, 0, 31) || + SemaFeatureCheck(*this, TheCall, "htm", + diag::err_ppc_builtin_requires_htm); case PPC::BI__builtin_tabortwci: case PPC::BI__builtin_tabortdci: - return SemaBuiltinConstantArgRange(TheCall, 0, 0, 31) || - SemaBuiltinConstantArgRange(TheCall, 2, 0, 31); + return SemaFeatureCheck(*this, TheCall, "htm", + diag::err_ppc_builtin_requires_htm) || + (SemaBuiltinConstantArgRange(TheCall, 0, 0, 31) || + SemaBuiltinConstantArgRange(TheCall, 2, 0, 31)); + case PPC::BI__builtin_tabort: + case PPC::BI__builtin_tcheck: + case PPC::BI__builtin_treclaim: + case PPC::BI__builtin_trechkpt: + case PPC::BI__builtin_tendall: + case PPC::BI__builtin_tresume: + case PPC::BI__builtin_tsuspend: + case PPC::BI__builtin_get_texasr: + case PPC::BI__builtin_get_texasru: + case PPC::BI__builtin_get_tfhar: + case PPC::BI__builtin_get_tfiar: + case PPC::BI__builtin_set_texasr: + case PPC::BI__builtin_set_texasru: + case PPC::BI__builtin_set_tfhar: + case PPC::BI__builtin_set_tfiar: + case PPC::BI__builtin_ttest: + return SemaFeatureCheck(*this, TheCall, "htm", + diag::err_ppc_builtin_requires_htm); // According to GCC 'Basic PowerPC Built-in Functions Available on ISA 2.05', // __builtin_(un)pack_longdouble are available only if long double uses IBM // extended double representation. @@ -5819,8 +5984,8 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, ? 0 : 1); - if (ValType->isExtIntType()) { - Diag(Ptr->getExprLoc(), diag::err_atomic_builtin_ext_int_prohibit); + if (ValType->isBitIntType()) { + Diag(Ptr->getExprLoc(), diag::err_atomic_builtin_bit_int_prohibit); return ExprError(); } @@ -6217,11 +6382,11 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) { // gracefully. TheCall->setType(ResultType); - // Prohibit use of _ExtInt with atomic builtins. - // The arguments would have already been converted to the first argument's - // type, so only need to check the first argument. - const auto *ExtIntValType = ValType->getAs(); - if (ExtIntValType && !llvm::isPowerOf2_64(ExtIntValType->getNumBits())) { + // Prohibit problematic uses of bit-precise integer types with atomic + // builtins. The arguments would have already been converted to the first + // argument's type, so only need to check the first argument. + const auto *BitIntValType = ValType->getAs(); + if (BitIntValType && !llvm::isPowerOf2_64(BitIntValType->getNumBits())) { Diag(FirstArg->getExprLoc(), diag::err_atomic_builtin_ext_int_size); return ExprError(); } @@ -11249,7 +11414,7 @@ struct IntRange { false/*NonNegative*/); } - if (const auto *EIT = dyn_cast(T)) + if (const auto *EIT = dyn_cast(T)) return IntRange(EIT->getNumBits(), EIT->isUnsigned()); const BuiltinType *BT = cast(T); @@ -11275,7 +11440,7 @@ struct IntRange { if (const EnumType *ET = dyn_cast(T)) T = C.getCanonicalType(ET->getDecl()->getIntegerType()).getTypePtr(); - if (const auto *EIT = dyn_cast(T)) + if (const auto *EIT = dyn_cast(T)) return IntRange(EIT->getNumBits(), EIT->isUnsigned()); const BuiltinType *BT = cast(T); @@ -16697,26 +16862,19 @@ static bool checkMathBuiltinElementType(Sema &S, SourceLocation Loc, return false; } -bool Sema::SemaBuiltinElementwiseMathOneArg(CallExpr *TheCall) { +bool Sema::PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall) { if (checkArgCount(*this, TheCall, 1)) return true; ExprResult A = UsualUnaryConversions(TheCall->getArg(0)); - SourceLocation ArgLoc = TheCall->getArg(0)->getBeginLoc(); if (A.isInvalid()) return true; TheCall->setArg(0, A.get()); QualType TyA = A.get()->getType(); - if (checkMathBuiltinElementType(*this, ArgLoc, TyA)) - return true; - QualType EltTy = TyA; - if (auto *VecTy = EltTy->getAs()) - EltTy = VecTy->getElementType(); - if (EltTy->isUnsignedIntegerType()) - return Diag(ArgLoc, diag::err_builtin_invalid_arg_type) - << 1 << /*signed integer or float ty*/ 3 << TyA; + if (checkMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA)) + return true; TheCall->setType(TyA); return false; @@ -16752,7 +16910,7 @@ bool Sema::SemaBuiltinElementwiseMath(CallExpr *TheCall) { return false; } -bool Sema::SemaBuiltinReduceMath(CallExpr *TheCall) { +bool Sema::PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall) { if (checkArgCount(*this, TheCall, 1)) return true; @@ -16761,14 +16919,6 @@ bool Sema::SemaBuiltinReduceMath(CallExpr *TheCall) { return true; TheCall->setArg(0, A.get()); - const VectorType *TyA = A.get()->getType()->getAs(); - if (!TyA) { - SourceLocation ArgLoc = TheCall->getArg(0)->getBeginLoc(); - return Diag(ArgLoc, diag::err_builtin_invalid_arg_type) - << 1 << /* vector ty*/ 4 << A.get()->getType(); - } - - TheCall->setType(TyA->getElementType()); return false; } diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index 083a67db7a91..93c07ccc891f 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -569,7 +569,6 @@ void PreferredTypeBuilder::enterMemAccess(Sema &S, SourceLocation Tok, return; // Keep the expected type, only update the location. ExpectedLoc = Tok; - return; } void PreferredTypeBuilder::enterUnary(Sema &S, SourceLocation Tok, diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp index b999b08d1662..e89cecd08cca 100644 --- a/clang/lib/Sema/SemaCoroutine.cpp +++ b/clang/lib/Sema/SemaCoroutine.cpp @@ -237,9 +237,9 @@ static bool isValidCoroutineContext(Sema &S, SourceLocation Loc, // placeholder type shall not be a coroutine." if (FD->getReturnType()->isUndeducedType()) DiagInvalid(DiagAutoRet); - // [dcl.fct.def.coroutine]p1: "The parameter-declaration-clause of the - // coroutine shall not terminate with an ellipsis that is not part of a - // parameter-declaration." + // [dcl.fct.def.coroutine]p1 + // The parameter-declaration-clause of the coroutine shall not terminate with + // an ellipsis that is not part of a parameter-declaration. if (FD->isVariadic()) DiagInvalid(DiagVarargs); @@ -579,8 +579,12 @@ VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) { /*TopLevelOfInitList=*/false, /*TreatUnavailableAsInvalid=*/false); - // Attempt to initialize the promise type with the arguments. - // If that fails, fall back to the promise type's default constructor. + // [dcl.fct.def.coroutine]5.7 + // promise-constructor-arguments is determined as follows: overload + // resolution is performed on a promise constructor call created by + // assembling an argument list q_1 ... q_n . If a viable constructor is + // found ([over.match.viable]), then promise-constructor-arguments is ( q_1 + // , ..., q_n ), otherwise promise-constructor-arguments is empty. if (InitSeq) { ExprResult Result = InitSeq.Perform(*this, Entity, Kind, CtorArgExprs); if (Result.isInvalid()) { @@ -648,6 +652,10 @@ static void checkNoThrow(Sema &S, const Stmt *E, return; } if (ThrowingDecls.empty()) { + // [dcl.fct.def.coroutine]p15 + // The expression co_­await promise.final_­suspend() shall not be + // potentially-throwing ([except.spec]). + // // First time seeing an error, emit the error message. S.Diag(cast(S.CurContext)->getLocation(), diag::err_coroutine_promise_final_suspend_requires_nothrow); @@ -995,9 +1003,8 @@ static Expr *buildStdNoThrowDeclRef(Sema &S, SourceLocation Loc) { LookupResult Result(S, &S.PP.getIdentifierTable().get("nothrow"), Loc, Sema::LookupOrdinaryName); if (!S.LookupQualifiedName(Result, Std)) { - // FIXME: should have been included already. - // If we require it to include then this diagnostic is no longer - // needed. + // is not requred to include , so we couldn't omit + // the check here. S.Diag(Loc, diag::err_implicit_coroutine_std_nothrow_type_not_found); return nullptr; } @@ -1029,9 +1036,21 @@ static FunctionDecl *findDeleteForPromise(Sema &S, SourceLocation Loc, auto *PointeeRD = PromiseType->getAsCXXRecordDecl(); assert(PointeeRD && "PromiseType must be a CxxRecordDecl type"); + // [dcl.fct.def.coroutine]p12 + // The deallocation function's name is looked up by searching for it in the + // scope of the promise type. If nothing is found, a search is performed in + // the global scope. if (S.FindDeallocationFunction(Loc, PointeeRD, DeleteName, OperatorDelete)) return nullptr; + // FIXME: We didn't implement following selection: + // [dcl.fct.def.coroutine]p12 + // If both a usual deallocation function with only a pointer parameter and a + // usual deallocation function with both a pointer parameter and a size + // parameter are found, then the selected deallocation function shall be the + // one with two parameters. Otherwise, the selected deallocation function + // shall be the function with one parameter. + if (!OperatorDelete) { // Look for a global declaration. const bool CanProvideSize = S.isCompleteType(Loc, PromiseType); @@ -1062,8 +1081,8 @@ void Sema::CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body) { return; } - // Coroutines [stmt.return]p1: - // A return statement shall not appear in a coroutine. + // [stmt.return.coroutine]p1: + // A coroutine shall not enclose a return statement ([stmt.return]). if (Fn->FirstReturnLoc.isValid()) { assert(Fn->FirstCoroutineStmtLoc.isValid() && "first coroutine location not set"); @@ -1164,12 +1183,15 @@ bool CoroutineStmtBuilder::makeReturnOnAllocFailure() { assert(!IsPromiseDependentType && "cannot make statement while the promise type is dependent"); - // [dcl.fct.def.coroutine]/8 - // The unqualified-id get_return_object_on_allocation_failure is looked up in - // the scope of class P by class member access lookup (3.4.5). ... - // If an allocation function returns nullptr, ... the coroutine return value - // is obtained by a call to ... get_return_object_on_allocation_failure(). - + // [dcl.fct.def.coroutine]p10 + // If a search for the name get_­return_­object_­on_­allocation_­failure in + // the scope of the promise type ([class.member.lookup]) finds any + // declarations, then the result of a call to an allocation function used to + // obtain storage for the coroutine state is assumed to return nullptr if it + // fails to obtain storage, ... If the allocation function returns nullptr, + // ... and the return value is obtained by a call to + // T::get_­return_­object_­on_­allocation_­failure(), where T is the + // promise type. DeclarationName DN = S.PP.getIdentifierInfo("get_return_object_on_allocation_failure"); LookupResult Found(S, DN, Loc, Sema::LookupMemberName); @@ -1215,12 +1237,11 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() { const bool RequiresNoThrowAlloc = ReturnStmtOnAllocFailure != nullptr; - // [dcl.fct.def.coroutine]/7 - // Lookup allocation functions using a parameter list composed of the - // requested size of the coroutine state being allocated, followed by - // the coroutine function's arguments. If a matching allocation function - // exists, use it. Otherwise, use an allocation function that just takes - // the requested size. + // According to [dcl.fct.def.coroutine]p9, Lookup allocation functions using a + // parameter list composed of the requested size of the coroutine state being + // allocated, followed by the coroutine function's arguments. If a matching + // allocation function exists, use it. Otherwise, use an allocation function + // that just takes the requested size. FunctionDecl *OperatorNew = nullptr; FunctionDecl *OperatorDelete = nullptr; @@ -1228,21 +1249,32 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() { bool PassAlignment = false; SmallVector PlacementArgs; - // [dcl.fct.def.coroutine]/7 - // "The allocation function’s name is looked up in the scope of P. - // [...] If the lookup finds an allocation function in the scope of P, - // overload resolution is performed on a function call created by assembling - // an argument list. The first argument is the amount of space requested, - // and has type std::size_t. The lvalues p1 ... pn are the succeeding - // arguments." + // [dcl.fct.def.coroutine]p9 + // An implementation may need to allocate additional storage for a + // coroutine. + // This storage is known as the coroutine state and is obtained by calling a + // non-array allocation function ([basic.stc.dynamic.allocation]). The + // allocation function's name is looked up by searching for it in the scope of + // the promise type. + // - If any declarations are found, overload resolution is performed on a + // function call created by assembling an argument list. The first argument is + // the amount of space requested, and has type std::size_t. The + // lvalues p1 ... pn are the succeeding arguments. // // ...where "p1 ... pn" are defined earlier as: // - // [dcl.fct.def.coroutine]/3 - // "For a coroutine f that is a non-static member function, let P1 denote the - // type of the implicit object parameter (13.3.1) and P2 ... Pn be the types - // of the function parameters; otherwise let P1 ... Pn be the types of the - // function parameters. Let p1 ... pn be lvalues denoting those objects." + // [dcl.fct.def.coroutine]p3 + // The promise type of a coroutine is `std::coroutine_traits` + // , where R is the return type of the function, and `P1, ..., Pn` are the + // sequence of types of the non-object function parameters, preceded by the + // type of the object parameter ([dcl.fct]) if the coroutine is a non-static + // member function. [dcl.fct.def.coroutine]p4 In the following, p_i is an + // lvalue of type P_i, where p1 denotes the object parameter and p_i+1 denotes + // the i-th non-object function parameter for a non-static member function, + // and p_i denotes the i-th function parameter otherwise. For a non-static + // member function, q_1 is an lvalue that denotes *this; any other q_i is an + // lvalue that denotes the parameter copy corresponding to p_i. if (auto *MD = dyn_cast(&FD)) { if (MD->isInstance() && !isLambdaCallOperator(MD)) { ExprResult ThisExpr = S.ActOnCXXThis(Loc); @@ -1273,10 +1305,10 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() { /*isArray*/ false, PassAlignment, PlacementArgs, OperatorNew, UnusedResult, /*Diagnose*/ false); - // [dcl.fct.def.coroutine]/7 - // "If no matching function is found, overload resolution is performed again - // on a function call created by passing just the amount of space required as - // an argument of type std::size_t." + // [dcl.fct.def.coroutine]p9 + // If no viable function is found ([over.match.viable]), overload resolution + // is performed again on a function call created by passing just the amount of + // space required as an argument of type std::size_t. if (!OperatorNew && !PlacementArgs.empty()) { PlacementArgs.clear(); S.FindAllocationFunctions(Loc, SourceRange(), /*NewScope*/ Sema::AFS_Class, @@ -1285,10 +1317,11 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() { OperatorNew, UnusedResult, /*Diagnose*/ false); } - // [dcl.fct.def.coroutine]/7 - // "The allocation function’s name is looked up in the scope of P. If this - // lookup fails, the allocation function’s name is looked up in the global - // scope." + // [dcl.fct.def.coroutine]p9 + // The allocation function's name is looked up by searching for it in the + // scope of the promise type. + // - If any declarations are found, ... + // - Otherwise, a search is performed in the global scope. if (!OperatorNew) { S.FindAllocationFunctions(Loc, SourceRange(), /*NewScope*/ Sema::AFS_Global, /*DeleteScope*/ Sema::AFS_Both, PromiseType, @@ -1328,8 +1361,12 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() { } } - if ((OperatorDelete = findDeleteForPromise(S, Loc, PromiseType)) == nullptr) + if ((OperatorDelete = findDeleteForPromise(S, Loc, PromiseType)) == nullptr) { + // FIXME: We should add an error here. According to: + // [dcl.fct.def.coroutine]p12 + // If no usual deallocation function is found, the program is ill-formed. return false; + } Expr *FramePtr = S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_frame, {}); @@ -1368,7 +1405,11 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() { SmallVector DeleteArgs{CoroFree}; - // Check if we need to pass the size. + // [dcl.fct.def.coroutine]p12 + // The selected deallocation function shall be called with the address of + // the block of storage to be reclaimed as its first argument. If a + // deallocation function with a parameter of type std::size_t is + // used, the size of the block is passed as the corresponding argument. const auto *OpDeleteType = OpDeleteQualType.getTypePtr()->castAs(); if (OpDeleteType->getNumParams() > 1) @@ -1391,9 +1432,13 @@ bool CoroutineStmtBuilder::makeOnFallthrough() { assert(!IsPromiseDependentType && "cannot make statement while the promise type is dependent"); - // [dcl.fct.def.coroutine]/4 - // The unqualified-ids 'return_void' and 'return_value' are looked up in - // the scope of class P. If both are found, the program is ill-formed. + // [dcl.fct.def.coroutine]/p6 + // If searches for the names return_­void and return_­value in the scope of + // the promise type each find any declarations, the program is ill-formed. + // [Note 1: If return_­void is found, flowing off the end of a coroutine is + // equivalent to a co_­return with no operand. Otherwise, flowing off the end + // of a coroutine results in undefined behavior ([stmt.return.coroutine]). — + // end note] bool HasRVoid, HasRValue; LookupResult LRVoid = lookupMember(S, "return_void", PromiseRecordDecl, Loc, HasRVoid); @@ -1414,18 +1459,20 @@ bool CoroutineStmtBuilder::makeOnFallthrough() { << LRValue.getLookupName(); return false; } else if (!HasRVoid && !HasRValue) { - // FIXME: The PDTS currently specifies this case as UB, not ill-formed. - // However we still diagnose this as an error since until the PDTS is fixed. - S.Diag(FD.getLocation(), - diag::err_coroutine_promise_requires_return_function) - << PromiseRecordDecl; - S.Diag(PromiseRecordDecl->getLocation(), diag::note_defined_here) - << PromiseRecordDecl; - return false; + // We need to set 'Fallthrough'. Otherwise the other analysis part might + // think the coroutine has defined a return_value method. So it might emit + // **false** positive warning. e.g., + // + // promise_without_return_func foo() { + // co_await something(); + // } + // + // Then AnalysisBasedWarning would emit a warning about `foo()` lacking a + // co_return statements, which isn't correct. + Fallthrough = S.ActOnNullStmt(PromiseRecordDecl->getLocation()); + if (Fallthrough.isInvalid()) + return false; } else if (HasRVoid) { - // If the unqualified-id return_void is found, flowing off the end of a - // coroutine is equivalent to a co_return with no operand. Otherwise, - // flowing off the end of a coroutine results in undefined behavior. Fallthrough = S.BuildCoreturnStmt(FD.getLocation(), nullptr, /*IsImplicit*/false); Fallthrough = S.ActOnFinishFullStmt(Fallthrough.get()); @@ -1481,8 +1528,9 @@ bool CoroutineStmtBuilder::makeOnException() { } bool CoroutineStmtBuilder::makeReturnObject() { - // Build implicit 'p.get_return_object()' expression and form initialization - // of return type from it. + // [dcl.fct.def.coroutine]p7 + // The expression promise.get_­return_­object() is used to initialize the + // returned reference or prvalue result object of a call to a coroutine. ExprResult ReturnObject = buildPromiseCall(S, Fn.CoroutinePromise, Loc, "get_return_object", None); if (ReturnObject.isInvalid()) @@ -1620,6 +1668,12 @@ bool Sema::buildCoroutineParameterMoves(SourceLocation Loc) { if (!ScopeInfo->CoroutineParameterMoves.empty()) return false; + // [dcl.fct.def.coroutine]p13 + // When a coroutine is invoked, after initializing its parameters + // ([expr.call]), a copy is created for each coroutine parameter. For a + // parameter of type cv T, the copy is a variable of type cv T with + // automatic storage duration that is direct-initialized from an xvalue of + // type T referring to the parameter. for (auto *PD : FD->parameters()) { if (PD->getType()->isDependentType()) continue; @@ -1636,7 +1690,9 @@ bool Sema::buildCoroutineParameterMoves(SourceLocation Loc) { CExpr = castForMoving(*this, PDRefExpr.get()); else CExpr = PDRefExpr.get(); - + // [dcl.fct.def.coroutine]p13 + // The initialization and destruction of each parameter copy occurs in the + // context of the called coroutine. auto D = buildVarDecl(*this, Loc, PD->getType(), PD->getIdentifier()); AddInitializerToDecl(D, CExpr, /*DirectInit=*/true); @@ -1661,43 +1717,53 @@ ClassTemplateDecl *Sema::lookupCoroutineTraits(SourceLocation KwLoc, SourceLocation FuncLoc, NamespaceDecl *&Namespace) { if (!StdCoroutineTraitsCache) { - NamespaceDecl *CoroNamespace = getStdNamespace(); - LookupResult Result(*this, &PP.getIdentifierTable().get("coroutine_traits"), - FuncLoc, LookupOrdinaryName); + // Because coroutines moved from std::experimental in the TS to std in + // C++20, we look in both places to give users time to transition their + // TS-specific code to C++20. Diagnostics are given when the TS usage is + // discovered. + // TODO: Become stricter when is removed. - if (!CoroNamespace || !LookupQualifiedName(Result, CoroNamespace)) { - /// Look up in namespace std::experimental, for compatibility. - /// TODO: Remove this extra lookup when is - /// removed. - CoroNamespace = lookupStdExperimentalNamespace(); - if (!CoroNamespace || !LookupQualifiedName(Result, CoroNamespace)) { - Diag(KwLoc, diag::err_implied_coroutine_type_not_found) - << "std::coroutine_traits"; - return nullptr; - } - Diag(KwLoc, diag::warn_deprecated_coroutine_namespace) - << "coroutine_traits"; - } else { - /// When we found coroutine_traits in std namespace. Make sure there is no - /// misleading definition in std::experimental namespace. - NamespaceDecl *ExpNamespace = lookupStdExperimentalNamespace(); - LookupResult ExpResult(*this, - &PP.getIdentifierTable().get("coroutine_traits"), - FuncLoc, LookupOrdinaryName); - if (ExpNamespace && LookupQualifiedName(ExpResult, ExpNamespace)) { - Diag(KwLoc, - diag::err_mixed_use_std_and_experimental_namespace_for_coroutine); - return nullptr; - } + auto const &TraitIdent = PP.getIdentifierTable().get("coroutine_traits"); + + NamespaceDecl *StdSpace = getStdNamespace(); + LookupResult ResStd(*this, &TraitIdent, FuncLoc, LookupOrdinaryName); + bool InStd = StdSpace && LookupQualifiedName(ResStd, StdSpace); + + NamespaceDecl *ExpSpace = lookupStdExperimentalNamespace(); + LookupResult ResExp(*this, &TraitIdent, FuncLoc, LookupOrdinaryName); + bool InExp = ExpSpace && LookupQualifiedName(ResExp, ExpSpace); + + if (!InStd && !InExp) { + // The goggles, they found nothing! + Diag(KwLoc, diag::err_implied_coroutine_type_not_found) + << "std::coroutine_traits"; + return nullptr; } + if (!InStd) { + // Found only in std::experimental. + Diag(KwLoc, diag::warn_deprecated_coroutine_namespace) + << "coroutine_traits"; + } else if (InExp) { + // Found in std and std::experimental. + Diag(KwLoc, + diag::err_mixed_use_std_and_experimental_namespace_for_coroutine); + Diag(KwLoc, diag::warn_deprecated_coroutine_namespace) + << "coroutine_traits"; + return nullptr; + } + + // Prefer ::std to std::experimental. + auto &Result = InStd ? ResStd : ResExp; + CoroTraitsNamespaceCache = InStd ? StdSpace : ExpSpace; + + // coroutine_traits is required to be a class template. if (!(StdCoroutineTraitsCache = Result.getAsSingle())) { Result.suppressDiagnostics(); NamedDecl *Found = *Result.begin(); Diag(Found->getLocation(), diag::err_malformed_std_coroutine_traits); return nullptr; } - CoroTraitsNamespaceCache = CoroNamespace; } Namespace = CoroTraitsNamespaceCache; return StdCoroutineTraitsCache; diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 7be71ca49ea2..3c58f1d19c04 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -372,6 +372,7 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, } NamedDecl *IIDecl = nullptr; + UsingShadowDecl *FoundUsingShadow = nullptr; switch (Result.getResultKind()) { case LookupResult::NotFound: case LookupResult::NotFoundInCurrentInstantiation: @@ -441,8 +442,10 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, (AllowDeducedTemplate && getAsTypeTemplateDecl(RealRes))) { if (!IIDecl || // Make the selection of the recovery decl deterministic. - RealRes->getLocation() < IIDecl->getLocation()) + RealRes->getLocation() < IIDecl->getLocation()) { IIDecl = RealRes; + FoundUsingShadow = dyn_cast(*Res); + } } } @@ -465,6 +468,7 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, case LookupResult::Found: IIDecl = Result.getFoundDecl(); + FoundUsingShadow = dyn_cast(*Result.begin()); break; } @@ -491,14 +495,20 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, (void)DiagnoseUseOfDecl(IDecl, NameLoc); if (!HasTrailingDot) T = Context.getObjCInterfaceType(IDecl); + FoundUsingShadow = nullptr; // FIXME: Target must be a TypeDecl. } else if (auto *UD = dyn_cast(IIDecl)) { (void)DiagnoseUseOfDecl(UD, NameLoc); // Recover with 'int' T = Context.IntTy; + FoundUsingShadow = nullptr; } else if (AllowDeducedTemplate) { - if (auto *TD = getAsTypeTemplateDecl(IIDecl)) + if (auto *TD = getAsTypeTemplateDecl(IIDecl)) { + // FIXME: TemplateName should include FoundUsingShadow sugar. T = Context.getDeducedTemplateSpecializationType(TemplateName(TD), QualType(), false); + // Don't wrap in a further UsingType. + FoundUsingShadow = nullptr; + } } if (T.isNull()) { @@ -507,6 +517,9 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, return nullptr; } + if (FoundUsingShadow) + T = Context.getUsingType(FoundUsingShadow, T); + // NOTE: avoid constructing an ElaboratedType(Loc) if this is a // constructor or destructor name (in such a case, the scope specifier // will be attached to the enclosing Expr or Decl node). @@ -843,21 +856,6 @@ static bool isTagTypeWithMissingTag(Sema &SemaRef, LookupResult &Result, return false; } -/// Build a ParsedType for a simple-type-specifier with a nested-name-specifier. -static ParsedType buildNestedType(Sema &S, CXXScopeSpec &SS, - QualType T, SourceLocation NameLoc) { - ASTContext &Context = S.Context; - - TypeLocBuilder Builder; - Builder.pushTypeSpec(T).setNameLoc(NameLoc); - - T = S.getElaboratedType(ETK_None, SS, T); - ElaboratedTypeLoc ElabTL = Builder.push(T); - ElabTL.setElaboratedKeywordLoc(SourceLocation()); - ElabTL.setQualifierLoc(SS.getWithLocInContext(Context)); - return S.CreateParsedType(T, Builder.getTypeSourceInfo(Context, T)); -} - Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS, IdentifierInfo *&Name, SourceLocation NameLoc, @@ -1134,14 +1132,28 @@ Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS, : NameClassification::TypeTemplate(Template); } + auto BuildTypeFor = [&](TypeDecl *Type, NamedDecl *Found) { + QualType T = Context.getTypeDeclType(Type); + if (const auto *USD = dyn_cast(Found)) + T = Context.getUsingType(USD, T); + + if (SS.isEmpty()) // No elaborated type, trivial location info + return ParsedType::make(T); + + TypeLocBuilder Builder; + Builder.pushTypeSpec(T).setNameLoc(NameLoc); + T = getElaboratedType(ETK_None, SS, T); + ElaboratedTypeLoc ElabTL = Builder.push(T); + ElabTL.setElaboratedKeywordLoc(SourceLocation()); + ElabTL.setQualifierLoc(SS.getWithLocInContext(Context)); + return CreateParsedType(T, Builder.getTypeSourceInfo(Context, T)); + }; + NamedDecl *FirstDecl = (*Result.begin())->getUnderlyingDecl(); if (TypeDecl *Type = dyn_cast(FirstDecl)) { DiagnoseUseOfDecl(Type, NameLoc); MarkAnyDeclReferenced(Type->getLocation(), Type, /*OdrUse=*/false); - QualType T = Context.getTypeDeclType(Type); - if (SS.isNotEmpty()) - return buildNestedType(*this, SS, T, NameLoc); - return ParsedType::make(T); + return BuildTypeFor(Type, *Result.begin()); } ObjCInterfaceDecl *Class = dyn_cast(FirstDecl); @@ -1190,10 +1202,7 @@ Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS, isTagTypeWithMissingTag(*this, Result, S, SS, Name, NameLoc)) { TypeDecl *Type = Result.getAsSingle(); DiagnoseUseOfDecl(Type, NameLoc); - QualType T = Context.getTypeDeclType(Type); - if (SS.isNotEmpty()) - return buildNestedType(*this, SS, T, NameLoc); - return ParsedType::make(T); + return BuildTypeFor(Type, *Result.begin()); } // If we already know which single declaration is referenced, just annotate @@ -8504,7 +8513,14 @@ static NamedDecl *DiagnoseInvalidRedeclaration( << NewFD->getParamDecl(Idx - 1)->getType(); } else if (FDisConst != NewFDisConst) { SemaRef.Diag(FD->getLocation(), diag::note_member_def_close_const_match) - << NewFDisConst << FD->getSourceRange().getEnd(); + << NewFDisConst << FD->getSourceRange().getEnd() + << (NewFDisConst + ? FixItHint::CreateRemoval(ExtraArgs.D.getFunctionTypeInfo() + .getConstQualifierLoc()) + : FixItHint::CreateInsertion(ExtraArgs.D.getFunctionTypeInfo() + .getRParenLoc() + .getLocWithOffset(1), + " const")); } else SemaRef.Diag(FD->getLocation(), IsMember ? diag::note_member_def_close_match @@ -9195,6 +9211,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, << Name << RemoveRange << FixItHint::CreateRemoval(RemoveRange) << FixItHint::CreateInsertion(InsertLoc, "<>"); + Invalid = true; } } } else { @@ -15359,7 +15376,7 @@ bool Sema::CheckEnumUnderlyingType(TypeSourceInfo *TI) { if (BT->isInteger()) return false; - if (T->isExtIntType()) + if (T->isBitIntType()) return false; return Diag(UnderlyingLoc, diag::err_enum_invalid_underlying) << T; @@ -18277,7 +18294,7 @@ static void CheckForDuplicateEnumValues(Sema &S, ArrayRef Elements, // Emit one note for each of the remaining enum constants with // the same value. - for (auto *ECD : llvm::make_range(Vec->begin() + 1, Vec->end())) + for (auto *ECD : llvm::drop_begin(*Vec)) S.Diag(ECD->getLocation(), diag::note_duplicate_element) << ECD << toString(ECD->getInitVal(), 10) << ECD->getSourceRange(); diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 4df8687aff89..b6bd2e69629d 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -1001,6 +1001,84 @@ class ArgumentDependenceChecker }; } +static void handleDiagnoseAsBuiltinAttr(Sema &S, Decl *D, + const ParsedAttr &AL) { + const auto *DeclFD = cast(D); + + if (const auto *MethodDecl = dyn_cast(DeclFD)) + if (!MethodDecl->isStatic()) { + S.Diag(AL.getLoc(), diag::err_attribute_no_member_function) << AL; + return; + } + + auto DiagnoseType = [&](unsigned Index, AttributeArgumentNType T) { + SourceLocation Loc = [&]() { + auto Union = AL.getArg(Index - 1); + if (Union.is()) + return Union.get()->getBeginLoc(); + return Union.get()->Loc; + }(); + + S.Diag(Loc, diag::err_attribute_argument_n_type) << AL << Index << T; + }; + + FunctionDecl *AttrFD = [&]() -> FunctionDecl * { + if (!AL.isArgExpr(0)) + return nullptr; + auto *F = dyn_cast_or_null(AL.getArgAsExpr(0)); + if (!F) + return nullptr; + return dyn_cast_or_null(F->getFoundDecl()); + }(); + + if (!AttrFD || !AttrFD->getBuiltinID(true)) { + DiagnoseType(1, AANT_ArgumentBuiltinFunction); + return; + } + + if (AttrFD->getNumParams() != AL.getNumArgs() - 1) { + S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments_for) + << AL << AttrFD << AttrFD->getNumParams(); + return; + } + + SmallVector Indices; + + for (unsigned I = 1; I < AL.getNumArgs(); ++I) { + if (!AL.isArgExpr(I)) { + DiagnoseType(I + 1, AANT_ArgumentIntegerConstant); + return; + } + + const Expr *IndexExpr = AL.getArgAsExpr(I); + uint32_t Index; + + if (!checkUInt32Argument(S, AL, IndexExpr, Index, I + 1, false)) + return; + + if (Index > DeclFD->getNumParams()) { + S.Diag(AL.getLoc(), diag::err_attribute_bounds_for_function) + << AL << Index << DeclFD << DeclFD->getNumParams(); + return; + } + + QualType T1 = AttrFD->getParamDecl(I - 1)->getType(); + QualType T2 = DeclFD->getParamDecl(Index - 1)->getType(); + + if (T1.getCanonicalType().getUnqualifiedType() != + T2.getCanonicalType().getUnqualifiedType()) { + S.Diag(IndexExpr->getBeginLoc(), diag::err_attribute_parameter_types) + << AL << Index << DeclFD << T2 << I << AttrFD << T1; + return; + } + + Indices.push_back(Index - 1); + } + + D->addAttr(::new (S.Context) DiagnoseAsBuiltinAttr( + S.Context, AL, AttrFD, Indices.data(), Indices.size())); +} + static void handleDiagnoseIfAttr(Sema &S, Decl *D, const ParsedAttr &AL) { S.Diag(AL.getLoc(), diag::ext_clang_diagnose_if); @@ -4502,7 +4580,7 @@ void Sema::AddModeAttr(Decl *D, const AttributeCommonInfo &CI, return; } bool IntegralOrAnyEnumType = (OldElemTy->isIntegralOrEnumerationType() && - !OldElemTy->isExtIntType()) || + !OldElemTy->isBitIntType()) || OldElemTy->getAs(); if (!OldElemTy->getAs() && !OldElemTy->isComplexType() && @@ -8159,6 +8237,9 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, case ParsedAttr::AT_DiagnoseIf: handleDiagnoseIfAttr(S, D, AL); break; + case ParsedAttr::AT_DiagnoseAsBuiltin: + handleDiagnoseAsBuiltinAttr(S, D, AL); + break; case ParsedAttr::AT_NoBuiltin: handleNoBuiltinAttr(S, D, AL); break; diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 2658e9698688..01f0079198c7 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -8435,9 +8435,6 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD, DefaultedComparisonKind DCK) { assert(DCK != DefaultedComparisonKind::None && "not a defaulted comparison"); - CXXRecordDecl *RD = dyn_cast(FD->getLexicalDeclContext()); - assert(RD && "defaulted comparison is not defaulted in a class"); - // Perform any unqualified lookups we're going to need to default this // function. if (S) { @@ -8455,43 +8452,17 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD, // const C&, or // -- a friend of C having two parameters of type const C& or two // parameters of type C. - QualType ExpectedParmType1 = Context.getRecordType(RD); - QualType ExpectedParmType2 = - Context.getLValueReferenceType(ExpectedParmType1.withConst()); - if (isa(FD)) - ExpectedParmType1 = ExpectedParmType2; - for (const ParmVarDecl *Param : FD->parameters()) { - if (!Param->getType()->isDependentType() && - !Context.hasSameType(Param->getType(), ExpectedParmType1) && - !Context.hasSameType(Param->getType(), ExpectedParmType2)) { - // Don't diagnose an implicit 'operator=='; we will have diagnosed the - // corresponding defaulted 'operator<=>' already. - if (!FD->isImplicit()) { - Diag(FD->getLocation(), diag::err_defaulted_comparison_param) - << (int)DCK << Param->getType() << ExpectedParmType1 - << !isa(FD) - << ExpectedParmType2 << Param->getSourceRange(); - } - return true; - } - } - if (FD->getNumParams() == 2 && - !Context.hasSameType(FD->getParamDecl(0)->getType(), - FD->getParamDecl(1)->getType())) { - if (!FD->isImplicit()) { - Diag(FD->getLocation(), diag::err_defaulted_comparison_param_mismatch) - << (int)DCK - << FD->getParamDecl(0)->getType() - << FD->getParamDecl(0)->getSourceRange() - << FD->getParamDecl(1)->getType() - << FD->getParamDecl(1)->getSourceRange(); - } - return true; - } - // ... non-static const member ... - if (auto *MD = dyn_cast(FD)) { + CXXRecordDecl *RD = dyn_cast(FD->getLexicalDeclContext()); + bool IsMethod = isa(FD); + if (IsMethod) { + auto *MD = cast(FD); assert(!MD->isStatic() && "comparison function cannot be a static member"); + + // If we're out-of-class, this is the class we're comparing. + if (!RD) + RD = MD->getParent(); + if (!MD->isConst()) { SourceLocation InsertLoc; if (FunctionTypeLoc Loc = MD->getFunctionTypeLoc()) @@ -8500,7 +8471,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD, // corresponding defaulted 'operator<=>' already. if (!MD->isImplicit()) { Diag(MD->getLocation(), diag::err_defaulted_comparison_non_const) - << (int)DCK << FixItHint::CreateInsertion(InsertLoc, " const"); + << (int)DCK << FixItHint::CreateInsertion(InsertLoc, " const"); } // Add the 'const' to the type to recover. @@ -8510,9 +8481,98 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD, MD->setType(Context.getFunctionType(FPT->getReturnType(), FPT->getParamTypes(), EPI)); } - } else { - // A non-member function declared in a class must be a friend. + } + + if (FD->getNumParams() != (IsMethod ? 1 : 2)) { + // Let's not worry about using a variadic template pack here -- who would do + // such a thing? + Diag(FD->getLocation(), diag::err_defaulted_comparison_num_args) + << int(IsMethod) << int(DCK); + return true; + } + + const ParmVarDecl *KnownParm = nullptr; + for (const ParmVarDecl *Param : FD->parameters()) { + QualType ParmTy = Param->getType(); + if (ParmTy->isDependentType()) + continue; + if (!KnownParm) { + auto CTy = ParmTy; + // Is it `T const &`? + bool Ok = !IsMethod; + QualType ExpectedTy; + if (RD) + ExpectedTy = Context.getRecordType(RD); + if (auto *Ref = CTy->getAs()) { + CTy = Ref->getPointeeType(); + if (RD) + ExpectedTy.addConst(); + Ok = true; + } + + // Is T a class? + if (!Ok) { + } else if (RD) { + if (!RD->isDependentType() && !Context.hasSameType(CTy, ExpectedTy)) + Ok = false; + } else if (auto *CRD = CTy->getAsRecordDecl()) { + RD = cast(CRD); + } else { + Ok = false; + } + + if (Ok) { + KnownParm = Param; + } else { + // Don't diagnose an implicit 'operator=='; we will have diagnosed the + // corresponding defaulted 'operator<=>' already. + if (!FD->isImplicit()) { + if (RD) { + QualType PlainTy = Context.getRecordType(RD); + QualType RefTy = + Context.getLValueReferenceType(PlainTy.withConst()); + Diag(FD->getLocation(), diag::err_defaulted_comparison_param) + << int(DCK) << ParmTy << RefTy << int(!IsMethod) << PlainTy + << Param->getSourceRange(); + } else { + assert(!IsMethod && "should know expected type for method"); + Diag(FD->getLocation(), + diag::err_defaulted_comparison_param_unknown) + << int(DCK) << ParmTy << Param->getSourceRange(); + } + } + return true; + } + } else if (!Context.hasSameType(KnownParm->getType(), ParmTy)) { + Diag(FD->getLocation(), diag::err_defaulted_comparison_param_mismatch) + << int(DCK) << KnownParm->getType() << KnownParm->getSourceRange() + << ParmTy << Param->getSourceRange(); + return true; + } + } + + assert(RD && "must have determined class"); + if (IsMethod) { + } else if (isa(FD->getLexicalDeclContext())) { + // In-class, must be a friend decl. assert(FD->getFriendObjectKind() && "expected a friend declaration"); + } else { + // Out of class, require the defaulted comparison to be a friend (of a + // complete type). + if (RequireCompleteType(FD->getLocation(), Context.getRecordType(RD), + diag::err_defaulted_comparison_not_friend, int(DCK), + int(1))) + return true; + + if (llvm::find_if(RD->friends(), [&](const FriendDecl *F) { + return FD->getCanonicalDecl() == + F->getFriendDecl()->getCanonicalDecl(); + }) == RD->friends().end()) { + Diag(FD->getLocation(), diag::err_defaulted_comparison_not_friend) + << int(DCK) << int(0) << RD; + Diag(RD->getCanonicalDecl()->getLocation(), diag::note_declared_at); + return true; + } } // C++2a [class.eq]p1, [class.rel]p1: @@ -8670,7 +8730,10 @@ void Sema::DefineDefaultedComparison(SourceLocation UseLoc, FunctionDecl *FD, { // Build and set up the function body. - CXXRecordDecl *RD = cast(FD->getLexicalParent()); + // The first parameter has type maybe-ref-to maybe-const T, use that to get + // the type of the class being compared. + auto PT = FD->getParamDecl(0)->getType(); + CXXRecordDecl *RD = PT.getNonReferenceType()->getAsCXXRecordDecl(); SourceLocation BodyLoc = FD->getEndLoc().isValid() ? FD->getEndLoc() : FD->getLocation(); StmtResult Body = @@ -16146,6 +16209,23 @@ Decl *Sema::ActOnStartLinkageSpecification(Scope *S, SourceLocation ExternLoc, LinkageSpecDecl *D = LinkageSpecDecl::Create(Context, CurContext, ExternLoc, LangStr->getExprLoc(), Language, LBraceLoc.isValid()); + + /// C++ [module.unit]p7.2.3 + /// - Otherwise, if the declaration + /// - ... + /// - ... + /// - appears within a linkage-specification, + /// it is attached to the global module. + /// + /// If the declaration is already in global module fragment, we don't + /// need to attach it again. + if (getLangOpts().CPlusPlusModules && isCurrentModulePurview()) { + Module *GlobalModule = + PushGlobalModuleFragment(ExternLoc, /*IsImplicit=*/true); + D->setModuleOwnershipKind(Decl::ModuleOwnershipKind::ModulePrivate); + D->setLocalOwningModule(GlobalModule); + } + CurContext->addDecl(D); PushDeclContext(S, D); return D; @@ -16162,6 +16242,14 @@ Decl *Sema::ActOnFinishLinkageSpecification(Scope *S, LinkageSpecDecl* LSDecl = cast(LinkageSpec); LSDecl->setRBraceLoc(RBraceLoc); } + + // If the current module doesn't has Parent, it implies that the + // LinkageSpec isn't in the module created by itself. So we don't + // need to pop it. + if (getLangOpts().CPlusPlusModules && getCurrentModule() && + getCurrentModule()->isGlobalModule() && getCurrentModule()->Parent) + PopGlobalModuleFragment(); + PopDeclContext(); return LinkageSpec; } @@ -17155,13 +17243,6 @@ void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) { return; } - if (DefKind.isComparison() && - !isa(FD->getLexicalDeclContext())) { - Diag(FD->getLocation(), diag::err_defaulted_comparison_out_of_class) - << (int)DefKind.asComparison(); - return; - } - // Issue compatibility warning. We already warned if the operator is // 'operator<=>' when parsing the '<=>' token. if (DefKind.isComparison() && @@ -17183,31 +17264,40 @@ void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) { // that we've marked it as defaulted. FD->setWillHaveBody(false); - // If this definition appears within the record, do the checking when - // the record is complete. This is always the case for a defaulted - // comparison. - if (DefKind.isComparison()) - return; - auto *MD = cast(FD); + if (DefKind.isComparison()) { + // If this comparison's defaulting occurs within the definition of its + // lexical class context, we have to do the checking when complete. + if (auto const *RD = dyn_cast(FD->getLexicalDeclContext())) + if (!RD->isCompleteDefinition()) + return; + } - const FunctionDecl *Primary = FD; - if (const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern()) - // Ask the template instantiation pattern that actually had the - // '= default' on it. - Primary = Pattern; - - // If the method was defaulted on its first declaration, we will have + // If this member fn was defaulted on its first declaration, we will have // already performed the checking in CheckCompletedCXXClass. Such a // declaration doesn't trigger an implicit definition. - if (Primary->getCanonicalDecl()->isDefaulted()) - return; + if (isa(FD)) { + const FunctionDecl *Primary = FD; + if (const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern()) + // Ask the template instantiation pattern that actually had the + // '= default' on it. + Primary = Pattern; + if (Primary->getCanonicalDecl()->isDefaulted()) + return; + } - // FIXME: Once we support defining comparisons out of class, check for a - // defaulted comparison here. - if (CheckExplicitlyDefaultedSpecialMember(MD, DefKind.asSpecialMember())) - MD->setInvalidDecl(); - else - DefineDefaultedFunction(*this, MD, DefaultLoc); + if (DefKind.isComparison()) { + if (CheckExplicitlyDefaultedComparison(nullptr, FD, DefKind.asComparison())) + FD->setInvalidDecl(); + else + DefineDefaultedComparison(DefaultLoc, FD, DefKind.asComparison()); + } else { + auto *MD = cast(FD); + + if (CheckExplicitlyDefaultedSpecialMember(MD, DefKind.asSpecialMember())) + MD->setInvalidDecl(); + else + DefineDefaultedFunction(*this, MD, DefaultLoc); + } } static void SearchForReturnInStmt(Sema &Self, Stmt *S) { diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index b305d4e5b92f..d32b3f217aa0 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -4384,7 +4384,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T, case Type::ObjCObjectPointer: case Type::ObjCTypeParam: case Type::Pipe: - case Type::ExtInt: + case Type::BitInt: llvm_unreachable("type class is never variably-modified!"); case Type::Adjusted: T = cast(Ty)->getOriginalType(); @@ -4443,6 +4443,9 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T, case Type::Decltype: T = cast(Ty)->desugar(); break; + case Type::Using: + T = cast(Ty)->desugar(); + break; case Type::Auto: case Type::DeducedTemplateSpecialization: T = cast(Ty)->getDeducedType(); @@ -8388,9 +8391,10 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS, // If both operands have arithmetic type, do the usual arithmetic conversions // to find a common type: C99 6.5.15p3,5. if (LHSTy->isArithmeticType() && RHSTy->isArithmeticType()) { - // Disallow invalid arithmetic conversions, such as those between ExtInts of - // different sizes, or between ExtInts and other types. - if (ResTy.isNull() && (LHSTy->isExtIntType() || RHSTy->isExtIntType())) { + // Disallow invalid arithmetic conversions, such as those between bit- + // precise integers types of different sizes, or between a bit-precise + // integer and another type. + if (ResTy.isNull() && (LHSTy->isBitIntType() || RHSTy->isBitIntType())) { Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands) << LHSTy << RHSTy << LHS.get()->getSourceRange() << RHS.get()->getSourceRange(); @@ -10974,7 +10978,7 @@ static void DiagnoseBadShiftValues(Sema& S, ExprResult &LHS, ExprResult &RHS, QualType LHSExprType = LHS.get()->getType(); uint64_t LeftSize = S.Context.getTypeSize(LHSExprType); - if (LHSExprType->isExtIntType()) + if (LHSExprType->isBitIntType()) LeftSize = S.Context.getIntWidth(LHSExprType); else if (LHSExprType->isFixedPointType()) { auto FXSema = S.Context.getFixedPointSemantics(LHSExprType); @@ -12260,27 +12264,32 @@ QualType Sema::GetSignedVectorType(QualType V) { if (isa(VTy)) { if (TypeSize == Context.getTypeSize(Context.CharTy)) return Context.getExtVectorType(Context.CharTy, VTy->getNumElements()); - else if (TypeSize == Context.getTypeSize(Context.ShortTy)) + if (TypeSize == Context.getTypeSize(Context.ShortTy)) return Context.getExtVectorType(Context.ShortTy, VTy->getNumElements()); - else if (TypeSize == Context.getTypeSize(Context.IntTy)) + if (TypeSize == Context.getTypeSize(Context.IntTy)) return Context.getExtVectorType(Context.IntTy, VTy->getNumElements()); - else if (TypeSize == Context.getTypeSize(Context.LongTy)) + if (TypeSize == Context.getTypeSize(Context.Int128Ty)) + return Context.getExtVectorType(Context.Int128Ty, VTy->getNumElements()); + if (TypeSize == Context.getTypeSize(Context.LongTy)) return Context.getExtVectorType(Context.LongTy, VTy->getNumElements()); assert(TypeSize == Context.getTypeSize(Context.LongLongTy) && "Unhandled vector element size in vector compare"); return Context.getExtVectorType(Context.LongLongTy, VTy->getNumElements()); } + if (TypeSize == Context.getTypeSize(Context.Int128Ty)) + return Context.getVectorType(Context.Int128Ty, VTy->getNumElements(), + VectorType::GenericVector); if (TypeSize == Context.getTypeSize(Context.LongLongTy)) return Context.getVectorType(Context.LongLongTy, VTy->getNumElements(), VectorType::GenericVector); - else if (TypeSize == Context.getTypeSize(Context.LongTy)) + if (TypeSize == Context.getTypeSize(Context.LongTy)) return Context.getVectorType(Context.LongTy, VTy->getNumElements(), VectorType::GenericVector); - else if (TypeSize == Context.getTypeSize(Context.IntTy)) + if (TypeSize == Context.getTypeSize(Context.IntTy)) return Context.getVectorType(Context.IntTy, VTy->getNumElements(), VectorType::GenericVector); - else if (TypeSize == Context.getTypeSize(Context.ShortTy)) + if (TypeSize == Context.getTypeSize(Context.ShortTy)) return Context.getVectorType(Context.ShortTy, VTy->getNumElements(), VectorType::GenericVector); assert(TypeSize == Context.getTypeSize(Context.CharTy) && diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index d25f329f85e4..54f0242d2ca1 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -1346,7 +1346,7 @@ bool Sema::CheckCXXThisCapture(SourceLocation Loc, const bool Explicit, // implicitly capturing the *enclosing object* by reference (see loop // above)). assert((!ByCopy || - dyn_cast(FunctionScopes[MaxFunctionScopesIndex])) && + isa(FunctionScopes[MaxFunctionScopesIndex])) && "Only a lambda can capture the enclosing object (referred to by " "*this) by copy"); QualType ThisTy = getCurrentThisType(); diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp index 0711e6d89383..635e93ba8460 100644 --- a/clang/lib/Sema/SemaLookup.cpp +++ b/clang/lib/Sema/SemaLookup.cpp @@ -2935,7 +2935,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) { case Type::ExtVector: case Type::ConstantMatrix: case Type::Complex: - case Type::ExtInt: + case Type::BitInt: break; // Non-deduced auto types only get here for error cases. diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index af95b1a93cc4..a4b9f3c242c1 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -68,15 +68,8 @@ Sema::ActOnGlobalModuleFragmentDecl(SourceLocation ModuleLoc) { // We start in the global module; all those declarations are implicitly // module-private (though they do not have module linkage). - auto &Map = PP.getHeaderSearchInfo().getModuleMap(); - auto *GlobalModule = Map.createGlobalModuleFragmentForModuleUnit(ModuleLoc); - assert(GlobalModule && "module creation should not fail"); - - // Enter the scope of the global module. - ModuleScopes.push_back({}); - ModuleScopes.back().BeginLoc = ModuleLoc; - ModuleScopes.back().Module = GlobalModule; - VisibleModules.setVisible(GlobalModule, ModuleLoc); + Module *GlobalModule = + PushGlobalModuleFragment(ModuleLoc, /*IsImplicit=*/false); // All declarations created from now on are owned by the global module. auto *TU = Context.getTranslationUnitDecl(); @@ -390,11 +383,18 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc, if (!ModuleScopes.empty()) Context.addModuleInitializer(ModuleScopes.back().Module, Import); - // Re-export the module if needed. if (!ModuleScopes.empty() && ModuleScopes.back().ModuleInterface) { + // Re-export the module if the imported module is exported. + // Note that we don't need to add re-exported module to Imports field + // since `Exports` implies the module is imported already. if (ExportLoc.isValid() || getEnclosingExportDecl(Import)) getCurrentModule()->Exports.emplace_back(Mod, false); + else + getCurrentModule()->Imports.insert(Mod); } else if (ExportLoc.isValid()) { + // [module.interface]p1: + // An export-declaration shall inhabit a namespace scope and appear in the + // purview of a module interface unit. Diag(ExportLoc, diag::err_export_not_in_module_interface); } @@ -708,3 +708,26 @@ Decl *Sema::ActOnFinishExportDecl(Scope *S, Decl *D, SourceLocation RBraceLoc) { return D; } + +Module *Sema::PushGlobalModuleFragment(SourceLocation BeginLoc, + bool IsImplicit) { + ModuleMap &Map = PP.getHeaderSearchInfo().getModuleMap(); + Module *GlobalModule = + Map.createGlobalModuleFragmentForModuleUnit(BeginLoc, getCurrentModule()); + assert(GlobalModule && "module creation should not fail"); + + // Enter the scope of the global module. + ModuleScopes.push_back({BeginLoc, GlobalModule, + /*ModuleInterface=*/false, + /*ImplicitGlobalModuleFragment=*/IsImplicit, + /*VisibleModuleSet*/{}}); + VisibleModules.setVisible(GlobalModule, BeginLoc); + + return GlobalModule; +} + +void Sema::PopGlobalModuleFragment() { + assert(!ModuleScopes.empty() && getCurrentModule()->isGlobalModule() && + "left the wrong module scope, which is not global module fragment"); + ModuleScopes.pop_back(); +} diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp index 22ae5f59d41b..ba0481874577 100644 --- a/clang/lib/Sema/SemaOpenMP.cpp +++ b/clang/lib/Sema/SemaOpenMP.cpp @@ -255,14 +255,14 @@ class DSAStackTy { return &Stack.back().first[Size - 1]; } const SharingMapTy *getTopOfStackOrNull() const { - return const_cast(*this).getTopOfStackOrNull(); + return const_cast(*this).getTopOfStackOrNull(); } SharingMapTy &getTopOfStack() { assert(!isStackEmpty() && "no current directive"); return *getTopOfStackOrNull(); } const SharingMapTy &getTopOfStack() const { - return const_cast(*this).getTopOfStack(); + return const_cast(*this).getTopOfStack(); } SharingMapTy *getSecondOnStackOrNull() { @@ -272,7 +272,7 @@ class DSAStackTy { return &Stack.back().first[Size - 2]; } const SharingMapTy *getSecondOnStackOrNull() const { - return const_cast(*this).getSecondOnStackOrNull(); + return const_cast(*this).getSecondOnStackOrNull(); } /// Get the stack element at a certain level (previously returned by @@ -286,7 +286,7 @@ class DSAStackTy { return Stack.back().first[Level]; } const SharingMapTy &getStackElemAtLevel(unsigned Level) const { - return const_cast(*this).getStackElemAtLevel(Level); + return const_cast(*this).getStackElemAtLevel(Level); } DSAVarData getDSA(const_iterator &Iter, ValueDecl *D) const; @@ -354,9 +354,7 @@ class DSAStackTy { const SharingMapTy *Top = getTopOfStackOrNull(); return Top && Top->BodyComplete; } - void setBodyComplete() { - getTopOfStack().BodyComplete = true; - } + void setBodyComplete() { getTopOfStack().BodyComplete = true; } bool isForceVarCapturing() const { return ForceCapturing; } void setForceVarCapturing(bool V) { ForceCapturing = V; } @@ -392,6 +390,7 @@ class DSAStackTy { class ParentDirectiveScope { DSAStackTy &Self; bool Active; + public: ParentDirectiveScope(DSAStackTy &Self, bool Activate) : Self(Self), Active(false) { @@ -433,8 +432,7 @@ class DSAStackTy { } /// Marks (or clears) declaration as possibly loop counter. void resetPossibleLoopCounter(const Decl *D = nullptr) { - getTopOfStack().PossiblyLoopCounter = - D ? D->getCanonicalDecl() : D; + getTopOfStack().PossiblyLoopCounter = D ? D->getCanonicalDecl() : D; } /// Gets the possible loop counter decl. const Decl *getPossiblyLoopCunter() const { @@ -631,13 +629,10 @@ class DSAStackTy { } /// Add requires decl to internal vector - void addRequiresDecl(OMPRequiresDecl *RD) { - RequiresDecls.push_back(RD); - } + void addRequiresDecl(OMPRequiresDecl *RD) { RequiresDecls.push_back(RD); } /// Checks if the defined 'requires' directive has specified type of clause. - template - bool hasRequiresDeclWithClause() const { + template bool hasRequiresDeclWithClause() const { return llvm::any_of(RequiresDecls, [](const OMPRequiresDecl *D) { return llvm::any_of(D->clauselists(), [](const OMPClause *C) { return isa(C); @@ -680,9 +675,7 @@ class DSAStackTy { /// Returns the location of the first encountered atomic directive in the /// module. - SourceLocation getAtomicDirectiveLoc() const { - return AtomicLocation; - } + SourceLocation getAtomicDirectiveLoc() const { return AtomicLocation; } // Return previously encountered target region locations. ArrayRef getEncounteredTargetLocs() const { @@ -706,8 +699,7 @@ class DSAStackTy { } /// Set default data mapping attribute to Modifier:Kind void setDefaultDMAAttr(OpenMPDefaultmapClauseModifier M, - OpenMPDefaultmapClauseKind Kind, - SourceLocation Loc) { + OpenMPDefaultmapClauseKind Kind, SourceLocation Loc) { DefaultmapInfo &DMI = getTopOfStack().DefaultmapMap[Kind]; DMI.ImplicitBehavior = M; DMI.SLoc = Loc; @@ -749,12 +741,10 @@ class DSAStackTy { : getStackElemAtLevel(Level).DefaultAttr; } DefaultDataSharingAttributes getDefaultDSA() const { - return isStackEmpty() ? DSA_unspecified - : getTopOfStack().DefaultAttr; + return isStackEmpty() ? DSA_unspecified : getTopOfStack().DefaultAttr; } SourceLocation getDefaultDSALocation() const { - return isStackEmpty() ? SourceLocation() - : getTopOfStack().DefaultAttrLoc; + return isStackEmpty() ? SourceLocation() : getTopOfStack().DefaultAttrLoc; } OpenMPDefaultmapClauseModifier getDefaultmapModifier(OpenMPDefaultmapClauseKind Kind) const { @@ -1457,8 +1447,7 @@ void DSAStackTy::addTaskgroupReductionData(const ValueDecl *D, SourceRange SR, "Additional reduction info may be specified only once for reduction " "items."); ReductionData.set(BOK, SR); - Expr *&TaskgroupReductionRef = - getTopOfStack().TaskgroupReductionRef; + Expr *&TaskgroupReductionRef = getTopOfStack().TaskgroupReductionRef; if (!TaskgroupReductionRef) { VarDecl *VD = buildVarDecl(SemaRef, SR.getBegin(), SemaRef.Context.VoidPtrTy, ".task_red."); @@ -1483,8 +1472,7 @@ void DSAStackTy::addTaskgroupReductionData(const ValueDecl *D, SourceRange SR, "Additional reduction info may be specified only once for reduction " "items."); ReductionData.set(ReductionRef, SR); - Expr *&TaskgroupReductionRef = - getTopOfStack().TaskgroupReductionRef; + Expr *&TaskgroupReductionRef = getTopOfStack().TaskgroupReductionRef; if (!TaskgroupReductionRef) { VarDecl *VD = buildVarDecl(SemaRef, SR.getBegin(), SemaRef.Context.VoidPtrTy, ".task_red."); @@ -1595,10 +1583,9 @@ static bool rejectConstNotMutableType(Sema &SemaRef, const ValueDecl *D, ASTContext &Context = SemaRef.getASTContext(); bool IsClassType; if (isConstNotMutableType(SemaRef, Type, AcceptIfMutable, &IsClassType)) { - unsigned Diag = ListItemNotVar - ? diag::err_omp_const_list_item - : IsClassType ? diag::err_omp_const_not_mutable_variable - : diag::err_omp_const_variable; + unsigned Diag = ListItemNotVar ? diag::err_omp_const_list_item + : IsClassType ? diag::err_omp_const_not_mutable_variable + : diag::err_omp_const_variable; SemaRef.Diag(ELoc, Diag) << getOpenMPClauseName(CKind); if (!ListItemNotVar && D) { const VarDecl *VD = dyn_cast(D); @@ -1658,8 +1645,7 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D, }); if (IterTarget != end()) { const_iterator ParentIterTarget = IterTarget + 1; - for (const_iterator Iter = begin(); - Iter != ParentIterTarget; ++Iter) { + for (const_iterator Iter = begin(); Iter != ParentIterTarget; ++Iter) { if (isOpenMPLocal(VD, Iter)) { DVar.RefExpr = buildDeclRefExpr(SemaRef, VD, D->getType().getNonReferenceType(), @@ -1677,9 +1663,9 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D, return DVar; } const_iterator End = end(); - if (!SemaRef.isOpenMPCapturedByRef( - D, std::distance(ParentIterTarget, End), - /*OpenMPCaptureLevel=*/0)) { + if (!SemaRef.isOpenMPCapturedByRef(D, + std::distance(ParentIterTarget, End), + /*OpenMPCaptureLevel=*/0)) { DVar.RefExpr = buildDeclRefExpr(SemaRef, VD, D->getType().getNonReferenceType(), IterTarget->ConstructLoc); @@ -1891,9 +1877,7 @@ void Sema::InitDataSharingAttributesStack() { #define DSAStack static_cast(VarDataSharingAttributesStack) -void Sema::pushOpenMPFunctionRegion() { - DSAStack->pushFunction(); -} +void Sema::pushOpenMPFunctionRegion() { DSAStack->pushFunction(); } void Sema::popOpenMPFunctionRegion(const FunctionScopeInfo *OldFSI) { DSAStack->popFunction(OldFSI); @@ -2070,8 +2054,8 @@ bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level, DSAStack->checkMappableExprComponentListsForDeclAtLevel( D, Level, - [&IsVariableUsedInMapClause, &IsVariableAssociatedWithSection, D]( - OMPClauseMappableExprCommon::MappableExprComponentListRef + [&IsVariableUsedInMapClause, &IsVariableAssociatedWithSection, + D](OMPClauseMappableExprCommon::MappableExprComponentListRef MapExprComponents, OpenMPClauseKind WhereFoundClauseKind) { // Only the map clause information influences how a variable is @@ -2248,7 +2232,7 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo, bool OpenMPFound = false; for (unsigned I = StopAt + 1; I > 0; --I) { FunctionScopeInfo *FSI = FunctionScopes[I - 1]; - if(!isa(FSI)) + if (!isa(FSI)) return nullptr; if (auto *RSI = dyn_cast(FSI)) if (RSI->CapRegionKind == CR_OpenMP) { @@ -2322,9 +2306,7 @@ void Sema::startOpenMPCXXRangeFor() { OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level, unsigned CapLevel) const { assert(LangOpts.OpenMP && "OpenMP is not allowed"); - if (DSAStack->hasExplicitDirective( - [](OpenMPDirectiveKind K) { return isOpenMPTaskingDirective(K); }, - Level)) { + if (DSAStack->hasExplicitDirective(isOpenMPTaskingDirective, Level)) { bool IsTriviallyCopyable = D->getType().getNonReferenceType().isTriviallyCopyableType(Context) && !D->getType() @@ -2351,8 +2333,7 @@ OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level, } } if (isOpenMPLoopDirective(DSAStack->getCurrentDirective())) { - if (DSAStack->getAssociatedLoops() > 0 && - !DSAStack->isLoopStarted()) { + if (DSAStack->getAssociatedLoops() > 0 && !DSAStack->isLoopStarted()) { DSAStack->resetPossibleLoopCounter(D); DSAStack->loopStart(); return OMPC_private; @@ -2519,16 +2500,16 @@ void Sema::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller, << HostDevTy; return; } - if (!LangOpts.OpenMPIsDevice && DevTy && - *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) { - // Diagnose nohost function called during host codegen. - StringRef NoHostDevTy = getOpenMPSimpleClauseTypeName( - OMPC_device_type, OMPC_DEVICE_TYPE_nohost); - Diag(Loc, diag::err_omp_wrong_device_function_call) << NoHostDevTy << 1; - Diag(*OMPDeclareTargetDeclAttr::getLocation(FD), - diag::note_omp_marked_device_type_here) - << NoHostDevTy; - } + if (!LangOpts.OpenMPIsDevice && DevTy && + *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) { + // Diagnose nohost function called during host codegen. + StringRef NoHostDevTy = getOpenMPSimpleClauseTypeName( + OMPC_device_type, OMPC_DEVICE_TYPE_nohost); + Diag(Loc, diag::err_omp_wrong_device_function_call) << NoHostDevTy << 1; + Diag(*OMPDeclareTargetDeclAttr::getLocation(FD), + diag::note_omp_marked_device_type_here) + << NoHostDevTy; + } } void Sema::StartOpenMPDSABlock(OpenMPDirectiveKind DKind, @@ -2779,7 +2760,6 @@ class VarDeclFilterCCC final : public CorrectionCandidateCallback { std::unique_ptr clone() override { return std::make_unique(*this); } - }; class VarOrFuncDeclFilterCCC final : public CorrectionCandidateCallback { @@ -3151,9 +3131,10 @@ applyOMPAllocateAttribute(Sema &S, VarDecl *VD, ML->DeclarationMarkedOpenMPAllocate(VD, A); } -Sema::DeclGroupPtrTy Sema::ActOnOpenMPAllocateDirective( - SourceLocation Loc, ArrayRef VarList, - ArrayRef Clauses, DeclContext *Owner) { +Sema::DeclGroupPtrTy +Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef VarList, + ArrayRef Clauses, + DeclContext *Owner) { assert(Clauses.size() <= 2 && "Expected at most two clauses."); Expr *Alignment = nullptr; Expr *Allocator = nullptr; @@ -3500,7 +3481,8 @@ class DSAAttrChecker final : public StmtVisitor { return; if (auto *VD = dyn_cast(E->getDecl())) { // Check the datasharing rules for the expressions in the clauses. - if (!CS) { + if (!CS || (isa(VD) && !CS->capturesVariable(VD) && + !Stack->getTopDSA(VD, /*FromParent=*/false).RefExpr)) { if (auto *CED = dyn_cast(VD)) if (!CED->hasAttr()) { Visit(CED->getInit()); @@ -3819,6 +3801,10 @@ class DSAAttrChecker final : public StmtVisitor { } void VisitOMPExecutableDirective(OMPExecutableDirective *S) { for (OMPClause *C : S->clauses()) { + // Skip analysis of arguments of private clauses for task|target + // directives. + if (isa_and_nonnull(C)) + continue; // Skip analysis of arguments of implicitly defined firstprivate clause // for task|target directives. // Skip analysis of arguments of implicitly defined map clause for target @@ -3841,6 +3827,18 @@ class DSAAttrChecker final : public StmtVisitor { VisitStmt(S); } + void VisitCallExpr(CallExpr *S) { + for (Stmt *C : S->arguments()) { + if (C) { + // Check implicitly captured variables in the task-based directives to + // check if they must be firstprivatized. + Visit(C); + } + } + if (Expr *Callee = S->getCallee()) + if (auto *CE = dyn_cast(Callee->IgnoreParenImpCasts())) + Visit(CE->getBase()); + } void VisitStmt(Stmt *S) { for (Stmt *C : S->children()) { if (C) { @@ -5089,8 +5087,8 @@ static std::pair getPrivateItem(Sema &S, Expr *&RefExpr, !isa(ME->getBase()->IgnoreParenImpCasts()) || !isa(ME->getMemberDecl()))) { if (IsArrayExpr != NoArrayExpr) { - S.Diag(ELoc, diag::err_omp_expected_base_var_name) << IsArrayExpr - << ERange; + S.Diag(ELoc, diag::err_omp_expected_base_var_name) + << IsArrayExpr << ERange; } else { S.Diag(ELoc, AllowArraySection @@ -5134,8 +5132,7 @@ static void checkAllocateClauses(Sema &S, DSAStackTy *Stack, "Expected non-dependent context."); auto AllocateRange = llvm::make_filter_range(Clauses, OMPAllocateClause::classof); - llvm::DenseMap, CanonicalDeclPtr> - DeclToCopy; + llvm::DenseMap, CanonicalDeclPtr> DeclToCopy; auto PrivateRange = llvm::make_filter_range(Clauses, [](const OMPClause *C) { return isOpenMPPrivate(C->getClauseKind()); }); @@ -6018,7 +6015,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective( break; case OMPD_parallel_master: Res = ActOnOpenMPParallelMasterDirective(ClausesWithImplicit, AStmt, - StartLoc, EndLoc); + StartLoc, EndLoc); AllowedNameModifiers.push_back(OMPD_parallel); break; case OMPD_parallel_sections: @@ -6357,6 +6354,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective( case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -7626,10 +7624,10 @@ bool OpenMPIterationSpaceChecker::setStep(Expr *NewStep, bool Subtract) { // != with increment is treated as <; != with decrement is treated as > if (!TestIsLessOp.hasValue()) TestIsLessOp = IsConstPos || (IsUnsigned && !Subtract); - if (UB && (IsConstZero || - (TestIsLessOp.getValue() ? - (IsConstNeg || (IsUnsigned && Subtract)) : - (IsConstPos || (IsUnsigned && !Subtract))))) { + if (UB && + (IsConstZero || (TestIsLessOp.getValue() + ? (IsConstNeg || (IsUnsigned && Subtract)) + : (IsConstPos || (IsUnsigned && !Subtract))))) { SemaRef.Diag(NewStep->getExprLoc(), diag::err_omp_loop_incr_not_compatible) << LCDecl << TestIsLessOp.getValue() << NewStep->getSourceRange(); @@ -8068,11 +8066,13 @@ calculateNumIters(Sema &SemaRef, Scope *S, SourceLocation DefaultLoc, return nullptr; llvm::APSInt LRes, SRes; bool IsLowerConst = false, IsStepConst = false; - if (Optional Res = Lower->getIntegerConstantExpr(SemaRef.Context)) { + if (Optional Res = + Lower->getIntegerConstantExpr(SemaRef.Context)) { LRes = *Res; IsLowerConst = true; } - if (Optional Res = Step->getIntegerConstantExpr(SemaRef.Context)) { + if (Optional Res = + Step->getIntegerConstantExpr(SemaRef.Context)) { SRes = *Res; IsStepConst = true; } @@ -8110,7 +8110,8 @@ calculateNumIters(Sema &SemaRef, Scope *S, SourceLocation DefaultLoc, } llvm::APSInt URes; bool IsUpperConst = false; - if (Optional Res = Upper->getIntegerConstantExpr(SemaRef.Context)) { + if (Optional Res = + Upper->getIntegerConstantExpr(SemaRef.Context)) { URes = *Res; IsUpperConst = true; } @@ -8566,10 +8567,12 @@ Expr *OpenMPIterationSpaceChecker::buildPreCond( // TODO: this can be improved by calculating min/max values but not sure that // it will be very effective. if (CondDependOnLC || InitDependOnLC) - return SemaRef.PerformImplicitConversion( - SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(), - SemaRef.Context.BoolTy, /*Action=*/Sema::AA_Casting, - /*AllowExplicit=*/true).get(); + return SemaRef + .PerformImplicitConversion( + SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(), + SemaRef.Context.BoolTy, /*Action=*/Sema::AA_Casting, + /*AllowExplicit=*/true) + .get(); // Try to build LB UB, where is <, >, <=, or >=. Sema::TentativeAnalysisScope Trap(SemaRef); @@ -8579,12 +8582,11 @@ Expr *OpenMPIterationSpaceChecker::buildPreCond( if (!NewLB.isUsable() || !NewUB.isUsable()) return nullptr; - ExprResult CondExpr = - SemaRef.BuildBinOp(S, DefaultLoc, - TestIsLessOp.getValue() ? - (TestIsStrictOp ? BO_LT : BO_LE) : - (TestIsStrictOp ? BO_GT : BO_GE), - NewLB.get(), NewUB.get()); + ExprResult CondExpr = SemaRef.BuildBinOp( + S, DefaultLoc, + TestIsLessOp.getValue() ? (TestIsStrictOp ? BO_LT : BO_LE) + : (TestIsStrictOp ? BO_GT : BO_GE), + NewLB.get(), NewUB.get()); if (CondExpr.isUsable()) { if (!SemaRef.Context.hasSameUnqualifiedType(CondExpr.get()->getType(), SemaRef.Context.BoolTy)) @@ -8804,6 +8806,9 @@ static bool checkOpenMPIterationSpace( } assert(((For && For->getBody()) || (CXXFor && CXXFor->getBody())) && "No loop body."); + // Postpone analysis in dependent contexts for ranged for loops. + if (CXXFor && SemaRef.CurContext->isDependentContext()) + return false; OpenMPIterationSpaceChecker ISC(SemaRef, SupportsNonRectangular, DSA, For ? For->getForLoc() : CXXFor->getForLoc()); @@ -9684,9 +9689,8 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr, ExprResult Iter; // Compute prod - ExprResult Prod = - SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(); - for (unsigned int K = Cnt+1; K < NestedLoopCount; ++K) + ExprResult Prod = SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(); + for (unsigned int K = Cnt + 1; K < NestedLoopCount; ++K) Prod = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Mul, Prod.get(), IterSpaces[K].NumIterations); @@ -9694,8 +9698,8 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr, // If there is at least one more inner loop to avoid // multiplication by 1. if (Cnt + 1 < NestedLoopCount) - Iter = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Div, - Acc.get(), Prod.get()); + Iter = + SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Div, Acc.get(), Prod.get()); else Iter = Acc; if (!Iter.isUsable()) { @@ -9708,12 +9712,11 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr, // Check if there is at least one more inner loop to avoid // multiplication by 1. if (Cnt + 1 < NestedLoopCount) - Prod = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Mul, - Iter.get(), Prod.get()); + Prod = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Mul, Iter.get(), + Prod.get()); else Prod = Iter; - Acc = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Sub, - Acc.get(), Prod.get()); + Acc = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Sub, Acc.get(), Prod.get()); // Build update: IS.CounterVar(Private) = IS.Start + Iter * IS.Step auto *VD = cast(cast(IS.CounterVar)->getDecl()); @@ -10000,7 +10003,7 @@ StmtResult Sema::ActOnOpenMPSectionsDirective(ArrayRef Clauses, return StmtError(); // All associated statements must be '#pragma omp section' except for // the first one. - for (Stmt *SectionStmt : llvm::make_range(std::next(S.begin()), S.end())) { + for (Stmt *SectionStmt : llvm::drop_begin(S)) { if (!SectionStmt || !isa(SectionStmt)) { if (SectionStmt) Diag(SectionStmt->getBeginLoc(), @@ -10382,7 +10385,7 @@ Sema::ActOnOpenMPParallelSectionsDirective(ArrayRef Clauses, return StmtError(); // All associated statements must be '#pragma omp section' except for // the first one. - for (Stmt *SectionStmt : llvm::make_range(std::next(S.begin()), S.end())) { + for (Stmt *SectionStmt : llvm::drop_begin(S)) { if (!SectionStmt || !isa(SectionStmt)) { if (SectionStmt) Diag(SectionStmt->getBeginLoc(), @@ -10752,7 +10755,6 @@ class OpenMPAtomicUpdateChecker { bool checkBinaryOperation(BinaryOperator *AtomicBinOp, unsigned DiagId = 0, unsigned NoteId = 0); }; -} // namespace bool OpenMPAtomicUpdateChecker::checkBinaryOperation( BinaryOperator *AtomicBinOp, unsigned DiagId, unsigned NoteId) { @@ -10913,6 +10915,7 @@ bool OpenMPAtomicUpdateChecker::checkStatement(Stmt *S, unsigned DiagId, } return ErrorFound != NoError; } +} // namespace StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, Stmt *AStmt, @@ -10933,9 +10936,12 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, OpenMPClauseKind MemOrderKind = OMPC_unknown; SourceLocation MemOrderLoc; for (const OMPClause *C : Clauses) { - if (C->getClauseKind() == OMPC_read || C->getClauseKind() == OMPC_write || - C->getClauseKind() == OMPC_update || - C->getClauseKind() == OMPC_capture) { + switch (C->getClauseKind()) { + case OMPC_read: + case OMPC_write: + case OMPC_update: + case OMPC_capture: + case OMPC_compare: { if (AtomicKind != OMPC_unknown) { Diag(C->getBeginLoc(), diag::err_omp_atomic_several_clauses) << SourceRange(C->getBeginLoc(), C->getEndLoc()); @@ -10945,12 +10951,13 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, AtomicKind = C->getClauseKind(); AtomicKindLoc = C->getBeginLoc(); } + break; } - if (C->getClauseKind() == OMPC_seq_cst || - C->getClauseKind() == OMPC_acq_rel || - C->getClauseKind() == OMPC_acquire || - C->getClauseKind() == OMPC_release || - C->getClauseKind() == OMPC_relaxed) { + case OMPC_seq_cst: + case OMPC_acq_rel: + case OMPC_acquire: + case OMPC_release: + case OMPC_relaxed: { if (MemOrderKind != OMPC_unknown) { Diag(C->getBeginLoc(), diag::err_omp_several_mem_order_clauses) << getOpenMPDirectiveName(OMPD_atomic) << 0 @@ -10961,6 +10968,13 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, MemOrderKind = C->getClauseKind(); MemOrderLoc = C->getBeginLoc(); } + break; + } + // The following clauses are allowed, but we don't need to do anything here. + case OMPC_hint: + break; + default: + llvm_unreachable("unknown clause is encountered"); } } // OpenMP 5.0, 2.17.7 atomic Construct, Restrictions @@ -11075,8 +11089,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, if (ErrorFound != NoError) { Diag(ErrorLoc, diag::err_omp_atomic_read_not_expression_statement) << ErrorRange; - Diag(NoteLoc, diag::note_omp_atomic_read_write) << ErrorFound - << NoteRange; + Diag(NoteLoc, diag::note_omp_atomic_read_write) + << ErrorFound << NoteRange; return StmtError(); } if (CurContext->isDependentContext()) @@ -11137,8 +11151,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, if (ErrorFound != NoError) { Diag(ErrorLoc, diag::err_omp_atomic_write_not_expression_statement) << ErrorRange; - Diag(NoteLoc, diag::note_omp_atomic_read_write) << ErrorFound - << NoteRange; + Diag(NoteLoc, diag::note_omp_atomic_read_write) + << ErrorFound << NoteRange; return StmtError(); } if (CurContext->isDependentContext()) @@ -11154,9 +11168,10 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, // x = expr binop x; OpenMPAtomicUpdateChecker Checker(*this); if (Checker.checkStatement( - Body, (AtomicKind == OMPC_update) - ? diag::err_omp_atomic_update_not_expression_statement - : diag::err_omp_atomic_not_expression_statement, + Body, + (AtomicKind == OMPC_update) + ? diag::err_omp_atomic_update_not_expression_statement + : diag::err_omp_atomic_not_expression_statement, diag::note_omp_atomic_update)) return StmtError(); if (!CurContext->isDependentContext()) { @@ -11370,15 +11385,21 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses, SourceRange(Body->getBeginLoc(), Body->getBeginLoc()); ErrorFound = NotACompoundStatement; } - if (ErrorFound != NoError) { - Diag(ErrorLoc, diag::err_omp_atomic_capture_not_compound_statement) - << ErrorRange; - Diag(NoteLoc, diag::note_omp_atomic_capture) << ErrorFound << NoteRange; - return StmtError(); - } - if (CurContext->isDependentContext()) - UE = V = E = X = nullptr; } + if (ErrorFound != NoError) { + Diag(ErrorLoc, diag::err_omp_atomic_capture_not_compound_statement) + << ErrorRange; + Diag(NoteLoc, diag::note_omp_atomic_capture) << ErrorFound << NoteRange; + return StmtError(); + } + if (CurContext->isDependentContext()) + UE = V = E = X = nullptr; + } else if (AtomicKind == OMPC_compare) { + // TODO: For now we emit an error here and in emitOMPAtomicExpr we ignore + // code gen. + unsigned DiagID = Diags.getCustomDiagID( + DiagnosticsEngine::Error, "atomic compare is not supported for now"); + Diag(AtomicKindLoc, DiagID); } setFunctionHasBranchProtectedScope(); @@ -12283,8 +12304,8 @@ StmtResult Sema::ActOnOpenMPTargetParallelForSimdDirective( // define the nested loops number. unsigned NestedLoopCount = checkOpenMPLoop( OMPD_target_parallel_for_simd, getCollapseNumberExpr(Clauses), - getOrderedNumberExpr(Clauses), CS, *this, *DSAStack, - VarsWithImplicitDSA, B); + getOrderedNumberExpr(Clauses), CS, *this, *DSAStack, VarsWithImplicitDSA, + B); if (NestedLoopCount == 0) return StmtError(); @@ -13459,6 +13480,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr, case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -14290,6 +14312,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause( case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -14751,6 +14774,7 @@ OMPClause *Sema::ActOnOpenMPSimpleClause( case OMPC_read: case OMPC_write: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -15056,6 +15080,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprWithArgClause( case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -15244,6 +15269,9 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind, case OMPC_capture: Res = ActOnOpenMPCaptureClause(StartLoc, EndLoc); break; + case OMPC_compare: + Res = ActOnOpenMPCompareClause(StartLoc, EndLoc); + break; case OMPC_seq_cst: Res = ActOnOpenMPSeqCstClause(StartLoc, EndLoc); break; @@ -15390,6 +15418,11 @@ OMPClause *Sema::ActOnOpenMPCaptureClause(SourceLocation StartLoc, return new (Context) OMPCaptureClause(StartLoc, EndLoc); } +OMPClause *Sema::ActOnOpenMPCompareClause(SourceLocation StartLoc, + SourceLocation EndLoc) { + return new (Context) OMPCompareClause(StartLoc, EndLoc); +} + OMPClause *Sema::ActOnOpenMPSeqCstClause(SourceLocation StartLoc, SourceLocation EndLoc) { return new (Context) OMPSeqCstClause(StartLoc, EndLoc); @@ -15858,6 +15891,7 @@ OMPClause *Sema::ActOnOpenMPVarListClause( case OMPC_write: case OMPC_update: case OMPC_capture: + case OMPC_compare: case OMPC_seq_cst: case OMPC_acq_rel: case OMPC_acquire: @@ -15981,9 +16015,8 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef VarList, Diag(ELoc, diag::err_omp_variably_modified_type_not_supported) << getOpenMPClauseName(OMPC_private) << Type << getOpenMPDirectiveName(CurrDir); - bool IsDecl = - !VD || - VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly; + bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) == + VarDecl::DeclarationOnly; Diag(D->getLocation(), IsDecl ? diag::note_previous_decl : diag::note_defined_here) << D; @@ -16247,9 +16280,8 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef VarList, Diag(ELoc, diag::err_omp_variably_modified_type_not_supported) << getOpenMPClauseName(OMPC_firstprivate) << Type << getOpenMPDirectiveName(DSAStack->getCurrentDirective()); - bool IsDecl = - !VD || - VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly; + bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) == + VarDecl::DeclarationOnly; Diag(D->getLocation(), IsDecl ? diag::note_previous_decl : diag::note_defined_here) << D; @@ -17433,8 +17465,8 @@ static bool actOnOMPReductionKindClause( llvm::APInt InitValue = (BOK != BO_LT) ? IsSigned ? llvm::APInt::getSignedMinValue(Size) : llvm::APInt::getMinValue(Size) - : IsSigned ? llvm::APInt::getSignedMaxValue(Size) - : llvm::APInt::getMaxValue(Size); + : IsSigned ? llvm::APInt::getSignedMaxValue(Size) + : llvm::APInt::getMaxValue(Size); Init = IntegerLiteral::Create(Context, InitValue, IntTy, ELoc); if (Type->isPointerType()) { // Cast to pointer type. @@ -17845,9 +17877,8 @@ bool Sema::CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc, !Ty->isIntegralType(Context) && !Ty->isPointerType())) { Diag(ELoc, diag::err_omp_linear_expected_int_or_ptr) << Type; if (D) { - bool IsDecl = - !VD || - VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly; + bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) == + VarDecl::DeclarationOnly; Diag(D->getLocation(), IsDecl ? diag::note_previous_decl : diag::note_defined_here) << D; @@ -18059,13 +18090,12 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV, Update = SemaRef.ActOnFinishFullExpr(Update.get(), DE->getBeginLoc(), /*DiscardedValue*/ false); - // Build final: Var = InitExpr + NumIterations * Step + // Build final: Var = PrivCopy; ExprResult Final; if (!Info.first) - Final = - buildCounterUpdate(SemaRef, S, RefExpr->getExprLoc(), CapturedRef, - InitExpr, NumIterations, Step, /*Subtract=*/false, - /*IsNonRectangularLB=*/false); + Final = SemaRef.BuildBinOp( + S, RefExpr->getExprLoc(), BO_Assign, CapturedRef, + SemaRef.DefaultLvalueConversion(*CurPrivate).get()); else Final = *CurPrivate; Final = SemaRef.ActOnFinishFullExpr(Final.get(), DE->getBeginLoc(), @@ -18124,9 +18154,8 @@ OMPClause *Sema::ActOnOpenMPAlignedClause( if (!Ty || (!Ty->isArrayType() && !Ty->isPointerType())) { Diag(ELoc, diag::err_omp_aligned_expected_array_or_ptr) << QType << getLangOpts().CPlusPlus << ERange; - bool IsDecl = - !VD || - VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly; + bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) == + VarDecl::DeclarationOnly; Diag(D->getLocation(), IsDecl ? diag::note_previous_decl : diag::note_defined_here) << D; @@ -18327,9 +18356,8 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef VarList, Diag(ELoc, diag::err_omp_variably_modified_type_not_supported) << getOpenMPClauseName(OMPC_copyprivate) << Type << getOpenMPDirectiveName(DSAStack->getCurrentDirective()); - bool IsDecl = - !VD || - VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly; + bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) == + VarDecl::DeclarationOnly; Diag(D->getLocation(), IsDecl ? diag::note_previous_decl : diag::note_defined_here) << D; @@ -18636,22 +18664,19 @@ Sema::ActOnOpenMPDependClause(Expr *DepModifier, OpenMPDependClauseKind DepKind, if (!RefExpr->isValueDependent() && !RefExpr->isTypeDependent() && !RefExpr->isInstantiationDependent() && !RefExpr->containsUnexpandedParameterPack() && - (OMPDependTFound && - DSAStack->getOMPDependT().getTypePtr() == ExprTy.getTypePtr())) { + (!RefExpr->IgnoreParenImpCasts()->isLValue() || + (OMPDependTFound && + DSAStack->getOMPDependT().getTypePtr() == ExprTy.getTypePtr()))) { Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item) - << (LangOpts.OpenMP >= 50 ? 1 : 0) << 1 - << RefExpr->getSourceRange(); + << (LangOpts.OpenMP >= 50 ? 1 : 0) + << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange(); continue; } auto *ASE = dyn_cast(SimpleExpr); - if (!RefExpr->IgnoreParenImpCasts()->isLValue() || - (ASE && !ASE->getBase()->isTypeDependent() && - !ASE->getBase() - ->getType() - .getNonReferenceType() - ->isPointerType() && - !ASE->getBase()->getType().getNonReferenceType()->isArrayType())) { + if (ASE && !ASE->getBase()->isTypeDependent() && + !ASE->getBase()->getType().getNonReferenceType()->isPointerType() && + !ASE->getBase()->getType().getNonReferenceType()->isArrayType()) { Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item) << (LangOpts.OpenMP >= 50 ? 1 : 0) << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange(); @@ -18934,7 +18959,7 @@ class MapBaseChecker final : public StmtVisitor { if (!isa(ME->getMemberDecl())) { if (!NoDiagnose) { SemaRef.Diag(ELoc, diag::err_omp_expected_access_to_data_field) - << ME->getSourceRange(); + << ME->getSourceRange(); return false; } if (RelevantExpr) @@ -18950,7 +18975,7 @@ class MapBaseChecker final : public StmtVisitor { if (FD->isBitField()) { if (!NoDiagnose) { SemaRef.Diag(ELoc, diag::err_omp_bit_fields_forbidden_in_clause) - << ME->getSourceRange() << getOpenMPClauseName(CKind); + << ME->getSourceRange() << getOpenMPClauseName(CKind); return false; } if (RelevantExpr) @@ -18970,7 +18995,7 @@ class MapBaseChecker final : public StmtVisitor { if (CurType->isUnionType()) { if (!NoDiagnose) { SemaRef.Diag(ELoc, diag::err_omp_union_type_not_allowed) - << ME->getSourceRange(); + << ME->getSourceRange(); return false; } return RelevantExpr || Visit(E); @@ -18997,7 +19022,7 @@ class MapBaseChecker final : public StmtVisitor { if (!E->getType()->isAnyPointerType() && !E->getType()->isArrayType()) { if (!NoDiagnose) { SemaRef.Diag(ELoc, diag::err_omp_expected_base_var_name) - << 0 << AE->getSourceRange(); + << 0 << AE->getSourceRange(); return false; } return RelevantExpr || Visit(E); @@ -19006,8 +19031,7 @@ class MapBaseChecker final : public StmtVisitor { // If we got an array subscript that express the whole dimension we // can have any array expressions before. If it only expressing part of // the dimension, we can only have unitary-size array expressions. - if (checkArrayExpressionDoesNotReferToWholeSize(SemaRef, AE, - E->getType())) + if (checkArrayExpressionDoesNotReferToWholeSize(SemaRef, AE, E->getType())) AllowWholeSizeArraySection = false; if (const auto *TE = dyn_cast(E->IgnoreParenCasts())) { @@ -19037,7 +19061,7 @@ class MapBaseChecker final : public StmtVisitor { "Array sections cannot be implicitly mapped."); Expr *E = OASE->getBase()->IgnoreParenImpCasts(); QualType CurType = - OMPArraySectionExpr::getBaseOriginalType(E).getCanonicalType(); + OMPArraySectionExpr::getBaseOriginalType(E).getCanonicalType(); // OpenMP 4.5 [2.15.5.1, map Clause, Restrictions, C++, p.1] // If the type of a list item is a reference to a type T then the type @@ -19049,14 +19073,14 @@ class MapBaseChecker final : public StmtVisitor { if (!IsPointer && !CurType->isArrayType()) { SemaRef.Diag(ELoc, diag::err_omp_expected_base_var_name) - << 0 << OASE->getSourceRange(); + << 0 << OASE->getSourceRange(); return false; } bool NotWhole = - checkArrayExpressionDoesNotReferToWholeSize(SemaRef, OASE, CurType); + checkArrayExpressionDoesNotReferToWholeSize(SemaRef, OASE, CurType); bool NotUnity = - checkArrayExpressionDoesNotReferToUnitySize(SemaRef, OASE, CurType); + checkArrayExpressionDoesNotReferToUnitySize(SemaRef, OASE, CurType); if (AllowWholeSizeArraySection) { // Any array section is currently allowed. Allowing a whole size array @@ -19079,9 +19103,9 @@ class MapBaseChecker final : public StmtVisitor { // compatible with the properties of the current array section. if (NoDiagnose) return false; - SemaRef.Diag( - ELoc, diag::err_array_section_does_not_specify_contiguous_storage) - << OASE->getSourceRange(); + SemaRef.Diag(ELoc, + diag::err_array_section_does_not_specify_contiguous_storage) + << OASE->getSourceRange(); return false; } @@ -19180,9 +19204,7 @@ class MapBaseChecker final : public StmtVisitor { emitErrorMsg(); return false; } - const Expr *getFoundBase() const { - return RelevantExpr; - } + const Expr *getFoundBase() const { return RelevantExpr; } explicit MapBaseChecker( Sema &SemaRef, OpenMPClauseKind CKind, OpenMPDirectiveKind DKind, OMPClauseMappableExprCommon::MappableExprComponentList &Components, @@ -19193,9 +19215,9 @@ class MapBaseChecker final : public StmtVisitor { } // namespace /// Return the expression of the base of the mappable expression or null if it -/// cannot be determined and do all the necessary checks to see if the expression -/// is valid as a standalone mappable expression. In the process, record all the -/// components of the expression. +/// cannot be determined and do all the necessary checks to see if the +/// expression is valid as a standalone mappable expression. In the process, +/// record all the components of the expression. static const Expr *checkMapClauseExpressionBase( Sema &SemaRef, Expr *E, OMPClauseMappableExprCommon::MappableExprComponentList &CurComponents, @@ -19385,9 +19407,9 @@ static bool checkMapConflicts( return true; } if (CI->getAssociatedExpression()->getStmtClass() != - SI->getAssociatedExpression()->getStmtClass() || - CI->getAssociatedDeclaration()->getCanonicalDecl() == - SI->getAssociatedDeclaration()->getCanonicalDecl()) { + SI->getAssociatedExpression()->getStmtClass() || + CI->getAssociatedDeclaration()->getCanonicalDecl() == + SI->getAssociatedDeclaration()->getCanonicalDecl()) { assert(CI != CE && SI != SE); SemaRef.Diag(DerivedLoc, diag::err_omp_same_pointer_dereferenced) << DerivedLoc; @@ -19608,7 +19630,7 @@ struct MappableVarListInfo { VarBaseDeclarations.reserve(VarList.size()); } }; -} +} // namespace // Check the validity of the provided variable list for the provided clause kind // \a CKind. In the check process the valid expressions, mappable expression @@ -21341,8 +21363,7 @@ OMPClause *Sema::ActOnOpenMPInclusiveClause(ArrayRef VarList, // A list item that appears in the inclusive or exclusive clause must appear // in a reduction clause with the inscan modifier on the enclosing // worksharing-loop, worksharing-loop SIMD, or simd construct. - if (DVar.CKind != OMPC_reduction || - DVar.Modifier != OMPC_REDUCTION_inscan) + if (DVar.CKind != OMPC_reduction || DVar.Modifier != OMPC_REDUCTION_inscan) Diag(ELoc, diag::err_omp_inclusive_exclusive_not_reduction) << RefExpr->getSourceRange(); diff --git a/clang/lib/Sema/SemaStmtAsm.cpp b/clang/lib/Sema/SemaStmtAsm.cpp index 603611b2d86b..49f7a8d573d5 100644 --- a/clang/lib/Sema/SemaStmtAsm.cpp +++ b/clang/lib/Sema/SemaStmtAsm.cpp @@ -296,9 +296,9 @@ StmtResult Sema::ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple, checkExprMemoryConstraintCompat(*this, OutputExpr, Info, false)) return StmtError(); - // Disallow _ExtInt, since the backends tend to have difficulties with - // non-normal sizes. - if (OutputExpr->getType()->isExtIntType()) + // Disallow bit-precise integer types, since the backends tend to have + // difficulties with abnormal sizes. + if (OutputExpr->getType()->isBitIntType()) return StmtError( Diag(OutputExpr->getBeginLoc(), diag::err_asm_invalid_type) << OutputExpr->getType() << 0 /*Input*/ @@ -429,7 +429,7 @@ StmtResult Sema::ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple, } } - if (InputExpr->getType()->isExtIntType()) + if (InputExpr->getType()->isBitIntType()) return StmtError( Diag(InputExpr->getBeginLoc(), diag::err_asm_invalid_type) << InputExpr->getType() << 1 /*Output*/ @@ -924,7 +924,7 @@ StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc, SourceLocation LBraceLoc, setFunctionHasBranchProtectedScope(); for (uint64_t I = 0; I < NumOutputs + NumInputs; ++I) { - if (Exprs[I]->getType()->isExtIntType()) + if (Exprs[I]->getType()->isBitIntType()) return StmtError( Diag(Exprs[I]->getBeginLoc(), diag::err_asm_invalid_type) << Exprs[I]->getType() << (I < NumOutputs) diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index f4fd2ea5aa8e..2482f6d404ea 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -6142,12 +6142,12 @@ bool UnnamedLocalNoLinkageFinder::VisitPipeType(const PipeType* T) { return false; } -bool UnnamedLocalNoLinkageFinder::VisitExtIntType(const ExtIntType *T) { +bool UnnamedLocalNoLinkageFinder::VisitBitIntType(const BitIntType *T) { return false; } -bool UnnamedLocalNoLinkageFinder::VisitDependentExtIntType( - const DependentExtIntType *T) { +bool UnnamedLocalNoLinkageFinder::VisitDependentBitIntType( + const DependentBitIntType *T) { return false; } @@ -7089,7 +7089,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, QualType IntegerType = ParamType; if (const EnumType *Enum = IntegerType->getAs()) IntegerType = Enum->getDecl()->getIntegerType(); - Value = Value.extOrTrunc(IntegerType->isExtIntType() + Value = Value.extOrTrunc(IntegerType->isBitIntType() ? Context.getIntWidth(IntegerType) : Context.getTypeSize(IntegerType)); @@ -7184,7 +7184,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, // Coerce the template argument's value to the value it will have // based on the template parameter's type. - unsigned AllowedBits = IntegerType->isExtIntType() + unsigned AllowedBits = IntegerType->isBitIntType() ? Context.getIntWidth(IntegerType) : Context.getTypeSize(IntegerType); if (Value.getBitWidth() != AllowedBits) diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index 81edae10335d..e9636d2b942e 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -1597,7 +1597,7 @@ static Sema::TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch( case Type::ObjCObject: case Type::ObjCInterface: case Type::ObjCObjectPointer: - case Type::ExtInt: + case Type::BitInt: return (TDF & TDF_SkipNonDependent) || ((TDF & TDF_IgnoreQualifiers) ? S.Context.hasSameUnqualifiedType(P, A) @@ -2144,10 +2144,10 @@ static Sema::TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch( return Sema::TDK_NonDeducedMismatch; } - case Type::DependentExtInt: { - const auto *IP = P->castAs(); + case Type::DependentBitInt: { + const auto *IP = P->castAs(); - if (const auto *IA = A->getAs()) { + if (const auto *IA = A->getAs()) { if (IP->isUnsigned() != IA->isUnsigned()) return Sema::TDK_NonDeducedMismatch; @@ -2164,7 +2164,7 @@ static Sema::TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch( Deduced); } - if (const auto *IA = A->getAs()) { + if (const auto *IA = A->getAs()) { if (IP->isUnsigned() != IA->isUnsigned()) return Sema::TDK_NonDeducedMismatch; return Sema::TDK_Success; @@ -5949,9 +5949,9 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T, cast(T)->getDeducedType(), OnlyDeduced, Depth, Used); break; - case Type::DependentExtInt: + case Type::DependentBitInt: MarkUsedTemplateParameters(Ctx, - cast(T)->getNumBitsExpr(), + cast(T)->getNumBitsExpr(), OnlyDeduced, Depth, Used); break; @@ -5966,7 +5966,7 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T, case Type::ObjCObjectPointer: case Type::UnresolvedUsing: case Type::Pipe: - case Type::ExtInt: + case Type::BitInt: #define TYPE(Class, Base) #define ABSTRACT_TYPE(Class, Base) #define DEPENDENT_TYPE(Class, Base) diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp index c0bb310e64fb..51c79e93ab0a 100644 --- a/clang/lib/Sema/SemaTemplateVariadic.cpp +++ b/clang/lib/Sema/SemaTemplateVariadic.cpp @@ -870,7 +870,7 @@ bool Sema::containsUnexpandedParameterPacks(Declarator &D) { case TST_typeofExpr: case TST_decltype: - case TST_extint: + case TST_bitint: if (DS.getRepAsExpr() && DS.getRepAsExpr()->containsUnexpandedParameterPack()) return true; diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index d2ee669debd0..7a038301a249 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -1435,12 +1435,11 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) { } break; } - case DeclSpec::TST_extint: { - if (!S.Context.getTargetInfo().hasExtIntType()) - S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported) - << "_ExtInt"; + case DeclSpec::TST_bitint: { + if (!S.Context.getTargetInfo().hasBitIntType()) + S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported) << "_BitInt"; Result = - S.BuildExtIntType(DS.getTypeSpecSign() == TypeSpecifierSign::Unsigned, + S.BuildBitIntType(DS.getTypeSpecSign() == TypeSpecifierSign::Unsigned, DS.getRepAsExpr(), DS.getBeginLoc()); if (Result.isNull()) { Result = Context.IntTy; @@ -2237,7 +2236,7 @@ QualType Sema::BuildWritePipeType(QualType T, SourceLocation Loc) { return Context.getWritePipeType(T); } -/// Build a extended int type. +/// Build a bit-precise integer type. /// /// \param IsUnsigned Boolean representing the signedness of the type. /// @@ -2245,10 +2244,10 @@ QualType Sema::BuildWritePipeType(QualType T, SourceLocation Loc) { /// that. /// /// \param Loc Location of the keyword. -QualType Sema::BuildExtIntType(bool IsUnsigned, Expr *BitWidth, +QualType Sema::BuildBitIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc) { if (BitWidth->isInstantiationDependent()) - return Context.getDependentExtIntType(IsUnsigned, BitWidth); + return Context.getDependentBitIntType(IsUnsigned, BitWidth); llvm::APSInt Bits(32); ExprResult ICE = @@ -2259,22 +2258,22 @@ QualType Sema::BuildExtIntType(bool IsUnsigned, Expr *BitWidth, int64_t NumBits = Bits.getSExtValue(); if (!IsUnsigned && NumBits < 2) { - Diag(Loc, diag::err_ext_int_bad_size) << 0; + Diag(Loc, diag::err_bit_int_bad_size) << 0; return QualType(); } if (IsUnsigned && NumBits < 1) { - Diag(Loc, diag::err_ext_int_bad_size) << 1; + Diag(Loc, diag::err_bit_int_bad_size) << 1; return QualType(); } if (NumBits > llvm::IntegerType::MAX_INT_BITS) { - Diag(Loc, diag::err_ext_int_max_size) << IsUnsigned - << llvm::IntegerType::MAX_INT_BITS; + Diag(Loc, diag::err_bit_int_max_size) + << IsUnsigned << llvm::IntegerType::MAX_INT_BITS; return QualType(); } - return Context.getExtIntType(IsUnsigned, NumBits); + return Context.getBitIntType(IsUnsigned, NumBits); } /// Check whether the specified array bound can be evaluated using the relevant @@ -3941,6 +3940,20 @@ static CallingConv getCCForDeclaratorChunk( break; } } + } else if (S.getLangOpts().CUDA) { + // If we're compiling CUDA/HIP code and targeting SPIR-V we need to make + // sure the kernels will be marked with the right calling convention so that + // they will be visible by the APIs that ingest SPIR-V. + llvm::Triple Triple = S.Context.getTargetInfo().getTriple(); + if (Triple.getArch() == llvm::Triple::spirv32 || + Triple.getArch() == llvm::Triple::spirv64) { + for (const ParsedAttr &AL : D.getDeclSpec().getAttributes()) { + if (AL.getKind() == ParsedAttr::AT_CUDAGlobal) { + CC = CC_OpenCLKernel; + break; + } + } + } } return CC; @@ -6077,11 +6090,11 @@ namespace { TL.getValueLoc().initializeFullCopy(TInfo->getTypeLoc()); } - void VisitExtIntTypeLoc(ExtIntTypeLoc TL) { + void VisitExtIntTypeLoc(BitIntTypeLoc TL) { TL.setNameLoc(DS.getTypeSpecTypeLoc()); } - void VisitDependentExtIntTypeLoc(DependentExtIntTypeLoc TL) { + void VisitDependentExtIntTypeLoc(DependentBitIntTypeLoc TL) { TL.setNameLoc(DS.getTypeSpecTypeLoc()); } @@ -6211,7 +6224,7 @@ namespace { assert(Chunk.Kind == DeclaratorChunk::Pipe); TL.setKWLoc(Chunk.Loc); } - void VisitExtIntTypeLoc(ExtIntTypeLoc TL) { + void VisitBitIntTypeLoc(BitIntTypeLoc TL) { TL.setNameLoc(Chunk.Loc); } void VisitMacroQualifiedTypeLoc(MacroQualifiedTypeLoc TL) { @@ -6528,7 +6541,6 @@ static void HandleBTFTypeTagAttribute(QualType &Type, const ParsedAttr &Attr, StringRef BTFTypeTag = StrLiteral->getString(); Type = State.getAttributedType( ::new (Ctx) BTFTypeTagAttr(Ctx, Attr, BTFTypeTag), Type, Type); - return; } /// HandleAddressSpaceTypeAttribute - Process an address_space attribute on the @@ -9079,9 +9091,8 @@ QualType Sema::BuildAtomicType(QualType T, SourceLocation Loc) { else if (!T.isTriviallyCopyableType(Context)) // Some other non-trivially-copyable type (probably a C++ class) DisallowedKind = 7; - else if (T->isExtIntType()) { - DisallowedKind = 8; - } + else if (T->isBitIntType()) + DisallowedKind = 8; if (DisallowedKind != -1) { Diag(Loc, diag::err_atomic_specifier_bad_type) << DisallowedKind << T; diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 7f3326c13263..298a3f7a83d8 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -933,6 +933,11 @@ class TreeTransform { /// the UnresolvedUsingTypenameDecl was transformed to. QualType RebuildUnresolvedUsingType(SourceLocation NameLoc, Decl *D); + /// Build a new type found via an alias. + QualType RebuildUsingType(UsingShadowDecl *Found, QualType Underlying) { + return SemaRef.Context.getUsingType(Found, Underlying); + } + /// Build a new typedef type. QualType RebuildTypedefType(TypedefNameDecl *Typedef) { return SemaRef.Context.getTypeDeclType(Typedef); @@ -1195,12 +1200,12 @@ class TreeTransform { QualType RebuildPipeType(QualType ValueType, SourceLocation KWLoc, bool isReadPipe); - /// Build an extended int given its value type. - QualType RebuildExtIntType(bool IsUnsigned, unsigned NumBits, + /// Build a bit-precise int given its value type. + QualType RebuildBitIntType(bool IsUnsigned, unsigned NumBits, SourceLocation Loc); - /// Build a dependent extended int given its value type. - QualType RebuildDependentExtIntType(bool IsUnsigned, Expr *NumBitsExpr, + /// Build a dependent bit-precise int given its value type. + QualType RebuildDependentBitIntType(bool IsUnsigned, Expr *NumBitsExpr, SourceLocation Loc); /// Build a new template name given a nested name specifier, a flag @@ -6072,9 +6077,9 @@ QualType TreeTransform::TransformFunctionNoProtoType( return Result; } -template QualType -TreeTransform::TransformUnresolvedUsingType(TypeLocBuilder &TLB, - UnresolvedUsingTypeLoc TL) { +template +QualType TreeTransform::TransformUnresolvedUsingType( + TypeLocBuilder &TLB, UnresolvedUsingTypeLoc TL) { const UnresolvedUsingType *T = TL.getTypePtr(); Decl *D = getDerived().TransformDecl(TL.getNameLoc(), T->getDecl()); if (!D) @@ -6095,6 +6100,32 @@ TreeTransform::TransformUnresolvedUsingType(TypeLocBuilder &TLB, return Result; } +template +QualType TreeTransform::TransformUsingType(TypeLocBuilder &TLB, + UsingTypeLoc TL) { + const UsingType *T = TL.getTypePtr(); + + auto *Found = cast_or_null(getDerived().TransformDecl( + TL.getLocalSourceRange().getBegin(), T->getFoundDecl())); + if (!Found) + return QualType(); + + QualType Underlying = getDerived().TransformType(T->desugar()); + if (Underlying.isNull()) + return QualType(); + + QualType Result = TL.getType(); + if (getDerived().AlwaysRebuild() || Found != T->getFoundDecl() || + Underlying != T->getUnderlyingType()) { + Result = getDerived().RebuildUsingType(Found, Underlying); + if (Result.isNull()) + return QualType(); + } + + TLB.pushTypeSpec(Result).setNameLoc(TL.getNameLoc()); + return Result; +} + template QualType TreeTransform::TransformTypedefType(TypeLocBuilder &TLB, TypedefTypeLoc TL) { @@ -6430,27 +6461,27 @@ QualType TreeTransform::TransformPipeType(TypeLocBuilder &TLB, } template -QualType TreeTransform::TransformExtIntType(TypeLocBuilder &TLB, - ExtIntTypeLoc TL) { - const ExtIntType *EIT = TL.getTypePtr(); +QualType TreeTransform::TransformBitIntType(TypeLocBuilder &TLB, + BitIntTypeLoc TL) { + const BitIntType *EIT = TL.getTypePtr(); QualType Result = TL.getType(); if (getDerived().AlwaysRebuild()) { - Result = getDerived().RebuildExtIntType(EIT->isUnsigned(), + Result = getDerived().RebuildBitIntType(EIT->isUnsigned(), EIT->getNumBits(), TL.getNameLoc()); if (Result.isNull()) return QualType(); } - ExtIntTypeLoc NewTL = TLB.push(Result); + BitIntTypeLoc NewTL = TLB.push(Result); NewTL.setNameLoc(TL.getNameLoc()); return Result; } template -QualType TreeTransform::TransformDependentExtIntType( - TypeLocBuilder &TLB, DependentExtIntTypeLoc TL) { - const DependentExtIntType *EIT = TL.getTypePtr(); +QualType TreeTransform::TransformDependentBitIntType( + TypeLocBuilder &TLB, DependentBitIntTypeLoc TL) { + const DependentBitIntType *EIT = TL.getTypePtr(); EnterExpressionEvaluationContext Unevaluated( SemaRef, Sema::ExpressionEvaluationContext::ConstantEvaluated); @@ -6463,18 +6494,18 @@ QualType TreeTransform::TransformDependentExtIntType( QualType Result = TL.getType(); if (getDerived().AlwaysRebuild() || BitsExpr.get() != EIT->getNumBitsExpr()) { - Result = getDerived().RebuildDependentExtIntType( + Result = getDerived().RebuildDependentBitIntType( EIT->isUnsigned(), BitsExpr.get(), TL.getNameLoc()); if (Result.isNull()) return QualType(); } - if (isa(Result)) { - DependentExtIntTypeLoc NewTL = TLB.push(Result); + if (isa(Result)) { + DependentBitIntTypeLoc NewTL = TLB.push(Result); NewTL.setNameLoc(TL.getNameLoc()); } else { - ExtIntTypeLoc NewTL = TLB.push(Result); + BitIntTypeLoc NewTL = TLB.push(Result); NewTL.setNameLoc(TL.getNameLoc()); } return Result; @@ -9429,6 +9460,13 @@ TreeTransform::TransformOMPCaptureClause(OMPCaptureClause *C) { return C; } +template +OMPClause * +TreeTransform::TransformOMPCompareClause(OMPCompareClause *C) { + // No need to rebuild this clause, no template-dependent parameters. + return C; +} + template OMPClause * TreeTransform::TransformOMPSeqCstClause(OMPSeqCstClause *C) { @@ -14462,7 +14500,6 @@ QualType TreeTransform::RebuildUnresolvedUsingType(SourceLocation Loc, if (D->isInvalidDecl()) return QualType(); // FIXME: Doesn't account for ObjCInterfaceDecl! - TypeDecl *Ty; if (auto *UPD = dyn_cast(D)) { // A valid resolved using typename pack expansion decl can have multiple // UsingDecls, but they must each have exactly one type, and it must be @@ -14498,17 +14535,18 @@ QualType TreeTransform::RebuildUnresolvedUsingType(SourceLocation Loc, // A valid resolved using typename decl points to exactly one type decl. assert(++Using->shadow_begin() == Using->shadow_end()); - NamedDecl *Target = Using->shadow_begin()->getTargetDecl(); - if (SemaRef.DiagnoseUseOfDecl(Target, Loc)) + UsingShadowDecl *Shadow = *Using->shadow_begin(); + if (SemaRef.DiagnoseUseOfDecl(Shadow->getTargetDecl(), Loc)) return QualType(); - Ty = cast(Target); + return SemaRef.Context.getUsingType( + Shadow, SemaRef.Context.getTypeDeclType( + cast(Shadow->getTargetDecl()))); } else { assert(isa(D) && "UnresolvedUsingTypenameDecl transformed to non-using decl"); - Ty = cast(D); + return SemaRef.Context.getTypeDeclType( + cast(D)); } - - return SemaRef.Context.getTypeDeclType(Ty); } template @@ -14557,20 +14595,20 @@ QualType TreeTransform::RebuildPipeType(QualType ValueType, } template -QualType TreeTransform::RebuildExtIntType(bool IsUnsigned, +QualType TreeTransform::RebuildBitIntType(bool IsUnsigned, unsigned NumBits, SourceLocation Loc) { llvm::APInt NumBitsAP(SemaRef.Context.getIntWidth(SemaRef.Context.IntTy), NumBits, true); IntegerLiteral *Bits = IntegerLiteral::Create(SemaRef.Context, NumBitsAP, SemaRef.Context.IntTy, Loc); - return SemaRef.BuildExtIntType(IsUnsigned, Bits, Loc); + return SemaRef.BuildBitIntType(IsUnsigned, Bits, Loc); } template -QualType TreeTransform::RebuildDependentExtIntType( +QualType TreeTransform::RebuildDependentBitIntType( bool IsUnsigned, Expr *NumBitsExpr, SourceLocation Loc) { - return SemaRef.BuildExtIntType(IsUnsigned, NumBitsExpr, Loc); + return SemaRef.BuildBitIntType(IsUnsigned, NumBitsExpr, Loc); } template diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index a033bccbe506..f93e0d2ed1c4 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -6607,6 +6607,10 @@ void TypeLocReader::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) { TL.setNameLoc(readSourceLocation()); } +void TypeLocReader::VisitUsingTypeLoc(UsingTypeLoc TL) { + TL.setNameLoc(readSourceLocation()); +} + void TypeLocReader::VisitTypedefTypeLoc(TypedefTypeLoc TL) { TL.setNameLoc(readSourceLocation()); } @@ -6772,11 +6776,11 @@ void TypeLocReader::VisitPipeTypeLoc(PipeTypeLoc TL) { TL.setKWLoc(readSourceLocation()); } -void TypeLocReader::VisitExtIntTypeLoc(clang::ExtIntTypeLoc TL) { +void TypeLocReader::VisitBitIntTypeLoc(clang::BitIntTypeLoc TL) { TL.setNameLoc(readSourceLocation()); } -void TypeLocReader::VisitDependentExtIntTypeLoc( - clang::DependentExtIntTypeLoc TL) { +void TypeLocReader::VisitDependentBitIntTypeLoc( + clang::DependentBitIntTypeLoc TL) { TL.setNameLoc(readSourceLocation()); } @@ -11761,6 +11765,9 @@ OMPClause *OMPClauseReader::readClause() { case llvm::omp::OMPC_capture: C = new (Context) OMPCaptureClause(); break; + case llvm::omp::OMPC_compare: + C = new (Context) OMPCompareClause(); + break; case llvm::omp::OMPC_seq_cst: C = new (Context) OMPSeqCstClause(); break; @@ -12119,6 +12126,8 @@ void OMPClauseReader::VisitOMPUpdateClause(OMPUpdateClause *C) { void OMPClauseReader::VisitOMPCaptureClause(OMPCaptureClause *) {} +void OMPClauseReader::VisitOMPCompareClause(OMPCompareClause *) {} + void OMPClauseReader::VisitOMPSeqCstClause(OMPSeqCstClause *) {} void OMPClauseReader::VisitOMPAcqRelClause(OMPAcqRelClause *) {} diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index 62a31f299d6b..2144befcdb14 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2948,6 +2948,7 @@ uint64_t ASTReader::getGlobalBitOffset(ModuleFile &M, uint64_t LocalOffset) { static bool isSameTemplateParameterList(const ASTContext &C, const TemplateParameterList *X, const TemplateParameterList *Y); +static bool isSameEntity(NamedDecl *X, NamedDecl *Y); /// Determine whether two template parameters are similar enough /// that they may be used in declarations of the same template. @@ -2967,7 +2968,9 @@ static bool isSameTemplateParameter(const NamedDecl *X, if (!TXTC != !TYTC) return false; if (TXTC && TYTC) { - if (TXTC->getNamedConcept() != TYTC->getNamedConcept()) + auto *NCX = TXTC->getNamedConcept(); + auto *NCY = TYTC->getNamedConcept(); + if (!NCX || !NCY || !isSameEntity(NCX, NCY)) return false; if (TXTC->hasExplicitTemplateArgs() != TYTC->hasExplicitTemplateArgs()) return false; @@ -3111,11 +3114,12 @@ static bool hasSameOverloadableAttrs(const FunctionDecl *A, /// Determine whether the two declarations refer to the same entity. static bool isSameEntity(NamedDecl *X, NamedDecl *Y) { - assert(X->getDeclName() == Y->getDeclName() && "Declaration name mismatch!"); - if (X == Y) return true; + if (X->getDeclName() != Y->getDeclName()) + return false; + // Must be in the same context. // // Note that we can't use DeclContext::Equals here, because the DeclContexts diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index a1972f5c6496..65a780e67510 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -396,6 +396,10 @@ void TypeLocWriter::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) { Record.AddSourceLocation(TL.getNameLoc()); } +void TypeLocWriter::VisitUsingTypeLoc(UsingTypeLoc TL) { + Record.AddSourceLocation(TL.getNameLoc()); +} + void TypeLocWriter::VisitTypedefTypeLoc(TypedefTypeLoc TL) { Record.AddSourceLocation(TL.getNameLoc()); } @@ -562,11 +566,11 @@ void TypeLocWriter::VisitPipeTypeLoc(PipeTypeLoc TL) { Record.AddSourceLocation(TL.getKWLoc()); } -void TypeLocWriter::VisitExtIntTypeLoc(clang::ExtIntTypeLoc TL) { +void TypeLocWriter::VisitBitIntTypeLoc(clang::BitIntTypeLoc TL) { Record.AddSourceLocation(TL.getNameLoc()); } -void TypeLocWriter::VisitDependentExtIntTypeLoc( - clang::DependentExtIntTypeLoc TL) { +void TypeLocWriter::VisitDependentBitIntTypeLoc( + clang::DependentBitIntTypeLoc TL) { Record.AddSourceLocation(TL.getNameLoc()); } @@ -6248,6 +6252,8 @@ void OMPClauseWriter::VisitOMPUpdateClause(OMPUpdateClause *C) { void OMPClauseWriter::VisitOMPCaptureClause(OMPCaptureClause *) {} +void OMPClauseWriter::VisitOMPCompareClause(OMPCompareClause *) {} + void OMPClauseWriter::VisitOMPSeqCstClause(OMPSeqCstClause *) {} void OMPClauseWriter::VisitOMPAcqRelClause(OMPAcqRelClause *) {} diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp index 13781b336426..4a56156de4b2 100644 --- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp @@ -66,7 +66,8 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call, case Builtin::BI__builtin_expect: case Builtin::BI__builtin_expect_with_probability: case Builtin::BI__builtin_assume_aligned: - case Builtin::BI__builtin_addressof: { + case Builtin::BI__builtin_addressof: + case Builtin::BI__builtin_function_start: { // For __builtin_unpredictable, __builtin_expect, // __builtin_expect_with_probability and __builtin_assume_aligned, // just return the value of the subexpression. diff --git a/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp index 8da482a2aec9..6ea39fb95e9a 100644 --- a/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp @@ -56,9 +56,8 @@ class ConversionChecker : public Checker> { void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast, CheckerContext &C) const { - // TODO: For now we only warn about DeclRefExpr, to avoid noise. Warn for - // calculations also. - if (!isa(Cast->IgnoreParenImpCasts())) + // Don't warn for implicit conversions to bool + if (Cast->getType()->isBooleanType()) return; // Don't warn for loss of sign/precision in macros. @@ -70,6 +69,9 @@ void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast, const Stmt *Parent = PM.getParent(Cast); if (!Parent) return; + // Dont warn if this is part of an explicit cast + if (isa(Parent)) + return; bool LossOfSign = false; bool LossOfPrecision = false; @@ -78,8 +80,10 @@ void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast, if (const auto *B = dyn_cast(Parent)) { BinaryOperator::Opcode Opc = B->getOpcode(); if (Opc == BO_Assign) { - LossOfSign = isLossOfSign(Cast, C); - LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C); + if (!Cast->IgnoreParenImpCasts()->isEvaluatable(C.getASTContext())) { + LossOfSign = isLossOfSign(Cast, C); + LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C); + } } else if (Opc == BO_AddAssign || Opc == BO_SubAssign) { // No loss of sign. LossOfPrecision = isLossOfPrecision(Cast, B->getLHS()->getType(), C); @@ -98,7 +102,12 @@ void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast, } else if (B->isRelationalOp() || B->isMultiplicativeOp()) { LossOfSign = isLossOfSign(Cast, C); } - } else if (isa(Parent)) { + } else if (isa(Parent)) { + if (!Cast->IgnoreParenImpCasts()->isEvaluatable(C.getASTContext())) { + LossOfSign = isLossOfSign(Cast, C); + LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C); + } + } else { LossOfSign = isLossOfSign(Cast, C); LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C); } diff --git a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp index e3f4be0726c8..6e4801aa8e91 100644 --- a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp @@ -389,7 +389,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call, llvm::raw_string_ostream OS(SBuf); OS << "Function '" << FuncDecl->getDeclName() << "' returns an open handle"; - return OS.str(); + return SBuf; } else return ""; }); @@ -405,7 +405,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call, llvm::raw_string_ostream OS(SBuf); OS << "Function '" << FuncDecl->getDeclName() << "' returns an unowned handle"; - return OS.str(); + return SBuf; } else return ""; }); @@ -439,7 +439,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call, llvm::raw_string_ostream OS(SBuf); OS << "Handle released through " << ParamDiagIdx << llvm::getOrdinalSuffix(ParamDiagIdx) << " parameter"; - return OS.str(); + return SBuf; } else return ""; }); @@ -453,7 +453,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call, llvm::raw_string_ostream OS(SBuf); OS << "Handle allocated through " << ParamDiagIdx << llvm::getOrdinalSuffix(ParamDiagIdx) << " parameter"; - return OS.str(); + return SBuf; } else return ""; }); @@ -467,7 +467,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call, llvm::raw_string_ostream OS(SBuf); OS << "Unowned handle allocated through " << ParamDiagIdx << llvm::getOrdinalSuffix(ParamDiagIdx) << " parameter"; - return OS.str(); + return SBuf; } else return ""; }); diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h b/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h index 6a40f8eda5fa..b4352b450c7f 100644 --- a/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h +++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h @@ -25,8 +25,6 @@ bool isStdSmartPtrCall(const CallEvent &Call); bool isStdSmartPtr(const CXXRecordDecl *RD); bool isStdSmartPtr(const Expr *E); -bool isStdSmartPtr(const CXXRecordDecl *RD); - /// Returns whether the smart pointer is null or not. bool isNullSmartPtr(const ProgramStateRef State, const MemRegion *ThisRegion); diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp index 764dad3e7ab4..ae46709340d3 100644 --- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp +++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp @@ -762,9 +762,9 @@ void CXXInstanceCall::getInitialStackFrameContents( QualType Ty = Ctx.getPointerType(Ctx.getRecordType(Class)); // FIXME: CallEvent maybe shouldn't be directly accessing StoreManager. - bool Failed; - ThisVal = StateMgr.getStoreManager().attemptDownCast(ThisVal, Ty, Failed); - if (Failed) { + Optional V = + StateMgr.getStoreManager().evalBaseToDerived(ThisVal, Ty); + if (!V.hasValue()) { // We might have suffered some sort of placement new earlier, so // we're constructing in a completely unexpected storage. // Fall back to a generic pointer cast for this-value. @@ -772,7 +772,8 @@ void CXXInstanceCall::getInitialStackFrameContents( const CXXRecordDecl *StaticClass = StaticMD->getParent(); QualType StaticTy = Ctx.getPointerType(Ctx.getRecordType(StaticClass)); ThisVal = SVB.evalCast(ThisVal, Ty, StaticTy); - } + } else + ThisVal = *V; } if (!ThisVal.isUnknown()) diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp index 69d67cf9b465..637e4edfd778 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp @@ -439,14 +439,15 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, if (CastE->isGLValue()) resultType = getContext().getPointerType(resultType); - bool Failed = false; + bool Failed = true; - // Check if the value being cast evaluates to 0. - if (val.isZeroConstant()) - Failed = true; - // Else, evaluate the cast. - else - val = getStoreManager().attemptDownCast(val, T, Failed); + // Check if the value being cast does not evaluates to 0. + if (!val.isZeroConstant()) + if (Optional V = + StateMgr.getStoreManager().evalBaseToDerived(val, T)) { + val = *V; + Failed = false; + } if (Failed) { if (T->isReferenceType()) { @@ -478,14 +479,13 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, if (CastE->isGLValue()) resultType = getContext().getPointerType(resultType); - bool Failed = false; - if (!val.isConstant()) { - val = getStoreManager().attemptDownCast(val, T, Failed); + Optional V = getStoreManager().evalBaseToDerived(val, T); + val = V ? *V : UnknownVal(); } // Failed to cast or the result is unknown, fall back to conservative. - if (Failed || val.isUnknown()) { + if (val.isUnknown()) { val = svalBuilder.conjureSymbolVal(nullptr, CastE, LCtx, resultType, currBldrCtx->blockCount()); diff --git a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp index 3b847d6f0d87..b4578385a147 100644 --- a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp +++ b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp @@ -410,7 +410,7 @@ std::string HTMLDiagnostics::GenerateHTML(const PathDiagnostic& D, Rewriter &R, } // Append files to the main report file in the order they appear in the path - for (auto I : llvm::make_range(FileIDs.begin() + 1, FileIDs.end())) { + for (auto I : llvm::drop_begin(FileIDs)) { std::string s; llvm::raw_string_ostream os(s); @@ -437,7 +437,7 @@ std::string HTMLDiagnostics::GenerateHTML(const PathDiagnostic& D, Rewriter &R, for (auto BI : *Buf) os << BI; - return os.str(); + return file; } void HTMLDiagnostics::dumpCoverageData( @@ -534,7 +534,7 @@ document.addEventListener("DOMContentLoaded", function() { )<<<"; - return os.str(); + return s; } void HTMLDiagnostics::FinalizeHTML(const PathDiagnostic& D, Rewriter &R, @@ -1202,7 +1202,7 @@ std::string getSpanBeginForControl(const char *ClassName, unsigned Index) { std::string Result; llvm::raw_string_ostream OS(Result); OS << ""; - return OS.str(); + return Result; } std::string getSpanBeginForControlStart(unsigned Index) { diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp index f77fcb030a15..58bea12f8783 100644 --- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp +++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp @@ -444,7 +444,7 @@ std::string MemRegion::getString() const { std::string s; llvm::raw_string_ostream os(s); dumpToStream(os); - return os.str(); + return s; } void MemRegion::dumpToStream(raw_ostream &os) const { diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp index 23c67c64f975..4b0d4942e528 100644 --- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp +++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp @@ -110,6 +110,14 @@ class OperatorRelationsTable { RangeSet::ContainerType RangeSet::Factory::EmptySet{}; +RangeSet RangeSet::Factory::add(RangeSet LHS, RangeSet RHS) { + ContainerType Result; + Result.reserve(LHS.size() + RHS.size()); + std::merge(LHS.begin(), LHS.end(), RHS.begin(), RHS.end(), + std::back_inserter(Result)); + return makePersistent(std::move(Result)); +} + RangeSet RangeSet::Factory::add(RangeSet Original, Range Element) { ContainerType Result; Result.reserve(Original.size() + 1); @@ -126,6 +134,186 @@ RangeSet RangeSet::Factory::add(RangeSet Original, const llvm::APSInt &Point) { return add(Original, Range(Point)); } +RangeSet RangeSet::Factory::unite(RangeSet LHS, RangeSet RHS) { + ContainerType Result = unite(*LHS.Impl, *RHS.Impl); + return makePersistent(std::move(Result)); +} + +RangeSet RangeSet::Factory::unite(RangeSet Original, Range R) { + ContainerType Result; + Result.push_back(R); + Result = unite(*Original.Impl, Result); + return makePersistent(std::move(Result)); +} + +RangeSet RangeSet::Factory::unite(RangeSet Original, llvm::APSInt Point) { + return unite(Original, Range(ValueFactory.getValue(Point))); +} + +RangeSet RangeSet::Factory::unite(RangeSet Original, llvm::APSInt From, + llvm::APSInt To) { + return unite(Original, + Range(ValueFactory.getValue(From), ValueFactory.getValue(To))); +} + +template +void swapIterators(T &First, T &FirstEnd, T &Second, T &SecondEnd) { + std::swap(First, Second); + std::swap(FirstEnd, SecondEnd); +} + +RangeSet::ContainerType RangeSet::Factory::unite(const ContainerType &LHS, + const ContainerType &RHS) { + if (LHS.empty()) + return RHS; + if (RHS.empty()) + return LHS; + + using llvm::APSInt; + using iterator = ContainerType::const_iterator; + + iterator First = LHS.begin(); + iterator FirstEnd = LHS.end(); + iterator Second = RHS.begin(); + iterator SecondEnd = RHS.end(); + APSIntType Ty = APSIntType(First->From()); + const APSInt Min = Ty.getMinValue(); + + // Handle a corner case first when both range sets start from MIN. + // This helps to avoid complicated conditions below. Specifically, this + // particular check for `MIN` is not needed in the loop below every time + // when we do `Second->From() - One` operation. + if (Min == First->From() && Min == Second->From()) { + if (First->To() > Second->To()) { + // [ First ]---> + // [ Second ]-----> + // MIN^ + // The Second range is entirely inside the First one. + + // Check if Second is the last in its RangeSet. + if (++Second == SecondEnd) + // [ First ]--[ First + 1 ]---> + // [ Second ]---------------------> + // MIN^ + // The Union is equal to First's RangeSet. + return LHS; + } else { + // case 1: [ First ]-----> + // case 2: [ First ]---> + // [ Second ]---> + // MIN^ + // The First range is entirely inside or equal to the Second one. + + // Check if First is the last in its RangeSet. + if (++First == FirstEnd) + // [ First ]-----------------------> + // [ Second ]--[ Second + 1 ]----> + // MIN^ + // The Union is equal to Second's RangeSet. + return RHS; + } + } + + const APSInt One = Ty.getValue(1); + ContainerType Result; + + // This is called when there are no ranges left in one of the ranges. + // Append the rest of the ranges from another range set to the Result + // and return with that. + const auto AppendTheRest = [&Result](iterator I, iterator E) { + Result.append(I, E); + return Result; + }; + + while (true) { + // We want to keep the following invariant at all times: + // ---[ First ------> + // -----[ Second ---> + if (First->From() > Second->From()) + swapIterators(First, FirstEnd, Second, SecondEnd); + + // The Union definitely starts with First->From(). + // ----------[ First ------> + // ------------[ Second ---> + // ----------[ Union ------> + // UnionStart^ + const llvm::APSInt &UnionStart = First->From(); + + // Loop where the invariant holds. + while (true) { + // Skip all enclosed ranges. + // ---[ First ]---> + // -----[ Second ]--[ Second + 1 ]--[ Second + N ]-----> + while (First->To() >= Second->To()) { + // Check if Second is the last in its RangeSet. + if (++Second == SecondEnd) { + // Append the Union. + // ---[ Union ]---> + // -----[ Second ]-----> + // --------[ First ]---> + // UnionEnd^ + Result.emplace_back(UnionStart, First->To()); + // ---[ Union ]-----------------> + // --------------[ First + 1]---> + // Append all remaining ranges from the First's RangeSet. + return AppendTheRest(++First, FirstEnd); + } + } + + // Check if First and Second are disjoint. It means that we find + // the end of the Union. Exit the loop and append the Union. + // ---[ First ]=-------------> + // ------------=[ Second ]---> + // ----MinusOne^ + if (First->To() < Second->From() - One) + break; + + // First is entirely inside the Union. Go next. + // ---[ Union -----------> + // ---- [ First ]--------> + // -------[ Second ]-----> + // Check if First is the last in its RangeSet. + if (++First == FirstEnd) { + // Append the Union. + // ---[ Union ]---> + // -----[ First ]-------> + // --------[ Second ]---> + // UnionEnd^ + Result.emplace_back(UnionStart, Second->To()); + // ---[ Union ]------------------> + // --------------[ Second + 1]---> + // Append all remaining ranges from the Second's RangeSet. + return AppendTheRest(++Second, SecondEnd); + } + + // We know that we are at one of the two cases: + // case 1: --[ First ]---------> + // case 2: ----[ First ]-------> + // --------[ Second ]----------> + // In both cases First starts after Second->From(). + // Make sure that the loop invariant holds. + swapIterators(First, FirstEnd, Second, SecondEnd); + } + + // Here First and Second are disjoint. + // Append the Union. + // ---[ Union ]---------------> + // -----------------[ Second ]---> + // ------[ First ]---------------> + // UnionEnd^ + Result.emplace_back(UnionStart, First->To()); + + // Check if First is the last in its RangeSet. + if (++First == FirstEnd) + // ---[ Union ]---------------> + // --------------[ Second ]---> + // Append all remaining ranges from the Second's RangeSet. + return AppendTheRest(Second, SecondEnd); + } + + llvm_unreachable("Normally, we should not reach here"); +} + RangeSet RangeSet::Factory::getRangeSet(Range From) { ContainerType Result; Result.push_back(From); @@ -155,13 +343,6 @@ RangeSet::ContainerType *RangeSet::Factory::construct(ContainerType &&From) { return new (Buffer) ContainerType(std::move(From)); } -RangeSet RangeSet::Factory::add(RangeSet LHS, RangeSet RHS) { - ContainerType Result; - std::merge(LHS.begin(), LHS.end(), RHS.begin(), RHS.end(), - std::back_inserter(Result)); - return makePersistent(std::move(Result)); -} - const llvm::APSInt &RangeSet::getMinValue() const { assert(!isEmpty()); return begin()->From(); @@ -325,11 +506,6 @@ RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS, const_iterator First = LHS.begin(), Second = RHS.begin(), FirstEnd = LHS.end(), SecondEnd = RHS.end(); - const auto SwapIterators = [&First, &FirstEnd, &Second, &SecondEnd]() { - std::swap(First, Second); - std::swap(FirstEnd, SecondEnd); - }; - // If we ran out of ranges in one set, but not in the other, // it means that those elements are definitely not in the // intersection. @@ -339,7 +515,7 @@ RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS, // ----[ First ----------------------> // --------[ Second -----------------> if (Second->From() < First->From()) - SwapIterators(); + swapIterators(First, FirstEnd, Second, SecondEnd); // Loop where the invariant holds: do { @@ -373,7 +549,7 @@ RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS, if (Second->To() > First->To()) { // Here we make a decision to keep First as the "longer" // range. - SwapIterators(); + swapIterators(First, FirstEnd, Second, SecondEnd); } // At this point, we have the following situation: @@ -2191,42 +2367,6 @@ LLVM_NODISCARD ProgramStateRef reAssume(ProgramStateRef State, Constraint->getMaxValue(), true); } -// Simplify the given symbol with the help of the SValBuilder. In -// SValBuilder::symplifySval, we traverse the symbol tree and query the -// constraint values for the sub-trees and if a value is a constant we do the -// constant folding. Compound symbols might collapse to simpler symbol tree -// that is still possible to further simplify. Thus, we do the simplification on -// a new symbol tree until we reach the simplest form, i.e. the fixpoint. -// -// Consider the following symbol `(b * b) * b * b` which has this tree: -// * -// / \ -// * b -// / \ -// / b -// (b * b) -// Now, if the `b * b == 1` new constraint is added then during the first -// iteration we have the following transformations: -// * * -// / \ / \ -// * b --> b b -// / \ -// / b -// 1 -// We need another iteration to reach the final result `1`. -LLVM_NODISCARD -static SVal simplifyUntilFixpoint(SValBuilder &SVB, ProgramStateRef State, - const SymbolRef Sym) { - SVal Val = SVB.makeSymbolVal(Sym); - SVal SimplifiedVal = SVB.simplifySVal(State, Val); - // Do the simplification until we can. - while (SimplifiedVal != Val) { - Val = SimplifiedVal; - SimplifiedVal = SVB.simplifySVal(State, Val); - } - return SimplifiedVal; -} - // Iterate over all symbols and try to simplify them. Once a symbol is // simplified then we check if we can merge the simplified symbol's equivalence // class to this class. This way, we simplify not just the symbols but the @@ -2238,8 +2378,7 @@ EquivalenceClass::simplify(SValBuilder &SVB, RangeSet::Factory &F, SymbolSet ClassMembers = Class.getClassMembers(State); for (const SymbolRef &MemberSym : ClassMembers) { - const SVal SimplifiedMemberVal = - simplifyUntilFixpoint(SVB, State, MemberSym); + const SVal SimplifiedMemberVal = simplifyToSVal(State, MemberSym); const SymbolRef SimplifiedMemberSym = SimplifiedMemberVal.getAsSymbol(); // The symbol is collapsed to a constant, check if the current State is diff --git a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp index 71bfc86ab8f7..8edcef319088 100644 --- a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp +++ b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp @@ -459,13 +459,23 @@ SVal SValBuilder::evalBinOp(ProgramStateRef state, BinaryOperator::Opcode op, return evalBinOpLN(state, op, *LV, rhs.castAs(), type); } - if (Optional RV = rhs.getAs()) { - // Support pointer arithmetic where the addend is on the left - // and the pointer on the right. - assert(op == BO_Add); + if (const Optional RV = rhs.getAs()) { + const auto IsCommutative = [](BinaryOperatorKind Op) { + return Op == BO_Mul || Op == BO_Add || Op == BO_And || Op == BO_Xor || + Op == BO_Or; + }; - // Commute the operands. - return evalBinOpLN(state, op, *RV, lhs.castAs(), type); + if (IsCommutative(op)) { + // Swap operands. + return evalBinOpLN(state, op, *RV, lhs.castAs(), type); + } + + // If the right operand is a concrete int location then we have nothing + // better but to treat it as a simple nonloc. + if (auto RV = rhs.getAs()) { + const nonloc::ConcreteInt RhsAsLoc = makeIntVal(RV->getValue()); + return evalBinOpNN(state, op, lhs.castAs(), RhsAsLoc, type); + } } return evalBinOpNN(state, op, lhs.castAs(), rhs.castAs(), diff --git a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp index 4ca35dd06ae5..dad8a7b3caae 100644 --- a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp +++ b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp @@ -21,6 +21,35 @@ using namespace ento; namespace { class SimpleSValBuilder : public SValBuilder { + + // With one `simplifySValOnce` call, a compound symbols might collapse to + // simpler symbol tree that is still possible to further simplify. Thus, we + // do the simplification on a new symbol tree until we reach the simplest + // form, i.e. the fixpoint. + // Consider the following symbol `(b * b) * b * b` which has this tree: + // * + // / \ + // * b + // / \ + // / b + // (b * b) + // Now, if the `b * b == 1` new constraint is added then during the first + // iteration we have the following transformations: + // * * + // / \ / \ + // * b --> b b + // / \ + // / b + // 1 + // We need another iteration to reach the final result `1`. + SVal simplifyUntilFixpoint(ProgramStateRef State, SVal Val); + + // Recursively descends into symbolic expressions and replaces symbols + // with their known values (in the sense of the getKnownValue() method). + // We traverse the symbol tree and query the constraint values for the + // sub-trees and if a value is a constant we do the constant folding. + SVal simplifySValOnce(ProgramStateRef State, SVal V); + public: SimpleSValBuilder(llvm::BumpPtrAllocator &alloc, ASTContext &context, ProgramStateManager &stateMgr) @@ -40,8 +69,6 @@ class SimpleSValBuilder : public SValBuilder { /// (integer) value, that value is returned. Otherwise, returns NULL. const llvm::APSInt *getKnownValue(ProgramStateRef state, SVal V) override; - /// Recursively descends into symbolic expressions and replaces symbols - /// with their known values (in the sense of the getKnownValue() method). SVal simplifySVal(ProgramStateRef State, SVal V) override; SVal MakeSymIntVal(const SymExpr *LHS, BinaryOperator::Opcode op, @@ -1105,7 +1132,20 @@ const llvm::APSInt *SimpleSValBuilder::getKnownValue(ProgramStateRef state, return nullptr; } +SVal SimpleSValBuilder::simplifyUntilFixpoint(ProgramStateRef State, SVal Val) { + SVal SimplifiedVal = simplifySValOnce(State, Val); + while (SimplifiedVal != Val) { + Val = SimplifiedVal; + SimplifiedVal = simplifySValOnce(State, Val); + } + return SimplifiedVal; +} + SVal SimpleSValBuilder::simplifySVal(ProgramStateRef State, SVal V) { + return simplifyUntilFixpoint(State, V); +} + +SVal SimpleSValBuilder::simplifySValOnce(ProgramStateRef State, SVal V) { // For now, this function tries to constant-fold symbols inside a // nonloc::SymbolVal, and does nothing else. More simplifications should // be possible, such as constant-folding an index in an ElementRegion. diff --git a/clang/lib/StaticAnalyzer/Core/Store.cpp b/clang/lib/StaticAnalyzer/Core/Store.cpp index 05feb1325c93..2bcdb0faf5da 100644 --- a/clang/lib/StaticAnalyzer/Core/Store.cpp +++ b/clang/lib/StaticAnalyzer/Core/Store.cpp @@ -314,10 +314,7 @@ static const CXXRecordDecl *getCXXRecordType(const MemRegion *MR) { return nullptr; } -SVal StoreManager::attemptDownCast(SVal Base, QualType TargetType, - bool &Failed) { - Failed = false; - +Optional StoreManager::evalBaseToDerived(SVal Base, QualType TargetType) { const MemRegion *MR = Base.getAsRegion(); if (!MR) return UnknownVal(); @@ -392,7 +389,9 @@ SVal StoreManager::attemptDownCast(SVal Base, QualType TargetType, } // We failed if the region we ended up with has perfect type info. - Failed = isa(MR); + if (isa(MR)) + return None; + return UnknownVal(); } diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp index f692c68045ee..f6ddcb763f9d 100644 --- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp +++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp @@ -35,6 +35,7 @@ #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h" #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -493,13 +494,11 @@ void AnalysisConsumer::HandleDeclsCallGraph(const unsigned LocalTUDeclsSize) { } } -static bool isBisonFile(ASTContext &C) { +static bool fileContainsString(StringRef Substring, ASTContext &C) { const SourceManager &SM = C.getSourceManager(); FileID FID = SM.getMainFileID(); StringRef Buffer = SM.getBufferOrFake(FID).getBuffer(); - if (Buffer.startswith("/* A Bison parser, made by")) - return true; - return false; + return Buffer.contains(Substring); } void AnalysisConsumer::runAnalysisOnTranslationUnit(ASTContext &C) { @@ -546,38 +545,48 @@ void AnalysisConsumer::reportAnalyzerProgress(StringRef S) { } void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) { - // Don't run the actions if an error has occurred with parsing the file. DiagnosticsEngine &Diags = PP.getDiagnostics(); if (Diags.hasErrorOccurred() || Diags.hasFatalErrorOccurred()) return; - if (isBisonFile(C)) { - reportAnalyzerProgress("Skipping bison-generated file\n"); - } else if (Opts->DisableAllCheckers) { + // Explicitly destroy the PathDiagnosticConsumer. This will flush its output. + // FIXME: This should be replaced with something that doesn't rely on + // side-effects in PathDiagnosticConsumer's destructor. This is required when + // used with option -disable-free. + const auto DiagFlusherScopeExit = + llvm::make_scope_exit([this] { Mgr.reset(); }); - // Don't analyze if the user explicitly asked for no checks to be performed - // on this file. - reportAnalyzerProgress("All checks are disabled using a supplied option\n"); - } else { - // Otherwise, just run the analysis. - runAnalysisOnTranslationUnit(C); + if (Opts->ShouldIgnoreBisonGeneratedFiles && + fileContainsString("/* A Bison parser, made by", C)) { + reportAnalyzerProgress("Skipping bison-generated file\n"); + return; } + if (Opts->ShouldIgnoreFlexGeneratedFiles && + fileContainsString("/* A lexical scanner generated by flex", C)) { + reportAnalyzerProgress("Skipping flex-generated file\n"); + return; + } + + // Don't analyze if the user explicitly asked for no checks to be performed + // on this file. + if (Opts->DisableAllCheckers) { + reportAnalyzerProgress("All checks are disabled using a supplied option\n"); + return; + } + + // Otherwise, just run the analysis. + runAnalysisOnTranslationUnit(C); + // Count how many basic blocks we have not covered. NumBlocksInAnalyzedFunctions = FunctionSummaries.getTotalNumBasicBlocks(); NumVisitedBlocksInAnalyzedFunctions = FunctionSummaries.getTotalNumVisitedBasicBlocks(); if (NumBlocksInAnalyzedFunctions > 0) PercentReachableBlocks = - (FunctionSummaries.getTotalNumVisitedBasicBlocks() * 100) / + (FunctionSummaries.getTotalNumVisitedBasicBlocks() * 100) / NumBlocksInAnalyzedFunctions; - - // Explicitly destroy the PathDiagnosticConsumer. This will flush its output. - // FIXME: This should be replaced with something that doesn't rely on - // side-effects in PathDiagnosticConsumer's destructor. This is required when - // used with option -disable-free. - Mgr.reset(); } AnalysisConsumer::AnalysisMode diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index f7c711690d7e..acceec690c11 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -9,66 +9,54 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" #include "clang/Lex/DependencyDirectivesSourceMinimizer.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" #include "llvm/Support/Threading.h" using namespace clang; using namespace tooling; using namespace dependencies; -CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( - StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { +llvm::ErrorOr +CachedFileSystemEntry::initFile(StringRef Filename, llvm::vfs::FileSystem &FS) { // Load the file and its content from the file system. - llvm::ErrorOr> MaybeFile = - FS.openFileForRead(Filename); + auto MaybeFile = FS.openFileForRead(Filename); if (!MaybeFile) return MaybeFile.getError(); - llvm::ErrorOr Stat = (*MaybeFile)->status(); - if (!Stat) - return Stat.getError(); + auto File = std::move(*MaybeFile); - llvm::vfs::File &F = **MaybeFile; - llvm::ErrorOr> MaybeBuffer = - F.getBuffer(Stat->getName()); + auto MaybeStat = File->status(); + if (!MaybeStat) + return MaybeStat.getError(); + auto Stat = std::move(*MaybeStat); + + auto MaybeBuffer = File->getBuffer(Stat.getName()); if (!MaybeBuffer) return MaybeBuffer.getError(); + auto Buffer = std::move(*MaybeBuffer); + + OriginalContents = std::move(Buffer); + return Stat; +} + +void CachedFileSystemEntry::minimizeFile() { + assert(OriginalContents && "minimizing missing contents"); llvm::SmallString<1024> MinimizedFileContents; // Minimize the file down to directives that might affect the dependencies. - const auto &Buffer = *MaybeBuffer; SmallVector Tokens; - if (!Minimize || minimizeSourceToDependencyDirectives( - Buffer->getBuffer(), MinimizedFileContents, Tokens)) { - // Use the original file unless requested otherwise, or - // if the minimization failed. - // FIXME: Propage the diagnostic if desired by the client. - CachedFileSystemEntry Result; - Result.MaybeStat = std::move(*Stat); - Result.Contents.reserve(Buffer->getBufferSize() + 1); - Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); - // Implicitly null terminate the contents for Clang's lexer. - Result.Contents.push_back('\0'); - Result.Contents.pop_back(); - return Result; + if (minimizeSourceToDependencyDirectives(OriginalContents->getBuffer(), + MinimizedFileContents, Tokens)) { + // FIXME: Propagate the diagnostic if desired by the client. + // Use the original file if the minimization failed. + MinimizedContentsStorage = + llvm::MemoryBuffer::getMemBuffer(*OriginalContents); + MinimizedContentsAccess.store(MinimizedContentsStorage.get()); + return; } - CachedFileSystemEntry Result; - size_t Size = MinimizedFileContents.size(); - Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), - Stat->getLastModificationTime(), - Stat->getUser(), Stat->getGroup(), Size, - Stat->getType(), Stat->getPermissions()); // The contents produced by the minimizer must be null terminated. assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && "not null terminated contents"); - // Even though there's an implicit null terminator in the minimized contents, - // we want to temporarily make it explicit. This will ensure that the - // std::move will preserve it even if it needs to do a copy if the - // SmallString still has the small capacity. - MinimizedFileContents.push_back('\0'); - Result.Contents = std::move(MinimizedFileContents); - // Now make the null terminator implicit again, so that Clang's lexer can find - // it right where the buffer ends. - Result.Contents.pop_back(); // Compute the skipped PP ranges that speedup skipping over inactive // preprocessor blocks. @@ -86,20 +74,20 @@ CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( } Mapping[Range.Offset] = Range.Length; } - Result.PPSkippedRangeMapping = std::move(Mapping); + PPSkippedRangeMapping = std::move(Mapping); - return Result; + MinimizedContentsStorage = std::make_unique( + std::move(MinimizedFileContents)); + // The algorithm in `getOrCreateFileSystemEntry` uses the presence of + // minimized contents to decide whether an entry is up-to-date or not. + // If it is up-to-date, the skipped range mappings must be already computed. + // This is why we need to store the minimized contents **after** storing the + // skipped range mappings. Failing to do so would lead to a data race. + MinimizedContentsAccess.store(MinimizedContentsStorage.get()); } -CachedFileSystemEntry -CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { - assert(Stat.isDirectory() && "not a directory!"); - auto Result = CachedFileSystemEntry(); - Result.MaybeStat = std::move(Stat); - return Result; -} - -DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() { +DependencyScanningFilesystemSharedCache:: + DependencyScanningFilesystemSharedCache() { // This heuristic was chosen using a empirical testing on a // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache // sharding gives a performance edge by reducing the lock contention. @@ -111,19 +99,13 @@ DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() { } DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & -DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) { +DependencyScanningFilesystemSharedCache::get(StringRef Key) { CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; - std::unique_lock LockGuard(Shard.CacheLock); + std::lock_guard LockGuard(Shard.CacheLock); auto It = Shard.Cache.try_emplace(Key); return It.first->getValue(); } -DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & -DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) { - SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal; - return Cache.get(Key); -} - /// Whitelist file extensions that should be minimized, treating no extension as /// a source file that should be minimized. /// @@ -134,15 +116,14 @@ static bool shouldMinimizeBasedOnExtension(StringRef Filename) { if (Ext.empty()) return true; // C++ standard library return llvm::StringSwitch(Ext) - .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) - .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) - .CasesLower(".m", ".mm", true) - .CasesLower(".i", ".ii", ".mi", ".mmi", true) - .CasesLower(".def", ".inc", true) - .Default(false); + .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) + .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) + .CasesLower(".m", ".mm", true) + .CasesLower(".i", ".ii", ".mi", ".mmi", true) + .CasesLower(".def", ".inc", true) + .Default(false); } - static bool shouldCacheStatFailures(StringRef Filename) { StringRef Ext = llvm::sys::path::extension(Filename); if (Ext.empty()) @@ -167,38 +148,33 @@ bool DependencyScanningWorkerFilesystem::shouldMinimize(StringRef RawFilename) { return !NotToBeMinimized.contains(Filename); } -CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry( - llvm::ErrorOr &&MaybeStatus, StringRef Filename, - bool ShouldMinimize) { - if (!MaybeStatus) - return CachedFileSystemEntry(MaybeStatus.getError()); - - if (MaybeStatus->isDirectory()) - return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus)); - - return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(), - ShouldMinimize); +void CachedFileSystemEntry::init(llvm::ErrorOr &&MaybeStatus, + StringRef Filename, + llvm::vfs::FileSystem &FS) { + if (!MaybeStatus || MaybeStatus->isDirectory()) + MaybeStat = std::move(MaybeStatus); + else + MaybeStat = initFile(Filename, FS); } -llvm::ErrorOr +llvm::ErrorOr DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( - const StringRef Filename) { - bool ShouldMinimize = shouldMinimize(Filename); + StringRef Filename) { + bool ShouldBeMinimized = shouldMinimize(Filename); - if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize)) - return Entry; + const auto *Entry = LocalCache.getCachedEntry(Filename); + if (Entry && !Entry->needsUpdate(ShouldBeMinimized)) + return EntryRef(ShouldBeMinimized, *Entry); // FIXME: Handle PCM/PCH files. // FIXME: Handle module map files. - DependencyScanningFilesystemSharedCache::SharedFileSystemEntry - &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize); - const CachedFileSystemEntry *Result; + auto &SharedCacheEntry = SharedCache.get(Filename); { - std::unique_lock LockGuard(SharedCacheEntry.ValueLock); + std::lock_guard LockGuard(SharedCacheEntry.ValueLock); CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; - if (!CacheEntry.isValid()) { + if (!CacheEntry.isInitialized()) { auto MaybeStatus = getUnderlyingFS().status(Filename); if (!MaybeStatus && !shouldCacheStatFailures(Filename)) // HACK: We need to always restat non source files if the stat fails. @@ -206,27 +182,30 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( // files before building them, and then looks for them again. If we // cache the stat failure, it won't see them the second time. return MaybeStatus.getError(); - CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename, - ShouldMinimize); + CacheEntry.init(std::move(MaybeStatus), Filename, getUnderlyingFS()); } - Result = &CacheEntry; + // Checking `needsUpdate` verifies the entry represents an opened file. + // Only checking `needsMinimization` could lead to minimization of files + // that we failed to load (such files don't have `OriginalContents`). + if (CacheEntry.needsUpdate(ShouldBeMinimized)) + CacheEntry.minimizeFile(); } // Store the result in the local cache. - Cache.setCachedEntry(Filename, ShouldMinimize, Result); - return Result; + Entry = &SharedCacheEntry.Value; + return EntryRef(ShouldBeMinimized, *Entry); } llvm::ErrorOr DependencyScanningWorkerFilesystem::status(const Twine &Path) { SmallString<256> OwnedFilename; StringRef Filename = Path.toStringRef(OwnedFilename); - const llvm::ErrorOr Result = - getOrCreateFileSystemEntry(Filename); + + llvm::ErrorOr Result = getOrCreateFileSystemEntry(Filename); if (!Result) return Result.getError(); - return (*Result)->getStatus(); + return Result->getStatus(); } namespace { @@ -240,7 +219,7 @@ class MinimizedVFSFile final : public llvm::vfs::File { : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} static llvm::ErrorOr> - create(const CachedFileSystemEntry *Entry, + create(EntryRef Entry, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings); llvm::ErrorOr status() override { return Stat; } @@ -261,21 +240,22 @@ class MinimizedVFSFile final : public llvm::vfs::File { } // end anonymous namespace llvm::ErrorOr> MinimizedVFSFile::create( - const CachedFileSystemEntry *Entry, - ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { - if (Entry->isDirectory()) - return llvm::ErrorOr>( - std::make_error_code(std::errc::is_a_directory)); - llvm::ErrorOr Contents = Entry->getContents(); + EntryRef Entry, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { + if (Entry.isDirectory()) + return std::make_error_code(std::errc::is_a_directory); + + llvm::ErrorOr Contents = Entry.getContents(); if (!Contents) return Contents.getError(); auto Result = std::make_unique( - llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), + llvm::MemoryBuffer::getMemBuffer(*Contents, Entry.getName(), /*RequiresNullTerminator=*/false), - *Entry->getStatus()); - if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings) - (*PPSkipMappings)[Result->Buffer->getBufferStart()] = - &Entry->getPPSkippedRangeMapping(); + *Entry.getStatus()); + + const auto *EntrySkipMappings = Entry.getPPSkippedRangeMapping(); + if (EntrySkipMappings && !EntrySkipMappings->empty() && PPSkipMappings) + (*PPSkipMappings)[Result->Buffer->getBufferStart()] = EntrySkipMappings; + return llvm::ErrorOr>( std::unique_ptr(std::move(Result))); } @@ -285,8 +265,7 @@ DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { SmallString<256> OwnedFilename; StringRef Filename = Path.toStringRef(OwnedFilename); - const llvm::ErrorOr Result = - getOrCreateFileSystemEntry(Filename); + llvm::ErrorOr Result = getOrCreateFileSystemEntry(Filename); if (!Result) return Result.getError(); return MinimizedVFSFile::create(Result.get(), PPSkipMappings); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 383a850301a1..086215e7a573 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -37,9 +37,13 @@ CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths( CI.getLangOpts()->resetNonModularOptions(); CI.getPreprocessorOpts().resetNonModularOptions(); - // Remove options incompatible with explicit module build. + // Remove options incompatible with explicit module build or are likely to + // differ between identical modules discovered from different translation + // units. CI.getFrontendOpts().Inputs.clear(); CI.getFrontendOpts().OutputFile.clear(); + CI.getCodeGenOpts().MainFileName.clear(); + CI.getCodeGenOpts().DwarfDebugFlags.clear(); CI.getFrontendOpts().ProgramAction = frontend::GenerateModule; CI.getLangOpts()->ModuleName = Deps.ID.ModuleName; @@ -233,7 +237,13 @@ ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { .getHeaderSearchInfo() .getModuleMap() .getModuleMapFileForUniquing(M); - MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : ""); + + if (ModuleMap) { + StringRef Path = ModuleMap->tryGetRealPathName(); + if (Path.empty()) + Path = ModuleMap->getName(); + MD.ClangModuleMapFile = std::string(Path); + } serialization::ModuleFile *MF = MDC.ScanInstance.getASTReader()->getModuleManager().lookup( diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp index 8a31e776d030..e2014f965c90 100644 --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -927,5 +927,5 @@ std::string TokenBuffer::dumpForTests() const { M.EndExpanded); } } - return OS.str(); + return Dump; } diff --git a/clang/lib/Tooling/Syntax/Tree.cpp b/clang/lib/Tooling/Syntax/Tree.cpp index 1e3a90f3a316..c813865e95cd 100644 --- a/clang/lib/Tooling/Syntax/Tree.cpp +++ b/clang/lib/Tooling/Syntax/Tree.cpp @@ -263,7 +263,7 @@ std::string syntax::Node::dumpTokens(const SourceManager &SM) const { OS << " "; } }); - return OS.str(); + return Storage; } void syntax::Node::assertInvariants() const { diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp index 56dc628869a4..893c17d91708 100644 --- a/clang/tools/clang-format/ClangFormat.cpp +++ b/clang/tools/clang-format/ClangFormat.cpp @@ -460,7 +460,7 @@ static bool format(StringRef FileName) { // To format JSON insert a variable to trick the code into thinking its // JavaScript. - if (FormatStyle->isJson()) { + if (FormatStyle->isJson() && !FormatStyle->DisableFormat) { auto Err = Replaces.add(tooling::Replacement( tooling::Replacement(AssumedFileName, 0, 0, "x = "))); if (Err) { diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp index c9129ee9e502..a7bfb07e002b 100644 --- a/clang/tools/driver/driver.cpp +++ b/clang/tools/driver/driver.cpp @@ -120,7 +120,7 @@ static void ApplyOneQAOverride(raw_ostream &OS, OS << "### Adding argument " << Str << " at end\n"; Args.push_back(Str); } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { + Edit.slice(2, Edit.size() - 1).contains('/')) { StringRef MatchPattern = Edit.substr(2).split('/').first; StringRef ReplPattern = Edit.substr(2).split('/').second; ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp index f5b6f4f01688..1a2532fbf53f 100644 --- a/clang/utils/TableGen/MveEmitter.cpp +++ b/clang/utils/TableGen/MveEmitter.cpp @@ -349,13 +349,8 @@ class PredicateType : public CRegularNamedType { bool requiresFloat() const override { return false; }; bool requiresMVE() const override { return true; } std::string llvmName() const override { - // Use <4 x i1> instead of <2 x i1> for two-lane vector types. See - // the comment in llvm/lib/Target/ARM/ARMInstrMVE.td for further - // explanation. - unsigned ModifiedLanes = (Lanes == 2 ? 4 : Lanes); - - return "llvm::FixedVectorType::get(Builder.getInt1Ty(), " + - utostr(ModifiedLanes) + ")"; + return "llvm::FixedVectorType::get(Builder.getInt1Ty(), " + utostr(Lanes) + + ")"; } static bool classof(const Type *T) { diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 008b8dde5820..44719126b596 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -653,15 +653,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. + * The 59th bit indicates whether to use debug info to correlate profiles. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) +#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias diff --git a/compiler-rt/include/sanitizer/dfsan_interface.h b/compiler-rt/include/sanitizer/dfsan_interface.h index d6209a3ea2b2..bc0652c99a14 100644 --- a/compiler-rt/include/sanitizer/dfsan_interface.h +++ b/compiler-rt/include/sanitizer/dfsan_interface.h @@ -54,6 +54,10 @@ dfsan_origin dfsan_get_origin(long data); /// Retrieves the label associated with the data at the given address. dfsan_label dfsan_read_label(const void *addr, size_t size); +/// Return the origin associated with the first taint byte in the size bytes +/// from the address addr. +dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, size_t size); + /// Returns whether the given label label contains the label elem. int dfsan_has_label(dfsan_label label, dfsan_label elem); @@ -87,6 +91,9 @@ void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2, /// prints description at the beginning of the trace. If origin tracking is not /// on, or the address is not labeled, it prints nothing. void dfsan_print_origin_trace(const void *addr, const char *description); +/// As above, but use an origin id from dfsan_get_origin() instead of address. +/// Does not include header line with taint label and address information. +void dfsan_print_origin_id_trace(dfsan_origin origin); /// Prints the origin trace of the label at the address \p addr to a /// pre-allocated output buffer. If origin tracking is not on, or the address is @@ -124,6 +131,10 @@ void dfsan_print_origin_trace(const void *addr, const char *description); /// return value is not less than \p out_buf_size. size_t dfsan_sprint_origin_trace(const void *addr, const char *description, char *out_buf, size_t out_buf_size); +/// As above, but use an origin id from dfsan_get_origin() instead of address. +/// Does not include header line with taint label and address information. +size_t dfsan_sprint_origin_id_trace(dfsan_origin origin, char *out_buf, + size_t out_buf_size); /// Prints the stack trace leading to this call to a pre-allocated output /// buffer. diff --git a/compiler-rt/lib/asan/asan_activation.cpp b/compiler-rt/lib/asan/asan_activation.cpp index 795df95a5414..1757838600ca 100644 --- a/compiler-rt/lib/asan/asan_activation.cpp +++ b/compiler-rt/lib/asan/asan_activation.cpp @@ -112,7 +112,7 @@ void AsanDeactivate() { disabled.quarantine_size_mb = 0; disabled.thread_local_quarantine_size_kb = 0; // Redzone must be at least Max(16, granularity) bytes long. - disabled.min_redzone = Max(16, (int)SHADOW_GRANULARITY); + disabled.min_redzone = Max(16, (int)ASAN_SHADOW_GRANULARITY); disabled.max_redzone = disabled.min_redzone; disabled.alloc_dealloc_mismatch = false; disabled.may_return_null = true; diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp index 3fa36742060b..1ff7091460ad 100644 --- a/compiler-rt/lib/asan/asan_allocator.cpp +++ b/compiler-rt/lib/asan/asan_allocator.cpp @@ -210,8 +210,7 @@ struct QuarantineCallback { CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE); } - PoisonShadow(m->Beg(), - RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), + PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), kAsanHeapLeftRedzoneMagic); // Statistics. @@ -305,7 +304,6 @@ struct Allocator { QuarantineCache fallback_quarantine_cache; uptr max_user_defined_malloc_size; - atomic_uint8_t rss_limit_exceeded; // ------------------- Options -------------------------- atomic_uint16_t min_redzone; @@ -345,14 +343,6 @@ struct Allocator { : kMaxAllowedMallocSize; } - bool RssLimitExceeded() { - return atomic_load(&rss_limit_exceeded, memory_order_relaxed); - } - - void SetRssLimitExceeded(bool limit_exceeded) { - atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); - } - void RePoisonChunk(uptr chunk) { // This could be a user-facing chunk (with redzones), or some internal // housekeeping chunk, like TransferBatch. Start by assuming the former. @@ -366,7 +356,7 @@ struct Allocator { if (chunk < beg && beg < end && end <= chunk_end) { // Looks like a valid AsanChunk in use, poison redzones only. PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic); - uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY); + uptr end_aligned_down = RoundDownTo(end, ASAN_SHADOW_GRANULARITY); FastPoisonShadowPartialRightRedzone( end_aligned_down, end - end_aligned_down, chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic); @@ -484,14 +474,14 @@ struct Allocator { AllocType alloc_type, bool can_fill) { if (UNLIKELY(!asan_inited)) AsanInitFromRtl(); - if (RssLimitExceeded()) { + if (UNLIKELY(IsRssLimitExceeded())) { if (AllocatorMayReturnNull()) return nullptr; ReportRssLimitExceeded(stack); } Flags &fl = *flags(); CHECK(stack); - const uptr min_alignment = SHADOW_GRANULARITY; + const uptr min_alignment = ASAN_SHADOW_GRANULARITY; const uptr user_requested_alignment_log = ComputeUserRequestedAlignmentLog(alignment); if (alignment < min_alignment) @@ -572,7 +562,7 @@ struct Allocator { m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack)); uptr size_rounded_down_to_granularity = - RoundDownTo(size, SHADOW_GRANULARITY); + RoundDownTo(size, ASAN_SHADOW_GRANULARITY); // Unpoison the bulk of the memory region. if (size_rounded_down_to_granularity) PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); @@ -580,7 +570,7 @@ struct Allocator { if (size != size_rounded_down_to_granularity && CanPoisonMemory()) { u8 *shadow = (u8 *)MemToShadow(user_beg + size_rounded_down_to_granularity); - *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY - 1)) : 0; + *shadow = fl.poison_partial ? (size & (ASAN_SHADOW_GRANULARITY - 1)) : 0; } AsanStats &thread_stats = GetCurrentThreadStats(); @@ -650,8 +640,7 @@ struct Allocator { } // Poison the region. - PoisonShadow(m->Beg(), - RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), + PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), kAsanHeapFreeMagic); AsanStats &thread_stats = GetCurrentThreadStats(); @@ -1071,10 +1060,6 @@ void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS { instance.ForceUnlock(); } -void AsanSoftRssLimitExceededCallback(bool limit_exceeded) { - instance.SetRssLimitExceeded(limit_exceeded); -} - } // namespace __asan // --- Implementation of LSan-specific functions --- {{{1 diff --git a/compiler-rt/lib/asan/asan_debugging.cpp b/compiler-rt/lib/asan/asan_debugging.cpp index 0b4bf52f2490..f078f1041a87 100644 --- a/compiler-rt/lib/asan/asan_debugging.cpp +++ b/compiler-rt/lib/asan/asan_debugging.cpp @@ -141,7 +141,7 @@ uptr __asan_get_free_stack(uptr addr, uptr *trace, uptr size, u32 *thread_id) { SANITIZER_INTERFACE_ATTRIBUTE void __asan_get_shadow_mapping(uptr *shadow_scale, uptr *shadow_offset) { if (shadow_scale) - *shadow_scale = SHADOW_SCALE; + *shadow_scale = ASAN_SHADOW_SCALE; if (shadow_offset) - *shadow_offset = SHADOW_OFFSET; + *shadow_offset = ASAN_SHADOW_OFFSET; } diff --git a/compiler-rt/lib/asan/asan_errors.cpp b/compiler-rt/lib/asan/asan_errors.cpp index 7cd9fe911afa..a22bf130d823 100644 --- a/compiler-rt/lib/asan/asan_errors.cpp +++ b/compiler-rt/lib/asan/asan_errors.cpp @@ -329,7 +329,7 @@ void ErrorBadParamsToAnnotateContiguousContainer::Print() { " old_mid : %p\n" " new_mid : %p\n", (void *)beg, (void *)end, (void *)old_mid, (void *)new_mid); - uptr granularity = SHADOW_GRANULARITY; + uptr granularity = ASAN_SHADOW_GRANULARITY; if (!IsAligned(beg, granularity)) Report("ERROR: beg is not aligned by %zu\n", granularity); stack->Print(); @@ -410,7 +410,8 @@ ErrorGeneric::ErrorGeneric(u32 tid, uptr pc_, uptr bp_, uptr sp_, uptr addr, if (AddrIsInMem(addr)) { u8 *shadow_addr = (u8 *)MemToShadow(addr); // If we are accessing 16 bytes, look at the second shadow byte. - if (*shadow_addr == 0 && access_size > SHADOW_GRANULARITY) shadow_addr++; + if (*shadow_addr == 0 && access_size > ASAN_SHADOW_GRANULARITY) + shadow_addr++; // If we are in the partial right redzone, look at the next shadow byte. if (*shadow_addr > 0 && *shadow_addr < 128) shadow_addr++; bool far_from_bounds = false; @@ -501,10 +502,11 @@ static void PrintLegend(InternalScopedString *str) { str->append( "Shadow byte legend (one shadow byte represents %d " "application bytes):\n", - (int)SHADOW_GRANULARITY); + (int)ASAN_SHADOW_GRANULARITY); PrintShadowByte(str, " Addressable: ", 0); str->append(" Partially addressable: "); - for (u8 i = 1; i < SHADOW_GRANULARITY; i++) PrintShadowByte(str, "", i, " "); + for (u8 i = 1; i < ASAN_SHADOW_GRANULARITY; i++) + PrintShadowByte(str, "", i, " "); str->append("\n"); PrintShadowByte(str, " Heap left redzone: ", kAsanHeapLeftRedzoneMagic); diff --git a/compiler-rt/lib/asan/asan_fake_stack.cpp b/compiler-rt/lib/asan/asan_fake_stack.cpp index 07681c10de91..08d81c72597c 100644 --- a/compiler-rt/lib/asan/asan_fake_stack.cpp +++ b/compiler-rt/lib/asan/asan_fake_stack.cpp @@ -28,8 +28,8 @@ static const u64 kAllocaRedzoneMask = 31UL; // For small size classes inline PoisonShadow for better performance. ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) { u64 *shadow = reinterpret_cast(MemToShadow(ptr)); - if (SHADOW_SCALE == 3 && class_id <= 6) { - // This code expects SHADOW_SCALE=3. + if (ASAN_SHADOW_SCALE == 3 && class_id <= 6) { + // This code expects ASAN_SHADOW_SCALE=3. for (uptr i = 0; i < (((uptr)1) << class_id); i++) { shadow[i] = magic; // Make sure this does not become memset. @@ -294,10 +294,10 @@ void __asan_alloca_poison(uptr addr, uptr size) { uptr LeftRedzoneAddr = addr - kAllocaRedzoneSize; uptr PartialRzAddr = addr + size; uptr RightRzAddr = (PartialRzAddr + kAllocaRedzoneMask) & ~kAllocaRedzoneMask; - uptr PartialRzAligned = PartialRzAddr & ~(SHADOW_GRANULARITY - 1); + uptr PartialRzAligned = PartialRzAddr & ~(ASAN_SHADOW_GRANULARITY - 1); FastPoisonShadow(LeftRedzoneAddr, kAllocaRedzoneSize, kAsanAllocaLeftMagic); FastPoisonShadowPartialRightRedzone( - PartialRzAligned, PartialRzAddr % SHADOW_GRANULARITY, + PartialRzAligned, PartialRzAddr % ASAN_SHADOW_GRANULARITY, RightRzAddr - PartialRzAligned, kAsanAllocaRightMagic); FastPoisonShadow(RightRzAddr, kAllocaRedzoneSize, kAsanAllocaRightMagic); } @@ -305,7 +305,8 @@ void __asan_alloca_poison(uptr addr, uptr size) { SANITIZER_INTERFACE_ATTRIBUTE void __asan_allocas_unpoison(uptr top, uptr bottom) { if ((!top) || (top > bottom)) return; - REAL(memset)(reinterpret_cast(MemToShadow(top)), 0, - (bottom - top) / SHADOW_GRANULARITY); + REAL(memset) + (reinterpret_cast(MemToShadow(top)), 0, + (bottom - top) / ASAN_SHADOW_GRANULARITY); } } // extern "C" diff --git a/compiler-rt/lib/asan/asan_flags.cpp b/compiler-rt/lib/asan/asan_flags.cpp index c64e46470287..9ea899f84b4b 100644 --- a/compiler-rt/lib/asan/asan_flags.cpp +++ b/compiler-rt/lib/asan/asan_flags.cpp @@ -140,9 +140,9 @@ void InitializeFlags() { SanitizerToolName); Die(); } - // Ensure that redzone is at least SHADOW_GRANULARITY. - if (f->redzone < (int)SHADOW_GRANULARITY) - f->redzone = SHADOW_GRANULARITY; + // Ensure that redzone is at least ASAN_SHADOW_GRANULARITY. + if (f->redzone < (int)ASAN_SHADOW_GRANULARITY) + f->redzone = ASAN_SHADOW_GRANULARITY; // Make "strict_init_order" imply "check_initialization_order". // TODO(samsonov): Use a single runtime flag for an init-order checker. if (f->strict_init_order) { diff --git a/compiler-rt/lib/asan/asan_globals.cpp b/compiler-rt/lib/asan/asan_globals.cpp index 5f56fe6f457d..ecc2600f039a 100644 --- a/compiler-rt/lib/asan/asan_globals.cpp +++ b/compiler-rt/lib/asan/asan_globals.cpp @@ -61,14 +61,13 @@ ALWAYS_INLINE void PoisonShadowForGlobal(const Global *g, u8 value) { } ALWAYS_INLINE void PoisonRedZones(const Global &g) { - uptr aligned_size = RoundUpTo(g.size, SHADOW_GRANULARITY); + uptr aligned_size = RoundUpTo(g.size, ASAN_SHADOW_GRANULARITY); FastPoisonShadow(g.beg + aligned_size, g.size_with_redzone - aligned_size, kAsanGlobalRedzoneMagic); if (g.size != aligned_size) { FastPoisonShadowPartialRightRedzone( - g.beg + RoundDownTo(g.size, SHADOW_GRANULARITY), - g.size % SHADOW_GRANULARITY, - SHADOW_GRANULARITY, + g.beg + RoundDownTo(g.size, ASAN_SHADOW_GRANULARITY), + g.size % ASAN_SHADOW_GRANULARITY, ASAN_SHADOW_GRANULARITY, kAsanGlobalRedzoneMagic); } } diff --git a/compiler-rt/lib/asan/asan_interface.inc b/compiler-rt/lib/asan/asan_interface.inc index ea28fc8ae87c..89ef552b7117 100644 --- a/compiler-rt/lib/asan/asan_interface.inc +++ b/compiler-rt/lib/asan/asan_interface.inc @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// // Asan interface list. //===----------------------------------------------------------------------===// + INTERFACE_FUNCTION(__asan_addr_is_in_fake_stack) INTERFACE_FUNCTION(__asan_address_is_poisoned) INTERFACE_FUNCTION(__asan_after_dynamic_init) diff --git a/compiler-rt/lib/asan/asan_linux.cpp b/compiler-rt/lib/asan/asan_linux.cpp index ad3693d5e6a2..1d92c530bd11 100644 --- a/compiler-rt/lib/asan/asan_linux.cpp +++ b/compiler-rt/lib/asan/asan_linux.cpp @@ -107,7 +107,7 @@ uptr FindDynamicShadowStart() { return FindPremappedShadowStart(shadow_size_bytes); #endif - return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE, + return MapDynamicShadow(shadow_size_bytes, ASAN_SHADOW_SCALE, /*min_shadow_base_alignment*/ 0, kHighMemEnd); } diff --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp index c6950547f089..9161f728d44c 100644 --- a/compiler-rt/lib/asan/asan_mac.cpp +++ b/compiler-rt/lib/asan/asan_mac.cpp @@ -55,7 +55,7 @@ void *AsanDoesNotSupportStaticLinkage() { } uptr FindDynamicShadowStart() { - return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE, + return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE, /*min_shadow_base_alignment*/ 0, kHighMemEnd); } diff --git a/compiler-rt/lib/asan/asan_mapping.h b/compiler-rt/lib/asan/asan_mapping.h index e5a7f2007aea..6ca6ee00e5c9 100644 --- a/compiler-rt/lib/asan/asan_mapping.h +++ b/compiler-rt/lib/asan/asan_mapping.h @@ -13,8 +13,6 @@ #ifndef ASAN_MAPPING_H #define ASAN_MAPPING_H -#include "asan_internal.h" - // The full explanation of the memory mapping could be found here: // https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm // @@ -151,149 +149,145 @@ // || `[0x30000000, 0x35ffffff]` || LowShadow || // || `[0x00000000, 0x2fffffff]` || LowMem || -#if defined(ASAN_SHADOW_SCALE) -static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE; -#else -static const u64 kDefaultShadowScale = 3; -#endif -static const u64 kDefaultShadowSentinel = ~(uptr)0; -static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000 -static const u64 kDefaultShadowOffset64 = 1ULL << 44; -static const u64 kDefaultShort64bitShadowOffset = - 0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G. -static const u64 kAArch64_ShadowOffset64 = 1ULL << 36; -static const u64 kRiscv64_ShadowOffset64 = 0xd55550000; -static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; -static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; -static const u64 kPPC64_ShadowOffset64 = 1ULL << 44; -static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52; -static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43; // 0x80000000000 -static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 -static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 -static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 -static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 -static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 - -#define SHADOW_SCALE kDefaultShadowScale +#define ASAN_SHADOW_SCALE 3 #if SANITIZER_FUCHSIA -# define SHADOW_OFFSET (0) +# define ASAN_SHADOW_OFFSET_CONST (0) #elif SANITIZER_WORDSIZE == 32 # if SANITIZER_ANDROID -# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address +# define ASAN_SHADOW_OFFSET_DYNAMIC # elif defined(__mips__) -# define SHADOW_OFFSET kMIPS32_ShadowOffset32 +# define ASAN_SHADOW_OFFSET_CONST 0x0aaa0000 # elif SANITIZER_FREEBSD -# define SHADOW_OFFSET kFreeBSD_ShadowOffset32 +# define ASAN_SHADOW_OFFSET_CONST 0x40000000 # elif SANITIZER_NETBSD -# define SHADOW_OFFSET kNetBSD_ShadowOffset32 +# define ASAN_SHADOW_OFFSET_CONST 0x40000000 # elif SANITIZER_WINDOWS -# define SHADOW_OFFSET kWindowsShadowOffset32 +# define ASAN_SHADOW_OFFSET_CONST 0x30000000 # elif SANITIZER_IOS -# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address +# define ASAN_SHADOW_OFFSET_DYNAMIC # else -# define SHADOW_OFFSET kDefaultShadowOffset32 +# define ASAN_SHADOW_OFFSET_CONST 0x20000000 # endif #else # if SANITIZER_IOS -# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address +# define ASAN_SHADOW_OFFSET_DYNAMIC # elif SANITIZER_MAC && defined(__aarch64__) -# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address -#elif SANITIZER_RISCV64 -#define SHADOW_OFFSET kRiscv64_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_DYNAMIC +# elif SANITIZER_RISCV64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000000d55550000 # elif defined(__aarch64__) -# define SHADOW_OFFSET kAArch64_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000001000000000 # elif defined(__powerpc64__) -# define SHADOW_OFFSET kPPC64_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000 # elif defined(__s390x__) -# define SHADOW_OFFSET kSystemZ_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0010000000000000 # elif SANITIZER_FREEBSD -# define SHADOW_OFFSET kFreeBSD_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000 # elif SANITIZER_NETBSD -# define SHADOW_OFFSET kNetBSD_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000 # elif SANITIZER_MAC -# define SHADOW_OFFSET kDefaultShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000 # elif defined(__mips64) -# define SHADOW_OFFSET kMIPS64_ShadowOffset64 -#elif defined(__sparc__) -#define SHADOW_OFFSET kSPARC64_ShadowOffset64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000002000000000 +# elif defined(__sparc__) +# define ASAN_SHADOW_OFFSET_CONST 0x0000080000000000 # elif SANITIZER_WINDOWS64 -# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address +# define ASAN_SHADOW_OFFSET_DYNAMIC # else -# define SHADOW_OFFSET kDefaultShort64bitShadowOffset +# if ASAN_SHADOW_SCALE != 3 +# error "Value below is based on shadow scale = 3." +# error "Original formula was: 0x7FFFFFFF & (~0xFFFULL << SHADOW_SCALE)." +# endif +# define ASAN_SHADOW_OFFSET_CONST 0x000000007fff8000 # endif #endif -#if SANITIZER_ANDROID && defined(__arm__) -# define ASAN_PREMAP_SHADOW 1 -#else -# define ASAN_PREMAP_SHADOW 0 -#endif +#if defined(__cplusplus) +# include "asan_internal.h" -#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) +static const u64 kDefaultShadowSentinel = ~(uptr)0; -#define DO_ASAN_MAPPING_PROFILE 0 // Set to 1 to profile the functions below. +# if defined(ASAN_SHADOW_OFFSET_CONST) +static const u64 kConstShadowOffset = ASAN_SHADOW_OFFSET_CONST; +# define ASAN_SHADOW_OFFSET kConstShadowOffset +# elif defined(ASAN_SHADOW_OFFSET_DYNAMIC) +# define ASAN_SHADOW_OFFSET __asan_shadow_memory_dynamic_address +# else +# error "ASAN_SHADOW_OFFSET can't be determined." +# endif -#if DO_ASAN_MAPPING_PROFILE -# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++; -#else -# define PROFILE_ASAN_MAPPING() -#endif +# if SANITIZER_ANDROID && defined(__arm__) +# define ASAN_PREMAP_SHADOW 1 +# else +# define ASAN_PREMAP_SHADOW 0 +# endif + +# define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE) + +# define DO_ASAN_MAPPING_PROFILE 0 // Set to 1 to profile the functions below. + +# if DO_ASAN_MAPPING_PROFILE +# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++; +# else +# define PROFILE_ASAN_MAPPING() +# endif // If 1, all shadow boundaries are constants. // Don't set to 1 other than for testing. -#define ASAN_FIXED_MAPPING 0 +# define ASAN_FIXED_MAPPING 0 namespace __asan { extern uptr AsanMappingProfile[]; -#if ASAN_FIXED_MAPPING +# if ASAN_FIXED_MAPPING // Fixed mapping for 64-bit Linux. Mostly used for performance comparison // with non-fixed mapping. As of r175253 (Feb 2013) the performance // difference between fixed and non-fixed mapping is below the noise level. static uptr kHighMemEnd = 0x7fffffffffffULL; -static uptr kMidMemBeg = 0x3000000000ULL; -static uptr kMidMemEnd = 0x4fffffffffULL; -#else +static uptr kMidMemBeg = 0x3000000000ULL; +static uptr kMidMemEnd = 0x4fffffffffULL; +# else extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init. -#endif +# endif } // namespace __asan -#if defined(__sparc__) && SANITIZER_WORDSIZE == 64 -# include "asan_mapping_sparc64.h" -#else -#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) +# if defined(__sparc__) && SANITIZER_WORDSIZE == 64 +# include "asan_mapping_sparc64.h" +# else +# define MEM_TO_SHADOW(mem) \ + (((mem) >> ASAN_SHADOW_SCALE) + (ASAN_SHADOW_OFFSET)) -#define kLowMemBeg 0 -#define kLowMemEnd (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0) +# define kLowMemBeg 0 +# define kLowMemEnd (ASAN_SHADOW_OFFSET ? ASAN_SHADOW_OFFSET - 1 : 0) -#define kLowShadowBeg SHADOW_OFFSET -#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd) +# define kLowShadowBeg ASAN_SHADOW_OFFSET +# define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd) -#define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1) +# define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1) -#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg) -#define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd) +# define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg) +# define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd) -# define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg) -# define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd) +# define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg) +# define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd) // With the zero shadow base we can not actually map pages starting from 0. // This constant is somewhat arbitrary. -#define kZeroBaseShadowStart 0 -#define kZeroBaseMaxShadowStart (1 << 18) +# define kZeroBaseShadowStart 0 +# define kZeroBaseMaxShadowStart (1 << 18) -#define kShadowGapBeg (kLowShadowEnd ? kLowShadowEnd + 1 \ - : kZeroBaseShadowStart) -#define kShadowGapEnd ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1) +# define kShadowGapBeg \ + (kLowShadowEnd ? kLowShadowEnd + 1 : kZeroBaseShadowStart) +# define kShadowGapEnd ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1) -#define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0) -#define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0) +# define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0) +# define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0) -#define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0) -#define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0) +# define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0) +# define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0) namespace __asan { @@ -331,29 +325,31 @@ static inline bool AddrIsInShadowGap(uptr a) { PROFILE_ASAN_MAPPING(); if (kMidMemBeg) { if (a <= kShadowGapEnd) - return SHADOW_OFFSET == 0 || a >= kShadowGapBeg; + return ASAN_SHADOW_OFFSET == 0 || a >= kShadowGapBeg; return (a >= kShadowGap2Beg && a <= kShadowGap2End) || (a >= kShadowGap3Beg && a <= kShadowGap3End); } // In zero-based shadow mode we treat addresses near zero as addresses // in shadow gap as well. - if (SHADOW_OFFSET == 0) + if (ASAN_SHADOW_OFFSET == 0) return a <= kShadowGapEnd; return a >= kShadowGapBeg && a <= kShadowGapEnd; } } // namespace __asan -#endif +# endif namespace __asan { -static inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; } +static inline uptr MemToShadowSize(uptr size) { + return size >> ASAN_SHADOW_SCALE; +} static inline bool AddrIsInMem(uptr a) { PROFILE_ASAN_MAPPING(); return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a) || - (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a)); + (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a)); } static inline uptr MemToShadow(uptr p) { @@ -369,17 +365,17 @@ static inline bool AddrIsInShadow(uptr a) { static inline bool AddrIsAlignedByGranularity(uptr a) { PROFILE_ASAN_MAPPING(); - return (a & (SHADOW_GRANULARITY - 1)) == 0; + return (a & (ASAN_SHADOW_GRANULARITY - 1)) == 0; } static inline bool AddressIsPoisoned(uptr a) { PROFILE_ASAN_MAPPING(); const uptr kAccessSize = 1; - u8 *shadow_address = (u8*)MEM_TO_SHADOW(a); + u8 *shadow_address = (u8 *)MEM_TO_SHADOW(a); s8 shadow_value = *shadow_address; if (shadow_value) { - u8 last_accessed_byte = (a & (SHADOW_GRANULARITY - 1)) - + kAccessSize - 1; + u8 last_accessed_byte = + (a & (ASAN_SHADOW_GRANULARITY - 1)) + kAccessSize - 1; return (last_accessed_byte >= shadow_value); } return false; @@ -390,4 +386,6 @@ static const uptr kAsanMappingProfileSize = __LINE__; } // namespace __asan +#endif // __cplusplus + #endif // ASAN_MAPPING_H diff --git a/compiler-rt/lib/asan/asan_mapping_sparc64.h b/compiler-rt/lib/asan/asan_mapping_sparc64.h index 432a1816f797..90261d301f7f 100644 --- a/compiler-rt/lib/asan/asan_mapping_sparc64.h +++ b/compiler-rt/lib/asan/asan_mapping_sparc64.h @@ -25,13 +25,14 @@ // The idea is to chop the high bits before doing the scaling, so the two // parts become contiguous again and the usual scheme can be applied. -#define MEM_TO_SHADOW(mem) \ - ((((mem) << HIGH_BITS) >> (HIGH_BITS + (SHADOW_SCALE))) + (SHADOW_OFFSET)) +#define MEM_TO_SHADOW(mem) \ + ((((mem) << HIGH_BITS) >> (HIGH_BITS + (ASAN_SHADOW_SCALE))) + \ + (ASAN_SHADOW_OFFSET)) #define kLowMemBeg 0 -#define kLowMemEnd (SHADOW_OFFSET - 1) +#define kLowMemEnd (ASAN_SHADOW_OFFSET - 1) -#define kLowShadowBeg SHADOW_OFFSET +#define kLowShadowBeg ASAN_SHADOW_OFFSET #define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd) // But of course there is the huge hole between the high shadow memory, diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp index d97af91e692d..bbc7db4709e1 100644 --- a/compiler-rt/lib/asan/asan_poisoning.cpp +++ b/compiler-rt/lib/asan/asan_poisoning.cpp @@ -35,7 +35,7 @@ void PoisonShadow(uptr addr, uptr size, u8 value) { CHECK(AddrIsAlignedByGranularity(addr)); CHECK(AddrIsInMem(addr)); CHECK(AddrIsAlignedByGranularity(addr + size)); - CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY)); + CHECK(AddrIsInMem(addr + size - ASAN_SHADOW_GRANULARITY)); CHECK(REAL(memset)); FastPoisonShadow(addr, size, value); } @@ -52,12 +52,12 @@ void PoisonShadowPartialRightRedzone(uptr addr, struct ShadowSegmentEndpoint { u8 *chunk; - s8 offset; // in [0, SHADOW_GRANULARITY) + s8 offset; // in [0, ASAN_SHADOW_GRANULARITY) s8 value; // = *chunk; explicit ShadowSegmentEndpoint(uptr address) { chunk = (u8*)MemToShadow(address); - offset = address & (SHADOW_GRANULARITY - 1); + offset = address & (ASAN_SHADOW_GRANULARITY - 1); value = *chunk; } }; @@ -72,14 +72,14 @@ void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) { } CHECK(size); CHECK_LE(size, 4096); - CHECK(IsAligned(end, SHADOW_GRANULARITY)); - if (!IsAligned(ptr, SHADOW_GRANULARITY)) { + CHECK(IsAligned(end, ASAN_SHADOW_GRANULARITY)); + if (!IsAligned(ptr, ASAN_SHADOW_GRANULARITY)) { *(u8 *)MemToShadow(ptr) = - poison ? static_cast(ptr % SHADOW_GRANULARITY) : 0; - ptr |= SHADOW_GRANULARITY - 1; + poison ? static_cast(ptr % ASAN_SHADOW_GRANULARITY) : 0; + ptr |= ASAN_SHADOW_GRANULARITY - 1; ptr++; } - for (; ptr < end; ptr += SHADOW_GRANULARITY) + for (; ptr < end; ptr += ASAN_SHADOW_GRANULARITY) *(u8*)MemToShadow(ptr) = poison ? kAsanIntraObjectRedzone : 0; } @@ -181,12 +181,12 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) { if (!AddrIsInMem(end)) return end; CHECK_LT(beg, end); - uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY); - uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY); + uptr aligned_b = RoundUpTo(beg, ASAN_SHADOW_GRANULARITY); + uptr aligned_e = RoundDownTo(end, ASAN_SHADOW_GRANULARITY); uptr shadow_beg = MemToShadow(aligned_b); uptr shadow_end = MemToShadow(aligned_e); // First check the first and the last application bytes, - // then check the SHADOW_GRANULARITY-aligned region by calling + // then check the ASAN_SHADOW_GRANULARITY-aligned region by calling // mem_is_zero on the corresponding shadow. if (!__asan::AddressIsPoisoned(beg) && !__asan::AddressIsPoisoned(end - 1) && (shadow_end <= shadow_beg || @@ -285,7 +285,7 @@ uptr __asan_load_cxx_array_cookie(uptr *p) { // assumes that left border of region to be poisoned is properly aligned. static void PoisonAlignedStackMemory(uptr addr, uptr size, bool do_poison) { if (size == 0) return; - uptr aligned_size = size & ~(SHADOW_GRANULARITY - 1); + uptr aligned_size = size & ~(ASAN_SHADOW_GRANULARITY - 1); PoisonShadow(addr, aligned_size, do_poison ? kAsanStackUseAfterScopeMagic : 0); if (size == aligned_size) @@ -351,7 +351,7 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p, uptr end = reinterpret_cast(end_p); uptr old_mid = reinterpret_cast(old_mid_p); uptr new_mid = reinterpret_cast(new_mid_p); - uptr granularity = SHADOW_GRANULARITY; + uptr granularity = ASAN_SHADOW_GRANULARITY; if (!(beg <= old_mid && beg <= new_mid && old_mid <= end && new_mid <= end && IsAligned(beg, granularity))) { GET_STACK_TRACE_FATAL_HERE; diff --git a/compiler-rt/lib/asan/asan_poisoning.h b/compiler-rt/lib/asan/asan_poisoning.h index 3d536f2d3097..600bd011f304 100644 --- a/compiler-rt/lib/asan/asan_poisoning.h +++ b/compiler-rt/lib/asan/asan_poisoning.h @@ -44,8 +44,8 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, common_flags()->clear_shadow_mmap_threshold); #else uptr shadow_beg = MEM_TO_SHADOW(aligned_beg); - uptr shadow_end = MEM_TO_SHADOW( - aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1; + uptr shadow_end = + MEM_TO_SHADOW(aligned_beg + aligned_size - ASAN_SHADOW_GRANULARITY) + 1; // FIXME: Page states are different on Windows, so using the same interface // for mapping shadow and zeroing out pages doesn't "just work", so we should // probably provide higher-level interface for these operations. @@ -78,11 +78,12 @@ ALWAYS_INLINE void FastPoisonShadowPartialRightRedzone( DCHECK(CanPoisonMemory()); bool poison_partial = flags()->poison_partial; u8 *shadow = (u8*)MEM_TO_SHADOW(aligned_addr); - for (uptr i = 0; i < redzone_size; i += SHADOW_GRANULARITY, shadow++) { - if (i + SHADOW_GRANULARITY <= size) { + for (uptr i = 0; i < redzone_size; i += ASAN_SHADOW_GRANULARITY, shadow++) { + if (i + ASAN_SHADOW_GRANULARITY <= size) { *shadow = 0; // fully addressable } else if (i >= size) { - *shadow = (SHADOW_GRANULARITY == 128) ? 0xff : value; // unaddressable + *shadow = + (ASAN_SHADOW_GRANULARITY == 128) ? 0xff : value; // unaddressable } else { // first size-i bytes are addressable *shadow = poison_partial ? static_cast(size - i) : 0; diff --git a/compiler-rt/lib/asan/asan_premap_shadow.cpp b/compiler-rt/lib/asan/asan_premap_shadow.cpp index 666bb9b34bd3..bed2f62a2251 100644 --- a/compiler-rt/lib/asan/asan_premap_shadow.cpp +++ b/compiler-rt/lib/asan/asan_premap_shadow.cpp @@ -26,7 +26,7 @@ namespace __asan { // Conservative upper limit. uptr PremapShadowSize() { uptr granularity = GetMmapGranularity(); - return RoundUpTo(GetMaxVirtualAddress() >> SHADOW_SCALE, granularity); + return RoundUpTo(GetMaxVirtualAddress() >> ASAN_SHADOW_SCALE, granularity); } // Returns an address aligned to 8 pages, such that one page on the left and diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp index 5be8ef0f6d1c..f0bbbf32e6a6 100644 --- a/compiler-rt/lib/asan/asan_rtl.cpp +++ b/compiler-rt/lib/asan/asan_rtl.cpp @@ -146,11 +146,11 @@ ASAN_REPORT_ERROR_N(store, true) #define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \ uptr sp = MEM_TO_SHADOW(addr); \ - uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast(sp) \ - : *reinterpret_cast(sp); \ + uptr s = size <= ASAN_SHADOW_GRANULARITY ? *reinterpret_cast(sp) \ + : *reinterpret_cast(sp); \ if (UNLIKELY(s)) { \ - if (UNLIKELY(size >= SHADOW_GRANULARITY || \ - ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \ + if (UNLIKELY(size >= ASAN_SHADOW_GRANULARITY || \ + ((s8)((addr & (ASAN_SHADOW_GRANULARITY - 1)) + size - 1)) >= \ (s8)s)) { \ ReportGenericErrorWrapper(addr, is_write, size, exp_arg, fatal); \ } \ @@ -309,7 +309,7 @@ static void InitializeHighMemEnd() { kHighMemEnd = GetMaxUserVirtualAddress(); // Increase kHighMemEnd to make sure it's properly // aligned together with kHighMemBeg: - kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; + kHighMemEnd |= (GetMmapGranularity() << ASAN_SHADOW_SCALE) - 1; #endif // !ASAN_FIXED_MAPPING CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0); } @@ -361,29 +361,16 @@ void PrintAddressSpaceLayout() { Printf("malloc_context_size=%zu\n", (uptr)common_flags()->malloc_context_size); - Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE); - Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY); - Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET); - CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); + Printf("SHADOW_SCALE: %d\n", (int)ASAN_SHADOW_SCALE); + Printf("SHADOW_GRANULARITY: %d\n", (int)ASAN_SHADOW_GRANULARITY); + Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)ASAN_SHADOW_OFFSET); + CHECK(ASAN_SHADOW_SCALE >= 3 && ASAN_SHADOW_SCALE <= 7); if (kMidMemBeg) CHECK(kMidShadowBeg > kLowShadowEnd && kMidMemBeg > kMidShadowEnd && kHighShadowBeg > kMidMemEnd); } -#if defined(__thumb__) && defined(__linux__) -#define START_BACKGROUND_THREAD_IN_ASAN_INTERNAL -#endif - -#ifndef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL -static bool UNUSED __local_asan_dyninit = [] { - MaybeStartBackgroudThread(); - SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); - - return false; -}(); -#endif - static void AsanInitInternal() { if (LIKELY(asan_inited)) return; SanitizerToolName = "AddressSanitizer"; @@ -434,7 +421,7 @@ static void AsanInitInternal() { MaybeReexec(); // Setup internal allocator callback. - SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); + SetLowLevelAllocateMinAlignment(ASAN_SHADOW_GRANULARITY); SetLowLevelAllocateCallback(OnLowLevelAllocate); InitializeAsanInterceptors(); @@ -458,10 +445,8 @@ static void AsanInitInternal() { allocator_options.SetFrom(flags(), common_flags()); InitializeAllocator(allocator_options); -#ifdef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL - MaybeStartBackgroudThread(); - SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); -#endif + if (SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL) + MaybeStartBackgroudThread(); // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited // should be set to 1 prior to initializing the threads. @@ -557,7 +542,7 @@ void UnpoisonStack(uptr bottom, uptr top, const char *type) { top - bottom); return; } - PoisonShadow(bottom, RoundUpTo(top - bottom, SHADOW_GRANULARITY), 0); + PoisonShadow(bottom, RoundUpTo(top - bottom, ASAN_SHADOW_GRANULARITY), 0); } static void UnpoisonDefaultStack() { diff --git a/compiler-rt/lib/asan/asan_rtl_x86_64.S b/compiler-rt/lib/asan/asan_rtl_x86_64.S new file mode 100644 index 000000000000..d27db745ed67 --- /dev/null +++ b/compiler-rt/lib/asan/asan_rtl_x86_64.S @@ -0,0 +1,146 @@ +#include "asan_mapping.h" +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__x86_64__) +#include "sanitizer_common/sanitizer_platform.h" + +.section .text +.file "asan_rtl_x86_64.S" + +#define NAME(n, reg, op, s, i) n##_##op##_##i##_##s##_##reg + +#define FNAME(reg, op, s, i) NAME(__asan_check, reg, op, s, i) +#define RLABEL(reg, op, s, i) NAME(.return, reg, op, s, i) +#define CLABEL(reg, op, s, i) NAME(.check, reg, op, s, i) +#define FLABEL(reg, op, s, i) NAME(.fail, reg, op, s, i) + +#define BEGINF(reg, op, s, i) \ +.globl FNAME(reg, op, s, i) ;\ +.hidden FNAME(reg, op, s, i) ;\ +ASM_TYPE_FUNCTION(FNAME(reg, op, s, i)) ;\ +.cfi_startproc ;\ +FNAME(reg, op, s, i): ;\ + +#define ENDF .cfi_endproc ;\ + +// Access check functions for 1,2 and 4 byte types, which require extra checks. +#define ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, s) \ + mov %##reg,%r10 ;\ + shr $0x3,%r10 ;\ + movsbl ASAN_SHADOW_OFFSET_CONST(%r10),%r10d ;\ + test %r10d,%r10d ;\ + jne CLABEL(reg, op, s, add) ;\ +RLABEL(reg, op, s, add): ;\ + retq ;\ + +#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, i) \ +CLABEL(reg, op, 1, i): ;\ + push %rcx ;\ + mov %##reg,%rcx ;\ + and $0x7,%ecx ;\ + cmp %r10d,%ecx ;\ + pop %rcx ;\ + jl RLABEL(reg, op, 1, i);\ + mov %##reg,%rdi ;\ + jmp __asan_report_##op##1@PLT ;\ + +#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, i) \ +CLABEL(reg, op, 2, i): ;\ + push %rcx ;\ + mov %##reg,%rcx ;\ + and $0x7,%ecx ;\ + add $0x1,%ecx ;\ + cmp %r10d,%ecx ;\ + pop %rcx ;\ + jl RLABEL(reg, op, 2, i);\ + mov %##reg,%rdi ;\ + jmp __asan_report_##op##2@PLT ;\ + +#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, i) \ +CLABEL(reg, op, 4, i): ;\ + push %rcx ;\ + mov %##reg,%rcx ;\ + and $0x7,%ecx ;\ + add $0x3,%ecx ;\ + cmp %r10d,%ecx ;\ + pop %rcx ;\ + jl RLABEL(reg, op, 4, i);\ + mov %##reg,%rdi ;\ + jmp __asan_report_##op##4@PLT ;\ + +#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, op) \ +BEGINF(reg, op, 1, add) ;\ + ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 1) ;\ + ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, add) ;\ +ENDF + +#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, op) \ +BEGINF(reg, op, 2, add) ;\ + ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 2) ;\ + ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, add) ;\ +ENDF + +#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, op) \ +BEGINF(reg, op, 4, add) ;\ + ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 4) ;\ + ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, add) ;\ +ENDF + +// Access check functions for 8 and 16 byte types: no extra checks required. +#define ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, s, c) \ + mov %##reg,%r10 ;\ + shr $0x3,%r10 ;\ + ##c $0x0,ASAN_SHADOW_OFFSET_CONST(%r10) ;\ + jne FLABEL(reg, op, s, add) ;\ + retq ;\ + +#define ASAN_MEMORY_ACCESS_FAIL(reg, op, s, i) \ +FLABEL(reg, op, s, i): ;\ + mov %##reg,%rdi ;\ + jmp __asan_report_##op##s@PLT;\ + +#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, op) \ +BEGINF(reg, op, 8, add) ;\ + ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, 8, cmpb) ;\ + ASAN_MEMORY_ACCESS_FAIL(reg, op, 8, add) ;\ +ENDF + +#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, op) \ +BEGINF(reg, op, 16, add) ;\ + ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, 16, cmpw) ;\ + ASAN_MEMORY_ACCESS_FAIL(reg, op, 16, add) ;\ +ENDF + +#define ASAN_MEMORY_ACCESS_CALLBACKS_ADD(reg) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, load) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, store) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, load) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, store) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, load) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, store) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, load) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, store) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, load) \ +ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, store) \ + + +// Instantiate all but R10 and R11 callbacks. We are using PLTSafe class with +// the intrinsic, which guarantees that the code generation will never emit +// R10 or R11 callback. +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RAX) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBX) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RCX) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDX) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RSI) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDI) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBP) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R8) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R9) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R12) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R13) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R14) +ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R15) + +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/compiler-rt/lib/asan/asan_thread.cpp b/compiler-rt/lib/asan/asan_thread.cpp index 930139968ec3..2b06c3c4e7c0 100644 --- a/compiler-rt/lib/asan/asan_thread.cpp +++ b/compiler-rt/lib/asan/asan_thread.cpp @@ -305,7 +305,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) { uptr stack_size = 0; GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size, &tls_begin_, &tls_size); - stack_top_ = RoundDownTo(stack_bottom_ + stack_size, SHADOW_GRANULARITY); + stack_top_ = RoundDownTo(stack_bottom_ + stack_size, ASAN_SHADOW_GRANULARITY); tls_end_ = tls_begin_ + tls_size; dtls_ = DTLS_Get(); @@ -321,8 +321,8 @@ void AsanThread::ClearShadowForThreadStackAndTLS() { if (stack_top_ != stack_bottom_) PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) { - uptr tls_begin_aligned = RoundDownTo(tls_begin_, SHADOW_GRANULARITY); - uptr tls_end_aligned = RoundUpTo(tls_end_, SHADOW_GRANULARITY); + uptr tls_begin_aligned = RoundDownTo(tls_begin_, ASAN_SHADOW_GRANULARITY); + uptr tls_end_aligned = RoundUpTo(tls_end_, ASAN_SHADOW_GRANULARITY); FastPoisonShadowPartialRightRedzone(tls_begin_aligned, tls_end_ - tls_begin_aligned, tls_end_aligned - tls_end_, 0); @@ -346,27 +346,27 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, return true; } uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. - uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY); + uptr mem_ptr = RoundDownTo(aligned_addr, ASAN_SHADOW_GRANULARITY); u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); u8 *shadow_bottom = (u8*)MemToShadow(bottom); while (shadow_ptr >= shadow_bottom && *shadow_ptr != kAsanStackLeftRedzoneMagic) { shadow_ptr--; - mem_ptr -= SHADOW_GRANULARITY; + mem_ptr -= ASAN_SHADOW_GRANULARITY; } while (shadow_ptr >= shadow_bottom && *shadow_ptr == kAsanStackLeftRedzoneMagic) { shadow_ptr--; - mem_ptr -= SHADOW_GRANULARITY; + mem_ptr -= ASAN_SHADOW_GRANULARITY; } if (shadow_ptr < shadow_bottom) { return false; } - uptr* ptr = (uptr*)(mem_ptr + SHADOW_GRANULARITY); + uptr *ptr = (uptr *)(mem_ptr + ASAN_SHADOW_GRANULARITY); CHECK(ptr[0] == kCurrentStackFrameMagic); access->offset = addr - (uptr)ptr; access->frame_pc = ptr[2]; diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp index 1577c83cf994..53a0e3bfd385 100644 --- a/compiler-rt/lib/asan/asan_win.cpp +++ b/compiler-rt/lib/asan/asan_win.cpp @@ -253,7 +253,7 @@ void *AsanDoesNotSupportStaticLinkage() { } uptr FindDynamicShadowStart() { - return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE, + return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE, /*min_shadow_base_alignment*/ 0, kHighMemEnd); } diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c index 53e2d89708dc..cf12aa021d3d 100644 --- a/compiler-rt/lib/builtins/cpu_model.c +++ b/compiler-rt/lib/builtins/cpu_model.c @@ -798,6 +798,10 @@ _Bool __aarch64_have_lse_atomics #ifndef HWCAP_ATOMICS #define HWCAP_ATOMICS (1 << 8) #endif +#if defined(__ANDROID__) +#include +#include +#endif static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) { #if defined(__FreeBSD__) unsigned long hwcap; @@ -805,8 +809,20 @@ static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) { __aarch64_have_lse_atomics = result == 0 && (hwcap & HWCAP_ATOMICS) != 0; #else unsigned long hwcap = getauxval(AT_HWCAP); - __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0; -#endif + _Bool result = (hwcap & HWCAP_ATOMICS) != 0; +#if defined(__ANDROID__) + if (result) { + char arch[PROP_VALUE_MAX]; + if (__system_property_get("ro.arch", arch) > 0 && + strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0) { + // Some cores of Exynos 9810 are ARMv8.2 and others are ARMv8.0, + // so disable the lse atomics completely. + result = false; + } + } +#endif // defined(__ANDROID__) + __aarch64_have_lse_atomics = result; +#endif // defined(__FreeBSD__) } #endif // defined(__has_include) #endif // __has_include() diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp index ce2c04df83a8..ee7221c7b9a8 100644 --- a/compiler-rt/lib/dfsan/dfsan.cpp +++ b/compiler-rt/lib/dfsan/dfsan.cpp @@ -630,22 +630,16 @@ void PrintInvalidOriginWarning(dfsan_label label, const void *address) { d.Warning(), label, address, d.Default()); } -bool PrintOriginTraceToStr(const void *addr, const char *description, - InternalScopedString *out) { - CHECK(out); - CHECK(dfsan_get_track_origins()); +void PrintInvalidOriginIdWarning(dfsan_origin origin) { Decorator d; + Printf( + " %sOrigin Id %d has invalid origin tracking. This can " + "be a DFSan bug.%s\n", + d.Warning(), origin, d.Default()); +} - const dfsan_label label = *__dfsan::shadow_for(addr); - CHECK(label); - - const dfsan_origin origin = *__dfsan::origin_for(addr); - - out->append(" %sTaint value 0x%x (at %p) origin tracking (%s)%s\n", - d.Origin(), label, addr, description ? description : "", - d.Default()); - - Origin o = Origin::FromRawId(origin); +bool PrintOriginTraceFramesToStr(Origin o, InternalScopedString *out) { + Decorator d; bool found = false; while (o.isChainedOrigin()) { @@ -668,6 +662,25 @@ bool PrintOriginTraceToStr(const void *addr, const char *description, return found; } +bool PrintOriginTraceToStr(const void *addr, const char *description, + InternalScopedString *out) { + CHECK(out); + CHECK(dfsan_get_track_origins()); + Decorator d; + + const dfsan_label label = *__dfsan::shadow_for(addr); + CHECK(label); + + const dfsan_origin origin = *__dfsan::origin_for(addr); + + out->append(" %sTaint value 0x%x (at %p) origin tracking (%s)%s\n", + d.Origin(), label, addr, description ? description : "", + d.Default()); + + Origin o = Origin::FromRawId(origin); + return PrintOriginTraceFramesToStr(o, out); +} + } // namespace extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace( @@ -725,6 +738,50 @@ dfsan_sprint_origin_trace(const void *addr, const char *description, return trace.length(); } +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_id_trace( + dfsan_origin origin) { + if (!dfsan_get_track_origins()) { + PrintNoOriginTrackingWarning(); + return; + } + Origin o = Origin::FromRawId(origin); + + InternalScopedString trace; + bool success = PrintOriginTraceFramesToStr(o, &trace); + + if (trace.length()) + Printf("%s", trace.data()); + + if (!success) + PrintInvalidOriginIdWarning(origin); +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr dfsan_sprint_origin_id_trace( + dfsan_origin origin, char *out_buf, uptr out_buf_size) { + CHECK(out_buf); + + if (!dfsan_get_track_origins()) { + PrintNoOriginTrackingWarning(); + return 0; + } + Origin o = Origin::FromRawId(origin); + + InternalScopedString trace; + bool success = PrintOriginTraceFramesToStr(o, &trace); + + if (!success) { + PrintInvalidOriginIdWarning(origin); + return 0; + } + + if (out_buf_size) { + internal_strncpy(out_buf, trace.data(), out_buf_size - 1); + out_buf[out_buf_size - 1] = '\0'; + } + + return trace.length(); +} + extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin dfsan_get_init_origin(const void *addr) { if (!dfsan_get_track_origins()) diff --git a/compiler-rt/lib/dfsan/dfsan_allocator.cpp b/compiler-rt/lib/dfsan/dfsan_allocator.cpp index b2e94564446e..c50aee7a55a0 100644 --- a/compiler-rt/lib/dfsan/dfsan_allocator.cpp +++ b/compiler-rt/lib/dfsan/dfsan_allocator.cpp @@ -87,6 +87,12 @@ static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) { BufferedStackTrace stack; ReportAllocationSizeTooBig(size, max_malloc_size, &stack); } + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + BufferedStackTrace stack; + ReportRssLimitExceeded(&stack); + } DFsanThread *t = GetCurrentThread(); void *allocated; if (t) { diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt index eef7c48948cc..fc2dd02ccf5f 100644 --- a/compiler-rt/lib/dfsan/done_abilist.txt +++ b/compiler-rt/lib/dfsan/done_abilist.txt @@ -30,12 +30,18 @@ fun:dfsan_flush=uninstrumented fun:dfsan_flush=discard fun:dfsan_print_origin_trace=uninstrumented fun:dfsan_print_origin_trace=discard +fun:dfsan_print_origin_id_trace=uninstrumented +fun:dfsan_print_origin_id_trace=discard fun:dfsan_sprint_origin_trace=uninstrumented fun:dfsan_sprint_origin_trace=discard +fun:dfsan_sprint_origin_id_trace=uninstrumented +fun:dfsan_sprint_origin_id_trace=discard fun:dfsan_sprint_stack_trace=uninstrumented fun:dfsan_sprint_stack_trace=discard fun:dfsan_get_origin=uninstrumented fun:dfsan_get_origin=custom +fun:dfsan_read_origin_of_first_taint=uninstrumented +fun:dfsan_read_origin_of_first_taint=discard fun:dfsan_get_init_origin=uninstrumented fun:dfsan_get_init_origin=discard fun:dfsan_get_track_origins=uninstrumented diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp index 9e1729964e27..84e183f2384f 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp +++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp @@ -132,6 +132,11 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, } ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack); } + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportRssLimitExceeded(stack); + } alignment = Max(alignment, kShadowAlignment); uptr size = TaggedSize(orig_size); diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp index f96ed8804102..8dc886e587e7 100644 --- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp +++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp @@ -47,6 +47,12 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), return res; } +INTERCEPTOR(int, pthread_join, void *t, void **arg) { + return REAL(pthread_join)(t, arg); +} + +DEFINE_REAL_PTHREAD_FUNCTIONS + DEFINE_REAL(int, vfork) DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) @@ -189,7 +195,8 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(vfork); #endif // __linux__ INTERCEPT_FUNCTION(pthread_create); -#endif + INTERCEPT_FUNCTION(pthread_join); +# endif inited = 1; } diff --git a/compiler-rt/lib/lsan/lsan.h b/compiler-rt/lib/lsan/lsan.h index 1e82ad72f005..af8efa6153a5 100644 --- a/compiler-rt/lib/lsan/lsan.h +++ b/compiler-rt/lib/lsan/lsan.h @@ -13,17 +13,17 @@ #include "lsan_thread.h" #if SANITIZER_POSIX -#include "lsan_posix.h" +# include "lsan_posix.h" #elif SANITIZER_FUCHSIA -#include "lsan_fuchsia.h" +# include "lsan_fuchsia.h" #endif #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#define GET_STACK_TRACE(max_size, fast) \ - __sanitizer::BufferedStackTrace stack; \ - stack.Unwind(StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), nullptr, fast, max_size); +#define GET_STACK_TRACE(max_size, fast) \ + __sanitizer::BufferedStackTrace stack; \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, fast, \ + max_size); #define GET_STACK_TRACE_FATAL \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) @@ -40,11 +40,12 @@ void InitializeInterceptors(); void ReplaceSystemMalloc(); void LsanOnDeadlySignal(int signo, void *siginfo, void *context); -#define ENSURE_LSAN_INITED do { \ - CHECK(!lsan_init_is_running); \ - if (!lsan_inited) \ - __lsan_init(); \ -} while (0) +#define ENSURE_LSAN_INITED \ + do { \ + CHECK(!lsan_init_is_running); \ + if (!lsan_inited) \ + __lsan_init(); \ + } while (0) } // namespace __lsan diff --git a/compiler-rt/lib/lsan/lsan_allocator.cpp b/compiler-rt/lib/lsan/lsan_allocator.cpp index 91e34ebb3214..ea4c6c9cf647 100644 --- a/compiler-rt/lib/lsan/lsan_allocator.cpp +++ b/compiler-rt/lib/lsan/lsan_allocator.cpp @@ -27,11 +27,11 @@ extern "C" void *memset(void *ptr, int value, uptr num); namespace __lsan { #if defined(__i386__) || defined(__arm__) -static const uptr kMaxAllowedMallocSize = 1UL << 30; +static const uptr kMaxAllowedMallocSize = 1ULL << 30; #elif defined(__mips64) || defined(__aarch64__) -static const uptr kMaxAllowedMallocSize = 4UL << 30; +static const uptr kMaxAllowedMallocSize = 4ULL << 30; #else -static const uptr kMaxAllowedMallocSize = 8UL << 30; +static const uptr kMaxAllowedMallocSize = 8ULL << 30; #endif static Allocator allocator; @@ -88,6 +88,11 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment, size = 1; if (size > max_malloc_size) return ReportAllocationSizeTooBig(size, stack); + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportRssLimitExceeded(&stack); + } void *p = allocator.Allocate(GetAllocatorCache(), size, alignment); if (UNLIKELY(!p)) { SetAllocatorOutOfMemory(); diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp index 308dbb3e41da..fd7aa38d99db 100644 --- a/compiler-rt/lib/lsan/lsan_common.cpp +++ b/compiler-rt/lib/lsan/lsan_common.cpp @@ -34,7 +34,6 @@ Mutex global_mutex; Flags lsan_flags; - void DisableCounterUnderflow() { if (common_flags()->detect_leaks) { Report("Unmatched call to __lsan_enable().\n"); @@ -43,44 +42,48 @@ void DisableCounterUnderflow() { } void Flags::SetDefaults() { -#define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; -#include "lsan_flags.inc" -#undef LSAN_FLAG +# define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +# include "lsan_flags.inc" +# undef LSAN_FLAG } void RegisterLsanFlags(FlagParser *parser, Flags *f) { -#define LSAN_FLAG(Type, Name, DefaultValue, Description) \ - RegisterFlag(parser, #Name, Description, &f->Name); -#include "lsan_flags.inc" -#undef LSAN_FLAG +# define LSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +# include "lsan_flags.inc" +# undef LSAN_FLAG } -#define LOG_POINTERS(...) \ - do { \ - if (flags()->log_pointers) Report(__VA_ARGS__); \ - } while (0) +# define LOG_POINTERS(...) \ + do { \ + if (flags()->log_pointers) \ + Report(__VA_ARGS__); \ + } while (0) -#define LOG_THREADS(...) \ - do { \ - if (flags()->log_threads) Report(__VA_ARGS__); \ - } while (0) +# define LOG_THREADS(...) \ + do { \ + if (flags()->log_threads) \ + Report(__VA_ARGS__); \ + } while (0) class LeakSuppressionContext { bool parsed = false; SuppressionContext context; bool suppressed_stacks_sorted = true; InternalMmapVector suppressed_stacks; + const LoadedModule *suppress_module = nullptr; - Suppression *GetSuppressionForAddr(uptr addr); void LazyInit(); + Suppression *GetSuppressionForAddr(uptr addr); + bool SuppressInvalid(const StackTrace &stack); + bool SuppressByRule(const StackTrace &stack, uptr hit_count, uptr total_size); public: LeakSuppressionContext(const char *supprression_types[], int suppression_types_num) : context(supprression_types, suppression_types_num) {} - Suppression *GetSuppressionForStack(u32 stack_trace_id, - const StackTrace &stack); + bool Suppress(u32 stack_trace_id, uptr hit_count, uptr total_size); const InternalMmapVector &GetSortedSuppressedStacks() { if (!suppressed_stacks_sorted) { @@ -95,17 +98,17 @@ class LeakSuppressionContext { ALIGNED(64) static char suppression_placeholder[sizeof(LeakSuppressionContext)]; static LeakSuppressionContext *suppression_ctx = nullptr; static const char kSuppressionLeak[] = "leak"; -static const char *kSuppressionTypes[] = { kSuppressionLeak }; +static const char *kSuppressionTypes[] = {kSuppressionLeak}; static const char kStdSuppressions[] = -#if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT +# if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT // definition. "leak:*pthread_exit*\n" -#endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT -#if SANITIZER_MAC +# endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT +# if SANITIZER_MAC // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173 "leak:*_os_trace*\n" -#endif +# endif // TLS leak in some glibc versions, described in // https://sourceware.org/bugzilla/show_bug.cgi?id=12650. "leak:*tls_get_addr*\n"; @@ -123,9 +126,92 @@ void LeakSuppressionContext::LazyInit() { if (&__lsan_default_suppressions) context.Parse(__lsan_default_suppressions()); context.Parse(kStdSuppressions); + if (flags()->use_tls && flags()->use_ld_allocations) + suppress_module = GetLinker(); } } +Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) { + Suppression *s = nullptr; + + // Suppress by module name. + if (const char *module_name = + Symbolizer::GetOrInit()->GetModuleNameForPc(addr)) + if (context.Match(module_name, kSuppressionLeak, &s)) + return s; + + // Suppress by file or function name. + SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr); + for (SymbolizedStack *cur = frames; cur; cur = cur->next) { + if (context.Match(cur->info.function, kSuppressionLeak, &s) || + context.Match(cur->info.file, kSuppressionLeak, &s)) { + break; + } + } + frames->ClearAll(); + return s; +} + +static uptr GetCallerPC(const StackTrace &stack) { + // The top frame is our malloc/calloc/etc. The next frame is the caller. + if (stack.size >= 2) + return stack.trace[1]; + return 0; +} + +// On Linux, treats all chunks allocated from ld-linux.so as reachable, which +// covers dynamically allocated TLS blocks, internal dynamic loader's loaded +// modules accounting etc. +// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules. +// They are allocated with a __libc_memalign() call in allocate_and_init() +// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those +// blocks, but we can make sure they come from our own allocator by intercepting +// __libc_memalign(). On top of that, there is no easy way to reach them. Their +// addresses are stored in a dynamically allocated array (the DTV) which is +// referenced from the static TLS. Unfortunately, we can't just rely on the DTV +// being reachable from the static TLS, and the dynamic TLS being reachable from +// the DTV. This is because the initial DTV is allocated before our interception +// mechanism kicks in, and thus we don't recognize it as allocated memory. We +// can't special-case it either, since we don't know its size. +// Our solution is to include in the root set all allocations made from +// ld-linux.so (which is where allocate_and_init() is implemented). This is +// guaranteed to include all dynamic TLS blocks (and possibly other allocations +// which we don't care about). +// On all other platforms, this simply checks to ensure that the caller pc is +// valid before reporting chunks as leaked. +bool LeakSuppressionContext::SuppressInvalid(const StackTrace &stack) { + uptr caller_pc = GetCallerPC(stack); + // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark + // it as reachable, as we can't properly report its allocation stack anyway. + return !caller_pc || + (suppress_module && suppress_module->containsAddress(caller_pc)); +} + +bool LeakSuppressionContext::SuppressByRule(const StackTrace &stack, + uptr hit_count, uptr total_size) { + for (uptr i = 0; i < stack.size; i++) { + Suppression *s = GetSuppressionForAddr( + StackTrace::GetPreviousInstructionPc(stack.trace[i])); + if (s) { + s->weight += total_size; + atomic_fetch_add(&s->hit_count, hit_count, memory_order_relaxed); + return true; + } + } + return false; +} + +bool LeakSuppressionContext::Suppress(u32 stack_trace_id, uptr hit_count, + uptr total_size) { + LazyInit(); + StackTrace stack = StackDepotGet(stack_trace_id); + if (!SuppressInvalid(stack) && !SuppressByRule(stack, hit_count, total_size)) + return false; + suppressed_stacks_sorted = false; + suppressed_stacks.push_back(stack_trace_id); + return true; +} + static LeakSuppressionContext *GetSuppressionContext() { CHECK(suppression_ctx); return suppression_ctx; @@ -146,9 +232,9 @@ void InitCommonLsan() { } } -class Decorator: public __sanitizer::SanitizerCommonDecorator { +class Decorator : public __sanitizer::SanitizerCommonDecorator { public: - Decorator() : SanitizerCommonDecorator() { } + Decorator() : SanitizerCommonDecorator() {} const char *Error() { return Red(); } const char *Leak() { return Blue(); } }; @@ -157,19 +243,19 @@ static inline bool CanBeAHeapPointer(uptr p) { // Since our heap is located in mmap-ed memory, we can assume a sensible lower // bound on heap addresses. const uptr kMinAddress = 4 * 4096; - if (p < kMinAddress) return false; -#if defined(__x86_64__) + if (p < kMinAddress) + return false; +# if defined(__x86_64__) // Accept only canonical form user-space addresses. return ((p >> 47) == 0); -#elif defined(__mips64) +# elif defined(__mips64) return ((p >> 40) == 0); -#elif defined(__aarch64__) - unsigned runtimeVMA = - (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); +# elif defined(__aarch64__) + unsigned runtimeVMA = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); return ((p >> runtimeVMA) == 0); -#else +# else return true; -#endif +# endif } // Scans the memory range, looking for byte patterns that point into allocator @@ -178,8 +264,7 @@ static inline bool CanBeAHeapPointer(uptr p) { // (|tag| = kReachable) and finding indirectly leaked chunks // (|tag| = kIndirectlyLeaked). In the second case, there's no flood fill, // so |frontier| = 0. -void ScanRangeForPointers(uptr begin, uptr end, - Frontier *frontier, +void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier, const char *region_type, ChunkTag tag) { CHECK(tag == kReachable || tag == kIndirectlyLeaked); const uptr alignment = flags()->pointer_alignment(); @@ -190,13 +275,17 @@ void ScanRangeForPointers(uptr begin, uptr end, pp = pp + alignment - pp % alignment; for (; pp + sizeof(void *) <= end; pp += alignment) { void *p = *reinterpret_cast(pp); - if (!CanBeAHeapPointer(reinterpret_cast(p))) continue; + if (!CanBeAHeapPointer(reinterpret_cast(p))) + continue; uptr chunk = PointsIntoChunk(p); - if (!chunk) continue; + if (!chunk) + continue; // Pointers to self don't count. This matters when tag == kIndirectlyLeaked. - if (chunk == begin) continue; + if (chunk == begin) + continue; LsanMetadata m(chunk); - if (m.tag() == kReachable || m.tag() == kIgnored) continue; + if (m.tag() == kReachable || m.tag() == kIgnored) + continue; // Do this check relatively late so we can log only the interesting cases. if (!flags()->use_poisoned && WordIsPoisoned(pp)) { @@ -234,23 +323,23 @@ void ScanGlobalRange(uptr begin, uptr end, Frontier *frontier) { } } -void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) { +void ForEachExtraStackRangeCb(uptr begin, uptr end, void *arg) { Frontier *frontier = reinterpret_cast(arg); ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable); } -#if SANITIZER_FUCHSIA +# if SANITIZER_FUCHSIA // Fuchsia handles all threads together with its own callback. static void ProcessThreads(SuspendedThreadsList const &, Frontier *) {} -#else +# else -#if SANITIZER_ANDROID +# if SANITIZER_ANDROID // FIXME: Move this out into *libcdep.cpp extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_iterate_dynamic_tls( pid_t, void (*cb)(void *, void *, uptr, void *), void *); -#endif +# endif static void ProcessThreadRegistry(Frontier *frontier) { InternalMmapVector ptrs; @@ -282,9 +371,9 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, LOG_THREADS("Processing thread %llu.\n", os_id); uptr stack_begin, stack_end, tls_begin, tls_end, cache_begin, cache_end; DTLS *dtls; - bool thread_found = GetThreadRangesLocked(os_id, &stack_begin, &stack_end, - &tls_begin, &tls_end, - &cache_begin, &cache_end, &dtls); + bool thread_found = + GetThreadRangesLocked(os_id, &stack_begin, &stack_end, &tls_begin, + &tls_end, &cache_begin, &cache_end, &dtls); if (!thread_found) { // If a thread can't be found in the thread registry, it's probably in the // process of destruction. Log this event and move on. @@ -298,7 +387,8 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, Report("Unable to get registers from thread %llu.\n", os_id); // If unable to get SP, consider the entire stack to be reachable unless // GetRegistersAndSP failed with ESRCH. - if (have_registers == REGISTERS_UNAVAILABLE_FATAL) continue; + if (have_registers == REGISTERS_UNAVAILABLE_FATAL) + continue; sp = stack_begin; } @@ -353,7 +443,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, kReachable); } } -#if SANITIZER_ANDROID +# if SANITIZER_ANDROID auto *cb = +[](void *dtls_begin, void *dtls_end, uptr /*dso_idd*/, void *arg) -> void { ScanRangeForPointers(reinterpret_cast(dtls_begin), @@ -366,7 +456,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, // thread is suspended in the middle of updating its DTLS. IOWs, we // could scan already freed memory. (probably fine for now) __libc_iterate_dynamic_tls(os_id, cb, frontier); -#else +# else if (dtls && !DTLSInDestruction(dtls)) { ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) { uptr dtls_beg = dtv.beg; @@ -383,7 +473,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, // this and continue. LOG_THREADS("Thread %llu has DTLS under destruction.\n", os_id); } -#endif +# endif } } @@ -391,13 +481,14 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, ProcessThreadRegistry(frontier); } -#endif // SANITIZER_FUCHSIA +# endif // SANITIZER_FUCHSIA void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, uptr region_begin, uptr region_end, bool is_readable) { uptr intersection_begin = Max(root_region.begin, region_begin); uptr intersection_end = Min(region_end, root_region.begin + root_region.size); - if (intersection_begin >= intersection_end) return; + if (intersection_begin >= intersection_end) + return; LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n", (void *)root_region.begin, (void *)(root_region.begin + root_region.size), @@ -420,7 +511,8 @@ static void ProcessRootRegion(Frontier *frontier, // Scans root regions for heap pointers. static void ProcessRootRegions(Frontier *frontier) { - if (!flags()->use_root_regions) return; + if (!flags()->use_root_regions) + return; for (uptr i = 0; i < root_regions.size(); i++) ProcessRootRegion(frontier, root_regions[i]); } @@ -477,68 +569,6 @@ static void CollectIgnoredCb(uptr chunk, void *arg) { } } -static uptr GetCallerPC(const StackTrace &stack) { - // The top frame is our malloc/calloc/etc. The next frame is the caller. - if (stack.size >= 2) - return stack.trace[1]; - return 0; -} - -struct InvalidPCParam { - Frontier *frontier; - bool skip_linker_allocations; -}; - -// ForEachChunk callback. If the caller pc is invalid or is within the linker, -// mark as reachable. Called by ProcessPlatformSpecificAllocations. -static void MarkInvalidPCCb(uptr chunk, void *arg) { - CHECK(arg); - InvalidPCParam *param = reinterpret_cast(arg); - chunk = GetUserBegin(chunk); - LsanMetadata m(chunk); - if (m.allocated() && m.tag() != kReachable && m.tag() != kIgnored) { - u32 stack_id = m.stack_trace_id(); - uptr caller_pc = 0; - if (stack_id > 0) - caller_pc = GetCallerPC(StackDepotGet(stack_id)); - // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark - // it as reachable, as we can't properly report its allocation stack anyway. - if (caller_pc == 0 || (param->skip_linker_allocations && - GetLinker()->containsAddress(caller_pc))) { - m.set_tag(kReachable); - param->frontier->push_back(chunk); - } - } -} - -// On Linux, treats all chunks allocated from ld-linux.so as reachable, which -// covers dynamically allocated TLS blocks, internal dynamic loader's loaded -// modules accounting etc. -// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules. -// They are allocated with a __libc_memalign() call in allocate_and_init() -// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those -// blocks, but we can make sure they come from our own allocator by intercepting -// __libc_memalign(). On top of that, there is no easy way to reach them. Their -// addresses are stored in a dynamically allocated array (the DTV) which is -// referenced from the static TLS. Unfortunately, we can't just rely on the DTV -// being reachable from the static TLS, and the dynamic TLS being reachable from -// the DTV. This is because the initial DTV is allocated before our interception -// mechanism kicks in, and thus we don't recognize it as allocated memory. We -// can't special-case it either, since we don't know its size. -// Our solution is to include in the root set all allocations made from -// ld-linux.so (which is where allocate_and_init() is implemented). This is -// guaranteed to include all dynamic TLS blocks (and possibly other allocations -// which we don't care about). -// On all other platforms, this simply checks to ensure that the caller pc is -// valid before reporting chunks as leaked. -static void ProcessPC(Frontier *frontier) { - InvalidPCParam arg; - arg.frontier = frontier; - arg.skip_linker_allocations = - flags()->use_tls && flags()->use_ld_allocations && GetLinker() != nullptr; - ForEachChunk(MarkInvalidPCCb, &arg); -} - // Sets the appropriate tag on each chunk. static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads, Frontier *frontier) { @@ -554,9 +584,6 @@ static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads, ProcessRootRegions(frontier); FloodFillTag(frontier, kReachable); - CHECK_EQ(0, frontier->size()); - ProcessPC(frontier); - // The check here is relatively expensive, so we do this in a separate flood // fill. That way we can skip the check for chunks that are reachable // otherwise. @@ -583,14 +610,13 @@ static void ResetTagsCb(uptr chunk, void *arg) { // a LeakReport. static void CollectLeaksCb(uptr chunk, void *arg) { CHECK(arg); - LeakReport *leak_report = reinterpret_cast(arg); + LeakedChunks *leaks = reinterpret_cast(arg); chunk = GetUserBegin(chunk); LsanMetadata m(chunk); - if (!m.allocated()) return; - if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked) { - leak_report->AddLeakedChunk(chunk, m.stack_trace_id(), m.requested_size(), - m.tag()); - } + if (!m.allocated()) + return; + if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked) + leaks->push_back({chunk, m.stack_trace_id(), m.requested_size(), m.tag()}); } void LeakSuppressionContext::PrintMatchedSuppressions() { @@ -622,13 +648,13 @@ static void ReportIfNotSuspended(ThreadContextBase *tctx, void *arg) { } } -#if SANITIZER_FUCHSIA +# if SANITIZER_FUCHSIA // Fuchsia provides a libc interface that guarantees all threads are // covered, and SuspendedThreadList is never really used. static void ReportUnsuspendedThreads(const SuspendedThreadsList &) {} -#else // !SANITIZER_FUCHSIA +# else // !SANITIZER_FUCHSIA static void ReportUnsuspendedThreads( const SuspendedThreadsList &suspended_threads) { @@ -642,7 +668,7 @@ static void ReportUnsuspendedThreads( &ReportIfNotSuspended, &threads); } -#endif // !SANITIZER_FUCHSIA +# endif // !SANITIZER_FUCHSIA static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads, void *arg) { @@ -651,7 +677,7 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads, CHECK(!param->success); ReportUnsuspendedThreads(suspended_threads); ClassifyAllChunks(suspended_threads, ¶m->frontier); - ForEachChunk(CollectLeaksCb, ¶m->leak_report); + ForEachChunk(CollectLeaksCb, ¶m->leaks); // Clean up for subsequent leak checks. This assumes we did not overwrite any // kIgnored tags. ForEachChunk(ResetTagsCb, nullptr); @@ -700,17 +726,20 @@ static bool CheckForLeaks() { "etc)\n"); Die(); } + LeakReport leak_report; + leak_report.AddLeakedChunks(param.leaks); + // No new suppressions stacks, so rerun will not help and we can report. - if (!param.leak_report.ApplySuppressions()) - return PrintResults(param.leak_report); + if (!leak_report.ApplySuppressions()) + return PrintResults(leak_report); // No indirect leaks to report, so we are done here. - if (!param.leak_report.IndirectUnsuppressedLeakCount()) - return PrintResults(param.leak_report); + if (!leak_report.IndirectUnsuppressedLeakCount()) + return PrintResults(leak_report); if (i >= 8) { Report("WARNING: LeakSanitizer gave up on indirect leaks suppression.\n"); - return PrintResults(param.leak_report); + return PrintResults(leak_report); } // We found a new previously unseen suppressed call stack. Rerun to make @@ -726,10 +755,12 @@ bool HasReportedLeaks() { return has_reported_leaks; } void DoLeakCheck() { Lock l(&global_mutex); static bool already_done; - if (already_done) return; + if (already_done) + return; already_done = true; has_reported_leaks = CheckForLeaks(); - if (has_reported_leaks) HandleLeaks(); + if (has_reported_leaks) + HandleLeaks(); } static int DoRecoverableLeakCheck() { @@ -740,80 +771,50 @@ static int DoRecoverableLeakCheck() { void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); } -Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) { - Suppression *s = nullptr; - - // Suppress by module name. - if (const char *module_name = - Symbolizer::GetOrInit()->GetModuleNameForPc(addr)) - if (context.Match(module_name, kSuppressionLeak, &s)) - return s; - - // Suppress by file or function name. - SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr); - for (SymbolizedStack *cur = frames; cur; cur = cur->next) { - if (context.Match(cur->info.function, kSuppressionLeak, &s) || - context.Match(cur->info.file, kSuppressionLeak, &s)) { - break; - } - } - frames->ClearAll(); - return s; -} - -Suppression *LeakSuppressionContext::GetSuppressionForStack( - u32 stack_trace_id, const StackTrace &stack) { - LazyInit(); - for (uptr i = 0; i < stack.size; i++) { - Suppression *s = GetSuppressionForAddr( - StackTrace::GetPreviousInstructionPc(stack.trace[i])); - if (s) { - suppressed_stacks_sorted = false; - suppressed_stacks.push_back(stack_trace_id); - return s; - } - } - return nullptr; -} - ///// LeakReport implementation. ///// // A hard limit on the number of distinct leaks, to avoid quadratic complexity // in LeakReport::AddLeakedChunk(). We don't expect to ever see this many leaks // in real-world applications. -// FIXME: Get rid of this limit by changing the implementation of LeakReport to -// use a hash table. +// FIXME: Get rid of this limit by moving logic into DedupLeaks. const uptr kMaxLeaksConsidered = 5000; -void LeakReport::AddLeakedChunk(uptr chunk, u32 stack_trace_id, - uptr leaked_size, ChunkTag tag) { - CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked); +void LeakReport::AddLeakedChunks(const LeakedChunks &chunks) { + for (const LeakedChunk &leak : chunks) { + uptr chunk = leak.chunk; + u32 stack_trace_id = leak.stack_trace_id; + uptr leaked_size = leak.leaked_size; + ChunkTag tag = leak.tag; + CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked); - if (u32 resolution = flags()->resolution) { - StackTrace stack = StackDepotGet(stack_trace_id); - stack.size = Min(stack.size, resolution); - stack_trace_id = StackDepotPut(stack); - } - - bool is_directly_leaked = (tag == kDirectlyLeaked); - uptr i; - for (i = 0; i < leaks_.size(); i++) { - if (leaks_[i].stack_trace_id == stack_trace_id && - leaks_[i].is_directly_leaked == is_directly_leaked) { - leaks_[i].hit_count++; - leaks_[i].total_size += leaked_size; - break; + if (u32 resolution = flags()->resolution) { + StackTrace stack = StackDepotGet(stack_trace_id); + stack.size = Min(stack.size, resolution); + stack_trace_id = StackDepotPut(stack); + } + + bool is_directly_leaked = (tag == kDirectlyLeaked); + uptr i; + for (i = 0; i < leaks_.size(); i++) { + if (leaks_[i].stack_trace_id == stack_trace_id && + leaks_[i].is_directly_leaked == is_directly_leaked) { + leaks_[i].hit_count++; + leaks_[i].total_size += leaked_size; + break; + } + } + if (i == leaks_.size()) { + if (leaks_.size() == kMaxLeaksConsidered) + return; + Leak leak = {next_id_++, /* hit_count */ 1, + leaked_size, stack_trace_id, + is_directly_leaked, /* is_suppressed */ false}; + leaks_.push_back(leak); + } + if (flags()->report_objects) { + LeakedObject obj = {leaks_[i].id, chunk, leaked_size}; + leaked_objects_.push_back(obj); } - } - if (i == leaks_.size()) { - if (leaks_.size() == kMaxLeaksConsidered) return; - Leak leak = { next_id_++, /* hit_count */ 1, leaked_size, stack_trace_id, - is_directly_leaked, /* is_suppressed */ false }; - leaks_.push_back(leak); - } - if (flags()->report_objects) { - LeakedObject obj = {leaks_[i].id, chunk, leaked_size}; - leaked_objects_.push_back(obj); } } @@ -828,9 +829,10 @@ void LeakReport::ReportTopLeaks(uptr num_leaks_to_report) { CHECK(leaks_.size() <= kMaxLeaksConsidered); Printf("\n"); if (leaks_.size() == kMaxLeaksConsidered) - Printf("Too many leaks! Only the first %zu leaks encountered will be " - "reported.\n", - kMaxLeaksConsidered); + Printf( + "Too many leaks! Only the first %zu leaks encountered will be " + "reported.\n", + kMaxLeaksConsidered); uptr unsuppressed_count = UnsuppressedLeakCount(); if (num_leaks_to_report > 0 && num_leaks_to_report < unsuppressed_count) @@ -838,10 +840,12 @@ void LeakReport::ReportTopLeaks(uptr num_leaks_to_report) { Sort(leaks_.data(), leaks_.size(), &LeakComparator); uptr leaks_reported = 0; for (uptr i = 0; i < leaks_.size(); i++) { - if (leaks_[i].is_suppressed) continue; + if (leaks_[i].is_suppressed) + continue; PrintReportForLeak(i); leaks_reported++; - if (leaks_reported == num_leaks_to_report) break; + if (leaks_reported == num_leaks_to_report) + break; } if (leaks_reported < unsuppressed_count) { uptr remaining = unsuppressed_count - leaks_reported; @@ -880,9 +884,10 @@ void LeakReport::PrintSummary() { CHECK(leaks_.size() <= kMaxLeaksConsidered); uptr bytes = 0, allocations = 0; for (uptr i = 0; i < leaks_.size(); i++) { - if (leaks_[i].is_suppressed) continue; - bytes += leaks_[i].total_size; - allocations += leaks_[i].hit_count; + if (leaks_[i].is_suppressed) + continue; + bytes += leaks_[i].total_size; + allocations += leaks_[i].hit_count; } InternalScopedString summary; summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes, @@ -894,12 +899,8 @@ uptr LeakReport::ApplySuppressions() { LeakSuppressionContext *suppressions = GetSuppressionContext(); uptr new_suppressions = false; for (uptr i = 0; i < leaks_.size(); i++) { - Suppression *s = suppressions->GetSuppressionForStack( - leaks_[i].stack_trace_id, StackDepotGet(leaks_[i].stack_trace_id)); - if (s) { - s->weight += leaks_[i].total_size; - atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) + - leaks_[i].hit_count); + if (suppressions->Suppress(leaks_[i].stack_trace_id, leaks_[i].hit_count, + leaks_[i].total_size)) { leaks_[i].is_suppressed = true; ++new_suppressions; } @@ -910,7 +911,8 @@ uptr LeakReport::ApplySuppressions() { uptr LeakReport::UnsuppressedLeakCount() { uptr result = 0; for (uptr i = 0; i < leaks_.size(); i++) - if (!leaks_[i].is_suppressed) result++; + if (!leaks_[i].is_suppressed) + result++; return result; } @@ -922,16 +924,16 @@ uptr LeakReport::IndirectUnsuppressedLeakCount() { return result; } -} // namespace __lsan -#else // CAN_SANITIZE_LEAKS +} // namespace __lsan +#else // CAN_SANITIZE_LEAKS namespace __lsan { -void InitCommonLsan() { } -void DoLeakCheck() { } -void DoRecoverableLeakCheckVoid() { } -void DisableInThisThread() { } -void EnableInThisThread() { } -} -#endif // CAN_SANITIZE_LEAKS +void InitCommonLsan() {} +void DoLeakCheck() {} +void DoRecoverableLeakCheckVoid() {} +void DisableInThisThread() {} +void EnableInThisThread() {} +} // namespace __lsan +#endif // CAN_SANITIZE_LEAKS using namespace __lsan; @@ -948,11 +950,13 @@ void __lsan_ignore_object(const void *p) { if (res == kIgnoreObjectInvalid) VReport(1, "__lsan_ignore_object(): no heap object found at %p", p); if (res == kIgnoreObjectAlreadyIgnored) - VReport(1, "__lsan_ignore_object(): " - "heap object at %p is already being ignored\n", p); + VReport(1, + "__lsan_ignore_object(): " + "heap object at %p is already being ignored\n", + p); if (res == kIgnoreObjectSuccess) VReport(1, "__lsan_ignore_object(): ignoring heap object at %p\n", p); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -962,7 +966,7 @@ void __lsan_register_root_region(const void *begin, uptr size) { RootRegion region = {reinterpret_cast(begin), size}; root_regions.push_back(region); VReport(1, "Registered root region at %p of size %zu\n", begin, size); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -988,7 +992,7 @@ void __lsan_unregister_root_region(const void *begin, uptr size) { begin, size); Die(); } -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -1010,7 +1014,7 @@ void __lsan_do_leak_check() { #if CAN_SANITIZE_LEAKS if (common_flags()->detect_leaks) __lsan::DoLeakCheck(); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -1018,7 +1022,7 @@ int __lsan_do_recoverable_leak_check() { #if CAN_SANITIZE_LEAKS if (common_flags()->detect_leaks) return __lsan::DoRecoverableLeakCheck(); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS return 0; } @@ -1027,14 +1031,14 @@ SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_options, void) { } #if !SANITIZER_SUPPORTS_WEAK_HOOKS -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -int __lsan_is_turned_off() { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int +__lsan_is_turned_off() { return 0; } -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -const char *__lsan_default_suppressions() { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char * +__lsan_default_suppressions() { return ""; } #endif -} // extern "C" +} // extern "C" diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h index f9b55e4e8006..61b64d4dc30f 100644 --- a/compiler-rt/lib/lsan/lsan_common.h +++ b/compiler-rt/lib/lsan/lsan_common.h @@ -33,21 +33,21 @@ // Exclude leak-detection on arm32 for Android because `__aeabi_read_tp` // is missing. This caused a link error. #if SANITIZER_ANDROID && (__ANDROID_API__ < 28 || defined(__arm__)) -#define CAN_SANITIZE_LEAKS 0 +# define CAN_SANITIZE_LEAKS 0 #elif (SANITIZER_LINUX || SANITIZER_MAC) && (SANITIZER_WORDSIZE == 64) && \ (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \ defined(__powerpc64__) || defined(__s390x__)) -#define CAN_SANITIZE_LEAKS 1 +# define CAN_SANITIZE_LEAKS 1 #elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_MAC) -#define CAN_SANITIZE_LEAKS 1 +# define CAN_SANITIZE_LEAKS 1 #elif defined(__arm__) && SANITIZER_LINUX -#define CAN_SANITIZE_LEAKS 1 +# define CAN_SANITIZE_LEAKS 1 #elif SANITIZER_RISCV64 && SANITIZER_LINUX -#define CAN_SANITIZE_LEAKS 1 +# define CAN_SANITIZE_LEAKS 1 #elif SANITIZER_NETBSD || SANITIZER_FUCHSIA -#define CAN_SANITIZE_LEAKS 1 +# define CAN_SANITIZE_LEAKS 1 #else -#define CAN_SANITIZE_LEAKS 0 +# define CAN_SANITIZE_LEAKS 0 #endif namespace __sanitizer { @@ -82,6 +82,15 @@ extern Flags lsan_flags; inline Flags *flags() { return &lsan_flags; } void RegisterLsanFlags(FlagParser *parser, Flags *f); +struct LeakedChunk { + uptr chunk; + u32 stack_trace_id; + uptr leaked_size; + ChunkTag tag; +}; + +using LeakedChunks = InternalMmapVector; + struct Leak { u32 id; uptr hit_count; @@ -101,8 +110,7 @@ struct LeakedObject { class LeakReport { public: LeakReport() {} - void AddLeakedChunk(uptr chunk, u32 stack_trace_id, uptr leaked_size, - ChunkTag tag); + void AddLeakedChunks(const LeakedChunks &chunks); void ReportTopLeaks(uptr max_leaks); void PrintSummary(); uptr ApplySuppressions(); @@ -136,7 +144,7 @@ struct RootRegion { // threads and enumerating roots. struct CheckForLeaksParam { Frontier frontier; - LeakReport leak_report; + LeakedChunks leaks; bool success = false; }; @@ -224,6 +232,22 @@ bool WordIsPoisoned(uptr addr); // Wrappers for ThreadRegistry access. void LockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS; void UnlockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS; + +struct ScopedStopTheWorldLock { + ScopedStopTheWorldLock() { + LockThreadRegistry(); + LockAllocator(); + } + + ~ScopedStopTheWorldLock() { + UnlockAllocator(); + UnlockThreadRegistry(); + } + + ScopedStopTheWorldLock &operator=(const ScopedStopTheWorldLock &) = delete; + ScopedStopTheWorldLock(const ScopedStopTheWorldLock &) = delete; +}; + ThreadRegistry *GetThreadRegistryLocked(); bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end, uptr *tls_begin, uptr *tls_end, uptr *cache_begin, diff --git a/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp b/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp index 2d35fa5b1cff..e5e8cd2b4c23 100644 --- a/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp +++ b/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp @@ -58,8 +58,7 @@ int ExitHook(int status) { void LockStuffAndStopTheWorld(StopTheWorldCallback callback, CheckForLeaksParam *argument) { - LockThreadRegistry(); - LockAllocator(); + ScopedStopTheWorldLock lock; struct Params { InternalMmapVector allocator_caches; @@ -149,9 +148,6 @@ void LockStuffAndStopTheWorld(StopTheWorldCallback callback, params->callback(SuspendedThreadsListFuchsia(), params->argument); }, ¶ms); - - UnlockAllocator(); - UnlockThreadRegistry(); } } // namespace __lsan diff --git a/compiler-rt/lib/lsan/lsan_common_linux.cpp b/compiler-rt/lib/lsan/lsan_common_linux.cpp index 3af586e220f6..692ad35169e1 100644 --- a/compiler-rt/lib/lsan/lsan_common_linux.cpp +++ b/compiler-rt/lib/lsan/lsan_common_linux.cpp @@ -122,12 +122,9 @@ void HandleLeaks() { static int LockStuffAndStopTheWorldCallback(struct dl_phdr_info *info, size_t size, void *data) { - LockThreadRegistry(); - LockAllocator(); + ScopedStopTheWorldLock lock; DoStopTheWorldParam *param = reinterpret_cast(data); StopTheWorld(param->callback, param->argument); - UnlockAllocator(); - UnlockThreadRegistry(); return 1; } diff --git a/compiler-rt/lib/lsan/lsan_common_mac.cpp b/compiler-rt/lib/lsan/lsan_common_mac.cpp index 4301dcc615d7..6e8a6dfe16b4 100644 --- a/compiler-rt/lib/lsan/lsan_common_mac.cpp +++ b/compiler-rt/lib/lsan/lsan_common_mac.cpp @@ -195,11 +195,8 @@ void HandleLeaks() {} void LockStuffAndStopTheWorld(StopTheWorldCallback callback, CheckForLeaksParam *argument) { - LockThreadRegistry(); - LockAllocator(); + ScopedStopTheWorldLock lock; StopTheWorld(callback, argument); - UnlockAllocator(); - UnlockThreadRegistry(); } } // namespace __lsan diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp index ee723f210c9d..205e85685a7f 100644 --- a/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -479,6 +479,12 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, return res; } +INTERCEPTOR(int, pthread_join, void *t, void **arg) { + return REAL(pthread_join)(t, arg); +} + +DEFINE_REAL_PTHREAD_FUNCTIONS + INTERCEPTOR(void, _exit, int status) { if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode; REAL(_exit)(status); @@ -511,6 +517,7 @@ void InitializeInterceptors() { LSAN_MAYBE_INTERCEPT_MALLINFO; LSAN_MAYBE_INTERCEPT_MALLOPT; INTERCEPT_FUNCTION(pthread_create); + INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(_exit); LSAN_MAYBE_INTERCEPT__LWP_EXIT; diff --git a/compiler-rt/lib/memprof/memprof_allocator.cpp b/compiler-rt/lib/memprof/memprof_allocator.cpp index 059ce283b8c9..adbdf365bc4c 100644 --- a/compiler-rt/lib/memprof/memprof_allocator.cpp +++ b/compiler-rt/lib/memprof/memprof_allocator.cpp @@ -218,7 +218,6 @@ struct Allocator { AllocatorCache fallback_allocator_cache; uptr max_user_defined_malloc_size; - atomic_uint8_t rss_limit_exceeded; // Holds the mapping of stack ids to MemInfoBlocks. MIBMapTy MIBMap; @@ -301,20 +300,12 @@ struct Allocator { : kMaxAllowedMallocSize; } - bool RssLimitExceeded() { - return atomic_load(&rss_limit_exceeded, memory_order_relaxed); - } - - void SetRssLimitExceeded(bool limit_exceeded) { - atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); - } - // -------------------- Allocation/Deallocation routines --------------- void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, AllocType alloc_type) { if (UNLIKELY(!memprof_inited)) MemprofInitFromRtl(); - if (RssLimitExceeded()) { + if (UNLIKELY(IsRssLimitExceeded())) { if (AllocatorMayReturnNull()) return nullptr; ReportRssLimitExceeded(stack); @@ -662,10 +653,6 @@ uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) { return usable_size; } -void MemprofSoftRssLimitExceededCallback(bool limit_exceeded) { - instance.SetRssLimitExceeded(limit_exceeded); -} - } // namespace __memprof // ---------------------- Interface ---------------- {{{1 diff --git a/compiler-rt/lib/memprof/memprof_allocator.h b/compiler-rt/lib/memprof/memprof_allocator.h index f1438baaa20e..001502cde08a 100644 --- a/compiler-rt/lib/memprof/memprof_allocator.h +++ b/compiler-rt/lib/memprof/memprof_allocator.h @@ -98,7 +98,6 @@ int memprof_posix_memalign(void **memptr, uptr alignment, uptr size, uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp); void PrintInternalAllocatorStats(); -void MemprofSoftRssLimitExceededCallback(bool exceeded); } // namespace __memprof #endif // MEMPROF_ALLOCATOR_H diff --git a/compiler-rt/lib/memprof/memprof_rtl.cpp b/compiler-rt/lib/memprof/memprof_rtl.cpp index fb2ef37e51a2..c3d1c5f096fb 100644 --- a/compiler-rt/lib/memprof/memprof_rtl.cpp +++ b/compiler-rt/lib/memprof/memprof_rtl.cpp @@ -133,13 +133,6 @@ void PrintAddressSpaceLayout() { CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); } -static bool UNUSED __local_memprof_dyninit = [] { - MaybeStartBackgroudThread(); - SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback); - - return false; -}(); - static void MemprofInitInternal() { if (LIKELY(memprof_inited)) return; diff --git a/compiler-rt/lib/msan/msan_allocator.cpp b/compiler-rt/lib/msan/msan_allocator.cpp index a97bd8371e08..dc006457a59f 100644 --- a/compiler-rt/lib/msan/msan_allocator.cpp +++ b/compiler-rt/lib/msan/msan_allocator.cpp @@ -160,6 +160,11 @@ static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, } ReportAllocationSizeTooBig(size, max_malloc_size, stack); } + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportRssLimitExceeded(stack); + } MsanThread *t = GetCurrentThread(); void *allocated; if (t) { diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp index eaa3b3ae9404..df0cdecf79c7 100644 --- a/compiler-rt/lib/msan/msan_interceptors.cpp +++ b/compiler-rt/lib/msan/msan_interceptors.cpp @@ -1065,6 +1065,8 @@ INTERCEPTOR(int, pthread_join, void *th, void **retval) { return res; } +DEFINE_REAL_PTHREAD_FUNCTIONS + extern char *tzname[2]; INTERCEPTOR(void, tzset, int fake) { @@ -1705,6 +1707,7 @@ void InitializeInterceptors() { #else INTERCEPT_FUNCTION(pthread_create); #endif + INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(pthread_key_create); #if SANITIZER_NETBSD diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c index 6df65f66df73..7c1d357d96fe 100644 --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -38,7 +38,7 @@ __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) { } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) { - return __llvm_profile_raw_version; + return INSTR_PROF_RAW_VERSION_VAR; } COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 80db2527461e..bf99521d4da7 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -95,6 +95,14 @@ static uintptr_t signextIfWin64(void *V) { COMPILER_RT_VISIBILITY int __llvm_profile_merge_from_buffer(const char *ProfileData, uint64_t ProfileSize) { + if (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) { + PROF_ERR( + "%s\n", + "Debug info correlation does not support profile merging at runtime. " + "Instead, merge raw profiles using the llvm-profdata tool."); + return 1; + } + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; uint64_t *SrcCountersStart; diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 9cb05570989d..e5c0dc1479d9 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -259,16 +259,19 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const uint64_t *CountersBegin, const uint64_t *CountersEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { + int DebugInfoCorrelate = + (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; /* Calculate size of sections. */ - const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t DataSize = + DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd); const uint64_t CountersSize = CountersEnd - CountersBegin; - const uint64_t NamesSize = NamesEnd - NamesBegin; + const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; /* Create the header. */ __llvm_profile_header Header; - if (!DataSize) + if (!DataSize && (!DebugInfoCorrelate || !CountersSize)) return 0; /* Determine how much padding is needed before/after the counters and after @@ -289,6 +292,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, Header.CountersDelta = (uint32_t)Header.CountersDelta; #endif + /* The data and names sections are omitted in lightweight mode. */ + if (DebugInfoCorrelate) { + Header.CountersDelta = 0; + Header.NamesDelta = 0; + } + /* Write the profile header. */ ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}}; if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) @@ -300,11 +309,13 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Write the profile data. */ ProfDataIOVec IOVecData[] = { - {DataBegin, sizeof(__llvm_profile_data), DataSize, 0}, + {DebugInfoCorrelate ? NULL : DataBegin, sizeof(__llvm_profile_data), + DataSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint64_t), CountersSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, - {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, + {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, + sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) return -1; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp index af0b0949a88e..c5a5fb7371dd 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp @@ -17,6 +17,7 @@ #include "sanitizer_allocator_internal.h" #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_platform.h" namespace __sanitizer { @@ -195,4 +196,14 @@ void PrintHintAllocatorCannotReturnNull() { "allocator_may_return_null=1\n"); } +static atomic_uint8_t rss_limit_exceeded; + +bool IsRssLimitExceeded() { + return atomic_load(&rss_limit_exceeded, memory_order_relaxed); +} + +void SetRssLimitExceeded(bool limit_exceeded) { + atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); +} + } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index ec23465d9584..76b936ff5eaa 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -70,6 +70,9 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) { #include "sanitizer_allocator_secondary.h" #include "sanitizer_allocator_combined.h" +bool IsRssLimitExceeded(); +void SetRssLimitExceeded(bool limit_exceeded); + } // namespace __sanitizer #endif // SANITIZER_ALLOCATOR_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp index 5fae8e33b905..e9379b7bdc96 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp @@ -138,9 +138,17 @@ void LoadedModule::set(const char *module_name, uptr base_address, set(module_name, base_address); arch_ = arch; internal_memcpy(uuid_, uuid, sizeof(uuid_)); + uuid_size_ = kModuleUUIDSize; instrumented_ = instrumented; } +void LoadedModule::setUuid(const char *uuid, uptr size) { + if (size > kModuleUUIDSize) + size = kModuleUUIDSize; + internal_memcpy(uuid_, uuid, size); + uuid_size_ = size; +} + void LoadedModule::clear() { InternalFree(full_name_); base_address_ = 0; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 6ec6bb4bd856..9ddb099a8dbc 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -326,12 +326,6 @@ void SetUserDieCallback(DieCallbackType callback); void SetCheckUnwindCallback(void (*callback)()); -// Callback will be called if soft_rss_limit_mb is given and the limit is -// exceeded (exceeded==true) or if rss went down below the limit -// (exceeded==false). -// The callback should be registered once at the tool init time. -void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)); - // Functions related to signal handling. typedef void (*SignalHandlerType)(int, void *, void *); HandleSignalMode GetHandleSignalMode(int signum); @@ -673,11 +667,9 @@ void Sort(T *v, uptr size, Compare comp = {}) { // Works like std::lower_bound: finds the first element that is not less // than the val. -template > -uptr InternalLowerBound(const Container &v, - const typename Container::value_type &val, - Compare comp = {}) { +uptr InternalLowerBound(const Container &v, const T &val, Compare comp = {}) { uptr first = 0; uptr last = v.size(); while (last > first) { @@ -778,7 +770,7 @@ inline const char *ModuleArchToString(ModuleArch arch) { return ""; } -const uptr kModuleUUIDSize = 16; +const uptr kModuleUUIDSize = 32; const uptr kMaxSegName = 16; // Represents a binary loaded into virtual memory (e.g. this can be an @@ -790,6 +782,7 @@ class LoadedModule { base_address_(0), max_executable_address_(0), arch_(kModuleArchUnknown), + uuid_size_(0), instrumented_(false) { internal_memset(uuid_, 0, kModuleUUIDSize); ranges_.clear(); @@ -797,6 +790,7 @@ class LoadedModule { void set(const char *module_name, uptr base_address); void set(const char *module_name, uptr base_address, ModuleArch arch, u8 uuid[kModuleUUIDSize], bool instrumented); + void setUuid(const char *uuid, uptr size); void clear(); void addAddressRange(uptr beg, uptr end, bool executable, bool writable, const char *name = nullptr); @@ -807,6 +801,7 @@ class LoadedModule { uptr max_executable_address() const { return max_executable_address_; } ModuleArch arch() const { return arch_; } const u8 *uuid() const { return uuid_; } + uptr uuid_size() const { return uuid_size_; } bool instrumented() const { return instrumented_; } struct AddressRange { @@ -835,6 +830,7 @@ class LoadedModule { uptr base_address_; uptr max_executable_address_; ModuleArch arch_; + uptr uuid_size_; u8 uuid_[kModuleUUIDSize]; bool instrumented_; IntrusiveList ranges_; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index d219734fa0a3..b0ab08dff1db 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -7857,12 +7857,12 @@ INTERCEPTOR(void, setbuf, __sanitizer_FILE *stream, char *buf) { unpoison_file(stream); } -INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, int mode) { +INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, SIZE_T size) { void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode); - REAL(setbuffer)(stream, buf, mode); + COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, size); + REAL(setbuffer)(stream, buf, size); if (buf) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size); } if (stream) unpoison_file(stream); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc index 38f9531148d4..a5259be9335a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc @@ -11,3 +11,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush) +INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle) +INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_inline_frames) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp index bc4b477e350f..c4cc0e45193e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp @@ -10,25 +10,21 @@ // run-time libraries. //===----------------------------------------------------------------------===// +#include "sanitizer_allocator.h" #include "sanitizer_allocator_interface.h" #include "sanitizer_common.h" #include "sanitizer_flags.h" #include "sanitizer_procmaps.h" - +#include "sanitizer_stackdepot.h" namespace __sanitizer { -static void (*SoftRssLimitExceededCallback)(bool exceeded); -void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) { - CHECK_EQ(SoftRssLimitExceededCallback, nullptr); - SoftRssLimitExceededCallback = Callback; -} - #if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO // Weak default implementation for when sanitizer_stackdepot is not linked in. SANITIZER_WEAK_ATTRIBUTE StackDepotStats StackDepotGetStats() { return {}; } void *BackgroundThread(void *arg) { + VPrintf(1, "%s: Started BackgroundThread\n", SanitizerToolName); const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb; const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb; const bool heap_profile = common_flags()->heap_profile; @@ -66,13 +62,11 @@ void *BackgroundThread(void *arg) { reached_soft_rss_limit = true; Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n", SanitizerToolName, soft_rss_limit_mb, current_rss_mb); - if (SoftRssLimitExceededCallback) - SoftRssLimitExceededCallback(true); + SetRssLimitExceeded(true); } else if (soft_rss_limit_mb >= current_rss_mb && reached_soft_rss_limit) { reached_soft_rss_limit = false; - if (SoftRssLimitExceededCallback) - SoftRssLimitExceededCallback(false); + SetRssLimitExceeded(false); } } if (heap_profile && @@ -83,6 +77,38 @@ void *BackgroundThread(void *arg) { } } } + +void MaybeStartBackgroudThread() { + // Need to implement/test on other platforms. + // Start the background thread if one of the rss limits is given. + if (!common_flags()->hard_rss_limit_mb && + !common_flags()->soft_rss_limit_mb && + !common_flags()->heap_profile) return; + if (!&real_pthread_create) { + VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName); + return; // Can't spawn the thread anyway. + } + + static bool started = false; + if (!started) { + started = true; + internal_start_thread(BackgroundThread, nullptr); + } +} + +# if !SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL +# pragma clang diagnostic push +// We avoid global-constructors to be sure that globals are ready when +// sanitizers need them. This can happend before global constructors executed. +// Here we don't mind if thread is started on later stages. +# pragma clang diagnostic ignored "-Wglobal-constructors" +static struct BackgroudThreadStarted { + BackgroudThreadStarted() { MaybeStartBackgroudThread(); } +} background_thread_strarter UNUSED; +# pragma clang diagnostic pop +# endif +#else +void MaybeStartBackgroudThread() {} #endif void WriteToSyslog(const char *msg) { @@ -105,18 +131,6 @@ void WriteToSyslog(const char *msg) { WriteOneLineToSyslog(p); } -void MaybeStartBackgroudThread() { -#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \ - !SANITIZER_GO // Need to implement/test on other platforms. - // Start the background thread if one of the rss limits is given. - if (!common_flags()->hard_rss_limit_mb && - !common_flags()->soft_rss_limit_mb && - !common_flags()->heap_profile) return; - if (!&real_pthread_create) return; // Can't spawn the thread anyway. - internal_start_thread(BackgroundThread, nullptr); -#endif -} - static void (*sandboxing_callback)(); void SetSandboxingCallback(void (*f)()) { sandboxing_callback = f; @@ -185,10 +199,22 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start, #endif // !SANITIZER_FUCHSIA +#if !SANITIZER_WINDOWS && !SANITIZER_GO +// Weak default implementation for when sanitizer_stackdepot is not linked in. +SANITIZER_WEAK_ATTRIBUTE void StackDepotStopBackgroundThread() {} +static void StopStackDepotBackgroundThread() { + StackDepotStopBackgroundThread(); +} +#else +// SANITIZER_WEAK_ATTRIBUTE is unsupported. +static void StopStackDepotBackgroundThread() {} +#endif + } // namespace __sanitizer SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify, __sanitizer_sandbox_arguments *args) { + __sanitizer::StopStackDepotBackgroundThread(); __sanitizer::PlatformPrepareForSandboxing(args); if (__sanitizer::sandboxing_callback) __sanitizer::sandboxing_callback(); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc index 95da82b1a1da..0ca91aff8dd4 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc @@ -179,6 +179,7 @@ COMMON_FLAG(bool, use_madv_dontdump, true, "in core file.") COMMON_FLAG(bool, symbolize_inline_frames, true, "Print inlined frames in stacktraces. Defaults to true.") +COMMON_FLAG(bool, demangle, true, "Print demangled symbols.") COMMON_FLAG(bool, symbolize_vs_style, false, "Print file locations in Visual Studio style (e.g: " " file(10,42): ...") @@ -191,6 +192,8 @@ COMMON_FLAG(const char *, stack_trace_format, "DEFAULT", "Format string used to render stack frames. " "See sanitizer_stacktrace_printer.h for the format description. " "Use DEFAULT to get default format.") +COMMON_FLAG(int, compress_stack_depot, 0, + "Compress stack depot to save memory.") COMMON_FLAG(bool, no_huge_pages_for_shadow, true, "If true, the shadow is not allowed to use huge pages. ") COMMON_FLAG(bool, strict_string_checks, false, diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp index 9b5f6f1da1a1..66a0fd64a05a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp @@ -275,11 +275,11 @@ void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { } bool MprotectNoAccess(uptr addr, uptr size) { - return _zx_vmar_protect(_zx_vmar_root_self(), 0, Addr, Size) == ZX_OK; + return _zx_vmar_protect(_zx_vmar_root_self(), 0, addr, size) == ZX_OK; } bool MprotectReadOnly(uptr addr, uptr size) { - return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, Addr, Size) == + return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, addr, size) == ZX_OK; } @@ -484,6 +484,9 @@ u32 GetNumberOfCPUs() { return zx_system_get_num_cpus(); } uptr GetRSS() { UNIMPLEMENTED(); } +void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; } +void internal_join_thread(void *th) {} + void InitializePlatformCommonFlags(CommonFlags *cf) {} } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp index 2d787332a445..a92ea01ccccc 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp @@ -162,6 +162,12 @@ ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { // on any thread, setuid call hangs. // See test/sanitizer_common/TestCases/Linux/setuid.c. internal_sigdelset(&set, 33); +# endif +# if SANITIZER_LINUX + // Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls. + // If this signal is blocked, such calls cannot be handled and the process may + // hang. + internal_sigdelset(&set, 31); # endif SetSigProcMask(&set, &saved_); if (copy) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 6a235db0ee2e..ebd60e0b10f2 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -55,6 +55,9 @@ struct ScopedBlockSignals { explicit ScopedBlockSignals(__sanitizer_sigset_t *copy); ~ScopedBlockSignals(); + ScopedBlockSignals &operator=(const ScopedBlockSignals &) = delete; + ScopedBlockSignals(const ScopedBlockSignals &) = delete; + private: __sanitizer_sigset_t saved_; }; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp index 7ce9e25da342..3c15c35cf488 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -603,6 +603,32 @@ static int AddModuleSegments(const char *module_name, dl_phdr_info *info, bool writable = phdr->p_flags & PF_W; cur_module.addAddressRange(cur_beg, cur_end, executable, writable); + } else if (phdr->p_type == PT_NOTE) { + uptr off = 0; + while (off < phdr->p_memsz - sizeof(ElfW(Nhdr))) { + auto *nhdr = reinterpret_cast(info->dlpi_addr + + phdr->p_vaddr + off); + constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte. + static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4."); + if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) { + if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz > + phdr->p_memsz) { + // Something is very wrong, bail out instead of reading potentially + // arbitrary memory. + break; + } + const char *name = + reinterpret_cast(nhdr) + sizeof(*nhdr); + if (internal_memcmp(name, "GNU", 3) == 0) { + const char *value = reinterpret_cast(nhdr) + + sizeof(*nhdr) + kGnuNamesz; + cur_module.setUuid(value, nhdr->n_descsz); + break; + } + } + off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) + + RoundUpTo(nhdr->n_descsz, 4); + } } } modules->push_back(cur_module); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_lzw.h b/compiler-rt/lib/sanitizer_common/sanitizer_lzw.h new file mode 100644 index 000000000000..42acfbdcea09 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_lzw.h @@ -0,0 +1,159 @@ +//===-- sanitizer_lzw.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Lempel–Ziv–Welch encoding/decoding +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_LZW_H +#define SANITIZER_LZW_H + +#include "sanitizer_dense_map.h" + +namespace __sanitizer { + +using LzwCodeType = u32; + +template +ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) { + using Substring = + detail::DenseMapPair; + + // Sentinel value for substrings of len 1. + static constexpr LzwCodeType kNoPrefix = + Min(DenseMapInfo::getEmptyKey().first, + DenseMapInfo::getTombstoneKey().first) - + 1; + DenseMap prefix_to_code; + { + // Add all substring of len 1 as initial dictionary. + InternalMmapVector dict_len1; + for (auto it = begin; it != end; ++it) + if (prefix_to_code.try_emplace({kNoPrefix, *it}, 0).second) + dict_len1.push_back(*it); + + // Slightly helps with later delta encoding. + Sort(dict_len1.data(), dict_len1.size()); + + // For large sizeof(T) we have to store dict_len1. Smaller types like u8 can + // just generate them. + *out = dict_len1.size(); + ++out; + + for (uptr i = 0; i != dict_len1.size(); ++i) { + // Remap after the Sort. + prefix_to_code[{kNoPrefix, dict_len1[i]}] = i; + *out = dict_len1[i]; + ++out; + } + CHECK_EQ(prefix_to_code.size(), dict_len1.size()); + } + + if (begin == end) + return out; + + // Main LZW encoding loop. + LzwCodeType match = prefix_to_code.find({kNoPrefix, *begin})->second; + ++begin; + for (auto it = begin; it != end; ++it) { + // Extend match with the new item. + auto ins = prefix_to_code.try_emplace({match, *it}, prefix_to_code.size()); + if (ins.second) { + // This is a new substring, but emit the code for the current match + // (before extend). This allows LZW decoder to recover the dictionary. + *out = match; + ++out; + // Reset the match to a single item, which must be already in the map. + match = prefix_to_code.find({kNoPrefix, *it})->second; + } else { + // Already known, use as the current match. + match = ins.first->second; + } + } + + *out = match; + ++out; + + return out; +} + +template +ItOut LzwDecode(ItIn begin, ItIn end, ItOut out) { + if (begin == end) + return out; + + // Load dictionary of len 1 substrings. Theses correspont to lowest codes. + InternalMmapVector dict_len1(*begin); + ++begin; + + if (begin == end) + return out; + + for (auto& v : dict_len1) { + v = *begin; + ++begin; + } + + // Substrings of len 2 and up. Indexes are shifted because [0, + // dict_len1.size()) stored in dict_len1. Substings get here after being + // emitted to the output, so we can use output position. + InternalMmapVector> + code_to_substr; + + // Copies already emitted substrings into the output again. + auto copy = [&code_to_substr, &dict_len1](LzwCodeType code, ItOut out) { + if (code < dict_len1.size()) { + *out = dict_len1[code]; + ++out; + return out; + } + const auto& s = code_to_substr[code - dict_len1.size()]; + + for (ItOut it = s.first; it != s.second; ++it, ++out) *out = *it; + return out; + }; + + // Returns lens of the substring with the given code. + auto code_to_len = [&code_to_substr, &dict_len1](LzwCodeType code) -> uptr { + if (code < dict_len1.size()) + return 1; + const auto& s = code_to_substr[code - dict_len1.size()]; + return s.second - s.first; + }; + + // Main LZW decoding loop. + LzwCodeType prev_code = *begin; + ++begin; + out = copy(prev_code, out); + for (auto it = begin; it != end; ++it) { + LzwCodeType code = *it; + auto start = out; + if (code == dict_len1.size() + code_to_substr.size()) { + // Special LZW case. The code is not in the dictionary yet. This is + // possible only when the new substring is the same as previous one plus + // the first item of the previous substring. We can emit that in two + // steps. + out = copy(prev_code, out); + *out = *start; + ++out; + } else { + out = copy(code, out); + } + + // Every time encoded emits the code, it also creates substing of len + 1 + // including the first item of the just emmited substring. Do the same here. + uptr len = code_to_len(prev_code); + code_to_substr.push_back({start - len, start + 1}); + + prev_code = code; + } + return out; +} + +} // namespace __sanitizer +#endif diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp index f9b5c531aeee..a2fc310ad1a2 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp @@ -888,7 +888,7 @@ bool SignalContext::IsTrueFaultingAddress() const { (uptr)ptrauth_strip( \ (void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0) #else - #define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r + #define AARCH64_GET_REG(r) (uptr)ucontext->uc_mcontext->__ss.__##r #endif static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { @@ -1223,7 +1223,7 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale, uptr largest_gap_found = 0; uptr max_occupied_addr = 0; - VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size); + VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size); uptr shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity, &largest_gap_found, &max_occupied_addr); @@ -1232,20 +1232,21 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale, VReport( 2, "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n", - largest_gap_found, max_occupied_addr); + (void *)largest_gap_found, (void *)max_occupied_addr); uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment); if (new_max_vm < max_occupied_addr) { Report("Unable to find a memory range for dynamic shadow.\n"); Report( "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, " "new_max_vm = %p\n", - space_size, largest_gap_found, max_occupied_addr, new_max_vm); + (void *)space_size, (void *)largest_gap_found, + (void *)max_occupied_addr, (void *)new_max_vm); CHECK(0 && "cannot place shadow"); } RestrictMemoryToMaxAddress(new_max_vm); high_mem_end = new_max_vm - 1; space_size = (high_mem_end >> shadow_scale) + left_padding; - VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size); + VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size); shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity, nullptr, nullptr); if (shadow_start == 0) { @@ -1325,7 +1326,7 @@ void SignalContext::DumpAllRegisters(void *context) { # define DUMPREG64(r) \ Printf("%s = 0x%016llx ", #r, ucontext->uc_mcontext->__ss.__ ## r); # define DUMPREGA64(r) \ - Printf(" %s = 0x%016llx ", #r, AARCH64_GET_REG(r)); + Printf(" %s = 0x%016lx ", #r, AARCH64_GET_REG(r)); # define DUMPREG32(r) \ Printf("%s = 0x%08x ", #r, ucontext->uc_mcontext->__ss.__ ## r); # define DUMPREG_(r) Printf(" "); DUMPREG(r); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index 3153de34e5a3..8de765cf6669 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -22,103 +22,103 @@ // function declarations into a .S file which doesn't compile. // https://crbug.com/1162741 #if __has_include() && !defined(__ANDROID__) -#include +# include #endif #if defined(__linux__) -# define SANITIZER_LINUX 1 +# define SANITIZER_LINUX 1 #else -# define SANITIZER_LINUX 0 +# define SANITIZER_LINUX 0 #endif #if defined(__GLIBC__) -# define SANITIZER_GLIBC 1 +# define SANITIZER_GLIBC 1 #else -# define SANITIZER_GLIBC 0 +# define SANITIZER_GLIBC 0 #endif #if defined(__FreeBSD__) -# define SANITIZER_FREEBSD 1 +# define SANITIZER_FREEBSD 1 #else -# define SANITIZER_FREEBSD 0 +# define SANITIZER_FREEBSD 0 #endif #if defined(__NetBSD__) -# define SANITIZER_NETBSD 1 +# define SANITIZER_NETBSD 1 #else -# define SANITIZER_NETBSD 0 +# define SANITIZER_NETBSD 0 #endif #if defined(__sun__) && defined(__svr4__) -# define SANITIZER_SOLARIS 1 +# define SANITIZER_SOLARIS 1 #else -# define SANITIZER_SOLARIS 0 +# define SANITIZER_SOLARIS 0 #endif #if defined(__APPLE__) -# define SANITIZER_MAC 1 -# include -# if TARGET_OS_OSX -# define SANITIZER_OSX 1 -# else -# define SANITIZER_OSX 0 -# endif -# if TARGET_OS_IPHONE -# define SANITIZER_IOS 1 -# else -# define SANITIZER_IOS 0 -# endif -# if TARGET_OS_SIMULATOR -# define SANITIZER_IOSSIM 1 -# else -# define SANITIZER_IOSSIM 0 -# endif +# define SANITIZER_MAC 1 +# include +# if TARGET_OS_OSX +# define SANITIZER_OSX 1 +# else +# define SANITIZER_OSX 0 +# endif +# if TARGET_OS_IPHONE +# define SANITIZER_IOS 1 +# else +# define SANITIZER_IOS 0 +# endif +# if TARGET_OS_SIMULATOR +# define SANITIZER_IOSSIM 1 +# else +# define SANITIZER_IOSSIM 0 +# endif #else -# define SANITIZER_MAC 0 -# define SANITIZER_IOS 0 -# define SANITIZER_IOSSIM 0 -# define SANITIZER_OSX 0 +# define SANITIZER_MAC 0 +# define SANITIZER_IOS 0 +# define SANITIZER_IOSSIM 0 +# define SANITIZER_OSX 0 #endif #if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH -# define SANITIZER_WATCHOS 1 +# define SANITIZER_WATCHOS 1 #else -# define SANITIZER_WATCHOS 0 +# define SANITIZER_WATCHOS 0 #endif #if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV -# define SANITIZER_TVOS 1 +# define SANITIZER_TVOS 1 #else -# define SANITIZER_TVOS 0 +# define SANITIZER_TVOS 0 #endif #if defined(_WIN32) -# define SANITIZER_WINDOWS 1 +# define SANITIZER_WINDOWS 1 #else -# define SANITIZER_WINDOWS 0 +# define SANITIZER_WINDOWS 0 #endif #if defined(_WIN64) -# define SANITIZER_WINDOWS64 1 +# define SANITIZER_WINDOWS64 1 #else -# define SANITIZER_WINDOWS64 0 +# define SANITIZER_WINDOWS64 0 #endif #if defined(__ANDROID__) -# define SANITIZER_ANDROID 1 +# define SANITIZER_ANDROID 1 #else -# define SANITIZER_ANDROID 0 +# define SANITIZER_ANDROID 0 #endif #if defined(__Fuchsia__) -# define SANITIZER_FUCHSIA 1 +# define SANITIZER_FUCHSIA 1 #else -# define SANITIZER_FUCHSIA 0 +# define SANITIZER_FUCHSIA 0 #endif -#define SANITIZER_POSIX \ +#define SANITIZER_POSIX \ (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \ - SANITIZER_NETBSD || SANITIZER_SOLARIS) + SANITIZER_NETBSD || SANITIZER_SOLARIS) #if __LP64__ || defined(_WIN64) # define SANITIZER_WORDSIZE 64 @@ -127,58 +127,64 @@ #endif #if SANITIZER_WORDSIZE == 64 -# define FIRST_32_SECOND_64(a, b) (b) +# define FIRST_32_SECOND_64(a, b) (b) #else -# define FIRST_32_SECOND_64(a, b) (a) +# define FIRST_32_SECOND_64(a, b) (a) #endif #if defined(__x86_64__) && !defined(_LP64) -# define SANITIZER_X32 1 +# define SANITIZER_X32 1 #else -# define SANITIZER_X32 0 +# define SANITIZER_X32 0 +#endif + +#if defined(__x86_64__) || defined(_M_X64) +# define SANITIZER_X64 1 +#else +# define SANITIZER_X64 0 #endif #if defined(__i386__) || defined(_M_IX86) -# define SANITIZER_I386 1 +# define SANITIZER_I386 1 #else -# define SANITIZER_I386 0 +# define SANITIZER_I386 0 #endif #if defined(__mips__) -# define SANITIZER_MIPS 1 -# if defined(__mips64) -# define SANITIZER_MIPS32 0 -# define SANITIZER_MIPS64 1 -# else -# define SANITIZER_MIPS32 1 -# define SANITIZER_MIPS64 0 -# endif +# define SANITIZER_MIPS 1 +# if defined(__mips64) +# define SANITIZER_MIPS32 0 +# define SANITIZER_MIPS64 1 +# else +# define SANITIZER_MIPS32 1 +# define SANITIZER_MIPS64 0 +# endif #else -# define SANITIZER_MIPS 0 -# define SANITIZER_MIPS32 0 -# define SANITIZER_MIPS64 0 +# define SANITIZER_MIPS 0 +# define SANITIZER_MIPS32 0 +# define SANITIZER_MIPS64 0 #endif #if defined(__s390__) -# define SANITIZER_S390 1 -# if defined(__s390x__) -# define SANITIZER_S390_31 0 -# define SANITIZER_S390_64 1 -# else -# define SANITIZER_S390_31 1 -# define SANITIZER_S390_64 0 -# endif +# define SANITIZER_S390 1 +# if defined(__s390x__) +# define SANITIZER_S390_31 0 +# define SANITIZER_S390_64 1 +# else +# define SANITIZER_S390_31 1 +# define SANITIZER_S390_64 0 +# endif #else -# define SANITIZER_S390 0 -# define SANITIZER_S390_31 0 -# define SANITIZER_S390_64 0 +# define SANITIZER_S390 0 +# define SANITIZER_S390_31 0 +# define SANITIZER_S390_64 0 #endif #if defined(__powerpc__) -# define SANITIZER_PPC 1 -# if defined(__powerpc64__) -# define SANITIZER_PPC32 0 -# define SANITIZER_PPC64 1 +# define SANITIZER_PPC 1 +# if defined(__powerpc64__) +# define SANITIZER_PPC32 0 +# define SANITIZER_PPC64 1 // 64-bit PPC has two ABIs (v1 and v2). The old powerpc64 target is // big-endian, and uses v1 ABI (known for its function descriptors), // while the new powerpc64le target is little-endian and uses v2. @@ -186,43 +192,49 @@ // (eg. big-endian v2), but you won't find such combinations in the wild // (it'd require bootstrapping a whole system, which would be quite painful // - there's no target triple for that). LLVM doesn't support them either. -# if _CALL_ELF == 2 -# define SANITIZER_PPC64V1 0 -# define SANITIZER_PPC64V2 1 +# if _CALL_ELF == 2 +# define SANITIZER_PPC64V1 0 +# define SANITIZER_PPC64V2 1 +# else +# define SANITIZER_PPC64V1 1 +# define SANITIZER_PPC64V2 0 +# endif # else -# define SANITIZER_PPC64V1 1 -# define SANITIZER_PPC64V2 0 +# define SANITIZER_PPC32 1 +# define SANITIZER_PPC64 0 +# define SANITIZER_PPC64V1 0 +# define SANITIZER_PPC64V2 0 # endif -# else -# define SANITIZER_PPC32 1 +#else +# define SANITIZER_PPC 0 +# define SANITIZER_PPC32 0 # define SANITIZER_PPC64 0 # define SANITIZER_PPC64V1 0 # define SANITIZER_PPC64V2 0 -# endif -#else -# define SANITIZER_PPC 0 -# define SANITIZER_PPC32 0 -# define SANITIZER_PPC64 0 -# define SANITIZER_PPC64V1 0 -# define SANITIZER_PPC64V2 0 #endif -#if defined(__arm__) -# define SANITIZER_ARM 1 +#if defined(__arm__) || defined(_M_ARM) +# define SANITIZER_ARM 1 #else -# define SANITIZER_ARM 0 +# define SANITIZER_ARM 0 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +# define SANITIZER_ARM64 1 +#else +# define SANITIZER_ARM64 0 #endif #if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32 -# define SANITIZER_SOLARIS32 1 +# define SANITIZER_SOLARIS32 1 #else -# define SANITIZER_SOLARIS32 0 +# define SANITIZER_SOLARIS32 0 #endif #if defined(__riscv) && (__riscv_xlen == 64) -#define SANITIZER_RISCV64 1 +# define SANITIZER_RISCV64 1 #else -#define SANITIZER_RISCV64 0 +# define SANITIZER_RISCV64 0 #endif // By default we allow to use SizeClassAllocator64 on 64-bit platform. @@ -231,50 +243,52 @@ // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here. #ifndef SANITIZER_CAN_USE_ALLOCATOR64 -# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA -# define SANITIZER_CAN_USE_ALLOCATOR64 1 -# elif defined(__mips64) || defined(__aarch64__) -# define SANITIZER_CAN_USE_ALLOCATOR64 0 -# else -# define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64) -# endif +# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA +# define SANITIZER_CAN_USE_ALLOCATOR64 1 +# elif defined(__mips64) || defined(__aarch64__) +# define SANITIZER_CAN_USE_ALLOCATOR64 0 +# else +# define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64) +# endif #endif // The range of addresses which can be returned my mmap. // FIXME: this value should be different on different platforms. Larger values // will still work but will consume more memory for TwoLevelByteMap. #if defined(__mips__) -#if SANITIZER_GO && defined(__mips64) -#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) -#else -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) -#endif -#elif SANITIZER_RISCV64 -#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38) -#elif defined(__aarch64__) -# if SANITIZER_MAC -# if SANITIZER_OSX || SANITIZER_IOSSIM -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +# if SANITIZER_GO && defined(__mips64) +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) # else - // Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36) +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) +# endif +#elif SANITIZER_RISCV64 +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38) +#elif defined(__aarch64__) +# if SANITIZER_MAC +# if SANITIZER_OSX || SANITIZER_IOSSIM +# define SANITIZER_MMAP_RANGE_SIZE \ + FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +# else +// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM +# define SANITIZER_MMAP_RANGE_SIZE \ + FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36) +# endif +# else +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) # endif -# else -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) -# endif #elif defined(__sparc__) -#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52) +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52) #else -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) #endif // Whether the addresses are sign-extended from the VMA range to the word. // The SPARC64 Linux port implements this to split the VMA space into two // non-contiguous halves with a huge hole in the middle. #if defined(__sparc__) && SANITIZER_WORDSIZE == 64 -#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1 +# define SANITIZER_SIGN_EXTENDED_ADDRESSES 1 #else -#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0 +# define SANITIZER_SIGN_EXTENDED_ADDRESSES 0 #endif // The AArch64 and RISC-V linux ports use the canonical syscall set as @@ -297,15 +311,15 @@ // Since we don't want to include libc headers here, we check the // target only. #if defined(__arm__) || SANITIZER_X32 || defined(__sparc__) -#define SANITIZER_USES_UID16_SYSCALLS 1 +# define SANITIZER_USES_UID16_SYSCALLS 1 #else -#define SANITIZER_USES_UID16_SYSCALLS 0 +# define SANITIZER_USES_UID16_SYSCALLS 0 #endif #if defined(__mips__) -# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10) +# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10) #else -# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) +# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) #endif /// \macro MSC_PREREQ @@ -314,15 +328,15 @@ /// * 1800: Microsoft Visual Studio 2013 / 12.0 /// * 1900: Microsoft Visual Studio 2015 / 14.0 #ifdef _MSC_VER -# define MSC_PREREQ(version) (_MSC_VER >= (version)) +# define MSC_PREREQ(version) (_MSC_VER >= (version)) #else -# define MSC_PREREQ(version) 0 +# define MSC_PREREQ(version) 0 #endif #if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS) -# define SANITIZER_NON_UNIQUE_TYPEINFO 0 +# define SANITIZER_NON_UNIQUE_TYPEINFO 0 #else -# define SANITIZER_NON_UNIQUE_TYPEINFO 1 +# define SANITIZER_NON_UNIQUE_TYPEINFO 1 #endif // On linux, some architectures had an ABI transition from 64-bit long double @@ -330,11 +344,11 @@ // involving long doubles come in two versions, and we need to pass the // correct one to dlvsym when intercepting them. #if SANITIZER_LINUX && (SANITIZER_S390 || SANITIZER_PPC32 || SANITIZER_PPC64V1) -#define SANITIZER_NLDBL_VERSION "GLIBC_2.4" +# define SANITIZER_NLDBL_VERSION "GLIBC_2.4" #endif #if SANITIZER_GO == 0 -# define SANITIZER_GO 0 +# define SANITIZER_GO 0 #endif // On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks. @@ -342,40 +356,39 @@ // dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize // that this allocation happens in dynamic linker and should be ignored. #if SANITIZER_PPC || defined(__thumb__) -# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1 +# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1 #else -# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0 +# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0 #endif -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || \ - SANITIZER_SOLARIS -# define SANITIZER_MADVISE_DONTNEED MADV_FREE +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS +# define SANITIZER_MADVISE_DONTNEED MADV_FREE #else -# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED +# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED #endif // Older gcc have issues aligning to a constexpr, and require an integer. // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others. #if defined(__powerpc__) || defined(__powerpc64__) -# define SANITIZER_CACHE_LINE_SIZE 128 +# define SANITIZER_CACHE_LINE_SIZE 128 #else -# define SANITIZER_CACHE_LINE_SIZE 64 +# define SANITIZER_CACHE_LINE_SIZE 64 #endif // Enable offline markup symbolizer for Fuchsia. #if SANITIZER_FUCHSIA # define SANITIZER_SYMBOLIZER_MARKUP 1 #else -#define SANITIZER_SYMBOLIZER_MARKUP 0 +# define SANITIZER_SYMBOLIZER_MARKUP 0 #endif // Enable ability to support sanitizer initialization that is // compatible with the sanitizer library being loaded via // `dlopen()`. #if SANITIZER_MAC -#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1 +# define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1 #else -#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0 +# define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0 #endif // SANITIZER_SUPPORTS_THREADLOCAL @@ -392,4 +405,15 @@ # endif #endif -#endif // SANITIZER_PLATFORM_H +#if defined(__thumb__) && defined(__linux__) +// Workaround for +// https://lab.llvm.org/buildbot/#/builders/clang-thumbv7-full-2stage +// or +// https://lab.llvm.org/staging/#/builders/clang-thumbv7-full-2stage +// It fails *rss_limit_mb_test* without meaningful errors. +# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 1 +#else +# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 0 +#endif + +#endif // SANITIZER_PLATFORM_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp index b1c15d8c2834..4791a3a35bdb 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp @@ -10,6 +10,10 @@ #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_leb128.h" +#include "sanitizer_lzw.h" +#include "sanitizer_placement_new.h" #include "sanitizer_stacktrace.h" namespace __sanitizer { @@ -52,7 +56,7 @@ StackTrace StackStore::Load(Id id) { uptr idx = IdToOffset(id); uptr block_idx = GetBlockIdx(idx); CHECK_LT(block_idx, ARRAY_SIZE(blocks_)); - const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(); + const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(this); if (!stack_trace) return {}; stack_trace += GetInBlockIdx(idx); @@ -61,11 +65,7 @@ StackTrace StackStore::Load(Id id) { } uptr StackStore::Allocated() const { - uptr next_block = GetBlockIdx( - RoundUpTo(atomic_load_relaxed(&total_frames_), kBlockSizeFrames)); - uptr res = 0; - for (uptr i = 0; i < next_block; ++i) res += blocks_[i].Allocated(); - return res + sizeof(*this); + return atomic_load_relaxed(&allocated_) + sizeof(*this); } uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) { @@ -79,7 +79,7 @@ uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) { // Fits into the a single block. CHECK_LT(block_idx, ARRAY_SIZE(blocks_)); *idx = start; - return blocks_[block_idx].GetOrCreate() + GetInBlockIdx(start); + return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start); } // Retry. We can't use range allocated in two different blocks. @@ -92,43 +92,157 @@ uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) { } } +void *StackStore::Map(uptr size, const char *mem_type) { + atomic_fetch_add(&allocated_, size, memory_order_relaxed); + return MmapNoReserveOrDie(size, mem_type); +} + +void StackStore::Unmap(void *addr, uptr size) { + atomic_fetch_sub(&allocated_, size, memory_order_relaxed); + UnmapOrDie(addr, size); +} + uptr StackStore::Pack(Compression type) { uptr res = 0; - for (BlockInfo &b : blocks_) res += b.Pack(type); + for (BlockInfo &b : blocks_) res += b.Pack(type, this); return res; } +void StackStore::LockAll() { + for (BlockInfo &b : blocks_) b.Lock(); +} + +void StackStore::UnlockAll() { + for (BlockInfo &b : blocks_) b.Unlock(); +} + void StackStore::TestOnlyUnmap() { - for (BlockInfo &b : blocks_) b.TestOnlyUnmap(); + for (BlockInfo &b : blocks_) b.TestOnlyUnmap(this); internal_memset(this, 0, sizeof(*this)); } uptr *StackStore::BlockInfo::Get() const { // Idiomatic double-checked locking uses memory_order_acquire here. But - // relaxed is find for us, justification is similar to + // relaxed is fine for us, justification is similar to // TwoLevelMap::GetOrCreate. return reinterpret_cast(atomic_load_relaxed(&data_)); } -uptr *StackStore::BlockInfo::Create() { +uptr *StackStore::BlockInfo::Create(StackStore *store) { SpinMutexLock l(&mtx_); uptr *ptr = Get(); if (!ptr) { - ptr = reinterpret_cast( - MmapNoReserveOrDie(kBlockSizeBytes, "StackStore")); + ptr = reinterpret_cast(store->Map(kBlockSizeBytes, "StackStore")); atomic_store(&data_, reinterpret_cast(ptr), memory_order_release); } return ptr; } -uptr *StackStore::BlockInfo::GetOrCreate() { +uptr *StackStore::BlockInfo::GetOrCreate(StackStore *store) { uptr *ptr = Get(); if (LIKELY(ptr)) return ptr; - return Create(); + return Create(store); } -uptr *StackStore::BlockInfo::GetOrUnpack() { +class SLeb128Encoder { + public: + SLeb128Encoder(u8 *begin, u8 *end) : begin(begin), end(end) {} + + bool operator==(const SLeb128Encoder &other) const { + return begin == other.begin; + } + + bool operator!=(const SLeb128Encoder &other) const { + return begin != other.begin; + } + + SLeb128Encoder &operator=(uptr v) { + sptr diff = v - previous; + begin = EncodeSLEB128(diff, begin, end); + previous = v; + return *this; + } + SLeb128Encoder &operator*() { return *this; } + SLeb128Encoder &operator++() { return *this; } + + u8 *base() const { return begin; } + + private: + u8 *begin; + u8 *end; + uptr previous = 0; +}; + +class SLeb128Decoder { + public: + SLeb128Decoder(const u8 *begin, const u8 *end) : begin(begin), end(end) {} + + bool operator==(const SLeb128Decoder &other) const { + return begin == other.begin; + } + + bool operator!=(const SLeb128Decoder &other) const { + return begin != other.begin; + } + + uptr operator*() { + sptr diff; + begin = DecodeSLEB128(begin, end, &diff); + previous += diff; + return previous; + } + SLeb128Decoder &operator++() { return *this; } + + SLeb128Decoder operator++(int) { return *this; } + + private: + const u8 *begin; + const u8 *end; + uptr previous = 0; +}; + +static u8 *CompressDelta(const uptr *from, const uptr *from_end, u8 *to, + u8 *to_end) { + SLeb128Encoder encoder(to, to_end); + for (; from != from_end; ++from, ++encoder) *encoder = *from; + return encoder.base(); +} + +static uptr *UncompressDelta(const u8 *from, const u8 *from_end, uptr *to, + uptr *to_end) { + SLeb128Decoder decoder(from, from_end); + SLeb128Decoder end(from_end, from_end); + for (; decoder != end; ++to, ++decoder) *to = *decoder; + CHECK_EQ(to, to_end); + return to; +} + +static u8 *CompressLzw(const uptr *from, const uptr *from_end, u8 *to, + u8 *to_end) { + SLeb128Encoder encoder(to, to_end); + encoder = LzwEncode(from, from_end, encoder); + return encoder.base(); +} + +static uptr *UncompressLzw(const u8 *from, const u8 *from_end, uptr *to, + uptr *to_end) { + SLeb128Decoder decoder(from, from_end); + SLeb128Decoder end(from_end, from_end); + to = LzwDecode(decoder, end, to); + CHECK_EQ(to, to_end); + return to; +} + +namespace { +struct PackedHeader { + uptr size; + StackStore::Compression type; + u8 data[]; +}; +} // namespace + +uptr *StackStore::BlockInfo::GetOrUnpack(StackStore *store) { SpinMutexLock l(&mtx_); switch (state) { case State::Storing: @@ -140,15 +254,43 @@ uptr *StackStore::BlockInfo::GetOrUnpack() { break; } - uptr *ptr = Get(); + u8 *ptr = reinterpret_cast(Get()); CHECK_NE(nullptr, ptr); - // Fake unpacking. - for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i]; + const PackedHeader *header = reinterpret_cast(ptr); + CHECK_LE(header->size, kBlockSizeBytes); + CHECK_GE(header->size, sizeof(PackedHeader)); + + uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached()); + + uptr *unpacked = + reinterpret_cast(store->Map(kBlockSizeBytes, "StackStoreUnpack")); + + uptr *unpacked_end; + switch (header->type) { + case Compression::Delta: + unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked, + unpacked + kBlockSizeFrames); + break; + case Compression::LZW: + unpacked_end = UncompressLzw(header->data, ptr + header->size, unpacked, + unpacked + kBlockSizeFrames); + break; + default: + UNREACHABLE("Unexpected type"); + break; + } + + CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked); + + MprotectReadOnly(reinterpret_cast(unpacked), kBlockSizeBytes); + atomic_store(&data_, reinterpret_cast(unpacked), memory_order_release); + store->Unmap(ptr, packed_size_aligned); + state = State::Unpacked; return Get(); } -uptr StackStore::BlockInfo::Pack(Compression type) { +uptr StackStore::BlockInfo::Pack(Compression type, StackStore *store) { if (type == Compression::None) return 0; @@ -165,26 +307,55 @@ uptr StackStore::BlockInfo::Pack(Compression type) { if (!ptr || !Stored(0)) return 0; - // Fake packing. - for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i]; - state = State::Packed; - return kBlockSizeBytes - kBlockSizeBytes / 10; -} + u8 *packed = + reinterpret_cast(store->Map(kBlockSizeBytes, "StackStorePack")); + PackedHeader *header = reinterpret_cast(packed); + u8 *alloc_end = packed + kBlockSizeBytes; -uptr StackStore::BlockInfo::Allocated() const { - SpinMutexLock l(&mtx_); - switch (state) { - case State::Packed: - return kBlockSizeBytes / 10; - case State::Unpacked: - case State::Storing: - return kBlockSizeBytes; + u8 *packed_end = nullptr; + switch (type) { + case Compression::Delta: + packed_end = + CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end); + break; + case Compression::LZW: + packed_end = + CompressLzw(ptr, ptr + kBlockSizeFrames, header->data, alloc_end); + break; + default: + UNREACHABLE("Unexpected type"); + break; } + + header->type = type; + header->size = packed_end - packed; + + VPrintf(1, "Packed block of %zu KiB to %zu KiB\n", kBlockSizeBytes >> 10, + header->size >> 10); + + if (kBlockSizeBytes - header->size < kBlockSizeBytes / 8) { + VPrintf(1, "Undo and keep block unpacked\n"); + MprotectReadOnly(reinterpret_cast(ptr), kBlockSizeBytes); + store->Unmap(packed, kBlockSizeBytes); + state = State::Unpacked; + return 0; + } + + uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached()); + store->Unmap(packed + packed_size_aligned, + kBlockSizeBytes - packed_size_aligned); + MprotectReadOnly(reinterpret_cast(packed), packed_size_aligned); + + atomic_store(&data_, reinterpret_cast(packed), memory_order_release); + store->Unmap(ptr, kBlockSizeBytes); + + state = State::Packed; + return kBlockSizeBytes - packed_size_aligned; } -void StackStore::BlockInfo::TestOnlyUnmap() { +void StackStore::BlockInfo::TestOnlyUnmap(StackStore *store) { if (uptr *ptr = Get()) - UnmapOrDie(ptr, StackStore::kBlockSizeBytes); + store->Unmap(ptr, kBlockSizeBytes); } bool StackStore::BlockInfo::Stored(uptr n) { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h index e0bc4e9c4a45..1bfad811f712 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h @@ -25,7 +25,8 @@ class StackStore { public: enum class Compression : u8 { None = 0, - Test, + Delta, + LZW, }; constexpr StackStore() = default; @@ -45,6 +46,9 @@ class StackStore { // Returns the number of released bytes. uptr Pack(Compression type); + void LockAll(); + void UnlockAll(); + void TestOnlyUnmap(); private: @@ -71,9 +75,15 @@ class StackStore { uptr *Alloc(uptr count, uptr *idx, uptr *pack); + void *Map(uptr size, const char *mem_type); + void Unmap(void *addr, uptr size); + // Total number of allocated frames. atomic_uintptr_t total_frames_ = {}; + // Tracks total allocated memory in bytes. + atomic_uintptr_t allocated_ = {}; + // Each block will hold pointer to exactly kBlockSizeFrames. class BlockInfo { atomic_uintptr_t data_; @@ -89,17 +99,18 @@ class StackStore { }; State state GUARDED_BY(mtx_); - uptr *Create(); + uptr *Create(StackStore *store); public: uptr *Get() const; - uptr *GetOrCreate(); - uptr *GetOrUnpack(); - uptr Pack(Compression type); - uptr Allocated() const; - void TestOnlyUnmap(); + uptr *GetOrCreate(StackStore *store); + uptr *GetOrUnpack(StackStore *store); + uptr Pack(Compression type, StackStore *store); + void TestOnlyUnmap(StackStore *store); bool Stored(uptr n); bool IsPacked() const; + void Lock() NO_THREAD_SAFETY_ANALYSIS { mtx_.Lock(); } + void Unlock() NO_THREAD_SAFETY_ANALYSIS { mtx_.Unlock(); } }; BlockInfo blocks_[kBlockCount] = {}; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp index 527221b0c85c..c755b1829d2a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp @@ -12,8 +12,10 @@ #include "sanitizer_stackdepot.h" +#include "sanitizer_atomic.h" #include "sanitizer_common.h" #include "sanitizer_hash.h" +#include "sanitizer_mutex.h" #include "sanitizer_stack_store.h" #include "sanitizer_stackdepotbase.h" @@ -72,12 +74,125 @@ uptr StackDepotNode::allocated() { return stackStore.Allocated() + useCounts.MemoryUsage(); } +static void CompressStackStore() { + u64 start = MonotonicNanoTime(); + uptr diff = stackStore.Pack(static_cast( + Abs(common_flags()->compress_stack_depot))); + if (!diff) + return; + u64 finish = MonotonicNanoTime(); + uptr total_before = theDepot.GetStats().allocated + diff; + VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n", + SanitizerToolName, diff >> 10, total_before >> 10, + (finish - start) / 1000000); +} + +namespace { + +class CompressThread { + public: + constexpr CompressThread() = default; + void NewWorkNotify(); + void Stop(); + void LockAndStop() NO_THREAD_SAFETY_ANALYSIS; + void Unlock() NO_THREAD_SAFETY_ANALYSIS; + + private: + enum class State { + NotStarted = 0, + Started, + Failed, + Stopped, + }; + + void Run(); + + bool WaitForWork() { + semaphore_.Wait(); + return atomic_load(&run_, memory_order_acquire); + } + + Semaphore semaphore_ = {}; + StaticSpinMutex mutex_ = {}; + State state_ GUARDED_BY(mutex_) = State::NotStarted; + void *thread_ GUARDED_BY(mutex_) = nullptr; + atomic_uint8_t run_ = {}; +}; + +static CompressThread compress_thread; + +void CompressThread::NewWorkNotify() { + int compress = common_flags()->compress_stack_depot; + if (!compress) + return; + if (compress > 0 /* for testing or debugging */) { + SpinMutexLock l(&mutex_); + if (state_ == State::NotStarted) { + atomic_store(&run_, 1, memory_order_release); + CHECK_EQ(nullptr, thread_); + thread_ = internal_start_thread( + [](void *arg) -> void * { + reinterpret_cast(arg)->Run(); + return nullptr; + }, + this); + state_ = thread_ ? State::Started : State::Failed; + } + if (state_ == State::Started) { + semaphore_.Post(); + return; + } + } + CompressStackStore(); +} + +void CompressThread::Run() { + VPrintf(1, "%s: StackDepot compression thread started\n", SanitizerToolName); + while (WaitForWork()) CompressStackStore(); + VPrintf(1, "%s: StackDepot compression thread stopped\n", SanitizerToolName); +} + +void CompressThread::Stop() { + void *t = nullptr; + { + SpinMutexLock l(&mutex_); + if (state_ != State::Started) + return; + state_ = State::Stopped; + CHECK_NE(nullptr, thread_); + t = thread_; + thread_ = nullptr; + } + atomic_store(&run_, 0, memory_order_release); + semaphore_.Post(); + internal_join_thread(t); +} + +void CompressThread::LockAndStop() { + mutex_.Lock(); + if (state_ != State::Started) + return; + CHECK_NE(nullptr, thread_); + + atomic_store(&run_, 0, memory_order_release); + semaphore_.Post(); + internal_join_thread(thread_); + // Allow to restart after Unlock() if needed. + state_ = State::NotStarted; + thread_ = nullptr; +} + +void CompressThread::Unlock() { mutex_.Unlock(); } + +} // namespace + void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) { stack_hash = hash; uptr pack = 0; store_id = stackStore.Store(args, &pack); - if (pack) - stackStore.Pack(StackStore::Compression::None); + if (LIKELY(!pack)) + return; + compress_thread.NewWorkNotify(); } StackDepotNode::args_type StackDepotNode::load(u32 id) const { @@ -100,9 +215,13 @@ StackTrace StackDepotGet(u32 id) { void StackDepotLockAll() { theDepot.LockAll(); + compress_thread.LockAndStop(); + stackStore.LockAll(); } void StackDepotUnlockAll() { + stackStore.UnlockAll(); + compress_thread.Unlock(); theDepot.UnlockAll(); } @@ -112,6 +231,8 @@ void StackDepotPrintAll() { #endif } +void StackDepotStopBackgroundThread() { compress_thread.Stop(); } + StackDepotHandle StackDepotNode::get_handle(u32 id) { return StackDepotHandle(&theDepot.nodes[id], id); } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h index 56d655d9404c..cca6fd534688 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h @@ -42,6 +42,7 @@ StackTrace StackDepotGet(u32 id); void StackDepotLockAll(); void StackDepotUnlockAll(); void StackDepotPrintAll(); +void StackDepotStopBackgroundThread(); void StackDepotTestOnlyUnmap(); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp index c6356dae23c1..2d0eccc1602a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp @@ -104,6 +104,19 @@ static const char *DemangleFunctionName(const char *function) { return function; } +static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace, + InternalScopedString *buffer) { + if (info.uuid_size) { + if (PrefixSpace) + buffer->append(" "); + buffer->append("(BuildId: "); + for (uptr i = 0; i < info.uuid_size; ++i) { + buffer->append("%02x", info.uuid[i]); + } + buffer->append(")"); + } +} + static const char kDefaultFormat[] = " #%n %p %F %L"; void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, @@ -140,6 +153,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, case 'o': buffer->append("0x%zx", info->module_offset); break; + case 'b': + MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer); + break; case 'f': buffer->append("%s", DemangleFunctionName(StripFunctionName( info->function, strip_func_prefix))); @@ -181,6 +197,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, } else if (info->module) { RenderModuleLocation(buffer, info->module, info->module_offset, info->module_arch, strip_path_prefix); + + MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer); } else { buffer->append("()"); } @@ -193,6 +211,7 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, // Always strip the module name for %M. RenderModuleLocation(buffer, StripModuleName(info->module), info->module_offset, info->module_arch, ""); + MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer); } else { buffer->append("(%p)", (void *)address); } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp new file mode 100644 index 000000000000..e12b9e5bee06 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp @@ -0,0 +1,175 @@ +//===-- sanitizer_stoptheworld_win.cpp ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// See sanitizer_stoptheworld.h for details. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_WINDOWS + +# define WIN32_LEAN_AND_MEAN +# include +// windows.h needs to be included before tlhelp32.h +# include + +# include "sanitizer_stoptheworld.h" + +namespace __sanitizer { + +namespace { + +struct SuspendedThreadsListWindows final : public SuspendedThreadsList { + InternalMmapVector threadHandles; + InternalMmapVector threadIds; + + SuspendedThreadsListWindows() { + threadIds.reserve(1024); + threadHandles.reserve(1024); + } + + PtraceRegistersStatus GetRegistersAndSP(uptr index, + InternalMmapVector *buffer, + uptr *sp) const override; + + tid_t GetThreadID(uptr index) const override; + uptr ThreadCount() const override; +}; + +// Stack Pointer register names on different architectures +# if SANITIZER_X64 +# define SP_REG Rsp +# elif SANITIZER_I386 +# define SP_REG Esp +# elif SANITIZER_ARM | SANITIZER_ARM64 +# define SP_REG Sp +# else +# error Architecture not supported! +# endif + +PtraceRegistersStatus SuspendedThreadsListWindows::GetRegistersAndSP( + uptr index, InternalMmapVector *buffer, uptr *sp) const { + CHECK_LT(index, threadHandles.size()); + + buffer->resize(RoundUpTo(sizeof(CONTEXT), sizeof(uptr)) / sizeof(uptr)); + CONTEXT *thread_context = reinterpret_cast(buffer->data()); + thread_context->ContextFlags = CONTEXT_ALL; + CHECK(GetThreadContext(threadHandles[index], thread_context)); + *sp = thread_context->SP_REG; + + return REGISTERS_AVAILABLE; +} + +tid_t SuspendedThreadsListWindows::GetThreadID(uptr index) const { + CHECK_LT(index, threadIds.size()); + return threadIds[index]; +} + +uptr SuspendedThreadsListWindows::ThreadCount() const { + return threadIds.size(); +} + +struct RunThreadArgs { + StopTheWorldCallback callback; + void *argument; +}; + +DWORD WINAPI RunThread(void *argument) { + RunThreadArgs *run_args = (RunThreadArgs *)argument; + + const DWORD this_thread = GetCurrentThreadId(); + const DWORD this_process = GetCurrentProcessId(); + + SuspendedThreadsListWindows suspended_threads_list; + bool new_thread_found; + + do { + // Take a snapshot of all Threads + const HANDLE threads = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + CHECK(threads != INVALID_HANDLE_VALUE); + + THREADENTRY32 thread_entry; + thread_entry.dwSize = sizeof(thread_entry); + new_thread_found = false; + + if (!Thread32First(threads, &thread_entry)) + break; + + do { + if (thread_entry.th32ThreadID == this_thread || + thread_entry.th32OwnerProcessID != this_process) + continue; + + bool suspended_thread = false; + for (const auto thread_id : suspended_threads_list.threadIds) { + if (thread_id == thread_entry.th32ThreadID) { + suspended_thread = true; + break; + } + } + + // Skip the Thread if it was already suspended + if (suspended_thread) + continue; + + const HANDLE thread = + OpenThread(THREAD_ALL_ACCESS, FALSE, thread_entry.th32ThreadID); + CHECK(thread); + + if (SuspendThread(thread) == -1) { + DWORD last_error = GetLastError(); + + VPrintf(1, "Could not suspend thread %lu (error %lu)", + thread_entry.th32ThreadID, last_error); + continue; + } + + suspended_threads_list.threadIds.push_back(thread_entry.th32ThreadID); + suspended_threads_list.threadHandles.push_back(thread); + new_thread_found = true; + } while (Thread32Next(threads, &thread_entry)); + + CloseHandle(threads); + + // Between the call to `CreateToolhelp32Snapshot` and suspending the + // relevant Threads, new Threads could have potentially been created. So + // continue to find and suspend new Threads until we don't find any. + } while (new_thread_found); + + // Now all Threads of this Process except of this Thread should be suspended. + // Execute the callback function. + run_args->callback(suspended_threads_list, run_args->argument); + + // Resume all Threads + for (const auto suspended_thread_handle : + suspended_threads_list.threadHandles) { + CHECK_NE(ResumeThread(suspended_thread_handle), -1); + CloseHandle(suspended_thread_handle); + } + + return 0; +} + +} // namespace + +void StopTheWorld(StopTheWorldCallback callback, void *argument) { + struct RunThreadArgs arg = {callback, argument}; + DWORD trace_thread_id; + + auto trace_thread = + CreateThread(nullptr, 0, RunThread, &arg, 0, &trace_thread_id); + CHECK(trace_thread); + + WaitForSingleObject(trace_thread, INFINITE); + CloseHandle(trace_thread); +} + +} // namespace __sanitizer + +#endif // SANITIZER_WINDOWS diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp index 0c4b84c767aa..d3cffaa6eeff 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp @@ -11,10 +11,11 @@ //===----------------------------------------------------------------------===// #include "sanitizer_allocator_internal.h" -#include "sanitizer_platform.h" +#include "sanitizer_common.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" #include "sanitizer_placement_new.h" +#include "sanitizer_platform.h" #include "sanitizer_symbolizer_internal.h" namespace __sanitizer { @@ -30,6 +31,7 @@ void AddressInfo::Clear() { InternalFree(file); internal_memset(this, 0, sizeof(AddressInfo)); function_offset = kUnknown; + uuid_size = 0; } void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset, @@ -37,6 +39,16 @@ void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset, module = internal_strdup(mod_name); module_offset = mod_offset; module_arch = mod_arch; + uuid_size = 0; +} + +void AddressInfo::FillModuleInfo(const LoadedModule &mod) { + module = internal_strdup(mod.full_name()); + module_offset = address - mod.base_address(); + module_arch = mod.arch(); + if (mod.uuid_size()) + internal_memcpy(uuid, mod.uuid(), mod.uuid_size()); + uuid_size = mod.uuid_size(); } SymbolizedStack::SymbolizedStack() : next(nullptr), info() {} @@ -126,10 +138,4 @@ Symbolizer::SymbolizerScope::~SymbolizerScope() { sym_->end_hook_(); } -void Symbolizer::LateInitializeTools() { - for (auto &tool : tools_) { - tool.LateInitialize(); - } -} - } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h index 42bd157fa627..bad4761e345f 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h @@ -32,6 +32,8 @@ struct AddressInfo { char *module; uptr module_offset; ModuleArch module_arch; + u8 uuid[kModuleUUIDSize]; + uptr uuid_size; static const uptr kUnknown = ~(uptr)0; char *function; @@ -45,6 +47,8 @@ struct AddressInfo { // Deletes all strings and resets all fields. void Clear(); void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch); + void FillModuleInfo(const LoadedModule &mod); + uptr module_base() const { return address - module_offset; } }; // Linked list of symbolized frames (each frame is described by AddressInfo). @@ -209,9 +213,6 @@ class Symbolizer final { private: const Symbolizer *sym_; }; - - // Calls `LateInitialize()` on all items in `tools_`. - void LateInitializeTools(); }; #ifdef SANITIZER_WINDOWS diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h index b8670941a05e..df122ed3425c 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -70,11 +70,6 @@ class SymbolizerTool { return nullptr; } - // Called during the LateInitialize phase of Sanitizer initialization. - // Usually this is a safe place to call code that might need to use user - // memory allocators. - virtual void LateInitialize() {} - protected: ~SymbolizerTool() {} }; @@ -91,7 +86,7 @@ class SymbolizerProcess { ~SymbolizerProcess() {} /// The maximum number of arguments required to invoke a tool process. - static const unsigned kArgVMax = 6; + static const unsigned kArgVMax = 16; // Customizable by subclasses. virtual bool StartSymbolizerSubprocess(); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp index 3fc994fd3deb..8bbd4af0c7c2 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp @@ -84,15 +84,12 @@ const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter, SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) { Lock l(&mu_); - const char *module_name = nullptr; - uptr module_offset; - ModuleArch arch; SymbolizedStack *res = SymbolizedStack::New(addr); - if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset, - &arch)) + auto *mod = FindModuleForAddress(addr); + if (!mod) return res; // Always fill data about module name and offset. - res->info.FillModuleInfo(module_name, module_offset, arch); + res->info.FillModuleInfo(*mod); for (auto &tool : tools_) { SymbolizerScope sym_scope(this); if (tool.SymbolizePC(addr, res)) { @@ -277,14 +274,17 @@ class LLVMSymbolizerProcess final : public SymbolizerProcess { const char* const kSymbolizerArch = "--default-arch=unknown"; #endif - const char *const inline_flag = common_flags()->symbolize_inline_frames - ? "--inlines" - : "--no-inlines"; + const char *const demangle_flag = + common_flags()->demangle ? "--demangle" : "--no-demangle"; + const char *const inline_flag = + common_flags()->symbolize_inline_frames ? "--inlines" : "--no-inlines"; int i = 0; argv[i++] = path_to_binary; + argv[i++] = demangle_flag; argv[i++] = inline_flag; argv[i++] = kSymbolizerArch; argv[i++] = nullptr; + CHECK_LE(i, kArgVMax); } }; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp index 5c25b28b5dc9..ac811c8a9136 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -58,13 +57,6 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) { return true; } -#define K_ATOS_ENV_VAR "__check_mach_ports_lookup" - -// This cannot live in `AtosSymbolizerProcess` because instances of that object -// are allocated by the internal allocator which under ASan is poisoned with -// kAsanInternalHeapMagic. -static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000"; - class AtosSymbolizerProcess final : public SymbolizerProcess { public: explicit AtosSymbolizerProcess(const char *path) @@ -72,51 +64,13 @@ class AtosSymbolizerProcess final : public SymbolizerProcess { pid_str_[0] = '\0'; } - void LateInitialize() { - if (SANITIZER_IOSSIM) { - // `putenv()` may call malloc/realloc so it is only safe to do this - // during LateInitialize() or later (i.e. we can't do this in the - // constructor). We also can't do this in `StartSymbolizerSubprocess()` - // because in TSan we switch allocators when we're symbolizing. - // We use `putenv()` rather than `setenv()` so that we can later directly - // write into the storage without LibC getting involved to change what the - // variable is set to - int result = putenv(kAtosMachPortEnvEntry); - CHECK_EQ(result, 0); - } - } - private: bool StartSymbolizerSubprocess() override { - // Configure sandbox before starting atos process. - // Put the string command line argument in the object so that it outlives // the call to GetArgV. - internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid()); - - if (SANITIZER_IOSSIM) { - // `atos` in the simulator is restricted in its ability to retrieve the - // task port for the target process (us) so we need to do extra work - // to pass our task port to it. - mach_port_t ports[]{mach_task_self()}; - kern_return_t ret = - mach_ports_register(mach_task_self(), ports, /*count=*/1); - CHECK_EQ(ret, KERN_SUCCESS); - - // Set environment variable that signals to `atos` that it should look - // for our task port. We can't call `setenv()` here because it might call - // malloc/realloc. To avoid that we instead update the - // `mach_port_env_var_entry_` variable with our current PID. - uptr count = internal_snprintf(kAtosMachPortEnvEntry, - sizeof(kAtosMachPortEnvEntry), - K_ATOS_ENV_VAR "=%s", pid_str_); - CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_)); - // Document our assumption but without calling `getenv()` in normal - // builds. - DCHECK(getenv(K_ATOS_ENV_VAR)); - DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0); - } + internal_snprintf(pid_str_, sizeof(pid_str_), "%d", (int)internal_getpid()); + // Configure sandbox before starting atos process. return SymbolizerProcess::StartSymbolizerSubprocess(); } @@ -137,13 +91,10 @@ class AtosSymbolizerProcess final : public SymbolizerProcess { argv[i++] = "-d"; } argv[i++] = nullptr; + CHECK_LE(i, kArgVMax); } char pid_str_[16]; - // Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`. - static_assert(sizeof(kAtosMachPortEnvEntry) == - (sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)), - "sizes should match"); }; #undef K_ATOS_ENV_VAR @@ -249,8 +200,6 @@ bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { return true; } -void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); } - } // namespace __sanitizer #endif // SANITIZER_MAC diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h index 401d30fa5033..d5abe9d98c1f 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h @@ -35,7 +35,6 @@ class AtosSymbolizer final : public SymbolizerTool { bool SymbolizePC(uptr addr, SymbolizedStack *stack) override; bool SymbolizeData(uptr addr, DataInfo *info) override; - void LateInitialize() override; private: AtosSymbolizerProcess *process_; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp index 9a5b4a8c54c7..1ec0c5cad7a2 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp @@ -100,9 +100,7 @@ Symbolizer *Symbolizer::PlatformInit() { return new (symbolizer_allocator_) Symbolizer({}); } -void Symbolizer::LateInitialize() { - Symbolizer::GetOrInit()->LateInitializeTools(); -} +void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); } void StartReportDeadlySignal() {} void ReportDeadlySignal(const SignalContext &sig, u32 tid, diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp index 4cd4b4636f0a..5f6e4cc3180e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp @@ -213,9 +213,14 @@ class Addr2LineProcess final : public SymbolizerProcess { const char *(&argv)[kArgVMax]) const override { int i = 0; argv[i++] = path_to_binary; - argv[i++] = "-iCfe"; + if (common_flags()->demangle) + argv[i++] = "-C"; + if (common_flags()->symbolize_inline_frames) + argv[i++] = "-i"; + argv[i++] = "-fe"; argv[i++] = module_name_; argv[i++] = nullptr; + CHECK_LE(i, kArgVMax); } bool ReachedEndOfOutput(const char *buffer, uptr length) const override; @@ -312,37 +317,42 @@ class Addr2LinePool final : public SymbolizerTool { FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX); }; -#if SANITIZER_SUPPORTS_WEAK_HOOKS +# if SANITIZER_SUPPORTS_WEAK_HOOKS extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool __sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset, - char *Buffer, int MaxLength, - bool SymbolizeInlineFrames); -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset, - char *Buffer, int MaxLength); -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void __sanitizer_symbolize_flush(); -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -int __sanitizer_symbolize_demangle(const char *Name, char *Buffer, - int MaxLength); + char *Buffer, int MaxLength); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool +__sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset, + char *Buffer, int MaxLength); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_symbolize_flush(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int +__sanitizer_symbolize_demangle(const char *Name, char *Buffer, int MaxLength); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool +__sanitizer_symbolize_set_demangle(bool Demangle); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool +__sanitizer_symbolize_set_inline_frames(bool InlineFrames); } // extern "C" class InternalSymbolizer final : public SymbolizerTool { public: static InternalSymbolizer *get(LowLevelAllocator *alloc) { - if (__sanitizer_symbolize_code != 0 && - __sanitizer_symbolize_data != 0) { - return new(*alloc) InternalSymbolizer(); - } + if (__sanitizer_symbolize_set_demangle) + CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle)); + if (__sanitizer_symbolize_set_inline_frames) + CHECK(__sanitizer_symbolize_set_inline_frames( + common_flags()->symbolize_inline_frames)); + if (__sanitizer_symbolize_code && __sanitizer_symbolize_data) + return new (*alloc) InternalSymbolizer(); return 0; } bool SymbolizePC(uptr addr, SymbolizedStack *stack) override { bool result = __sanitizer_symbolize_code( - stack->info.module, stack->info.module_offset, buffer_, kBufferSize, - common_flags()->symbolize_inline_frames); - if (result) ParseSymbolizePCOutput(buffer_, stack); + stack->info.module, stack->info.module_offset, buffer_, kBufferSize); + if (result) + ParseSymbolizePCOutput(buffer_, stack); return result; } @@ -365,7 +375,7 @@ class InternalSymbolizer final : public SymbolizerTool { if (__sanitizer_symbolize_demangle) { for (uptr res_length = 1024; res_length <= InternalSizeClassMap::kMaxSize;) { - char *res_buff = static_cast(InternalAlloc(res_length)); + char *res_buff = static_cast(InternalAlloc(res_length)); uptr req_length = __sanitizer_symbolize_demangle(name, res_buff, res_length); if (req_length > res_length) { @@ -380,19 +390,19 @@ class InternalSymbolizer final : public SymbolizerTool { } private: - InternalSymbolizer() { } + InternalSymbolizer() {} static const int kBufferSize = 16 * 1024; char buffer_[kBufferSize]; }; -#else // SANITIZER_SUPPORTS_WEAK_HOOKS +# else // SANITIZER_SUPPORTS_WEAK_HOOKS class InternalSymbolizer final : public SymbolizerTool { public: static InternalSymbolizer *get(LowLevelAllocator *alloc) { return 0; } }; -#endif // SANITIZER_SUPPORTS_WEAK_HOOKS +# endif // SANITIZER_SUPPORTS_WEAK_HOOKS const char *Symbolizer::PlatformDemangle(const char *name) { return DemangleSwiftAndCXX(name); @@ -492,7 +502,7 @@ Symbolizer *Symbolizer::PlatformInit() { } void Symbolizer::LateInitialize() { - Symbolizer::GetOrInit()->LateInitializeTools(); + Symbolizer::GetOrInit(); InitializeSwiftDemangler(); } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp index 702d901353db..c647ab107ec5 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp @@ -318,7 +318,7 @@ Symbolizer *Symbolizer::PlatformInit() { } void Symbolizer::LateInitialize() { - Symbolizer::GetOrInit()->LateInitializeTools(); + Symbolizer::GetOrInit(); } } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp index 2e1cd0238812..278f6defca95 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp @@ -110,7 +110,7 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads, max_threads_(max_threads), thread_quarantine_size_(thread_quarantine_size), max_reuse_(max_reuse), - mtx_(), + mtx_(MutexThreadRegistry), total_threads_(0), alive_threads_(0), max_alive_threads_(0), @@ -365,4 +365,20 @@ void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) { CHECK(live_.try_emplace(user_id, tctx->tid).second); } +u32 ThreadRegistry::OnFork(u32 tid) { + ThreadRegistryLock l(this); + // We only purge user_id (pthread_t) of live threads because + // they cause CHECK failures if new threads with matching pthread_t + // created after fork. + // Potentially we could purge more info (ThreadContextBase themselves), + // but it's hard to test and easy to introduce new issues by doing this. + for (auto *tctx : threads_) { + if (tctx->tid == tid || !tctx->user_id) + continue; + CHECK(live_.erase(tctx->user_id)); + tctx->user_id = 0; + } + return alive_threads_; +} + } // namespace __sanitizer diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h index a259b324220f..9975d78ec0bb 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h @@ -104,6 +104,8 @@ class MUTEX ThreadRegistry { return threads_.empty() ? nullptr : threads_[tid]; } + u32 NumThreadsLocked() const { return threads_.size(); } + u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg); typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg); @@ -131,6 +133,11 @@ class MUTEX ThreadRegistry { u32 ConsumeThreadUserId(uptr user_id); void SetThreadUserId(u32 tid, uptr user_id); + // OnFork must be called in the child process after fork to purge old + // threads that don't exist anymore (except for the current thread tid). + // Returns number of alive threads before fork. + u32 OnFork(u32 tid); + private: const ThreadContextFactory context_factory_; const u32 max_threads_; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp index 1a31ce02af4c..cfe6cc2b394b 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp @@ -16,7 +16,6 @@ #define WIN32_LEAN_AND_MEAN #define NOGDI -#include #include #include #include @@ -571,7 +570,9 @@ void Abort() { internal__exit(3); } -bool CreateDir(const char *pathname) { return _mkdir(pathname) == 0; } +bool CreateDir(const char *pathname) { + return CreateDirectoryA(pathname, nullptr) != 0; +} #if !SANITIZER_GO // Read the file to extract the ImageBase field from the PE header. If ASLR is diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp index 3809880d50b4..80cab36426c5 100644 --- a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp +++ b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp @@ -17,10 +17,17 @@ #include "llvm/DebugInfo/Symbolize/DIPrinter.h" #include "llvm/DebugInfo/Symbolize/Symbolize.h" +static llvm::symbolize::LLVMSymbolizer *Symbolizer = nullptr; +static bool Demangle = true; +static bool InlineFrames = true; + static llvm::symbolize::LLVMSymbolizer *getDefaultSymbolizer() { - static llvm::symbolize::LLVMSymbolizer *DefaultSymbolizer = - new llvm::symbolize::LLVMSymbolizer(); - return DefaultSymbolizer; + if (Symbolizer) + return Symbolizer; + llvm::symbolize::LLVMSymbolizer::Options Opts; + Opts.Demangle = Demangle; + Symbolizer = new llvm::symbolize::LLVMSymbolizer(Opts); + return Symbolizer; } static llvm::symbolize::PrinterConfig getDefaultPrinterConfig() { @@ -43,8 +50,7 @@ extern "C" { typedef uint64_t u64; bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset, - char *Buffer, int MaxLength, - bool SymbolizeInlineFrames) { + char *Buffer, int MaxLength) { std::string Result; { llvm::raw_string_ostream OS(Result); @@ -55,7 +61,7 @@ bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset, // TODO: it is neccessary to set proper SectionIndex here. // object::SectionedAddress::UndefSection works for only absolute addresses. - if (SymbolizeInlineFrames) { + if (InlineFrames) { auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode( ModuleName, {ModuleOffset, llvm::object::SectionedAddress::UndefSection}); @@ -93,7 +99,10 @@ bool __sanitizer_symbolize_data(const char *ModuleName, uint64_t ModuleOffset, Result.c_str()) < MaxLength; } -void __sanitizer_symbolize_flush() { getDefaultSymbolizer()->flush(); } +void __sanitizer_symbolize_flush() { + if (Symbolizer) + Symbolizer->flush(); +} int __sanitizer_symbolize_demangle(const char *Name, char *Buffer, int MaxLength) { @@ -105,6 +114,19 @@ int __sanitizer_symbolize_demangle(const char *Name, char *Buffer, : 0; } +bool __sanitizer_symbolize_set_demangle(bool Value) { + // Must be called before LLVMSymbolizer created. + if (Symbolizer) + return false; + Demangle = Value; + return true; +} + +bool __sanitizer_symbolize_set_inline_frames(bool Value) { + InlineFrames = Value; + return true; +} + // Override __cxa_atexit and ignore callbacks. // This prevents crashes in a configuration when the symbolizer // is built into sanitizer runtime and consequently into the test process. diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh index d1d61fb7ab2a..599f063b45c9 100755 --- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh +++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh @@ -143,7 +143,7 @@ if [[ ! -d ${LLVM_BUILD} ]]; then $LLVM_SRC fi cd ${LLVM_BUILD} -ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMMC LLVMDemangle LLVMTextAPI +ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMDebuginfod LLVMMC LLVMDemangle LLVMTextAPI cd ${BUILD_DIR} rm -rf ${SYMBOLIZER_BUILD} @@ -155,7 +155,12 @@ SYMBOLIZER_FLAGS="$LLVM_FLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std $CXX $SYMBOLIZER_FLAGS ${SRC_DIR}/sanitizer_symbolize.cpp ${SRC_DIR}/sanitizer_wrappers.cpp -c $AR rc symbolizer.a sanitizer_symbolize.o sanitizer_wrappers.o -SYMBOLIZER_API_LIST=__sanitizer_symbolize_code,__sanitizer_symbolize_data,__sanitizer_symbolize_flush,__sanitizer_symbolize_demangle +SYMBOLIZER_API_LIST=__sanitizer_symbolize_code +SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_data +SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_flush +SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_demangle +SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_set_demangle +SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_set_inline_frames LIBCXX_ARCHIVE_DIR=$(dirname $(find $LIBCXX_BUILD -name libc++.a | head -n1)) @@ -170,6 +175,7 @@ $SCRIPT_DIR/ar_to_bc.sh $LIBCXX_ARCHIVE_DIR/libc++.a \ $LLVM_BUILD/lib/libLLVMDebugInfoPDB.a \ $LLVM_BUILD/lib/libLLVMDebugInfoMSF.a \ $LLVM_BUILD/lib/libLLVMDebugInfoCodeView.a \ + $LLVM_BUILD/lib/libLLVMDebuginfod.a \ $LLVM_BUILD/lib/libLLVMDemangle.a \ $LLVM_BUILD/lib/libLLVMMC.a \ $LLVM_BUILD/lib/libLLVMTextAPI.a \ diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt index 29b2960e11fe..0bb38ba951a8 100644 --- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt +++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt @@ -38,6 +38,8 @@ __sanitizer_symbolize_code T __sanitizer_symbolize_data T __sanitizer_symbolize_demangle T __sanitizer_symbolize_flush T +__sanitizer_symbolize_set_demangle T +__sanitizer_symbolize_set_inline_frames T __strdup U __udivdi3 U __umoddi3 U @@ -51,8 +53,8 @@ catgets U catopen U ceil U ceilf U -clock_gettime U cfgetospeed U +clock_gettime U dl_iterate_phdr U dlsym U dup U @@ -76,15 +78,17 @@ getcwd U getenv U getpagesize U getpid U +getpwuid U getrlimit U gettimeofday U +getuid U ioctl U isalnum U isalpha U isatty U islower U -isspace U isprint U +isspace U isupper U isxdigit U log10 U @@ -111,12 +115,17 @@ posix_spawn_file_actions_addopen U posix_spawn_file_actions_destroy U posix_spawn_file_actions_init U qsort U +raise U rand U readlink U realloc U remove U +rename U setrlimit U setvbuf U +sigaction U +sigaltstack U +sigemptyset U sigfillset U sigprocmask U snprintf U @@ -146,6 +155,7 @@ strtold_l U strtoll_l U strtoull_l U syscall U +sysconf U tcgetattr U uname U ungetc U diff --git a/compiler-rt/lib/sanitizer_common/weak_symbols.txt b/compiler-rt/lib/sanitizer_common/weak_symbols.txt index 5a2b275184f4..d07f81bc8c12 100644 --- a/compiler-rt/lib/sanitizer_common/weak_symbols.txt +++ b/compiler-rt/lib/sanitizer_common/weak_symbols.txt @@ -6,3 +6,5 @@ ___sanitizer_symbolize_code ___sanitizer_symbolize_data ___sanitizer_symbolize_demangle ___sanitizer_symbolize_flush +___sanitizer_symbolize_set_demangle +___sanitizer_symbolize_set_inline_frames diff --git a/compiler-rt/lib/tsan/go/tsan_go.cpp b/compiler-rt/lib/tsan/go/tsan_go.cpp index 104c5b325aee..c689a51fb5e1 100644 --- a/compiler-rt/lib/tsan/go/tsan_go.cpp +++ b/compiler-rt/lib/tsan/go/tsan_go.cpp @@ -214,7 +214,7 @@ void __tsan_malloc(ThreadState *thr, uptr pc, uptr p, uptr sz) { } void __tsan_free(uptr p, uptr sz) { - ctx->metamap.FreeRange(get_cur_proc(), p, sz); + ctx->metamap.FreeRange(get_cur_proc(), p, sz, false); } void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) { diff --git a/compiler-rt/lib/tsan/rtl-old/tsan.syms.extra b/compiler-rt/lib/tsan/rtl-old/tsan.syms.extra new file mode 100644 index 000000000000..4838bb0a7279 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan.syms.extra @@ -0,0 +1,31 @@ +__tsan_init +__tsan_flush_memory +__tsan_read* +__tsan_write* +__tsan_vptr* +__tsan_func* +__tsan_atomic* +__tsan_java* +__tsan_unaligned* +__tsan_release +__tsan_acquire +__tsan_mutex_create +__tsan_mutex_destroy +__tsan_mutex_pre_lock +__tsan_mutex_post_lock +__tsan_mutex_pre_unlock +__tsan_mutex_post_unlock +__tsan_mutex_pre_signal +__tsan_mutex_post_signal +__tsan_mutex_pre_divert +__tsan_mutex_post_divert +__tsan_get_current_fiber +__tsan_create_fiber +__tsan_destroy_fiber +__tsan_switch_to_fiber +__tsan_set_fiber_name +__ubsan_* +Annotate* +WTFAnnotate* +RunningOnValgrind +ValgrindSlowdown diff --git a/compiler-rt/lib/tsan/rtl/tsan_clock.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_clock.cpp similarity index 100% rename from compiler-rt/lib/tsan/rtl/tsan_clock.cpp rename to compiler-rt/lib/tsan/rtl-old/tsan_clock.cpp diff --git a/compiler-rt/lib/tsan/rtl/tsan_clock.h b/compiler-rt/lib/tsan/rtl-old/tsan_clock.h similarity index 100% rename from compiler-rt/lib/tsan/rtl/tsan_clock.h rename to compiler-rt/lib/tsan/rtl-old/tsan_clock.h diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp new file mode 100644 index 000000000000..1d3c3849a446 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp @@ -0,0 +1,262 @@ +//===-- tsan_debugging.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// TSan debugging API implementation. +//===----------------------------------------------------------------------===// +#include "tsan_interface.h" +#include "tsan_report.h" +#include "tsan_rtl.h" + +#include "sanitizer_common/sanitizer_stackdepot.h" + +using namespace __tsan; + +static const char *ReportTypeDescription(ReportType typ) { + switch (typ) { + case ReportTypeRace: return "data-race"; + case ReportTypeVptrRace: return "data-race-vptr"; + case ReportTypeUseAfterFree: return "heap-use-after-free"; + case ReportTypeVptrUseAfterFree: return "heap-use-after-free-vptr"; + case ReportTypeExternalRace: return "external-race"; + case ReportTypeThreadLeak: return "thread-leak"; + case ReportTypeMutexDestroyLocked: return "locked-mutex-destroy"; + case ReportTypeMutexDoubleLock: return "mutex-double-lock"; + case ReportTypeMutexInvalidAccess: return "mutex-invalid-access"; + case ReportTypeMutexBadUnlock: return "mutex-bad-unlock"; + case ReportTypeMutexBadReadLock: return "mutex-bad-read-lock"; + case ReportTypeMutexBadReadUnlock: return "mutex-bad-read-unlock"; + case ReportTypeSignalUnsafe: return "signal-unsafe-call"; + case ReportTypeErrnoInSignal: return "errno-in-signal-handler"; + case ReportTypeDeadlock: return "lock-order-inversion"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); +} + +static const char *ReportLocationTypeDescription(ReportLocationType typ) { + switch (typ) { + case ReportLocationGlobal: return "global"; + case ReportLocationHeap: return "heap"; + case ReportLocationStack: return "stack"; + case ReportLocationTLS: return "tls"; + case ReportLocationFD: return "fd"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); +} + +static void CopyTrace(SymbolizedStack *first_frame, void **trace, + uptr trace_size) { + uptr i = 0; + for (SymbolizedStack *frame = first_frame; frame != nullptr; + frame = frame->next) { + trace[i++] = (void *)frame->info.address; + if (i >= trace_size) break; + } +} + +// Meant to be called by the debugger. +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_report() { + return const_cast(cur_thread()->current_report); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_data(void *report, const char **description, int *count, + int *stack_count, int *mop_count, int *loc_count, + int *mutex_count, int *thread_count, + int *unique_tid_count, void **sleep_trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + *description = ReportTypeDescription(rep->typ); + *count = rep->count; + *stack_count = rep->stacks.Size(); + *mop_count = rep->mops.Size(); + *loc_count = rep->locs.Size(); + *mutex_count = rep->mutexes.Size(); + *thread_count = rep->threads.Size(); + *unique_tid_count = rep->unique_tids.Size(); + if (rep->sleep) CopyTrace(rep->sleep->frames, sleep_trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_tag(void *report, uptr *tag) { + const ReportDesc *rep = (ReportDesc *)report; + *tag = rep->tag; + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_stack(void *report, uptr idx, void **trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->stacks.Size()); + ReportStack *stack = rep->stacks[idx]; + if (stack) CopyTrace(stack->frames, trace, trace_size); + return stack ? 1 : 0; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr, + int *size, int *write, int *atomic, void **trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->mops.Size()); + ReportMop *mop = rep->mops[idx]; + *tid = mop->tid; + *addr = (void *)mop->addr; + *size = mop->size; + *write = mop->write ? 1 : 0; + *atomic = mop->atomic ? 1 : 0; + if (mop->stack) CopyTrace(mop->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_loc(void *report, uptr idx, const char **type, + void **addr, uptr *start, uptr *size, int *tid, + int *fd, int *suppressable, void **trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->locs.Size()); + ReportLocation *loc = rep->locs[idx]; + *type = ReportLocationTypeDescription(loc->type); + *addr = (void *)loc->global.start; + *start = loc->heap_chunk_start; + *size = loc->heap_chunk_size; + *tid = loc->tid; + *fd = loc->fd; + *suppressable = loc->suppressable; + if (loc->stack) CopyTrace(loc->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_loc_object_type(void *report, uptr idx, + const char **object_type) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->locs.Size()); + ReportLocation *loc = rep->locs[idx]; + *object_type = GetObjectTypeFromTag(loc->external_tag); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr, + int *destroyed, void **trace, uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->mutexes.Size()); + ReportMutex *mutex = rep->mutexes[idx]; + *mutex_id = mutex->id; + *addr = (void *)mutex->addr; + *destroyed = mutex->destroyed; + if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id, + int *running, const char **name, int *parent_tid, + void **trace, uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->threads.Size()); + ReportThread *thread = rep->threads[idx]; + *tid = thread->id; + *os_id = thread->os_id; + *running = thread->running; + *name = thread->name; + *parent_tid = thread->parent_tid; + if (thread->stack) CopyTrace(thread->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->unique_tids.Size()); + *tid = rep->unique_tids[idx]; + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +const char *__tsan_locate_address(uptr addr, char *name, uptr name_size, + uptr *region_address_ptr, + uptr *region_size_ptr) { + uptr region_address = 0; + uptr region_size = 0; + const char *region_kind = nullptr; + if (name && name_size > 0) name[0] = 0; + + if (IsMetaMem(reinterpret_cast(addr))) { + region_kind = "meta shadow"; + } else if (IsShadowMem(reinterpret_cast(addr))) { + region_kind = "shadow"; + } else { + bool is_stack = false; + MBlock *b = 0; + Allocator *a = allocator(); + if (a->PointerIsMine((void *)addr)) { + void *block_begin = a->GetBlockBegin((void *)addr); + if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin); + } + + if (b != 0) { + region_address = (uptr)allocator()->GetBlockBegin((void *)addr); + region_size = b->siz; + region_kind = "heap"; + } else { + // TODO(kuba.brecka): We should not lock. This is supposed to be called + // from within the debugger when other threads are stopped. + ctx->thread_registry.Lock(); + ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack); + ctx->thread_registry.Unlock(); + if (tctx) { + region_kind = is_stack ? "stack" : "tls"; + } else { + region_kind = "global"; + DataInfo info; + if (Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) { + internal_strncpy(name, info.name, name_size); + region_address = info.start; + region_size = info.size; + } + } + } + } + + CHECK(region_kind); + if (region_address_ptr) *region_address_ptr = region_address; + if (region_size_ptr) *region_size_ptr = region_size; + return region_kind; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id, + tid_t *os_id) { + MBlock *b = 0; + Allocator *a = allocator(); + if (a->PointerIsMine((void *)addr)) { + void *block_begin = a->GetBlockBegin((void *)addr); + if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin); + } + if (b == 0) return 0; + + *thread_id = b->tid; + // No locking. This is supposed to be called from within the debugger when + // other threads are stopped. + ThreadContextBase *tctx = ctx->thread_registry.GetThreadLocked(b->tid); + *os_id = tctx->os_id; + + StackTrace stack = StackDepotGet(b->stk); + size = Min(size, (uptr)stack.size); + for (uptr i = 0; i < size; i++) trace[i] = stack.trace[stack.size - i - 1]; + return size; +} diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_defs.h b/compiler-rt/lib/tsan/rtl-old/tsan_defs.h new file mode 100644 index 000000000000..4712c2be1813 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_defs.h @@ -0,0 +1,236 @@ +//===-- tsan_defs.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#ifndef TSAN_DEFS_H +#define TSAN_DEFS_H + +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_mutex.h" +#include "ubsan/ubsan_platform.h" + +#ifndef TSAN_VECTORIZE +# define TSAN_VECTORIZE __SSE4_2__ +#endif + +#if TSAN_VECTORIZE +// transitively includes , +// and it's prohibited to include std headers into tsan runtime. +// So we do this dirty trick. +# define _MM_MALLOC_H_INCLUDED +# define __MM_MALLOC_H +# include +# include +# define VECTOR_ALIGNED ALIGNED(16) +typedef __m128i m128; +#else +# define VECTOR_ALIGNED +#endif + +// Setup defaults for compile definitions. +#ifndef TSAN_NO_HISTORY +# define TSAN_NO_HISTORY 0 +#endif + +#ifndef TSAN_CONTAINS_UBSAN +# if CAN_SANITIZE_UB && !SANITIZER_GO +# define TSAN_CONTAINS_UBSAN 1 +# else +# define TSAN_CONTAINS_UBSAN 0 +# endif +#endif + +namespace __tsan { + +constexpr uptr kByteBits = 8; + +// Thread slot ID. +enum class Sid : u8 {}; +constexpr uptr kThreadSlotCount = 256; +constexpr Sid kFreeSid = static_cast(255); + +// Abstract time unit, vector clock element. +enum class Epoch : u16 {}; +constexpr uptr kEpochBits = 14; +constexpr Epoch kEpochZero = static_cast(0); +constexpr Epoch kEpochOver = static_cast(1 << kEpochBits); + +const int kClkBits = 42; +const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1; + +struct ClockElem { + u64 epoch : kClkBits; + u64 reused : 64 - kClkBits; // tid reuse count +}; + +struct ClockBlock { + static const uptr kSize = 512; + static const uptr kTableSize = kSize / sizeof(u32); + static const uptr kClockCount = kSize / sizeof(ClockElem); + static const uptr kRefIdx = kTableSize - 1; + static const uptr kBlockIdx = kTableSize - 2; + + union { + u32 table[kTableSize]; + ClockElem clock[kClockCount]; + }; + + ClockBlock() { + } +}; + +const int kTidBits = 13; +// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is +// occupied by reference counter, so total number of elements we can store +// in SyncClock is kClockCount * (kTableSize - 1). +const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount; +#if !SANITIZER_GO +const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit. +#else +const unsigned kMaxTidInClock = kMaxTid; // Go does not track freed memory. +#endif +const uptr kShadowStackSize = 64 * 1024; + +// Count of shadow values in a shadow cell. +const uptr kShadowCnt = 4; + +// That many user bytes are mapped onto a single shadow cell. +const uptr kShadowCell = 8; + +// Single shadow value. +typedef u64 RawShadow; +const uptr kShadowSize = sizeof(RawShadow); + +// Shadow memory is kShadowMultiplier times larger than user memory. +const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell; + +// That many user bytes are mapped onto a single meta shadow cell. +// Must be less or equal to minimal memory allocator alignment. +const uptr kMetaShadowCell = 8; + +// Size of a single meta shadow value (u32). +const uptr kMetaShadowSize = 4; + +// All addresses and PCs are assumed to be compressable to that many bits. +const uptr kCompressedAddrBits = 44; + +#if TSAN_NO_HISTORY +const bool kCollectHistory = false; +#else +const bool kCollectHistory = true; +#endif + +// The following "build consistency" machinery ensures that all source files +// are built in the same configuration. Inconsistent builds lead to +// hard to debug crashes. +#if SANITIZER_DEBUG +void build_consistency_debug(); +#else +void build_consistency_release(); +#endif + +static inline void USED build_consistency() { +#if SANITIZER_DEBUG + build_consistency_debug(); +#else + build_consistency_release(); +#endif +} + +template +T min(T a, T b) { + return a < b ? a : b; +} + +template +T max(T a, T b) { + return a > b ? a : b; +} + +template +T RoundUp(T p, u64 align) { + DCHECK_EQ(align & (align - 1), 0); + return (T)(((u64)p + align - 1) & ~(align - 1)); +} + +template +T RoundDown(T p, u64 align) { + DCHECK_EQ(align & (align - 1), 0); + return (T)((u64)p & ~(align - 1)); +} + +// Zeroizes high part, returns 'bits' lsb bits. +template +T GetLsb(T v, int bits) { + return (T)((u64)v & ((1ull << bits) - 1)); +} + +struct MD5Hash { + u64 hash[2]; + bool operator==(const MD5Hash &other) const; +}; + +MD5Hash md5_hash(const void *data, uptr size); + +struct Processor; +struct ThreadState; +class ThreadContext; +struct Context; +struct ReportStack; +class ReportDesc; +class RegionAlloc; + +typedef uptr AccessType; + +enum : AccessType { + kAccessWrite = 0, + kAccessRead = 1 << 0, + kAccessAtomic = 1 << 1, + kAccessVptr = 1 << 2, // read or write of an object virtual table pointer + kAccessFree = 1 << 3, // synthetic memory access during memory freeing + kAccessExternalPC = 1 << 4, // access PC can have kExternalPCBit set +}; + +// Descriptor of user's memory block. +struct MBlock { + u64 siz : 48; + u64 tag : 16; + StackID stk; + Tid tid; +}; + +COMPILER_CHECK(sizeof(MBlock) == 16); + +enum ExternalTag : uptr { + kExternalTagNone = 0, + kExternalTagSwiftModifyingAccess = 1, + kExternalTagFirstUserAvailable = 2, + kExternalTagMax = 1024, + // Don't set kExternalTagMax over 65,536, since MBlock only stores tags + // as 16-bit values, see tsan_defs.h. +}; + +enum MutexType { + MutexTypeTrace = MutexLastCommon, + MutexTypeReport, + MutexTypeSyncVar, + MutexTypeAnnotations, + MutexTypeAtExit, + MutexTypeFired, + MutexTypeRacy, + MutexTypeGlobalProc, + MutexTypeInternalAlloc, +}; + +} // namespace __tsan + +#endif // TSAN_DEFS_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h b/compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h new file mode 100644 index 000000000000..9e15f74a0615 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h @@ -0,0 +1,156 @@ +//===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// A DenseSlabAlloc is a freelist-based allocator of fixed-size objects. +// DenseSlabAllocCache is a thread-local cache for DenseSlabAlloc. +// The only difference with traditional slab allocators is that DenseSlabAlloc +// allocates/free indices of objects and provide a functionality to map +// the index onto the real pointer. The index is u32, that is, 2 times smaller +// than uptr (hense the Dense prefix). +//===----------------------------------------------------------------------===// +#ifndef TSAN_DENSE_ALLOC_H +#define TSAN_DENSE_ALLOC_H + +#include "sanitizer_common/sanitizer_common.h" +#include "tsan_defs.h" + +namespace __tsan { + +class DenseSlabAllocCache { + static const uptr kSize = 128; + typedef u32 IndexT; + uptr pos; + IndexT cache[kSize]; + template + friend class DenseSlabAlloc; +}; + +template +class DenseSlabAlloc { + public: + typedef DenseSlabAllocCache Cache; + typedef typename Cache::IndexT IndexT; + + static_assert((kL1Size & (kL1Size - 1)) == 0, + "kL1Size must be a power-of-two"); + static_assert((kL2Size & (kL2Size - 1)) == 0, + "kL2Size must be a power-of-two"); + static_assert((kL1Size * kL2Size) <= (1ull << (sizeof(IndexT) * 8)), + "kL1Size/kL2Size are too large"); + static_assert(((kL1Size * kL2Size - 1) & kReserved) == 0, + "reserved bits don't fit"); + static_assert(sizeof(T) > sizeof(IndexT), + "it doesn't make sense to use dense alloc"); + + DenseSlabAlloc(LinkerInitialized, const char *name) : name_(name) {} + + explicit DenseSlabAlloc(const char *name) + : DenseSlabAlloc(LINKER_INITIALIZED, name) { + // It can be very large. + // Don't page it in for linker initialized objects. + internal_memset(map_, 0, sizeof(map_)); + } + + ~DenseSlabAlloc() { + for (uptr i = 0; i < kL1Size; i++) { + if (map_[i] != 0) + UnmapOrDie(map_[i], kL2Size * sizeof(T)); + } + } + + IndexT Alloc(Cache *c) { + if (c->pos == 0) + Refill(c); + return c->cache[--c->pos]; + } + + void Free(Cache *c, IndexT idx) { + DCHECK_NE(idx, 0); + if (c->pos == Cache::kSize) + Drain(c); + c->cache[c->pos++] = idx; + } + + T *Map(IndexT idx) { + DCHECK_NE(idx, 0); + DCHECK_LE(idx, kL1Size * kL2Size); + return &map_[idx / kL2Size][idx % kL2Size]; + } + + void FlushCache(Cache *c) { + if (!c->pos) + return; + SpinMutexLock lock(&mtx_); + while (c->pos) { + IndexT idx = c->cache[--c->pos]; + *(IndexT*)Map(idx) = freelist_; + freelist_ = idx; + } + } + + void InitCache(Cache *c) { + c->pos = 0; + internal_memset(c->cache, 0, sizeof(c->cache)); + } + + uptr AllocatedMemory() const { + return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T); + } + + private: + T *map_[kL1Size]; + SpinMutex mtx_; + IndexT freelist_ = {0}; + atomic_uintptr_t fillpos_ = {0}; + const char *const name_; + + void Refill(Cache *c) { + SpinMutexLock lock(&mtx_); + if (freelist_ == 0) { + uptr fillpos = atomic_load_relaxed(&fillpos_); + if (fillpos == kL1Size) { + Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", + name_, kL1Size, kL2Size); + Die(); + } + VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_, + fillpos, kL1Size, kL2Size); + T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_); + // Reserve 0 as invalid index. + IndexT start = fillpos == 0 ? 1 : 0; + for (IndexT i = start; i < kL2Size; i++) { + new(batch + i) T; + *(IndexT *)(batch + i) = i + 1 + fillpos * kL2Size; + } + *(IndexT*)(batch + kL2Size - 1) = 0; + freelist_ = fillpos * kL2Size + start; + map_[fillpos] = batch; + atomic_store_relaxed(&fillpos_, fillpos + 1); + } + for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) { + IndexT idx = freelist_; + c->cache[c->pos++] = idx; + freelist_ = *(IndexT*)Map(idx); + } + } + + void Drain(Cache *c) { + SpinMutexLock lock(&mtx_); + for (uptr i = 0; i < Cache::kSize / 2; i++) { + IndexT idx = c->cache[--c->pos]; + *(IndexT*)Map(idx) = freelist_; + freelist_ = idx; + } + } +}; + +} // namespace __tsan + +#endif // TSAN_DENSE_ALLOC_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h b/compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h new file mode 100644 index 000000000000..94e0b50fed36 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h @@ -0,0 +1,73 @@ +//===-- tsan_dispatch_defs.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_DISPATCH_DEFS_H +#define TSAN_DISPATCH_DEFS_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +typedef struct dispatch_object_s {} *dispatch_object_t; + +#define DISPATCH_DECL(name) \ + typedef struct name##_s : public dispatch_object_s {} *name##_t + +DISPATCH_DECL(dispatch_queue); +DISPATCH_DECL(dispatch_source); +DISPATCH_DECL(dispatch_group); +DISPATCH_DECL(dispatch_data); +DISPATCH_DECL(dispatch_semaphore); +DISPATCH_DECL(dispatch_io); + +typedef void (*dispatch_function_t)(void *arg); +typedef void (^dispatch_block_t)(void); +typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data, + int error); + +typedef long dispatch_once_t; +typedef __sanitizer::u64 dispatch_time_t; +typedef int dispatch_fd_t; +typedef unsigned long dispatch_io_type_t; +typedef unsigned long dispatch_io_close_flags_t; + +extern "C" { +void *dispatch_get_context(dispatch_object_t object); +void dispatch_retain(dispatch_object_t object); +void dispatch_release(dispatch_object_t object); + +extern const dispatch_block_t _dispatch_data_destructor_free; +extern const dispatch_block_t _dispatch_data_destructor_munmap; +} // extern "C" + +#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr +#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free +#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap + +#if __has_attribute(noescape) +# define DISPATCH_NOESCAPE __attribute__((__noescape__)) +#else +# define DISPATCH_NOESCAPE +#endif + +#if SANITIZER_MAC +# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import)) +#else +# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak)) +#endif + + +// Data types used in dispatch APIs +typedef unsigned long size_t; +typedef unsigned long uintptr_t; +typedef __sanitizer::s64 off_t; +typedef __sanitizer::u16 mode_t; +typedef long long_t; + +#endif // TSAN_DISPATCH_DEFS_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_external.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_external.cpp new file mode 100644 index 000000000000..19ae174f20a5 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_external.cpp @@ -0,0 +1,126 @@ +//===-- tsan_external.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_ptrauth.h" + +#if !SANITIZER_GO +# include "tsan_interceptors.h" +#endif + +namespace __tsan { + +#define CALLERPC ((uptr)__builtin_return_address(0)) + +struct TagData { + const char *object_type; + const char *header; +}; + +static TagData registered_tags[kExternalTagMax] = { + {}, + {"Swift variable", "Swift access race"}, +}; +static atomic_uint32_t used_tags{kExternalTagFirstUserAvailable}; +static TagData *GetTagData(uptr tag) { + // Invalid/corrupted tag? Better return NULL and let the caller deal with it. + if (tag >= atomic_load(&used_tags, memory_order_relaxed)) return nullptr; + return ®istered_tags[tag]; +} + +const char *GetObjectTypeFromTag(uptr tag) { + TagData *tag_data = GetTagData(tag); + return tag_data ? tag_data->object_type : nullptr; +} + +const char *GetReportHeaderFromTag(uptr tag) { + TagData *tag_data = GetTagData(tag); + return tag_data ? tag_data->header : nullptr; +} + +void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) { + FuncEntry(thr, (uptr)®istered_tags[tag]); +} + +uptr TagFromShadowStackFrame(uptr pc) { + uptr tag_count = atomic_load(&used_tags, memory_order_relaxed); + void *pc_ptr = (void *)pc; + if (pc_ptr < GetTagData(0) || pc_ptr > GetTagData(tag_count - 1)) + return 0; + return (TagData *)pc_ptr - GetTagData(0); +} + +#if !SANITIZER_GO + +void ExternalAccess(void *addr, uptr caller_pc, void *tag, AccessType typ) { + CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed)); + ThreadState *thr = cur_thread(); + if (caller_pc) FuncEntry(thr, caller_pc); + InsertShadowStackFrameForTag(thr, (uptr)tag); + bool in_ignored_lib; + if (!caller_pc || !libignore()->IsIgnored(caller_pc, &in_ignored_lib)) + MemoryAccess(thr, CALLERPC, (uptr)addr, 1, typ); + FuncExit(thr); + if (caller_pc) FuncExit(thr); +} + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_external_register_tag(const char *object_type) { + uptr new_tag = atomic_fetch_add(&used_tags, 1, memory_order_relaxed); + CHECK_LT(new_tag, kExternalTagMax); + GetTagData(new_tag)->object_type = internal_strdup(object_type); + char header[127] = {0}; + internal_snprintf(header, sizeof(header), "race on %s", object_type); + GetTagData(new_tag)->header = internal_strdup(header); + return (void *)new_tag; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_register_header(void *tag, const char *header) { + CHECK_GE((uptr)tag, kExternalTagFirstUserAvailable); + CHECK_LT((uptr)tag, kExternalTagMax); + atomic_uintptr_t *header_ptr = + (atomic_uintptr_t *)&GetTagData((uptr)tag)->header; + header = internal_strdup(header); + char *old_header = + (char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst); + Free(old_header); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_assign_tag(void *addr, void *tag) { + CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed)); + Allocator *a = allocator(); + MBlock *b = nullptr; + if (a->PointerIsMine((void *)addr)) { + void *block_begin = a->GetBlockBegin((void *)addr); + if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin); + } + if (b) { + b->tag = (uptr)tag; + } +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_read(void *addr, void *caller_pc, void *tag) { + ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, kAccessRead); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_write(void *addr, void *caller_pc, void *tag) { + ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, kAccessWrite); +} +} // extern "C" + +#endif // !SANITIZER_GO + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp new file mode 100644 index 000000000000..255ffa8daf76 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp @@ -0,0 +1,316 @@ +//===-- tsan_fd.cpp -------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "tsan_fd.h" +#include "tsan_rtl.h" +#include + +namespace __tsan { + +const int kTableSizeL1 = 1024; +const int kTableSizeL2 = 1024; +const int kTableSize = kTableSizeL1 * kTableSizeL2; + +struct FdSync { + atomic_uint64_t rc; +}; + +struct FdDesc { + FdSync *sync; + Tid creation_tid; + StackID creation_stack; +}; + +struct FdContext { + atomic_uintptr_t tab[kTableSizeL1]; + // Addresses used for synchronization. + FdSync globsync; + FdSync filesync; + FdSync socksync; + u64 connectsync; +}; + +static FdContext fdctx; + +static bool bogusfd(int fd) { + // Apparently a bogus fd value. + return fd < 0 || fd >= kTableSize; +} + +static FdSync *allocsync(ThreadState *thr, uptr pc) { + FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync), + kDefaultAlignment, false); + atomic_store(&s->rc, 1, memory_order_relaxed); + return s; +} + +static FdSync *ref(FdSync *s) { + if (s && atomic_load(&s->rc, memory_order_relaxed) != (u64)-1) + atomic_fetch_add(&s->rc, 1, memory_order_relaxed); + return s; +} + +static void unref(ThreadState *thr, uptr pc, FdSync *s) { + if (s && atomic_load(&s->rc, memory_order_relaxed) != (u64)-1) { + if (atomic_fetch_sub(&s->rc, 1, memory_order_acq_rel) == 1) { + CHECK_NE(s, &fdctx.globsync); + CHECK_NE(s, &fdctx.filesync); + CHECK_NE(s, &fdctx.socksync); + user_free(thr, pc, s, false); + } + } +} + +static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) { + CHECK_GE(fd, 0); + CHECK_LT(fd, kTableSize); + atomic_uintptr_t *pl1 = &fdctx.tab[fd / kTableSizeL2]; + uptr l1 = atomic_load(pl1, memory_order_consume); + if (l1 == 0) { + uptr size = kTableSizeL2 * sizeof(FdDesc); + // We need this to reside in user memory to properly catch races on it. + void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false); + internal_memset(p, 0, size); + MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size); + if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel)) + l1 = (uptr)p; + else + user_free(thr, pc, p, false); + } + FdDesc *fds = reinterpret_cast(l1); + return &fds[fd % kTableSizeL2]; +} + +// pd must be already ref'ed. +static void init(ThreadState *thr, uptr pc, int fd, FdSync *s, + bool write = true) { + FdDesc *d = fddesc(thr, pc, fd); + // As a matter of fact, we don't intercept all close calls. + // See e.g. libc __res_iclose(). + if (d->sync) { + unref(thr, pc, d->sync); + d->sync = 0; + } + if (flags()->io_sync == 0) { + unref(thr, pc, s); + } else if (flags()->io_sync == 1) { + d->sync = s; + } else if (flags()->io_sync == 2) { + unref(thr, pc, s); + d->sync = &fdctx.globsync; + } + d->creation_tid = thr->tid; + d->creation_stack = CurrentStackId(thr, pc); + if (write) { + // To catch races between fd usage and open. + MemoryRangeImitateWrite(thr, pc, (uptr)d, 8); + } else { + // See the dup-related comment in FdClose. + MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); + } +} + +void FdInit() { + atomic_store(&fdctx.globsync.rc, (u64)-1, memory_order_relaxed); + atomic_store(&fdctx.filesync.rc, (u64)-1, memory_order_relaxed); + atomic_store(&fdctx.socksync.rc, (u64)-1, memory_order_relaxed); +} + +void FdOnFork(ThreadState *thr, uptr pc) { + // On fork() we need to reset all fd's, because the child is going + // close all them, and that will cause races between previous read/write + // and the close. + for (int l1 = 0; l1 < kTableSizeL1; l1++) { + FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed); + if (tab == 0) + break; + for (int l2 = 0; l2 < kTableSizeL2; l2++) { + FdDesc *d = &tab[l2]; + MemoryResetRange(thr, pc, (uptr)d, 8); + } + } +} + +bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack) { + for (int l1 = 0; l1 < kTableSizeL1; l1++) { + FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed); + if (tab == 0) + break; + if (addr >= (uptr)tab && addr < (uptr)(tab + kTableSizeL2)) { + int l2 = (addr - (uptr)tab) / sizeof(FdDesc); + FdDesc *d = &tab[l2]; + *fd = l1 * kTableSizeL1 + l2; + *tid = d->creation_tid; + *stack = d->creation_stack; + return true; + } + } + return false; +} + +void FdAcquire(ThreadState *thr, uptr pc, int fd) { + if (bogusfd(fd)) + return; + FdDesc *d = fddesc(thr, pc, fd); + FdSync *s = d->sync; + DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s); + MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); + if (s) + Acquire(thr, pc, (uptr)s); +} + +void FdRelease(ThreadState *thr, uptr pc, int fd) { + if (bogusfd(fd)) + return; + FdDesc *d = fddesc(thr, pc, fd); + FdSync *s = d->sync; + DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s); + MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); + if (s) + Release(thr, pc, (uptr)s); +} + +void FdAccess(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdAccess(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + FdDesc *d = fddesc(thr, pc, fd); + MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); +} + +void FdClose(ThreadState *thr, uptr pc, int fd, bool write) { + DPrintf("#%d: FdClose(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + FdDesc *d = fddesc(thr, pc, fd); + if (write) { + // To catch races between fd usage and close. + MemoryAccess(thr, pc, (uptr)d, 8, kAccessWrite); + } else { + // This path is used only by dup2/dup3 calls. + // We do read instead of write because there is a number of legitimate + // cases where write would lead to false positives: + // 1. Some software dups a closed pipe in place of a socket before closing + // the socket (to prevent races actually). + // 2. Some daemons dup /dev/null in place of stdin/stdout. + // On the other hand we have not seen cases when write here catches real + // bugs. + MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); + } + // We need to clear it, because if we do not intercept any call out there + // that creates fd, we will hit false postives. + MemoryResetRange(thr, pc, (uptr)d, 8); + unref(thr, pc, d->sync); + d->sync = 0; + d->creation_tid = kInvalidTid; + d->creation_stack = kInvalidStackID; +} + +void FdFileCreate(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdFileCreate(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + init(thr, pc, fd, &fdctx.filesync); +} + +void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write) { + DPrintf("#%d: FdDup(%d, %d)\n", thr->tid, oldfd, newfd); + if (bogusfd(oldfd) || bogusfd(newfd)) + return; + // Ignore the case when user dups not yet connected socket. + FdDesc *od = fddesc(thr, pc, oldfd); + MemoryAccess(thr, pc, (uptr)od, 8, kAccessRead); + FdClose(thr, pc, newfd, write); + init(thr, pc, newfd, ref(od->sync), write); +} + +void FdPipeCreate(ThreadState *thr, uptr pc, int rfd, int wfd) { + DPrintf("#%d: FdCreatePipe(%d, %d)\n", thr->tid, rfd, wfd); + FdSync *s = allocsync(thr, pc); + init(thr, pc, rfd, ref(s)); + init(thr, pc, wfd, ref(s)); + unref(thr, pc, s); +} + +void FdEventCreate(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdEventCreate(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + init(thr, pc, fd, allocsync(thr, pc)); +} + +void FdSignalCreate(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdSignalCreate(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + init(thr, pc, fd, 0); +} + +void FdInotifyCreate(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdInotifyCreate(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + init(thr, pc, fd, 0); +} + +void FdPollCreate(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdPollCreate(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + init(thr, pc, fd, allocsync(thr, pc)); +} + +void FdSocketCreate(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + // It can be a UDP socket. + init(thr, pc, fd, &fdctx.socksync); +} + +void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd) { + DPrintf("#%d: FdSocketAccept(%d, %d)\n", thr->tid, fd, newfd); + if (bogusfd(fd)) + return; + // Synchronize connect->accept. + Acquire(thr, pc, (uptr)&fdctx.connectsync); + init(thr, pc, newfd, &fdctx.socksync); +} + +void FdSocketConnecting(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdSocketConnecting(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + // Synchronize connect->accept. + Release(thr, pc, (uptr)&fdctx.connectsync); +} + +void FdSocketConnect(ThreadState *thr, uptr pc, int fd) { + DPrintf("#%d: FdSocketConnect(%d)\n", thr->tid, fd); + if (bogusfd(fd)) + return; + init(thr, pc, fd, &fdctx.socksync); +} + +uptr File2addr(const char *path) { + (void)path; + static u64 addr; + return (uptr)&addr; +} + +uptr Dir2addr(const char *path) { + (void)path; + static u64 addr; + return (uptr)&addr; +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_fd.h b/compiler-rt/lib/tsan/rtl-old/tsan_fd.h new file mode 100644 index 000000000000..d9648178481c --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_fd.h @@ -0,0 +1,64 @@ +//===-- tsan_fd.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// This file handles synchronization via IO. +// People use IO for synchronization along the lines of: +// +// int X; +// int client_socket; // initialized elsewhere +// int server_socket; // initialized elsewhere +// +// Thread 1: +// X = 42; +// send(client_socket, ...); +// +// Thread 2: +// if (recv(server_socket, ...) > 0) +// assert(X == 42); +// +// This file determines the scope of the file descriptor (pipe, socket, +// all local files, etc) and executes acquire and release operations on +// the scope as necessary. Some scopes are very fine grained (e.g. pipe +// operations synchronize only with operations on the same pipe), while +// others are corse-grained (e.g. all operations on local files synchronize +// with each other). +//===----------------------------------------------------------------------===// +#ifndef TSAN_FD_H +#define TSAN_FD_H + +#include "tsan_rtl.h" + +namespace __tsan { + +void FdInit(); +void FdAcquire(ThreadState *thr, uptr pc, int fd); +void FdRelease(ThreadState *thr, uptr pc, int fd); +void FdAccess(ThreadState *thr, uptr pc, int fd); +void FdClose(ThreadState *thr, uptr pc, int fd, bool write = true); +void FdFileCreate(ThreadState *thr, uptr pc, int fd); +void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write); +void FdPipeCreate(ThreadState *thr, uptr pc, int rfd, int wfd); +void FdEventCreate(ThreadState *thr, uptr pc, int fd); +void FdSignalCreate(ThreadState *thr, uptr pc, int fd); +void FdInotifyCreate(ThreadState *thr, uptr pc, int fd); +void FdPollCreate(ThreadState *thr, uptr pc, int fd); +void FdSocketCreate(ThreadState *thr, uptr pc, int fd); +void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd); +void FdSocketConnecting(ThreadState *thr, uptr pc, int fd); +void FdSocketConnect(ThreadState *thr, uptr pc, int fd); +bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack); +void FdOnFork(ThreadState *thr, uptr pc); + +uptr File2addr(const char *path); +uptr Dir2addr(const char *path); + +} // namespace __tsan + +#endif // TSAN_INTERFACE_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp new file mode 100644 index 000000000000..ee89862d17bd --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp @@ -0,0 +1,126 @@ +//===-- tsan_flags.cpp ----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "tsan_flags.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" +#include "ubsan/ubsan_flags.h" + +namespace __tsan { + +// Can be overriden in frontend. +#ifdef TSAN_EXTERNAL_HOOKS +extern "C" const char* __tsan_default_options(); +#else +SANITIZER_WEAK_DEFAULT_IMPL +const char *__tsan_default_options() { + return ""; +} +#endif + +void Flags::SetDefaults() { +#define TSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "tsan_flags.inc" +#undef TSAN_FLAG + // DDFlags + second_deadlock_stack = false; +} + +void RegisterTsanFlags(FlagParser *parser, Flags *f) { +#define TSAN_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "tsan_flags.inc" +#undef TSAN_FLAG + // DDFlags + RegisterFlag(parser, "second_deadlock_stack", + "Report where each mutex is locked in deadlock reports", + &f->second_deadlock_stack); +} + +void InitializeFlags(Flags *f, const char *env, const char *env_option_name) { + SetCommonFlagsDefaults(); + { + // Override some common flags defaults. + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("TSAN_SYMBOLIZER_PATH"); + cf.allow_addr2line = true; + if (SANITIZER_GO) { + // Does not work as expected for Go: runtime handles SIGABRT and crashes. + cf.abort_on_error = false; + // Go does not have mutexes. + cf.detect_deadlocks = false; + } + cf.print_suppressions = false; + cf.stack_trace_format = " #%n %f %S %M"; + cf.exitcode = 66; + cf.intercept_tls_get_addr = true; + OverrideCommonFlags(cf); + } + + f->SetDefaults(); + + FlagParser parser; + RegisterTsanFlags(&parser, f); + RegisterCommonFlags(&parser); + +#if TSAN_CONTAINS_UBSAN + __ubsan::Flags *uf = __ubsan::flags(); + uf->SetDefaults(); + + FlagParser ubsan_parser; + __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); + RegisterCommonFlags(&ubsan_parser); +#endif + + // Let a frontend override. + parser.ParseString(__tsan_default_options()); +#if TSAN_CONTAINS_UBSAN + const char *ubsan_default_options = __ubsan_default_options(); + ubsan_parser.ParseString(ubsan_default_options); +#endif + // Override from command line. + parser.ParseString(env, env_option_name); +#if TSAN_CONTAINS_UBSAN + ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); +#endif + + // Sanity check. + if (!f->report_bugs) { + f->report_thread_leaks = false; + f->report_destroy_locked = false; + f->report_signal_unsafe = false; + } + + InitializeCommonFlags(); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); + + if (f->history_size < 0 || f->history_size > 7) { + Printf("ThreadSanitizer: incorrect value for history_size" + " (must be [0..7])\n"); + Die(); + } + + if (f->io_sync < 0 || f->io_sync > 2) { + Printf("ThreadSanitizer: incorrect value for io_sync" + " (must be [0..2])\n"); + Die(); + } +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_flags.h b/compiler-rt/lib/tsan/rtl-old/tsan_flags.h new file mode 100644 index 000000000000..da27d5b992bc --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_flags.h @@ -0,0 +1,34 @@ +//===-- tsan_flags.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// NOTE: This file may be included into user code. +//===----------------------------------------------------------------------===// + +#ifndef TSAN_FLAGS_H +#define TSAN_FLAGS_H + +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" + +namespace __tsan { + +struct Flags : DDFlags { +#define TSAN_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "tsan_flags.inc" +#undef TSAN_FLAG + + void SetDefaults(); + void ParseFromString(const char *str); +}; + +void InitializeFlags(Flags *flags, const char *env, + const char *env_option_name = nullptr); +} // namespace __tsan + +#endif // TSAN_FLAGS_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_flags.inc b/compiler-rt/lib/tsan/rtl-old/tsan_flags.inc new file mode 100644 index 000000000000..7954a4307fa1 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_flags.inc @@ -0,0 +1,84 @@ +//===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TSan runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_FLAG +# error "Define TSAN_FLAG prior to including this file!" +#endif + +// TSAN_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +TSAN_FLAG(bool, enable_annotations, true, + "Enable dynamic annotations, otherwise they are no-ops.") +// Suppress a race report if we've already output another race report +// with the same stack. +TSAN_FLAG(bool, suppress_equal_stacks, true, + "Suppress a race report if we've already output another race report " + "with the same stack.") +TSAN_FLAG(bool, suppress_equal_addresses, true, + "Suppress a race report if we've already output another race report " + "on the same address.") + +TSAN_FLAG(bool, report_bugs, true, + "Turns off bug reporting entirely (useful for benchmarking).") +TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?") +TSAN_FLAG(bool, report_destroy_locked, true, + "Report destruction of a locked mutex?") +TSAN_FLAG(bool, report_mutex_bugs, true, + "Report incorrect usages of mutexes and mutex annotations?") +TSAN_FLAG(bool, report_signal_unsafe, true, + "Report violations of async signal-safety " + "(e.g. malloc() call from a signal handler).") +TSAN_FLAG(bool, report_atomic_races, true, + "Report races between atomic and plain memory accesses.") +TSAN_FLAG( + bool, force_seq_cst_atomics, false, + "If set, all atomics are effectively sequentially consistent (seq_cst), " + "regardless of what user actually specified.") +TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.") +TSAN_FLAG(int, atexit_sleep_ms, 1000, + "Sleep in main thread before exiting for that many ms " + "(useful to catch \"at exit\" races).") +TSAN_FLAG(const char *, profile_memory, "", + "If set, periodically write memory profile to that file.") +TSAN_FLAG(int, flush_memory_ms, 0, "Flush shadow memory every X ms.") +TSAN_FLAG(int, flush_symbolizer_ms, 5000, "Flush symbolizer caches every X ms.") +TSAN_FLAG( + int, memory_limit_mb, 0, + "Resident memory limit in MB to aim at." + "If the process consumes more memory, then TSan will flush shadow memory.") +TSAN_FLAG(bool, stop_on_start, false, + "Stops on start until __tsan_resume() is called (for debugging).") +TSAN_FLAG(bool, running_on_valgrind, false, + "Controls whether RunningOnValgrind() returns true or false.") +// There are a lot of goroutines in Go, so we use smaller history. +TSAN_FLAG( + int, history_size, SANITIZER_GO ? 1 : 3, + "Per-thread history size, controls how many previous memory accesses " + "are remembered per thread. Possible values are [0..7]. " + "history_size=0 amounts to 32K memory accesses. Each next value doubles " + "the amount of memory accesses, up to history_size=7 that amounts to " + "4M memory accesses. The default value is 2 (128K memory accesses).") +TSAN_FLAG(int, io_sync, 1, + "Controls level of synchronization implied by IO operations. " + "0 - no synchronization " + "1 - reasonable level of synchronization (write->read)" + "2 - global synchronization of all IO operations.") +TSAN_FLAG(bool, die_after_fork, true, + "Die after multi-threaded fork if the child creates new threads.") +TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") +TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false, + "Ignore reads and writes from all interceptors.") +TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false, + "Interceptors should only detect races when called from instrumented " + "modules.") +TSAN_FLAG(bool, shared_ptr_interceptor, true, + "Track atomic reference counting in libc++ shared_ptr and weak_ptr.") diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp new file mode 100644 index 000000000000..1fca1cf4f9fc --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp @@ -0,0 +1,38 @@ +//===-- tsan_ignoreset.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_ignoreset.h" + +namespace __tsan { + +const uptr IgnoreSet::kMaxSize; + +IgnoreSet::IgnoreSet() + : size_() { +} + +void IgnoreSet::Add(StackID stack_id) { + if (size_ == kMaxSize) + return; + for (uptr i = 0; i < size_; i++) { + if (stacks_[i] == stack_id) + return; + } + stacks_[size_++] = stack_id; +} + +StackID IgnoreSet::At(uptr i) const { + CHECK_LT(i, size_); + CHECK_LE(size_, kMaxSize); + return stacks_[i]; +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h new file mode 100644 index 000000000000..4e2511291ce4 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h @@ -0,0 +1,36 @@ +//===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// IgnoreSet holds a set of stack traces where ignores were enabled. +//===----------------------------------------------------------------------===// +#ifndef TSAN_IGNORESET_H +#define TSAN_IGNORESET_H + +#include "tsan_defs.h" + +namespace __tsan { + +class IgnoreSet { + public: + IgnoreSet(); + void Add(StackID stack_id); + void Reset() { size_ = 0; } + uptr Size() const { return size_; } + StackID At(uptr i) const; + + private: + static constexpr uptr kMaxSize = 16; + uptr size_; + StackID stacks_[kMaxSize]; +}; + +} // namespace __tsan + +#endif // TSAN_IGNORESET_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ilist.h b/compiler-rt/lib/tsan/rtl-old/tsan_ilist.h new file mode 100644 index 000000000000..d7d8be219dbe --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_ilist.h @@ -0,0 +1,189 @@ +//===-- tsan_ilist.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_ILIST_H +#define TSAN_ILIST_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __tsan { + +class INode { + public: + INode() = default; + + private: + INode* next_ = nullptr; + INode* prev_ = nullptr; + + template + friend class IList; + INode(const INode&) = delete; + void operator=(const INode&) = delete; +}; + +// Intrusive doubly-linked list. +// +// The node class (MyNode) needs to include "INode foo" field, +// then the list can be declared as IList. +// This design allows to link MyNode into multiple lists using +// different INode fields. +// The optional Elem template argument allows to specify node MDT +// (most derived type) if it's different from MyNode. +template +class IList { + public: + IList(); + + void PushFront(Elem* e); + void PushBack(Elem* e); + void Remove(Elem* e); + + Elem* PopFront(); + Elem* PopBack(); + Elem* Front(); + Elem* Back(); + + // Prev links point towards front of the queue. + Elem* Prev(Elem* e); + // Next links point towards back of the queue. + Elem* Next(Elem* e); + + uptr Size() const; + bool Empty() const; + bool Queued(Elem* e) const; + + private: + INode node_; + uptr size_ = 0; + + void Push(Elem* e, INode* after); + static INode* ToNode(Elem* e); + static Elem* ToElem(INode* n); + + IList(const IList&) = delete; + void operator=(const IList&) = delete; +}; + +template +IList::IList() { + node_.next_ = node_.prev_ = &node_; +} + +template +void IList::PushFront(Elem* e) { + Push(e, &node_); +} + +template +void IList::PushBack(Elem* e) { + Push(e, node_.prev_); +} + +template +void IList::Push(Elem* e, INode* after) { + INode* n = ToNode(e); + DCHECK_EQ(n->next_, nullptr); + DCHECK_EQ(n->prev_, nullptr); + INode* next = after->next_; + n->next_ = next; + n->prev_ = after; + next->prev_ = n; + after->next_ = n; + size_++; +} + +template +void IList::Remove(Elem* e) { + INode* n = ToNode(e); + INode* next = n->next_; + INode* prev = n->prev_; + DCHECK(next); + DCHECK(prev); + DCHECK(size_); + next->prev_ = prev; + prev->next_ = next; + n->prev_ = n->next_ = nullptr; + size_--; +} + +template +Elem* IList::PopFront() { + Elem* e = Front(); + if (e) + Remove(e); + return e; +} + +template +Elem* IList::PopBack() { + Elem* e = Back(); + if (e) + Remove(e); + return e; +} + +template +Elem* IList::Front() { + return size_ ? ToElem(node_.next_) : nullptr; +} + +template +Elem* IList::Back() { + return size_ ? ToElem(node_.prev_) : nullptr; +} + +template +Elem* IList::Prev(Elem* e) { + INode* n = ToNode(e); + DCHECK(n->prev_); + return n->prev_ != &node_ ? ToElem(n->prev_) : nullptr; +} + +template +Elem* IList::Next(Elem* e) { + INode* n = ToNode(e); + DCHECK(n->next_); + return n->next_ != &node_ ? ToElem(n->next_) : nullptr; +} + +template +uptr IList::Size() const { + return size_; +} + +template +bool IList::Empty() const { + return size_ == 0; +} + +template +bool IList::Queued(Elem* e) const { + INode* n = ToNode(e); + DCHECK_EQ(!n->next_, !n->prev_); + return n->next_; +} + +template +INode* IList::ToNode(Elem* e) { + return &(e->*Node); +} + +template +Elem* IList::ToElem(INode* n) { + return static_cast(reinterpret_cast( + reinterpret_cast(n) - + reinterpret_cast(&(reinterpret_cast(0)->*Node)))); +} + +} // namespace __tsan + +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h new file mode 100644 index 000000000000..61dbb81ffec4 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h @@ -0,0 +1,93 @@ +#ifndef TSAN_INTERCEPTORS_H +#define TSAN_INTERCEPTORS_H + +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "tsan_rtl.h" + +namespace __tsan { + +class ScopedInterceptor { + public: + ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); + ~ScopedInterceptor(); + void DisableIgnores() { + if (UNLIKELY(ignoring_)) + DisableIgnoresImpl(); + } + void EnableIgnores() { + if (UNLIKELY(ignoring_)) + EnableIgnoresImpl(); + } + + private: + ThreadState *const thr_; + bool in_ignored_lib_; + bool ignoring_; + + void DisableIgnoresImpl(); + void EnableIgnoresImpl(); +}; + +LibIgnore *libignore(); + +#if !SANITIZER_GO +inline bool in_symbolizer() { + return UNLIKELY(cur_thread_init()->in_symbolizer); +} +#endif + +} // namespace __tsan + +#define SCOPED_INTERCEPTOR_RAW(func, ...) \ + ThreadState *thr = cur_thread_init(); \ + ScopedInterceptor si(thr, #func, GET_CALLER_PC()); \ + UNUSED const uptr pc = GET_CURRENT_PC(); + +#ifdef __powerpc64__ +// Debugging of crashes on powerpc after commit: +// c80604f7a3 ("tsan: remove real func check from interceptors") +// Somehow replacing if with DCHECK leads to strange failures in: +// SanitizerCommon-tsan-powerpc64le-Linux :: Linux/ptrace.cpp +// https://lab.llvm.org/buildbot/#/builders/105 +// https://lab.llvm.org/buildbot/#/builders/121 +// https://lab.llvm.org/buildbot/#/builders/57 +# define CHECK_REAL_FUNC(func) \ + if (REAL(func) == 0) { \ + Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ + Die(); \ + } +#else +# define CHECK_REAL_FUNC(func) DCHECK(REAL(func)) +#endif + +#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ + SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ + CHECK_REAL_FUNC(func); \ + if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \ + return REAL(func)(__VA_ARGS__); + +#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \ + si.DisableIgnores(); + +#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \ + si.EnableIgnores(); + +#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) + +#if SANITIZER_NETBSD +# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \ + TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func)); +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \ + TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func2)); +#else +# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) +#endif + +#endif // TSAN_INTERCEPTORS_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp new file mode 100644 index 000000000000..cbbb7ecb2397 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp @@ -0,0 +1,814 @@ +//===-- tsan_interceptors_libdispatch.cpp ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Support for intercepting libdispatch (GCD). +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_common.h" +#include "interception/interception.h" +#include "tsan_interceptors.h" +#include "tsan_rtl.h" + +#include "BlocksRuntime/Block.h" +#include "tsan_dispatch_defs.h" + +#if SANITIZER_MAC +# include +#endif + +namespace __tsan { + typedef u16 uint16_t; + +typedef struct { + dispatch_queue_t queue; + void *orig_context; + dispatch_function_t orig_work; + bool free_context_in_callback; + bool submitted_synchronously; + bool is_barrier_block; + uptr non_queue_sync_object; +} block_context_t; + +// The offsets of different fields of the dispatch_queue_t structure, exported +// by libdispatch.dylib. +extern "C" struct dispatch_queue_offsets_s { + const uint16_t dqo_version; + const uint16_t dqo_label; + const uint16_t dqo_label_size; + const uint16_t dqo_flags; + const uint16_t dqo_flags_size; + const uint16_t dqo_serialnum; + const uint16_t dqo_serialnum_size; + const uint16_t dqo_width; + const uint16_t dqo_width_size; + const uint16_t dqo_running; + const uint16_t dqo_running_size; + const uint16_t dqo_suspend_cnt; + const uint16_t dqo_suspend_cnt_size; + const uint16_t dqo_target_queue; + const uint16_t dqo_target_queue_size; + const uint16_t dqo_priority; + const uint16_t dqo_priority_size; +} dispatch_queue_offsets; + +static bool IsQueueSerial(dispatch_queue_t q) { + CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); + uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); + CHECK_NE(width, 0); + return width == 1; +} + +static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) { + CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); + dispatch_queue_t tq = *( + dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue); + return tq; +} + +static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { + dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source); + CHECK_NE(tq, 0); + return tq; +} + +static block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, void *orig_context, + dispatch_function_t orig_work) { + block_context_t *new_context = + (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); + new_context->queue = queue; + new_context->orig_context = orig_context; + new_context->orig_work = orig_work; + new_context->free_context_in_callback = true; + new_context->submitted_synchronously = false; + new_context->is_barrier_block = false; + new_context->non_queue_sync_object = 0; + return new_context; +} + +#define GET_QUEUE_SYNC_VARS(context, q) \ + bool is_queue_serial = q && IsQueueSerial(q); \ + uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \ + uptr serial_sync = (uptr)sync_ptr; \ + uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \ + bool serial_task = context->is_barrier_block || is_queue_serial + +static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, + block_context_t *context) { + uptr submit_sync = (uptr)context; + Acquire(thr, pc, submit_sync); + + dispatch_queue_t q = context->queue; + do { + GET_QUEUE_SYNC_VARS(context, q); + if (serial_sync) Acquire(thr, pc, serial_sync); + if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync); + + if (q) q = GetTargetQueueFromQueue(q); + } while (q); +} + +static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, + block_context_t *context) { + uptr submit_sync = (uptr)context; + if (context->submitted_synchronously) Release(thr, pc, submit_sync); + + dispatch_queue_t q = context->queue; + do { + GET_QUEUE_SYNC_VARS(context, q); + if (serial_task && serial_sync) Release(thr, pc, serial_sync); + if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync); + + if (q) q = GetTargetQueueFromQueue(q); + } while (q); +} + +static void dispatch_callback_wrap(void *param) { + SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); + block_context_t *context = (block_context_t *)param; + + dispatch_sync_pre_execute(thr, pc, context); + + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + context->orig_work(context->orig_context); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + + dispatch_sync_post_execute(thr, pc, context); + + if (context->free_context_in_callback) user_free(thr, pc, context); +} + +static void invoke_block(void *param) { + dispatch_block_t block = (dispatch_block_t)param; + block(); +} + +static void invoke_and_release_block(void *param) { + dispatch_block_t block = (dispatch_block_t)param; + block(); + Block_release(block); +} + +#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + dispatch_block_t heap_block = Block_copy(block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + block_context_t *new_context = \ + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ + new_context->is_barrier_block = barrier; \ + Release(thr, pc, (uptr)new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + } + +#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ + DISPATCH_NOESCAPE dispatch_block_t block) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, block); \ + block_context_t new_context = { \ + q, block, &invoke_block, false, true, barrier, 0}; \ + Release(thr, pc, (uptr)&new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + Acquire(thr, pc, (uptr)&new_context); \ + } + +#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ + dispatch_function_t work) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ + block_context_t *new_context = \ + AllocContext(thr, pc, q, context, work); \ + new_context->is_barrier_block = barrier; \ + Release(thr, pc, (uptr)new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name)(q, new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + } + +#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ + dispatch_function_t work) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ + block_context_t new_context = { \ + q, context, work, false, true, barrier, 0}; \ + Release(thr, pc, (uptr)&new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name)(q, &new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + Acquire(thr, pc, (uptr)&new_context); \ + } + +#define DISPATCH_INTERCEPT(name, barrier) \ + DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ + DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ + DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ + DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) + +// We wrap dispatch_async, dispatch_sync and friends where we allocate a new +// context, which is used to synchronize (we release the context before +// submitting, and the callback acquires it before executing the original +// callback). +DISPATCH_INTERCEPT(dispatch, false) +DISPATCH_INTERCEPT(dispatch_barrier, true) + +// dispatch_async_and_wait() and friends were introduced in macOS 10.14. +// Linking of these interceptors fails when using an older SDK. +#if !SANITIZER_MAC || defined(__MAC_10_14) +// macOS 10.14 is greater than our minimal deployment target. To ensure we +// generate a weak reference so the TSan dylib continues to work on older +// systems, we need to forward declare the intercepted functions as "weak +// imports". Note that this file is multi-platform, so we cannot include the +// actual header file (#include ). +SANITIZER_WEAK_IMPORT void dispatch_async_and_wait( + dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block); +SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f( + dispatch_queue_t queue, void *context, dispatch_function_t work); +SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait( + dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block); +SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f( + dispatch_queue_t queue, void *context, dispatch_function_t work); + +DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false) +DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false) +DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true) +DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true) +#endif + + +DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, dispatch_function_t work) + +TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, + dispatch_queue_t queue, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + block_context_t *new_context = + AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); + Release(thr, pc, (uptr)new_context); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); +} + +TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, + dispatch_function_t work) { + SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); + WRAP(dispatch_after)(when, queue, ^(void) { + work(context); + }); +} + +// GCD's dispatch_once implementation has a fast path that contains a racy read +// and it's inlined into user's code. Furthermore, this fast path doesn't +// establish a proper happens-before relations between the initialization and +// code following the call to dispatch_once. We could deal with this in +// instrumented code, but there's not much we can do about it in system +// libraries. Let's disable the fast path (by never storing the value ~0 to +// predicate), so the interceptor is always called, and let's add proper release +// and acquire semantics. Since TSan does not see its own atomic stores, the +// race on predicate won't be reported - the only accesses to it that TSan sees +// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is +// both a macro and a real function, we want to intercept the function, so we +// need to undefine the macro. +#undef dispatch_once +TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, + DISPATCH_NOESCAPE dispatch_block_t block) { + SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); + atomic_uint32_t *a = reinterpret_cast(predicate); + u32 v = atomic_load(a, memory_order_acquire); + if (v == 0 && + atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + block(); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, (uptr)a); + atomic_store(a, 2, memory_order_release); + } else { + while (v != 2) { + internal_sched_yield(); + v = atomic_load(a, memory_order_acquire); + } + Acquire(thr, pc, (uptr)a); + } +} + +#undef dispatch_once_f +TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, + void *context, dispatch_function_t function) { + SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + WRAP(dispatch_once)(predicate, ^(void) { + function(context); + }); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); +} + +TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, + dispatch_semaphore_t dsema) { + SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); + Release(thr, pc, (uptr)dsema); + return REAL(dispatch_semaphore_signal)(dsema); +} + +TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, + dispatch_time_t timeout) { + SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); + long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); + if (result == 0) Acquire(thr, pc, (uptr)dsema); + return result; +} + +TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, + dispatch_time_t timeout) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); + long_t result = REAL(dispatch_group_wait)(group, timeout); + if (result == 0) Acquire(thr, pc, (uptr)group); + return result; +} + +// Used, but not intercepted. +extern "C" void dispatch_group_enter(dispatch_group_t group); + +TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); + // Acquired in the group notification callback in dispatch_group_notify[_f]. + Release(thr, pc, (uptr)group); + REAL(dispatch_group_leave)(group); +} + +TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, + dispatch_queue_t queue, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); + dispatch_retain(group); + dispatch_group_enter(group); + __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); + WRAP(dispatch_async)(queue, ^(void) { + block_copy(); + Block_release(block_copy); + WRAP(dispatch_group_leave)(group); + dispatch_release(group); + }); +} + +TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, + dispatch_queue_t queue, void *context, + dispatch_function_t work) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); + dispatch_retain(group); + dispatch_group_enter(group); + WRAP(dispatch_async)(queue, ^(void) { + work(context); + WRAP(dispatch_group_leave)(group); + dispatch_release(group); + }); +} + +DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) + +TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, + dispatch_queue_t q, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); + + // To make sure the group is still available in the callback (otherwise + // it can be already destroyed). Will be released in the callback. + dispatch_retain(group); + + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(^(void) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); + // Released when leaving the group (dispatch_group_leave). + Acquire(thr, pc, (uptr)group); + } + dispatch_release(group); + block(); + }); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + block_context_t *new_context = + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); + new_context->is_barrier_block = true; + Release(thr, pc, (uptr)new_context); + REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); +} + +TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) { + WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, + dispatch_source_t source, dispatch_block_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); + if (handler == nullptr) + return REAL(dispatch_source_set_event_handler)(source, nullptr); + dispatch_queue_t q = GetTargetQueueFromSource(source); + __block block_context_t new_context = { + q, handler, &invoke_block, false, false, false, 0 }; + dispatch_block_t new_handler = Block_copy(^(void) { + new_context.orig_context = handler; // To explicitly capture "handler". + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_source_set_event_handler)(source, new_handler); + Block_release(new_handler); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, + dispatch_source_t source, dispatch_function_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); + if (handler == nullptr) + return REAL(dispatch_source_set_event_handler)(source, nullptr); + dispatch_block_t block = ^(void) { + handler(dispatch_get_context(source)); + }; + WRAP(dispatch_source_set_event_handler)(source, block); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, + dispatch_source_t source, dispatch_block_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); + if (handler == nullptr) + return REAL(dispatch_source_set_cancel_handler)(source, nullptr); + dispatch_queue_t q = GetTargetQueueFromSource(source); + __block block_context_t new_context = { + q, handler, &invoke_block, false, false, false, 0}; + dispatch_block_t new_handler = Block_copy(^(void) { + new_context.orig_context = handler; // To explicitly capture "handler". + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_source_set_cancel_handler)(source, new_handler); + Block_release(new_handler); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, + dispatch_source_t source, dispatch_function_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, + handler); + if (handler == nullptr) + return REAL(dispatch_source_set_cancel_handler)(source, nullptr); + dispatch_block_t block = ^(void) { + handler(dispatch_get_context(source)); + }; + WRAP(dispatch_source_set_cancel_handler)(source, block); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, + dispatch_source_t source, dispatch_block_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, + handler); + if (handler == nullptr) + return REAL(dispatch_source_set_registration_handler)(source, nullptr); + dispatch_queue_t q = GetTargetQueueFromSource(source); + __block block_context_t new_context = { + q, handler, &invoke_block, false, false, false, 0}; + dispatch_block_t new_handler = Block_copy(^(void) { + new_context.orig_context = handler; // To explicitly capture "handler". + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_source_set_registration_handler)(source, new_handler); + Block_release(new_handler); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, + dispatch_source_t source, dispatch_function_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, + handler); + if (handler == nullptr) + return REAL(dispatch_source_set_registration_handler)(source, nullptr); + dispatch_block_t block = ^(void) { + handler(dispatch_get_context(source)); + }; + WRAP(dispatch_source_set_registration_handler)(source, block); +} + +TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, + dispatch_queue_t queue, + DISPATCH_NOESCAPE void (^block)(size_t)) { + SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); + + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; + + Release(thr, pc, parent_to_child_sync); + void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply); + Acquire(thr, pc, parent_to_child_sync); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + block(iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync); + }; + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply)(iterations, queue, new_block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync); +} + +static void invoke_block_iteration(void *param, size_t iteration) { + auto block = (void (^)(size_t)) param; + block(iteration); +} + +TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, + dispatch_queue_t queue, void *context, + void (*work)(void *, size_t)) { + SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); + + // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch + // implements dispatch_apply in terms of dispatch_apply_f. + u8 sync1, sync2; + uptr parent_to_child_sync = (uptr)&sync1; + uptr child_to_parent_sync = (uptr)&sync2; + + Release(thr, pc, parent_to_child_sync); + void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); + Acquire(thr, pc, parent_to_child_sync); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + work(context, iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync); + }; + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync); +} + +DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) +DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) + +TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, + size_t size, dispatch_queue_t q, dispatch_block_t destructor) { + SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); + if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) + return REAL(dispatch_data_create)(buffer, size, q, destructor); + + if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) + destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; + else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) + destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; + + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(destructor); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + block_context_t *new_context = + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); + uptr submit_sync = (uptr)new_context; + Release(thr, pc, submit_sync); + return REAL(dispatch_data_create)(buffer, size, q, ^(void) { + dispatch_callback_wrap(new_context); + }); +} + +typedef void (^fd_handler_t)(dispatch_data_t data, int error); +typedef void (^cleanup_handler_t)(int error); + +TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, + dispatch_queue_t q, fd_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_read)(fd, length, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, + dispatch_queue_t q, fd_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_write)(fd, data, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, + size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + dispatch_io_handler_t new_h = + Block_copy(^(bool done, dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(done, data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_io_read)(channel, offset, length, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, + dispatch_data_t data, dispatch_queue_t q, + dispatch_io_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + dispatch_io_handler_t new_h = + Block_copy(^(bool done, dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(done, data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_io_write)(channel, offset, data, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, + dispatch_block_t barrier) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); + __block block_context_t new_context = { + nullptr, nullptr, &invoke_block, false, false, false, 0}; + new_context.non_queue_sync_object = (uptr)channel; + new_context.is_barrier_block = true; + dispatch_block_t new_block = Block_copy(^(void) { + new_context.orig_context = ^(void) { + barrier(); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_io_barrier)(channel, new_block); + Block_release(new_block); +} + +TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, + dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); + __block dispatch_io_t new_channel = nullptr; + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + cleanup_handler_t new_h = Block_copy(^(int error) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); + Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. + } + new_context.orig_context = ^(void) { + h(error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); + Block_release(new_h); + return new_channel; +} + +TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, + dispatch_io_type_t type, const char *path, int oflag, + mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, + q, h); + __block dispatch_io_t new_channel = nullptr; + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + cleanup_handler_t new_h = Block_copy(^(int error) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); + Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. + } + new_context.orig_context = ^(void) { + h(error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + new_channel = + REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); + Block_release(new_h); + return new_channel; +} + +TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, + dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, + cleanup_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); + __block dispatch_io_t new_channel = nullptr; + __block block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + cleanup_handler_t new_h = Block_copy(^(int error) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); + Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. + } + new_context.orig_context = ^(void) { + h(error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); + Block_release(new_h); + return new_channel; +} + +TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, + dispatch_io_close_flags_t flags) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); + Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. + return REAL(dispatch_io_close)(channel, flags); +} + +// Resuming a suspended queue needs to synchronize with all subsequent +// executions of blocks in that queue. +TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { + SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o); + Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync + // in dispatch_sync_pre_execute + return REAL(dispatch_resume)(o); +} + +void InitializeLibdispatchInterceptors() { + INTERCEPT_FUNCTION(dispatch_async); + INTERCEPT_FUNCTION(dispatch_async_f); + INTERCEPT_FUNCTION(dispatch_sync); + INTERCEPT_FUNCTION(dispatch_sync_f); + INTERCEPT_FUNCTION(dispatch_barrier_async); + INTERCEPT_FUNCTION(dispatch_barrier_async_f); + INTERCEPT_FUNCTION(dispatch_barrier_sync); + INTERCEPT_FUNCTION(dispatch_barrier_sync_f); + INTERCEPT_FUNCTION(dispatch_async_and_wait); + INTERCEPT_FUNCTION(dispatch_async_and_wait_f); + INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait); + INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f); + INTERCEPT_FUNCTION(dispatch_after); + INTERCEPT_FUNCTION(dispatch_after_f); + INTERCEPT_FUNCTION(dispatch_once); + INTERCEPT_FUNCTION(dispatch_once_f); + INTERCEPT_FUNCTION(dispatch_semaphore_signal); + INTERCEPT_FUNCTION(dispatch_semaphore_wait); + INTERCEPT_FUNCTION(dispatch_group_wait); + INTERCEPT_FUNCTION(dispatch_group_leave); + INTERCEPT_FUNCTION(dispatch_group_async); + INTERCEPT_FUNCTION(dispatch_group_async_f); + INTERCEPT_FUNCTION(dispatch_group_notify); + INTERCEPT_FUNCTION(dispatch_group_notify_f); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler); + INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); + INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); + INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); + INTERCEPT_FUNCTION(dispatch_apply); + INTERCEPT_FUNCTION(dispatch_apply_f); + INTERCEPT_FUNCTION(dispatch_data_create); + INTERCEPT_FUNCTION(dispatch_read); + INTERCEPT_FUNCTION(dispatch_write); + INTERCEPT_FUNCTION(dispatch_io_read); + INTERCEPT_FUNCTION(dispatch_io_write); + INTERCEPT_FUNCTION(dispatch_io_barrier); + INTERCEPT_FUNCTION(dispatch_io_create); + INTERCEPT_FUNCTION(dispatch_io_create_with_path); + INTERCEPT_FUNCTION(dispatch_io_create_with_io); + INTERCEPT_FUNCTION(dispatch_io_close); + INTERCEPT_FUNCTION(dispatch_resume); +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp new file mode 100644 index 000000000000..ed064150d005 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp @@ -0,0 +1,521 @@ +//===-- tsan_interceptors_mac.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific interceptors. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "interception/interception.h" +#include "tsan_interceptors.h" +#include "tsan_interface.h" +#include "tsan_interface_ann.h" +#include "sanitizer_common/sanitizer_addrhashmap.h" + +#include +#include +#include +#include +#include + +#if defined(__has_include) && __has_include() +#include +#endif // #if defined(__has_include) && __has_include() + +typedef long long_t; + +extern "C" { +int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); +int setcontext(const ucontext_t *ucp); +} + +namespace __tsan { + +// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, +// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are +// actually aliases of each other, and we cannot have different interceptors for +// them, because they're actually the same function. Thus, we have to stay +// conservative and treat the non-barrier versions as mo_acq_rel. +static constexpr morder kMacOrderBarrier = mo_acq_rel; +static constexpr morder kMacOrderNonBarrier = mo_acq_rel; +static constexpr morder kMacFailureOrder = mo_relaxed; + +#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ + } + +#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ + } + +#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ + } + +#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ + mo) \ + TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ + } + +#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ + m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderBarrier) \ + m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ + kMacOrderBarrier) + +#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ + m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderBarrier) \ + m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ + __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) + +OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, + OSATOMIC_INTERCEPTOR_PLUS_X) +OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, + OSATOMIC_INTERCEPTOR_PLUS_1) +OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, + OSATOMIC_INTERCEPTOR_MINUS_1) +OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, + OSATOMIC_INTERCEPTOR) +OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, + OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) +OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, + OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) + +#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ + TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ + return tsan_atomic_f##_compare_exchange_strong( \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + kMacOrderNonBarrier, kMacFailureOrder); \ + } \ + \ + TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ + t volatile *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ + return tsan_atomic_f##_compare_exchange_strong( \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + kMacOrderBarrier, kMacFailureOrder); \ + } + +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, + long_t) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, + void *) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, + int32_t) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, + int64_t) + +#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ + TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ + volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ + char bit = 0x80u >> (n & 7); \ + char mask = clear ? ~bit : bit; \ + char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ + return orig_byte & bit; \ + } + +#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ + OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ + OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) + +OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) +OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, + true) + +TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, + size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); + __tsan_release(item); + REAL(OSAtomicEnqueue)(list, item, offset); +} + +TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); + void *item = REAL(OSAtomicDequeue)(list, offset); + if (item) __tsan_acquire(item); + return item; +} + +// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. +#if !SANITIZER_IOS + +TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, + size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); + __tsan_release(item); + REAL(OSAtomicFifoEnqueue)(list, item, offset); +} + +TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, + size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); + void *item = REAL(OSAtomicFifoDequeue)(list, offset); + if (item) __tsan_acquire(item); + return item; +} + +#endif + +TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockLock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); + REAL(OSSpinLockLock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockTry)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); + bool result = REAL(OSSpinLockTry)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockUnlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); + Release(thr, pc, (uptr)lock); + REAL(OSSpinLockUnlock)(lock); +} + +TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_lock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); + REAL(os_lock_lock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_trylock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); + bool result = REAL(os_lock_trylock)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_unlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); + Release(thr, pc, (uptr)lock); + REAL(os_lock_unlock)(lock); +} + +TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) { + if (!cur_thread()->is_inited || cur_thread()->is_dead) { + return REAL(os_unfair_lock_lock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock); + REAL(os_unfair_lock_lock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock, + u32 options) { + if (!cur_thread()->is_inited || cur_thread()->is_dead) { + return REAL(os_unfair_lock_lock_with_options)(lock, options); + } + SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options); + REAL(os_unfair_lock_lock_with_options)(lock, options); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) { + if (!cur_thread()->is_inited || cur_thread()->is_dead) { + return REAL(os_unfair_lock_trylock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock); + bool result = REAL(os_unfair_lock_trylock)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) { + if (!cur_thread()->is_inited || cur_thread()->is_dead) { + return REAL(os_unfair_lock_unlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock); + Release(thr, pc, (uptr)lock); + REAL(os_unfair_lock_unlock)(lock); +} + +#if defined(__has_include) && __has_include() + +TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, + xpc_connection_t connection, xpc_handler_t handler) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, + handler); + Release(thr, pc, (uptr)connection); + xpc_handler_t new_handler = ^(xpc_object_t object) { + { + SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); + Acquire(thr, pc, (uptr)connection); + } + handler(object); + }; + REAL(xpc_connection_set_event_handler)(connection, new_handler); +} + +TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, + dispatch_block_t barrier) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); + Release(thr, pc, (uptr)connection); + dispatch_block_t new_barrier = ^() { + { + SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); + Acquire(thr, pc, (uptr)connection); + } + barrier(); + }; + REAL(xpc_connection_send_barrier)(connection, new_barrier); +} + +TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, + xpc_connection_t connection, xpc_object_t message, + dispatch_queue_t replyq, xpc_handler_t handler) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, + message, replyq, handler); + Release(thr, pc, (uptr)connection); + xpc_handler_t new_handler = ^(xpc_object_t object) { + { + SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); + Acquire(thr, pc, (uptr)connection); + } + handler(object); + }; + REAL(xpc_connection_send_message_with_reply) + (connection, message, replyq, new_handler); +} + +TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection); + Release(thr, pc, (uptr)connection); + REAL(xpc_connection_cancel)(connection); +} + +#endif // #if defined(__has_include) && __has_include() + +// Determines whether the Obj-C object pointer is a tagged pointer. Tagged +// pointers encode the object data directly in their pointer bits and do not +// have an associated memory allocation. The Obj-C runtime uses tagged pointers +// to transparently optimize small objects. +static bool IsTaggedObjCPointer(id obj) { + const uptr kPossibleTaggedBits = 0x8000000000000001ull; + return ((uptr)obj & kPossibleTaggedBits) != 0; +} + +// Returns an address which can be used to inform TSan about synchronization +// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid +// address in the process space. We do a small allocation here to obtain a +// stable address (the array backing the hash map can change). The memory is +// never free'd (leaked) and allocation and locking are slow, but this code only +// runs for @synchronized with tagged pointers, which is very rare. +static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { + typedef AddrHashMap Map; + static Map Addresses; + Map::Handle h(&Addresses, addr); + if (h.created()) { + ThreadIgnoreBegin(thr, pc); + *h = (uptr) user_alloc(thr, pc, /*size=*/1); + ThreadIgnoreEnd(thr); + } + return *h; +} + +// Returns an address on which we can synchronize given an Obj-C object pointer. +// For normal object pointers, this is just the address of the object in memory. +// Tagged pointers are not backed by an actual memory allocation, so we need to +// synthesize a valid address. +static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { + if (IsTaggedObjCPointer(obj)) + return GetOrCreateSyncAddress((uptr)obj, thr, pc); + return (uptr)obj; +} + +TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { + SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); + if (!obj) return REAL(objc_sync_enter)(obj); + uptr addr = SyncAddressForObjCObject(obj, thr, pc); + MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); + int result = REAL(objc_sync_enter)(obj); + CHECK_EQ(result, OBJC_SYNC_SUCCESS); + MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); + return result; +} + +TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { + SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); + if (!obj) return REAL(objc_sync_exit)(obj); + uptr addr = SyncAddressForObjCObject(obj, thr, pc); + MutexUnlock(thr, pc, addr); + int result = REAL(objc_sync_exit)(obj); + if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); + return result; +} + +TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { + { + SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); + } + // Because of swapcontext() semantics we have no option but to copy its + // implementation here + if (!oucp || !ucp) { + errno = EINVAL; + return -1; + } + ThreadState *thr = cur_thread(); + const int UCF_SWAPPED = 0x80000000; + oucp->uc_onstack &= ~UCF_SWAPPED; + thr->ignore_interceptors++; + int ret = getcontext(oucp); + if (!(oucp->uc_onstack & UCF_SWAPPED)) { + thr->ignore_interceptors--; + if (!ret) { + oucp->uc_onstack |= UCF_SWAPPED; + ret = setcontext(ucp); + } + } + return ret; +} + +// On macOS, libc++ is always linked dynamically, so intercepting works the +// usual way. +#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR + +namespace { +struct fake_shared_weak_count { + volatile a64 shared_owners; + volatile a64 shared_weak_owners; + virtual void _unused_0x0() = 0; + virtual void _unused_0x8() = 0; + virtual void on_zero_shared() = 0; + virtual void _unused_0x18() = 0; + virtual void on_zero_shared_weak() = 0; + virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor +}; +} // namespace + +// The following code adds libc++ interceptors for: +// void __shared_weak_count::__release_shared() _NOEXCEPT; +// bool __shared_count::__release_shared() _NOEXCEPT; +// Shared and weak pointers in C++ maintain reference counts via atomics in +// libc++.dylib, which are TSan-invisible, and this leads to false positives in +// destructor code. These interceptors re-implements the whole functions so that +// the mo_acq_rel semantics of the atomic decrement are visible. +// +// Unfortunately, the interceptors cannot simply Acquire/Release some sync +// object and call the original function, because it would have a race between +// the sync and the destruction of the object. Calling both under a lock will +// not work because the destructor can invoke this interceptor again (and even +// in a different thread, so recursive locks don't help). + +STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, + fake_shared_weak_count *o) { + if (!flags()->shared_ptr_interceptor) + return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); + + SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, + o); + if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { + Acquire(thr, pc, (uptr)&o->shared_owners); + o->on_zero_shared(); + if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == + 0) { + Acquire(thr, pc, (uptr)&o->shared_weak_owners); + o->on_zero_shared_weak(); + } + } +} + +STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv, + fake_shared_weak_count *o) { + if (!flags()->shared_ptr_interceptor) + return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o); + + SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o); + if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { + Acquire(thr, pc, (uptr)&o->shared_owners); + o->on_zero_shared(); + return true; + } + return false; +} + +namespace { +struct call_once_callback_args { + void (*orig_func)(void *arg); + void *orig_arg; + void *flag; +}; + +void call_once_callback_wrapper(void *arg) { + call_once_callback_args *new_args = (call_once_callback_args *)arg; + new_args->orig_func(new_args->orig_arg); + __tsan_release(new_args->flag); +} +} // namespace + +// This adds a libc++ interceptor for: +// void __call_once(volatile unsigned long&, void*, void(*)(void*)); +// C++11 call_once is implemented via an internal function __call_once which is +// inside libc++.dylib, and the atomic release store inside it is thus +// TSan-invisible. To avoid false positives, this interceptor wraps the callback +// function and performs an explicit Release after the user code has run. +STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, + void *arg, void (*func)(void *arg)) { + call_once_callback_args new_args = {func, arg, flag}; + REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, + call_once_callback_wrapper); +} + +} // namespace __tsan + +#endif // SANITIZER_MAC diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp new file mode 100644 index 000000000000..6d62ff6a8382 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp @@ -0,0 +1,53 @@ +//===-- tsan_interceptors_mach_vm.cpp -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Interceptors for mach_vm_* user space memory routines on Darwin. +//===----------------------------------------------------------------------===// + +#include "interception/interception.h" +#include "tsan_interceptors.h" +#include "tsan_platform.h" + +#include + +namespace __tsan { + +static bool intersects_with_shadow(mach_vm_address_t address, + mach_vm_size_t size, int flags) { + // VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE. + if (flags & VM_FLAGS_ANYWHERE) return false; + return !IsAppMem(address) || !IsAppMem(address + size - 1); +} + +TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target, + mach_vm_address_t *address, mach_vm_size_t size, int flags) { + SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags); + if (target != mach_task_self()) + return REAL(mach_vm_allocate)(target, address, size, flags); + if (address && intersects_with_shadow(*address, size, flags)) + return KERN_NO_SPACE; + kern_return_t kr = REAL(mach_vm_allocate)(target, address, size, flags); + if (kr == KERN_SUCCESS) + MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size); + return kr; +} + +TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target, + mach_vm_address_t address, mach_vm_size_t size) { + SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size); + if (target != mach_task_self()) + return REAL(mach_vm_deallocate)(target, address, size); + kern_return_t kr = REAL(mach_vm_deallocate)(target, address, size); + if (kr == KERN_SUCCESS && address) + UnmapShadow(thr, address, size); + return kr; +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp new file mode 100644 index 000000000000..cf3dc90d96a1 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp @@ -0,0 +1,3015 @@ +//===-- tsan_interceptors_posix.cpp ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// FIXME: move as many interceptors as possible into +// sanitizer_common/sanitizer_common_interceptors.inc +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_posix.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" +#include "interception/interception.h" +#include "tsan_interceptors.h" +#include "tsan_interface.h" +#include "tsan_platform.h" +#include "tsan_suppressions.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" +#include "tsan_fd.h" + +#include + +using namespace __tsan; + +#if SANITIZER_FREEBSD || SANITIZER_MAC +#define stdout __stdoutp +#define stderr __stderrp +#endif + +#if SANITIZER_NETBSD +#define dirfd(dirp) (*(int *)(dirp)) +#define fileno_unlocked(fp) \ + (((__sanitizer_FILE *)fp)->_file == -1 \ + ? -1 \ + : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file)) + +#define stdout ((__sanitizer_FILE*)&__sF[1]) +#define stderr ((__sanitizer_FILE*)&__sF[2]) + +#define nanosleep __nanosleep50 +#define vfork __vfork14 +#endif + +#ifdef __mips__ +const int kSigCount = 129; +#else +const int kSigCount = 65; +#endif + +#ifdef __mips__ +struct ucontext_t { + u64 opaque[768 / sizeof(u64) + 1]; +}; +#else +struct ucontext_t { + // The size is determined by looking at sizeof of real ucontext_t on linux. + u64 opaque[936 / sizeof(u64) + 1]; +}; +#endif + +#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \ + defined(__s390x__) +#define PTHREAD_ABI_BASE "GLIBC_2.3.2" +#elif defined(__aarch64__) || SANITIZER_PPC64V2 +#define PTHREAD_ABI_BASE "GLIBC_2.17" +#endif + +extern "C" int pthread_attr_init(void *attr); +extern "C" int pthread_attr_destroy(void *attr); +DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) +extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); +extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void), + void (*child)(void)); +extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); +extern "C" int pthread_setspecific(unsigned key, const void *v); +DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) +DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) +DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) +DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) +extern "C" int pthread_equal(void *t1, void *t2); +extern "C" void *pthread_self(); +extern "C" void _exit(int status); +#if !SANITIZER_NETBSD +extern "C" int fileno_unlocked(void *stream); +extern "C" int dirfd(void *dirp); +#endif +#if SANITIZER_NETBSD +extern __sanitizer_FILE __sF[]; +#else +extern __sanitizer_FILE *stdout, *stderr; +#endif +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD +const int PTHREAD_MUTEX_RECURSIVE = 1; +const int PTHREAD_MUTEX_RECURSIVE_NP = 1; +#else +const int PTHREAD_MUTEX_RECURSIVE = 2; +const int PTHREAD_MUTEX_RECURSIVE_NP = 2; +#endif +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD +const int EPOLL_CTL_ADD = 1; +#endif +const int SIGILL = 4; +const int SIGTRAP = 5; +const int SIGABRT = 6; +const int SIGFPE = 8; +const int SIGSEGV = 11; +const int SIGPIPE = 13; +const int SIGTERM = 15; +#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD +const int SIGBUS = 10; +const int SIGSYS = 12; +#else +const int SIGBUS = 7; +const int SIGSYS = 31; +#endif +void *const MAP_FAILED = (void*)-1; +#if SANITIZER_NETBSD +const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; +#elif !SANITIZER_MAC +const int PTHREAD_BARRIER_SERIAL_THREAD = -1; +#endif +const int MAP_FIXED = 0x10; +typedef long long_t; +typedef __sanitizer::u16 mode_t; + +// From /usr/include/unistd.h +# define F_ULOCK 0 /* Unlock a previously locked region. */ +# define F_LOCK 1 /* Lock a region for exclusive use. */ +# define F_TLOCK 2 /* Test and lock a region for exclusive use. */ +# define F_TEST 3 /* Test a region for other processes locks. */ + +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD +const int SA_SIGINFO = 0x40; +const int SIG_SETMASK = 3; +#elif defined(__mips__) +const int SA_SIGINFO = 8; +const int SIG_SETMASK = 3; +#else +const int SA_SIGINFO = 4; +const int SIG_SETMASK = 2; +#endif + +#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ + (!cur_thread_init()->is_inited) + +namespace __tsan { +struct SignalDesc { + bool armed; + __sanitizer_siginfo siginfo; + ucontext_t ctx; +}; + +struct ThreadSignalContext { + int int_signal_send; + atomic_uintptr_t in_blocking_func; + SignalDesc pending_signals[kSigCount]; + // emptyset and oldset are too big for stack. + __sanitizer_sigset_t emptyset; + __sanitizer_sigset_t oldset; +}; + +// The sole reason tsan wraps atexit callbacks is to establish synchronization +// between callback setup and callback execution. +struct AtExitCtx { + void (*f)(); + void *arg; + uptr pc; +}; + +// InterceptorContext holds all global data required for interceptors. +// It's explicitly constructed in InitializeInterceptors with placement new +// and is never destroyed. This allows usage of members with non-trivial +// constructors and destructors. +struct InterceptorContext { + // The object is 64-byte aligned, because we want hot data to be located + // in a single cache line if possible (it's accessed in every interceptor). + ALIGNED(64) LibIgnore libignore; + __sanitizer_sigaction sigactions[kSigCount]; +#if !SANITIZER_MAC && !SANITIZER_NETBSD + unsigned finalize_key; +#endif + + Mutex atexit_mu; + Vector AtExitStack; + + InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {} +}; + +static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; +InterceptorContext *interceptor_ctx() { + return reinterpret_cast(&interceptor_placeholder[0]); +} + +LibIgnore *libignore() { + return &interceptor_ctx()->libignore; +} + +void InitializeLibIgnore() { + const SuppressionContext &supp = *Suppressions(); + const uptr n = supp.SuppressionCount(); + for (uptr i = 0; i < n; i++) { + const Suppression *s = supp.SuppressionAt(i); + if (0 == internal_strcmp(s->type, kSuppressionLib)) + libignore()->AddIgnoredLibrary(s->templ); + } + if (flags()->ignore_noninstrumented_modules) + libignore()->IgnoreNoninstrumentedModules(true); + libignore()->OnLibraryLoaded(0); +} + +// The following two hooks can be used by for cooperative scheduling when +// locking. +#ifdef TSAN_EXTERNAL_HOOKS +void OnPotentiallyBlockingRegionBegin(); +void OnPotentiallyBlockingRegionEnd(); +#else +SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} +SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} +#endif + +} // namespace __tsan + +static ThreadSignalContext *SigCtx(ThreadState *thr) { + ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx; + if (ctx == 0 && !thr->is_dead) { + ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext"); + MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); + thr->signal_ctx = ctx; + } + return ctx; +} + +ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, + uptr pc) + : thr_(thr), in_ignored_lib_(false), ignoring_(false) { + LazyInitialize(thr); + if (!thr_->is_inited) return; + if (!thr_->ignore_interceptors) FuncEntry(thr, pc); + DPrintf("#%d: intercept %s()\n", thr_->tid, fname); + ignoring_ = + !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || + libignore()->IsIgnored(pc, &in_ignored_lib_)); + EnableIgnores(); +} + +ScopedInterceptor::~ScopedInterceptor() { + if (!thr_->is_inited) return; + DisableIgnores(); + if (!thr_->ignore_interceptors) { + ProcessPendingSignals(thr_); + FuncExit(thr_); + CheckedMutex::CheckNoLocks(); + } +} + +NOINLINE +void ScopedInterceptor::EnableIgnoresImpl() { + ThreadIgnoreBegin(thr_, 0); + if (flags()->ignore_noninstrumented_modules) + thr_->suppress_reports++; + if (in_ignored_lib_) { + DCHECK(!thr_->in_ignored_lib); + thr_->in_ignored_lib = true; + } +} + +NOINLINE +void ScopedInterceptor::DisableIgnoresImpl() { + ThreadIgnoreEnd(thr_); + if (flags()->ignore_noninstrumented_modules) + thr_->suppress_reports--; + if (in_ignored_lib_) { + DCHECK(thr_->in_ignored_lib); + thr_->in_ignored_lib = false; + } +} + +#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) +#if SANITIZER_FREEBSD +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) +#elif SANITIZER_NETBSD +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ + INTERCEPT_FUNCTION(__libc_##func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ + INTERCEPT_FUNCTION(__libc_thr_##func) +#else +# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) +# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) +#endif + +#define READ_STRING_OF_LEN(thr, pc, s, len, n) \ + MemoryAccessRange((thr), (pc), (uptr)(s), \ + common_flags()->strict_string_checks ? (len) + 1 : (n), false) + +#define READ_STRING(thr, pc, s, n) \ + READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n)) + +#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) + +struct BlockingCall { + explicit BlockingCall(ThreadState *thr) + : thr(thr) + , ctx(SigCtx(thr)) { + for (;;) { + atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); + if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0) + break; + atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); + ProcessPendingSignals(thr); + } + // When we are in a "blocking call", we process signals asynchronously + // (right when they arrive). In this context we do not expect to be + // executing any user/runtime code. The known interceptor sequence when + // this is not true is: pthread_join -> munmap(stack). It's fine + // to ignore munmap in this case -- we handle stack shadow separately. + thr->ignore_interceptors++; + } + + ~BlockingCall() { + thr->ignore_interceptors--; + atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); + } + + ThreadState *thr; + ThreadSignalContext *ctx; +}; + +TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { + SCOPED_TSAN_INTERCEPTOR(sleep, sec); + unsigned res = BLOCK_REAL(sleep)(sec); + AfterSleep(thr, pc); + return res; +} + +TSAN_INTERCEPTOR(int, usleep, long_t usec) { + SCOPED_TSAN_INTERCEPTOR(usleep, usec); + int res = BLOCK_REAL(usleep)(usec); + AfterSleep(thr, pc); + return res; +} + +TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { + SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); + int res = BLOCK_REAL(nanosleep)(req, rem); + AfterSleep(thr, pc); + return res; +} + +TSAN_INTERCEPTOR(int, pause, int fake) { + SCOPED_TSAN_INTERCEPTOR(pause, fake); + return BLOCK_REAL(pause)(fake); +} + +// Note: we specifically call the function in such strange way +// with "installed_at" because in reports it will appear between +// callback frames and the frame that installed the callback. +static void at_exit_callback_installed_at() { + AtExitCtx *ctx; + { + // Ensure thread-safety. + Lock l(&interceptor_ctx()->atexit_mu); + + // Pop AtExitCtx from the top of the stack of callback functions + uptr element = interceptor_ctx()->AtExitStack.Size() - 1; + ctx = interceptor_ctx()->AtExitStack[element]; + interceptor_ctx()->AtExitStack.PopBack(); + } + + ThreadState *thr = cur_thread(); + Acquire(thr, ctx->pc, (uptr)ctx); + FuncEntry(thr, ctx->pc); + ((void(*)())ctx->f)(); + FuncExit(thr); + Free(ctx); +} + +static void cxa_at_exit_callback_installed_at(void *arg) { + ThreadState *thr = cur_thread(); + AtExitCtx *ctx = (AtExitCtx*)arg; + Acquire(thr, ctx->pc, (uptr)arg); + FuncEntry(thr, ctx->pc); + ((void(*)(void *arg))ctx->f)(ctx->arg); + FuncExit(thr); + Free(ctx); +} + +static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), + void *arg, void *dso); + +#if !SANITIZER_ANDROID +TSAN_INTERCEPTOR(int, atexit, void (*f)()) { + if (in_symbolizer()) + return 0; + // We want to setup the atexit callback even if we are in ignored lib + // or after fork. + SCOPED_INTERCEPTOR_RAW(atexit, f); + return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0); +} +#endif + +TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { + if (in_symbolizer()) + return 0; + SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); + return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso); +} + +static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), + void *arg, void *dso) { + auto *ctx = New(); + ctx->f = f; + ctx->arg = arg; + ctx->pc = pc; + Release(thr, pc, (uptr)ctx); + // Memory allocation in __cxa_atexit will race with free during exit, + // because we do not see synchronization around atexit callback list. + ThreadIgnoreBegin(thr, pc); + int res; + if (!dso) { + // NetBSD does not preserve the 2nd argument if dso is equal to 0 + // Store ctx in a local stack-like structure + + // Ensure thread-safety. + Lock l(&interceptor_ctx()->atexit_mu); + // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail + // due to atexit_mu held on exit from the calloc interceptor. + ScopedIgnoreInterceptors ignore; + + res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at, + 0, 0); + // Push AtExitCtx on the top of the stack of callback functions + if (!res) { + interceptor_ctx()->AtExitStack.PushBack(ctx); + } + } else { + res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso); + } + ThreadIgnoreEnd(thr); + return res; +} + +#if !SANITIZER_MAC && !SANITIZER_NETBSD +static void on_exit_callback_installed_at(int status, void *arg) { + ThreadState *thr = cur_thread(); + AtExitCtx *ctx = (AtExitCtx*)arg; + Acquire(thr, ctx->pc, (uptr)arg); + FuncEntry(thr, ctx->pc); + ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); + FuncExit(thr); + Free(ctx); +} + +TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { + if (in_symbolizer()) + return 0; + SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); + auto *ctx = New(); + ctx->f = (void(*)())f; + ctx->arg = arg; + ctx->pc = GET_CALLER_PC(); + Release(thr, pc, (uptr)ctx); + // Memory allocation in __cxa_atexit will race with free during exit, + // because we do not see synchronization around atexit callback list. + ThreadIgnoreBegin(thr, pc); + int res = REAL(on_exit)(on_exit_callback_installed_at, ctx); + ThreadIgnoreEnd(thr); + return res; +} +#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit) +#else +#define TSAN_MAYBE_INTERCEPT_ON_EXIT +#endif + +// Cleanup old bufs. +static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { + for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { + JmpBuf *buf = &thr->jmp_bufs[i]; + if (buf->sp <= sp) { + uptr sz = thr->jmp_bufs.Size(); + internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); + thr->jmp_bufs.PopBack(); + i--; + } + } +} + +static void SetJmp(ThreadState *thr, uptr sp) { + if (!thr->is_inited) // called from libc guts during bootstrap + return; + // Cleanup old bufs. + JmpBufGarbageCollect(thr, sp); + // Remember the buf. + JmpBuf *buf = thr->jmp_bufs.PushBack(); + buf->sp = sp; + buf->shadow_stack_pos = thr->shadow_stack_pos; + ThreadSignalContext *sctx = SigCtx(thr); + buf->int_signal_send = sctx ? sctx->int_signal_send : 0; + buf->in_blocking_func = sctx ? + atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : + false; + buf->in_signal_handler = atomic_load(&thr->in_signal_handler, + memory_order_relaxed); +} + +static void LongJmp(ThreadState *thr, uptr *env) { + uptr sp = ExtractLongJmpSp(env); + // Find the saved buf with matching sp. + for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { + JmpBuf *buf = &thr->jmp_bufs[i]; + if (buf->sp == sp) { + CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); + // Unwind the stack. + while (thr->shadow_stack_pos > buf->shadow_stack_pos) + FuncExit(thr); + ThreadSignalContext *sctx = SigCtx(thr); + if (sctx) { + sctx->int_signal_send = buf->int_signal_send; + atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, + memory_order_relaxed); + } + atomic_store(&thr->in_signal_handler, buf->in_signal_handler, + memory_order_relaxed); + JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp + return; + } + } + Printf("ThreadSanitizer: can't find longjmp buf\n"); + CHECK(0); +} + +// FIXME: put everything below into a common extern "C" block? +extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); } + +#if SANITIZER_MAC +TSAN_INTERCEPTOR(int, setjmp, void *env); +TSAN_INTERCEPTOR(int, _setjmp, void *env); +TSAN_INTERCEPTOR(int, sigsetjmp, void *env); +#else // SANITIZER_MAC + +#if SANITIZER_NETBSD +#define setjmp_symname __setjmp14 +#define sigsetjmp_symname __sigsetjmp14 +#else +#define setjmp_symname setjmp +#define sigsetjmp_symname sigsetjmp +#endif + +#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x +#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) +#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) +#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) + +#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname) +#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname) + +// Not called. Merely to satisfy TSAN_INTERCEPT(). +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int TSAN_INTERCEPTOR_SETJMP(void *env); +extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { + CHECK(0); + return 0; +} + +// FIXME: any reason to have a separate declaration? +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int __interceptor__setjmp(void *env); +extern "C" int __interceptor__setjmp(void *env) { + CHECK(0); + return 0; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int TSAN_INTERCEPTOR_SIGSETJMP(void *env); +extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { + CHECK(0); + return 0; +} + +#if !SANITIZER_NETBSD +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +int __interceptor___sigsetjmp(void *env); +extern "C" int __interceptor___sigsetjmp(void *env) { + CHECK(0); + return 0; +} +#endif + +extern "C" int setjmp_symname(void *env); +extern "C" int _setjmp(void *env); +extern "C" int sigsetjmp_symname(void *env); +#if !SANITIZER_NETBSD +extern "C" int __sigsetjmp(void *env); +#endif +DEFINE_REAL(int, setjmp_symname, void *env) +DEFINE_REAL(int, _setjmp, void *env) +DEFINE_REAL(int, sigsetjmp_symname, void *env) +#if !SANITIZER_NETBSD +DEFINE_REAL(int, __sigsetjmp, void *env) +#endif +#endif // SANITIZER_MAC + +#if SANITIZER_NETBSD +#define longjmp_symname __longjmp14 +#define siglongjmp_symname __siglongjmp14 +#else +#define longjmp_symname longjmp +#define siglongjmp_symname siglongjmp +#endif + +TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { + // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, + // bad things will happen. We will jump over ScopedInterceptor dtor and can + // leave thr->in_ignored_lib set. + { + SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val); + } + LongJmp(cur_thread(), env); + REAL(longjmp_symname)(env, val); +} + +TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { + { + SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val); + } + LongJmp(cur_thread(), env); + REAL(siglongjmp_symname)(env, val); +} + +#if SANITIZER_NETBSD +TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { + { + SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); + } + LongJmp(cur_thread(), env); + REAL(_longjmp)(env, val); +} +#endif + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(void*, malloc, uptr size) { + if (in_symbolizer()) + return InternalAlloc(size); + void *p = 0; + { + SCOPED_INTERCEPTOR_RAW(malloc, size); + p = user_alloc(thr, pc, size); + } + invoke_malloc_hook(p, size); + return p; +} + +// In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept +// __libc_memalign so that (1) we can detect races (2) free will not be called +// on libc internally allocated blocks. +TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { + SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz); + return user_memalign(thr, pc, align, sz); +} + +TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { + if (in_symbolizer()) + return InternalCalloc(size, n); + void *p = 0; + { + SCOPED_INTERCEPTOR_RAW(calloc, size, n); + p = user_calloc(thr, pc, size, n); + } + invoke_malloc_hook(p, n * size); + return p; +} + +TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { + if (in_symbolizer()) + return InternalRealloc(p, size); + if (p) + invoke_free_hook(p); + { + SCOPED_INTERCEPTOR_RAW(realloc, p, size); + p = user_realloc(thr, pc, p, size); + } + invoke_malloc_hook(p, size); + return p; +} + +TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { + if (in_symbolizer()) + return InternalReallocArray(p, size, n); + if (p) + invoke_free_hook(p); + { + SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); + p = user_reallocarray(thr, pc, p, size, n); + } + invoke_malloc_hook(p, size); + return p; +} + +TSAN_INTERCEPTOR(void, free, void *p) { + if (p == 0) + return; + if (in_symbolizer()) + return InternalFree(p); + invoke_free_hook(p); + SCOPED_INTERCEPTOR_RAW(free, p); + user_free(thr, pc, p); +} + +TSAN_INTERCEPTOR(void, cfree, void *p) { + if (p == 0) + return; + if (in_symbolizer()) + return InternalFree(p); + invoke_free_hook(p); + SCOPED_INTERCEPTOR_RAW(cfree, p); + user_free(thr, pc, p); +} + +TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { + SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); + return user_alloc_usable_size(p); +} +#endif + +TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) { + SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); + uptr srclen = internal_strlen(src); + MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); + MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); + return REAL(strcpy)(dst, src); +} + +TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { + SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); + uptr srclen = internal_strnlen(src, n); + MemoryAccessRange(thr, pc, (uptr)dst, n, true); + MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); + return REAL(strncpy)(dst, src, n); +} + +TSAN_INTERCEPTOR(char*, strdup, const char *str) { + SCOPED_TSAN_INTERCEPTOR(strdup, str); + // strdup will call malloc, so no instrumentation is required here. + return REAL(strdup)(str); +} + +// Zero out addr if it points into shadow memory and was provided as a hint +// only, i.e., MAP_FIXED is not set. +static bool fix_mmap_addr(void **addr, long_t sz, int flags) { + if (*addr) { + if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { + if (flags & MAP_FIXED) { + errno = errno_EINVAL; + return false; + } else { + *addr = 0; + } + } + } + return true; +} + +template +static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap, + void *addr, SIZE_T sz, int prot, int flags, + int fd, OFF64_T off) { + if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; + void *res = real_mmap(addr, sz, prot, flags, fd, off); + if (res != MAP_FAILED) { + if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) { + Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n", + addr, (void*)sz, res); + Die(); + } + if (fd > 0) FdAccess(thr, pc, fd); + MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz); + } + return res; +} + +TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { + SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); + UnmapShadow(thr, (uptr)addr, sz); + int res = REAL(munmap)(addr, sz); + return res; +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { + SCOPED_INTERCEPTOR_RAW(memalign, align, sz); + return user_memalign(thr, pc, align, sz); +} +#define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) +#else +#define TSAN_MAYBE_INTERCEPT_MEMALIGN +#endif + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { + if (in_symbolizer()) + return InternalAlloc(sz, nullptr, align); + SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); + return user_aligned_alloc(thr, pc, align, sz); +} + +TSAN_INTERCEPTOR(void*, valloc, uptr sz) { + if (in_symbolizer()) + return InternalAlloc(sz, nullptr, GetPageSizeCached()); + SCOPED_INTERCEPTOR_RAW(valloc, sz); + return user_valloc(thr, pc, sz); +} +#endif + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { + if (in_symbolizer()) { + uptr PageSize = GetPageSizeCached(); + sz = sz ? RoundUpTo(sz, PageSize) : PageSize; + return InternalAlloc(sz, nullptr, PageSize); + } + SCOPED_INTERCEPTOR_RAW(pvalloc, sz); + return user_pvalloc(thr, pc, sz); +} +#define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) +#else +#define TSAN_MAYBE_INTERCEPT_PVALLOC +#endif + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { + if (in_symbolizer()) { + void *p = InternalAlloc(sz, nullptr, align); + if (!p) + return errno_ENOMEM; + *memptr = p; + return 0; + } + SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); + return user_posix_memalign(thr, pc, memptr, align, sz); +} +#endif + +// Both __cxa_guard_acquire and pthread_once 0-initialize +// the object initially. pthread_once does not have any +// other ABI requirements. __cxa_guard_acquire assumes +// that any non-0 value in the first byte means that +// initialization is completed. Contents of the remaining +// bytes are up to us. +constexpr u32 kGuardInit = 0; +constexpr u32 kGuardDone = 1; +constexpr u32 kGuardRunning = 1 << 16; +constexpr u32 kGuardWaiter = 1 << 17; + +static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g, + bool blocking_hooks = true) { + if (blocking_hooks) + OnPotentiallyBlockingRegionBegin(); + auto on_exit = at_scope_exit([blocking_hooks] { + if (blocking_hooks) + OnPotentiallyBlockingRegionEnd(); + }); + + for (;;) { + u32 cmp = atomic_load(g, memory_order_acquire); + if (cmp == kGuardInit) { + if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning, + memory_order_relaxed)) + return 1; + } else if (cmp == kGuardDone) { + if (!thr->in_ignored_lib) + Acquire(thr, pc, (uptr)g); + return 0; + } else { + if ((cmp & kGuardWaiter) || + atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter, + memory_order_relaxed)) + FutexWait(g, cmp | kGuardWaiter); + } + } +} + +static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g, + u32 v) { + if (!thr->in_ignored_lib) + Release(thr, pc, (uptr)g); + u32 old = atomic_exchange(g, v, memory_order_release); + if (old & kGuardWaiter) + FutexWake(g, 1 << 30); +} + +// __cxa_guard_acquire and friends need to be intercepted in a special way - +// regular interceptors will break statically-linked libstdc++. Linux +// interceptors are especially defined as weak functions (so that they don't +// cause link errors when user defines them as well). So they silently +// auto-disable themselves when such symbol is already present in the binary. If +// we link libstdc++ statically, it will bring own __cxa_guard_acquire which +// will silently replace our interceptor. That's why on Linux we simply export +// these interceptors with INTERFACE_ATTRIBUTE. +// On OS X, we don't support statically linking, so we just use a regular +// interceptor. +#if SANITIZER_MAC +#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR +#else +#define STDCXX_INTERCEPTOR(rettype, name, ...) \ + extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) +#endif + +// Used in thread-safe function static initialization. +STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { + SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); + return guard_acquire(thr, pc, g); +} + +STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { + SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); + guard_release(thr, pc, g, kGuardDone); +} + +STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { + SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); + guard_release(thr, pc, g, kGuardInit); +} + +namespace __tsan { +void DestroyThreadState() { + ThreadState *thr = cur_thread(); + Processor *proc = thr->proc(); + ThreadFinish(thr); + ProcUnwire(proc, thr); + ProcDestroy(proc); + DTLS_Destroy(); + cur_thread_finalize(); +} + +void PlatformCleanUpThreadState(ThreadState *thr) { + ThreadSignalContext *sctx = thr->signal_ctx; + if (sctx) { + thr->signal_ctx = 0; + UnmapOrDie(sctx, sizeof(*sctx)); + } +} +} // namespace __tsan + +#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD +static void thread_finalize(void *v) { + uptr iter = (uptr)v; + if (iter > 1) { + if (pthread_setspecific(interceptor_ctx()->finalize_key, + (void*)(iter - 1))) { + Printf("ThreadSanitizer: failed to set thread key\n"); + Die(); + } + return; + } + DestroyThreadState(); +} +#endif + + +struct ThreadParam { + void* (*callback)(void *arg); + void *param; + Tid tid; + Semaphore created; + Semaphore started; +}; + +extern "C" void *__tsan_thread_start_func(void *arg) { + ThreadParam *p = (ThreadParam*)arg; + void* (*callback)(void *arg) = p->callback; + void *param = p->param; + { + ThreadState *thr = cur_thread_init(); + // Thread-local state is not initialized yet. + ScopedIgnoreInterceptors ignore; +#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD + ThreadIgnoreBegin(thr, 0); + if (pthread_setspecific(interceptor_ctx()->finalize_key, + (void *)GetPthreadDestructorIterations())) { + Printf("ThreadSanitizer: failed to set thread key\n"); + Die(); + } + ThreadIgnoreEnd(thr); +#endif + p->created.Wait(); + Processor *proc = ProcCreate(); + ProcWire(proc, thr); + ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular); + p->started.Post(); + } + void *res = callback(param); + // Prevent the callback from being tail called, + // it mixes up stack traces. + volatile int foo = 42; + foo++; + return res; +} + +TSAN_INTERCEPTOR(int, pthread_create, + void *th, void *attr, void *(*callback)(void*), void * param) { + SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); + + MaybeSpawnBackgroundThread(); + + if (ctx->after_multithreaded_fork) { + if (flags()->die_after_fork) { + Report("ThreadSanitizer: starting new threads after multi-threaded " + "fork is not supported. Dying (set die_after_fork=0 to override)\n"); + Die(); + } else { + VPrintf(1, + "ThreadSanitizer: starting new threads after multi-threaded " + "fork is not supported (pid %lu). Continuing because of " + "die_after_fork=0, but you are on your own\n", + internal_getpid()); + } + } + __sanitizer_pthread_attr_t myattr; + if (attr == 0) { + pthread_attr_init(&myattr); + attr = &myattr; + } + int detached = 0; + REAL(pthread_attr_getdetachstate)(attr, &detached); + AdjustStackSize(attr); + + ThreadParam p; + p.callback = callback; + p.param = param; + p.tid = kMainTid; + int res = -1; + { + // Otherwise we see false positives in pthread stack manipulation. + ScopedIgnoreInterceptors ignore; + ThreadIgnoreBegin(thr, pc); + res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); + ThreadIgnoreEnd(thr); + } + if (res == 0) { + p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached)); + CHECK_NE(p.tid, kMainTid); + // Synchronization on p.tid serves two purposes: + // 1. ThreadCreate must finish before the new thread starts. + // Otherwise the new thread can call pthread_detach, but the pthread_t + // identifier is not yet registered in ThreadRegistry by ThreadCreate. + // 2. ThreadStart must finish before this thread continues. + // Otherwise, this thread can call pthread_detach and reset thr->sync + // before the new thread got a chance to acquire from it in ThreadStart. + p.created.Post(); + p.started.Wait(); + } + if (attr == &myattr) + pthread_attr_destroy(&myattr); + return res; +} + +TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { + SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); + Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = BLOCK_REAL(pthread_join)(th, ret); + ThreadIgnoreEnd(thr); + if (res == 0) { + ThreadJoin(thr, pc, tid); + } + return res; +} + +DEFINE_REAL_PTHREAD_FUNCTIONS + +TSAN_INTERCEPTOR(int, pthread_detach, void *th) { + SCOPED_INTERCEPTOR_RAW(pthread_detach, th); + Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); + int res = REAL(pthread_detach)(th); + if (res == 0) { + ThreadDetach(thr, pc, tid); + } + return res; +} + +TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { + { + SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(thr, &cur_thread_placeholder); +#endif + } + REAL(pthread_exit)(retval); +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { + SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret); + Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = REAL(pthread_tryjoin_np)(th, ret); + ThreadIgnoreEnd(thr); + if (res == 0) + ThreadJoin(thr, pc, tid); + else + ThreadNotJoined(thr, pc, tid, (uptr)th); + return res; +} + +TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, + const struct timespec *abstime) { + SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime); + Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); + ThreadIgnoreEnd(thr); + if (res == 0) + ThreadJoin(thr, pc, tid); + else + ThreadNotJoined(thr, pc, tid, (uptr)th); + return res; +} +#endif + +// Problem: +// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). +// pthread_cond_t has different size in the different versions. +// If call new REAL functions for old pthread_cond_t, they will corrupt memory +// after pthread_cond_t (old cond is smaller). +// If we call old REAL functions for new pthread_cond_t, we will lose some +// functionality (e.g. old functions do not support waiting against +// CLOCK_REALTIME). +// Proper handling would require to have 2 versions of interceptors as well. +// But this is messy, in particular requires linker scripts when sanitizer +// runtime is linked into a shared library. +// Instead we assume we don't have dynamic libraries built against old +// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag +// that allows to work with old libraries (but this mode does not support +// some features, e.g. pthread_condattr_getpshared). +static void *init_cond(void *c, bool force = false) { + // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. + // So we allocate additional memory on the side large enough to hold + // any pthread_cond_t object. Always call new REAL functions, but pass + // the aux object to them. + // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes + // first word of pthread_cond_t to zero. + // It's all relevant only for linux. + if (!common_flags()->legacy_pthread_cond) + return c; + atomic_uintptr_t *p = (atomic_uintptr_t*)c; + uptr cond = atomic_load(p, memory_order_acquire); + if (!force && cond != 0) + return (void*)cond; + void *newcond = WRAP(malloc)(pthread_cond_t_sz); + internal_memset(newcond, 0, pthread_cond_t_sz); + if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, + memory_order_acq_rel)) + return newcond; + WRAP(free)(newcond); + return (void*)cond; +} + +namespace { + +template +struct CondMutexUnlockCtx { + ScopedInterceptor *si; + ThreadState *thr; + uptr pc; + void *m; + void *c; + const Fn &fn; + + int Cancel() const { return fn(); } + void Unlock() const; +}; + +template +void CondMutexUnlockCtx::Unlock() const { + // pthread_cond_wait interceptor has enabled async signal delivery + // (see BlockingCall below). Disable async signals since we are running + // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run + // since the thread is cancelled, so we have to manually execute them + // (the thread still can run some user code due to pthread_cleanup_push). + ThreadSignalContext *ctx = SigCtx(thr); + CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1); + atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); + MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); + // Undo BlockingCall ctor effects. + thr->ignore_interceptors--; + si->~ScopedInterceptor(); +} +} // namespace + +INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { + void *cond = init_cond(c, true); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); + return REAL(pthread_cond_init)(cond, a); +} + +template +int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn, + void *c, void *m) { + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + MutexUnlock(thr, pc, (uptr)m); + int res = 0; + // This ensures that we handle mutex lock even in case of pthread_cancel. + // See test/tsan/cond_cancel.cpp. + { + // Enable signal delivery while the thread is blocked. + BlockingCall bc(thr); + CondMutexUnlockCtx arg = {si, thr, pc, m, c, fn}; + res = call_pthread_cancel_with_cleanup( + [](void *arg) -> int { + return ((const CondMutexUnlockCtx *)arg)->Cancel(); + }, + [](void *arg) { ((const CondMutexUnlockCtx *)arg)->Unlock(); }, + &arg); + } + if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); + MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); + return res; +} + +INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); + return cond_wait( + thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond, + m); +} + +INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); + return cond_wait( + thr, pc, &si, + [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond, + m); +} + +#if SANITIZER_LINUX +INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m, + __sanitizer_clockid_t clock, void *abstime) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime); + return cond_wait( + thr, pc, &si, + [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); }, + cond, m); +} +#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait) +#else +#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT +#endif + +#if SANITIZER_MAC +INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, + void *reltime) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); + return cond_wait( + thr, pc, &si, + [=]() { + return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime); + }, + cond, m); +} +#endif + +INTERCEPTOR(int, pthread_cond_signal, void *c) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + return REAL(pthread_cond_signal)(cond); +} + +INTERCEPTOR(int, pthread_cond_broadcast, void *c) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + return REAL(pthread_cond_broadcast)(cond); +} + +INTERCEPTOR(int, pthread_cond_destroy, void *c) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); + int res = REAL(pthread_cond_destroy)(cond); + if (common_flags()->legacy_pthread_cond) { + // Free our aux cond and zero the pointer to not leave dangling pointers. + WRAP(free)(cond); + atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { + SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); + int res = REAL(pthread_mutex_init)(m, a); + if (res == 0) { + u32 flagz = 0; + if (a) { + int type = 0; + if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) + if (type == PTHREAD_MUTEX_RECURSIVE || + type == PTHREAD_MUTEX_RECURSIVE_NP) + flagz |= MutexFlagWriteReentrant; + } + MutexCreate(thr, pc, (uptr)m, flagz); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); + int res = REAL(pthread_mutex_destroy)(m); + if (res == 0 || res == errno_EBUSY) { + MutexDestroy(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); + int res = REAL(pthread_mutex_trylock)(m); + if (res == errno_EOWNERDEAD) + MutexRepair(thr, pc, (uptr)m); + if (res == 0 || res == errno_EOWNERDEAD) + MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); + return res; +} + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { + SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); + int res = REAL(pthread_mutex_timedlock)(m, abstime); + if (res == 0) { + MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); + } + return res; +} +#endif + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { + SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); + int res = REAL(pthread_spin_init)(m, pshared); + if (res == 0) { + MutexCreate(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); + int res = REAL(pthread_spin_destroy)(m); + if (res == 0) { + MutexDestroy(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); + MutexPreLock(thr, pc, (uptr)m); + int res = REAL(pthread_spin_lock)(m); + if (res == 0) { + MutexPostLock(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); + int res = REAL(pthread_spin_trylock)(m); + if (res == 0) { + MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); + MutexUnlock(thr, pc, (uptr)m); + int res = REAL(pthread_spin_unlock)(m); + return res; +} +#endif + +TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); + int res = REAL(pthread_rwlock_init)(m, a); + if (res == 0) { + MutexCreate(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); + int res = REAL(pthread_rwlock_destroy)(m); + if (res == 0) { + MutexDestroy(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); + MutexPreReadLock(thr, pc, (uptr)m); + int res = REAL(pthread_rwlock_rdlock)(m); + if (res == 0) { + MutexPostReadLock(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); + int res = REAL(pthread_rwlock_tryrdlock)(m); + if (res == 0) { + MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock); + } + return res; +} + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); + int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); + if (res == 0) { + MutexPostReadLock(thr, pc, (uptr)m); + } + return res; +} +#endif + +TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); + MutexPreLock(thr, pc, (uptr)m); + int res = REAL(pthread_rwlock_wrlock)(m); + if (res == 0) { + MutexPostLock(thr, pc, (uptr)m); + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); + int res = REAL(pthread_rwlock_trywrlock)(m); + if (res == 0) { + MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); + } + return res; +} + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); + int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); + if (res == 0) { + MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); + } + return res; +} +#endif + +TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { + SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); + MutexReadOrWriteUnlock(thr, pc, (uptr)m); + int res = REAL(pthread_rwlock_unlock)(m); + return res; +} + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { + SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); + MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); + int res = REAL(pthread_barrier_init)(b, a, count); + return res; +} + +TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { + SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); + MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); + int res = REAL(pthread_barrier_destroy)(b); + return res; +} + +TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { + SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); + Release(thr, pc, (uptr)b); + MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); + int res = REAL(pthread_barrier_wait)(b); + MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); + if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { + Acquire(thr, pc, (uptr)b); + } + return res; +} +#endif + +TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { + SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); + if (o == 0 || f == 0) + return errno_EINVAL; + atomic_uint32_t *a; + + if (SANITIZER_MAC) + a = static_cast((void *)((char *)o + sizeof(long_t))); + else if (SANITIZER_NETBSD) + a = static_cast + ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz)); + else + a = static_cast(o); + + // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks + // result in crashes due to too little stack space. + if (guard_acquire(thr, pc, a, !SANITIZER_MAC)) { + (*f)(); + guard_release(thr, pc, a, kGuardDone); + } + return 0; +} + +#if SANITIZER_LINUX && !SANITIZER_ANDROID +TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { + SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); + if (fd > 0) + FdAccess(thr, pc, fd); + return REAL(__fxstat)(version, fd, buf); +} +#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) +#else +#define TSAN_MAYBE_INTERCEPT___FXSTAT +#endif + +TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD + SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); + if (fd > 0) + FdAccess(thr, pc, fd); + return REAL(fstat)(fd, buf); +#else + SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); + if (fd > 0) + FdAccess(thr, pc, fd); + return REAL(__fxstat)(0, fd, buf); +#endif +} + +#if SANITIZER_LINUX && !SANITIZER_ANDROID +TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { + SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); + if (fd > 0) + FdAccess(thr, pc, fd); + return REAL(__fxstat64)(version, fd, buf); +} +#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) +#else +#define TSAN_MAYBE_INTERCEPT___FXSTAT64 +#endif + +#if SANITIZER_LINUX && !SANITIZER_ANDROID +TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { + SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); + if (fd > 0) + FdAccess(thr, pc, fd); + return REAL(__fxstat64)(0, fd, buf); +} +#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) +#else +#define TSAN_MAYBE_INTERCEPT_FSTAT64 +#endif + +TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) { + va_list ap; + va_start(ap, oflag); + mode_t mode = va_arg(ap, int); + va_end(ap); + SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode); + READ_STRING(thr, pc, name, 0); + int fd = REAL(open)(name, oflag, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) { + va_list ap; + va_start(ap, oflag); + mode_t mode = va_arg(ap, int); + va_end(ap); + SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode); + READ_STRING(thr, pc, name, 0); + int fd = REAL(open64)(name, oflag, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} +#define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) +#else +#define TSAN_MAYBE_INTERCEPT_OPEN64 +#endif + +TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { + SCOPED_TSAN_INTERCEPTOR(creat, name, mode); + READ_STRING(thr, pc, name, 0); + int fd = REAL(creat)(name, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { + SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); + READ_STRING(thr, pc, name, 0); + int fd = REAL(creat64)(name, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} +#define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) +#else +#define TSAN_MAYBE_INTERCEPT_CREAT64 +#endif + +TSAN_INTERCEPTOR(int, dup, int oldfd) { + SCOPED_TSAN_INTERCEPTOR(dup, oldfd); + int newfd = REAL(dup)(oldfd); + if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) + FdDup(thr, pc, oldfd, newfd, true); + return newfd; +} + +TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { + SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); + int newfd2 = REAL(dup2)(oldfd, newfd); + if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) + FdDup(thr, pc, oldfd, newfd2, false); + return newfd2; +} + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { + SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); + int newfd2 = REAL(dup3)(oldfd, newfd, flags); + if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) + FdDup(thr, pc, oldfd, newfd2, false); + return newfd2; +} +#endif + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { + SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); + int fd = REAL(eventfd)(initval, flags); + if (fd >= 0) + FdEventCreate(thr, pc, fd); + return fd; +} +#define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) +#else +#define TSAN_MAYBE_INTERCEPT_EVENTFD +#endif + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { + SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); + if (fd >= 0) + FdClose(thr, pc, fd); + fd = REAL(signalfd)(fd, mask, flags); + if (fd >= 0) + FdSignalCreate(thr, pc, fd); + return fd; +} +#define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) +#else +#define TSAN_MAYBE_INTERCEPT_SIGNALFD +#endif + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, inotify_init, int fake) { + SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); + int fd = REAL(inotify_init)(fake); + if (fd >= 0) + FdInotifyCreate(thr, pc, fd); + return fd; +} +#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) +#else +#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT +#endif + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, inotify_init1, int flags) { + SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); + int fd = REAL(inotify_init1)(flags); + if (fd >= 0) + FdInotifyCreate(thr, pc, fd); + return fd; +} +#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) +#else +#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 +#endif + +TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { + SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); + int fd = REAL(socket)(domain, type, protocol); + if (fd >= 0) + FdSocketCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { + SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); + int res = REAL(socketpair)(domain, type, protocol, fd); + if (res == 0 && fd[0] >= 0 && fd[1] >= 0) + FdPipeCreate(thr, pc, fd[0], fd[1]); + return res; +} + +TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { + SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); + FdSocketConnecting(thr, pc, fd); + int res = REAL(connect)(fd, addr, addrlen); + if (res == 0 && fd >= 0) + FdSocketConnect(thr, pc, fd); + return res; +} + +TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { + SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); + int res = REAL(bind)(fd, addr, addrlen); + if (fd > 0 && res == 0) + FdAccess(thr, pc, fd); + return res; +} + +TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { + SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); + int res = REAL(listen)(fd, backlog); + if (fd > 0 && res == 0) + FdAccess(thr, pc, fd); + return res; +} + +TSAN_INTERCEPTOR(int, close, int fd) { + SCOPED_TSAN_INTERCEPTOR(close, fd); + if (fd >= 0) + FdClose(thr, pc, fd); + return REAL(close)(fd); +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, __close, int fd) { + SCOPED_TSAN_INTERCEPTOR(__close, fd); + if (fd >= 0) + FdClose(thr, pc, fd); + return REAL(__close)(fd); +} +#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) +#else +#define TSAN_MAYBE_INTERCEPT___CLOSE +#endif + +// glibc guts +#if SANITIZER_LINUX && !SANITIZER_ANDROID +TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { + SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); + int fds[64]; + int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); + for (int i = 0; i < cnt; i++) { + if (fds[i] > 0) + FdClose(thr, pc, fds[i]); + } + REAL(__res_iclose)(state, free_addr); +} +#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) +#else +#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE +#endif + +TSAN_INTERCEPTOR(int, pipe, int *pipefd) { + SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); + int res = REAL(pipe)(pipefd); + if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) + FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); + return res; +} + +#if !SANITIZER_MAC +TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { + SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); + int res = REAL(pipe2)(pipefd, flags); + if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) + FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); + return res; +} +#endif + +TSAN_INTERCEPTOR(int, unlink, char *path) { + SCOPED_TSAN_INTERCEPTOR(unlink, path); + Release(thr, pc, File2addr(path)); + int res = REAL(unlink)(path); + return res; +} + +TSAN_INTERCEPTOR(void*, tmpfile, int fake) { + SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); + void *res = REAL(tmpfile)(fake); + if (res) { + int fd = fileno_unlocked(res); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + } + return res; +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { + SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); + void *res = REAL(tmpfile64)(fake); + if (res) { + int fd = fileno_unlocked(res); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + } + return res; +} +#define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) +#else +#define TSAN_MAYBE_INTERCEPT_TMPFILE64 +#endif + +static void FlushStreams() { + // Flushing all the streams here may freeze the process if a child thread is + // performing file stream operations at the same time. + REAL(fflush)(stdout); + REAL(fflush)(stderr); +} + +TSAN_INTERCEPTOR(void, abort, int fake) { + SCOPED_TSAN_INTERCEPTOR(abort, fake); + FlushStreams(); + REAL(abort)(fake); +} + +TSAN_INTERCEPTOR(int, rmdir, char *path) { + SCOPED_TSAN_INTERCEPTOR(rmdir, path); + Release(thr, pc, Dir2addr(path)); + int res = REAL(rmdir)(path); + return res; +} + +TSAN_INTERCEPTOR(int, closedir, void *dirp) { + SCOPED_TSAN_INTERCEPTOR(closedir, dirp); + if (dirp) { + int fd = dirfd(dirp); + FdClose(thr, pc, fd); + } + return REAL(closedir)(dirp); +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, epoll_create, int size) { + SCOPED_TSAN_INTERCEPTOR(epoll_create, size); + int fd = REAL(epoll_create)(size); + if (fd >= 0) + FdPollCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, epoll_create1, int flags) { + SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); + int fd = REAL(epoll_create1)(flags); + if (fd >= 0) + FdPollCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { + SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); + if (epfd >= 0) + FdAccess(thr, pc, epfd); + if (epfd >= 0 && fd >= 0) + FdAccess(thr, pc, fd); + if (op == EPOLL_CTL_ADD && epfd >= 0) + FdRelease(thr, pc, epfd); + int res = REAL(epoll_ctl)(epfd, op, fd, ev); + return res; +} + +TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { + SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); + if (epfd >= 0) + FdAccess(thr, pc, epfd); + int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); + if (res > 0 && epfd >= 0) + FdAcquire(thr, pc, epfd); + return res; +} + +TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, + void *sigmask) { + SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask); + if (epfd >= 0) + FdAccess(thr, pc, epfd); + int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask); + if (res > 0 && epfd >= 0) + FdAcquire(thr, pc, epfd); + return res; +} + +#define TSAN_MAYBE_INTERCEPT_EPOLL \ + TSAN_INTERCEPT(epoll_create); \ + TSAN_INTERCEPT(epoll_create1); \ + TSAN_INTERCEPT(epoll_ctl); \ + TSAN_INTERCEPT(epoll_wait); \ + TSAN_INTERCEPT(epoll_pwait) +#else +#define TSAN_MAYBE_INTERCEPT_EPOLL +#endif + +// The following functions are intercepted merely to process pending signals. +// If program blocks signal X, we must deliver the signal before the function +// returns. Similarly, if program unblocks a signal (or returns from sigsuspend) +// it's better to deliver the signal straight away. +TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { + SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); + return REAL(sigsuspend)(mask); +} + +TSAN_INTERCEPTOR(int, sigblock, int mask) { + SCOPED_TSAN_INTERCEPTOR(sigblock, mask); + return REAL(sigblock)(mask); +} + +TSAN_INTERCEPTOR(int, sigsetmask, int mask) { + SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask); + return REAL(sigsetmask)(mask); +} + +TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); + return REAL(pthread_sigmask)(how, set, oldset); +} + +namespace __tsan { + +static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) { + VarSizeStackTrace stack; + // StackTrace::GetNestInstructionPc(pc) is used because return address is + // expected, OutputReport() will undo this. + ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); + ThreadRegistryLock l(&ctx->thread_registry); + ScopedReport rep(ReportTypeErrnoInSignal); + if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { + rep.AddStack(stack, true); + OutputReport(thr, rep); + } +} + +static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, + int sig, __sanitizer_siginfo *info, + void *uctx) { + __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; + if (acquire) + Acquire(thr, 0, (uptr)&sigactions[sig]); + // Signals are generally asynchronous, so if we receive a signals when + // ignores are enabled we should disable ignores. This is critical for sync + // and interceptors, because otherwise we can miss synchronization and report + // false races. + int ignore_reads_and_writes = thr->ignore_reads_and_writes; + int ignore_interceptors = thr->ignore_interceptors; + int ignore_sync = thr->ignore_sync; + // For symbolizer we only process SIGSEGVs synchronously + // (bug in symbolizer or in tsan). But we want to reset + // in_symbolizer to fail gracefully. Symbolizer and user code + // use different memory allocators, so if we don't reset + // in_symbolizer we can get memory allocated with one being + // feed with another, which can cause more crashes. + int in_symbolizer = thr->in_symbolizer; + if (!ctx->after_multithreaded_fork) { + thr->ignore_reads_and_writes = 0; + thr->fast_state.ClearIgnoreBit(); + thr->ignore_interceptors = 0; + thr->ignore_sync = 0; + thr->in_symbolizer = 0; + } + // Ensure that the handler does not spoil errno. + const int saved_errno = errno; + errno = 99; + // This code races with sigaction. Be careful to not read sa_sigaction twice. + // Also need to remember pc for reporting before the call, + // because the handler can reset it. + volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO) + ? (uptr)sigactions[sig].sigaction + : (uptr)sigactions[sig].handler; + if (pc != sig_dfl && pc != sig_ign) { + // The callback can be either sa_handler or sa_sigaction. + // They have different signatures, but we assume that passing + // additional arguments to sa_handler works and is harmless. + ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); + } + if (!ctx->after_multithreaded_fork) { + thr->ignore_reads_and_writes = ignore_reads_and_writes; + if (ignore_reads_and_writes) + thr->fast_state.SetIgnoreBit(); + thr->ignore_interceptors = ignore_interceptors; + thr->ignore_sync = ignore_sync; + thr->in_symbolizer = in_symbolizer; + } + // We do not detect errno spoiling for SIGTERM, + // because some SIGTERM handlers do spoil errno but reraise SIGTERM, + // tsan reports false positive in such case. + // It's difficult to properly detect this situation (reraise), + // because in async signal processing case (when handler is called directly + // from rtl_generic_sighandler) we have not yet received the reraised + // signal; and it looks too fragile to intercept all ways to reraise a signal. + if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM && + errno != 99) + ReportErrnoSpoiling(thr, pc); + errno = saved_errno; +} + +void ProcessPendingSignalsImpl(ThreadState *thr) { + atomic_store(&thr->pending_signals, 0, memory_order_relaxed); + ThreadSignalContext *sctx = SigCtx(thr); + if (sctx == 0) + return; + atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); + internal_sigfillset(&sctx->emptyset); + int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset); + CHECK_EQ(res, 0); + for (int sig = 0; sig < kSigCount; sig++) { + SignalDesc *signal = &sctx->pending_signals[sig]; + if (signal->armed) { + signal->armed = false; + CallUserSignalHandler(thr, false, true, sig, &signal->siginfo, + &signal->ctx); + } + } + res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); + CHECK_EQ(res, 0); + atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); +} + +} // namespace __tsan + +static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { + return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || + sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || + // If we are sending signal to ourselves, we must process it now. + (sctx && sig == sctx->int_signal_send); +} + +void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { + ThreadState *thr = cur_thread_init(); + ThreadSignalContext *sctx = SigCtx(thr); + if (sig < 0 || sig >= kSigCount) { + VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); + return; + } + // Don't mess with synchronous signals. + const bool sync = is_sync_signal(sctx, sig); + if (sync || + // If we are in blocking function, we can safely process it now + // (but check if we are in a recursive interceptor, + // i.e. pthread_join()->munmap()). + (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { + atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); + if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { + atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); + CallUserSignalHandler(thr, sync, true, sig, info, ctx); + atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); + } else { + // Be very conservative with when we do acquire in this case. + // It's unsafe to do acquire in async handlers, because ThreadState + // can be in inconsistent state. + // SIGSYS looks relatively safe -- it's synchronous and can actually + // need some global state. + bool acq = (sig == SIGSYS); + CallUserSignalHandler(thr, sync, acq, sig, info, ctx); + } + atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); + return; + } + + if (sctx == 0) + return; + SignalDesc *signal = &sctx->pending_signals[sig]; + if (signal->armed == false) { + signal->armed = true; + internal_memcpy(&signal->siginfo, info, sizeof(*info)); + internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); + atomic_store(&thr->pending_signals, 1, memory_order_relaxed); + } +} + +TSAN_INTERCEPTOR(int, raise, int sig) { + SCOPED_TSAN_INTERCEPTOR(raise, sig); + ThreadSignalContext *sctx = SigCtx(thr); + CHECK_NE(sctx, 0); + int prev = sctx->int_signal_send; + sctx->int_signal_send = sig; + int res = REAL(raise)(sig); + CHECK_EQ(sctx->int_signal_send, sig); + sctx->int_signal_send = prev; + return res; +} + +TSAN_INTERCEPTOR(int, kill, int pid, int sig) { + SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); + ThreadSignalContext *sctx = SigCtx(thr); + CHECK_NE(sctx, 0); + int prev = sctx->int_signal_send; + if (pid == (int)internal_getpid()) { + sctx->int_signal_send = sig; + } + int res = REAL(kill)(pid, sig); + if (pid == (int)internal_getpid()) { + CHECK_EQ(sctx->int_signal_send, sig); + sctx->int_signal_send = prev; + } + return res; +} + +TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { + SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); + ThreadSignalContext *sctx = SigCtx(thr); + CHECK_NE(sctx, 0); + int prev = sctx->int_signal_send; + bool self = pthread_equal(tid, pthread_self()); + if (self) + sctx->int_signal_send = sig; + int res = REAL(pthread_kill)(tid, sig); + if (self) { + CHECK_EQ(sctx->int_signal_send, sig); + sctx->int_signal_send = prev; + } + return res; +} + +TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { + SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); + // It's intercepted merely to process pending signals. + return REAL(gettimeofday)(tv, tz); +} + +TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, + void *hints, void *rv) { + SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); + // We miss atomic synchronization in getaddrinfo, + // and can report false race between malloc and free + // inside of getaddrinfo. So ignore memory accesses. + ThreadIgnoreBegin(thr, pc); + int res = REAL(getaddrinfo)(node, service, hints, rv); + ThreadIgnoreEnd(thr); + return res; +} + +TSAN_INTERCEPTOR(int, fork, int fake) { + if (in_symbolizer()) + return REAL(fork)(fake); + SCOPED_INTERCEPTOR_RAW(fork, fake); + return REAL(fork)(fake); +} + +void atfork_prepare() { + if (in_symbolizer()) + return; + ThreadState *thr = cur_thread(); + const uptr pc = StackTrace::GetCurrentPc(); + ForkBefore(thr, pc); +} + +void atfork_parent() { + if (in_symbolizer()) + return; + ThreadState *thr = cur_thread(); + const uptr pc = StackTrace::GetCurrentPc(); + ForkParentAfter(thr, pc); +} + +void atfork_child() { + if (in_symbolizer()) + return; + ThreadState *thr = cur_thread(); + const uptr pc = StackTrace::GetCurrentPc(); + ForkChildAfter(thr, pc, true); + FdOnFork(thr, pc); +} + +TSAN_INTERCEPTOR(int, vfork, int fake) { + // Some programs (e.g. openjdk) call close for all file descriptors + // in the child process. Under tsan it leads to false positives, because + // address space is shared, so the parent process also thinks that + // the descriptors are closed (while they are actually not). + // This leads to false positives due to missed synchronization. + // Strictly saying this is undefined behavior, because vfork child is not + // allowed to call any functions other than exec/exit. But this is what + // openjdk does, so we want to handle it. + // We could disable interceptors in the child process. But it's not possible + // to simply intercept and wrap vfork, because vfork child is not allowed + // to return from the function that calls vfork, and that's exactly what + // we would do. So this would require some assembly trickery as well. + // Instead we simply turn vfork into fork. + return WRAP(fork)(fake); +} + +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags, + void *arg, int *parent_tid, void *tls, pid_t *child_tid) { + SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls, + child_tid); + struct Arg { + int (*fn)(void *); + void *arg; + }; + auto wrapper = +[](void *p) -> int { + auto *thr = cur_thread(); + uptr pc = GET_CURRENT_PC(); + // Start the background thread for fork, but not for clone. + // For fork we did this always and it's known to work (or user code has + // adopted). But if we do this for the new clone interceptor some code + // (sandbox2) fails. So model we used to do for years and don't start the + // background thread after clone. + ForkChildAfter(thr, pc, false); + FdOnFork(thr, pc); + auto *arg = static_cast(p); + return arg->fn(arg->arg); + }; + ForkBefore(thr, pc); + Arg arg_wrapper = {fn, arg}; + int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls, + child_tid); + ForkParentAfter(thr, pc); + return pid; +} +#endif + +#if !SANITIZER_MAC && !SANITIZER_ANDROID +typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, + void *data); +struct dl_iterate_phdr_data { + ThreadState *thr; + uptr pc; + dl_iterate_phdr_cb_t cb; + void *data; +}; + +static bool IsAppNotRodata(uptr addr) { + return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata; +} + +static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, + void *data) { + dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data; + // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later + // accessible in dl_iterate_phdr callback. But we don't see synchronization + // inside of dynamic linker, so we "unpoison" it here in order to not + // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough + // because some libc functions call __libc_dlopen. + if (info && IsAppNotRodata((uptr)info->dlpi_name)) + MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, + internal_strlen(info->dlpi_name)); + int res = cbdata->cb(info, size, cbdata->data); + // Perform the check one more time in case info->dlpi_name was overwritten + // by user callback. + if (info && IsAppNotRodata((uptr)info->dlpi_name)) + MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, + internal_strlen(info->dlpi_name)); + return res; +} + +TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { + SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data); + dl_iterate_phdr_data cbdata; + cbdata.thr = thr; + cbdata.pc = pc; + cbdata.cb = cb; + cbdata.data = data; + int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata); + return res; +} +#endif + +static int OnExit(ThreadState *thr) { + int status = Finalize(thr); + FlushStreams(); + return status; +} + +struct TsanInterceptorContext { + ThreadState *thr; + const uptr pc; +}; + +#if !SANITIZER_MAC +static void HandleRecvmsg(ThreadState *thr, uptr pc, + __sanitizer_msghdr *msg) { + int fds[64]; + int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); + for (int i = 0; i < cnt; i++) + FdEventCreate(thr, pc, fds[i]); +} +#endif + +#include "sanitizer_common/sanitizer_platform_interceptors.h" +// Causes interceptor recursion (getaddrinfo() and fopen()) +#undef SANITIZER_INTERCEPT_GETADDRINFO +// We define our own. +#if SANITIZER_INTERCEPT_TLS_GET_ADDR +#define NEED_TLS_GET_ADDR +#endif +#undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1 +#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK + +#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ + INTERCEPT_FUNCTION_VER(name, ver) +#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ + (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name)) + +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ + true) + +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ + ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ + false) + +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ + TsanInterceptorContext _ctx = {thr, pc}; \ + ctx = (void *)&_ctx; \ + (void)ctx; + +#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ + SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ + TsanInterceptorContext _ctx = {thr, pc}; \ + ctx = (void *)&_ctx; \ + (void)ctx; + +#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ + if (path) \ + Acquire(thr, pc, File2addr(path)); \ + if (file) { \ + int fd = fileno_unlocked(file); \ + if (fd >= 0) FdFileCreate(thr, pc, fd); \ + } + +#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ + if (file) { \ + int fd = fileno_unlocked(file); \ + if (fd >= 0) FdClose(thr, pc, fd); \ + } + +#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ + ({ \ + CheckNoDeepBind(filename, flag); \ + ThreadIgnoreBegin(thr, 0); \ + void *res = REAL(dlopen)(filename, flag); \ + ThreadIgnoreEnd(thr); \ + res; \ + }) + +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ + libignore()->OnLibraryLoaded(filename) + +#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ + libignore()->OnLibraryUnloaded() + +#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ + Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u) + +#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ + Release(((TsanInterceptorContext *) ctx)->thr, pc, u) + +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) + +#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) + +#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) + +#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ + FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) + +#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) + +#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ + ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) + +#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ + if (pthread_equal(pthread_self(), reinterpret_cast(thread))) \ + COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \ + else \ + __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name) + +#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) + +#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ + OnExit(((TsanInterceptorContext *) ctx)->thr) + +#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \ + MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)m) + +#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \ + MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)m) + +#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ + MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)m) + +#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ + MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)m) + +#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \ + MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)m) + +#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ + off) \ + do { \ + return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \ + off); \ + } while (false) + +#if !SANITIZER_MAC +#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ + HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, msg) +#endif + +#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ + if (TsanThread *t = GetCurrentThread()) { \ + *begin = t->tls_begin(); \ + *end = t->tls_end(); \ + } else { \ + *begin = *end = 0; \ + } + +#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() + +#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() + +#include "sanitizer_common/sanitizer_common_interceptors.inc" + +static int sigaction_impl(int sig, const __sanitizer_sigaction *act, + __sanitizer_sigaction *old); +static __sanitizer_sighandler_ptr signal_impl(int sig, + __sanitizer_sighandler_ptr h); + +#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ + { return sigaction_impl(signo, act, oldact); } + +#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ + { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } + +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +int sigaction_impl(int sig, const __sanitizer_sigaction *act, + __sanitizer_sigaction *old) { + // Note: if we call REAL(sigaction) directly for any reason without proxying + // the signal handler through sighandler, very bad things will happen. + // The handler will run synchronously and corrupt tsan per-thread state. + SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); + if (sig <= 0 || sig >= kSigCount) { + errno = errno_EINVAL; + return -1; + } + __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; + __sanitizer_sigaction old_stored; + if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored)); + __sanitizer_sigaction newact; + if (act) { + // Copy act into sigactions[sig]. + // Can't use struct copy, because compiler can emit call to memcpy. + // Can't use internal_memcpy, because it copies byte-by-byte, + // and signal handler reads the handler concurrently. It it can read + // some bytes from old value and some bytes from new value. + // Use volatile to prevent insertion of memcpy. + sigactions[sig].handler = + *(volatile __sanitizer_sighandler_ptr const *)&act->handler; + sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; + internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, + sizeof(sigactions[sig].sa_mask)); +#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD + sigactions[sig].sa_restorer = act->sa_restorer; +#endif + internal_memcpy(&newact, act, sizeof(newact)); + internal_sigfillset(&newact.sa_mask); + if ((act->sa_flags & SA_SIGINFO) || + ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) { + newact.sa_flags |= SA_SIGINFO; + newact.sigaction = sighandler; + } + ReleaseStore(thr, pc, (uptr)&sigactions[sig]); + act = &newact; + } + int res = REAL(sigaction)(sig, act, old); + if (res == 0 && old && old->sigaction == sighandler) + internal_memcpy(old, &old_stored, sizeof(*old)); + return res; +} + +static __sanitizer_sighandler_ptr signal_impl(int sig, + __sanitizer_sighandler_ptr h) { + __sanitizer_sigaction act; + act.handler = h; + internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); + act.sa_flags = 0; + __sanitizer_sigaction old; + int res = sigaction_symname(sig, &act, &old); + if (res) return (__sanitizer_sighandler_ptr)sig_err; + return old.handler; +} + +#define TSAN_SYSCALL() \ + ThreadState *thr = cur_thread(); \ + if (thr->ignore_interceptors) \ + return; \ + ScopedSyscall scoped_syscall(thr) + +struct ScopedSyscall { + ThreadState *thr; + + explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); } + + ~ScopedSyscall() { + ProcessPendingSignals(thr); + } +}; + +#if !SANITIZER_FREEBSD && !SANITIZER_MAC +static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { + TSAN_SYSCALL(); + MemoryAccessRange(thr, pc, p, s, write); +} + +static USED void syscall_acquire(uptr pc, uptr addr) { + TSAN_SYSCALL(); + Acquire(thr, pc, addr); + DPrintf("syscall_acquire(0x%zx))\n", addr); +} + +static USED void syscall_release(uptr pc, uptr addr) { + TSAN_SYSCALL(); + DPrintf("syscall_release(0x%zx)\n", addr); + Release(thr, pc, addr); +} + +static void syscall_fd_close(uptr pc, int fd) { + TSAN_SYSCALL(); + FdClose(thr, pc, fd); +} + +static USED void syscall_fd_acquire(uptr pc, int fd) { + TSAN_SYSCALL(); + FdAcquire(thr, pc, fd); + DPrintf("syscall_fd_acquire(%d)\n", fd); +} + +static USED void syscall_fd_release(uptr pc, int fd) { + TSAN_SYSCALL(); + DPrintf("syscall_fd_release(%d)\n", fd); + FdRelease(thr, pc, fd); +} + +static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); } + +static void syscall_post_fork(uptr pc, int pid) { + ThreadState *thr = cur_thread(); + if (pid == 0) { + // child + ForkChildAfter(thr, pc, true); + FdOnFork(thr, pc); + } else if (pid > 0) { + // parent + ForkParentAfter(thr, pc); + } else { + // error + ForkParentAfter(thr, pc); + } +} +#endif + +#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ + syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) + +#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ + syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) + +#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) + +#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) + +#define COMMON_SYSCALL_ACQUIRE(addr) \ + syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) + +#define COMMON_SYSCALL_RELEASE(addr) \ + syscall_release(GET_CALLER_PC(), (uptr)(addr)) + +#define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) + +#define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) + +#define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) + +#define COMMON_SYSCALL_PRE_FORK() \ + syscall_pre_fork(GET_CALLER_PC()) + +#define COMMON_SYSCALL_POST_FORK(res) \ + syscall_post_fork(GET_CALLER_PC(), res) + +#include "sanitizer_common/sanitizer_common_syscalls.inc" +#include "sanitizer_common/sanitizer_syscalls_netbsd.inc" + +#ifdef NEED_TLS_GET_ADDR + +static void handle_tls_addr(void *arg, void *res) { + ThreadState *thr = cur_thread(); + if (!thr) + return; + DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, + thr->tls_addr + thr->tls_size); + if (!dtv) + return; + // New DTLS block has been allocated. + MemoryResetRange(thr, 0, dtv->beg, dtv->size); +} + +#if !SANITIZER_S390 +// Define own interceptor instead of sanitizer_common's for three reasons: +// 1. It must not process pending signals. +// Signal handlers may contain MOVDQA instruction (see below). +// 2. It must be as simple as possible to not contain MOVDQA. +// 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which +// is empty for tsan (meant only for msan). +// Note: __tls_get_addr can be called with mis-aligned stack due to: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 +// So the interceptor must work with mis-aligned stack, in particular, does not +// execute MOVDQA with stack addresses. +TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { + void *res = REAL(__tls_get_addr)(arg); + handle_tls_addr(arg, res); + return res; +} +#else // SANITIZER_S390 +TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { + uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset)); + char *tp = static_cast(__builtin_thread_pointer()); + handle_tls_addr(arg, res + tp); + return res; +} +#endif +#endif + +#if SANITIZER_NETBSD +TSAN_INTERCEPTOR(void, _lwp_exit) { + SCOPED_TSAN_INTERCEPTOR(_lwp_exit); + DestroyThreadState(); + REAL(_lwp_exit)(); +} +#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit) +#else +#define TSAN_MAYBE_INTERCEPT__LWP_EXIT +#endif + +#if SANITIZER_FREEBSD +TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) { + SCOPED_TSAN_INTERCEPTOR(thr_exit, state); + DestroyThreadState(); + REAL(thr_exit(state)); +} +#define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit) +#else +#define TSAN_MAYBE_INTERCEPT_THR_EXIT +#endif + +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m) +TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()) +TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b, + void *c) + +namespace __tsan { + +static void finalize(void *arg) { + ThreadState *thr = cur_thread(); + int status = Finalize(thr); + // Make sure the output is not lost. + FlushStreams(); + if (status) + Die(); +} + +#if !SANITIZER_MAC && !SANITIZER_ANDROID +static void unreachable() { + Report("FATAL: ThreadSanitizer: unreachable called\n"); + Die(); +} +#endif + +// Define default implementation since interception of libdispatch is optional. +SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} + +void InitializeInterceptors() { +#if !SANITIZER_MAC + // We need to setup it early, because functions like dlsym() can call it. + REAL(memset) = internal_memset; + REAL(memcpy) = internal_memcpy; +#endif + + new(interceptor_ctx()) InterceptorContext(); + + InitializeCommonInterceptors(); + InitializeSignalInterceptors(); + InitializeLibdispatchInterceptors(); + +#if !SANITIZER_MAC + // We can not use TSAN_INTERCEPT to get setjmp addr, + // because it does &setjmp and setjmp is not present in some versions of libc. + using __interception::InterceptFunction; + InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); + InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); + InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, + 0); +#if !SANITIZER_NETBSD + InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); +#endif +#endif + + TSAN_INTERCEPT(longjmp_symname); + TSAN_INTERCEPT(siglongjmp_symname); +#if SANITIZER_NETBSD + TSAN_INTERCEPT(_longjmp); +#endif + + TSAN_INTERCEPT(malloc); + TSAN_INTERCEPT(__libc_memalign); + TSAN_INTERCEPT(calloc); + TSAN_INTERCEPT(realloc); + TSAN_INTERCEPT(reallocarray); + TSAN_INTERCEPT(free); + TSAN_INTERCEPT(cfree); + TSAN_INTERCEPT(munmap); + TSAN_MAYBE_INTERCEPT_MEMALIGN; + TSAN_INTERCEPT(valloc); + TSAN_MAYBE_INTERCEPT_PVALLOC; + TSAN_INTERCEPT(posix_memalign); + + TSAN_INTERCEPT(strcpy); + TSAN_INTERCEPT(strncpy); + TSAN_INTERCEPT(strdup); + + TSAN_INTERCEPT(pthread_create); + TSAN_INTERCEPT(pthread_join); + TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT(pthread_exit); + #if SANITIZER_LINUX + TSAN_INTERCEPT(pthread_tryjoin_np); + TSAN_INTERCEPT(pthread_timedjoin_np); + #endif + + TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE); + + TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT; + + TSAN_INTERCEPT(pthread_mutex_init); + TSAN_INTERCEPT(pthread_mutex_destroy); + TSAN_INTERCEPT(pthread_mutex_trylock); + TSAN_INTERCEPT(pthread_mutex_timedlock); + + TSAN_INTERCEPT(pthread_spin_init); + TSAN_INTERCEPT(pthread_spin_destroy); + TSAN_INTERCEPT(pthread_spin_lock); + TSAN_INTERCEPT(pthread_spin_trylock); + TSAN_INTERCEPT(pthread_spin_unlock); + + TSAN_INTERCEPT(pthread_rwlock_init); + TSAN_INTERCEPT(pthread_rwlock_destroy); + TSAN_INTERCEPT(pthread_rwlock_rdlock); + TSAN_INTERCEPT(pthread_rwlock_tryrdlock); + TSAN_INTERCEPT(pthread_rwlock_timedrdlock); + TSAN_INTERCEPT(pthread_rwlock_wrlock); + TSAN_INTERCEPT(pthread_rwlock_trywrlock); + TSAN_INTERCEPT(pthread_rwlock_timedwrlock); + TSAN_INTERCEPT(pthread_rwlock_unlock); + + TSAN_INTERCEPT(pthread_barrier_init); + TSAN_INTERCEPT(pthread_barrier_destroy); + TSAN_INTERCEPT(pthread_barrier_wait); + + TSAN_INTERCEPT(pthread_once); + + TSAN_INTERCEPT(fstat); + TSAN_MAYBE_INTERCEPT___FXSTAT; + TSAN_MAYBE_INTERCEPT_FSTAT64; + TSAN_MAYBE_INTERCEPT___FXSTAT64; + TSAN_INTERCEPT(open); + TSAN_MAYBE_INTERCEPT_OPEN64; + TSAN_INTERCEPT(creat); + TSAN_MAYBE_INTERCEPT_CREAT64; + TSAN_INTERCEPT(dup); + TSAN_INTERCEPT(dup2); + TSAN_INTERCEPT(dup3); + TSAN_MAYBE_INTERCEPT_EVENTFD; + TSAN_MAYBE_INTERCEPT_SIGNALFD; + TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; + TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; + TSAN_INTERCEPT(socket); + TSAN_INTERCEPT(socketpair); + TSAN_INTERCEPT(connect); + TSAN_INTERCEPT(bind); + TSAN_INTERCEPT(listen); + TSAN_MAYBE_INTERCEPT_EPOLL; + TSAN_INTERCEPT(close); + TSAN_MAYBE_INTERCEPT___CLOSE; + TSAN_MAYBE_INTERCEPT___RES_ICLOSE; + TSAN_INTERCEPT(pipe); + TSAN_INTERCEPT(pipe2); + + TSAN_INTERCEPT(unlink); + TSAN_INTERCEPT(tmpfile); + TSAN_MAYBE_INTERCEPT_TMPFILE64; + TSAN_INTERCEPT(abort); + TSAN_INTERCEPT(rmdir); + TSAN_INTERCEPT(closedir); + + TSAN_INTERCEPT(sigsuspend); + TSAN_INTERCEPT(sigblock); + TSAN_INTERCEPT(sigsetmask); + TSAN_INTERCEPT(pthread_sigmask); + TSAN_INTERCEPT(raise); + TSAN_INTERCEPT(kill); + TSAN_INTERCEPT(pthread_kill); + TSAN_INTERCEPT(sleep); + TSAN_INTERCEPT(usleep); + TSAN_INTERCEPT(nanosleep); + TSAN_INTERCEPT(pause); + TSAN_INTERCEPT(gettimeofday); + TSAN_INTERCEPT(getaddrinfo); + + TSAN_INTERCEPT(fork); + TSAN_INTERCEPT(vfork); +#if SANITIZER_LINUX + TSAN_INTERCEPT(clone); +#endif +#if !SANITIZER_ANDROID + TSAN_INTERCEPT(dl_iterate_phdr); +#endif + TSAN_MAYBE_INTERCEPT_ON_EXIT; + TSAN_INTERCEPT(__cxa_atexit); + TSAN_INTERCEPT(_exit); + +#ifdef NEED_TLS_GET_ADDR +#if !SANITIZER_S390 + TSAN_INTERCEPT(__tls_get_addr); +#else + TSAN_INTERCEPT(__tls_get_addr_internal); + TSAN_INTERCEPT(__tls_get_offset); +#endif +#endif + + TSAN_MAYBE_INTERCEPT__LWP_EXIT; + TSAN_MAYBE_INTERCEPT_THR_EXIT; + +#if !SANITIZER_MAC && !SANITIZER_ANDROID + // Need to setup it, because interceptors check that the function is resolved. + // But atexit is emitted directly into the module, so can't be resolved. + REAL(atexit) = (int(*)(void(*)()))unreachable; +#endif + + if (REAL(__cxa_atexit)(&finalize, 0, 0)) { + Printf("ThreadSanitizer: failed to setup atexit callback\n"); + Die(); + } + if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) { + Printf("ThreadSanitizer: failed to setup atfork callbacks\n"); + Die(); + } + +#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD + if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { + Printf("ThreadSanitizer: failed to create thread key\n"); + Die(); + } +#endif + + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask); + + FdInit(); +} + +} // namespace __tsan + +// Invisible barrier for tests. +// There were several unsuccessful iterations for this functionality: +// 1. Initially it was implemented in user code using +// REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on +// MacOS. Futexes are linux-specific for this matter. +// 2. Then we switched to atomics+usleep(10). But usleep produced parasitic +// "as-if synchronized via sleep" messages in reports which failed some +// output tests. +// 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- +// visible events, which lead to "failed to restore stack trace" failures. +// Note that no_sanitize_thread attribute does not turn off atomic interception +// so attaching it to the function defined in user code does not help. +// That's why we now have what we have. +constexpr u32 kBarrierThreadBits = 10; +constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( + atomic_uint32_t *barrier, u32 num_threads) { + if (num_threads >= kBarrierThreads) { + Printf("barrier_init: count is too large (%d)\n", num_threads); + Die(); + } + // kBarrierThreadBits lsb is thread count, + // the remaining are count of entered threads. + atomic_store(barrier, num_threads, memory_order_relaxed); +} + +static u32 barrier_epoch(u32 value) { + return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1)); +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( + atomic_uint32_t *barrier) { + u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed); + u32 old_epoch = barrier_epoch(old); + if (barrier_epoch(old + kBarrierThreads) != old_epoch) { + FutexWake(barrier, (1 << 30)); + return; + } + for (;;) { + u32 cur = atomic_load(barrier, memory_order_relaxed); + if (barrier_epoch(cur) != old_epoch) + return; + FutexWait(barrier, cur); + } +} diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp new file mode 100644 index 000000000000..048715185151 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp @@ -0,0 +1,106 @@ +//===-- tsan_interface.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "tsan_interface.h" +#include "tsan_interface_ann.h" +#include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_ptrauth.h" + +#define CALLERPC ((uptr)__builtin_return_address(0)) + +using namespace __tsan; + +void __tsan_init() { Initialize(cur_thread_init()); } + +void __tsan_flush_memory() { + FlushShadowMemory(); +} + +void __tsan_read16(void *addr) { + uptr pc = CALLERPC; + ThreadState *thr = cur_thread(); + MemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead); + MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead); +} + +void __tsan_write16(void *addr) { + uptr pc = CALLERPC; + ThreadState *thr = cur_thread(); + MemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite); + MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite); +} + +void __tsan_read16_pc(void *addr, void *pc) { + uptr pc_no_pac = STRIP_PAC_PC(pc); + ThreadState *thr = cur_thread(); + MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessRead); + MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessRead); +} + +void __tsan_write16_pc(void *addr, void *pc) { + uptr pc_no_pac = STRIP_PAC_PC(pc); + ThreadState *thr = cur_thread(); + MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessWrite); + MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessWrite); +} + +// __tsan_unaligned_read/write calls are emitted by compiler. + +void __tsan_unaligned_read16(const void *addr) { + uptr pc = CALLERPC; + ThreadState *thr = cur_thread(); + UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead); + UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead); +} + +void __tsan_unaligned_write16(void *addr) { + uptr pc = CALLERPC; + ThreadState *thr = cur_thread(); + UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite); + UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite); +} + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast(fiber), name); +} +} // extern "C" + +void __tsan_acquire(void *addr) { + Acquire(cur_thread(), CALLERPC, (uptr)addr); +} + +void __tsan_release(void *addr) { + Release(cur_thread(), CALLERPC, (uptr)addr); +} diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface.h b/compiler-rt/lib/tsan/rtl-old/tsan_interface.h new file mode 100644 index 000000000000..711f064174c2 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface.h @@ -0,0 +1,424 @@ +//===-- tsan_interface.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// The functions declared in this header will be inserted by the instrumentation +// module. +// This header can be included by the instrumented program or by TSan tests. +//===----------------------------------------------------------------------===// +#ifndef TSAN_INTERFACE_H +#define TSAN_INTERFACE_H + +#include +using __sanitizer::uptr; +using __sanitizer::tid_t; + +// This header should NOT include any other headers. +// All functions in this header are extern "C" and start with __tsan_. + +#ifdef __cplusplus +extern "C" { +#endif + +#if !SANITIZER_GO + +// This function should be called at the very beginning of the process, +// before any instrumented code is executed and before any call to malloc. +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init(); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_flush_memory(); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16(void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16(void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16_pc(void *addr, void *pc); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8_pc(void *addr, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16_pc(void *addr, void *pc); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_vptr_update(void **vptr_p, void *new_val); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit(); + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_begin(); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_end(); + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_external_register_tag(const char *object_type); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_register_header(void *tag, const char *header); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_assign_tag(void *addr, void *tag); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_read(void *addr, void *caller_pc, void *tag); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_external_write(void *addr, void *caller_pc, void *tag); + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_read_range(void *addr, unsigned long size); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_write_range(void *addr, unsigned long size); + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_read_range_pc(void *addr, unsigned long size, void *pc); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_write_range_pc(void *addr, unsigned long size, void *pc); + +// User may provide function that would be called right when TSan detects +// an error. The argument 'report' is an opaque pointer that can be used to +// gather additional information using other TSan report API functions. +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_on_report(void *report); + +// If TSan is currently reporting a detected issue on the current thread, +// returns an opaque pointer to the current report. Otherwise returns NULL. +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_report(); + +// Returns a report's description (issue type), number of duplicate issues +// found, counts of array data (stack traces, memory operations, locations, +// mutexes, threads, unique thread IDs) and a stack trace of a sleep() call (if +// one was involved in the issue). +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_data(void *report, const char **description, int *count, + int *stack_count, int *mop_count, int *loc_count, + int *mutex_count, int *thread_count, + int *unique_tid_count, void **sleep_trace, + uptr trace_size); + +/// Retrieves the "tag" from a report (for external-race report types). External +/// races can be associated with a tag which give them more meaning. For example +/// tag value '1' means "Swift access race". Tag value '0' indicated a plain +/// external race. +/// +/// \param report opaque pointer to the current report (obtained as argument in +/// __tsan_on_report, or from __tsan_get_current_report) +/// \param [out] tag points to storage that will be filled with the tag value +/// +/// \returns non-zero value on success, zero on failure +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_tag(void *report, uptr *tag); + +// Returns information about stack traces included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_stack(void *report, uptr idx, void **trace, + uptr trace_size); + +// Returns information about memory operations included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr, + int *size, int *write, int *atomic, void **trace, + uptr trace_size); + +// Returns information about locations included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_loc(void *report, uptr idx, const char **type, + void **addr, uptr *start, uptr *size, int *tid, + int *fd, int *suppressable, void **trace, + uptr trace_size); + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_loc_object_type(void *report, uptr idx, + const char **object_type); + +// Returns information about mutexes included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr, + int *destroyed, void **trace, uptr trace_size); + +// Returns information about threads included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id, + int *running, const char **name, int *parent_tid, + void **trace, uptr trace_size); + +// Returns information about unique thread IDs included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid); + +// Returns the type of the pointer (heap, stack, global, ...) and if possible +// also the starting address (e.g. of a heap allocation) and size. +SANITIZER_INTERFACE_ATTRIBUTE +const char *__tsan_locate_address(uptr addr, char *name, uptr name_size, + uptr *region_address, uptr *region_size); + +// Returns the allocation stack for a heap pointer. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id, + tid_t *os_id); + +#endif // SANITIZER_GO + +#ifdef __cplusplus +} // extern "C" +#endif + +namespace __tsan { + +// These should match declarations from public tsan_interface_atomic.h header. +typedef unsigned char a8; +typedef unsigned short a16; +typedef unsigned int a32; +typedef unsigned long long a64; +#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \ + || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \ + !defined(__mips64) && !defined(__s390x__) +__extension__ typedef __int128 a128; +# define __TSAN_HAS_INT128 1 +#else +# define __TSAN_HAS_INT128 0 +#endif + +// Part of ABI, do not change. +// https://github.com/llvm/llvm-project/blob/main/libcxx/include/atomic +typedef enum { + mo_relaxed, + mo_consume, + mo_acquire, + mo_release, + mo_acq_rel, + mo_seq_cst +} morder; + +struct ThreadState; + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_load(const volatile a8 *a, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_load(const volatile a16 *a, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_load(const volatile a32 *a, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_load(const volatile a64 *a, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_load(const volatile a128 *a, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, + morder mo, morder fmo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo, + morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, + morder mo, morder fmo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo, + morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, + morder mo, morder fmo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, + morder mo, morder fmo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic_thread_fence(morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic_signal_fence(morder mo); + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc, + u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc, + u8 *a); + +} // extern "C" + +} // namespace __tsan + +#endif // TSAN_INTERFACE_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface.inc b/compiler-rt/lib/tsan/rtl-old/tsan_interface.inc new file mode 100644 index 000000000000..0031800e851f --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface.inc @@ -0,0 +1,182 @@ +//===-- tsan_interface.inc --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_ptrauth.h" +#include "tsan_interface.h" +#include "tsan_rtl.h" + +#define CALLERPC ((uptr)__builtin_return_address(0)) + +using namespace __tsan; + +void __tsan_read1(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessRead); +} + +void __tsan_read2(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead); +} + +void __tsan_read4(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead); +} + +void __tsan_read8(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead); +} + +void __tsan_write1(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite); +} + +void __tsan_write2(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite); +} + +void __tsan_write4(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite); +} + +void __tsan_write8(void *addr) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite); +} + +void __tsan_read1_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC); +} + +void __tsan_read2_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessRead | kAccessExternalPC); +} + +void __tsan_read4_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessRead | kAccessExternalPC); +} + +void __tsan_read8_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessRead | kAccessExternalPC); +} + +void __tsan_write1_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessWrite | kAccessExternalPC); +} + +void __tsan_write2_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessWrite | kAccessExternalPC); +} + +void __tsan_write4_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessWrite | kAccessExternalPC); +} + +void __tsan_write8_pc(void *addr, void *pc) { + MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessWrite | kAccessExternalPC); +} + +ALWAYS_INLINE USED void __tsan_unaligned_read2(const void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead); +} + +ALWAYS_INLINE USED void __tsan_unaligned_read4(const void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead); +} + +ALWAYS_INLINE USED void __tsan_unaligned_read8(const void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead); +} + +ALWAYS_INLINE USED void __tsan_unaligned_write2(void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite); +} + +ALWAYS_INLINE USED void __tsan_unaligned_write4(void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite); +} + +ALWAYS_INLINE USED void __tsan_unaligned_write8(void *addr) { + UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite); +} + +extern "C" { +// __sanitizer_unaligned_load/store are for user instrumentation. +SANITIZER_INTERFACE_ATTRIBUTE +u16 __sanitizer_unaligned_load16(const uu16 *addr) { + __tsan_unaligned_read2(addr); + return *addr; +} + +SANITIZER_INTERFACE_ATTRIBUTE +u32 __sanitizer_unaligned_load32(const uu32 *addr) { + __tsan_unaligned_read4(addr); + return *addr; +} + +SANITIZER_INTERFACE_ATTRIBUTE +u64 __sanitizer_unaligned_load64(const uu64 *addr) { + __tsan_unaligned_read8(addr); + return *addr; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store16(uu16 *addr, u16 v) { + *addr = v; + __tsan_unaligned_write2(addr); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store32(uu32 *addr, u32 v) { + *addr = v; + __tsan_unaligned_write4(addr); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_unaligned_store64(uu64 *addr, u64 v) { + *addr = v; + __tsan_unaligned_write8(addr); +} +} + +void __tsan_vptr_update(void **vptr_p, void *new_val) { + if (*vptr_p == new_val) + return; + MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p), + kAccessWrite | kAccessVptr); +} + +void __tsan_vptr_read(void **vptr_p) { + MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p), + kAccessRead | kAccessVptr); +} + +void __tsan_func_entry(void *pc) { FuncEntry(cur_thread(), STRIP_PAC_PC(pc)); } + +void __tsan_func_exit() { FuncExit(cur_thread()); } + +void __tsan_ignore_thread_begin() { ThreadIgnoreBegin(cur_thread(), CALLERPC); } + +void __tsan_ignore_thread_end() { ThreadIgnoreEnd(cur_thread()); } + +void __tsan_read_range(void *addr, uptr size) { + MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false); +} + +void __tsan_write_range(void *addr, uptr size) { + MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true); +} + +void __tsan_read_range_pc(void *addr, uptr size, void *pc) { + MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false); +} + +void __tsan_write_range_pc(void *addr, uptr size, void *pc) { + MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true); +} diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp new file mode 100644 index 000000000000..6bd72e18d942 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp @@ -0,0 +1,438 @@ +//===-- tsan_interface_ann.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_vector.h" +#include "tsan_interface_ann.h" +#include "tsan_report.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" +#include "tsan_flags.h" +#include "tsan_platform.h" + +#define CALLERPC ((uptr)__builtin_return_address(0)) + +using namespace __tsan; + +namespace __tsan { + +class ScopedAnnotation { + public: + ScopedAnnotation(ThreadState *thr, const char *aname, uptr pc) + : thr_(thr) { + FuncEntry(thr_, pc); + DPrintf("#%d: annotation %s()\n", thr_->tid, aname); + } + + ~ScopedAnnotation() { + FuncExit(thr_); + CheckedMutex::CheckNoLocks(); + } + private: + ThreadState *const thr_; +}; + +#define SCOPED_ANNOTATION_RET(typ, ret) \ + if (!flags()->enable_annotations) \ + return ret; \ + ThreadState *thr = cur_thread(); \ + const uptr caller_pc = (uptr)__builtin_return_address(0); \ + ScopedAnnotation sa(thr, __func__, caller_pc); \ + const uptr pc = StackTrace::GetCurrentPc(); \ + (void)pc; + +#define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, ) + +static const int kMaxDescLen = 128; + +struct ExpectRace { + ExpectRace *next; + ExpectRace *prev; + atomic_uintptr_t hitcount; + atomic_uintptr_t addcount; + uptr addr; + uptr size; + char *file; + int line; + char desc[kMaxDescLen]; +}; + +struct DynamicAnnContext { + Mutex mtx; + ExpectRace benign; + + DynamicAnnContext() : mtx(MutexTypeAnnotations) {} +}; + +static DynamicAnnContext *dyn_ann_ctx; +static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)] ALIGNED(64); + +static void AddExpectRace(ExpectRace *list, + char *f, int l, uptr addr, uptr size, char *desc) { + ExpectRace *race = list->next; + for (; race != list; race = race->next) { + if (race->addr == addr && race->size == size) { + atomic_store_relaxed(&race->addcount, + atomic_load_relaxed(&race->addcount) + 1); + return; + } + } + race = static_cast(Alloc(sizeof(ExpectRace))); + race->addr = addr; + race->size = size; + race->file = f; + race->line = l; + race->desc[0] = 0; + atomic_store_relaxed(&race->hitcount, 0); + atomic_store_relaxed(&race->addcount, 1); + if (desc) { + int i = 0; + for (; i < kMaxDescLen - 1 && desc[i]; i++) + race->desc[i] = desc[i]; + race->desc[i] = 0; + } + race->prev = list; + race->next = list->next; + race->next->prev = race; + list->next = race; +} + +static ExpectRace *FindRace(ExpectRace *list, uptr addr, uptr size) { + for (ExpectRace *race = list->next; race != list; race = race->next) { + uptr maxbegin = max(race->addr, addr); + uptr minend = min(race->addr + race->size, addr + size); + if (maxbegin < minend) + return race; + } + return 0; +} + +static bool CheckContains(ExpectRace *list, uptr addr, uptr size) { + ExpectRace *race = FindRace(list, addr, size); + if (race == 0) + return false; + DPrintf("Hit expected/benign race: %s addr=%zx:%d %s:%d\n", + race->desc, race->addr, (int)race->size, race->file, race->line); + atomic_fetch_add(&race->hitcount, 1, memory_order_relaxed); + return true; +} + +static void InitList(ExpectRace *list) { + list->next = list; + list->prev = list; +} + +void InitializeDynamicAnnotations() { + dyn_ann_ctx = new(dyn_ann_ctx_placeholder) DynamicAnnContext; + InitList(&dyn_ann_ctx->benign); +} + +bool IsExpectedReport(uptr addr, uptr size) { + ReadLock lock(&dyn_ann_ctx->mtx); + return CheckContains(&dyn_ann_ctx->benign, addr, size); +} +} // namespace __tsan + +using namespace __tsan; + +extern "C" { +void INTERFACE_ATTRIBUTE AnnotateHappensBefore(char *f, int l, uptr addr) { + SCOPED_ANNOTATION(AnnotateHappensBefore); + Release(thr, pc, addr); +} + +void INTERFACE_ATTRIBUTE AnnotateHappensAfter(char *f, int l, uptr addr) { + SCOPED_ANNOTATION(AnnotateHappensAfter); + Acquire(thr, pc, addr); +} + +void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) { +} + +void INTERFACE_ATTRIBUTE AnnotateCondVarSignalAll(char *f, int l, uptr cv) { +} + +void INTERFACE_ATTRIBUTE AnnotateMutexIsNotPHB(char *f, int l, uptr mu) { +} + +void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv, + uptr lock) { +} + +void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) { + SCOPED_ANNOTATION(AnnotateRWLockCreate); + MutexCreate(thr, pc, m, MutexFlagWriteReentrant); +} + +void INTERFACE_ATTRIBUTE AnnotateRWLockCreateStatic(char *f, int l, uptr m) { + SCOPED_ANNOTATION(AnnotateRWLockCreateStatic); + MutexCreate(thr, pc, m, MutexFlagWriteReentrant | MutexFlagLinkerInit); +} + +void INTERFACE_ATTRIBUTE AnnotateRWLockDestroy(char *f, int l, uptr m) { + SCOPED_ANNOTATION(AnnotateRWLockDestroy); + MutexDestroy(thr, pc, m); +} + +void INTERFACE_ATTRIBUTE AnnotateRWLockAcquired(char *f, int l, uptr m, + uptr is_w) { + SCOPED_ANNOTATION(AnnotateRWLockAcquired); + if (is_w) + MutexPostLock(thr, pc, m, MutexFlagDoPreLockOnPostLock); + else + MutexPostReadLock(thr, pc, m, MutexFlagDoPreLockOnPostLock); +} + +void INTERFACE_ATTRIBUTE AnnotateRWLockReleased(char *f, int l, uptr m, + uptr is_w) { + SCOPED_ANNOTATION(AnnotateRWLockReleased); + if (is_w) + MutexUnlock(thr, pc, m); + else + MutexReadUnlock(thr, pc, m); +} + +void INTERFACE_ATTRIBUTE AnnotateTraceMemory(char *f, int l, uptr mem) { +} + +void INTERFACE_ATTRIBUTE AnnotateFlushState(char *f, int l) { +} + +void INTERFACE_ATTRIBUTE AnnotateNewMemory(char *f, int l, uptr mem, + uptr size) { +} + +void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) { +} + +void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) { +} + +void INTERFACE_ATTRIBUTE AnnotateEnableRaceDetection( + char *f, int l, int enable) { +} + +void INTERFACE_ATTRIBUTE AnnotateMutexIsUsedAsCondVar( + char *f, int l, uptr mu) { +} + +void INTERFACE_ATTRIBUTE AnnotatePCQGet( + char *f, int l, uptr pcq) { +} + +void INTERFACE_ATTRIBUTE AnnotatePCQPut( + char *f, int l, uptr pcq) { +} + +void INTERFACE_ATTRIBUTE AnnotatePCQDestroy( + char *f, int l, uptr pcq) { +} + +void INTERFACE_ATTRIBUTE AnnotatePCQCreate( + char *f, int l, uptr pcq) { +} + +void INTERFACE_ATTRIBUTE AnnotateExpectRace( + char *f, int l, uptr mem, char *desc) { +} + +static void BenignRaceImpl(char *f, int l, uptr mem, uptr size, char *desc) { + Lock lock(&dyn_ann_ctx->mtx); + AddExpectRace(&dyn_ann_ctx->benign, + f, l, mem, size, desc); + DPrintf("Add benign race: %s addr=%zx %s:%d\n", desc, mem, f, l); +} + +void INTERFACE_ATTRIBUTE AnnotateBenignRaceSized( + char *f, int l, uptr mem, uptr size, char *desc) { + SCOPED_ANNOTATION(AnnotateBenignRaceSized); + BenignRaceImpl(f, l, mem, size, desc); +} + +void INTERFACE_ATTRIBUTE AnnotateBenignRace( + char *f, int l, uptr mem, char *desc) { + SCOPED_ANNOTATION(AnnotateBenignRace); + BenignRaceImpl(f, l, mem, 1, desc); +} + +void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) { + SCOPED_ANNOTATION(AnnotateIgnoreReadsBegin); + ThreadIgnoreBegin(thr, pc); +} + +void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) { + SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd); + ThreadIgnoreEnd(thr); +} + +void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) { + SCOPED_ANNOTATION(AnnotateIgnoreWritesBegin); + ThreadIgnoreBegin(thr, pc); +} + +void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) { + SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd); + ThreadIgnoreEnd(thr); +} + +void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) { + SCOPED_ANNOTATION(AnnotateIgnoreSyncBegin); + ThreadIgnoreSyncBegin(thr, pc); +} + +void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) { + SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd); + ThreadIgnoreSyncEnd(thr); +} + +void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange( + char *f, int l, uptr addr, uptr size) { +} + +void INTERFACE_ATTRIBUTE AnnotateUnpublishMemoryRange( + char *f, int l, uptr addr, uptr size) { +} + +void INTERFACE_ATTRIBUTE AnnotateThreadName( + char *f, int l, char *name) { + SCOPED_ANNOTATION(AnnotateThreadName); + ThreadSetName(thr, name); +} + +// We deliberately omit the implementation of WTFAnnotateHappensBefore() and +// WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate +// atomic operations, which should be handled by ThreadSanitizer correctly. +void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) { +} + +void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) { +} + +void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized( + char *f, int l, uptr mem, uptr sz, char *desc) { + SCOPED_ANNOTATION(AnnotateBenignRaceSized); + BenignRaceImpl(f, l, mem, sz, desc); +} + +int INTERFACE_ATTRIBUTE RunningOnValgrind() { + return flags()->running_on_valgrind; +} + +double __attribute__((weak)) INTERFACE_ATTRIBUTE ValgrindSlowdown(void) { + return 10.0; +} + +const char INTERFACE_ATTRIBUTE* ThreadSanitizerQuery(const char *query) { + if (internal_strcmp(query, "pure_happens_before") == 0) + return "1"; + else + return "0"; +} + +void INTERFACE_ATTRIBUTE +AnnotateMemoryIsInitialized(char *f, int l, uptr mem, uptr sz) {} +void INTERFACE_ATTRIBUTE +AnnotateMemoryIsUninitialized(char *f, int l, uptr mem, uptr sz) {} + +// Note: the parameter is called flagz, because flags is already taken +// by the global function that returns flags. +INTERFACE_ATTRIBUTE +void __tsan_mutex_create(void *m, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_create); + MutexCreate(thr, pc, (uptr)m, flagz & MutexCreationFlagMask); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_destroy(void *m, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_destroy); + MutexDestroy(thr, pc, (uptr)m, flagz); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_pre_lock(void *m, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_pre_lock); + if (!(flagz & MutexFlagTryLock)) { + if (flagz & MutexFlagReadLock) + MutexPreReadLock(thr, pc, (uptr)m); + else + MutexPreLock(thr, pc, (uptr)m); + } + ThreadIgnoreBegin(thr, 0); + ThreadIgnoreSyncBegin(thr, 0); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) { + SCOPED_ANNOTATION(__tsan_mutex_post_lock); + ThreadIgnoreSyncEnd(thr); + ThreadIgnoreEnd(thr); + if (!(flagz & MutexFlagTryLockFailed)) { + if (flagz & MutexFlagReadLock) + MutexPostReadLock(thr, pc, (uptr)m, flagz); + else + MutexPostLock(thr, pc, (uptr)m, flagz, rec); + } +} + +INTERFACE_ATTRIBUTE +int __tsan_mutex_pre_unlock(void *m, unsigned flagz) { + SCOPED_ANNOTATION_RET(__tsan_mutex_pre_unlock, 0); + int ret = 0; + if (flagz & MutexFlagReadLock) { + CHECK(!(flagz & MutexFlagRecursiveUnlock)); + MutexReadUnlock(thr, pc, (uptr)m); + } else { + ret = MutexUnlock(thr, pc, (uptr)m, flagz); + } + ThreadIgnoreBegin(thr, 0); + ThreadIgnoreSyncBegin(thr, 0); + return ret; +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_post_unlock(void *m, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_post_unlock); + ThreadIgnoreSyncEnd(thr); + ThreadIgnoreEnd(thr); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_pre_signal(void *addr, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_pre_signal); + ThreadIgnoreBegin(thr, 0); + ThreadIgnoreSyncBegin(thr, 0); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_post_signal(void *addr, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_post_signal); + ThreadIgnoreSyncEnd(thr); + ThreadIgnoreEnd(thr); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_pre_divert(void *addr, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_pre_divert); + // Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal. + ThreadIgnoreSyncEnd(thr); + ThreadIgnoreEnd(thr); +} + +INTERFACE_ATTRIBUTE +void __tsan_mutex_post_divert(void *addr, unsigned flagz) { + SCOPED_ANNOTATION(__tsan_mutex_post_divert); + ThreadIgnoreBegin(thr, 0); + ThreadIgnoreSyncBegin(thr, 0); +} +} // extern "C" diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h new file mode 100644 index 000000000000..458d61f53356 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h @@ -0,0 +1,32 @@ +//===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Interface for dynamic annotations. +//===----------------------------------------------------------------------===// +#ifndef TSAN_INTERFACE_ANN_H +#define TSAN_INTERFACE_ANN_H + +#include + +// This header should NOT include any other headers. +// All functions in this header are extern "C" and start with __tsan_. + +#ifdef __cplusplus +extern "C" { +#endif + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_acquire(void *addr); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_release(void *addr); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TSAN_INTERFACE_ANN_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp new file mode 100644 index 000000000000..24ba3bb1f65d --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp @@ -0,0 +1,920 @@ +//===-- tsan_interface_atomic.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +// ThreadSanitizer atomic operations are based on C++11/C1x standards. +// For background see C++11 standard. A slightly older, publicly +// available draft of the standard (not entirely up-to-date, but close enough +// for casual browsing) is available here: +// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf +// The following page contains more background information: +// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ + +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_mutex.h" +#include "tsan_flags.h" +#include "tsan_interface.h" +#include "tsan_rtl.h" + +using namespace __tsan; + +#if !SANITIZER_GO && __TSAN_HAS_INT128 +// Protects emulation of 128-bit atomic operations. +static StaticSpinMutex mutex128; +#endif + +#if SANITIZER_DEBUG +static bool IsLoadOrder(morder mo) { + return mo == mo_relaxed || mo == mo_consume + || mo == mo_acquire || mo == mo_seq_cst; +} + +static bool IsStoreOrder(morder mo) { + return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; +} +#endif + +static bool IsReleaseOrder(morder mo) { + return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; +} + +static bool IsAcquireOrder(morder mo) { + return mo == mo_consume || mo == mo_acquire + || mo == mo_acq_rel || mo == mo_seq_cst; +} + +static bool IsAcqRelOrder(morder mo) { + return mo == mo_acq_rel || mo == mo_seq_cst; +} + +template T func_xchg(volatile T *v, T op) { + T res = __sync_lock_test_and_set(v, op); + // __sync_lock_test_and_set does not contain full barrier. + __sync_synchronize(); + return res; +} + +template T func_add(volatile T *v, T op) { + return __sync_fetch_and_add(v, op); +} + +template T func_sub(volatile T *v, T op) { + return __sync_fetch_and_sub(v, op); +} + +template T func_and(volatile T *v, T op) { + return __sync_fetch_and_and(v, op); +} + +template T func_or(volatile T *v, T op) { + return __sync_fetch_and_or(v, op); +} + +template T func_xor(volatile T *v, T op) { + return __sync_fetch_and_xor(v, op); +} + +template T func_nand(volatile T *v, T op) { + // clang does not support __sync_fetch_and_nand. + T cmp = *v; + for (;;) { + T newv = ~(cmp & op); + T cur = __sync_val_compare_and_swap(v, cmp, newv); + if (cmp == cur) + return cmp; + cmp = cur; + } +} + +template T func_cas(volatile T *v, T cmp, T xch) { + return __sync_val_compare_and_swap(v, cmp, xch); +} + +// clang does not support 128-bit atomic ops. +// Atomic ops are executed under tsan internal mutex, +// here we assume that the atomic variables are not accessed +// from non-instrumented code. +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \ + && __TSAN_HAS_INT128 +a128 func_xchg(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = op; + return cmp; +} + +a128 func_add(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = cmp + op; + return cmp; +} + +a128 func_sub(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = cmp - op; + return cmp; +} + +a128 func_and(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = cmp & op; + return cmp; +} + +a128 func_or(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = cmp | op; + return cmp; +} + +a128 func_xor(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = cmp ^ op; + return cmp; +} + +a128 func_nand(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); + a128 cmp = *v; + *v = ~(cmp & op); + return cmp; +} + +a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { + SpinMutexLock lock(&mutex128); + a128 cur = *v; + if (cur == cmp) + *v = xch; + return cur; +} +#endif + +template +static int AccessSize() { + if (sizeof(T) <= 1) + return 1; + else if (sizeof(T) <= 2) + return 2; + else if (sizeof(T) <= 4) + return 4; + else + return 8; + // For 16-byte atomics we also use 8-byte memory access, + // this leads to false negatives only in very obscure cases. +} + +#if !SANITIZER_GO +static atomic_uint8_t *to_atomic(const volatile a8 *a) { + return reinterpret_cast(const_cast(a)); +} + +static atomic_uint16_t *to_atomic(const volatile a16 *a) { + return reinterpret_cast(const_cast(a)); +} +#endif + +static atomic_uint32_t *to_atomic(const volatile a32 *a) { + return reinterpret_cast(const_cast(a)); +} + +static atomic_uint64_t *to_atomic(const volatile a64 *a) { + return reinterpret_cast(const_cast(a)); +} + +static memory_order to_mo(morder mo) { + switch (mo) { + case mo_relaxed: return memory_order_relaxed; + case mo_consume: return memory_order_consume; + case mo_acquire: return memory_order_acquire; + case mo_release: return memory_order_release; + case mo_acq_rel: return memory_order_acq_rel; + case mo_seq_cst: return memory_order_seq_cst; + } + DCHECK(0); + return memory_order_seq_cst; +} + +template +static T NoTsanAtomicLoad(const volatile T *a, morder mo) { + return atomic_load(to_atomic(a), to_mo(mo)); +} + +#if __TSAN_HAS_INT128 && !SANITIZER_GO +static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { + SpinMutexLock lock(&mutex128); + return *a; +} +#endif + +template +static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { + DCHECK(IsLoadOrder(mo)); + // This fast-path is critical for performance. + // Assume the access is atomic. + if (!IsAcquireOrder(mo)) { + MemoryAccess(thr, pc, (uptr)a, AccessSize(), + kAccessRead | kAccessAtomic); + return NoTsanAtomicLoad(a, mo); + } + // Don't create sync object if it does not exist yet. For example, an atomic + // pointer is initialized to nullptr and then periodically acquire-loaded. + T v = NoTsanAtomicLoad(a, mo); + SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); + if (s) { + ReadLock l(&s->mtx); + AcquireImpl(thr, pc, &s->clock); + // Re-read under sync mutex because we need a consistent snapshot + // of the value and the clock we acquire. + v = NoTsanAtomicLoad(a, mo); + } + MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessRead | kAccessAtomic); + return v; +} + +template +static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { + atomic_store(to_atomic(a), v, to_mo(mo)); +} + +#if __TSAN_HAS_INT128 && !SANITIZER_GO +static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { + SpinMutexLock lock(&mutex128); + *a = v; +} +#endif + +template +static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + DCHECK(IsStoreOrder(mo)); + MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessWrite | kAccessAtomic); + // This fast-path is critical for performance. + // Assume the access is atomic. + // Strictly saying even relaxed store cuts off release sequence, + // so must reset the clock. + if (!IsReleaseOrder(mo)) { + NoTsanAtomicStore(a, v, mo); + return; + } + __sync_synchronize(); + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + ReleaseStoreImpl(thr, pc, &s->clock); + NoTsanAtomicStore(a, v, mo); +} + +template +static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessWrite | kAccessAtomic); + if (LIKELY(mo == mo_relaxed)) + return F(a, v); + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + if (IsAcqRelOrder(mo)) + AcquireReleaseImpl(thr, pc, &s->clock); + else if (IsReleaseOrder(mo)) + ReleaseImpl(thr, pc, &s->clock); + else if (IsAcquireOrder(mo)) + AcquireImpl(thr, pc, &s->clock); + return F(a, v); +} + +template +static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { + return func_xchg(a, v); +} + +template +static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { + return func_add(a, v); +} + +template +static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { + return func_sub(a, v); +} + +template +static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { + return func_and(a, v); +} + +template +static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { + return func_or(a, v); +} + +template +static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { + return func_xor(a, v); +} + +template +static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { + return func_nand(a, v); +} + +template +static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, + morder mo) { + return AtomicRMW(thr, pc, a, v, mo); +} + +template +static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { + return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); +} + +#if __TSAN_HAS_INT128 +static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo) { + a128 old = *c; + a128 cur = func_cas(a, old, v); + if (cur == old) + return true; + *c = cur; + return false; +} +#endif + +template +static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { + NoTsanAtomicCAS(a, &c, v, mo, fmo); + return c; +} + +template +static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, + morder mo, morder fmo) { + // 31.7.2.18: "The failure argument shall not be memory_order_release + // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic + // (mo_relaxed) when those are used. + DCHECK(IsLoadOrder(fmo)); + + MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessWrite | kAccessAtomic); + if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { + T cc = *c; + T pr = func_cas(a, cc, v); + if (pr == cc) + return true; + *c = pr; + return false; + } + + bool release = IsReleaseOrder(mo); + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + RWLock l(&s->mtx, release); + T cc = *c; + T pr = func_cas(a, cc, v); + bool success = pr == cc; + if (!success) { + *c = pr; + mo = fmo; + } + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + + if (success && IsAcqRelOrder(mo)) + AcquireReleaseImpl(thr, pc, &s->clock); + else if (success && IsReleaseOrder(mo)) + ReleaseImpl(thr, pc, &s->clock); + else if (IsAcquireOrder(mo)) + AcquireImpl(thr, pc, &s->clock); + return success; +} + +template +static T AtomicCAS(ThreadState *thr, uptr pc, + volatile T *a, T c, T v, morder mo, morder fmo) { + AtomicCAS(thr, pc, a, &c, v, mo, fmo); + return c; +} + +#if !SANITIZER_GO +static void NoTsanAtomicFence(morder mo) { + __sync_synchronize(); +} + +static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { + // FIXME(dvyukov): not implemented. + __sync_synchronize(); +} +#endif + +// Interface functions follow. +#if !SANITIZER_GO + +// C/C++ + +static morder convert_morder(morder mo) { + if (flags()->force_seq_cst_atomics) + return (morder)mo_seq_cst; + + // Filter out additional memory order flags: + // MEMMODEL_SYNC = 1 << 15 + // __ATOMIC_HLE_ACQUIRE = 1 << 16 + // __ATOMIC_HLE_RELEASE = 1 << 17 + // + // HLE is an optimization, and we pretend that elision always fails. + // MEMMODEL_SYNC is used when lowering __sync_ atomics, + // since we use __sync_ atomics for actual atomic operations, + // we can safely ignore it as well. It also subtly affects semantics, + // but we don't model the difference. + return (morder)(mo & 0x7fff); +} + +# define ATOMIC_IMPL(func, ...) \ + ThreadState *const thr = cur_thread(); \ + ProcessPendingSignals(thr); \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \ + return NoTsanAtomic##func(__VA_ARGS__); \ + mo = convert_morder(mo); \ + return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__); + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { + ATOMIC_IMPL(Load, a, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { + ATOMIC_IMPL(Load, a, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { + ATOMIC_IMPL(Load, a, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { + ATOMIC_IMPL(Load, a, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { + ATOMIC_IMPL(Load, a, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(Store, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(Store, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(Store, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(Store, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(Store, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(Exchange, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(Exchange, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(Exchange, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(Exchange, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(Exchange, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(FetchAdd, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(FetchAdd, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(FetchAdd, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(FetchAdd, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(FetchAdd, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(FetchSub, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(FetchSub, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(FetchSub, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(FetchSub, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(FetchSub, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(FetchAnd, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(FetchAnd, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(FetchAnd, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(FetchAnd, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(FetchAnd, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(FetchOr, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(FetchOr, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(FetchOr, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(FetchOr, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(FetchOr, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(FetchXor, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(FetchXor, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(FetchXor, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(FetchXor, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(FetchXor, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { + ATOMIC_IMPL(FetchNand, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { + ATOMIC_IMPL(FetchNand, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { + ATOMIC_IMPL(FetchNand, a, v, mo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { + ATOMIC_IMPL(FetchNand, a, v, mo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { + ATOMIC_IMPL(FetchNand, a, v, mo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} + +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, + morder mo, morder fmo) { + ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +} +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); } + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic_signal_fence(morder mo) { +} +} // extern "C" + +#else // #if !SANITIZER_GO + +// Go + +# define ATOMIC(func, ...) \ + if (thr->ignore_sync) { \ + NoTsanAtomic##func(__VA_ARGS__); \ + } else { \ + FuncEntry(thr, cpc); \ + Atomic##func(thr, pc, __VA_ARGS__); \ + FuncExit(thr); \ + } + +# define ATOMIC_RET(func, ret, ...) \ + if (thr->ignore_sync) { \ + (ret) = NoTsanAtomic##func(__VA_ARGS__); \ + } else { \ + FuncEntry(thr, cpc); \ + (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ + FuncExit(thr); \ + } + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_compare_exchange( + ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + a32 cur = 0; + a32 cmp = *(a32*)(a+8); + ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); + *(bool*)(a+16) = (cur == cmp); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_compare_exchange( + ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + a64 cur = 0; + a64 cmp = *(a64*)(a+8); + ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); + *(bool*)(a+24) = (cur == cmp); +} +} // extern "C" +#endif // #if !SANITIZER_GO diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp new file mode 100644 index 000000000000..c090c1f08cbe --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp @@ -0,0 +1,258 @@ +//===-- tsan_interface_java.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "tsan_interface_java.h" +#include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_procmaps.h" + +using namespace __tsan; + +const jptr kHeapAlignment = 8; + +namespace __tsan { + +struct JavaContext { + const uptr heap_begin; + const uptr heap_size; + + JavaContext(jptr heap_begin, jptr heap_size) + : heap_begin(heap_begin) + , heap_size(heap_size) { + } +}; + +static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1]; +static JavaContext *jctx; + +MBlock *JavaHeapBlock(uptr addr, uptr *start) { + if (!jctx || addr < jctx->heap_begin || + addr >= jctx->heap_begin + jctx->heap_size) + return nullptr; + for (uptr p = RoundDown(addr, kMetaShadowCell); p >= jctx->heap_begin; + p -= kMetaShadowCell) { + MBlock *b = ctx->metamap.GetBlock(p); + if (!b) + continue; + if (p + b->siz <= addr) + return nullptr; + *start = p; + return b; + } + return nullptr; +} + +} // namespace __tsan + +#define JAVA_FUNC_ENTER(func) \ + ThreadState *thr = cur_thread(); \ + (void)thr; + +void __tsan_java_init(jptr heap_begin, jptr heap_size) { + JAVA_FUNC_ENTER(__tsan_java_init); + Initialize(thr); + DPrintf("#%d: java_init(0x%zx, 0x%zx)\n", thr->tid, heap_begin, heap_size); + DCHECK_EQ(jctx, 0); + DCHECK_GT(heap_begin, 0); + DCHECK_GT(heap_size, 0); + DCHECK_EQ(heap_begin % kHeapAlignment, 0); + DCHECK_EQ(heap_size % kHeapAlignment, 0); + DCHECK_LT(heap_begin, heap_begin + heap_size); + jctx = new(jctx_buf) JavaContext(heap_begin, heap_size); +} + +int __tsan_java_fini() { + JAVA_FUNC_ENTER(__tsan_java_fini); + DPrintf("#%d: java_fini()\n", thr->tid); + DCHECK_NE(jctx, 0); + // FIXME(dvyukov): this does not call atexit() callbacks. + int status = Finalize(thr); + DPrintf("#%d: java_fini() = %d\n", thr->tid, status); + return status; +} + +void __tsan_java_alloc(jptr ptr, jptr size) { + JAVA_FUNC_ENTER(__tsan_java_alloc); + DPrintf("#%d: java_alloc(0x%zx, 0x%zx)\n", thr->tid, ptr, size); + DCHECK_NE(jctx, 0); + DCHECK_NE(size, 0); + DCHECK_EQ(ptr % kHeapAlignment, 0); + DCHECK_EQ(size % kHeapAlignment, 0); + DCHECK_GE(ptr, jctx->heap_begin); + DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size); + + OnUserAlloc(thr, 0, ptr, size, false); +} + +void __tsan_java_free(jptr ptr, jptr size) { + JAVA_FUNC_ENTER(__tsan_java_free); + DPrintf("#%d: java_free(0x%zx, 0x%zx)\n", thr->tid, ptr, size); + DCHECK_NE(jctx, 0); + DCHECK_NE(size, 0); + DCHECK_EQ(ptr % kHeapAlignment, 0); + DCHECK_EQ(size % kHeapAlignment, 0); + DCHECK_GE(ptr, jctx->heap_begin); + DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size); + + ctx->metamap.FreeRange(thr->proc(), ptr, size); +} + +void __tsan_java_move(jptr src, jptr dst, jptr size) { + JAVA_FUNC_ENTER(__tsan_java_move); + DPrintf("#%d: java_move(0x%zx, 0x%zx, 0x%zx)\n", thr->tid, src, dst, size); + DCHECK_NE(jctx, 0); + DCHECK_NE(size, 0); + DCHECK_EQ(src % kHeapAlignment, 0); + DCHECK_EQ(dst % kHeapAlignment, 0); + DCHECK_EQ(size % kHeapAlignment, 0); + DCHECK_GE(src, jctx->heap_begin); + DCHECK_LE(src + size, jctx->heap_begin + jctx->heap_size); + DCHECK_GE(dst, jctx->heap_begin); + DCHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size); + DCHECK_NE(dst, src); + DCHECK_NE(size, 0); + + // Assuming it's not running concurrently with threads that do + // memory accesses and mutex operations (stop-the-world phase). + ctx->metamap.MoveMemory(src, dst, size); + + // Clear the destination shadow range. + // We used to move shadow from src to dst, but the trace format does not + // support that anymore as it contains addresses of accesses. + RawShadow *d = MemToShadow(dst); + RawShadow *dend = MemToShadow(dst + size); + internal_memset(d, 0, (dend - d) * sizeof(*d)); +} + +jptr __tsan_java_find(jptr *from_ptr, jptr to) { + JAVA_FUNC_ENTER(__tsan_java_find); + DPrintf("#%d: java_find(&0x%zx, 0x%zx)\n", thr->tid, *from_ptr, to); + DCHECK_EQ((*from_ptr) % kHeapAlignment, 0); + DCHECK_EQ(to % kHeapAlignment, 0); + DCHECK_GE(*from_ptr, jctx->heap_begin); + DCHECK_LE(to, jctx->heap_begin + jctx->heap_size); + for (uptr from = *from_ptr; from < to; from += kHeapAlignment) { + MBlock *b = ctx->metamap.GetBlock(from); + if (b) { + *from_ptr = from; + return b->siz; + } + } + return 0; +} + +void __tsan_java_finalize() { + JAVA_FUNC_ENTER(__tsan_java_finalize); + DPrintf("#%d: java_finalize()\n", thr->tid); + AcquireGlobal(thr); +} + +void __tsan_java_mutex_lock(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_mutex_lock); + DPrintf("#%d: java_mutex_lock(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + MutexPostLock(thr, 0, addr, + MutexFlagLinkerInit | MutexFlagWriteReentrant | + MutexFlagDoPreLockOnPostLock); +} + +void __tsan_java_mutex_unlock(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_mutex_unlock); + DPrintf("#%d: java_mutex_unlock(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + MutexUnlock(thr, 0, addr); +} + +void __tsan_java_mutex_read_lock(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_mutex_read_lock); + DPrintf("#%d: java_mutex_read_lock(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + MutexPostReadLock(thr, 0, addr, + MutexFlagLinkerInit | MutexFlagWriteReentrant | + MutexFlagDoPreLockOnPostLock); +} + +void __tsan_java_mutex_read_unlock(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_mutex_read_unlock); + DPrintf("#%d: java_mutex_read_unlock(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + MutexReadUnlock(thr, 0, addr); +} + +void __tsan_java_mutex_lock_rec(jptr addr, int rec) { + JAVA_FUNC_ENTER(__tsan_java_mutex_lock_rec); + DPrintf("#%d: java_mutex_lock_rec(0x%zx, %d)\n", thr->tid, addr, rec); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + DCHECK_GT(rec, 0); + + MutexPostLock(thr, 0, addr, + MutexFlagLinkerInit | MutexFlagWriteReentrant | + MutexFlagDoPreLockOnPostLock | MutexFlagRecursiveLock, + rec); +} + +int __tsan_java_mutex_unlock_rec(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_mutex_unlock_rec); + DPrintf("#%d: java_mutex_unlock_rec(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + return MutexUnlock(thr, 0, addr, MutexFlagRecursiveUnlock); +} + +void __tsan_java_acquire(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_acquire); + DPrintf("#%d: java_acquire(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + Acquire(thr, 0, addr); +} + +void __tsan_java_release(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_release); + DPrintf("#%d: java_release(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + Release(thr, 0, addr); +} + +void __tsan_java_release_store(jptr addr) { + JAVA_FUNC_ENTER(__tsan_java_release); + DPrintf("#%d: java_release_store(0x%zx)\n", thr->tid, addr); + DCHECK_NE(jctx, 0); + DCHECK_GE(addr, jctx->heap_begin); + DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size); + + ReleaseStore(thr, 0, addr); +} diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h new file mode 100644 index 000000000000..51b445251e09 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h @@ -0,0 +1,99 @@ +//===-- tsan_interface_java.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Interface for verification of Java or mixed Java/C++ programs. +// The interface is intended to be used from within a JVM and notify TSan +// about such events like Java locks and GC memory compaction. +// +// For plain memory accesses and function entry/exit a JVM is intended to use +// C++ interfaces: __tsan_readN/writeN and __tsan_func_enter/exit. +// +// For volatile memory accesses and atomic operations JVM is intended to use +// standard atomics API: __tsan_atomicN_load/store/etc. +// +// For usage examples see lit_tests/java_*.cpp +//===----------------------------------------------------------------------===// +#ifndef TSAN_INTERFACE_JAVA_H +#define TSAN_INTERFACE_JAVA_H + +#ifndef INTERFACE_ATTRIBUTE +# define INTERFACE_ATTRIBUTE __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef unsigned long jptr; + +// Must be called before any other callback from Java. +void __tsan_java_init(jptr heap_begin, jptr heap_size) INTERFACE_ATTRIBUTE; +// Must be called when the application exits. +// Not necessary the last callback (concurrently running threads are OK). +// Returns exit status or 0 if tsan does not want to override it. +int __tsan_java_fini() INTERFACE_ATTRIBUTE; + +// Callback for memory allocations. +// May be omitted for allocations that are not subject to data races +// nor contain synchronization objects (e.g. String). +void __tsan_java_alloc(jptr ptr, jptr size) INTERFACE_ATTRIBUTE; +// Callback for memory free. +// Can be aggregated for several objects (preferably). +void __tsan_java_free(jptr ptr, jptr size) INTERFACE_ATTRIBUTE; +// Callback for memory move by GC. +// Can be aggregated for several objects (preferably). +// The ranges can overlap. +void __tsan_java_move(jptr src, jptr dst, jptr size) INTERFACE_ATTRIBUTE; +// This function must be called on the finalizer thread +// before executing a batch of finalizers. +// It ensures necessary synchronization between +// java object creation and finalization. +void __tsan_java_finalize() INTERFACE_ATTRIBUTE; +// Finds the first allocated memory block in the [*from_ptr, to) range, saves +// its address in *from_ptr and returns its size. Returns 0 if there are no +// allocated memory blocks in the range. +jptr __tsan_java_find(jptr *from_ptr, jptr to) INTERFACE_ATTRIBUTE; + +// Mutex lock. +// Addr is any unique address associated with the mutex. +// Can be called on recursive reentry. +void __tsan_java_mutex_lock(jptr addr) INTERFACE_ATTRIBUTE; +// Mutex unlock. +void __tsan_java_mutex_unlock(jptr addr) INTERFACE_ATTRIBUTE; +// Mutex read lock. +void __tsan_java_mutex_read_lock(jptr addr) INTERFACE_ATTRIBUTE; +// Mutex read unlock. +void __tsan_java_mutex_read_unlock(jptr addr) INTERFACE_ATTRIBUTE; +// Recursive mutex lock, intended for handling of Object.wait(). +// The 'rec' value must be obtained from the previous +// __tsan_java_mutex_unlock_rec(). +void __tsan_java_mutex_lock_rec(jptr addr, int rec) INTERFACE_ATTRIBUTE; +// Recursive mutex unlock, intended for handling of Object.wait(). +// The return value says how many times this thread called lock() +// w/o a pairing unlock() (i.e. how many recursive levels it unlocked). +// It must be passed back to __tsan_java_mutex_lock_rec() to restore +// the same recursion level. +int __tsan_java_mutex_unlock_rec(jptr addr) INTERFACE_ATTRIBUTE; + +// Raw acquire/release primitives. +// Can be used to establish happens-before edges on volatile/final fields, +// in atomic operations, etc. release_store is the same as release, but it +// breaks release sequence on addr (see C++ standard 1.10/7 for details). +void __tsan_java_acquire(jptr addr) INTERFACE_ATTRIBUTE; +void __tsan_java_release(jptr addr) INTERFACE_ATTRIBUTE; +void __tsan_java_release_store(jptr addr) INTERFACE_ATTRIBUTE; + +#ifdef __cplusplus +} // extern "C" +#endif + +#undef INTERFACE_ATTRIBUTE + +#endif // #ifndef TSAN_INTERFACE_JAVA_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp new file mode 100644 index 000000000000..0e861bf1f962 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp @@ -0,0 +1,71 @@ +//===-- tsan_malloc_mac.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific malloc interception. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "sanitizer_common/sanitizer_errno.h" +#include "tsan_interceptors.h" +#include "tsan_stack_trace.h" + +using namespace __tsan; +#define COMMON_MALLOC_ZONE_NAME "tsan" +#define COMMON_MALLOC_ENTER() +#define COMMON_MALLOC_SANITIZER_INITIALIZED (cur_thread()->is_inited) +#define COMMON_MALLOC_FORCE_LOCK() +#define COMMON_MALLOC_FORCE_UNLOCK() +#define COMMON_MALLOC_MEMALIGN(alignment, size) \ + void *p = \ + user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) +#define COMMON_MALLOC_MALLOC(size) \ + if (in_symbolizer()) return InternalAlloc(size); \ + SCOPED_INTERCEPTOR_RAW(malloc, size); \ + void *p = user_alloc(thr, pc, size) +#define COMMON_MALLOC_REALLOC(ptr, size) \ + if (in_symbolizer()) return InternalRealloc(ptr, size); \ + SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ + void *p = user_realloc(thr, pc, ptr, size) +#define COMMON_MALLOC_CALLOC(count, size) \ + if (in_symbolizer()) return InternalCalloc(count, size); \ + SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ + void *p = user_calloc(thr, pc, size, count) +#define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ + if (in_symbolizer()) { \ + void *p = InternalAlloc(size, nullptr, alignment); \ + if (!p) return errno_ENOMEM; \ + *memptr = p; \ + return 0; \ + } \ + SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \ + int res = user_posix_memalign(thr, pc, memptr, alignment, size); +#define COMMON_MALLOC_VALLOC(size) \ + if (in_symbolizer()) \ + return InternalAlloc(size, nullptr, GetPageSizeCached()); \ + SCOPED_INTERCEPTOR_RAW(valloc, size); \ + void *p = user_valloc(thr, pc, size) +#define COMMON_MALLOC_FREE(ptr) \ + if (in_symbolizer()) return InternalFree(ptr); \ + SCOPED_INTERCEPTOR_RAW(free, ptr); \ + user_free(thr, pc, ptr) +#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); +#define COMMON_MALLOC_FILL_STATS(zone, stats) +#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ + (void)zone_name; \ + Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); +#define COMMON_MALLOC_NAMESPACE __tsan +#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0 + +#include "sanitizer_common/sanitizer_malloc_mac.inc" + +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp new file mode 100644 index 000000000000..72857b773fed --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp @@ -0,0 +1,250 @@ +//===-- tsan_md5.cpp ------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_defs.h" + +namespace __tsan { + +#define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y)))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define I(x, y, z) ((y) ^ ((x) | ~(z))) + +#define STEP(f, a, b, c, d, x, t, s) \ + (a) += f((b), (c), (d)) + (x) + (t); \ + (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \ + (a) += (b); + +#define SET(n) \ + (*(const MD5_u32plus *)&ptr[(n) * 4]) +#define GET(n) \ + SET(n) + +typedef unsigned int MD5_u32plus; +typedef unsigned long ulong_t; + +typedef struct { + MD5_u32plus lo, hi; + MD5_u32plus a, b, c, d; + unsigned char buffer[64]; + MD5_u32plus block[16]; +} MD5_CTX; + +static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) { + const unsigned char *ptr = (const unsigned char *)data; + MD5_u32plus a, b, c, d; + MD5_u32plus saved_a, saved_b, saved_c, saved_d; + + a = ctx->a; + b = ctx->b; + c = ctx->c; + d = ctx->d; + + do { + saved_a = a; + saved_b = b; + saved_c = c; + saved_d = d; + + STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7) + STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12) + STEP(F, c, d, a, b, SET(2), 0x242070db, 17) + STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22) + STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7) + STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12) + STEP(F, c, d, a, b, SET(6), 0xa8304613, 17) + STEP(F, b, c, d, a, SET(7), 0xfd469501, 22) + STEP(F, a, b, c, d, SET(8), 0x698098d8, 7) + STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12) + STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17) + STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22) + STEP(F, a, b, c, d, SET(12), 0x6b901122, 7) + STEP(F, d, a, b, c, SET(13), 0xfd987193, 12) + STEP(F, c, d, a, b, SET(14), 0xa679438e, 17) + STEP(F, b, c, d, a, SET(15), 0x49b40821, 22) + + STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5) + STEP(G, d, a, b, c, GET(6), 0xc040b340, 9) + STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14) + STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20) + STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5) + STEP(G, d, a, b, c, GET(10), 0x02441453, 9) + STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14) + STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20) + STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5) + STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9) + STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14) + STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20) + STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5) + STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9) + STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14) + STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20) + + STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4) + STEP(H, d, a, b, c, GET(8), 0x8771f681, 11) + STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16) + STEP(H, b, c, d, a, GET(14), 0xfde5380c, 23) + STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4) + STEP(H, d, a, b, c, GET(4), 0x4bdecfa9, 11) + STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16) + STEP(H, b, c, d, a, GET(10), 0xbebfbc70, 23) + STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4) + STEP(H, d, a, b, c, GET(0), 0xeaa127fa, 11) + STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16) + STEP(H, b, c, d, a, GET(6), 0x04881d05, 23) + STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4) + STEP(H, d, a, b, c, GET(12), 0xe6db99e5, 11) + STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16) + STEP(H, b, c, d, a, GET(2), 0xc4ac5665, 23) + + STEP(I, a, b, c, d, GET(0), 0xf4292244, 6) + STEP(I, d, a, b, c, GET(7), 0x432aff97, 10) + STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15) + STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21) + STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6) + STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10) + STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15) + STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21) + STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6) + STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10) + STEP(I, c, d, a, b, GET(6), 0xa3014314, 15) + STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21) + STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6) + STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10) + STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15) + STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21) + + a += saved_a; + b += saved_b; + c += saved_c; + d += saved_d; + + ptr += 64; + } while (size -= 64); + + ctx->a = a; + ctx->b = b; + ctx->c = c; + ctx->d = d; + + return ptr; +} + +#undef F +#undef G +#undef H +#undef I +#undef STEP +#undef SET +#undef GET + +void MD5_Init(MD5_CTX *ctx) { + ctx->a = 0x67452301; + ctx->b = 0xefcdab89; + ctx->c = 0x98badcfe; + ctx->d = 0x10325476; + + ctx->lo = 0; + ctx->hi = 0; +} + +void MD5_Update(MD5_CTX *ctx, const void *data, ulong_t size) { + MD5_u32plus saved_lo; + ulong_t used, free; + + saved_lo = ctx->lo; + if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo) + ctx->hi++; + ctx->hi += size >> 29; + + used = saved_lo & 0x3f; + + if (used) { + free = 64 - used; + + if (size < free) { + internal_memcpy(&ctx->buffer[used], data, size); + return; + } + + internal_memcpy(&ctx->buffer[used], data, free); + data = (const unsigned char *)data + free; + size -= free; + body(ctx, ctx->buffer, 64); + } + + if (size >= 64) { + data = body(ctx, data, size & ~(ulong_t)0x3f); + size &= 0x3f; + } + + internal_memcpy(ctx->buffer, data, size); +} + +void MD5_Final(unsigned char *result, MD5_CTX *ctx) { + ulong_t used, free; + + used = ctx->lo & 0x3f; + + ctx->buffer[used++] = 0x80; + + free = 64 - used; + + if (free < 8) { + internal_memset(&ctx->buffer[used], 0, free); + body(ctx, ctx->buffer, 64); + used = 0; + free = 64; + } + + internal_memset(&ctx->buffer[used], 0, free - 8); + + ctx->lo <<= 3; + ctx->buffer[56] = ctx->lo; + ctx->buffer[57] = ctx->lo >> 8; + ctx->buffer[58] = ctx->lo >> 16; + ctx->buffer[59] = ctx->lo >> 24; + ctx->buffer[60] = ctx->hi; + ctx->buffer[61] = ctx->hi >> 8; + ctx->buffer[62] = ctx->hi >> 16; + ctx->buffer[63] = ctx->hi >> 24; + + body(ctx, ctx->buffer, 64); + + result[0] = ctx->a; + result[1] = ctx->a >> 8; + result[2] = ctx->a >> 16; + result[3] = ctx->a >> 24; + result[4] = ctx->b; + result[5] = ctx->b >> 8; + result[6] = ctx->b >> 16; + result[7] = ctx->b >> 24; + result[8] = ctx->c; + result[9] = ctx->c >> 8; + result[10] = ctx->c >> 16; + result[11] = ctx->c >> 24; + result[12] = ctx->d; + result[13] = ctx->d >> 8; + result[14] = ctx->d >> 16; + result[15] = ctx->d >> 24; + + internal_memset(ctx, 0, sizeof(*ctx)); +} + +MD5Hash md5_hash(const void *data, uptr size) { + MD5Hash res; + MD5_CTX ctx; + MD5_Init(&ctx); + MD5_Update(&ctx, data, size); + MD5_Final((unsigned char*)&res.hash[0], &ctx); + return res; +} +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp new file mode 100644 index 000000000000..75044c38d5d2 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp @@ -0,0 +1,436 @@ +//===-- tsan_mman.cpp -----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_report.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_mman.h" +#include "tsan_rtl.h" +#include "tsan_report.h" +#include "tsan_flags.h" + +// May be overriden by front-end. +SANITIZER_WEAK_DEFAULT_IMPL +void __sanitizer_malloc_hook(void *ptr, uptr size) { + (void)ptr; + (void)size; +} + +SANITIZER_WEAK_DEFAULT_IMPL +void __sanitizer_free_hook(void *ptr) { + (void)ptr; +} + +namespace __tsan { + +struct MapUnmapCallback { + void OnMap(uptr p, uptr size) const { } + void OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // Mark the corresponding shadow memory as not needed. + DontNeedShadowFor(p, size); + // Mark the corresponding meta shadow memory as not needed. + // Note the block does not contain any meta info at this point + // (this happens after free). + const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize; + const uptr kPageSize = GetPageSizeCached() * kMetaRatio; + // Block came from LargeMmapAllocator, so must be large. + // We rely on this in the calculations below. + CHECK_GE(size, 2 * kPageSize); + uptr diff = RoundUp(p, kPageSize) - p; + if (diff != 0) { + p += diff; + size -= diff; + } + diff = p + size - RoundDown(p + size, kPageSize); + if (diff != 0) + size -= diff; + uptr p_meta = (uptr)MemToMeta(p); + ReleaseMemoryPagesToOS(p_meta, p_meta + size / kMetaRatio); + } +}; + +static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64); +Allocator *allocator() { + return reinterpret_cast(&allocator_placeholder); +} + +struct GlobalProc { + Mutex mtx; + Processor *proc; + // This mutex represents the internal allocator combined for + // the purposes of deadlock detection. The internal allocator + // uses multiple mutexes, moreover they are locked only occasionally + // and they are spin mutexes which don't support deadlock detection. + // So we use this fake mutex to serve as a substitute for these mutexes. + CheckedMutex internal_alloc_mtx; + + GlobalProc() + : mtx(MutexTypeGlobalProc), + proc(ProcCreate()), + internal_alloc_mtx(MutexTypeInternalAlloc) {} +}; + +static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64); +GlobalProc *global_proc() { + return reinterpret_cast(&global_proc_placeholder); +} + +static void InternalAllocAccess() { + global_proc()->internal_alloc_mtx.Lock(); + global_proc()->internal_alloc_mtx.Unlock(); +} + +ScopedGlobalProcessor::ScopedGlobalProcessor() { + GlobalProc *gp = global_proc(); + ThreadState *thr = cur_thread(); + if (thr->proc()) + return; + // If we don't have a proc, use the global one. + // There are currently only two known case where this path is triggered: + // __interceptor_free + // __nptl_deallocate_tsd + // start_thread + // clone + // and: + // ResetRange + // __interceptor_munmap + // __deallocate_stack + // start_thread + // clone + // Ideally, we destroy thread state (and unwire proc) when a thread actually + // exits (i.e. when we join/wait it). Then we would not need the global proc + gp->mtx.Lock(); + ProcWire(gp->proc, thr); +} + +ScopedGlobalProcessor::~ScopedGlobalProcessor() { + GlobalProc *gp = global_proc(); + ThreadState *thr = cur_thread(); + if (thr->proc() != gp->proc) + return; + ProcUnwire(gp->proc, thr); + gp->mtx.Unlock(); +} + +void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS { + global_proc()->mtx.Lock(); + global_proc()->internal_alloc_mtx.Lock(); + InternalAllocatorLock(); +} + +void AllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS { + InternalAllocatorUnlock(); + global_proc()->internal_alloc_mtx.Unlock(); + global_proc()->mtx.Unlock(); +} + +static constexpr uptr kMaxAllowedMallocSize = 1ull << 40; +static uptr max_user_defined_malloc_size; + +void InitializeAllocator() { + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator()->Init(common_flags()->allocator_release_to_os_interval_ms); + max_user_defined_malloc_size = common_flags()->max_allocation_size_mb + ? common_flags()->max_allocation_size_mb + << 20 + : kMaxAllowedMallocSize; +} + +void InitializeAllocatorLate() { + new(global_proc()) GlobalProc(); +} + +void AllocatorProcStart(Processor *proc) { + allocator()->InitCache(&proc->alloc_cache); + internal_allocator()->InitCache(&proc->internal_alloc_cache); +} + +void AllocatorProcFinish(Processor *proc) { + allocator()->DestroyCache(&proc->alloc_cache); + internal_allocator()->DestroyCache(&proc->internal_alloc_cache); +} + +void AllocatorPrintStats() { + allocator()->PrintStats(); +} + +static void SignalUnsafeCall(ThreadState *thr, uptr pc) { + if (atomic_load_relaxed(&thr->in_signal_handler) == 0 || + !ShouldReport(thr, ReportTypeSignalUnsafe)) + return; + VarSizeStackTrace stack; + ObtainCurrentStack(thr, pc, &stack); + if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack)) + return; + ThreadRegistryLock l(&ctx->thread_registry); + ScopedReport rep(ReportTypeSignalUnsafe); + rep.AddStack(stack, true); + OutputReport(thr, rep); +} + + +void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align, + bool signal) { + if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize || + sz > max_user_defined_malloc_size) { + if (AllocatorMayReturnNull()) + return nullptr; + uptr malloc_limit = + Min(kMaxAllowedMallocSize, max_user_defined_malloc_size); + GET_STACK_TRACE_FATAL(thr, pc); + ReportAllocationSizeTooBig(sz, malloc_limit, &stack); + } + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + GET_STACK_TRACE_FATAL(thr, pc); + ReportRssLimitExceeded(&stack); + } + void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align); + if (UNLIKELY(!p)) { + SetAllocatorOutOfMemory(); + if (AllocatorMayReturnNull()) + return nullptr; + GET_STACK_TRACE_FATAL(thr, pc); + ReportOutOfMemory(sz, &stack); + } + if (ctx && ctx->initialized) + OnUserAlloc(thr, pc, (uptr)p, sz, true); + if (signal) + SignalUnsafeCall(thr, pc); + return p; +} + +void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { + ScopedGlobalProcessor sgp; + if (ctx && ctx->initialized) + OnUserFree(thr, pc, (uptr)p, true); + allocator()->Deallocate(&thr->proc()->alloc_cache, p); + if (signal) + SignalUnsafeCall(thr, pc); +} + +void *user_alloc(ThreadState *thr, uptr pc, uptr sz) { + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment)); +} + +void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) { + if (AllocatorMayReturnNull()) + return SetErrnoOnNull(nullptr); + GET_STACK_TRACE_FATAL(thr, pc); + ReportCallocOverflow(n, size, &stack); + } + void *p = user_alloc_internal(thr, pc, n * size); + if (p) + internal_memset(p, 0, n * size); + return SetErrnoOnNull(p); +} + +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { + if (UNLIKELY(CheckForCallocOverflow(size, n))) { + if (AllocatorMayReturnNull()) + return SetErrnoOnNull(nullptr); + GET_STACK_TRACE_FATAL(thr, pc); + ReportReallocArrayOverflow(size, n, &stack); + } + return user_realloc(thr, pc, p, size * n); +} + +void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { + DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p); + ctx->metamap.AllocBlock(thr, pc, p, sz); + if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited) + MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); + else + MemoryResetRange(thr, pc, (uptr)p, sz); +} + +void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { + CHECK_NE(p, (void*)0); + uptr sz = ctx->metamap.FreeBlock(thr->proc(), p); + DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz); + if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited) + MemoryRangeFreed(thr, pc, (uptr)p, sz); +} + +void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { + // FIXME: Handle "shrinking" more efficiently, + // it seems that some software actually does this. + if (!p) + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz)); + if (!sz) { + user_free(thr, pc, p); + return nullptr; + } + void *new_p = user_alloc_internal(thr, pc, sz); + if (new_p) { + uptr old_sz = user_alloc_usable_size(p); + internal_memcpy(new_p, p, min(old_sz, sz)); + user_free(thr, pc, p); + } + return SetErrnoOnNull(new_p); +} + +void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) { + if (UNLIKELY(!IsPowerOfTwo(align))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + GET_STACK_TRACE_FATAL(thr, pc); + ReportInvalidAllocationAlignment(align, &stack); + } + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align)); +} + +int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, + uptr sz) { + if (UNLIKELY(!CheckPosixMemalignAlignment(align))) { + if (AllocatorMayReturnNull()) + return errno_EINVAL; + GET_STACK_TRACE_FATAL(thr, pc); + ReportInvalidPosixMemalignAlignment(align, &stack); + } + void *ptr = user_alloc_internal(thr, pc, sz, align); + if (UNLIKELY(!ptr)) + // OOM error is already taken care of by user_alloc_internal. + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, align)); + *memptr = ptr; + return 0; +} + +void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + GET_STACK_TRACE_FATAL(thr, pc); + ReportInvalidAlignedAllocAlignment(sz, align, &stack); + } + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align)); +} + +void *user_valloc(ThreadState *thr, uptr pc, uptr sz) { + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached())); +} + +void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + GET_STACK_TRACE_FATAL(thr, pc); + ReportPvallocOverflow(sz, &stack); + } + // pvalloc(0) should allocate one page. + sz = sz ? RoundUpTo(sz, PageSize) : PageSize; + return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize)); +} + +uptr user_alloc_usable_size(const void *p) { + if (p == 0) + return 0; + MBlock *b = ctx->metamap.GetBlock((uptr)p); + if (!b) + return 0; // Not a valid pointer. + if (b->siz == 0) + return 1; // Zero-sized allocations are actually 1 byte. + return b->siz; +} + +void invoke_malloc_hook(void *ptr, uptr size) { + ThreadState *thr = cur_thread(); + if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) + return; + __sanitizer_malloc_hook(ptr, size); + RunMallocHooks(ptr, size); +} + +void invoke_free_hook(void *ptr) { + ThreadState *thr = cur_thread(); + if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) + return; + __sanitizer_free_hook(ptr); + RunFreeHooks(ptr); +} + +void *Alloc(uptr sz) { + ThreadState *thr = cur_thread(); + if (thr->nomalloc) { + thr->nomalloc = 0; // CHECK calls internal_malloc(). + CHECK(0); + } + InternalAllocAccess(); + return InternalAlloc(sz, &thr->proc()->internal_alloc_cache); +} + +void FreeImpl(void *p) { + ThreadState *thr = cur_thread(); + if (thr->nomalloc) { + thr->nomalloc = 0; // CHECK calls internal_malloc(). + CHECK(0); + } + InternalAllocAccess(); + InternalFree(p, &thr->proc()->internal_alloc_cache); +} + +} // namespace __tsan + +using namespace __tsan; + +extern "C" { +uptr __sanitizer_get_current_allocated_bytes() { + uptr stats[AllocatorStatCount]; + allocator()->GetStats(stats); + return stats[AllocatorStatAllocated]; +} + +uptr __sanitizer_get_heap_size() { + uptr stats[AllocatorStatCount]; + allocator()->GetStats(stats); + return stats[AllocatorStatMapped]; +} + +uptr __sanitizer_get_free_bytes() { + return 1; +} + +uptr __sanitizer_get_unmapped_bytes() { + return 1; +} + +uptr __sanitizer_get_estimated_allocated_size(uptr size) { + return size; +} + +int __sanitizer_get_ownership(const void *p) { + return allocator()->GetBlockBegin(p) != 0; +} + +uptr __sanitizer_get_allocated_size(const void *p) { + return user_alloc_usable_size(p); +} + +void __tsan_on_thread_idle() { + ThreadState *thr = cur_thread(); + thr->clock.ResetCached(&thr->proc()->clock_cache); + thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); + allocator()->SwallowCache(&thr->proc()->alloc_cache); + internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache); + ctx->metamap.OnProcIdle(thr->proc()); +} +} // extern "C" diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mman.h b/compiler-rt/lib/tsan/rtl-old/tsan_mman.h new file mode 100644 index 000000000000..db8488eabbe2 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_mman.h @@ -0,0 +1,78 @@ +//===-- tsan_mman.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_MMAN_H +#define TSAN_MMAN_H + +#include "tsan_defs.h" + +namespace __tsan { + +const uptr kDefaultAlignment = 16; + +void InitializeAllocator(); +void InitializeAllocatorLate(); +void ReplaceSystemMalloc(); +void AllocatorProcStart(Processor *proc); +void AllocatorProcFinish(Processor *proc); +void AllocatorPrintStats(); +void AllocatorLock(); +void AllocatorUnlock(); + +// For user allocations. +void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, + uptr align = kDefaultAlignment, bool signal = true); +// Does not accept NULL. +void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true); +// Interceptor implementations. +void *user_alloc(ThreadState *thr, uptr pc, uptr sz); +void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n); +void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz); +void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n); +void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz); +int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align, + uptr sz); +void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz); +void *user_valloc(ThreadState *thr, uptr pc, uptr sz); +void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz); +uptr user_alloc_usable_size(const void *p); + +// Invoking malloc/free hooks that may be installed by the user. +void invoke_malloc_hook(void *ptr, uptr size); +void invoke_free_hook(void *ptr); + +// For internal data structures. +void *Alloc(uptr sz); +void FreeImpl(void *p); + +template +T *New(Args &&...args) { + return new (Alloc(sizeof(T))) T(static_cast(args)...); +} + +template +void Free(T *&p) { + if (p == nullptr) + return; + FreeImpl(p); + p = nullptr; +} + +template +void DestroyAndFree(T *&p) { + if (p == nullptr) + return; + p->~T(); + Free(p); +} + +} // namespace __tsan +#endif // TSAN_MMAN_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp new file mode 100644 index 000000000000..735179686ba9 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp @@ -0,0 +1,132 @@ +//===-- tsan_mutexset.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_mutexset.h" + +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_rtl.h" + +namespace __tsan { + +MutexSet::MutexSet() { +} + +void MutexSet::Add(u64 id, bool write, u64 epoch) { + // Look up existing mutex with the same id. + for (uptr i = 0; i < size_; i++) { + if (descs_[i].id == id) { + descs_[i].count++; + descs_[i].epoch = epoch; + return; + } + } + // On overflow, find the oldest mutex and drop it. + if (size_ == kMaxSize) { + u64 minepoch = (u64)-1; + u64 mini = (u64)-1; + for (uptr i = 0; i < size_; i++) { + if (descs_[i].epoch < minepoch) { + minepoch = descs_[i].epoch; + mini = i; + } + } + RemovePos(mini); + CHECK_EQ(size_, kMaxSize - 1); + } + // Add new mutex descriptor. + descs_[size_].addr = 0; + descs_[size_].stack_id = kInvalidStackID; + descs_[size_].id = id; + descs_[size_].write = write; + descs_[size_].epoch = epoch; + descs_[size_].seq = seq_++; + descs_[size_].count = 1; + size_++; +} + +void MutexSet::Del(u64 id, bool write) { + for (uptr i = 0; i < size_; i++) { + if (descs_[i].id == id) { + if (--descs_[i].count == 0) + RemovePos(i); + return; + } + } +} + +void MutexSet::Remove(u64 id) { + for (uptr i = 0; i < size_; i++) { + if (descs_[i].id == id) { + RemovePos(i); + return; + } + } +} + +void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) { + // Look up existing mutex with the same id. + for (uptr i = 0; i < size_; i++) { + if (descs_[i].addr == addr) { + descs_[i].count++; + descs_[i].seq = seq_++; + return; + } + } + // On overflow, find the oldest mutex and drop it. + if (size_ == kMaxSize) { + uptr min = 0; + for (uptr i = 0; i < size_; i++) { + if (descs_[i].seq < descs_[min].seq) + min = i; + } + RemovePos(min); + CHECK_EQ(size_, kMaxSize - 1); + } + // Add new mutex descriptor. + descs_[size_].addr = addr; + descs_[size_].stack_id = stack_id; + descs_[size_].id = 0; + descs_[size_].write = write; + descs_[size_].epoch = 0; + descs_[size_].seq = seq_++; + descs_[size_].count = 1; + size_++; +} + +void MutexSet::DelAddr(uptr addr, bool destroy) { + for (uptr i = 0; i < size_; i++) { + if (descs_[i].addr == addr) { + if (destroy || --descs_[i].count == 0) + RemovePos(i); + return; + } + } +} + +void MutexSet::RemovePos(uptr i) { + CHECK_LT(i, size_); + descs_[i] = descs_[size_ - 1]; + size_--; +} + +uptr MutexSet::Size() const { + return size_; +} + +MutexSet::Desc MutexSet::Get(uptr i) const { + CHECK_LT(i, size_); + return descs_[i]; +} + +DynamicMutexSet::DynamicMutexSet() : ptr_(New()) {} +DynamicMutexSet::~DynamicMutexSet() { DestroyAndFree(ptr_); } + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h new file mode 100644 index 000000000000..93776a664135 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h @@ -0,0 +1,98 @@ +//===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// MutexSet holds the set of mutexes currently held by a thread. +//===----------------------------------------------------------------------===// +#ifndef TSAN_MUTEXSET_H +#define TSAN_MUTEXSET_H + +#include "tsan_defs.h" + +namespace __tsan { + +class MutexSet { + public: + // Holds limited number of mutexes. + // The oldest mutexes are discarded on overflow. + static constexpr uptr kMaxSize = 16; + struct Desc { + uptr addr; + StackID stack_id; + u64 id; + u64 epoch; + u32 seq; + u32 count; + bool write; + + Desc() { internal_memset(this, 0, sizeof(*this)); } + Desc(const Desc& other) { *this = other; } + Desc& operator=(const MutexSet::Desc& other) { + internal_memcpy(this, &other, sizeof(*this)); + return *this; + } + }; + + MutexSet(); + // The 'id' is obtained from SyncVar::GetId(). + void Add(u64 id, bool write, u64 epoch); + void Del(u64 id, bool write); + void Remove(u64 id); // Removes the mutex completely (if it's destroyed). + void AddAddr(uptr addr, StackID stack_id, bool write); + void DelAddr(uptr addr, bool destroy = false); + uptr Size() const; + Desc Get(uptr i) const; + + private: +#if !SANITIZER_GO + u32 seq_ = 0; + uptr size_ = 0; + Desc descs_[kMaxSize]; + + void RemovePos(uptr i); +#endif +}; + +// MutexSet is too large to live on stack. +// DynamicMutexSet can be use used to create local MutexSet's. +class DynamicMutexSet { + public: + DynamicMutexSet(); + ~DynamicMutexSet(); + MutexSet* operator->() { return ptr_; } + operator MutexSet*() { return ptr_; } + DynamicMutexSet(const DynamicMutexSet&) = delete; + DynamicMutexSet& operator=(const DynamicMutexSet&) = delete; + + private: + MutexSet* ptr_; +#if SANITIZER_GO + MutexSet set_; +#endif +}; + +// Go does not have mutexes, so do not spend memory and time. +// (Go sync.Mutex is actually a semaphore -- can be unlocked +// in different goroutine). +#if SANITIZER_GO +MutexSet::MutexSet() {} +void MutexSet::Add(u64 id, bool write, u64 epoch) {} +void MutexSet::Del(u64 id, bool write) {} +void MutexSet::Remove(u64 id) {} +void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {} +void MutexSet::DelAddr(uptr addr, bool destroy) {} +uptr MutexSet::Size() const { return 0; } +MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); } +DynamicMutexSet::DynamicMutexSet() : ptr_(&set_) {} +DynamicMutexSet::~DynamicMutexSet() {} +#endif + +} // namespace __tsan + +#endif // TSAN_MUTEXSET_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp new file mode 100644 index 000000000000..fc44a5221b5b --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp @@ -0,0 +1,199 @@ +//===-- tsan_new_delete.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Interceptors for operators new and delete. +//===----------------------------------------------------------------------===// +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_report.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "tsan_interceptors.h" +#include "tsan_rtl.h" + +using namespace __tsan; + +namespace std { +struct nothrow_t {}; +enum class align_val_t: __sanitizer::uptr {}; +} // namespace std + +DECLARE_REAL(void *, malloc, uptr size) +DECLARE_REAL(void, free, void *ptr) + +// TODO(alekseys): throw std::bad_alloc instead of dying on OOM. +#define OPERATOR_NEW_BODY(mangled_name, nothrow) \ + if (in_symbolizer()) \ + return InternalAlloc(size); \ + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ + p = user_alloc(thr, pc, size); \ + if (!nothrow && UNLIKELY(!p)) { \ + GET_STACK_TRACE_FATAL(thr, pc); \ + ReportOutOfMemory(size, &stack); \ + } \ + } \ + invoke_malloc_hook(p, size); \ + return p; + +#define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \ + if (in_symbolizer()) \ + return InternalAlloc(size, nullptr, (uptr)align); \ + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ + p = user_memalign(thr, pc, (uptr)align, size); \ + if (!nothrow && UNLIKELY(!p)) { \ + GET_STACK_TRACE_FATAL(thr, pc); \ + ReportOutOfMemory(size, &stack); \ + } \ + } \ + invoke_malloc_hook(p, size); \ + return p; + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size); +void *operator new(__sanitizer::uptr size) { + OPERATOR_NEW_BODY(_Znwm, false /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size); +void *operator new[](__sanitizer::uptr size) { + OPERATOR_NEW_BODY(_Znam, false /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size, std::nothrow_t const&); +void *operator new(__sanitizer::uptr size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t, true /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size, std::nothrow_t const&); +void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size, std::align_val_t align); +void *operator new(__sanitizer::uptr size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_t, false /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size, std::align_val_t align); +void *operator new[](__sanitizer::uptr size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_t, false /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new(__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&); +void *operator new(__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&) { + OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_tRKSt9nothrow_t, + true /*nothrow*/); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *operator new[](__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&); +void *operator new[](__sanitizer::uptr size, std::align_val_t align, + std::nothrow_t const&) { + OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_tRKSt9nothrow_t, + true /*nothrow*/); +} + +#define OPERATOR_DELETE_BODY(mangled_name) \ + if (ptr == 0) return; \ + if (in_symbolizer()) \ + return InternalFree(ptr); \ + invoke_free_hook(ptr); \ + SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ + user_free(thr, pc, ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr) NOEXCEPT; +void operator delete(void *ptr) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPv); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr) NOEXCEPT; +void operator delete[](void *ptr) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPv); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const&); +void operator delete(void *ptr, std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const&); +void operator delete[](void *ptr, std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT; +void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPvm); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT; +void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPvm); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT; +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT; +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&); +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_tRKSt9nothrow_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const&); +void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_tRKSt9nothrow_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete(void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT; +void operator delete(void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdlPvmSt11align_val_t); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void operator delete[](void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT; +void operator delete[](void *ptr, __sanitizer::uptr size, + std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY(_ZdaPvmSt11align_val_t); +} diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform.h b/compiler-rt/lib/tsan/rtl-old/tsan_platform.h new file mode 100644 index 000000000000..7ff0acace8f6 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform.h @@ -0,0 +1,988 @@ +//===-- tsan_platform.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Platform-specific code. +//===----------------------------------------------------------------------===// + +#ifndef TSAN_PLATFORM_H +#define TSAN_PLATFORM_H + +#if !defined(__LP64__) && !defined(_WIN64) +# error "Only 64-bit is supported" +#endif + +#include "tsan_defs.h" +#include "tsan_trace.h" + +namespace __tsan { + +enum { + // App memory is not mapped onto shadow memory range. + kBrokenMapping = 1 << 0, + // Mapping app memory and back does not produce the same address, + // this can lead to wrong addresses in reports and potentially + // other bad consequences. + kBrokenReverseMapping = 1 << 1, + // Mapping is non-linear for linear user range. + // This is bad and can lead to unpredictable memory corruptions, etc + // because range access functions assume linearity. + kBrokenLinearity = 1 << 2, +}; + +/* +C/C++ on linux/x86_64 and freebsd/x86_64 +0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB) +0040 0000 0000 - 0100 0000 0000: - +0100 0000 0000 - 2000 0000 0000: shadow +2000 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 5500 0000 0000: - +5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels +5680 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 7d00 0000 0000: - +7b00 0000 0000 - 7c00 0000 0000: heap +7c00 0000 0000 - 7e80 0000 0000: - +7e80 0000 0000 - 8000 0000 0000: modules and main thread stack + +C/C++ on netbsd/amd64 can reuse the same mapping: + * The address space starts from 0x1000 (option with 0x0) and ends with + 0x7f7ffffff000. + * LoAppMem-kHeapMemEnd can be reused as it is. + * No VDSO support. + * No MidAppMem region. + * No additional HeapMem region. + * HiAppMem contains the stack, loader, shared libraries and heap. + * Stack on NetBSD/amd64 has prereserved 128MB. + * Heap grows downwards (top-down). + * ASLR must be disabled per-process or globally. +*/ +struct Mapping48AddressSpace { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x340000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x200000000000ull; + static const uptr kHeapMemBeg = 0x7b0000000000ull; + static const uptr kHeapMemEnd = 0x7c0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x008000000000ull; + static const uptr kMidAppMemBeg = 0x550000000000ull; + static const uptr kMidAppMemEnd = 0x568000000000ull; + static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kHiAppMemEnd = 0x800000000000ull; + static const uptr kShadowMsk = 0x780000000000ull; + static const uptr kShadowXor = 0x040000000000ull; + static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kVdsoBeg = 0xf000000000000000ull; +}; + +/* +C/C++ on linux/mips64 (40-bit VMA) +0000 0000 00 - 0100 0000 00: - (4 GB) +0100 0000 00 - 0200 0000 00: main binary (4 GB) +0200 0000 00 - 2000 0000 00: - (120 GB) +2000 0000 00 - 4000 0000 00: shadow (128 GB) +4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB) +5000 0000 00 - aa00 0000 00: - (360 GB) +aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB) +ab00 0000 00 - b000 0000 00: - (20 GB) +b000 0000 00 - b200 0000 00: traces (8 GB) +b200 0000 00 - fe00 0000 00: - (304 GB) +fe00 0000 00 - ff00 0000 00: heap (4 GB) +ff00 0000 00 - ff80 0000 00: - (2 GB) +ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB) +*/ +struct MappingMips64_40 { + static const uptr kMetaShadowBeg = 0x4000000000ull; + static const uptr kMetaShadowEnd = 0x5000000000ull; + static const uptr kTraceMemBeg = 0xb000000000ull; + static const uptr kTraceMemEnd = 0xb200000000ull; + static const uptr kShadowBeg = 0x2000000000ull; + static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kHeapMemBeg = 0xfe00000000ull; + static const uptr kHeapMemEnd = 0xff00000000ull; + static const uptr kLoAppMemBeg = 0x0100000000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kMidAppMemBeg = 0xaa00000000ull; + static const uptr kMidAppMemEnd = 0xab00000000ull; + static const uptr kHiAppMemBeg = 0xff80000000ull; + static const uptr kHiAppMemEnd = 0xffffffffffull; + static const uptr kShadowMsk = 0xf800000000ull; + static const uptr kShadowXor = 0x0800000000ull; + static const uptr kShadowAdd = 0x0000000000ull; + static const uptr kVdsoBeg = 0xfffff00000ull; +}; + +/* +C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) +0000 0000 00 - 0100 0000 00: - (4 GB) +0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) +0200 0000 00 - 0300 0000 00: heap (4 GB) +0300 0000 00 - 0400 0000 00: - (4 GB) +0400 0000 00 - 0c00 0000 00: shadow memory (32 GB) +0c00 0000 00 - 0d00 0000 00: - (4 GB) +0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) +0e00 0000 00 - 0f00 0000 00: - (4 GB) +0f00 0000 00 - 0fc0 0000 00: traces (3 GB) +0fc0 0000 00 - 1000 0000 00: - +*/ +struct MappingAppleAarch64 { + static const uptr kLoAppMemBeg = 0x0100000000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kHeapMemBeg = 0x0200000000ull; + static const uptr kHeapMemEnd = 0x0300000000ull; + static const uptr kShadowBeg = 0x0400000000ull; + static const uptr kShadowEnd = 0x0c00000000ull; + static const uptr kMetaShadowBeg = 0x0d00000000ull; + static const uptr kMetaShadowEnd = 0x0e00000000ull; + static const uptr kTraceMemBeg = 0x0f00000000ull; + static const uptr kTraceMemEnd = 0x0fc0000000ull; + static const uptr kHiAppMemBeg = 0x0fc0000000ull; + static const uptr kHiAppMemEnd = 0x0fc0000000ull; + static const uptr kShadowMsk = 0x0ull; + static const uptr kShadowXor = 0x0ull; + static const uptr kShadowAdd = 0x0ull; + static const uptr kVdsoBeg = 0x7000000000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; +}; + +/* +C/C++ on linux/aarch64 (39-bit VMA) +0000 0010 00 - 0100 0000 00: main binary +0100 0000 00 - 0800 0000 00: - +0800 0000 00 - 2000 0000 00: shadow memory +2000 0000 00 - 3100 0000 00: - +3100 0000 00 - 3400 0000 00: metainfo +3400 0000 00 - 5500 0000 00: - +5500 0000 00 - 5600 0000 00: main binary (PIE) +5600 0000 00 - 6000 0000 00: - +6000 0000 00 - 6200 0000 00: traces +6200 0000 00 - 7d00 0000 00: - +7c00 0000 00 - 7d00 0000 00: heap +7d00 0000 00 - 7fff ffff ff: modules and main thread stack +*/ +struct MappingAarch64_39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0100000000ull; + static const uptr kShadowBeg = 0x0800000000ull; + static const uptr kShadowEnd = 0x2000000000ull; + static const uptr kMetaShadowBeg = 0x3100000000ull; + static const uptr kMetaShadowEnd = 0x3400000000ull; + static const uptr kMidAppMemBeg = 0x5500000000ull; + static const uptr kMidAppMemEnd = 0x5600000000ull; + static const uptr kTraceMemBeg = 0x6000000000ull; + static const uptr kTraceMemEnd = 0x6200000000ull; + static const uptr kHeapMemBeg = 0x7c00000000ull; + static const uptr kHeapMemEnd = 0x7d00000000ull; + static const uptr kHiAppMemBeg = 0x7e00000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffull; + static const uptr kShadowMsk = 0x7800000000ull; + static const uptr kShadowXor = 0x0200000000ull; + static const uptr kShadowAdd = 0x0000000000ull; + static const uptr kVdsoBeg = 0x7f00000000ull; +}; + +/* +C/C++ on linux/aarch64 (42-bit VMA) +00000 0010 00 - 01000 0000 00: main binary +01000 0000 00 - 10000 0000 00: - +10000 0000 00 - 20000 0000 00: shadow memory +20000 0000 00 - 26000 0000 00: - +26000 0000 00 - 28000 0000 00: metainfo +28000 0000 00 - 2aa00 0000 00: - +2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) +2ab00 0000 00 - 36200 0000 00: - +36200 0000 00 - 36240 0000 00: traces +36240 0000 00 - 3e000 0000 00: - +3e000 0000 00 - 3f000 0000 00: heap +3f000 0000 00 - 3ffff ffff ff: modules and main thread stack +*/ +struct MappingAarch64_42 { + static const uptr kBroken = kBrokenReverseMapping; + static const uptr kLoAppMemBeg = 0x00000001000ull; + static const uptr kLoAppMemEnd = 0x01000000000ull; + static const uptr kShadowBeg = 0x10000000000ull; + static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kMetaShadowBeg = 0x26000000000ull; + static const uptr kMetaShadowEnd = 0x28000000000ull; + static const uptr kMidAppMemBeg = 0x2aa00000000ull; + static const uptr kMidAppMemEnd = 0x2ab00000000ull; + static const uptr kTraceMemBeg = 0x36200000000ull; + static const uptr kTraceMemEnd = 0x36400000000ull; + static const uptr kHeapMemBeg = 0x3e000000000ull; + static const uptr kHeapMemEnd = 0x3f000000000ull; + static const uptr kHiAppMemBeg = 0x3f000000000ull; + static const uptr kHiAppMemEnd = 0x3ffffffffffull; + static const uptr kShadowMsk = 0x3c000000000ull; + static const uptr kShadowXor = 0x04000000000ull; + static const uptr kShadowAdd = 0x00000000000ull; + static const uptr kVdsoBeg = 0x37f00000000ull; +}; + +struct MappingAarch64_48 { + static const uptr kLoAppMemBeg = 0x0000000001000ull; + static const uptr kLoAppMemEnd = 0x0000200000000ull; + static const uptr kShadowBeg = 0x0002000000000ull; + static const uptr kShadowEnd = 0x0004000000000ull; + static const uptr kMetaShadowBeg = 0x0005000000000ull; + static const uptr kMetaShadowEnd = 0x0006000000000ull; + static const uptr kMidAppMemBeg = 0x0aaaa00000000ull; + static const uptr kMidAppMemEnd = 0x0aaaf00000000ull; + static const uptr kTraceMemBeg = 0x0f06000000000ull; + static const uptr kTraceMemEnd = 0x0f06200000000ull; + static const uptr kHeapMemBeg = 0x0ffff00000000ull; + static const uptr kHeapMemEnd = 0x0ffff00000000ull; + static const uptr kHiAppMemBeg = 0x0ffff00000000ull; + static const uptr kHiAppMemEnd = 0x1000000000000ull; + static const uptr kShadowMsk = 0x0fff800000000ull; + static const uptr kShadowXor = 0x0000800000000ull; + static const uptr kShadowAdd = 0x0000000000000ull; + static const uptr kVdsoBeg = 0xffff000000000ull; +}; + +/* +C/C++ on linux/powerpc64 (44-bit VMA) +0000 0000 0100 - 0001 0000 0000: main binary +0001 0000 0000 - 0001 0000 0000: - +0001 0000 0000 - 0b00 0000 0000: shadow +0b00 0000 0000 - 0b00 0000 0000: - +0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) +0d00 0000 0000 - 0d00 0000 0000: - +0d00 0000 0000 - 0f00 0000 0000: traces +0f00 0000 0000 - 0f00 0000 0000: - +0f00 0000 0000 - 0f50 0000 0000: heap +0f50 0000 0000 - 0f60 0000 0000: - +0f60 0000 0000 - 1000 0000 0000: modules and main thread stack +*/ +struct MappingPPC64_44 { + static const uptr kBroken = + kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity; + static const uptr kMetaShadowBeg = 0x0b0000000000ull; + static const uptr kMetaShadowEnd = 0x0d0000000000ull; + static const uptr kTraceMemBeg = 0x0d0000000000ull; + static const uptr kTraceMemEnd = 0x0f0000000000ull; + static const uptr kShadowBeg = 0x000100000000ull; + static const uptr kShadowEnd = 0x0b0000000000ull; + static const uptr kLoAppMemBeg = 0x000000000100ull; + static const uptr kLoAppMemEnd = 0x000100000000ull; + static const uptr kHeapMemBeg = 0x0f0000000000ull; + static const uptr kHeapMemEnd = 0x0f5000000000ull; + static const uptr kHiAppMemBeg = 0x0f6000000000ull; + static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits + static const uptr kShadowMsk = 0x0f0000000000ull; + static const uptr kShadowXor = 0x002100000000ull; + static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kVdsoBeg = 0x3c0000000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; +}; + +/* +C/C++ on linux/powerpc64 (46-bit VMA) +0000 0000 1000 - 0100 0000 0000: main binary +0100 0000 0000 - 0200 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) +2000 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2200 0000 0000: traces +2200 0000 0000 - 3d00 0000 0000: - +3d00 0000 0000 - 3e00 0000 0000: heap +3e00 0000 0000 - 3e80 0000 0000: - +3e80 0000 0000 - 4000 0000 0000: modules and main thread stack +*/ +struct MappingPPC64_46 { + static const uptr kMetaShadowBeg = 0x100000000000ull; + static const uptr kMetaShadowEnd = 0x200000000000ull; + static const uptr kTraceMemBeg = 0x200000000000ull; + static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x3d0000000000ull; + static const uptr kHeapMemEnd = 0x3e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x3e8000000000ull; + static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits + static const uptr kShadowMsk = 0x3c0000000000ull; + static const uptr kShadowXor = 0x020000000000ull; + static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kVdsoBeg = 0x7800000000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; +}; + +/* +C/C++ on linux/powerpc64 (47-bit VMA) +0000 0000 1000 - 0100 0000 0000: main binary +0100 0000 0000 - 0200 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) +2000 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2200 0000 0000: traces +2200 0000 0000 - 7d00 0000 0000: - +7d00 0000 0000 - 7e00 0000 0000: heap +7e00 0000 0000 - 7e80 0000 0000: - +7e80 0000 0000 - 8000 0000 0000: modules and main thread stack +*/ +struct MappingPPC64_47 { + static const uptr kMetaShadowBeg = 0x100000000000ull; + static const uptr kMetaShadowEnd = 0x200000000000ull; + static const uptr kTraceMemBeg = 0x200000000000ull; + static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x7d0000000000ull; + static const uptr kHeapMemEnd = 0x7e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits + static const uptr kShadowMsk = 0x7c0000000000ull; + static const uptr kShadowXor = 0x020000000000ull; + static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kVdsoBeg = 0x7800000000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; +}; + +/* +C/C++ on linux/s390x +While the kernel provides a 64-bit address space, we have to restrict ourselves +to 48 bits due to how e.g. SyncVar::GetId() works. +0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB +0e00 0000 0000 - 4000 0000 0000: - +4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app) +8000 0000 0000 - 9000 0000 0000: - +9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) +9800 0000 0000 - a000 0000 0000: - +a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads) +b000 0000 0000 - be00 0000 0000: - +be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator) +*/ +struct MappingS390x { + static const uptr kMetaShadowBeg = 0x900000000000ull; + static const uptr kMetaShadowEnd = 0x980000000000ull; + static const uptr kTraceMemBeg = 0xa00000000000ull; + static const uptr kTraceMemEnd = 0xb00000000000ull; + static const uptr kShadowBeg = 0x400000000000ull; + static const uptr kShadowEnd = 0x800000000000ull; + static const uptr kHeapMemBeg = 0xbe0000000000ull; + static const uptr kHeapMemEnd = 0xc00000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x0e0000000000ull; + static const uptr kHiAppMemBeg = 0xc00000004000ull; + static const uptr kHiAppMemEnd = 0xc00000004000ull; + static const uptr kShadowMsk = 0xb00000000000ull; + static const uptr kShadowXor = 0x100000000000ull; + static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kVdsoBeg = 0xfffffffff000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; +}; + +/* Go on linux, darwin and freebsd on x86_64 +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2380 0000 0000: shadow +2380 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 8000 0000 0000: - +*/ + +struct MappingGo48 { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x238000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x00e000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x200000000000ull; +}; + +/* Go on windows +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00f8 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 0100 0000 0000: - +0100 0000 0000 - 0500 0000 0000: shadow +0500 0000 0000 - 0700 0000 0000: traces +0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects) +07d0 0000 0000 - 8000 0000 0000: - +*/ + +struct MappingGoWindows { + static const uptr kMetaShadowBeg = 0x070000000000ull; + static const uptr kMetaShadowEnd = 0x077000000000ull; + static const uptr kTraceMemBeg = 0x050000000000ull; + static const uptr kTraceMemEnd = 0x070000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x050000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x00e000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x010000000000ull; +}; + +/* Go on linux/powerpc64 (46-bit VMA) +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2380 0000 0000: shadow +2380 0000 0000 - 2400 0000 0000: - +2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects) +3400 0000 0000 - 3600 0000 0000: - +3600 0000 0000 - 3800 0000 0000: traces +3800 0000 0000 - 4000 0000 0000: - +*/ + +struct MappingGoPPC64_46 { + static const uptr kMetaShadowBeg = 0x240000000000ull; + static const uptr kMetaShadowEnd = 0x340000000000ull; + static const uptr kTraceMemBeg = 0x360000000000ull; + static const uptr kTraceMemEnd = 0x380000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x238000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x00e000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x200000000000ull; +}; + +/* Go on linux/powerpc64 (47-bit VMA) +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 3000 0000 0000: shadow +3000 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 8000 0000 0000: - +*/ + +struct MappingGoPPC64_47 { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x00e000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x200000000000ull; +}; + +/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA) +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 3000 0000 0000: shadow +3000 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 8000 0000 0000: - +*/ +struct MappingGoAarch64 { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x00e000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x200000000000ull; +}; + +/* +Go on linux/mips64 (47-bit VMA) +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 3000 0000 0000: shadow +3000 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 8000 0000 0000: - +*/ +struct MappingGoMips64_47 { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x00e000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x200000000000ull; +}; + +/* +Go on linux/s390x +0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB +1000 0000 0000 - 4000 0000 0000: - +4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app) +8000 0000 0000 - 9000 0000 0000: - +9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) +9800 0000 0000 - a000 0000 0000: - +a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads) +*/ +struct MappingGoS390x { + static const uptr kMetaShadowBeg = 0x900000000000ull; + static const uptr kMetaShadowEnd = 0x980000000000ull; + static const uptr kTraceMemBeg = 0xa00000000000ull; + static const uptr kTraceMemEnd = 0xb00000000000ull; + static const uptr kShadowBeg = 0x400000000000ull; + static const uptr kShadowEnd = 0x800000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x100000000000ull; + static const uptr kMidAppMemBeg = 0; + static const uptr kMidAppMemEnd = 0; + static const uptr kHiAppMemBeg = 0; + static const uptr kHiAppMemEnd = 0; + static const uptr kHeapMemBeg = 0; + static const uptr kHeapMemEnd = 0; + static const uptr kVdsoBeg = 0; + static const uptr kShadowMsk = 0; + static const uptr kShadowXor = 0; + static const uptr kShadowAdd = 0x400000000000ull; +}; + +extern uptr vmaSize; + +template +ALWAYS_INLINE auto SelectMapping(Arg arg) { +#if SANITIZER_GO +# if defined(__powerpc64__) + switch (vmaSize) { + case 46: + return Func::template Apply(arg); + case 47: + return Func::template Apply(arg); + } +# elif defined(__mips64) + return Func::template Apply(arg); +# elif defined(__s390x__) + return Func::template Apply(arg); +# elif defined(__aarch64__) + return Func::template Apply(arg); +# elif SANITIZER_WINDOWS + return Func::template Apply(arg); +# else + return Func::template Apply(arg); +# endif +#else // SANITIZER_GO +# if defined(__x86_64__) || SANITIZER_IOSSIM || SANITIZER_MAC && !SANITIZER_IOS + return Func::template Apply(arg); +# elif defined(__aarch64__) && defined(__APPLE__) + return Func::template Apply(arg); +# elif defined(__aarch64__) && !defined(__APPLE__) + switch (vmaSize) { + case 39: + return Func::template Apply(arg); + case 42: + return Func::template Apply(arg); + case 48: + return Func::template Apply(arg); + } +# elif defined(__powerpc64__) + switch (vmaSize) { + case 44: + return Func::template Apply(arg); + case 46: + return Func::template Apply(arg); + case 47: + return Func::template Apply(arg); + } +# elif defined(__mips64) + return Func::template Apply(arg); +# elif defined(__s390x__) + return Func::template Apply(arg); +# else +# error "unsupported platform" +# endif +#endif + Die(); +} + +template +void ForEachMapping() { + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); + Func::template Apply(); +} + +enum MappingType { + kLoAppMemBeg, + kLoAppMemEnd, + kHiAppMemBeg, + kHiAppMemEnd, + kMidAppMemBeg, + kMidAppMemEnd, + kHeapMemBeg, + kHeapMemEnd, + kShadowBeg, + kShadowEnd, + kMetaShadowBeg, + kMetaShadowEnd, + kTraceMemBeg, + kTraceMemEnd, + kVdsoBeg, +}; + +struct MappingField { + template + static uptr Apply(MappingType type) { + switch (type) { + case kLoAppMemBeg: + return Mapping::kLoAppMemBeg; + case kLoAppMemEnd: + return Mapping::kLoAppMemEnd; + case kMidAppMemBeg: + return Mapping::kMidAppMemBeg; + case kMidAppMemEnd: + return Mapping::kMidAppMemEnd; + case kHiAppMemBeg: + return Mapping::kHiAppMemBeg; + case kHiAppMemEnd: + return Mapping::kHiAppMemEnd; + case kHeapMemBeg: + return Mapping::kHeapMemBeg; + case kHeapMemEnd: + return Mapping::kHeapMemEnd; + case kVdsoBeg: + return Mapping::kVdsoBeg; + case kShadowBeg: + return Mapping::kShadowBeg; + case kShadowEnd: + return Mapping::kShadowEnd; + case kMetaShadowBeg: + return Mapping::kMetaShadowBeg; + case kMetaShadowEnd: + return Mapping::kMetaShadowEnd; + case kTraceMemBeg: + return Mapping::kTraceMemBeg; + case kTraceMemEnd: + return Mapping::kTraceMemEnd; + } + Die(); + } +}; + +ALWAYS_INLINE +uptr LoAppMemBeg(void) { return SelectMapping(kLoAppMemBeg); } +ALWAYS_INLINE +uptr LoAppMemEnd(void) { return SelectMapping(kLoAppMemEnd); } + +ALWAYS_INLINE +uptr MidAppMemBeg(void) { return SelectMapping(kMidAppMemBeg); } +ALWAYS_INLINE +uptr MidAppMemEnd(void) { return SelectMapping(kMidAppMemEnd); } + +ALWAYS_INLINE +uptr HeapMemBeg(void) { return SelectMapping(kHeapMemBeg); } +ALWAYS_INLINE +uptr HeapMemEnd(void) { return SelectMapping(kHeapMemEnd); } + +ALWAYS_INLINE +uptr HiAppMemBeg(void) { return SelectMapping(kHiAppMemBeg); } +ALWAYS_INLINE +uptr HiAppMemEnd(void) { return SelectMapping(kHiAppMemEnd); } + +ALWAYS_INLINE +uptr VdsoBeg(void) { return SelectMapping(kVdsoBeg); } + +ALWAYS_INLINE +uptr ShadowBeg(void) { return SelectMapping(kShadowBeg); } +ALWAYS_INLINE +uptr ShadowEnd(void) { return SelectMapping(kShadowEnd); } + +ALWAYS_INLINE +uptr MetaShadowBeg(void) { return SelectMapping(kMetaShadowBeg); } +ALWAYS_INLINE +uptr MetaShadowEnd(void) { return SelectMapping(kMetaShadowEnd); } + +ALWAYS_INLINE +uptr TraceMemBeg(void) { return SelectMapping(kTraceMemBeg); } +ALWAYS_INLINE +uptr TraceMemEnd(void) { return SelectMapping(kTraceMemEnd); } + +struct IsAppMemImpl { + template + static bool Apply(uptr mem) { + return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || + (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || + (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || + (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); + } +}; + +ALWAYS_INLINE +bool IsAppMem(uptr mem) { return SelectMapping(mem); } + +struct IsShadowMemImpl { + template + static bool Apply(uptr mem) { + return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; + } +}; + +ALWAYS_INLINE +bool IsShadowMem(RawShadow *p) { + return SelectMapping(reinterpret_cast(p)); +} + +struct IsMetaMemImpl { + template + static bool Apply(uptr mem) { + return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; + } +}; + +ALWAYS_INLINE +bool IsMetaMem(const u32 *p) { + return SelectMapping(reinterpret_cast(p)); +} + +struct MemToShadowImpl { + template + static uptr Apply(uptr x) { + DCHECK(IsAppMemImpl::Apply(x)); + return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^ + Mapping::kShadowXor) * + kShadowMultiplier + + Mapping::kShadowAdd; + } +}; + +ALWAYS_INLINE +RawShadow *MemToShadow(uptr x) { + return reinterpret_cast(SelectMapping(x)); +} + +struct MemToMetaImpl { + template + static u32 *Apply(uptr x) { + DCHECK(IsAppMemImpl::Apply(x)); + return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) / + kMetaShadowCell * kMetaShadowSize) | + Mapping::kMetaShadowBeg); + } +}; + +ALWAYS_INLINE +u32 *MemToMeta(uptr x) { return SelectMapping(x); } + +struct ShadowToMemImpl { + template + static uptr Apply(uptr sp) { + if (!IsShadowMemImpl::Apply(sp)) + return 0; + // The shadow mapping is non-linear and we've lost some bits, so we don't + // have an easy way to restore the original app address. But the mapping is + // a bijection, so we try to restore the address as belonging to + // low/mid/high range consecutively and see if shadow->app->shadow mapping + // gives us the same address. + uptr p = + ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor; + if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd && + MemToShadowImpl::Apply(p) == sp) + return p; + if (Mapping::kMidAppMemBeg) { + uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk); + if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd && + MemToShadowImpl::Apply(p_mid) == sp) + return p_mid; + } + return p | Mapping::kShadowMsk; + } +}; + +ALWAYS_INLINE +uptr ShadowToMem(RawShadow *s) { + return SelectMapping(reinterpret_cast(s)); +} + +// Compresses addr to kCompressedAddrBits stored in least significant bits. +ALWAYS_INLINE uptr CompressAddr(uptr addr) { + return addr & ((1ull << kCompressedAddrBits) - 1); +} + +struct RestoreAddrImpl { + typedef uptr Result; + template + static Result Apply(uptr addr) { + // To restore the address we go over all app memory ranges and check if top + // 3 bits of the compressed addr match that of the app range. If yes, we + // assume that the compressed address come from that range and restore the + // missing top bits to match the app range address. + const uptr ranges[] = { + Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg, + Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd, + Mapping::kHeapMemBeg, Mapping::kHeapMemEnd, + }; + const uptr indicator = 0x0e0000000000ull; + const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator); + for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) { + uptr beg = ranges[i]; + uptr end = ranges[i + 1]; + if (beg == end) + continue; + for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) { + if ((addr & indicator) == (p & indicator)) + return addr | (p & ~(ind_lsb - 1)); + } + } + Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr); + Die(); + } +}; + +// Restores compressed addr from kCompressedAddrBits to full representation. +// This is called only during reporting and is not performance-critical. +inline uptr RestoreAddr(uptr addr) { + return SelectMapping(addr); +} + +// The additional page is to catch shadow stack overflow as paging fault. +// Windows wants 64K alignment for mmaps. +const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); + +struct GetThreadTraceImpl { + template + static uptr Apply(uptr tid) { + uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize; + DCHECK_LT(p, Mapping::kTraceMemEnd); + return p; + } +}; + +ALWAYS_INLINE +uptr GetThreadTrace(int tid) { return SelectMapping(tid); } + +struct GetThreadTraceHeaderImpl { + template + static uptr Apply(uptr tid) { + uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize + + kTraceSize * sizeof(Event); + DCHECK_LT(p, Mapping::kTraceMemEnd); + return p; + } +}; + +ALWAYS_INLINE +uptr GetThreadTraceHeader(int tid) { + return SelectMapping(tid); +} + +void InitializePlatform(); +void InitializePlatformEarly(); +void CheckAndProtect(); +void InitializeShadowMemoryPlatform(); +void FlushShadowMemory(); +void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns); +int ExtractResolvFDs(void *state, int *fds, int nfd); +int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); +uptr ExtractLongJmpSp(uptr *env); +void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size); + +int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), + void (*cleanup)(void *arg), void *arg); + +void DestroyThreadState(); +void PlatformCleanUpThreadState(ThreadState *thr); + +} // namespace __tsan + +#endif // TSAN_PLATFORM_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp new file mode 100644 index 000000000000..73ec14892d28 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp @@ -0,0 +1,545 @@ +//===-- tsan_platform_linux.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Linux- and BSD-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_netbsd.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_posix.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_stoptheworld.h" +#include "tsan_flags.h" +#include "tsan_platform.h" +#include "tsan_rtl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#if SANITIZER_LINUX +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if SANITIZER_LINUX +#define __need_res_state +#include +#endif + +#ifdef sa_handler +# undef sa_handler +#endif + +#ifdef sa_sigaction +# undef sa_sigaction +#endif + +#if SANITIZER_FREEBSD +extern "C" void *__libc_stack_end; +void *__libc_stack_end = 0; +#endif + +#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO +# define INIT_LONGJMP_XOR_KEY 1 +#else +# define INIT_LONGJMP_XOR_KEY 0 +#endif + +#if INIT_LONGJMP_XOR_KEY +#include "interception/interception.h" +// Must be declared outside of other namespaces. +DECLARE_REAL(int, _setjmp, void *env) +#endif + +namespace __tsan { + +#if INIT_LONGJMP_XOR_KEY +static void InitializeLongjmpXorKey(); +static uptr longjmp_xor_key; +#endif + +// Runtime detected VMA size. +uptr vmaSize; + +enum { + MemTotal, + MemShadow, + MemMeta, + MemFile, + MemMmap, + MemTrace, + MemHeap, + MemOther, + MemCount, +}; + +void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) { + mem[MemTotal] += rss; + if (p >= ShadowBeg() && p < ShadowEnd()) + mem[MemShadow] += rss; + else if (p >= MetaShadowBeg() && p < MetaShadowEnd()) + mem[MemMeta] += rss; + else if ((p >= LoAppMemBeg() && p < LoAppMemEnd()) || + (p >= MidAppMemBeg() && p < MidAppMemEnd()) || + (p >= HiAppMemBeg() && p < HiAppMemEnd())) + mem[file ? MemFile : MemMmap] += rss; + else if (p >= HeapMemBeg() && p < HeapMemEnd()) + mem[MemHeap] += rss; + else if (p >= TraceMemBeg() && p < TraceMemEnd()) + mem[MemTrace] += rss; + else + mem[MemOther] += rss; +} + +void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { + uptr mem[MemCount]; + internal_memset(mem, 0, sizeof(mem)); + GetMemoryProfile(FillProfileCallback, mem); + auto meta = ctx->metamap.GetMemoryStats(); + StackDepotStats stacks = StackDepotGetStats(); + uptr nthread, nlive; + ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive); + uptr internal_stats[AllocatorStatCount]; + internal_allocator()->GetStats(internal_stats); + // All these are allocated from the common mmap region. + mem[MemMmap] -= meta.mem_block + meta.sync_obj + stacks.allocated + + internal_stats[AllocatorStatMapped]; + if (s64(mem[MemMmap]) < 0) + mem[MemMmap] = 0; + internal_snprintf( + buf, buf_size, + "%llus: RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd" + " trace:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu" + " stacks=%zd[%zd] nthr=%zd/%zd\n", + uptime_ns / (1000 * 1000 * 1000), mem[MemTotal] >> 20, + mem[MemShadow] >> 20, mem[MemMeta] >> 20, mem[MemFile] >> 20, + mem[MemMmap] >> 20, mem[MemTrace] >> 20, mem[MemHeap] >> 20, + mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20, + meta.mem_block >> 20, meta.sync_obj >> 20, stacks.allocated >> 20, + stacks.n_uniq_ids, nlive, nthread); +} + +# if SANITIZER_LINUX +void FlushShadowMemoryCallback( + const SuspendedThreadsList &suspended_threads_list, + void *argument) { + ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd()); +} +#endif + +void FlushShadowMemory() { +#if SANITIZER_LINUX + StopTheWorld(FlushShadowMemoryCallback, 0); +#endif +} + +#if !SANITIZER_GO +// Mark shadow for .rodata sections with the special kShadowRodata marker. +// Accesses to .rodata can't race, so this saves time, memory and trace space. +static void MapRodata() { + // First create temp file. + const char *tmpdir = GetEnv("TMPDIR"); + if (tmpdir == 0) + tmpdir = GetEnv("TEST_TMPDIR"); +#ifdef P_tmpdir + if (tmpdir == 0) + tmpdir = P_tmpdir; +#endif + if (tmpdir == 0) + return; + char name[256]; + internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d", + tmpdir, (int)internal_getpid()); + uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); + if (internal_iserror(openrv)) + return; + internal_unlink(name); // Unlink it now, so that we can reuse the buffer. + fd_t fd = openrv; + // Fill the file with kShadowRodata. + const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow); + InternalMmapVector marker(kMarkerSize); + // volatile to prevent insertion of memset + for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize; + p++) + *p = kShadowRodata; + internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow)); + // Map the file into memory. + uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); + if (internal_iserror(page)) { + internal_close(fd); + return; + } + // Map the file into shadow of .rodata sections. + MemoryMappingLayout proc_maps(/*cache_enabled*/true); + // Reusing the buffer 'name'. + MemoryMappedSegment segment(name, ARRAY_SIZE(name)); + while (proc_maps.Next(&segment)) { + if (segment.filename[0] != 0 && segment.filename[0] != '[' && + segment.IsReadable() && segment.IsExecutable() && + !segment.IsWritable() && IsAppMem(segment.start)) { + // Assume it's .rodata + char *shadow_start = (char *)MemToShadow(segment.start); + char *shadow_end = (char *)MemToShadow(segment.end); + for (char *p = shadow_start; p < shadow_end; + p += marker.size() * sizeof(RawShadow)) { + internal_mmap( + p, Min(marker.size() * sizeof(RawShadow), shadow_end - p), + PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); + } + } + } + internal_close(fd); +} + +void InitializeShadowMemoryPlatform() { + MapRodata(); +} + +#endif // #if !SANITIZER_GO + +void InitializePlatformEarly() { + vmaSize = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); +#if defined(__aarch64__) +# if !SANITIZER_GO + if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize); + Die(); + } +#else + if (vmaSize != 48) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 48\n", vmaSize); + Die(); + } +#endif +#elif defined(__powerpc64__) +# if !SANITIZER_GO + if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize); + Die(); + } +# else + if (vmaSize != 46 && vmaSize != 47) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize); + Die(); + } +# endif +#elif defined(__mips64) +# if !SANITIZER_GO + if (vmaSize != 40) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 40\n", vmaSize); + Die(); + } +# else + if (vmaSize != 47) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 47\n", vmaSize); + Die(); + } +# endif +#endif +} + +void InitializePlatform() { + DisableCoreDumperIfNecessary(); + + // Go maps shadow memory lazily and works fine with limited address space. + // Unlimited stack is not a problem as well, because the executable + // is not compiled with -pie. +#if !SANITIZER_GO + { + bool reexec = false; + // TSan doesn't play well with unlimited stack size (as stack + // overlaps with shadow memory). If we detect unlimited stack size, + // we re-exec the program with limited stack size as a best effort. + if (StackSizeIsUnlimited()) { + const uptr kMaxStackSize = 32 * 1024 * 1024; + VReport(1, "Program is run with unlimited stack size, which wouldn't " + "work with ThreadSanitizer.\n" + "Re-execing with stack size limited to %zd bytes.\n", + kMaxStackSize); + SetStackSizeLimitInBytes(kMaxStackSize); + reexec = true; + } + + if (!AddressSpaceIsUnlimited()) { + Report("WARNING: Program is run with limited virtual address space," + " which wouldn't work with ThreadSanitizer.\n"); + Report("Re-execing with unlimited virtual address space.\n"); + SetAddressSpaceUnlimited(); + reexec = true; + } +#if SANITIZER_LINUX && defined(__aarch64__) + // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in + // linux kernel, the random gap between stack and mapped area is increased + // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover + // this big range, we should disable randomized virtual space on aarch64. + int old_personality = personality(0xffffffff); + if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { + VReport(1, "WARNING: Program is run with randomized virtual address " + "space, which wouldn't work with ThreadSanitizer.\n" + "Re-execing with fixed virtual address space.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + reexec = true; + } + // Initialize the xor key used in {sig}{set,long}jump. + InitializeLongjmpXorKey(); +#endif + if (reexec) + ReExec(); + } + + CheckAndProtect(); + InitTlsSize(); +#endif // !SANITIZER_GO +} + +#if !SANITIZER_GO +// Extract file descriptors passed to glibc internal __res_iclose function. +// This is required to properly "close" the fds, because we do not see internal +// closes within glibc. The code is a pure hack. +int ExtractResolvFDs(void *state, int *fds, int nfd) { +#if SANITIZER_LINUX && !SANITIZER_ANDROID + int cnt = 0; + struct __res_state *statp = (struct __res_state*)state; + for (int i = 0; i < MAXNS && cnt < nfd; i++) { + if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1) + fds[cnt++] = statp->_u._ext.nssocks[i]; + } + return cnt; +#else + return 0; +#endif +} + +// Extract file descriptors passed via UNIX domain sockets. +// This is required to properly handle "open" of these fds. +// see 'man recvmsg' and 'man 3 cmsg'. +int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { + int res = 0; + msghdr *msg = (msghdr*)msgp; + struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); + for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) + continue; + int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]); + for (int i = 0; i < n; i++) { + fds[res++] = ((int*)CMSG_DATA(cmsg))[i]; + if (res == nfd) + return res; + } + } + return res; +} + +// Reverse operation of libc stack pointer mangling +static uptr UnmangleLongJmpSp(uptr mangled_sp) { +#if defined(__x86_64__) +# if SANITIZER_LINUX + // Reverse of: + // xor %fs:0x30, %rsi + // rol $0x11, %rsi + uptr sp; + asm("ror $0x11, %0 \n" + "xor %%fs:0x30, %0 \n" + : "=r" (sp) + : "0" (mangled_sp)); + return sp; +# else + return mangled_sp; +# endif +#elif defined(__aarch64__) +# if SANITIZER_LINUX + return mangled_sp ^ longjmp_xor_key; +# else + return mangled_sp; +# endif +#elif defined(__powerpc64__) + // Reverse of: + // ld r4, -28696(r13) + // xor r4, r3, r4 + uptr xor_key; + asm("ld %0, -28696(%%r13)" : "=r" (xor_key)); + return mangled_sp ^ xor_key; +#elif defined(__mips__) + return mangled_sp; +#elif defined(__s390x__) + // tcbhead_t.stack_guard + uptr xor_key = ((uptr *)__builtin_thread_pointer())[5]; + return mangled_sp ^ xor_key; +#else + #error "Unknown platform" +#endif +} + +#if SANITIZER_NETBSD +# ifdef __x86_64__ +# define LONG_JMP_SP_ENV_SLOT 6 +# else +# error unsupported +# endif +#elif defined(__powerpc__) +# define LONG_JMP_SP_ENV_SLOT 0 +#elif SANITIZER_FREEBSD +# define LONG_JMP_SP_ENV_SLOT 2 +#elif SANITIZER_LINUX +# ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT 13 +# elif defined(__mips64) +# define LONG_JMP_SP_ENV_SLOT 1 +# elif defined(__s390x__) +# define LONG_JMP_SP_ENV_SLOT 9 +# else +# define LONG_JMP_SP_ENV_SLOT 6 +# endif +#endif + +uptr ExtractLongJmpSp(uptr *env) { + uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; + return UnmangleLongJmpSp(mangled_sp); +} + +#if INIT_LONGJMP_XOR_KEY +// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp +// functions) by XORing them with a random key. For AArch64 it is a global +// variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by +// issuing a setjmp and XORing the SP pointer values to derive the key. +static void InitializeLongjmpXorKey() { + // 1. Call REAL(setjmp), which stores the mangled SP in env. + jmp_buf env; + REAL(_setjmp)(env); + + // 2. Retrieve vanilla/mangled SP. + uptr sp; + asm("mov %0, sp" : "=r" (sp)); + uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT]; + + // 3. xor SPs to obtain key. + longjmp_xor_key = mangled_sp ^ sp; +} +#endif + +extern "C" void __tsan_tls_initialization() {} + +void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { + // Check that the thr object is in tls; + const uptr thr_beg = (uptr)thr; + const uptr thr_end = (uptr)thr + sizeof(*thr); + CHECK_GE(thr_beg, tls_addr); + CHECK_LE(thr_beg, tls_addr + tls_size); + CHECK_GE(thr_end, tls_addr); + CHECK_LE(thr_end, tls_addr + tls_size); + // Since the thr object is huge, skip it. + const uptr pc = StackTrace::GetNextInstructionPc( + reinterpret_cast(__tsan_tls_initialization)); + MemoryRangeImitateWrite(thr, pc, tls_addr, thr_beg - tls_addr); + MemoryRangeImitateWrite(thr, pc, thr_end, tls_addr + tls_size - thr_end); +} + +// Note: this function runs with async signals enabled, +// so it must not touch any tsan state. +int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), + void (*cleanup)(void *arg), void *arg) { + // pthread_cleanup_push/pop are hardcore macros mess. + // We can't intercept nor call them w/o including pthread.h. + int res; + pthread_cleanup_push(cleanup, arg); + res = fn(arg); + pthread_cleanup_pop(0); + return res; +} +#endif // !SANITIZER_GO + +#if !SANITIZER_GO +void ReplaceSystemMalloc() { } +#endif + +#if !SANITIZER_GO +#if SANITIZER_ANDROID +// On Android, one thread can call intercepted functions after +// DestroyThreadState(), so add a fake thread state for "dead" threads. +static ThreadState *dead_thread_state = nullptr; + +ThreadState *cur_thread() { + ThreadState* thr = reinterpret_cast(*get_android_tls_ptr()); + if (thr == nullptr) { + __sanitizer_sigset_t emptyset; + internal_sigfillset(&emptyset); + __sanitizer_sigset_t oldset; + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); + thr = reinterpret_cast(*get_android_tls_ptr()); + if (thr == nullptr) { + thr = reinterpret_cast(MmapOrDie(sizeof(ThreadState), + "ThreadState")); + *get_android_tls_ptr() = reinterpret_cast(thr); + if (dead_thread_state == nullptr) { + dead_thread_state = reinterpret_cast( + MmapOrDie(sizeof(ThreadState), "ThreadState")); + dead_thread_state->fast_state.SetIgnoreBit(); + dead_thread_state->ignore_interceptors = 1; + dead_thread_state->is_dead = true; + *const_cast(&dead_thread_state->tid) = -1; + CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState), + PROT_READ)); + } + } + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); + } + return thr; +} + +void set_cur_thread(ThreadState *thr) { + *get_android_tls_ptr() = reinterpret_cast(thr); +} + +void cur_thread_finalize() { + __sanitizer_sigset_t emptyset; + internal_sigfillset(&emptyset); + __sanitizer_sigset_t oldset; + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); + ThreadState* thr = reinterpret_cast(*get_android_tls_ptr()); + if (thr != dead_thread_state) { + *get_android_tls_ptr() = reinterpret_cast(dead_thread_state); + UnmapOrDie(thr, sizeof(ThreadState)); + } + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); +} +#endif // SANITIZER_ANDROID +#endif // if !SANITIZER_GO + +} // namespace __tsan + +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp new file mode 100644 index 000000000000..1465f9953c19 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp @@ -0,0 +1,326 @@ +//===-- tsan_platform_mac.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_posix.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_ptrauth.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "tsan_platform.h" +#include "tsan_rtl.h" +#include "tsan_flags.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace __tsan { + +#if !SANITIZER_GO +static char main_thread_state[sizeof(ThreadState)] ALIGNED( + SANITIZER_CACHE_LINE_SIZE); +static ThreadState *dead_thread_state; +static pthread_key_t thread_state_key; + +// We rely on the following documented, but Darwin-specific behavior to keep the +// reference to the ThreadState object alive in TLS: +// pthread_key_create man page: +// If, after all the destructors have been called for all non-NULL values with +// associated destructors, there are still some non-NULL values with +// associated destructors, then the process is repeated. If, after at least +// [PTHREAD_DESTRUCTOR_ITERATIONS] iterations of destructor calls for +// outstanding non-NULL values, there are still some non-NULL values with +// associated destructors, the implementation stops calling destructors. +static_assert(PTHREAD_DESTRUCTOR_ITERATIONS == 4, "Small number of iterations"); +static void ThreadStateDestructor(void *thr) { + int res = pthread_setspecific(thread_state_key, thr); + CHECK_EQ(res, 0); +} + +static void InitializeThreadStateStorage() { + int res; + CHECK_EQ(thread_state_key, 0); + res = pthread_key_create(&thread_state_key, ThreadStateDestructor); + CHECK_EQ(res, 0); + res = pthread_setspecific(thread_state_key, main_thread_state); + CHECK_EQ(res, 0); + + auto dts = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState"); + dts->fast_state.SetIgnoreBit(); + dts->ignore_interceptors = 1; + dts->is_dead = true; + const_cast(dts->tid) = kInvalidTid; + res = internal_mprotect(dts, sizeof(ThreadState), PROT_READ); // immutable + CHECK_EQ(res, 0); + dead_thread_state = dts; +} + +ThreadState *cur_thread() { + // Some interceptors get called before libpthread has been initialized and in + // these cases we must avoid calling any pthread APIs. + if (UNLIKELY(!thread_state_key)) { + return (ThreadState *)main_thread_state; + } + + // We only reach this line after InitializeThreadStateStorage() ran, i.e, + // after TSan (and therefore libpthread) have been initialized. + ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key); + if (UNLIKELY(!thr)) { + thr = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState"); + int res = pthread_setspecific(thread_state_key, thr); + CHECK_EQ(res, 0); + } + return thr; +} + +void set_cur_thread(ThreadState *thr) { + int res = pthread_setspecific(thread_state_key, thr); + CHECK_EQ(res, 0); +} + +void cur_thread_finalize() { + ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key); + CHECK(thr); + if (thr == (ThreadState *)main_thread_state) { + // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to + // exit the main thread. Let's keep the main thread's ThreadState. + return; + } + // Intercepted functions can still get called after cur_thread_finalize() + // (called from DestroyThreadState()), so put a fake thread state for "dead" + // threads. An alternative solution would be to release the ThreadState + // object from THREAD_DESTROY (which is delivered later and on the parent + // thread) instead of THREAD_TERMINATE. + int res = pthread_setspecific(thread_state_key, dead_thread_state); + CHECK_EQ(res, 0); + UnmapOrDie(thr, sizeof(ThreadState)); +} +#endif + +void FlushShadowMemory() { +} + +static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) { + vm_address_t address = start; + vm_address_t end_address = end; + uptr resident_pages = 0; + uptr dirty_pages = 0; + while (address < end_address) { + vm_size_t vm_region_size; + mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT; + vm_region_extended_info_data_t vm_region_info; + mach_port_t object_name; + kern_return_t ret = vm_region_64( + mach_task_self(), &address, &vm_region_size, VM_REGION_EXTENDED_INFO, + (vm_region_info_t)&vm_region_info, &count, &object_name); + if (ret != KERN_SUCCESS) break; + + resident_pages += vm_region_info.pages_resident; + dirty_pages += vm_region_info.pages_dirtied; + + address += vm_region_size; + } + *res = resident_pages * GetPageSizeCached(); + *dirty = dirty_pages * GetPageSizeCached(); +} + +void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { + uptr shadow_res, shadow_dirty; + uptr meta_res, meta_dirty; + uptr trace_res, trace_dirty; + RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty); + RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty); + RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty); + +#if !SANITIZER_GO + uptr low_res, low_dirty; + uptr high_res, high_dirty; + uptr heap_res, heap_dirty; + RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &low_res, &low_dirty); + RegionMemUsage(HiAppMemBeg(), HiAppMemEnd(), &high_res, &high_dirty); + RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty); +#else // !SANITIZER_GO + uptr app_res, app_dirty; + RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &app_res, &app_dirty); +#endif + + StackDepotStats stacks = StackDepotGetStats(); + uptr nthread, nlive; + ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive); + internal_snprintf( + buf, buf_size, + "shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" + "meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" + "traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" +# if !SANITIZER_GO + "low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" + "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" + "heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" +# else // !SANITIZER_GO + "app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" +# endif + "stacks: %zd unique IDs, %zd kB allocated\n" + "threads: %zd total, %zd live\n" + "------------------------------\n", + ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024, + MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024, + TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024, +# if !SANITIZER_GO + LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024, + HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024, + HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024, +# else // !SANITIZER_GO + LoAppMemBeg(), LoAppMemEnd(), app_res / 1024, app_dirty / 1024, +# endif + stacks.n_uniq_ids, stacks.allocated / 1024, nthread, nlive); +} + +# if !SANITIZER_GO +void InitializeShadowMemoryPlatform() { } + +// On OS X, GCD worker threads are created without a call to pthread_create. We +// need to properly register these threads with ThreadCreate and ThreadStart. +// These threads don't have a parent thread, as they are created "spuriously". +// We're using a libpthread API that notifies us about a newly created thread. +// The `thread == pthread_self()` check indicates this is actually a worker +// thread. If it's just a regular thread, this hook is called on the parent +// thread. +typedef void (*pthread_introspection_hook_t)(unsigned int event, + pthread_t thread, void *addr, + size_t size); +extern "C" pthread_introspection_hook_t pthread_introspection_hook_install( + pthread_introspection_hook_t hook); +static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1; +static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3; +static pthread_introspection_hook_t prev_pthread_introspection_hook; +static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, + void *addr, size_t size) { + if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { + if (thread == pthread_self()) { + // The current thread is a newly created GCD worker thread. + ThreadState *thr = cur_thread(); + Processor *proc = ProcCreate(); + ProcWire(proc, thr); + ThreadState *parent_thread_state = nullptr; // No parent. + Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); + CHECK_NE(tid, kMainTid); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); + } + } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { + CHECK_EQ(thread, pthread_self()); + ThreadState *thr = cur_thread(); + if (thr->tctx) { + DestroyThreadState(); + } + } + + if (prev_pthread_introspection_hook != nullptr) + prev_pthread_introspection_hook(event, thread, addr, size); +} +#endif + +void InitializePlatformEarly() { +# if !SANITIZER_GO && SANITIZER_IOS + uptr max_vm = GetMaxUserVirtualAddress() + 1; + if (max_vm != HiAppMemEnd()) { + Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n", + (void *)max_vm, (void *)HiAppMemEnd()); + Die(); + } +#endif +} + +static uptr longjmp_xor_key = 0; + +void InitializePlatform() { + DisableCoreDumperIfNecessary(); +#if !SANITIZER_GO + CheckAndProtect(); + + InitializeThreadStateStorage(); + + prev_pthread_introspection_hook = + pthread_introspection_hook_install(&my_pthread_introspection_hook); +#endif + + if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) { + // Libsystem currently uses a process-global key; this might change. + const unsigned kTLSLongjmpXorKeySlot = 0x7; + longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot); + } +} + +#ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT \ + ((GetMacosAlignedVersion() >= MacosVersion(10, 14)) ? 12 : 13) +#else +# define LONG_JMP_SP_ENV_SLOT 2 +#endif + +uptr ExtractLongJmpSp(uptr *env) { + uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; + uptr sp = mangled_sp ^ longjmp_xor_key; + sp = (uptr)ptrauth_auth_data((void *)sp, ptrauth_key_asdb, + ptrauth_string_discriminator("sp")); + return sp; +} + +#if !SANITIZER_GO +extern "C" void __tsan_tls_initialization() {} + +void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { + const uptr pc = StackTrace::GetNextInstructionPc( + reinterpret_cast(__tsan_tls_initialization)); + // Unlike Linux, we only store a pointer to the ThreadState object in TLS; + // just mark the entire range as written to. + MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size); +} +#endif + +#if !SANITIZER_GO +// Note: this function runs with async signals enabled, +// so it must not touch any tsan state. +int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), + void (*cleanup)(void *arg), void *arg) { + // pthread_cleanup_push/pop are hardcore macros mess. + // We can't intercept nor call them w/o including pthread.h. + int res; + pthread_cleanup_push(cleanup, arg); + res = fn(arg); + pthread_cleanup_pop(0); + return res; +} +#endif + +} // namespace __tsan + +#endif // SANITIZER_MAC diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp new file mode 100644 index 000000000000..763ac444377e --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp @@ -0,0 +1,147 @@ +//===-- tsan_platform_posix.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// POSIX-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_POSIX + +# include + +# include "sanitizer_common/sanitizer_common.h" +# include "sanitizer_common/sanitizer_errno.h" +# include "sanitizer_common/sanitizer_libc.h" +# include "sanitizer_common/sanitizer_procmaps.h" +# include "tsan_platform.h" +# include "tsan_rtl.h" + +namespace __tsan { + +static const char kShadowMemoryMappingWarning[] = + "FATAL: %s can not madvise shadow region [%zx, %zx] with %s (errno: %d)\n"; +static const char kShadowMemoryMappingHint[] = + "HINT: if %s is not supported in your environment, you may set " + "TSAN_OPTIONS=%s=0\n"; + +# if !SANITIZER_GO +static void DontDumpShadow(uptr addr, uptr size) { + if (common_flags()->use_madv_dontdump) + if (!DontDumpShadowMemory(addr, size)) { + Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size, + "MADV_DONTDUMP", errno); + Printf(kShadowMemoryMappingHint, "MADV_DONTDUMP", "use_madv_dontdump"); + Die(); + } +} + +void InitializeShadowMemory() { + // Map memory shadow. + if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), + "shadow")) { + Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); + Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); + Die(); + } + // This memory range is used for thread stacks and large user mmaps. + // Frequently a thread uses only a small part of stack and similarly + // a program uses a small part of large mmap. On some programs + // we see 20% memory usage reduction without huge pages for this range. + DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg()); + DPrintf("memory shadow: %zx-%zx (%zuGB)\n", + ShadowBeg(), ShadowEnd(), + (ShadowEnd() - ShadowBeg()) >> 30); + + // Map meta shadow. + const uptr meta = MetaShadowBeg(); + const uptr meta_size = MetaShadowEnd() - meta; + if (!MmapFixedSuperNoReserve(meta, meta_size, "meta shadow")) { + Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); + Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); + Die(); + } + DontDumpShadow(meta, meta_size); + DPrintf("meta shadow: %zx-%zx (%zuGB)\n", + meta, meta + meta_size, meta_size >> 30); + + InitializeShadowMemoryPlatform(); + + on_initialize = reinterpret_cast( + dlsym(RTLD_DEFAULT, "__tsan_on_initialize")); + on_finalize = + reinterpret_cast(dlsym(RTLD_DEFAULT, "__tsan_on_finalize")); +} + +static bool TryProtectRange(uptr beg, uptr end) { + CHECK_LE(beg, end); + if (beg == end) + return true; + return beg == (uptr)MmapFixedNoAccess(beg, end - beg); +} + +static void ProtectRange(uptr beg, uptr end) { + if (!TryProtectRange(beg, end)) { + Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end); + Printf("FATAL: Make sure you are not using unlimited stack\n"); + Die(); + } +} + +void CheckAndProtect() { + // Ensure that the binary is indeed compiled with -pie. + MemoryMappingLayout proc_maps(true); + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (IsAppMem(segment.start)) continue; + if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue; + if (segment.protection == 0) // Zero page or mprotected. + continue; + if (segment.start >= VdsoBeg()) // vdso + break; + Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n", + segment.start, segment.end); + Die(); + } + +# if defined(__aarch64__) && defined(__APPLE__) && SANITIZER_IOS + ProtectRange(HeapMemEnd(), ShadowBeg()); + ProtectRange(ShadowEnd(), MetaShadowBeg()); + ProtectRange(MetaShadowEnd(), TraceMemBeg()); +#else + ProtectRange(LoAppMemEnd(), ShadowBeg()); + ProtectRange(ShadowEnd(), MetaShadowBeg()); + if (MidAppMemBeg()) { + ProtectRange(MetaShadowEnd(), MidAppMemBeg()); + ProtectRange(MidAppMemEnd(), TraceMemBeg()); + } else { + ProtectRange(MetaShadowEnd(), TraceMemBeg()); + } + // Memory for traces is mapped lazily in MapThreadTrace. + // Protect the whole range for now, so that user does not map something here. + ProtectRange(TraceMemBeg(), TraceMemEnd()); + ProtectRange(TraceMemEnd(), HeapMemBeg()); + ProtectRange(HeapEnd(), HiAppMemBeg()); +#endif + +#if defined(__s390x__) + // Protect the rest of the address space. + const uptr user_addr_max_l4 = 0x0020000000000000ull; + const uptr user_addr_max_l5 = 0xfffffffffffff000ull; + // All the maintained s390x kernels support at least 4-level page tables. + ProtectRange(HiAppMemEnd(), user_addr_max_l4); + // Older s390x kernels may not support 5-level page tables. + TryProtectRange(user_addr_max_l4, user_addr_max_l5); +#endif +} +#endif + +} // namespace __tsan + +#endif // SANITIZER_POSIX diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp new file mode 100644 index 000000000000..fea893768c79 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp @@ -0,0 +1,36 @@ +//===-- tsan_platform_windows.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Windows-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_WINDOWS + +#include "tsan_platform.h" + +#include + +namespace __tsan { + +void FlushShadowMemory() { +} + +void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {} + +void InitializePlatformEarly() { +} + +void InitializePlatform() { +} + +} // namespace __tsan + +#endif // SANITIZER_WINDOWS diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h b/compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h new file mode 100644 index 000000000000..5b43f3ddada3 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h @@ -0,0 +1,96 @@ +#define r0 0 +#define r1 1 +#define r2 2 +#define r3 3 +#define r4 4 +#define r5 5 +#define r6 6 +#define r7 7 +#define r8 8 +#define r9 9 +#define r10 10 +#define r11 11 +#define r12 12 +#define r13 13 +#define r14 14 +#define r15 15 +#define r16 16 +#define r17 17 +#define r18 18 +#define r19 19 +#define r20 20 +#define r21 21 +#define r22 22 +#define r23 23 +#define r24 24 +#define r25 25 +#define r26 26 +#define r27 27 +#define r28 28 +#define r29 29 +#define r30 30 +#define r31 31 +#define f0 0 +#define f1 1 +#define f2 2 +#define f3 3 +#define f4 4 +#define f5 5 +#define f6 6 +#define f7 7 +#define f8 8 +#define f9 9 +#define f10 10 +#define f11 11 +#define f12 12 +#define f13 13 +#define f14 14 +#define f15 15 +#define f16 16 +#define f17 17 +#define f18 18 +#define f19 19 +#define f20 20 +#define f21 21 +#define f22 22 +#define f23 23 +#define f24 24 +#define f25 25 +#define f26 26 +#define f27 27 +#define f28 28 +#define f29 29 +#define f30 30 +#define f31 31 +#define v0 0 +#define v1 1 +#define v2 2 +#define v3 3 +#define v4 4 +#define v5 5 +#define v6 6 +#define v7 7 +#define v8 8 +#define v9 9 +#define v10 10 +#define v11 11 +#define v12 12 +#define v13 13 +#define v14 14 +#define v15 15 +#define v16 16 +#define v17 17 +#define v18 18 +#define v19 19 +#define v20 20 +#define v21 21 +#define v22 22 +#define v23 23 +#define v24 24 +#define v25 25 +#define v26 26 +#define v27 27 +#define v28 28 +#define v29 29 +#define v30 30 +#define v31 31 diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp new file mode 100644 index 000000000000..205bdbf93b20 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp @@ -0,0 +1,26 @@ +//===-- tsan_preinit.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer. +// +// Call __tsan_init at the very early stage of process startup. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "tsan_interface.h" + +#if SANITIZER_CAN_USE_PREINIT_ARRAY + +// The symbol is called __local_tsan_preinit, because it's not intended to be +// exported. +// This code linked into the main executable when -fsanitize=thread is in +// the link flags. It can only use exported interface functions. +__attribute__((section(".preinit_array"), used)) +void (*__local_tsan_preinit)(void) = __tsan_init; + +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_report.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_report.cpp new file mode 100644 index 000000000000..a926c3761ccf --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_report.cpp @@ -0,0 +1,479 @@ +//===-- tsan_report.cpp ---------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_report.h" +#include "tsan_platform.h" +#include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_file.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_report_decorator.h" +#include "sanitizer_common/sanitizer_stacktrace_printer.h" + +namespace __tsan { + +class Decorator: public __sanitizer::SanitizerCommonDecorator { + public: + Decorator() : SanitizerCommonDecorator() { } + const char *Access() { return Blue(); } + const char *ThreadDescription() { return Cyan(); } + const char *Location() { return Green(); } + const char *Sleep() { return Yellow(); } + const char *Mutex() { return Magenta(); } +}; + +ReportDesc::ReportDesc() + : tag(kExternalTagNone) + , stacks() + , mops() + , locs() + , mutexes() + , threads() + , unique_tids() + , sleep() + , count() { +} + +ReportMop::ReportMop() + : mset() { +} + +ReportDesc::~ReportDesc() { + // FIXME(dvyukov): it must be leaking a lot of memory. +} + +#if !SANITIZER_GO + +const int kThreadBufSize = 32; +const char *thread_name(char *buf, Tid tid) { + if (tid == kMainTid) + return "main thread"; + internal_snprintf(buf, kThreadBufSize, "thread T%d", tid); + return buf; +} + +static const char *ReportTypeString(ReportType typ, uptr tag) { + switch (typ) { + case ReportTypeRace: + return "data race"; + case ReportTypeVptrRace: + return "data race on vptr (ctor/dtor vs virtual call)"; + case ReportTypeUseAfterFree: + return "heap-use-after-free"; + case ReportTypeVptrUseAfterFree: + return "heap-use-after-free (virtual call vs free)"; + case ReportTypeExternalRace: { + const char *str = GetReportHeaderFromTag(tag); + return str ? str : "race on external object"; + } + case ReportTypeThreadLeak: + return "thread leak"; + case ReportTypeMutexDestroyLocked: + return "destroy of a locked mutex"; + case ReportTypeMutexDoubleLock: + return "double lock of a mutex"; + case ReportTypeMutexInvalidAccess: + return "use of an invalid mutex (e.g. uninitialized or destroyed)"; + case ReportTypeMutexBadUnlock: + return "unlock of an unlocked mutex (or by a wrong thread)"; + case ReportTypeMutexBadReadLock: + return "read lock of a write locked mutex"; + case ReportTypeMutexBadReadUnlock: + return "read unlock of a write locked mutex"; + case ReportTypeSignalUnsafe: + return "signal-unsafe call inside of a signal"; + case ReportTypeErrnoInSignal: + return "signal handler spoils errno"; + case ReportTypeDeadlock: + return "lock-order-inversion (potential deadlock)"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); +} + +#if SANITIZER_MAC +static const char *const kInterposedFunctionPrefix = "wrap_"; +#else +static const char *const kInterposedFunctionPrefix = "__interceptor_"; +#endif + +void PrintStack(const ReportStack *ent) { + if (ent == 0 || ent->frames == 0) { + Printf(" [failed to restore the stack]\n\n"); + return; + } + SymbolizedStack *frame = ent->frames; + for (int i = 0; frame && frame->info.address; frame = frame->next, i++) { + InternalScopedString res; + RenderFrame(&res, common_flags()->stack_trace_format, i, + frame->info.address, &frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix, kInterposedFunctionPrefix); + Printf("%s\n", res.data()); + } + Printf("\n"); +} + +static void PrintMutexSet(Vector const& mset) { + for (uptr i = 0; i < mset.Size(); i++) { + if (i == 0) + Printf(" (mutexes:"); + const ReportMopMutex m = mset[i]; + Printf(" %s M%llu", m.write ? "write" : "read", m.id); + Printf(i == mset.Size() - 1 ? ")" : ","); + } +} + +static const char *MopDesc(bool first, bool write, bool atomic) { + return atomic ? (first ? (write ? "Atomic write" : "Atomic read") + : (write ? "Previous atomic write" : "Previous atomic read")) + : (first ? (write ? "Write" : "Read") + : (write ? "Previous write" : "Previous read")); +} + +static const char *ExternalMopDesc(bool first, bool write) { + return first ? (write ? "Modifying" : "Read-only") + : (write ? "Previous modifying" : "Previous read-only"); +} + +static void PrintMop(const ReportMop *mop, bool first) { + Decorator d; + char thrbuf[kThreadBufSize]; + Printf("%s", d.Access()); + if (mop->external_tag == kExternalTagNone) { + Printf(" %s of size %d at %p by %s", + MopDesc(first, mop->write, mop->atomic), mop->size, + (void *)mop->addr, thread_name(thrbuf, mop->tid)); + } else { + const char *object_type = GetObjectTypeFromTag(mop->external_tag); + if (object_type == nullptr) + object_type = "external object"; + Printf(" %s access of %s at %p by %s", + ExternalMopDesc(first, mop->write), object_type, + (void *)mop->addr, thread_name(thrbuf, mop->tid)); + } + PrintMutexSet(mop->mset); + Printf(":\n"); + Printf("%s", d.Default()); + PrintStack(mop->stack); +} + +static void PrintLocation(const ReportLocation *loc) { + Decorator d; + char thrbuf[kThreadBufSize]; + bool print_stack = false; + Printf("%s", d.Location()); + if (loc->type == ReportLocationGlobal) { + const DataInfo &global = loc->global; + if (global.size != 0) + Printf(" Location is global '%s' of size %zu at %p (%s+0x%zx)\n\n", + global.name, global.size, reinterpret_cast(global.start), + StripModuleName(global.module), global.module_offset); + else + Printf(" Location is global '%s' at %p (%s+0x%zx)\n\n", global.name, + reinterpret_cast(global.start), + StripModuleName(global.module), global.module_offset); + } else if (loc->type == ReportLocationHeap) { + char thrbuf[kThreadBufSize]; + const char *object_type = GetObjectTypeFromTag(loc->external_tag); + if (!object_type) { + Printf(" Location is heap block of size %zu at %p allocated by %s:\n", + loc->heap_chunk_size, + reinterpret_cast(loc->heap_chunk_start), + thread_name(thrbuf, loc->tid)); + } else { + Printf(" Location is %s of size %zu at %p allocated by %s:\n", + object_type, loc->heap_chunk_size, + reinterpret_cast(loc->heap_chunk_start), + thread_name(thrbuf, loc->tid)); + } + print_stack = true; + } else if (loc->type == ReportLocationStack) { + Printf(" Location is stack of %s.\n\n", thread_name(thrbuf, loc->tid)); + } else if (loc->type == ReportLocationTLS) { + Printf(" Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid)); + } else if (loc->type == ReportLocationFD) { + Printf(" Location is file descriptor %d created by %s at:\n", + loc->fd, thread_name(thrbuf, loc->tid)); + print_stack = true; + } + Printf("%s", d.Default()); + if (print_stack) + PrintStack(loc->stack); +} + +static void PrintMutexShort(const ReportMutex *rm, const char *after) { + Decorator d; + Printf("%sM%lld%s%s", d.Mutex(), rm->id, d.Default(), after); +} + +static void PrintMutexShortWithAddress(const ReportMutex *rm, + const char *after) { + Decorator d; + Printf("%sM%lld (%p)%s%s", d.Mutex(), rm->id, + reinterpret_cast(rm->addr), d.Default(), after); +} + +static void PrintMutex(const ReportMutex *rm) { + Decorator d; + if (rm->destroyed) { + Printf("%s", d.Mutex()); + Printf(" Mutex M%llu is already destroyed.\n\n", rm->id); + Printf("%s", d.Default()); + } else { + Printf("%s", d.Mutex()); + Printf(" Mutex M%llu (%p) created at:\n", rm->id, + reinterpret_cast(rm->addr)); + Printf("%s", d.Default()); + PrintStack(rm->stack); + } +} + +static void PrintThread(const ReportThread *rt) { + Decorator d; + if (rt->id == kMainTid) // Little sense in describing the main thread. + return; + Printf("%s", d.ThreadDescription()); + Printf(" Thread T%d", rt->id); + if (rt->name && rt->name[0] != '\0') + Printf(" '%s'", rt->name); + char thrbuf[kThreadBufSize]; + const char *thread_status = rt->running ? "running" : "finished"; + if (rt->thread_type == ThreadType::Worker) { + Printf(" (tid=%llu, %s) is a GCD worker thread\n", rt->os_id, + thread_status); + Printf("\n"); + Printf("%s", d.Default()); + return; + } + Printf(" (tid=%llu, %s) created by %s", rt->os_id, thread_status, + thread_name(thrbuf, rt->parent_tid)); + if (rt->stack) + Printf(" at:"); + Printf("\n"); + Printf("%s", d.Default()); + PrintStack(rt->stack); +} + +static void PrintSleep(const ReportStack *s) { + Decorator d; + Printf("%s", d.Sleep()); + Printf(" As if synchronized via sleep:\n"); + Printf("%s", d.Default()); + PrintStack(s); +} + +static ReportStack *ChooseSummaryStack(const ReportDesc *rep) { + if (rep->mops.Size()) + return rep->mops[0]->stack; + if (rep->stacks.Size()) + return rep->stacks[0]; + if (rep->mutexes.Size()) + return rep->mutexes[0]->stack; + if (rep->threads.Size()) + return rep->threads[0]->stack; + return 0; +} + +static bool FrameIsInternal(const SymbolizedStack *frame) { + if (frame == 0) + return false; + const char *file = frame->info.file; + const char *module = frame->info.module; + if (file != 0 && + (internal_strstr(file, "tsan_interceptors_posix.cpp") || + internal_strstr(file, "sanitizer_common_interceptors.inc") || + internal_strstr(file, "tsan_interface_"))) + return true; + if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_"))) + return true; + return false; +} + +static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { + while (FrameIsInternal(frames) && frames->next) + frames = frames->next; + return frames; +} + +void PrintReport(const ReportDesc *rep) { + Decorator d; + Printf("==================\n"); + const char *rep_typ_str = ReportTypeString(rep->typ, rep->tag); + Printf("%s", d.Warning()); + Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str, + (int)internal_getpid()); + Printf("%s", d.Default()); + + if (rep->typ == ReportTypeDeadlock) { + char thrbuf[kThreadBufSize]; + Printf(" Cycle in lock order graph: "); + for (uptr i = 0; i < rep->mutexes.Size(); i++) + PrintMutexShortWithAddress(rep->mutexes[i], " => "); + PrintMutexShort(rep->mutexes[0], "\n\n"); + CHECK_GT(rep->mutexes.Size(), 0U); + CHECK_EQ(rep->mutexes.Size() * (flags()->second_deadlock_stack ? 2 : 1), + rep->stacks.Size()); + for (uptr i = 0; i < rep->mutexes.Size(); i++) { + Printf(" Mutex "); + PrintMutexShort(rep->mutexes[(i + 1) % rep->mutexes.Size()], + " acquired here while holding mutex "); + PrintMutexShort(rep->mutexes[i], " in "); + Printf("%s", d.ThreadDescription()); + Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i])); + Printf("%s", d.Default()); + if (flags()->second_deadlock_stack) { + PrintStack(rep->stacks[2*i]); + Printf(" Mutex "); + PrintMutexShort(rep->mutexes[i], + " previously acquired by the same thread here:\n"); + PrintStack(rep->stacks[2*i+1]); + } else { + PrintStack(rep->stacks[i]); + if (i == 0) + Printf(" Hint: use TSAN_OPTIONS=second_deadlock_stack=1 " + "to get more informative warning message\n\n"); + } + } + } else { + for (uptr i = 0; i < rep->stacks.Size(); i++) { + if (i) + Printf(" and:\n"); + PrintStack(rep->stacks[i]); + } + } + + for (uptr i = 0; i < rep->mops.Size(); i++) + PrintMop(rep->mops[i], i == 0); + + if (rep->sleep) + PrintSleep(rep->sleep); + + for (uptr i = 0; i < rep->locs.Size(); i++) + PrintLocation(rep->locs[i]); + + if (rep->typ != ReportTypeDeadlock) { + for (uptr i = 0; i < rep->mutexes.Size(); i++) + PrintMutex(rep->mutexes[i]); + } + + for (uptr i = 0; i < rep->threads.Size(); i++) + PrintThread(rep->threads[i]); + + if (rep->typ == ReportTypeThreadLeak && rep->count > 1) + Printf(" And %d more similar thread leaks.\n\n", rep->count - 1); + + if (ReportStack *stack = ChooseSummaryStack(rep)) { + if (SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames)) + ReportErrorSummary(rep_typ_str, frame->info); + } + + if (common_flags()->print_module_map == 2) + DumpProcessMap(); + + Printf("==================\n"); +} + +#else // #if !SANITIZER_GO + +const Tid kMainGoroutineId = 1; + +void PrintStack(const ReportStack *ent) { + if (ent == 0 || ent->frames == 0) { + Printf(" [failed to restore the stack]\n"); + return; + } + SymbolizedStack *frame = ent->frames; + for (int i = 0; frame; frame = frame->next, i++) { + const AddressInfo &info = frame->info; + Printf(" %s()\n %s:%d +0x%zx\n", info.function, + StripPathPrefix(info.file, common_flags()->strip_path_prefix), + info.line, info.module_offset); + } +} + +static void PrintMop(const ReportMop *mop, bool first) { + Printf("\n"); + Printf("%s at %p by ", + (first ? (mop->write ? "Write" : "Read") + : (mop->write ? "Previous write" : "Previous read")), + reinterpret_cast(mop->addr)); + if (mop->tid == kMainGoroutineId) + Printf("main goroutine:\n"); + else + Printf("goroutine %d:\n", mop->tid); + PrintStack(mop->stack); +} + +static void PrintLocation(const ReportLocation *loc) { + switch (loc->type) { + case ReportLocationHeap: { + Printf("\n"); + Printf("Heap block of size %zu at %p allocated by ", loc->heap_chunk_size, + reinterpret_cast(loc->heap_chunk_start)); + if (loc->tid == kMainGoroutineId) + Printf("main goroutine:\n"); + else + Printf("goroutine %d:\n", loc->tid); + PrintStack(loc->stack); + break; + } + case ReportLocationGlobal: { + Printf("\n"); + Printf("Global var %s of size %zu at %p declared at %s:%zu\n", + loc->global.name, loc->global.size, + reinterpret_cast(loc->global.start), loc->global.file, + loc->global.line); + break; + } + default: + break; + } +} + +static void PrintThread(const ReportThread *rt) { + if (rt->id == kMainGoroutineId) + return; + Printf("\n"); + Printf("Goroutine %d (%s) created at:\n", + rt->id, rt->running ? "running" : "finished"); + PrintStack(rt->stack); +} + +void PrintReport(const ReportDesc *rep) { + Printf("==================\n"); + if (rep->typ == ReportTypeRace) { + Printf("WARNING: DATA RACE"); + for (uptr i = 0; i < rep->mops.Size(); i++) + PrintMop(rep->mops[i], i == 0); + for (uptr i = 0; i < rep->locs.Size(); i++) + PrintLocation(rep->locs[i]); + for (uptr i = 0; i < rep->threads.Size(); i++) + PrintThread(rep->threads[i]); + } else if (rep->typ == ReportTypeDeadlock) { + Printf("WARNING: DEADLOCK\n"); + for (uptr i = 0; i < rep->mutexes.Size(); i++) { + Printf("Goroutine %d lock mutex %llu while holding mutex %llu:\n", 999, + rep->mutexes[i]->id, + rep->mutexes[(i + 1) % rep->mutexes.Size()]->id); + PrintStack(rep->stacks[2*i]); + Printf("\n"); + Printf("Mutex %llu was previously locked here:\n", + rep->mutexes[(i + 1) % rep->mutexes.Size()]->id); + PrintStack(rep->stacks[2*i + 1]); + Printf("\n"); + } + } + Printf("==================\n"); +} + +#endif + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_report.h b/compiler-rt/lib/tsan/rtl-old/tsan_report.h new file mode 100644 index 000000000000..d68c2db88828 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_report.h @@ -0,0 +1,127 @@ +//===-- tsan_report.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_REPORT_H +#define TSAN_REPORT_H + +#include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_thread_registry.h" +#include "sanitizer_common/sanitizer_vector.h" +#include "tsan_defs.h" + +namespace __tsan { + +enum ReportType { + ReportTypeRace, + ReportTypeVptrRace, + ReportTypeUseAfterFree, + ReportTypeVptrUseAfterFree, + ReportTypeExternalRace, + ReportTypeThreadLeak, + ReportTypeMutexDestroyLocked, + ReportTypeMutexDoubleLock, + ReportTypeMutexInvalidAccess, + ReportTypeMutexBadUnlock, + ReportTypeMutexBadReadLock, + ReportTypeMutexBadReadUnlock, + ReportTypeSignalUnsafe, + ReportTypeErrnoInSignal, + ReportTypeDeadlock +}; + +struct ReportStack { + SymbolizedStack *frames = nullptr; + bool suppressable = false; +}; + +struct ReportMopMutex { + u64 id; + bool write; +}; + +struct ReportMop { + int tid; + uptr addr; + int size; + bool write; + bool atomic; + uptr external_tag; + Vector mset; + ReportStack *stack; + + ReportMop(); +}; + +enum ReportLocationType { + ReportLocationGlobal, + ReportLocationHeap, + ReportLocationStack, + ReportLocationTLS, + ReportLocationFD +}; + +struct ReportLocation { + ReportLocationType type = ReportLocationGlobal; + DataInfo global = {}; + uptr heap_chunk_start = 0; + uptr heap_chunk_size = 0; + uptr external_tag = 0; + Tid tid = kInvalidTid; + int fd = 0; + bool suppressable = false; + ReportStack *stack = nullptr; +}; + +struct ReportThread { + Tid id; + tid_t os_id; + bool running; + ThreadType thread_type; + char *name; + Tid parent_tid; + ReportStack *stack; +}; + +struct ReportMutex { + u64 id; + uptr addr; + bool destroyed; + ReportStack *stack; +}; + +class ReportDesc { + public: + ReportType typ; + uptr tag; + Vector stacks; + Vector mops; + Vector locs; + Vector mutexes; + Vector threads; + Vector unique_tids; + ReportStack *sleep; + int count; + + ReportDesc(); + ~ReportDesc(); + + private: + ReportDesc(const ReportDesc&); + void operator = (const ReportDesc&); +}; + +// Format and output the report to the console/log. No additional logic. +void PrintReport(const ReportDesc *rep); +void PrintStack(const ReportStack *stack); + +} // namespace __tsan + +#endif // TSAN_REPORT_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp new file mode 100644 index 000000000000..c14af9788e32 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp @@ -0,0 +1,811 @@ +//===-- tsan_rtl.cpp ------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Main file (entry points) for the TSan run-time. +//===----------------------------------------------------------------------===// + +#include "tsan_rtl.h" + +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_file.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_symbolizer.h" +#include "tsan_defs.h" +#include "tsan_interface.h" +#include "tsan_mman.h" +#include "tsan_platform.h" +#include "tsan_suppressions.h" +#include "tsan_symbolize.h" +#include "ubsan/ubsan_init.h" + +volatile int __tsan_resumed = 0; + +extern "C" void __tsan_resume() { + __tsan_resumed = 1; +} + +SANITIZER_WEAK_DEFAULT_IMPL +void __tsan_test_only_on_fork() {} + +namespace __tsan { + +#if !SANITIZER_GO +void (*on_initialize)(void); +int (*on_finalize)(int); +#endif + +#if !SANITIZER_GO && !SANITIZER_MAC +__attribute__((tls_model("initial-exec"))) +THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED( + SANITIZER_CACHE_LINE_SIZE); +#endif +static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE); +Context *ctx; + +// Can be overriden by a front-end. +#ifdef TSAN_EXTERNAL_HOOKS +bool OnFinalize(bool failed); +void OnInitialize(); +#else +#include +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool OnFinalize(bool failed) { +#if !SANITIZER_GO + if (on_finalize) + return on_finalize(failed); +#endif + return failed; +} +SANITIZER_WEAK_CXX_DEFAULT_IMPL +void OnInitialize() { +#if !SANITIZER_GO + if (on_initialize) + on_initialize(); +#endif +} +#endif + +static ThreadContextBase *CreateThreadContext(Tid tid) { + // Map thread trace when context is created. + char name[50]; + internal_snprintf(name, sizeof(name), "trace %u", tid); + MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name); + const uptr hdr = GetThreadTraceHeader(tid); + internal_snprintf(name, sizeof(name), "trace header %u", tid); + MapThreadTrace(hdr, sizeof(Trace), name); + new((void*)hdr) Trace(); + // We are going to use only a small part of the trace with the default + // value of history_size. However, the constructor writes to the whole trace. + // Release the unused part. + uptr hdr_end = hdr + sizeof(Trace); + hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts()); + hdr_end = RoundUp(hdr_end, GetPageSizeCached()); + if (hdr_end < hdr + sizeof(Trace)) { + ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace)); + uptr unused = hdr + sizeof(Trace) - hdr_end; + if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) { + Report("ThreadSanitizer: failed to mprotect [0x%zx-0x%zx) \n", hdr_end, + unused); + CHECK("unable to mprotect" && 0); + } + } + return New(tid); +} + +#if !SANITIZER_GO +static const u32 kThreadQuarantineSize = 16; +#else +static const u32 kThreadQuarantineSize = 64; +#endif + +Context::Context() + : initialized(), + report_mtx(MutexTypeReport), + nreported(), + thread_registry(CreateThreadContext, kMaxTid, kThreadQuarantineSize, + kMaxTidReuse), + racy_mtx(MutexTypeRacy), + racy_stacks(), + racy_addresses(), + fired_suppressions_mtx(MutexTypeFired), + clock_alloc(LINKER_INITIALIZED, "clock allocator") { + fired_suppressions.reserve(8); +} + +// The objects are allocated in TLS, so one may rely on zero-initialization. +ThreadState::ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch, + unsigned reuse_count, uptr stk_addr, uptr stk_size, + uptr tls_addr, uptr tls_size) + : fast_state(tid, epoch) + // Do not touch these, rely on zero initialization, + // they may be accessed before the ctor. + // , ignore_reads_and_writes() + // , ignore_interceptors() + , + clock(tid, reuse_count) +#if !SANITIZER_GO + , + jmp_bufs() +#endif + , + tid(tid), + unique_id(unique_id), + stk_addr(stk_addr), + stk_size(stk_size), + tls_addr(tls_addr), + tls_size(tls_size) +#if !SANITIZER_GO + , + last_sleep_clock(tid) +#endif +{ + CHECK_EQ(reinterpret_cast(this) % SANITIZER_CACHE_LINE_SIZE, 0); +#if !SANITIZER_GO + // C/C++ uses fixed size shadow stack. + const int kInitStackSize = kShadowStackSize; + shadow_stack = static_cast( + MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack")); + SetShadowRegionHugePageMode(reinterpret_cast(shadow_stack), + kInitStackSize * sizeof(uptr)); +#else + // Go uses malloc-allocated shadow stack with dynamic size. + const int kInitStackSize = 8; + shadow_stack = static_cast(Alloc(kInitStackSize * sizeof(uptr))); +#endif + shadow_stack_pos = shadow_stack; + shadow_stack_end = shadow_stack + kInitStackSize; +} + +#if !SANITIZER_GO +void MemoryProfiler(u64 uptime) { + if (ctx->memprof_fd == kInvalidFd) + return; + InternalMmapVector buf(4096); + WriteMemoryProfile(buf.data(), buf.size(), uptime); + WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data())); +} + +void InitializeMemoryProfiler() { + ctx->memprof_fd = kInvalidFd; + const char *fname = flags()->profile_memory; + if (!fname || !fname[0]) + return; + if (internal_strcmp(fname, "stdout") == 0) { + ctx->memprof_fd = 1; + } else if (internal_strcmp(fname, "stderr") == 0) { + ctx->memprof_fd = 2; + } else { + InternalScopedString filename; + filename.append("%s.%d", fname, (int)internal_getpid()); + ctx->memprof_fd = OpenFile(filename.data(), WrOnly); + if (ctx->memprof_fd == kInvalidFd) { + Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", + filename.data()); + return; + } + } + MemoryProfiler(0); + MaybeSpawnBackgroundThread(); +} + +static void *BackgroundThread(void *arg) { + // This is a non-initialized non-user thread, nothing to see here. + // We don't use ScopedIgnoreInterceptors, because we want ignores to be + // enabled even when the thread function exits (e.g. during pthread thread + // shutdown code). + cur_thread_init()->ignore_interceptors++; + const u64 kMs2Ns = 1000 * 1000; + const u64 start = NanoTime(); + + u64 last_flush = NanoTime(); + uptr last_rss = 0; + for (int i = 0; + atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0; + i++) { + SleepForMillis(100); + u64 now = NanoTime(); + + // Flush memory if requested. + if (flags()->flush_memory_ms > 0) { + if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) { + VPrintf(1, "ThreadSanitizer: periodic memory flush\n"); + FlushShadowMemory(); + last_flush = NanoTime(); + } + } + if (flags()->memory_limit_mb > 0) { + uptr rss = GetRSS(); + uptr limit = uptr(flags()->memory_limit_mb) << 20; + VPrintf(1, "ThreadSanitizer: memory flush check" + " RSS=%llu LAST=%llu LIMIT=%llu\n", + (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20); + if (2 * rss > limit + last_rss) { + VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n"); + FlushShadowMemory(); + rss = GetRSS(); + VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20); + } + last_rss = rss; + } + + MemoryProfiler(now - start); + + // Flush symbolizer cache if requested. + if (flags()->flush_symbolizer_ms > 0) { + u64 last = atomic_load(&ctx->last_symbolize_time_ns, + memory_order_relaxed); + if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { + Lock l(&ctx->report_mtx); + ScopedErrorReportLock l2; + SymbolizeFlush(); + atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); + } + } + } + return nullptr; +} + +static void StartBackgroundThread() { + ctx->background_thread = internal_start_thread(&BackgroundThread, 0); +} + +#ifndef __mips__ +static void StopBackgroundThread() { + atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed); + internal_join_thread(ctx->background_thread); + ctx->background_thread = 0; +} +#endif +#endif + +void DontNeedShadowFor(uptr addr, uptr size) { + ReleaseMemoryPagesToOS(reinterpret_cast(MemToShadow(addr)), + reinterpret_cast(MemToShadow(addr + size))); +} + +#if !SANITIZER_GO +// We call UnmapShadow before the actual munmap, at that point we don't yet +// know if the provided address/size are sane. We can't call UnmapShadow +// after the actual munmap becuase at that point the memory range can +// already be reused for something else, so we can't rely on the munmap +// return value to understand is the values are sane. +// While calling munmap with insane values (non-canonical address, negative +// size, etc) is an error, the kernel won't crash. We must also try to not +// crash as the failure mode is very confusing (paging fault inside of the +// runtime on some derived shadow address). +static bool IsValidMmapRange(uptr addr, uptr size) { + if (size == 0) + return true; + if (static_cast(size) < 0) + return false; + if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) + return false; + // Check that if the start of the region belongs to one of app ranges, + // end of the region belongs to the same region. + const uptr ranges[][2] = { + {LoAppMemBeg(), LoAppMemEnd()}, + {MidAppMemBeg(), MidAppMemEnd()}, + {HiAppMemBeg(), HiAppMemEnd()}, + }; + for (auto range : ranges) { + if (addr >= range[0] && addr < range[1]) + return addr + size <= range[1]; + } + return false; +} + +void UnmapShadow(ThreadState *thr, uptr addr, uptr size) { + if (size == 0 || !IsValidMmapRange(addr, size)) + return; + DontNeedShadowFor(addr, size); + ScopedGlobalProcessor sgp; + ctx->metamap.ResetRange(thr->proc(), addr, size); +} +#endif + +void MapShadow(uptr addr, uptr size) { + // Global data is not 64K aligned, but there are no adjacent mappings, + // so we can get away with unaligned mapping. + // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment + const uptr kPageSize = GetPageSizeCached(); + uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize); + uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize); + if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, + "shadow")) + Die(); + + // Meta shadow is 2:1, so tread carefully. + static bool data_mapped = false; + static uptr mapped_meta_end = 0; + uptr meta_begin = (uptr)MemToMeta(addr); + uptr meta_end = (uptr)MemToMeta(addr + size); + meta_begin = RoundDownTo(meta_begin, 64 << 10); + meta_end = RoundUpTo(meta_end, 64 << 10); + if (!data_mapped) { + // First call maps data+bss. + data_mapped = true; + if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, + "meta shadow")) + Die(); + } else { + // Mapping continuous heap. + // Windows wants 64K alignment. + meta_begin = RoundDownTo(meta_begin, 64 << 10); + meta_end = RoundUpTo(meta_end, 64 << 10); + if (meta_end <= mapped_meta_end) + return; + if (meta_begin < mapped_meta_end) + meta_begin = mapped_meta_end; + if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, + "meta shadow")) + Die(); + mapped_meta_end = meta_end; + } + VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr, + addr + size, meta_begin, meta_end); +} + +void MapThreadTrace(uptr addr, uptr size, const char *name) { + DPrintf("#0: Mapping trace at 0x%zx-0x%zx(0x%zx)\n", addr, addr + size, size); + CHECK_GE(addr, TraceMemBeg()); + CHECK_LE(addr + size, TraceMemEnd()); + CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment + if (!MmapFixedSuperNoReserve(addr, size, name)) { + Printf("FATAL: ThreadSanitizer can not mmap thread trace (0x%zx/0x%zx)\n", + addr, size); + Die(); + } +} + +#if !SANITIZER_GO +static void OnStackUnwind(const SignalContext &sig, const void *, + BufferedStackTrace *stack) { + stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); +} + +static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) { + HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr); +} +#endif + +void CheckUnwind() { + // There is high probability that interceptors will check-fail as well, + // on the other hand there is no sense in processing interceptors + // since we are going to die soon. + ScopedIgnoreInterceptors ignore; +#if !SANITIZER_GO + cur_thread()->ignore_sync++; + cur_thread()->ignore_reads_and_writes++; +#endif + PrintCurrentStackSlow(StackTrace::GetCurrentPc()); +} + +bool is_initialized; + +void Initialize(ThreadState *thr) { + // Thread safe because done before all threads exist. + if (is_initialized) + return; + is_initialized = true; + // We are not ready to handle interceptors yet. + ScopedIgnoreInterceptors ignore; + SanitizerToolName = "ThreadSanitizer"; + // Install tool-specific callbacks in sanitizer_common. + SetCheckUnwindCallback(CheckUnwind); + + ctx = new(ctx_placeholder) Context; + const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"; + const char *options = GetEnv(env_name); + CacheBinaryName(); + CheckASLR(); + InitializeFlags(&ctx->flags, options, env_name); + AvoidCVE_2016_2143(); + __sanitizer::InitializePlatformEarly(); + __tsan::InitializePlatformEarly(); + +#if !SANITIZER_GO + // Re-exec ourselves if we need to set additional env or command line args. + MaybeReexec(); + + InitializeAllocator(); + ReplaceSystemMalloc(); +#endif + if (common_flags()->detect_deadlocks) + ctx->dd = DDetector::Create(flags()); + Processor *proc = ProcCreate(); + ProcWire(proc, thr); + InitializeInterceptors(); + InitializePlatform(); + InitializeDynamicAnnotations(); +#if !SANITIZER_GO + InitializeShadowMemory(); + InitializeAllocatorLate(); + InstallDeadlySignalHandlers(TsanOnDeadlySignal); +#endif + // Setup correct file descriptor for error reports. + __sanitizer_set_report_path(common_flags()->log_path); + InitializeSuppressions(); +#if !SANITIZER_GO + InitializeLibIgnore(); + Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); +#endif + + VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", + (int)internal_getpid()); + + // Initialize thread 0. + Tid tid = ThreadCreate(thr, 0, 0, true); + CHECK_EQ(tid, kMainTid); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); +#if TSAN_CONTAINS_UBSAN + __ubsan::InitAsPlugin(); +#endif + ctx->initialized = true; + +#if !SANITIZER_GO + Symbolizer::LateInitialize(); + InitializeMemoryProfiler(); +#endif + + if (flags()->stop_on_start) { + Printf("ThreadSanitizer is suspended at startup (pid %d)." + " Call __tsan_resume().\n", + (int)internal_getpid()); + while (__tsan_resumed == 0) {} + } + + OnInitialize(); +} + +void MaybeSpawnBackgroundThread() { + // On MIPS, TSan initialization is run before + // __pthread_initialize_minimal_internal() is finished, so we can not spawn + // new threads. +#if !SANITIZER_GO && !defined(__mips__) + static atomic_uint32_t bg_thread = {}; + if (atomic_load(&bg_thread, memory_order_relaxed) == 0 && + atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) { + StartBackgroundThread(); + SetSandboxingCallback(StopBackgroundThread); + } +#endif +} + + +int Finalize(ThreadState *thr) { + bool failed = false; + + if (common_flags()->print_module_map == 1) + DumpProcessMap(); + + if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1) + SleepForMillis(flags()->atexit_sleep_ms); + + // Wait for pending reports. + ctx->report_mtx.Lock(); + { ScopedErrorReportLock l; } + ctx->report_mtx.Unlock(); + +#if !SANITIZER_GO + if (Verbosity()) AllocatorPrintStats(); +#endif + + ThreadFinalize(thr); + + if (ctx->nreported) { + failed = true; +#if !SANITIZER_GO + Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); +#else + Printf("Found %d data race(s)\n", ctx->nreported); +#endif + } + + if (common_flags()->print_suppressions) + PrintMatchedSuppressions(); + + failed = OnFinalize(failed); + + return failed ? common_flags()->exitcode : 0; +} + +#if !SANITIZER_GO +void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { + ctx->thread_registry.Lock(); + ctx->report_mtx.Lock(); + ScopedErrorReportLock::Lock(); + AllocatorLock(); + // Suppress all reports in the pthread_atfork callbacks. + // Reports will deadlock on the report_mtx. + // We could ignore sync operations as well, + // but so far it's unclear if it will do more good or harm. + // Unnecessarily ignoring things can lead to false positives later. + thr->suppress_reports++; + // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and + // we'll assert in CheckNoLocks() unless we ignore interceptors. + // On OS X libSystem_atfork_prepare/parent/child callbacks are called + // after/before our callbacks and they call free. + thr->ignore_interceptors++; + // Disables memory write in OnUserAlloc/Free. + thr->ignore_reads_and_writes++; + + __tsan_test_only_on_fork(); +} + +void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { + thr->suppress_reports--; // Enabled in ForkBefore. + thr->ignore_interceptors--; + thr->ignore_reads_and_writes--; + AllocatorUnlock(); + ScopedErrorReportLock::Unlock(); + ctx->report_mtx.Unlock(); + ctx->thread_registry.Unlock(); +} + +void ForkChildAfter(ThreadState *thr, uptr pc, + bool start_thread) NO_THREAD_SAFETY_ANALYSIS { + thr->suppress_reports--; // Enabled in ForkBefore. + thr->ignore_interceptors--; + thr->ignore_reads_and_writes--; + AllocatorUnlock(); + ScopedErrorReportLock::Unlock(); + ctx->report_mtx.Unlock(); + ctx->thread_registry.Unlock(); + + uptr nthread = 0; + ctx->thread_registry.GetNumberOfThreads(0, 0, &nthread /* alive threads */); + VPrintf(1, "ThreadSanitizer: forked new process with pid %d," + " parent had %d threads\n", (int)internal_getpid(), (int)nthread); + if (nthread == 1) { + if (start_thread) + StartBackgroundThread(); + } else { + // We've just forked a multi-threaded process. We cannot reasonably function + // after that (some mutexes may be locked before fork). So just enable + // ignores for everything in the hope that we will exec soon. + ctx->after_multithreaded_fork = true; + thr->ignore_interceptors++; + ThreadIgnoreBegin(thr, pc); + ThreadIgnoreSyncBegin(thr, pc); + } +} +#endif + +#if SANITIZER_GO +NOINLINE +void GrowShadowStack(ThreadState *thr) { + const int sz = thr->shadow_stack_end - thr->shadow_stack; + const int newsz = 2 * sz; + auto *newstack = (uptr *)Alloc(newsz * sizeof(uptr)); + internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr)); + Free(thr->shadow_stack); + thr->shadow_stack = newstack; + thr->shadow_stack_pos = newstack + sz; + thr->shadow_stack_end = newstack + newsz; +} +#endif + +StackID CurrentStackId(ThreadState *thr, uptr pc) { + if (!thr->is_inited) // May happen during bootstrap. + return kInvalidStackID; + if (pc != 0) { +#if !SANITIZER_GO + DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); +#else + if (thr->shadow_stack_pos == thr->shadow_stack_end) + GrowShadowStack(thr); +#endif + thr->shadow_stack_pos[0] = pc; + thr->shadow_stack_pos++; + } + StackID id = StackDepotPut( + StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack)); + if (pc != 0) + thr->shadow_stack_pos--; + return id; +} + +namespace v3 { + +NOINLINE +void TraceSwitchPart(ThreadState *thr) { + Trace *trace = &thr->tctx->trace; + Event *pos = reinterpret_cast(atomic_load_relaxed(&thr->trace_pos)); + DCHECK_EQ(reinterpret_cast(pos + 1) & TracePart::kAlignment, 0); + auto *part = trace->parts.Back(); + DPrintf("TraceSwitchPart part=%p pos=%p\n", part, pos); + if (part) { + // We can get here when we still have space in the current trace part. + // The fast-path check in TraceAcquire has false positives in the middle of + // the part. Check if we are indeed at the end of the current part or not, + // and fill any gaps with NopEvent's. + Event *end = &part->events[TracePart::kSize]; + DCHECK_GE(pos, &part->events[0]); + DCHECK_LE(pos, end); + if (pos + 1 < end) { + if ((reinterpret_cast(pos) & TracePart::kAlignment) == + TracePart::kAlignment) + *pos++ = NopEvent; + *pos++ = NopEvent; + DCHECK_LE(pos + 2, end); + atomic_store_relaxed(&thr->trace_pos, reinterpret_cast(pos)); + // Ensure we setup trace so that the next TraceAcquire + // won't detect trace part end. + Event *ev; + CHECK(TraceAcquire(thr, &ev)); + return; + } + // We are indeed at the end. + for (; pos < end; pos++) *pos = NopEvent; + } +#if !SANITIZER_GO + if (ctx->after_multithreaded_fork) { + // We just need to survive till exec. + CHECK(part); + atomic_store_relaxed(&thr->trace_pos, + reinterpret_cast(&part->events[0])); + return; + } +#endif + part = new (MmapOrDie(sizeof(TracePart), "TracePart")) TracePart(); + part->trace = trace; + thr->trace_prev_pc = 0; + { + Lock lock(&trace->mtx); + trace->parts.PushBack(part); + atomic_store_relaxed(&thr->trace_pos, + reinterpret_cast(&part->events[0])); + } + // Make this part self-sufficient by restoring the current stack + // and mutex set in the beginning of the trace. + TraceTime(thr); + for (uptr *pos = &thr->shadow_stack[0]; pos < thr->shadow_stack_pos; pos++) + CHECK(TryTraceFunc(thr, *pos)); + for (uptr i = 0; i < thr->mset.Size(); i++) { + MutexSet::Desc d = thr->mset.Get(i); + TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0, + d.addr, d.stack_id); + } +} + +} // namespace v3 + +void TraceSwitch(ThreadState *thr) { +#if !SANITIZER_GO + if (ctx->after_multithreaded_fork) + return; +#endif + thr->nomalloc++; + Trace *thr_trace = ThreadTrace(thr->tid); + Lock l(&thr_trace->mtx); + unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts(); + TraceHeader *hdr = &thr_trace->headers[trace]; + hdr->epoch0 = thr->fast_state.epoch(); + ObtainCurrentStack(thr, 0, &hdr->stack0); + hdr->mset0 = thr->mset; + thr->nomalloc--; +} + +Trace *ThreadTrace(Tid tid) { return (Trace *)GetThreadTraceHeader(tid); } + +uptr TraceTopPC(ThreadState *thr) { + Event *events = (Event*)GetThreadTrace(thr->tid); + uptr pc = events[thr->fast_state.GetTracePos()]; + return pc; +} + +uptr TraceSize() { + return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1)); +} + +uptr TraceParts() { + return TraceSize() / kTracePartSize; +} + +#if !SANITIZER_GO +extern "C" void __tsan_trace_switch() { + TraceSwitch(cur_thread()); +} + +extern "C" void __tsan_report_race() { + ReportRace(cur_thread()); +} +#endif + +void ThreadIgnoreBegin(ThreadState *thr, uptr pc) { + DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid); + thr->ignore_reads_and_writes++; + CHECK_GT(thr->ignore_reads_and_writes, 0); + thr->fast_state.SetIgnoreBit(); +#if !SANITIZER_GO + if (pc && !ctx->after_multithreaded_fork) + thr->mop_ignore_set.Add(CurrentStackId(thr, pc)); +#endif +} + +void ThreadIgnoreEnd(ThreadState *thr) { + DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid); + CHECK_GT(thr->ignore_reads_and_writes, 0); + thr->ignore_reads_and_writes--; + if (thr->ignore_reads_and_writes == 0) { + thr->fast_state.ClearIgnoreBit(); +#if !SANITIZER_GO + thr->mop_ignore_set.Reset(); +#endif + } +} + +#if !SANITIZER_GO +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +uptr __tsan_testonly_shadow_stack_current_size() { + ThreadState *thr = cur_thread(); + return thr->shadow_stack_pos - thr->shadow_stack; +} +#endif + +void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) { + DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid); + thr->ignore_sync++; + CHECK_GT(thr->ignore_sync, 0); +#if !SANITIZER_GO + if (pc && !ctx->after_multithreaded_fork) + thr->sync_ignore_set.Add(CurrentStackId(thr, pc)); +#endif +} + +void ThreadIgnoreSyncEnd(ThreadState *thr) { + DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid); + CHECK_GT(thr->ignore_sync, 0); + thr->ignore_sync--; +#if !SANITIZER_GO + if (thr->ignore_sync == 0) + thr->sync_ignore_set.Reset(); +#endif +} + +bool MD5Hash::operator==(const MD5Hash &other) const { + return hash[0] == other.hash[0] && hash[1] == other.hash[1]; +} + +#if SANITIZER_DEBUG +void build_consistency_debug() {} +#else +void build_consistency_release() {} +#endif + +} // namespace __tsan + +#if SANITIZER_CHECK_DEADLOCKS +namespace __sanitizer { +using namespace __tsan; +MutexMeta mutex_meta[] = { + {MutexInvalid, "Invalid", {}}, + {MutexThreadRegistry, "ThreadRegistry", {}}, + {MutexTypeTrace, "Trace", {}}, + {MutexTypeReport, + "Report", + {MutexTypeSyncVar, MutexTypeGlobalProc, MutexTypeTrace}}, + {MutexTypeSyncVar, "SyncVar", {MutexTypeTrace}}, + {MutexTypeAnnotations, "Annotations", {}}, + {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}}, + {MutexTypeFired, "Fired", {MutexLeaf}}, + {MutexTypeRacy, "Racy", {MutexLeaf}}, + {MutexTypeGlobalProc, "GlobalProc", {}}, + {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}}, + {}, +}; + +void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); } +} // namespace __sanitizer +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl.h b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.h new file mode 100644 index 000000000000..c71b27e1cbf5 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.h @@ -0,0 +1,796 @@ +//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Main internal TSan header file. +// +// Ground rules: +// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static +// function-scope locals) +// - All functions/classes/etc reside in namespace __tsan, except for those +// declared in tsan_interface.h. +// - Platform-specific files should be used instead of ifdefs (*). +// - No system headers included in header files (*). +// - Platform specific headres included only into platform-specific files (*). +// +// (*) Except when inlining is critical for performance. +//===----------------------------------------------------------------------===// + +#ifndef TSAN_RTL_H +#define TSAN_RTL_H + +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_asm.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" +#include "sanitizer_common/sanitizer_libignore.h" +#include "sanitizer_common/sanitizer_suppressions.h" +#include "sanitizer_common/sanitizer_thread_registry.h" +#include "sanitizer_common/sanitizer_vector.h" +#include "tsan_clock.h" +#include "tsan_defs.h" +#include "tsan_flags.h" +#include "tsan_ignoreset.h" +#include "tsan_mman.h" +#include "tsan_mutexset.h" +#include "tsan_platform.h" +#include "tsan_report.h" +#include "tsan_shadow.h" +#include "tsan_stack_trace.h" +#include "tsan_sync.h" +#include "tsan_trace.h" + +#if SANITIZER_WORDSIZE != 64 +# error "ThreadSanitizer is supported only on 64-bit platforms" +#endif + +namespace __tsan { + +#if !SANITIZER_GO +struct MapUnmapCallback; +#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) + +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = 0; + typedef __sanitizer::CompactSizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = 20; + using AddressSpaceView = LocalAddressSpaceView; + typedef __tsan::MapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator32 PrimaryAllocator; +#else +struct AP64 { // Allocator64 parameters. Deliberately using a short name. +# if defined(__s390x__) + typedef MappingS390x Mapping; +# else + typedef Mapping48AddressSpace Mapping; +# endif + static const uptr kSpaceBeg = Mapping::kHeapMemBeg; + static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg; + static const uptr kMetadataSize = 0; + typedef DefaultSizeClassMap SizeClassMap; + typedef __tsan::MapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; +}; +typedef SizeClassAllocator64 PrimaryAllocator; +#endif +typedef CombinedAllocator Allocator; +typedef Allocator::AllocatorCache AllocatorCache; +Allocator *allocator(); +#endif + +struct ThreadSignalContext; + +struct JmpBuf { + uptr sp; + int int_signal_send; + bool in_blocking_func; + uptr in_signal_handler; + uptr *shadow_stack_pos; +}; + +// A Processor represents a physical thread, or a P for Go. +// It is used to store internal resources like allocate cache, and does not +// participate in race-detection logic (invisible to end user). +// In C++ it is tied to an OS thread just like ThreadState, however ideally +// it should be tied to a CPU (this way we will have fewer allocator caches). +// In Go it is tied to a P, so there are significantly fewer Processor's than +// ThreadState's (which are tied to Gs). +// A ThreadState must be wired with a Processor to handle events. +struct Processor { + ThreadState *thr; // currently wired thread, or nullptr +#if !SANITIZER_GO + AllocatorCache alloc_cache; + InternalAllocatorCache internal_alloc_cache; +#endif + DenseSlabAllocCache block_cache; + DenseSlabAllocCache sync_cache; + DenseSlabAllocCache clock_cache; + DDPhysicalThread *dd_pt; +}; + +#if !SANITIZER_GO +// ScopedGlobalProcessor temporary setups a global processor for the current +// thread, if it does not have one. Intended for interceptors that can run +// at the very thread end, when we already destroyed the thread processor. +struct ScopedGlobalProcessor { + ScopedGlobalProcessor(); + ~ScopedGlobalProcessor(); +}; +#endif + +// This struct is stored in TLS. +struct ThreadState { + FastState fast_state; + // Synch epoch represents the threads's epoch before the last synchronization + // action. It allows to reduce number of shadow state updates. + // For example, fast_synch_epoch=100, last write to addr X was at epoch=150, + // if we are processing write to X from the same thread at epoch=200, + // we do nothing, because both writes happen in the same 'synch epoch'. + // That is, if another memory access does not race with the former write, + // it does not race with the latter as well. + // QUESTION: can we can squeeze this into ThreadState::Fast? + // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are + // taken by epoch between synchs. + // This way we can save one load from tls. + u64 fast_synch_epoch; + // Technically `current` should be a separate THREADLOCAL variable; + // but it is placed here in order to share cache line with previous fields. + ThreadState* current; + // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. + // We do not distinguish beteween ignoring reads and writes + // for better performance. + int ignore_reads_and_writes; + atomic_sint32_t pending_signals; + int ignore_sync; + int suppress_reports; + // Go does not support ignores. +#if !SANITIZER_GO + IgnoreSet mop_ignore_set; + IgnoreSet sync_ignore_set; +#endif + uptr *shadow_stack; + uptr *shadow_stack_end; + uptr *shadow_stack_pos; + RawShadow *racy_shadow_addr; + RawShadow racy_state[2]; + MutexSet mset; + ThreadClock clock; +#if !SANITIZER_GO + Vector jmp_bufs; + int ignore_interceptors; +#endif + const Tid tid; + const int unique_id; + bool in_symbolizer; + bool in_ignored_lib; + bool is_inited; + bool is_dead; + bool is_freeing; + bool is_vptr_access; + const uptr stk_addr; + const uptr stk_size; + const uptr tls_addr; + const uptr tls_size; + ThreadContext *tctx; + + DDLogicalThread *dd_lt; + + // Current wired Processor, or nullptr. Required to handle any events. + Processor *proc1; +#if !SANITIZER_GO + Processor *proc() { return proc1; } +#else + Processor *proc(); +#endif + + atomic_uintptr_t in_signal_handler; + ThreadSignalContext *signal_ctx; + +#if !SANITIZER_GO + StackID last_sleep_stack_id; + ThreadClock last_sleep_clock; +#endif + + // Set in regions of runtime that must be signal-safe and fork-safe. + // If set, malloc must not be called. + int nomalloc; + + const ReportDesc *current_report; + + // Current position in tctx->trace.Back()->events (Event*). + atomic_uintptr_t trace_pos; + // PC of the last memory access, used to compute PC deltas in the trace. + uptr trace_prev_pc; + Sid sid; + Epoch epoch; + + explicit ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch, + unsigned reuse_count, uptr stk_addr, uptr stk_size, + uptr tls_addr, uptr tls_size); +} ALIGNED(SANITIZER_CACHE_LINE_SIZE); + +#if !SANITIZER_GO +#if SANITIZER_MAC || SANITIZER_ANDROID +ThreadState *cur_thread(); +void set_cur_thread(ThreadState *thr); +void cur_thread_finalize(); +inline ThreadState *cur_thread_init() { return cur_thread(); } +# else +__attribute__((tls_model("initial-exec"))) +extern THREADLOCAL char cur_thread_placeholder[]; +inline ThreadState *cur_thread() { + return reinterpret_cast(cur_thread_placeholder)->current; +} +inline ThreadState *cur_thread_init() { + ThreadState *thr = reinterpret_cast(cur_thread_placeholder); + if (UNLIKELY(!thr->current)) + thr->current = thr; + return thr->current; +} +inline void set_cur_thread(ThreadState *thr) { + reinterpret_cast(cur_thread_placeholder)->current = thr; +} +inline void cur_thread_finalize() { } +# endif // SANITIZER_MAC || SANITIZER_ANDROID +#endif // SANITIZER_GO + +class ThreadContext final : public ThreadContextBase { + public: + explicit ThreadContext(Tid tid); + ~ThreadContext(); + ThreadState *thr; + StackID creation_stack_id; + SyncClock sync; + // Epoch at which the thread had started. + // If we see an event from the thread stamped by an older epoch, + // the event is from a dead thread that shared tid with this thread. + u64 epoch0; + u64 epoch1; + + v3::Trace trace; + + // Override superclass callbacks. + void OnDead() override; + void OnJoined(void *arg) override; + void OnFinished() override; + void OnStarted(void *arg) override; + void OnCreated(void *arg) override; + void OnReset() override; + void OnDetached(void *arg) override; +}; + +struct RacyStacks { + MD5Hash hash[2]; + bool operator==(const RacyStacks &other) const; +}; + +struct RacyAddress { + uptr addr_min; + uptr addr_max; +}; + +struct FiredSuppression { + ReportType type; + uptr pc_or_addr; + Suppression *supp; +}; + +struct Context { + Context(); + + bool initialized; +#if !SANITIZER_GO + bool after_multithreaded_fork; +#endif + + MetaMap metamap; + + Mutex report_mtx; + int nreported; + atomic_uint64_t last_symbolize_time_ns; + + void *background_thread; + atomic_uint32_t stop_background_thread; + + ThreadRegistry thread_registry; + + Mutex racy_mtx; + Vector racy_stacks; + Vector racy_addresses; + // Number of fired suppressions may be large enough. + Mutex fired_suppressions_mtx; + InternalMmapVector fired_suppressions; + DDetector *dd; + + ClockAlloc clock_alloc; + + Flags flags; + fd_t memprof_fd; + + Mutex slot_mtx; +}; + +extern Context *ctx; // The one and the only global runtime context. + +ALWAYS_INLINE Flags *flags() { + return &ctx->flags; +} + +struct ScopedIgnoreInterceptors { + ScopedIgnoreInterceptors() { +#if !SANITIZER_GO + cur_thread()->ignore_interceptors++; +#endif + } + + ~ScopedIgnoreInterceptors() { +#if !SANITIZER_GO + cur_thread()->ignore_interceptors--; +#endif + } +}; + +const char *GetObjectTypeFromTag(uptr tag); +const char *GetReportHeaderFromTag(uptr tag); +uptr TagFromShadowStackFrame(uptr pc); + +class ScopedReportBase { + public: + void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack, + const MutexSet *mset); + void AddStack(StackTrace stack, bool suppressable = false); + void AddThread(const ThreadContext *tctx, bool suppressable = false); + void AddThread(Tid unique_tid, bool suppressable = false); + void AddUniqueTid(Tid unique_tid); + void AddMutex(const SyncVar *s); + u64 AddMutex(u64 id); + void AddLocation(uptr addr, uptr size); + void AddSleep(StackID stack_id); + void SetCount(int count); + + const ReportDesc *GetReport() const; + + protected: + ScopedReportBase(ReportType typ, uptr tag); + ~ScopedReportBase(); + + private: + ReportDesc *rep_; + // Symbolizer makes lots of intercepted calls. If we try to process them, + // at best it will cause deadlocks on internal mutexes. + ScopedIgnoreInterceptors ignore_interceptors_; + + void AddDeadMutex(u64 id); + + ScopedReportBase(const ScopedReportBase &) = delete; + void operator=(const ScopedReportBase &) = delete; +}; + +class ScopedReport : public ScopedReportBase { + public: + explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); + ~ScopedReport(); + + private: + ScopedErrorReportLock lock_; +}; + +bool ShouldReport(ThreadState *thr, ReportType typ); +ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack); +void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk, + MutexSet *mset, uptr *tag = nullptr); + +// The stack could look like: +// |
| | tag | +// This will extract the tag and keep: +// |
| | +template +void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) { + if (stack->size < 2) return; + uptr possible_tag_pc = stack->trace[stack->size - 2]; + uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc); + if (possible_tag == kExternalTagNone) return; + stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1]; + stack->size -= 1; + if (tag) *tag = possible_tag; +} + +template +void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack, + uptr *tag = nullptr) { + uptr size = thr->shadow_stack_pos - thr->shadow_stack; + uptr start = 0; + if (size + !!toppc > kStackTraceMax) { + start = size + !!toppc - kStackTraceMax; + size = kStackTraceMax - !!toppc; + } + stack->Init(&thr->shadow_stack[start], size, toppc); + ExtractTagFromStack(stack, tag); +} + +#define GET_STACK_TRACE_FATAL(thr, pc) \ + VarSizeStackTrace stack; \ + ObtainCurrentStack(thr, pc, &stack); \ + stack.ReverseOrder(); + +void MapShadow(uptr addr, uptr size); +void MapThreadTrace(uptr addr, uptr size, const char *name); +void DontNeedShadowFor(uptr addr, uptr size); +void UnmapShadow(ThreadState *thr, uptr addr, uptr size); +void InitializeShadowMemory(); +void InitializeInterceptors(); +void InitializeLibIgnore(); +void InitializeDynamicAnnotations(); + +void ForkBefore(ThreadState *thr, uptr pc); +void ForkParentAfter(ThreadState *thr, uptr pc); +void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread); + +void ReportRace(ThreadState *thr); +bool OutputReport(ThreadState *thr, const ScopedReport &srep); +bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace); +bool IsExpectedReport(uptr addr, uptr size); + +#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 +# define DPrintf Printf +#else +# define DPrintf(...) +#endif + +#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 +# define DPrintf2 Printf +#else +# define DPrintf2(...) +#endif + +StackID CurrentStackId(ThreadState *thr, uptr pc); +ReportStack *SymbolizeStackId(StackID stack_id); +void PrintCurrentStack(ThreadState *thr, uptr pc); +void PrintCurrentStackSlow(uptr pc); // uses libunwind +MBlock *JavaHeapBlock(uptr addr, uptr *start); + +void Initialize(ThreadState *thr); +void MaybeSpawnBackgroundThread(); +int Finalize(ThreadState *thr); + +void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); +void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write); + +void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, + int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic); +void MemoryAccessImpl(ThreadState *thr, uptr addr, + int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, + u64 *shadow_mem, Shadow cur); +void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, + uptr size, bool is_write); +void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ); + +const int kSizeLog1 = 0; +const int kSizeLog2 = 1; +const int kSizeLog4 = 2; +const int kSizeLog8 = 3; + +ALWAYS_INLINE +void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ) { + int size_log; + switch (size) { + case 1: + size_log = kSizeLog1; + break; + case 2: + size_log = kSizeLog2; + break; + case 4: + size_log = kSizeLog4; + break; + default: + DCHECK_EQ(size, 8); + size_log = kSizeLog8; + break; + } + bool is_write = !(typ & kAccessRead); + bool is_atomic = typ & kAccessAtomic; + if (typ & kAccessVptr) + thr->is_vptr_access = true; + if (typ & kAccessFree) + thr->is_freeing = true; + MemoryAccess(thr, pc, addr, size_log, is_write, is_atomic); + if (typ & kAccessVptr) + thr->is_vptr_access = false; + if (typ & kAccessFree) + thr->is_freeing = false; +} + +void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); +void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); +void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); +void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, + uptr size); + +void ThreadIgnoreBegin(ThreadState *thr, uptr pc); +void ThreadIgnoreEnd(ThreadState *thr); +void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc); +void ThreadIgnoreSyncEnd(ThreadState *thr); + +void FuncEntry(ThreadState *thr, uptr pc); +void FuncExit(ThreadState *thr); + +Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); +void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, + ThreadType thread_type); +void ThreadFinish(ThreadState *thr); +Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid); +void ThreadJoin(ThreadState *thr, uptr pc, Tid tid); +void ThreadDetach(ThreadState *thr, uptr pc, Tid tid); +void ThreadFinalize(ThreadState *thr); +void ThreadSetName(ThreadState *thr, const char *name); +int ThreadCount(ThreadState *thr); +void ProcessPendingSignalsImpl(ThreadState *thr); +void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid); + +Processor *ProcCreate(); +void ProcDestroy(Processor *proc); +void ProcWire(Processor *proc, ThreadState *thr); +void ProcUnwire(Processor *proc, ThreadState *thr); + +// Note: the parameter is called flagz, because flags is already taken +// by the global function that returns flags. +void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); +void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); +void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); +void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0, + int rec = 1); +int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); +void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); +void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); +void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); +void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); +void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD +void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr); + +void Acquire(ThreadState *thr, uptr pc, uptr addr); +// AcquireGlobal synchronizes the current thread with all other threads. +// In terms of happens-before relation, it draws a HB edge from all threads +// (where they happen to execute right now) to the current thread. We use it to +// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal +// right before executing finalizers. This provides a coarse, but simple +// approximation of the actual required synchronization. +void AcquireGlobal(ThreadState *thr); +void Release(ThreadState *thr, uptr pc, uptr addr); +void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr); +void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); +void AfterSleep(ThreadState *thr, uptr pc); +void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c); +void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); +void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c); +void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c); +void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); + +// The hacky call uses custom calling convention and an assembly thunk. +// It is considerably faster that a normal call for the caller +// if it is not executed (it is intended for slow paths from hot functions). +// The trick is that the call preserves all registers and the compiler +// does not treat it as a call. +// If it does not work for you, use normal call. +#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC +// The caller may not create the stack frame for itself at all, +// so we create a reserve stack frame for it (1024b must be enough). +#define HACKY_CALL(f) \ + __asm__ __volatile__("sub $1024, %%rsp;" \ + CFI_INL_ADJUST_CFA_OFFSET(1024) \ + ".hidden " #f "_thunk;" \ + "call " #f "_thunk;" \ + "add $1024, %%rsp;" \ + CFI_INL_ADJUST_CFA_OFFSET(-1024) \ + ::: "memory", "cc"); +#else +#define HACKY_CALL(f) f() +#endif + +void TraceSwitch(ThreadState *thr); +uptr TraceTopPC(ThreadState *thr); +uptr TraceSize(); +uptr TraceParts(); +Trace *ThreadTrace(Tid tid); + +extern "C" void __tsan_trace_switch(); +void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, + EventType typ, u64 addr) { + if (!kCollectHistory) + return; + // TraceSwitch accesses shadow_stack, but it's called infrequently, + // so we check it here proactively. + DCHECK(thr->shadow_stack); + DCHECK_GE((int)typ, 0); + DCHECK_LE((int)typ, 7); + DCHECK_EQ(GetLsb(addr, kEventPCBits), addr); + u64 pos = fs.GetTracePos(); + if (UNLIKELY((pos % kTracePartSize) == 0)) { +#if !SANITIZER_GO + HACKY_CALL(__tsan_trace_switch); +#else + TraceSwitch(thr); +#endif + } + Event *trace = (Event*)GetThreadTrace(fs.tid()); + Event *evp = &trace[pos]; + Event ev = (u64)addr | ((u64)typ << kEventPCBits); + *evp = ev; +} + +#if !SANITIZER_GO +uptr ALWAYS_INLINE HeapEnd() { + return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); +} +#endif + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + +ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) { + if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals))) + ProcessPendingSignalsImpl(thr); +} + +extern bool is_initialized; + +ALWAYS_INLINE +void LazyInitialize(ThreadState *thr) { + // If we can use .preinit_array, assume that __tsan_init + // called from .preinit_array initializes runtime before + // any instrumented code. +#if !SANITIZER_CAN_USE_PREINIT_ARRAY + if (UNLIKELY(!is_initialized)) + Initialize(thr); +#endif +} + +namespace v3 { + +void TraceSwitchPart(ThreadState *thr); +bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, + uptr size, AccessType typ, VarSizeStackTrace *pstk, + MutexSet *pmset, uptr *ptag); + +template +ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr, + EventT **ev) { + Event *pos = reinterpret_cast(atomic_load_relaxed(&thr->trace_pos)); +#if SANITIZER_DEBUG + // TraceSwitch acquires these mutexes, + // so we lock them here to detect deadlocks more reliably. + { Lock lock(&ctx->slot_mtx); } + { Lock lock(&thr->tctx->trace.mtx); } + TracePart *current = thr->tctx->trace.parts.Back(); + if (current) { + DCHECK_GE(pos, ¤t->events[0]); + DCHECK_LE(pos, ¤t->events[TracePart::kSize]); + } else { + DCHECK_EQ(pos, nullptr); + } +#endif + // TracePart is allocated with mmap and is at least 4K aligned. + // So the following check is a faster way to check for part end. + // It may have false positives in the middle of the trace, + // they are filtered out in TraceSwitch. + if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0)) + return false; + *ev = reinterpret_cast(pos); + return true; +} + +template +ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) { + DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]); + atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1)); +} + +template +void TraceEvent(ThreadState *thr, EventT ev) { + EventT *evp; + if (!TraceAcquire(thr, &evp)) { + TraceSwitchPart(thr); + UNUSED bool res = TraceAcquire(thr, &evp); + DCHECK(res); + } + *evp = ev; + TraceRelease(thr, evp); +} + +ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr, + uptr pc = 0) { + if (!kCollectHistory) + return true; + EventFunc *ev; + if (UNLIKELY(!TraceAcquire(thr, &ev))) + return false; + ev->is_access = 0; + ev->is_func = 1; + ev->pc = pc; + TraceRelease(thr, ev); + return true; +} + +WARN_UNUSED_RESULT +bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ); +WARN_UNUSED_RESULT +bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ); +void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ); +void TraceFunc(ThreadState *thr, uptr pc = 0); +void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, + StackID stk); +void TraceMutexUnlock(ThreadState *thr, uptr addr); +void TraceTime(ThreadState *thr); + +} // namespace v3 + +void GrowShadowStack(ThreadState *thr); + +ALWAYS_INLINE +void FuncEntry(ThreadState *thr, uptr pc) { + DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc); + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); + } + + // Shadow stack maintenance can be replaced with + // stack unwinding during trace switch (which presumably must be faster). + DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack); +#if !SANITIZER_GO + DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); +#else + if (thr->shadow_stack_pos == thr->shadow_stack_end) + GrowShadowStack(thr); +#endif + thr->shadow_stack_pos[0] = pc; + thr->shadow_stack_pos++; +} + +ALWAYS_INLINE +void FuncExit(ThreadState *thr) { + DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0); + } + + DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); +#if !SANITIZER_GO + DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); +#endif + thr->shadow_stack_pos--; +} + +#if !SANITIZER_GO +extern void (*on_initialize)(void); +extern int (*on_finalize)(int); +#endif + +} // namespace __tsan + +#endif // TSAN_RTL_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S new file mode 100644 index 000000000000..e0b4c71dfed9 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S @@ -0,0 +1,245 @@ +// The content of this file is AArch64-only: +#if defined(__aarch64__) + +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__APPLE__) +.align 2 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long _setjmp$non_lazy_ptr +_setjmp$non_lazy_ptr: +.indirect_symbol _setjmp +.long 0 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long __setjmp$non_lazy_ptr +__setjmp$non_lazy_ptr: +.indirect_symbol __setjmp +.long 0 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long _sigsetjmp$non_lazy_ptr +_sigsetjmp$non_lazy_ptr: +.indirect_symbol _sigsetjmp +.long 0 +#endif + +#if !defined(__APPLE__) +.section .text +#else +.section __TEXT,__text +.align 3 +#endif + +ASM_HIDDEN(__tsan_setjmp) +.comm _ZN14__interception11real_setjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): + CFI_STARTPROC + + // Save frame/link register + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save env parameter + str x0, [sp, 16] + CFI_OFFSET (0, -16) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + add x0, x29, 32 + + // call tsan interceptor + bl ASM_SYMBOL(__tsan_setjmp) + + // Restore env parameter + ldr x0, [sp, 16] + CFI_RESTORE (0) + + // Restore frame/link register + ldp x29, x30, [sp], 32 + CFI_RESTORE (29) + CFI_RESTORE (30) + CFI_DEF_CFA (31, 0) + + // tail jump to libc setjmp +#if !defined(__APPLE__) + adrp x1, :got:_ZN14__interception11real_setjmpE + ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE] + ldr x1, [x1] +#else + adrp x1, _setjmp$non_lazy_ptr@page + add x1, x1, _setjmp$non_lazy_ptr@pageoff + ldr x1, [x1] +#endif + br x1 + + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) + +.comm _ZN14__interception12real__setjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): + CFI_STARTPROC + + // Save frame/link register + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save env parameter + str x0, [sp, 16] + CFI_OFFSET (0, -16) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + add x0, x29, 32 + + // call tsan interceptor + bl ASM_SYMBOL(__tsan_setjmp) + + // Restore env parameter + ldr x0, [sp, 16] + CFI_RESTORE (0) + + // Restore frame/link register + ldp x29, x30, [sp], 32 + CFI_RESTORE (29) + CFI_RESTORE (30) + CFI_DEF_CFA (31, 0) + + // tail jump to libc setjmp +#if !defined(__APPLE__) + adrp x1, :got:_ZN14__interception12real__setjmpE + ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE] + ldr x1, [x1] +#else + adrp x1, __setjmp$non_lazy_ptr@page + add x1, x1, __setjmp$non_lazy_ptr@pageoff + ldr x1, [x1] +#endif + br x1 + + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) + +.comm _ZN14__interception14real_sigsetjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): + CFI_STARTPROC + + // Save frame/link register + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save env and savesigs parameter + stp x0, x1, [sp, 16] + CFI_OFFSET (0, -16) + CFI_OFFSET (1, -8) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + add x0, x29, 32 + + // call tsan interceptor + bl ASM_SYMBOL(__tsan_setjmp) + + // Restore env and savesigs parameter + ldp x0, x1, [sp, 16] + CFI_RESTORE (0) + CFI_RESTORE (1) + + // Restore frame/link register + ldp x29, x30, [sp], 32 + CFI_RESTORE (29) + CFI_RESTORE (30) + CFI_DEF_CFA (31, 0) + + // tail jump to libc sigsetjmp +#if !defined(__APPLE__) + adrp x2, :got:_ZN14__interception14real_sigsetjmpE + ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE] + ldr x2, [x2] +#else + adrp x2, _sigsetjmp$non_lazy_ptr@page + add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff + ldr x2, [x2] +#endif + br x2 + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) + +#if !defined(__APPLE__) +.comm _ZN14__interception16real___sigsetjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): + CFI_STARTPROC + + // Save frame/link register + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save env and savesigs parameter + stp x0, x1, [sp, 16] + CFI_OFFSET (0, -16) + CFI_OFFSET (1, -8) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + add x0, x29, 32 + + // call tsan interceptor + bl ASM_SYMBOL(__tsan_setjmp) + + // Restore env and savesigs parameter + ldp x0, x1, [sp, 16] + CFI_RESTORE (0) + CFI_RESTORE (1) + + // Restore frame/link register + ldp x29, x30, [sp], 32 + CFI_RESTORE (29) + CFI_RESTORE (30) + CFI_DEF_CFA (31, 0) + + // tail jump to libc __sigsetjmp +#if !defined(__APPLE__) + adrp x2, :got:_ZN14__interception16real___sigsetjmpE + ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE] + ldr x2, [x2] +#else + adrp x2, ASM_SYMBOL(__sigsetjmp)@page + add x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff +#endif + br x2 + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif + +NO_EXEC_STACK_DIRECTIVE + +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp new file mode 100644 index 000000000000..7365fdaa3038 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp @@ -0,0 +1,604 @@ +//===-- tsan_rtl_access.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Definitions of memory access and function entry/exit entry points. +//===----------------------------------------------------------------------===// + +#include "tsan_rtl.h" + +namespace __tsan { + +namespace v3 { + +ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, + uptr addr, uptr size, + AccessType typ) { + DCHECK(size == 1 || size == 2 || size == 4 || size == 8); + if (!kCollectHistory) + return true; + EventAccess *ev; + if (UNLIKELY(!TraceAcquire(thr, &ev))) + return false; + u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3; + uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1)); + thr->trace_prev_pc = pc; + if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) { + ev->is_access = 1; + ev->is_read = !!(typ & kAccessRead); + ev->is_atomic = !!(typ & kAccessAtomic); + ev->size_log = size_log; + ev->pc_delta = pc_delta; + DCHECK_EQ(ev->pc_delta, pc_delta); + ev->addr = CompressAddr(addr); + TraceRelease(thr, ev); + return true; + } + auto *evex = reinterpret_cast(ev); + evex->is_access = 0; + evex->is_func = 0; + evex->type = EventType::kAccessExt; + evex->is_read = !!(typ & kAccessRead); + evex->is_atomic = !!(typ & kAccessAtomic); + evex->size_log = size_log; + evex->addr = CompressAddr(addr); + evex->pc = pc; + TraceRelease(thr, evex); + return true; +} + +ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, + uptr addr, uptr size, + AccessType typ) { + if (!kCollectHistory) + return true; + EventAccessRange *ev; + if (UNLIKELY(!TraceAcquire(thr, &ev))) + return false; + thr->trace_prev_pc = pc; + ev->is_access = 0; + ev->is_func = 0; + ev->type = EventType::kAccessRange; + ev->is_read = !!(typ & kAccessRead); + ev->is_free = !!(typ & kAccessFree); + ev->size_lo = size; + ev->pc = CompressAddr(pc); + ev->addr = CompressAddr(addr); + ev->size_hi = size >> EventAccessRange::kSizeLoBits; + TraceRelease(thr, ev); + return true; +} + +void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ) { + if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ))) + return; + TraceSwitchPart(thr); + UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ); + DCHECK(res); +} + +void TraceFunc(ThreadState *thr, uptr pc) { + if (LIKELY(TryTraceFunc(thr, pc))) + return; + TraceSwitchPart(thr); + UNUSED bool res = TryTraceFunc(thr, pc); + DCHECK(res); +} + +void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, + StackID stk) { + DCHECK(type == EventType::kLock || type == EventType::kRLock); + if (!kCollectHistory) + return; + EventLock ev; + ev.is_access = 0; + ev.is_func = 0; + ev.type = type; + ev.pc = CompressAddr(pc); + ev.stack_lo = stk; + ev.stack_hi = stk >> EventLock::kStackIDLoBits; + ev._ = 0; + ev.addr = CompressAddr(addr); + TraceEvent(thr, ev); +} + +void TraceMutexUnlock(ThreadState *thr, uptr addr) { + if (!kCollectHistory) + return; + EventUnlock ev; + ev.is_access = 0; + ev.is_func = 0; + ev.type = EventType::kUnlock; + ev._ = 0; + ev.addr = CompressAddr(addr); + TraceEvent(thr, ev); +} + +void TraceTime(ThreadState *thr) { + if (!kCollectHistory) + return; + EventTime ev; + ev.is_access = 0; + ev.is_func = 0; + ev.type = EventType::kTime; + ev.sid = static_cast(thr->sid); + ev.epoch = static_cast(thr->epoch); + ev._ = 0; + TraceEvent(thr, ev); +} + +} // namespace v3 + +ALWAYS_INLINE +Shadow LoadShadow(u64 *p) { + u64 raw = atomic_load((atomic_uint64_t *)p, memory_order_relaxed); + return Shadow(raw); +} + +ALWAYS_INLINE +void StoreShadow(u64 *sp, u64 s) { + atomic_store((atomic_uint64_t *)sp, s, memory_order_relaxed); +} + +ALWAYS_INLINE +void StoreIfNotYetStored(u64 *sp, u64 *s) { + StoreShadow(sp, *s); + *s = 0; +} + +extern "C" void __tsan_report_race(); + +ALWAYS_INLINE +void HandleRace(ThreadState *thr, u64 *shadow_mem, Shadow cur, Shadow old) { + thr->racy_state[0] = cur.raw(); + thr->racy_state[1] = old.raw(); + thr->racy_shadow_addr = shadow_mem; +#if !SANITIZER_GO + HACKY_CALL(__tsan_report_race); +#else + ReportRace(thr); +#endif +} + +static inline bool HappensBefore(Shadow old, ThreadState *thr) { + return thr->clock.get(old.TidWithIgnore()) >= old.epoch(); +} + +ALWAYS_INLINE +void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog, + bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem, + Shadow cur) { + // This potentially can live in an MMX/SSE scratch register. + // The required intrinsics are: + // __m128i _mm_move_epi64(__m128i*); + // _mm_storel_epi64(u64*, __m128i); + u64 store_word = cur.raw(); + bool stored = false; + + // scan all the shadow values and dispatch to 4 categories: + // same, replace, candidate and race (see comments below). + // we consider only 3 cases regarding access sizes: + // equal, intersect and not intersect. initially I considered + // larger and smaller as well, it allowed to replace some + // 'candidates' with 'same' or 'replace', but I think + // it's just not worth it (performance- and complexity-wise). + + Shadow old(0); + + // It release mode we manually unroll the loop, + // because empirically gcc generates better code this way. + // However, we can't afford unrolling in debug mode, because the function + // consumes almost 4K of stack. Gtest gives only 4K of stack to death test + // threads, which is not enough for the unrolled loop. +#if SANITIZER_DEBUG + for (int idx = 0; idx < 4; idx++) { +# include "tsan_update_shadow_word.inc" + } +#else + int idx = 0; +# include "tsan_update_shadow_word.inc" + idx = 1; + if (stored) { +# include "tsan_update_shadow_word.inc" + } else { +# include "tsan_update_shadow_word.inc" + } + idx = 2; + if (stored) { +# include "tsan_update_shadow_word.inc" + } else { +# include "tsan_update_shadow_word.inc" + } + idx = 3; + if (stored) { +# include "tsan_update_shadow_word.inc" + } else { +# include "tsan_update_shadow_word.inc" + } +#endif + + // we did not find any races and had already stored + // the current access info, so we are done + if (LIKELY(stored)) + return; + // choose a random candidate slot and replace it + StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); + return; +RACE: + HandleRace(thr, shadow_mem, cur, old); + return; +} + +void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ) { + DCHECK(!(typ & kAccessAtomic)); + const bool kAccessIsWrite = !(typ & kAccessRead); + const bool kIsAtomic = false; + while (size) { + int size1 = 1; + int kAccessSizeLog = kSizeLog1; + if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) { + size1 = 8; + kAccessSizeLog = kSizeLog8; + } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) { + size1 = 4; + kAccessSizeLog = kSizeLog4; + } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) { + size1 = 2; + kAccessSizeLog = kSizeLog2; + } + MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic); + addr += size1; + size -= size1; + } +} + +ALWAYS_INLINE +bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) { + Shadow cur(a); + for (uptr i = 0; i < kShadowCnt; i++) { + Shadow old(LoadShadow(&s[i])); + if (Shadow::Addr0AndSizeAreEqual(cur, old) && + old.TidWithIgnore() == cur.TidWithIgnore() && + old.epoch() > sync_epoch && old.IsAtomic() == cur.IsAtomic() && + old.IsRead() <= cur.IsRead()) + return true; + } + return false; +} + +#if TSAN_VECTORIZE +# define SHUF(v0, v1, i0, i1, i2, i3) \ + _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), \ + _mm_castsi128_ps(v1), \ + (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64)) +ALWAYS_INLINE +bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) { + // This is an optimized version of ContainsSameAccessSlow. + // load current access into access[0:63] + const m128 access = _mm_cvtsi64_si128(a); + // duplicate high part of access in addr0: + // addr0[0:31] = access[32:63] + // addr0[32:63] = access[32:63] + // addr0[64:95] = access[32:63] + // addr0[96:127] = access[32:63] + const m128 addr0 = SHUF(access, access, 1, 1, 1, 1); + // load 4 shadow slots + const m128 shadow0 = _mm_load_si128((__m128i *)s); + const m128 shadow1 = _mm_load_si128((__m128i *)s + 1); + // load high parts of 4 shadow slots into addr_vect: + // addr_vect[0:31] = shadow0[32:63] + // addr_vect[32:63] = shadow0[96:127] + // addr_vect[64:95] = shadow1[32:63] + // addr_vect[96:127] = shadow1[96:127] + m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3); + if (!is_write) { + // set IsRead bit in addr_vect + const m128 rw_mask1 = _mm_cvtsi64_si128(1 << 15); + const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0); + addr_vect = _mm_or_si128(addr_vect, rw_mask); + } + // addr0 == addr_vect? + const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect); + // epoch1[0:63] = sync_epoch + const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch); + // epoch[0:31] = sync_epoch[0:31] + // epoch[32:63] = sync_epoch[0:31] + // epoch[64:95] = sync_epoch[0:31] + // epoch[96:127] = sync_epoch[0:31] + const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0); + // load low parts of shadow cell epochs into epoch_vect: + // epoch_vect[0:31] = shadow0[0:31] + // epoch_vect[32:63] = shadow0[64:95] + // epoch_vect[64:95] = shadow1[0:31] + // epoch_vect[96:127] = shadow1[64:95] + const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2); + // epoch_vect >= sync_epoch? + const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch); + // addr_res & epoch_res + const m128 res = _mm_and_si128(addr_res, epoch_res); + // mask[0] = res[7] + // mask[1] = res[15] + // ... + // mask[15] = res[127] + const int mask = _mm_movemask_epi8(res); + return mask != 0; +} +#endif + +ALWAYS_INLINE +bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) { +#if TSAN_VECTORIZE + bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write); + // NOTE: this check can fail if the shadow is concurrently mutated + // by other threads. But it still can be useful if you modify + // ContainsSameAccessFast and want to ensure that it's not completely broken. + // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write)); + return res; +#else + return ContainsSameAccessSlow(s, a, sync_epoch, is_write); +#endif +} + +ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, + int kAccessSizeLog, bool kAccessIsWrite, + bool kIsAtomic) { + RawShadow *shadow_mem = MemToShadow(addr); + DPrintf2( + "#%d: MemoryAccess: @%p %p size=%d" + " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", + (int)thr->fast_state.tid(), (void *)pc, (void *)addr, + (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, + (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2], + (uptr)shadow_mem[3]); +#if SANITIZER_DEBUG + if (!IsAppMem(addr)) { + Printf("Access to non app mem %zx\n", addr); + DCHECK(IsAppMem(addr)); + } + if (!IsShadowMem(shadow_mem)) { + Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); + DCHECK(IsShadowMem(shadow_mem)); + } +#endif + + if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { + // Access to .rodata section, no races here. + // Measurements show that it can be 10-20% of all memory accesses. + return; + } + + FastState fast_state = thr->fast_state; + if (UNLIKELY(fast_state.GetIgnoreBit())) { + return; + } + + Shadow cur(fast_state); + cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); + cur.SetWrite(kAccessIsWrite); + cur.SetAtomic(kIsAtomic); + + if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, + kAccessIsWrite))) { + return; + } + + if (kCollectHistory) { + fast_state.IncrementEpoch(); + thr->fast_state = fast_state; + TraceAddEvent(thr, fast_state, EventTypeMop, pc); + cur.IncrementEpoch(); + } + + MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, + shadow_mem, cur); +} + +// Called by MemoryAccessRange in tsan_rtl_thread.cpp +ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr, + int kAccessSizeLog, + bool kAccessIsWrite, bool kIsAtomic, + u64 *shadow_mem, Shadow cur) { + if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, + kAccessIsWrite))) { + return; + } + + MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, + shadow_mem, cur); +} + +static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, + u64 val) { + (void)thr; + (void)pc; + if (size == 0) + return; + // FIXME: fix me. + uptr offset = addr % kShadowCell; + if (offset) { + offset = kShadowCell - offset; + if (size <= offset) + return; + addr += offset; + size -= offset; + } + DCHECK_EQ(addr % 8, 0); + // If a user passes some insane arguments (memset(0)), + // let it just crash as usual. + if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) + return; + // Don't want to touch lots of shadow memory. + // If a program maps 10MB stack, there is no need reset the whole range. + size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); + // UnmapOrDie/MmapFixedNoReserve does not work on Windows. + if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) { + RawShadow *p = MemToShadow(addr); + CHECK(IsShadowMem(p)); + CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1)); + // FIXME: may overwrite a part outside the region + for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) { + p[i++] = val; + for (uptr j = 1; j < kShadowCnt; j++) p[i++] = 0; + } + } else { + // The region is big, reset only beginning and end. + const uptr kPageSize = GetPageSizeCached(); + RawShadow *begin = MemToShadow(addr); + RawShadow *end = begin + size / kShadowCell * kShadowCnt; + RawShadow *p = begin; + // Set at least first kPageSize/2 to page boundary. + while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) { + *p++ = val; + for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0; + } + // Reset middle part. + RawShadow *p1 = p; + p = RoundDown(end, kPageSize); + if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1)) + Die(); + // Set the ending. + while (p < end) { + *p++ = val; + for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0; + } + } +} + +void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { + MemoryRangeSet(thr, pc, addr, size, 0); +} + +void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { + // Processing more than 1k (4k of shadow) is expensive, + // can cause excessive memory consumption (user does not necessary touch + // the whole range) and most likely unnecessary. + if (size > 1024) + size = 1024; + CHECK_EQ(thr->is_freeing, false); + thr->is_freeing = true; + MemoryAccessRange(thr, pc, addr, size, true); + thr->is_freeing = false; + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + } + Shadow s(thr->fast_state); + s.ClearIgnoreBit(); + s.MarkAsFreed(); + s.SetWrite(true); + s.SetAddr0AndSizeLog(0, 3); + MemoryRangeSet(thr, pc, addr, size, s.raw()); +} + +void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + } + Shadow s(thr->fast_state); + s.ClearIgnoreBit(); + s.SetWrite(true); + s.SetAddr0AndSizeLog(0, 3); + MemoryRangeSet(thr, pc, addr, size, s.raw()); +} + +void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, + uptr size) { + if (thr->ignore_reads_and_writes == 0) + MemoryRangeImitateWrite(thr, pc, addr, size); + else + MemoryResetRange(thr, pc, addr, size); +} + +void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, + bool is_write) { + if (size == 0) + return; + + RawShadow *shadow_mem = MemToShadow(addr); + DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n", thr->tid, + (void *)pc, (void *)addr, (int)size, is_write); + +#if SANITIZER_DEBUG + if (!IsAppMem(addr)) { + Printf("Access to non app mem %zx\n", addr); + DCHECK(IsAppMem(addr)); + } + if (!IsAppMem(addr + size - 1)) { + Printf("Access to non app mem %zx\n", addr + size - 1); + DCHECK(IsAppMem(addr + size - 1)); + } + if (!IsShadowMem(shadow_mem)) { + Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); + DCHECK(IsShadowMem(shadow_mem)); + } + if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) { + Printf("Bad shadow addr %p (%zx)\n", shadow_mem + size * kShadowCnt / 8 - 1, + addr + size - 1); + DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)); + } +#endif + + if (*shadow_mem == kShadowRodata) { + DCHECK(!is_write); + // Access to .rodata section, no races here. + // Measurements show that it can be 10-20% of all memory accesses. + return; + } + + FastState fast_state = thr->fast_state; + if (fast_state.GetIgnoreBit()) + return; + + fast_state.IncrementEpoch(); + thr->fast_state = fast_state; + TraceAddEvent(thr, fast_state, EventTypeMop, pc); + + bool unaligned = (addr % kShadowCell) != 0; + + // Handle unaligned beginning, if any. + for (; addr % kShadowCell && size; addr++, size--) { + int const kAccessSizeLog = 0; + Shadow cur(fast_state); + cur.SetWrite(is_write); + cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog); + MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem, + cur); + } + if (unaligned) + shadow_mem += kShadowCnt; + // Handle middle part, if any. + for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) { + int const kAccessSizeLog = 3; + Shadow cur(fast_state); + cur.SetWrite(is_write); + cur.SetAddr0AndSizeLog(0, kAccessSizeLog); + MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem, + cur); + shadow_mem += kShadowCnt; + } + // Handle ending, if any. + for (; size; addr++, size--) { + int const kAccessSizeLog = 0; + Shadow cur(fast_state); + cur.SetWrite(is_write); + cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog); + MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem, + cur); + } +} + +} // namespace __tsan + +#if !SANITIZER_GO +// Must be included in this file to make sure everything is inlined. +# include "tsan_interface.inc" +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S new file mode 100644 index 000000000000..632b19d18158 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S @@ -0,0 +1,446 @@ +// The content of this file is x86_64-only: +#if defined(__x86_64__) + +#include "sanitizer_common/sanitizer_asm.h" + +#if !defined(__APPLE__) +.section .text +#else +.section __TEXT,__text +#endif + +ASM_HIDDEN(__tsan_trace_switch) +.globl ASM_SYMBOL(__tsan_trace_switch_thunk) +ASM_SYMBOL(__tsan_trace_switch_thunk): + CFI_STARTPROC + _CET_ENDBR + # Save scratch registers. + push %rax + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rax, 0) + push %rcx + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rcx, 0) + push %rdx + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdx, 0) + push %rsi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rsi, 0) + push %rdi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdi, 0) + push %r8 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r8, 0) + push %r9 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r9, 0) + push %r10 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r10, 0) + push %r11 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r11, 0) + # All XMM registers are caller-saved. + sub $0x100, %rsp + CFI_ADJUST_CFA_OFFSET(0x100) + vmovdqu %xmm0, 0x0(%rsp) + vmovdqu %xmm1, 0x10(%rsp) + vmovdqu %xmm2, 0x20(%rsp) + vmovdqu %xmm3, 0x30(%rsp) + vmovdqu %xmm4, 0x40(%rsp) + vmovdqu %xmm5, 0x50(%rsp) + vmovdqu %xmm6, 0x60(%rsp) + vmovdqu %xmm7, 0x70(%rsp) + vmovdqu %xmm8, 0x80(%rsp) + vmovdqu %xmm9, 0x90(%rsp) + vmovdqu %xmm10, 0xa0(%rsp) + vmovdqu %xmm11, 0xb0(%rsp) + vmovdqu %xmm12, 0xc0(%rsp) + vmovdqu %xmm13, 0xd0(%rsp) + vmovdqu %xmm14, 0xe0(%rsp) + vmovdqu %xmm15, 0xf0(%rsp) + # Align stack frame. + push %rbx # non-scratch + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rbx, 0) + mov %rsp, %rbx # save current rsp + CFI_DEF_CFA_REGISTER(%rbx) + shr $4, %rsp # clear 4 lsb, align to 16 + shl $4, %rsp + + call ASM_SYMBOL(__tsan_trace_switch) + + # Unalign stack frame back. + mov %rbx, %rsp # restore the original rsp + CFI_DEF_CFA_REGISTER(%rsp) + pop %rbx + CFI_ADJUST_CFA_OFFSET(-8) + # Restore scratch registers. + vmovdqu 0x0(%rsp), %xmm0 + vmovdqu 0x10(%rsp), %xmm1 + vmovdqu 0x20(%rsp), %xmm2 + vmovdqu 0x30(%rsp), %xmm3 + vmovdqu 0x40(%rsp), %xmm4 + vmovdqu 0x50(%rsp), %xmm5 + vmovdqu 0x60(%rsp), %xmm6 + vmovdqu 0x70(%rsp), %xmm7 + vmovdqu 0x80(%rsp), %xmm8 + vmovdqu 0x90(%rsp), %xmm9 + vmovdqu 0xa0(%rsp), %xmm10 + vmovdqu 0xb0(%rsp), %xmm11 + vmovdqu 0xc0(%rsp), %xmm12 + vmovdqu 0xd0(%rsp), %xmm13 + vmovdqu 0xe0(%rsp), %xmm14 + vmovdqu 0xf0(%rsp), %xmm15 + add $0x100, %rsp + CFI_ADJUST_CFA_OFFSET(-0x100) + pop %r11 + CFI_ADJUST_CFA_OFFSET(-8) + pop %r10 + CFI_ADJUST_CFA_OFFSET(-8) + pop %r9 + CFI_ADJUST_CFA_OFFSET(-8) + pop %r8 + CFI_ADJUST_CFA_OFFSET(-8) + pop %rdi + CFI_ADJUST_CFA_OFFSET(-8) + pop %rsi + CFI_ADJUST_CFA_OFFSET(-8) + pop %rdx + CFI_ADJUST_CFA_OFFSET(-8) + pop %rcx + CFI_ADJUST_CFA_OFFSET(-8) + pop %rax + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rax) + CFI_RESTORE(%rbx) + CFI_RESTORE(%rcx) + CFI_RESTORE(%rdx) + CFI_RESTORE(%rsi) + CFI_RESTORE(%rdi) + CFI_RESTORE(%r8) + CFI_RESTORE(%r9) + CFI_RESTORE(%r10) + CFI_RESTORE(%r11) + ret + CFI_ENDPROC + +ASM_HIDDEN(__tsan_report_race) +.globl ASM_SYMBOL(__tsan_report_race_thunk) +ASM_SYMBOL(__tsan_report_race_thunk): + CFI_STARTPROC + _CET_ENDBR + # Save scratch registers. + push %rax + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rax, 0) + push %rcx + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rcx, 0) + push %rdx + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdx, 0) + push %rsi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rsi, 0) + push %rdi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdi, 0) + push %r8 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r8, 0) + push %r9 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r9, 0) + push %r10 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r10, 0) + push %r11 + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%r11, 0) + # All XMM registers are caller-saved. + sub $0x100, %rsp + CFI_ADJUST_CFA_OFFSET(0x100) + vmovdqu %xmm0, 0x0(%rsp) + vmovdqu %xmm1, 0x10(%rsp) + vmovdqu %xmm2, 0x20(%rsp) + vmovdqu %xmm3, 0x30(%rsp) + vmovdqu %xmm4, 0x40(%rsp) + vmovdqu %xmm5, 0x50(%rsp) + vmovdqu %xmm6, 0x60(%rsp) + vmovdqu %xmm7, 0x70(%rsp) + vmovdqu %xmm8, 0x80(%rsp) + vmovdqu %xmm9, 0x90(%rsp) + vmovdqu %xmm10, 0xa0(%rsp) + vmovdqu %xmm11, 0xb0(%rsp) + vmovdqu %xmm12, 0xc0(%rsp) + vmovdqu %xmm13, 0xd0(%rsp) + vmovdqu %xmm14, 0xe0(%rsp) + vmovdqu %xmm15, 0xf0(%rsp) + # Align stack frame. + push %rbx # non-scratch + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rbx, 0) + mov %rsp, %rbx # save current rsp + CFI_DEF_CFA_REGISTER(%rbx) + shr $4, %rsp # clear 4 lsb, align to 16 + shl $4, %rsp + + call ASM_SYMBOL(__tsan_report_race) + + # Unalign stack frame back. + mov %rbx, %rsp # restore the original rsp + CFI_DEF_CFA_REGISTER(%rsp) + pop %rbx + CFI_ADJUST_CFA_OFFSET(-8) + # Restore scratch registers. + vmovdqu 0x0(%rsp), %xmm0 + vmovdqu 0x10(%rsp), %xmm1 + vmovdqu 0x20(%rsp), %xmm2 + vmovdqu 0x30(%rsp), %xmm3 + vmovdqu 0x40(%rsp), %xmm4 + vmovdqu 0x50(%rsp), %xmm5 + vmovdqu 0x60(%rsp), %xmm6 + vmovdqu 0x70(%rsp), %xmm7 + vmovdqu 0x80(%rsp), %xmm8 + vmovdqu 0x90(%rsp), %xmm9 + vmovdqu 0xa0(%rsp), %xmm10 + vmovdqu 0xb0(%rsp), %xmm11 + vmovdqu 0xc0(%rsp), %xmm12 + vmovdqu 0xd0(%rsp), %xmm13 + vmovdqu 0xe0(%rsp), %xmm14 + vmovdqu 0xf0(%rsp), %xmm15 + add $0x100, %rsp + CFI_ADJUST_CFA_OFFSET(-0x100) + pop %r11 + CFI_ADJUST_CFA_OFFSET(-8) + pop %r10 + CFI_ADJUST_CFA_OFFSET(-8) + pop %r9 + CFI_ADJUST_CFA_OFFSET(-8) + pop %r8 + CFI_ADJUST_CFA_OFFSET(-8) + pop %rdi + CFI_ADJUST_CFA_OFFSET(-8) + pop %rsi + CFI_ADJUST_CFA_OFFSET(-8) + pop %rdx + CFI_ADJUST_CFA_OFFSET(-8) + pop %rcx + CFI_ADJUST_CFA_OFFSET(-8) + pop %rax + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rax) + CFI_RESTORE(%rbx) + CFI_RESTORE(%rcx) + CFI_RESTORE(%rdx) + CFI_RESTORE(%rsi) + CFI_RESTORE(%rdi) + CFI_RESTORE(%r8) + CFI_RESTORE(%r9) + CFI_RESTORE(%r10) + CFI_RESTORE(%r11) + ret + CFI_ENDPROC + +ASM_HIDDEN(__tsan_setjmp) +#if defined(__NetBSD__) +.comm _ZN14__interception15real___setjmp14E,8,8 +#elif !defined(__APPLE__) +.comm _ZN14__interception11real_setjmpE,8,8 +#endif +#if defined(__NetBSD__) +.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) +ASM_SYMBOL_INTERCEPTOR(__setjmp14): +#else +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): +#endif + CFI_STARTPROC + _CET_ENDBR + // save env parameter + push %rdi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdi, 0) + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` +#if defined(__FreeBSD__) || defined(__NetBSD__) + lea 8(%rsp), %rdi +#elif defined(__linux__) || defined(__APPLE__) + lea 16(%rsp), %rdi +#else +# error "Unknown platform" +#endif + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + // restore env parameter + pop %rdi + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rdi) + // tail jump to libc setjmp + movl $0, %eax +#if defined(__NetBSD__) + movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#elif !defined(__APPLE__) + movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#else + jmp ASM_SYMBOL(setjmp) +#endif + CFI_ENDPROC +#if defined(__NetBSD__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) +#else +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) +#endif + +.comm _ZN14__interception12real__setjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): + CFI_STARTPROC + _CET_ENDBR + // save env parameter + push %rdi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdi, 0) + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` +#if defined(__FreeBSD__) || defined(__NetBSD__) + lea 8(%rsp), %rdi +#elif defined(__linux__) || defined(__APPLE__) + lea 16(%rsp), %rdi +#else +# error "Unknown platform" +#endif + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + // restore env parameter + pop %rdi + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rdi) + // tail jump to libc setjmp + movl $0, %eax +#if !defined(__APPLE__) + movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#else + jmp ASM_SYMBOL(_setjmp) +#endif + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) + +#if defined(__NetBSD__) +.comm _ZN14__interception18real___sigsetjmp14E,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14): +#else +.comm _ZN14__interception14real_sigsetjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): +#endif + CFI_STARTPROC + _CET_ENDBR + // save env parameter + push %rdi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdi, 0) + // save savesigs parameter + push %rsi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rsi, 0) + // align stack frame + sub $8, %rsp + CFI_ADJUST_CFA_OFFSET(8) + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` +#if defined(__FreeBSD__) || defined(__NetBSD__) + lea 24(%rsp), %rdi +#elif defined(__linux__) || defined(__APPLE__) + lea 32(%rsp), %rdi +#else +# error "Unknown platform" +#endif + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + // unalign stack frame + add $8, %rsp + CFI_ADJUST_CFA_OFFSET(-8) + // restore savesigs parameter + pop %rsi + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rsi) + // restore env parameter + pop %rdi + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rdi) + // tail jump to libc sigsetjmp + movl $0, %eax +#if defined(__NetBSD__) + movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#elif !defined(__APPLE__) + movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx + jmp *(%rdx) +#else + jmp ASM_SYMBOL(sigsetjmp) +#endif + CFI_ENDPROC +#if defined(__NetBSD__) +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) +#else +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +#endif + +#if !defined(__APPLE__) && !defined(__NetBSD__) +.comm _ZN14__interception16real___sigsetjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): + CFI_STARTPROC + _CET_ENDBR + // save env parameter + push %rdi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rdi, 0) + // save savesigs parameter + push %rsi + CFI_ADJUST_CFA_OFFSET(8) + CFI_REL_OFFSET(%rsi, 0) + // align stack frame + sub $8, %rsp + CFI_ADJUST_CFA_OFFSET(8) + // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` +#if defined(__FreeBSD__) + lea 24(%rsp), %rdi +#else + lea 32(%rsp), %rdi +#endif + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + // unalign stack frame + add $8, %rsp + CFI_ADJUST_CFA_OFFSET(-8) + // restore savesigs parameter + pop %rsi + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rsi) + // restore env parameter + pop %rdi + CFI_ADJUST_CFA_OFFSET(-8) + CFI_RESTORE(%rdi) + // tail jump to libc sigsetjmp + movl $0, %eax + movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx + jmp *(%rdx) + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif // !defined(__APPLE__) && !defined(__NetBSD__) + +NO_EXEC_STACK_DIRECTIVE + +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S new file mode 100644 index 000000000000..d0f7a3f9af98 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S @@ -0,0 +1,214 @@ +.section .text +.set noreorder + +.hidden __tsan_setjmp +.comm _ZN14__interception11real_setjmpE,8,8 +.globl setjmp +.type setjmp, @function +setjmp: + + // save env parameters + daddiu $sp,$sp,-40 + sd $s0,32($sp) + sd $ra,24($sp) + sd $fp,16($sp) + sd $gp,8($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(setjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(setjmp))) + move $s0,$gp + + // save jmp_buf + sd $a0,0($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,40 + + // restore jmp_buf + ld $a0,0($sp) + + // restore gp + move $gp,$s0 + + // load pointer of libc setjmp to t9 + dla $t9,(_ZN14__interception11real_setjmpE) + + // restore env parameters + ld $gp,8($sp) + ld $fp,16($sp) + ld $ra,24($sp) + ld $s0,32($sp) + daddiu $sp,$sp,40 + + // tail jump to libc setjmp + ld $t9,0($t9) + jr $t9 + nop + +.size setjmp, .-setjmp + +.hidden __tsan_setjmp +.globl _setjmp +.comm _ZN14__interception12real__setjmpE,8,8 +.type _setjmp, @function +_setjmp: + + // Save env parameters + daddiu $sp,$sp,-40 + sd $s0,32($sp) + sd $ra,24($sp) + sd $fp,16($sp) + sd $gp,8($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(_setjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(_setjmp))) + move $s0,$gp + + // save jmp_buf + sd $a0,0($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,40 + + // restore jmp_buf + ld $a0,0($sp) + + // restore gp + move $gp,$s0 + + // load pointer of libc _setjmp to t9 + dla $t9,(_ZN14__interception12real__setjmpE) + + // restore env parameters + ld $gp,8($sp) + ld $fp,16($sp) + ld $ra,24($sp) + ld $s0,32($sp) + daddiu $sp,$sp,40 + + // tail jump to libc _setjmp + ld $t9,0($t9) + jr $t9 + nop + +.size _setjmp, .-_setjmp + +.hidden __tsan_setjmp +.globl sigsetjmp +.comm _ZN14__interception14real_sigsetjmpE,8,8 +.type sigsetjmp, @function +sigsetjmp: + + // Save env parameters + daddiu $sp,$sp,-48 + sd $s0,40($sp) + sd $ra,32($sp) + sd $fp,24($sp) + sd $gp,16($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(sigsetjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(sigsetjmp))) + move $s0,$gp + + // save jmp_buf and savesig + sd $a0,0($sp) + sd $a1,8($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,48 + + // restore jmp_buf and savesig + ld $a0,0($sp) + ld $a1,8($sp) + + // restore gp + move $gp,$s0 + + // load pointer of libc sigsetjmp to t9 + dla $t9,(_ZN14__interception14real_sigsetjmpE) + + // restore env parameters + ld $gp,16($sp) + ld $fp,24($sp) + ld $ra,32($sp) + ld $s0,40($sp) + daddiu $sp,$sp,48 + + // tail jump to libc sigsetjmp + ld $t9,0($t9) + jr $t9 + nop + +.size sigsetjmp, .-sigsetjmp + +.hidden __tsan_setjmp +.comm _ZN14__interception16real___sigsetjmpE,8,8 +.globl __sigsetjmp +.type __sigsetjmp, @function +__sigsetjmp: + + // Save env parameters + daddiu $sp,$sp,-48 + sd $s0,40($sp) + sd $ra,32($sp) + sd $fp,24($sp) + sd $gp,16($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(__sigsetjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(__sigsetjmp))) + move $s0,$gp + + // save jmp_buf and savesig + sd $a0,0($sp) + sd $a1,8($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,48 + + // restore jmp_buf and savesig + ld $a0,0($sp) + ld $a1,8($sp) + + // restore gp + move $gp,$s0 + + // load pointer to libc __sigsetjmp in t9 + dla $t9,(_ZN14__interception16real___sigsetjmpE) + + // restore env parameters + ld $gp,16($sp) + ld $fp,24($sp) + ld $ra,32($sp) + ld $s0,40($sp) + daddiu $sp,$sp,48 + + // tail jump to libc __sigsetjmp + ld $t9,0($t9) + jr $t9 + nop + +.size __sigsetjmp, .-__sigsetjmp diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp new file mode 100644 index 000000000000..7d6b41116aa6 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp @@ -0,0 +1,555 @@ +//===-- tsan_rtl_mutex.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include +#include + +#include "tsan_rtl.h" +#include "tsan_flags.h" +#include "tsan_sync.h" +#include "tsan_report.h" +#include "tsan_symbolize.h" +#include "tsan_platform.h" + +namespace __tsan { + +void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r); + +struct Callback final : public DDCallback { + ThreadState *thr; + uptr pc; + + Callback(ThreadState *thr, uptr pc) + : thr(thr) + , pc(pc) { + DDCallback::pt = thr->proc()->dd_pt; + DDCallback::lt = thr->dd_lt; + } + + StackID Unwind() override { return CurrentStackId(thr, pc); } + int UniqueTid() override { return thr->unique_id; } +}; + +void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) { + Callback cb(thr, pc); + ctx->dd->MutexInit(&cb, &s->dd); + s->dd.ctx = s->GetId(); +} + +static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, + uptr addr, u64 mid) { + // In Go, these misuses are either impossible, or detected by std lib, + // or false positives (e.g. unlock in a different thread). + if (SANITIZER_GO) + return; + if (!ShouldReport(thr, typ)) + return; + ThreadRegistryLock l(&ctx->thread_registry); + ScopedReport rep(typ); + rep.AddMutex(mid); + VarSizeStackTrace trace; + ObtainCurrentStack(thr, pc, &trace); + rep.AddStack(trace, true); + rep.AddLocation(addr, 1); + OutputReport(thr, rep); +} + +void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { + DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz); + if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) { + CHECK(!thr->is_freeing); + thr->is_freeing = true; + MemoryAccess(thr, pc, addr, 1, kAccessWrite); + thr->is_freeing = false; + } + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock l(&s->mtx); + s->SetFlags(flagz & MutexCreationFlagMask); + // Save stack in the case the sync object was created before as atomic. + if (!SANITIZER_GO && s->creation_stack_id == 0) + s->creation_stack_id = CurrentStackId(thr, pc); +} + +void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { + DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr); + bool unlock_locked = false; + u64 mid = 0; + u64 last_lock = 0; + { + SyncVar *s = ctx->metamap.GetSyncIfExists(addr); + if (s == 0) + return; + Lock l(&s->mtx); + if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) || + ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) { + // Destroy is no-op for linker-initialized mutexes. + return; + } + if (common_flags()->detect_deadlocks) { + Callback cb(thr, pc); + ctx->dd->MutexDestroy(&cb, &s->dd); + ctx->dd->MutexInit(&cb, &s->dd); + } + if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid && + !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + unlock_locked = true; + } + mid = s->GetId(); + last_lock = s->last_lock; + if (!unlock_locked) + s->Reset(thr->proc()); // must not reset it before the report is printed + } + if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) { + ThreadRegistryLock l(&ctx->thread_registry); + ScopedReport rep(ReportTypeMutexDestroyLocked); + rep.AddMutex(mid); + VarSizeStackTrace trace; + ObtainCurrentStack(thr, pc, &trace); + rep.AddStack(trace, true); + FastState last(last_lock); + RestoreStack(last.tid(), last.epoch(), &trace, 0); + rep.AddStack(trace, true); + rep.AddLocation(addr, 1); + OutputReport(thr, rep); + + SyncVar *s = ctx->metamap.GetSyncIfExists(addr); + if (s != 0) { + Lock l(&s->mtx); + s->Reset(thr->proc()); + } + } + thr->mset.Remove(mid); + // Imitate a memory write to catch unlock-destroy races. + // Do this outside of sync mutex, because it can report a race which locks + // sync mutexes. + if (IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessWrite | kAccessFree); + // s will be destroyed and freed in MetaMap::FreeBlock. +} + +void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { + DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz); + if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + { + ReadLock l(&s->mtx); + s->UpdateFlags(flagz); + if (s->owner_tid != thr->tid) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeLock(&cb, &s->dd, true); + } + } + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } +} + +void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) { + DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n", + thr->tid, addr, flagz, rec); + if (flagz & MutexFlagRecursiveLock) + CHECK_GT(rec, 0); + else + rec = 1; + if (IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); + u64 mid = 0; + bool pre_lock = false; + bool first = false; + bool report_double_lock = false; + { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock l(&s->mtx); + s->UpdateFlags(flagz); + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId()); + if (s->owner_tid == kInvalidTid) { + CHECK_EQ(s->recursion, 0); + s->owner_tid = thr->tid; + s->last_lock = thr->fast_state.raw(); + } else if (s->owner_tid == thr->tid) { + CHECK_GT(s->recursion, 0); + } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_double_lock = true; + } + first = s->recursion == 0; + s->recursion += rec; + if (first) { + AcquireImpl(thr, pc, &s->clock); + AcquireImpl(thr, pc, &s->read_clock); + } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) { + } + thr->mset.Add(s->GetId(), true, thr->fast_state.epoch()); + if (first && common_flags()->detect_deadlocks) { + pre_lock = + (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock); + Callback cb(thr, pc); + if (pre_lock) + ctx->dd->MutexBeforeLock(&cb, &s->dd, true); + ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock); + } + mid = s->GetId(); + } + if (report_double_lock) + ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid); + if (first && pre_lock && common_flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } +} + +int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { + DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz); + if (IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); + u64 mid = 0; + bool report_bad_unlock = false; + int rec = 0; + { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); + if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) { + if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_unlock = true; + } + } else { + rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1; + s->recursion -= rec; + if (s->recursion == 0) { + s->owner_tid = kInvalidTid; + ReleaseStoreImpl(thr, pc, &s->clock); + } else { + } + } + thr->mset.Del(s->GetId(), true); + if (common_flags()->detect_deadlocks && s->recursion == 0 && + !report_bad_unlock) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true); + } + mid = s->GetId(); + } + if (report_bad_unlock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); + if (common_flags()->detect_deadlocks && !report_bad_unlock) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } + return rec; +} + +void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { + DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz); + if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) { + { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + ReadLock l(&s->mtx); + s->UpdateFlags(flagz); + Callback cb(thr, pc); + ctx->dd->MutexBeforeLock(&cb, &s->dd, false); + } + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } +} + +void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { + DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz); + if (IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); + u64 mid = 0; + bool report_bad_lock = false; + bool pre_lock = false; + { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + ReadLock l(&s->mtx); + s->UpdateFlags(flagz); + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId()); + if (s->owner_tid != kInvalidTid) { + if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_lock = true; + } + } + AcquireImpl(thr, pc, &s->clock); + s->last_lock = thr->fast_state.raw(); + thr->mset.Add(s->GetId(), false, thr->fast_state.epoch()); + if (common_flags()->detect_deadlocks) { + pre_lock = + (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock); + Callback cb(thr, pc); + if (pre_lock) + ctx->dd->MutexBeforeLock(&cb, &s->dd, false); + ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock); + } + mid = s->GetId(); + } + if (report_bad_lock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid); + if (pre_lock && common_flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } +} + +void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr); + if (IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); + u64 mid = 0; + bool report_bad_unlock = false; + { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); + if (s->owner_tid != kInvalidTid) { + if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_unlock = true; + } + } + ReleaseImpl(thr, pc, &s->read_clock); + if (common_flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false); + } + mid = s->GetId(); + } + thr->mset.Del(mid, false); + if (report_bad_unlock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid); + if (common_flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } +} + +void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr); + if (IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); + u64 mid = 0; + bool report_bad_unlock = false; + { + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock l(&s->mtx); + bool write = true; + if (s->owner_tid == kInvalidTid) { + // Seems to be read unlock. + write = false; + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); + ReleaseImpl(thr, pc, &s->read_clock); + } else if (s->owner_tid == thr->tid) { + // Seems to be write unlock. + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); + CHECK_GT(s->recursion, 0); + s->recursion--; + if (s->recursion == 0) { + s->owner_tid = kInvalidTid; + ReleaseStoreImpl(thr, pc, &s->clock); + } else { + } + } else if (!s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_unlock = true; + } + thr->mset.Del(s->GetId(), write); + if (common_flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write); + } + mid = s->GetId(); + } + if (report_bad_unlock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); + if (common_flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } +} + +void MutexRepair(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr); + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock l(&s->mtx); + s->owner_tid = kInvalidTid; + s->recursion = 0; +} + +void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr); + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, s->GetId()); +} + +void Acquire(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: Acquire %zx\n", thr->tid, addr); + if (thr->ignore_sync) + return; + SyncVar *s = ctx->metamap.GetSyncIfExists(addr); + if (!s) + return; + ReadLock l(&s->mtx); + AcquireImpl(thr, pc, &s->clock); +} + +static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) { + ThreadState *thr = reinterpret_cast(arg); + ThreadContext *tctx = static_cast(tctx_base); + u64 epoch = tctx->epoch1; + if (tctx->status == ThreadStatusRunning) { + epoch = tctx->thr->fast_state.epoch(); + tctx->thr->clock.NoteGlobalAcquire(epoch); + } + thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); +} + +void AcquireGlobal(ThreadState *thr) { + DPrintf("#%d: AcquireGlobal\n", thr->tid); + if (thr->ignore_sync) + return; + ThreadRegistryLock l(&ctx->thread_registry); + ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateClockCallback, thr); +} + +void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr); + if (thr->ignore_sync) + return; + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + ReleaseStoreAcquireImpl(thr, pc, &s->clock); +} + +void Release(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: Release %zx\n", thr->tid, addr); + if (thr->ignore_sync) + return; + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + ReleaseImpl(thr, pc, &s->clock); +} + +void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr); + if (thr->ignore_sync) + return; + SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); + Lock l(&s->mtx); + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + ReleaseStoreImpl(thr, pc, &s->clock); +} + +#if !SANITIZER_GO +static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { + ThreadState *thr = reinterpret_cast(arg); + ThreadContext *tctx = static_cast(tctx_base); + u64 epoch = tctx->epoch1; + if (tctx->status == ThreadStatusRunning) + epoch = tctx->thr->fast_state.epoch(); + thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); +} + +void AfterSleep(ThreadState *thr, uptr pc) { + DPrintf("#%d: AfterSleep\n", thr->tid); + if (thr->ignore_sync) + return; + thr->last_sleep_stack_id = CurrentStackId(thr, pc); + ThreadRegistryLock l(&ctx->thread_registry); + ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateSleepClockCallback, + thr); +} +#endif + +void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { + if (thr->ignore_sync) + return; + thr->clock.set(thr->fast_state.epoch()); + thr->clock.acquire(&thr->proc()->clock_cache, c); +} + +void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { + if (thr->ignore_sync) + return; + thr->clock.set(thr->fast_state.epoch()); + thr->fast_synch_epoch = thr->fast_state.epoch(); + thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c); +} + +void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { + if (thr->ignore_sync) + return; + thr->clock.set(thr->fast_state.epoch()); + thr->fast_synch_epoch = thr->fast_state.epoch(); + thr->clock.release(&thr->proc()->clock_cache, c); +} + +void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { + if (thr->ignore_sync) + return; + thr->clock.set(thr->fast_state.epoch()); + thr->fast_synch_epoch = thr->fast_state.epoch(); + thr->clock.ReleaseStore(&thr->proc()->clock_cache, c); +} + +void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { + if (thr->ignore_sync) + return; + thr->clock.set(thr->fast_state.epoch()); + thr->fast_synch_epoch = thr->fast_state.epoch(); + thr->clock.acq_rel(&thr->proc()->clock_cache, c); +} + +void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { + if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock)) + return; + ThreadRegistryLock l(&ctx->thread_registry); + ScopedReport rep(ReportTypeDeadlock); + for (int i = 0; i < r->n; i++) { + rep.AddMutex(r->loop[i].mtx_ctx0); + rep.AddUniqueTid((int)r->loop[i].thr_ctx); + rep.AddThread((int)r->loop[i].thr_ctx); + } + uptr dummy_pc = 0x42; + for (int i = 0; i < r->n; i++) { + for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) { + u32 stk = r->loop[i].stk[j]; + if (stk && stk != 0xffffffff) { + rep.AddStack(StackDepotGet(stk), true); + } else { + // Sometimes we fail to extract the stack trace (FIXME: investigate), + // but we should still produce some stack trace in the report. + rep.AddStack(StackTrace(&dummy_pc, 1), true); + } + } + } + OutputReport(thr, rep); +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S new file mode 100644 index 000000000000..8285e21aa1ec --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S @@ -0,0 +1,288 @@ +#include "tsan_ppc_regs.h" + + .section .text + .hidden __tsan_setjmp + .globl _setjmp + .type _setjmp, @function + .align 4 +#if _CALL_ELF == 2 +_setjmp: +#else + .section ".opd","aw" + .align 3 +_setjmp: + .quad .L._setjmp,.TOC.@tocbase,0 + .previous +#endif +.L._setjmp: + mflr r0 + stdu r1,-48(r1) + std r2,24(r1) + std r3,32(r1) + std r0,40(r1) + // r3 is the original stack pointer. + addi r3,r1,48 + // r4 is the mangled stack pointer (see glibc) + ld r4,-28696(r13) + xor r4,r3,r4 + // Materialize a TOC in case we were called from libc. + // For big-endian, we load the TOC from the OPD. For little- + // endian, we use the .TOC. symbol to find it. + nop + bcl 20,31,0f +0: + mflr r2 +#if _CALL_ELF == 2 + addis r2,r2,.TOC.-0b@ha + addi r2,r2,.TOC.-0b@l +#else + addis r2,r2,_setjmp-0b@ha + addi r2,r2,_setjmp-0b@l + ld r2,8(r2) +#endif + // Call the interceptor. + bl __tsan_setjmp + nop + // Restore regs needed for setjmp. + ld r3,32(r1) + ld r0,40(r1) + // Emulate the real setjmp function. We do this because we can't + // perform a sibcall: The real setjmp function trashes the TOC + // pointer, and with a sibcall we have no way to restore it. + // This way we can make sure our caller's stack pointer and + // link register are saved correctly in the jmpbuf. + ld r6,-28696(r13) + addi r5,r1,48 // original stack ptr of caller + xor r5,r6,r5 + std r5,0(r3) // mangled stack ptr of caller + ld r5,24(r1) + std r5,8(r3) // caller's saved TOC pointer + xor r0,r6,r0 + std r0,16(r3) // caller's mangled return address + mfcr r0 + // Nonvolatiles. + std r14,24(r3) + stfd f14,176(r3) + stw r0,172(r3) // CR + std r15,32(r3) + stfd f15,184(r3) + std r16,40(r3) + stfd f16,192(r3) + std r17,48(r3) + stfd f17,200(r3) + std r18,56(r3) + stfd f18,208(r3) + std r19,64(r3) + stfd f19,216(r3) + std r20,72(r3) + stfd f20,224(r3) + std r21,80(r3) + stfd f21,232(r3) + std r22,88(r3) + stfd f22,240(r3) + std r23,96(r3) + stfd f23,248(r3) + std r24,104(r3) + stfd f24,256(r3) + std r25,112(r3) + stfd f25,264(r3) + std r26,120(r3) + stfd f26,272(r3) + std r27,128(r3) + stfd f27,280(r3) + std r28,136(r3) + stfd f28,288(r3) + std r29,144(r3) + stfd f29,296(r3) + std r30,152(r3) + stfd f30,304(r3) + std r31,160(r3) + stfd f31,312(r3) + addi r5,r3,320 + mfspr r0,256 + stw r0,168(r3) // VRSAVE + addi r6,r5,16 + stvx v20,0,r5 + addi r5,r5,32 + stvx v21,0,r6 + addi r6,r6,32 + stvx v22,0,r5 + addi r5,r5,32 + stvx v23,0,r6 + addi r6,r6,32 + stvx v24,0,r5 + addi r5,r5,32 + stvx v25,0,r6 + addi r6,r6,32 + stvx v26,0,r5 + addi r5,r5,32 + stvx v27,0,r6 + addi r6,r6,32 + stvx v28,0,r5 + addi r5,r5,32 + stvx v29,0,r6 + addi r6,r6,32 + stvx v30,0,r5 + stvx v31,0,r6 + // Clear the "mask-saved" slot. + li r4,0 + stw r4,512(r3) + // Restore TOC, LR, and stack and return to caller. + ld r2,24(r1) + ld r0,40(r1) + addi r1,r1,48 + li r3,0 // This is the setjmp return path + mtlr r0 + blr + .size _setjmp, .-.L._setjmp + + .globl setjmp + .type setjmp, @function + .align 4 +setjmp: + b _setjmp + .size setjmp, .-setjmp + + // sigsetjmp is like setjmp, except that the mask in r4 needs + // to be saved at offset 512 of the jump buffer. + .globl __sigsetjmp + .type __sigsetjmp, @function + .align 4 +#if _CALL_ELF == 2 +__sigsetjmp: +#else + .section ".opd","aw" + .align 3 +__sigsetjmp: + .quad .L.__sigsetjmp,.TOC.@tocbase,0 + .previous +#endif +.L.__sigsetjmp: + mflr r0 + stdu r1,-64(r1) + std r2,24(r1) + std r3,32(r1) + std r4,40(r1) + std r0,48(r1) + // r3 is the original stack pointer. + addi r3,r1,64 + // r4 is the mangled stack pointer (see glibc) + ld r4,-28696(r13) + xor r4,r3,r4 + // Materialize a TOC in case we were called from libc. + // For big-endian, we load the TOC from the OPD. For little- + // endian, we use the .TOC. symbol to find it. + nop + bcl 20,31,1f +1: + mflr r2 +#if _CALL_ELF == 2 + addis r2,r2,.TOC.-1b@ha + addi r2,r2,.TOC.-1b@l +#else + addis r2,r2,_setjmp-1b@ha + addi r2,r2,_setjmp-1b@l + ld r2,8(r2) +#endif + // Call the interceptor. + bl __tsan_setjmp + nop + // Restore regs needed for __sigsetjmp. + ld r3,32(r1) + ld r4,40(r1) + ld r0,48(r1) + // Emulate the real sigsetjmp function. We do this because we can't + // perform a sibcall: The real sigsetjmp function trashes the TOC + // pointer, and with a sibcall we have no way to restore it. + // This way we can make sure our caller's stack pointer and + // link register are saved correctly in the jmpbuf. + ld r6,-28696(r13) + addi r5,r1,64 // original stack ptr of caller + xor r5,r6,r5 + std r5,0(r3) // mangled stack ptr of caller + ld r5,24(r1) + std r5,8(r3) // caller's saved TOC pointer + xor r0,r6,r0 + std r0,16(r3) // caller's mangled return address + mfcr r0 + // Nonvolatiles. + std r14,24(r3) + stfd f14,176(r3) + stw r0,172(r3) // CR + std r15,32(r3) + stfd f15,184(r3) + std r16,40(r3) + stfd f16,192(r3) + std r17,48(r3) + stfd f17,200(r3) + std r18,56(r3) + stfd f18,208(r3) + std r19,64(r3) + stfd f19,216(r3) + std r20,72(r3) + stfd f20,224(r3) + std r21,80(r3) + stfd f21,232(r3) + std r22,88(r3) + stfd f22,240(r3) + std r23,96(r3) + stfd f23,248(r3) + std r24,104(r3) + stfd f24,256(r3) + std r25,112(r3) + stfd f25,264(r3) + std r26,120(r3) + stfd f26,272(r3) + std r27,128(r3) + stfd f27,280(r3) + std r28,136(r3) + stfd f28,288(r3) + std r29,144(r3) + stfd f29,296(r3) + std r30,152(r3) + stfd f30,304(r3) + std r31,160(r3) + stfd f31,312(r3) + addi r5,r3,320 + mfspr r0,256 + stw r0,168(r3) // VRSAVE + addi r6,r5,16 + stvx v20,0,r5 + addi r5,r5,32 + stvx v21,0,r6 + addi r6,r6,32 + stvx v22,0,r5 + addi r5,r5,32 + stvx v23,0,r6 + addi r6,r6,32 + stvx v24,0,r5 + addi r5,r5,32 + stvx v25,0,r6 + addi r6,r6,32 + stvx v26,0,r5 + addi r5,r5,32 + stvx v27,0,r6 + addi r6,r6,32 + stvx v28,0,r5 + addi r5,r5,32 + stvx v29,0,r6 + addi r6,r6,32 + stvx v30,0,r5 + stvx v31,0,r6 + // Save into the "mask-saved" slot. + stw r4,512(r3) + // Restore TOC, LR, and stack and return to caller. + ld r2,24(r1) + ld r0,48(r1) + addi r1,r1,64 + li r3,0 // This is the sigsetjmp return path + mtlr r0 + blr + .size __sigsetjmp, .-.L.__sigsetjmp + + .globl sigsetjmp + .type sigsetjmp, @function + .align 4 +sigsetjmp: + b __sigsetjmp + .size sigsetjmp, .-sigsetjmp diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp new file mode 100644 index 000000000000..def61cca14d5 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp @@ -0,0 +1,60 @@ +//===-- tsan_rtl_proc.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" +#include "tsan_flags.h" + +namespace __tsan { + +Processor *ProcCreate() { + void *mem = InternalAlloc(sizeof(Processor)); + internal_memset(mem, 0, sizeof(Processor)); + Processor *proc = new(mem) Processor; + proc->thr = nullptr; +#if !SANITIZER_GO + AllocatorProcStart(proc); +#endif + if (common_flags()->detect_deadlocks) + proc->dd_pt = ctx->dd->CreatePhysicalThread(); + return proc; +} + +void ProcDestroy(Processor *proc) { + CHECK_EQ(proc->thr, nullptr); +#if !SANITIZER_GO + AllocatorProcFinish(proc); +#endif + ctx->clock_alloc.FlushCache(&proc->clock_cache); + ctx->metamap.OnProcIdle(proc); + if (common_flags()->detect_deadlocks) + ctx->dd->DestroyPhysicalThread(proc->dd_pt); + proc->~Processor(); + InternalFree(proc); +} + +void ProcWire(Processor *proc, ThreadState *thr) { + CHECK_EQ(thr->proc1, nullptr); + CHECK_EQ(proc->thr, nullptr); + thr->proc1 = proc; + proc->thr = thr; +} + +void ProcUnwire(Processor *proc, ThreadState *thr) { + CHECK_EQ(thr->proc1, proc); + CHECK_EQ(proc->thr, thr); + thr->proc1 = nullptr; + proc->thr = nullptr; +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp new file mode 100644 index 000000000000..f332a6a8d1d8 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp @@ -0,0 +1,984 @@ +//===-- tsan_rtl_report.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "tsan_platform.h" +#include "tsan_rtl.h" +#include "tsan_suppressions.h" +#include "tsan_symbolize.h" +#include "tsan_report.h" +#include "tsan_sync.h" +#include "tsan_mman.h" +#include "tsan_flags.h" +#include "tsan_fd.h" + +namespace __tsan { + +using namespace __sanitizer; + +static ReportStack *SymbolizeStack(StackTrace trace); + +// Can be overriden by an application/test to intercept reports. +#ifdef TSAN_EXTERNAL_HOOKS +bool OnReport(const ReportDesc *rep, bool suppressed); +#else +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool OnReport(const ReportDesc *rep, bool suppressed) { + (void)rep; + return suppressed; +} +#endif + +SANITIZER_WEAK_DEFAULT_IMPL +void __tsan_on_report(const ReportDesc *rep) { + (void)rep; +} + +static void StackStripMain(SymbolizedStack *frames) { + SymbolizedStack *last_frame = nullptr; + SymbolizedStack *last_frame2 = nullptr; + for (SymbolizedStack *cur = frames; cur; cur = cur->next) { + last_frame2 = last_frame; + last_frame = cur; + } + + if (last_frame2 == 0) + return; +#if !SANITIZER_GO + const char *last = last_frame->info.function; + const char *last2 = last_frame2->info.function; + // Strip frame above 'main' + if (last2 && 0 == internal_strcmp(last2, "main")) { + last_frame->ClearAll(); + last_frame2->next = nullptr; + // Strip our internal thread start routine. + } else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) { + last_frame->ClearAll(); + last_frame2->next = nullptr; + // Strip global ctors init, .preinit_array and main caller. + } else if (last && (0 == internal_strcmp(last, "__do_global_ctors_aux") || + 0 == internal_strcmp(last, "__libc_csu_init") || + 0 == internal_strcmp(last, "__libc_start_main"))) { + last_frame->ClearAll(); + last_frame2->next = nullptr; + // If both are 0, then we probably just failed to symbolize. + } else if (last || last2) { + // Ensure that we recovered stack completely. Trimmed stack + // can actually happen if we do not instrument some code, + // so it's only a debug print. However we must try hard to not miss it + // due to our fault. + DPrintf("Bottom stack frame is missed\n"); + } +#else + // The last frame always point into runtime (gosched0, goexit0, runtime.main). + last_frame->ClearAll(); + last_frame2->next = nullptr; +#endif +} + +ReportStack *SymbolizeStackId(u32 stack_id) { + if (stack_id == 0) + return 0; + StackTrace stack = StackDepotGet(stack_id); + if (stack.trace == nullptr) + return nullptr; + return SymbolizeStack(stack); +} + +static ReportStack *SymbolizeStack(StackTrace trace) { + if (trace.size == 0) + return 0; + SymbolizedStack *top = nullptr; + for (uptr si = 0; si < trace.size; si++) { + const uptr pc = trace.trace[si]; + uptr pc1 = pc; + // We obtain the return address, but we're interested in the previous + // instruction. + if ((pc & kExternalPCBit) == 0) + pc1 = StackTrace::GetPreviousInstructionPc(pc); + SymbolizedStack *ent = SymbolizeCode(pc1); + CHECK_NE(ent, 0); + SymbolizedStack *last = ent; + while (last->next) { + last->info.address = pc; // restore original pc for report + last = last->next; + } + last->info.address = pc; // restore original pc for report + last->next = top; + top = ent; + } + StackStripMain(top); + + auto *stack = New(); + stack->frames = top; + return stack; +} + +bool ShouldReport(ThreadState *thr, ReportType typ) { + // We set thr->suppress_reports in the fork context. + // Taking any locking in the fork context can lead to deadlocks. + // If any locks are already taken, it's too late to do this check. + CheckedMutex::CheckNoLocks(); + // For the same reason check we didn't lock thread_registry yet. + if (SANITIZER_DEBUG) + ThreadRegistryLock l(&ctx->thread_registry); + if (!flags()->report_bugs || thr->suppress_reports) + return false; + switch (typ) { + case ReportTypeSignalUnsafe: + return flags()->report_signal_unsafe; + case ReportTypeThreadLeak: +#if !SANITIZER_GO + // It's impossible to join phantom threads + // in the child after fork. + if (ctx->after_multithreaded_fork) + return false; +#endif + return flags()->report_thread_leaks; + case ReportTypeMutexDestroyLocked: + return flags()->report_destroy_locked; + default: + return true; + } +} + +ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) { + ctx->thread_registry.CheckLocked(); + rep_ = New(); + rep_->typ = typ; + rep_->tag = tag; + ctx->report_mtx.Lock(); +} + +ScopedReportBase::~ScopedReportBase() { + ctx->report_mtx.Unlock(); + DestroyAndFree(rep_); +} + +void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) { + ReportStack **rs = rep_->stacks.PushBack(); + *rs = SymbolizeStack(stack); + (*rs)->suppressable = suppressable; +} + +void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, + StackTrace stack, const MutexSet *mset) { + auto *mop = New(); + rep_->mops.PushBack(mop); + mop->tid = s.tid(); + mop->addr = addr + s.addr0(); + mop->size = s.size(); + mop->write = s.IsWrite(); + mop->atomic = s.IsAtomic(); + mop->stack = SymbolizeStack(stack); + mop->external_tag = external_tag; + if (mop->stack) + mop->stack->suppressable = true; + for (uptr i = 0; i < mset->Size(); i++) { + MutexSet::Desc d = mset->Get(i); + u64 mid = this->AddMutex(d.id); + ReportMopMutex mtx = {mid, d.write}; + mop->mset.PushBack(mtx); + } +} + +void ScopedReportBase::AddUniqueTid(Tid unique_tid) { + rep_->unique_tids.PushBack(unique_tid); +} + +void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { + for (uptr i = 0; i < rep_->threads.Size(); i++) { + if ((u32)rep_->threads[i]->id == tctx->tid) + return; + } + auto *rt = New(); + rep_->threads.PushBack(rt); + rt->id = tctx->tid; + rt->os_id = tctx->os_id; + rt->running = (tctx->status == ThreadStatusRunning); + rt->name = internal_strdup(tctx->name); + rt->parent_tid = tctx->parent_tid; + rt->thread_type = tctx->thread_type; + rt->stack = 0; + rt->stack = SymbolizeStackId(tctx->creation_stack_id); + if (rt->stack) + rt->stack->suppressable = suppressable; +} + +#if !SANITIZER_GO +static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) { + int unique_id = *(int *)arg; + return tctx->unique_id == (u32)unique_id; +} + +static ThreadContext *FindThreadByUidLocked(Tid unique_id) { + ctx->thread_registry.CheckLocked(); + return static_cast( + ctx->thread_registry.FindThreadContextLocked( + FindThreadByUidLockedCallback, &unique_id)); +} + +static ThreadContext *FindThreadByTidLocked(Tid tid) { + ctx->thread_registry.CheckLocked(); + return static_cast( + ctx->thread_registry.GetThreadLocked(tid)); +} + +static bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) { + uptr addr = (uptr)arg; + ThreadContext *tctx = static_cast(tctx_base); + if (tctx->status != ThreadStatusRunning) + return false; + ThreadState *thr = tctx->thr; + CHECK(thr); + return ((addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size) || + (addr >= thr->tls_addr && addr < thr->tls_addr + thr->tls_size)); +} + +ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { + ctx->thread_registry.CheckLocked(); + ThreadContext *tctx = + static_cast(ctx->thread_registry.FindThreadContextLocked( + IsInStackOrTls, (void *)addr)); + if (!tctx) + return 0; + ThreadState *thr = tctx->thr; + CHECK(thr); + *is_stack = (addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size); + return tctx; +} +#endif + +void ScopedReportBase::AddThread(Tid unique_tid, bool suppressable) { +#if !SANITIZER_GO + if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid)) + AddThread(tctx, suppressable); +#endif +} + +void ScopedReportBase::AddMutex(const SyncVar *s) { + for (uptr i = 0; i < rep_->mutexes.Size(); i++) { + if (rep_->mutexes[i]->id == s->uid) + return; + } + auto *rm = New(); + rep_->mutexes.PushBack(rm); + rm->id = s->uid; + rm->addr = s->addr; + rm->destroyed = false; + rm->stack = SymbolizeStackId(s->creation_stack_id); +} + +u64 ScopedReportBase::AddMutex(u64 id) { + u64 uid = 0; + u64 mid = id; + uptr addr = SyncVar::SplitId(id, &uid); + SyncVar *s = ctx->metamap.GetSyncIfExists(addr); + // Check that the mutex is still alive. + // Another mutex can be created at the same address, + // so check uid as well. + if (s && s->CheckId(uid)) { + Lock l(&s->mtx); + mid = s->uid; + AddMutex(s); + } else { + AddDeadMutex(id); + } + return mid; +} + +void ScopedReportBase::AddDeadMutex(u64 id) { + for (uptr i = 0; i < rep_->mutexes.Size(); i++) { + if (rep_->mutexes[i]->id == id) + return; + } + auto *rm = New(); + rep_->mutexes.PushBack(rm); + rm->id = id; + rm->addr = 0; + rm->destroyed = true; + rm->stack = 0; +} + +void ScopedReportBase::AddLocation(uptr addr, uptr size) { + if (addr == 0) + return; +#if !SANITIZER_GO + int fd = -1; + Tid creat_tid = kInvalidTid; + StackID creat_stack = 0; + if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) { + auto *loc = New(); + loc->type = ReportLocationFD; + loc->fd = fd; + loc->tid = creat_tid; + loc->stack = SymbolizeStackId(creat_stack); + rep_->locs.PushBack(loc); + ThreadContext *tctx = FindThreadByUidLocked(creat_tid); + if (tctx) + AddThread(tctx); + return; + } + MBlock *b = 0; + uptr block_begin = 0; + Allocator *a = allocator(); + if (a->PointerIsMine((void*)addr)) { + block_begin = (uptr)a->GetBlockBegin((void *)addr); + if (block_begin) + b = ctx->metamap.GetBlock(block_begin); + } + if (!b) + b = JavaHeapBlock(addr, &block_begin); + if (b != 0) { + ThreadContext *tctx = FindThreadByTidLocked(b->tid); + auto *loc = New(); + loc->type = ReportLocationHeap; + loc->heap_chunk_start = block_begin; + loc->heap_chunk_size = b->siz; + loc->external_tag = b->tag; + loc->tid = tctx ? tctx->tid : b->tid; + loc->stack = SymbolizeStackId(b->stk); + rep_->locs.PushBack(loc); + if (tctx) + AddThread(tctx); + return; + } + bool is_stack = false; + if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) { + auto *loc = New(); + loc->type = is_stack ? ReportLocationStack : ReportLocationTLS; + loc->tid = tctx->tid; + rep_->locs.PushBack(loc); + AddThread(tctx); + } +#endif + if (ReportLocation *loc = SymbolizeData(addr)) { + loc->suppressable = true; + rep_->locs.PushBack(loc); + return; + } +} + +#if !SANITIZER_GO +void ScopedReportBase::AddSleep(StackID stack_id) { + rep_->sleep = SymbolizeStackId(stack_id); +} +#endif + +void ScopedReportBase::SetCount(int count) { rep_->count = count; } + +const ReportDesc *ScopedReportBase::GetReport() const { return rep_; } + +ScopedReport::ScopedReport(ReportType typ, uptr tag) + : ScopedReportBase(typ, tag) {} + +ScopedReport::~ScopedReport() {} + +void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk, + MutexSet *mset, uptr *tag) { + // This function restores stack trace and mutex set for the thread/epoch. + // It does so by getting stack trace and mutex set at the beginning of + // trace part, and then replaying the trace till the given epoch. + Trace* trace = ThreadTrace(tid); + ReadLock l(&trace->mtx); + const int partidx = (epoch / kTracePartSize) % TraceParts(); + TraceHeader* hdr = &trace->headers[partidx]; + if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize) + return; + CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0); + const u64 epoch0 = RoundDown(epoch, TraceSize()); + const u64 eend = epoch % TraceSize(); + const u64 ebegin = RoundDown(eend, kTracePartSize); + DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n", + tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx); + Vector stack; + stack.Resize(hdr->stack0.size + 64); + for (uptr i = 0; i < hdr->stack0.size; i++) { + stack[i] = hdr->stack0.trace[i]; + DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]); + } + if (mset) + *mset = hdr->mset0; + uptr pos = hdr->stack0.size; + Event *events = (Event*)GetThreadTrace(tid); + for (uptr i = ebegin; i <= eend; i++) { + Event ev = events[i]; + EventType typ = (EventType)(ev >> kEventPCBits); + uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1)); + DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc); + if (typ == EventTypeMop) { + stack[pos] = pc; + } else if (typ == EventTypeFuncEnter) { + if (stack.Size() < pos + 2) + stack.Resize(pos + 2); + stack[pos++] = pc; + } else if (typ == EventTypeFuncExit) { + if (pos > 0) + pos--; + } + if (mset) { + if (typ == EventTypeLock) { + mset->Add(pc, true, epoch0 + i); + } else if (typ == EventTypeUnlock) { + mset->Del(pc, true); + } else if (typ == EventTypeRLock) { + mset->Add(pc, false, epoch0 + i); + } else if (typ == EventTypeRUnlock) { + mset->Del(pc, false); + } + } + for (uptr j = 0; j <= pos; j++) + DPrintf2(" #%zu: %zx\n", j, stack[j]); + } + if (pos == 0 && stack[0] == 0) + return; + pos++; + stk->Init(&stack[0], pos); + ExtractTagFromStack(stk, tag); +} + +namespace v3 { + +// Replays the trace up to last_pos position in the last part +// or up to the provided epoch/sid (whichever is earlier) +// and calls the provided function f for each event. +template +void TraceReplay(Trace *trace, TracePart *last, Event *last_pos, Sid sid, + Epoch epoch, Func f) { + TracePart *part = trace->parts.Front(); + Sid ev_sid = kFreeSid; + Epoch ev_epoch = kEpochOver; + for (;;) { + DCHECK_EQ(part->trace, trace); + // Note: an event can't start in the last element. + // Since an event can take up to 2 elements, + // we ensure we have at least 2 before adding an event. + Event *end = &part->events[TracePart::kSize - 1]; + if (part == last) + end = last_pos; + for (Event *evp = &part->events[0]; evp < end; evp++) { + Event *evp0 = evp; + if (!evp->is_access && !evp->is_func) { + switch (evp->type) { + case EventType::kTime: { + auto *ev = reinterpret_cast(evp); + ev_sid = static_cast(ev->sid); + ev_epoch = static_cast(ev->epoch); + if (ev_sid == sid && ev_epoch > epoch) + return; + break; + } + case EventType::kAccessExt: + FALLTHROUGH; + case EventType::kAccessRange: + FALLTHROUGH; + case EventType::kLock: + FALLTHROUGH; + case EventType::kRLock: + // These take 2 Event elements. + evp++; + break; + case EventType::kUnlock: + // This takes 1 Event element. + break; + } + } + CHECK_NE(ev_sid, kFreeSid); + CHECK_NE(ev_epoch, kEpochOver); + f(ev_sid, ev_epoch, evp0); + } + if (part == last) + return; + part = trace->parts.Next(part); + CHECK(part); + } + CHECK(0); +} + +static void RestoreStackMatch(VarSizeStackTrace *pstk, MutexSet *pmset, + Vector *stack, MutexSet *mset, uptr pc, + bool *found) { + DPrintf2(" MATCHED\n"); + *pmset = *mset; + stack->PushBack(pc); + pstk->Init(&(*stack)[0], stack->Size()); + stack->PopBack(); + *found = true; +} + +// Checks if addr1|size1 is fully contained in addr2|size2. +// We check for fully contained instread of just overlapping +// because a memory access is always traced once, but can be +// split into multiple accesses in the shadow. +static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2, + uptr size2) { + return addr1 >= addr2 && addr1 + size1 <= addr2 + size2; +} + +// Replays the trace of thread tid up to the target event identified +// by sid/epoch/addr/size/typ and restores and returns stack, mutex set +// and tag for that event. If there are multiple such events, it returns +// the last one. Returns false if the event is not present in the trace. +bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, + uptr size, AccessType typ, VarSizeStackTrace *pstk, + MutexSet *pmset, uptr *ptag) { + // This function restores stack trace and mutex set for the thread/epoch. + // It does so by getting stack trace and mutex set at the beginning of + // trace part, and then replaying the trace till the given epoch. + DPrintf2("RestoreStack: tid=%u sid=%u@%u addr=0x%zx/%zu typ=%x\n", tid, + static_cast(sid), static_cast(epoch), addr, size, + static_cast(typ)); + ctx->slot_mtx.CheckLocked(); // needed to prevent trace part recycling + ctx->thread_registry.CheckLocked(); + ThreadContext *tctx = + static_cast(ctx->thread_registry.GetThreadLocked(tid)); + Trace *trace = &tctx->trace; + // Snapshot first/last parts and the current position in the last part. + TracePart *first_part; + TracePart *last_part; + Event *last_pos; + { + Lock lock(&trace->mtx); + first_part = trace->parts.Front(); + if (!first_part) + return false; + last_part = trace->parts.Back(); + last_pos = trace->final_pos; + if (tctx->thr) + last_pos = (Event *)atomic_load_relaxed(&tctx->thr->trace_pos); + } + DynamicMutexSet mset; + Vector stack; + uptr prev_pc = 0; + bool found = false; + bool is_read = typ & kAccessRead; + bool is_atomic = typ & kAccessAtomic; + bool is_free = typ & kAccessFree; + TraceReplay( + trace, last_part, last_pos, sid, epoch, + [&](Sid ev_sid, Epoch ev_epoch, Event *evp) { + bool match = ev_sid == sid && ev_epoch == epoch; + if (evp->is_access) { + if (evp->is_func == 0 && evp->type == EventType::kAccessExt && + evp->_ == 0) // NopEvent + return; + auto *ev = reinterpret_cast(evp); + uptr ev_addr = RestoreAddr(ev->addr); + uptr ev_size = 1 << ev->size_log; + uptr ev_pc = + prev_pc + ev->pc_delta - (1 << (EventAccess::kPCBits - 1)); + prev_pc = ev_pc; + DPrintf2(" Access: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc, + ev_addr, ev_size, ev->is_read, ev->is_atomic); + if (match && type == EventType::kAccessExt && + IsWithinAccess(addr, size, ev_addr, ev_size) && + is_read == ev->is_read && is_atomic == ev->is_atomic && !is_free) + RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found); + return; + } + if (evp->is_func) { + auto *ev = reinterpret_cast(evp); + if (ev->pc) { + DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc); + stack.PushBack(ev->pc); + } else { + DPrintf2(" FuncExit\n"); + CHECK(stack.Size()); + stack.PopBack(); + } + return; + } + switch (evp->type) { + case EventType::kAccessExt: { + auto *ev = reinterpret_cast(evp); + uptr ev_addr = RestoreAddr(ev->addr); + uptr ev_size = 1 << ev->size_log; + prev_pc = ev->pc; + DPrintf2(" AccessExt: pc=0x%llx addr=0x%zx/%zu type=%u/%u\n", + ev->pc, ev_addr, ev_size, ev->is_read, ev->is_atomic); + if (match && type == EventType::kAccessExt && + IsWithinAccess(addr, size, ev_addr, ev_size) && + is_read == ev->is_read && is_atomic == ev->is_atomic && + !is_free) + RestoreStackMatch(pstk, pmset, &stack, mset, ev->pc, &found); + break; + } + case EventType::kAccessRange: { + auto *ev = reinterpret_cast(evp); + uptr ev_addr = RestoreAddr(ev->addr); + uptr ev_size = + (ev->size_hi << EventAccessRange::kSizeLoBits) + ev->size_lo; + uptr ev_pc = RestoreAddr(ev->pc); + prev_pc = ev_pc; + DPrintf2(" Range: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc, + ev_addr, ev_size, ev->is_read, ev->is_free); + if (match && type == EventType::kAccessExt && + IsWithinAccess(addr, size, ev_addr, ev_size) && + is_read == ev->is_read && !is_atomic && is_free == ev->is_free) + RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found); + break; + } + case EventType::kLock: + FALLTHROUGH; + case EventType::kRLock: { + auto *ev = reinterpret_cast(evp); + bool is_write = ev->type == EventType::kLock; + uptr ev_addr = RestoreAddr(ev->addr); + uptr ev_pc = RestoreAddr(ev->pc); + StackID stack_id = + (ev->stack_hi << EventLock::kStackIDLoBits) + ev->stack_lo; + DPrintf2(" Lock: pc=0x%zx addr=0x%zx stack=%u write=%d\n", ev_pc, + ev_addr, stack_id, is_write); + mset->AddAddr(ev_addr, stack_id, is_write); + // Events with ev_pc == 0 are written to the beginning of trace + // part as initial mutex set (are not real). + if (match && type == EventType::kLock && addr == ev_addr && ev_pc) + RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found); + break; + } + case EventType::kUnlock: { + auto *ev = reinterpret_cast(evp); + uptr ev_addr = RestoreAddr(ev->addr); + DPrintf2(" Unlock: addr=0x%zx\n", ev_addr); + mset->DelAddr(ev_addr); + break; + } + case EventType::kTime: + // TraceReplay already extracted sid/epoch from it, + // nothing else to do here. + break; + } + }); + ExtractTagFromStack(pstk, ptag); + return found; +} + +} // namespace v3 + +bool RacyStacks::operator==(const RacyStacks &other) const { + if (hash[0] == other.hash[0] && hash[1] == other.hash[1]) + return true; + if (hash[0] == other.hash[1] && hash[1] == other.hash[0]) + return true; + return false; +} + +static bool FindRacyStacks(const RacyStacks &hash) { + for (uptr i = 0; i < ctx->racy_stacks.Size(); i++) { + if (hash == ctx->racy_stacks[i]) { + VPrintf(2, "ThreadSanitizer: suppressing report as doubled (stack)\n"); + return true; + } + } + return false; +} + +static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) { + if (!flags()->suppress_equal_stacks) + return false; + RacyStacks hash; + hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr)); + hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr)); + { + ReadLock lock(&ctx->racy_mtx); + if (FindRacyStacks(hash)) + return true; + } + Lock lock(&ctx->racy_mtx); + if (FindRacyStacks(hash)) + return true; + ctx->racy_stacks.PushBack(hash); + return false; +} + +static bool FindRacyAddress(const RacyAddress &ra0) { + for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) { + RacyAddress ra2 = ctx->racy_addresses[i]; + uptr maxbeg = max(ra0.addr_min, ra2.addr_min); + uptr minend = min(ra0.addr_max, ra2.addr_max); + if (maxbeg < minend) { + VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n"); + return true; + } + } + return false; +} + +static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) { + if (!flags()->suppress_equal_addresses) + return false; + RacyAddress ra0 = {addr_min, addr_max}; + { + ReadLock lock(&ctx->racy_mtx); + if (FindRacyAddress(ra0)) + return true; + } + Lock lock(&ctx->racy_mtx); + if (FindRacyAddress(ra0)) + return true; + ctx->racy_addresses.PushBack(ra0); + return false; +} + +bool OutputReport(ThreadState *thr, const ScopedReport &srep) { + // These should have been checked in ShouldReport. + // It's too late to check them here, we have already taken locks. + CHECK(flags()->report_bugs); + CHECK(!thr->suppress_reports); + atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime()); + const ReportDesc *rep = srep.GetReport(); + CHECK_EQ(thr->current_report, nullptr); + thr->current_report = rep; + Suppression *supp = 0; + uptr pc_or_addr = 0; + for (uptr i = 0; pc_or_addr == 0 && i < rep->mops.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->mops[i]->stack, &supp); + for (uptr i = 0; pc_or_addr == 0 && i < rep->stacks.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->stacks[i], &supp); + for (uptr i = 0; pc_or_addr == 0 && i < rep->threads.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->threads[i]->stack, &supp); + for (uptr i = 0; pc_or_addr == 0 && i < rep->locs.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->locs[i], &supp); + if (pc_or_addr != 0) { + Lock lock(&ctx->fired_suppressions_mtx); + FiredSuppression s = {srep.GetReport()->typ, pc_or_addr, supp}; + ctx->fired_suppressions.push_back(s); + } + { + bool old_is_freeing = thr->is_freeing; + thr->is_freeing = false; + bool suppressed = OnReport(rep, pc_or_addr != 0); + thr->is_freeing = old_is_freeing; + if (suppressed) { + thr->current_report = nullptr; + return false; + } + } + PrintReport(rep); + __tsan_on_report(rep); + ctx->nreported++; + if (flags()->halt_on_error) + Die(); + thr->current_report = nullptr; + return true; +} + +bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace) { + ReadLock lock(&ctx->fired_suppressions_mtx); + for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) { + if (ctx->fired_suppressions[k].type != type) + continue; + for (uptr j = 0; j < trace.size; j++) { + FiredSuppression *s = &ctx->fired_suppressions[k]; + if (trace.trace[j] == s->pc_or_addr) { + if (s->supp) + atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed); + return true; + } + } + } + return false; +} + +static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) { + ReadLock lock(&ctx->fired_suppressions_mtx); + for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) { + if (ctx->fired_suppressions[k].type != type) + continue; + FiredSuppression *s = &ctx->fired_suppressions[k]; + if (addr == s->pc_or_addr) { + if (s->supp) + atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed); + return true; + } + } + return false; +} + +static bool RaceBetweenAtomicAndFree(ThreadState *thr) { + Shadow s0(thr->racy_state[0]); + Shadow s1(thr->racy_state[1]); + CHECK(!(s0.IsAtomic() && s1.IsAtomic())); + if (!s0.IsAtomic() && !s1.IsAtomic()) + return true; + if (s0.IsAtomic() && s1.IsFreed()) + return true; + if (s1.IsAtomic() && thr->is_freeing) + return true; + return false; +} + +void ReportRace(ThreadState *thr) { + CheckedMutex::CheckNoLocks(); + + // Symbolizer makes lots of intercepted calls. If we try to process them, + // at best it will cause deadlocks on internal mutexes. + ScopedIgnoreInterceptors ignore; + + if (!ShouldReport(thr, ReportTypeRace)) + return; + if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr)) + return; + + bool freed = false; + { + Shadow s(thr->racy_state[1]); + freed = s.GetFreedAndReset(); + thr->racy_state[1] = s.raw(); + } + + uptr addr = ShadowToMem(thr->racy_shadow_addr); + uptr addr_min = 0; + uptr addr_max = 0; + { + uptr a0 = addr + Shadow(thr->racy_state[0]).addr0(); + uptr a1 = addr + Shadow(thr->racy_state[1]).addr0(); + uptr e0 = a0 + Shadow(thr->racy_state[0]).size(); + uptr e1 = a1 + Shadow(thr->racy_state[1]).size(); + addr_min = min(a0, a1); + addr_max = max(e0, e1); + if (IsExpectedReport(addr_min, addr_max - addr_min)) + return; + } + if (HandleRacyAddress(thr, addr_min, addr_max)) + return; + + ReportType typ = ReportTypeRace; + if (thr->is_vptr_access && freed) + typ = ReportTypeVptrUseAfterFree; + else if (thr->is_vptr_access) + typ = ReportTypeVptrRace; + else if (freed) + typ = ReportTypeUseAfterFree; + + if (IsFiredSuppression(ctx, typ, addr)) + return; + + const uptr kMop = 2; + VarSizeStackTrace traces[kMop]; + uptr tags[kMop] = {kExternalTagNone}; + uptr toppc = TraceTopPC(thr); + if (toppc >> kEventPCBits) { + // This is a work-around for a known issue. + // The scenario where this happens is rather elaborate and requires + // an instrumented __sanitizer_report_error_summary callback and + // a __tsan_symbolize_external callback and a race during a range memory + // access larger than 8 bytes. MemoryAccessRange adds the current PC to + // the trace and starts processing memory accesses. A first memory access + // triggers a race, we report it and call the instrumented + // __sanitizer_report_error_summary, which adds more stuff to the trace + // since it is intrumented. Then a second memory access in MemoryAccessRange + // also triggers a race and we get here and call TraceTopPC to get the + // current PC, however now it contains some unrelated events from the + // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit + // event. Later we subtract -1 from it (in GetPreviousInstructionPc) + // and the resulting PC has kExternalPCBit set, so we pass it to + // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its + // rights to crash since the PC is completely bogus. + // test/tsan/double_race.cpp contains a test case for this. + toppc = 0; + } + ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]); + if (IsFiredSuppression(ctx, typ, traces[0])) + return; + + DynamicMutexSet mset2; + Shadow s2(thr->racy_state[1]); + RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]); + if (IsFiredSuppression(ctx, typ, traces[1])) + return; + + if (HandleRacyStacks(thr, traces)) + return; + + // If any of the accesses has a tag, treat this as an "external" race. + uptr tag = kExternalTagNone; + for (uptr i = 0; i < kMop; i++) { + if (tags[i] != kExternalTagNone) { + typ = ReportTypeExternalRace; + tag = tags[i]; + break; + } + } + + ThreadRegistryLock l0(&ctx->thread_registry); + ScopedReport rep(typ, tag); + for (uptr i = 0; i < kMop; i++) { + Shadow s(thr->racy_state[i]); + rep.AddMemoryAccess(addr, tags[i], s, traces[i], + i == 0 ? &thr->mset : mset2); + } + + for (uptr i = 0; i < kMop; i++) { + FastState s(thr->racy_state[i]); + ThreadContext *tctx = static_cast( + ctx->thread_registry.GetThreadLocked(s.tid())); + if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1) + continue; + rep.AddThread(tctx); + } + + rep.AddLocation(addr_min, addr_max - addr_min); + +#if !SANITIZER_GO + { + Shadow s(thr->racy_state[1]); + if (s.epoch() <= thr->last_sleep_clock.get(s.tid())) + rep.AddSleep(thr->last_sleep_stack_id); + } +#endif + + OutputReport(thr, rep); +} + +void PrintCurrentStack(ThreadState *thr, uptr pc) { + VarSizeStackTrace trace; + ObtainCurrentStack(thr, pc, &trace); + PrintStack(SymbolizeStack(trace)); +} + +// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes +// __sanitizer_print_stack_trace exists in the actual unwinded stack, but +// tail-call to PrintCurrentStackSlow breaks this assumption because +// __sanitizer_print_stack_trace disappears after tail-call. +// However, this solution is not reliable enough, please see dvyukov's comment +// http://reviews.llvm.org/D19148#406208 +// Also see PR27280 comment 2 and 3 for breaking examples and analysis. +ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) { +#if !SANITIZER_GO + uptr bp = GET_CURRENT_FRAME(); + auto *ptrace = New(); + ptrace->Unwind(pc, bp, nullptr, false); + + for (uptr i = 0; i < ptrace->size / 2; i++) { + uptr tmp = ptrace->trace_buffer[i]; + ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1]; + ptrace->trace_buffer[ptrace->size - i - 1] = tmp; + } + PrintStack(SymbolizeStack(*ptrace)); +#endif +} + +} // namespace __tsan + +using namespace __tsan; + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + PrintCurrentStackSlow(StackTrace::GetCurrentPc()); +} +} // extern "C" diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S new file mode 100644 index 000000000000..fcff35fbc7e0 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S @@ -0,0 +1,47 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#define CFA_OFFSET 160 +#define R2_REL_OFFSET 16 +#define R3_REL_OFFSET 24 +#define R14_REL_OFFSET 112 +#define R15_REL_OFFSET 120 +#define FRAME_SIZE 160 + +.text + +ASM_HIDDEN(__tsan_setjmp) + +.macro intercept symbol, real +.comm \real, 8, 8 +.globl ASM_SYMBOL_INTERCEPTOR(\symbol) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol)) +ASM_SYMBOL_INTERCEPTOR(\symbol): + CFI_STARTPROC + stmg %r2, %r3, R2_REL_OFFSET(%r15) + CFI_REL_OFFSET(%r2, R2_REL_OFFSET) + CFI_REL_OFFSET(%r3, R3_REL_OFFSET) + stmg %r14, %r15, R14_REL_OFFSET(%r15) + CFI_REL_OFFSET(%r14, R14_REL_OFFSET) + CFI_REL_OFFSET(%r15, R15_REL_OFFSET) + aghi %r15, -FRAME_SIZE + CFI_ADJUST_CFA_OFFSET(FRAME_SIZE) + la %r2, FRAME_SIZE(%r15) + brasl %r14, ASM_SYMBOL(__tsan_setjmp) + lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15) + CFI_RESTORE(%r14) + CFI_RESTORE(%r15) + CFI_DEF_CFA_OFFSET(CFA_OFFSET) + lmg %r2, %r3, R2_REL_OFFSET(%r15) + CFI_RESTORE(%r2) + CFI_RESTORE(%r3) + larl %r1, \real + lg %r1, 0(%r1) + br %r1 + CFI_ENDPROC + ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol)) +.endm + +intercept setjmp, _ZN14__interception11real_setjmpE +intercept _setjmp, _ZN14__interception12real__setjmpE +intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE +intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp new file mode 100644 index 000000000000..c8f7124c009d --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp @@ -0,0 +1,349 @@ +//===-- tsan_rtl_thread.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" +#include "tsan_platform.h" +#include "tsan_report.h" +#include "tsan_sync.h" + +namespace __tsan { + +// ThreadContext implementation. + +ThreadContext::ThreadContext(Tid tid) + : ThreadContextBase(tid), thr(), sync(), epoch0(), epoch1() {} + +#if !SANITIZER_GO +ThreadContext::~ThreadContext() { +} +#endif + +void ThreadContext::OnReset() { + CHECK_EQ(sync.size(), 0); + uptr trace_p = GetThreadTrace(tid); + ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event)); + //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace)); +} + +#if !SANITIZER_GO +struct ThreadLeak { + ThreadContext *tctx; + int count; +}; + +static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) { + auto &leaks = *static_cast *>(arg); + auto *tctx = static_cast(tctx_base); + if (tctx->detached || tctx->status != ThreadStatusFinished) + return; + for (uptr i = 0; i < leaks.Size(); i++) { + if (leaks[i].tctx->creation_stack_id == tctx->creation_stack_id) { + leaks[i].count++; + return; + } + } + leaks.PushBack({tctx, 1}); +} +#endif + +#if !SANITIZER_GO +static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) { + if (tctx->tid == kMainTid) { + Printf("ThreadSanitizer: main thread finished with ignores enabled\n"); + } else { + Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled," + " created at:\n", tctx->tid, tctx->name); + PrintStack(SymbolizeStackId(tctx->creation_stack_id)); + } + Printf(" One of the following ignores was not ended" + " (in order of probability)\n"); + for (uptr i = 0; i < set->Size(); i++) { + Printf(" Ignore was enabled at:\n"); + PrintStack(SymbolizeStackId(set->At(i))); + } + Die(); +} + +static void ThreadCheckIgnore(ThreadState *thr) { + if (ctx->after_multithreaded_fork) + return; + if (thr->ignore_reads_and_writes) + ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set); + if (thr->ignore_sync) + ReportIgnoresEnabled(thr->tctx, &thr->sync_ignore_set); +} +#else +static void ThreadCheckIgnore(ThreadState *thr) {} +#endif + +void ThreadFinalize(ThreadState *thr) { + ThreadCheckIgnore(thr); +#if !SANITIZER_GO + if (!ShouldReport(thr, ReportTypeThreadLeak)) + return; + ThreadRegistryLock l(&ctx->thread_registry); + Vector leaks; + ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks, + &leaks); + for (uptr i = 0; i < leaks.Size(); i++) { + ScopedReport rep(ReportTypeThreadLeak); + rep.AddThread(leaks[i].tctx, true); + rep.SetCount(leaks[i].count); + OutputReport(thr, rep); + } +#endif +} + +int ThreadCount(ThreadState *thr) { + uptr result; + ctx->thread_registry.GetNumberOfThreads(0, 0, &result); + return (int)result; +} + +struct OnCreatedArgs { + ThreadState *thr; + uptr pc; +}; + +Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { + OnCreatedArgs args = { thr, pc }; + u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers. + Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent_tid, &args); + DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid); + return tid; +} + +void ThreadContext::OnCreated(void *arg) { + thr = 0; + if (tid == kMainTid) + return; + OnCreatedArgs *args = static_cast(arg); + if (!args->thr) // GCD workers don't have a parent thread. + return; + args->thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); + ReleaseImpl(args->thr, 0, &sync); + creation_stack_id = CurrentStackId(args->thr, args->pc); +} + +extern "C" void __tsan_stack_initialization() {} + +struct OnStartedArgs { + ThreadState *thr; + uptr stk_addr; + uptr stk_size; + uptr tls_addr; + uptr tls_size; +}; + +void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, + ThreadType thread_type) { + uptr stk_addr = 0; + uptr stk_size = 0; + uptr tls_addr = 0; + uptr tls_size = 0; +#if !SANITIZER_GO + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr, + &tls_size); +#endif + + ThreadRegistry *tr = &ctx->thread_registry; + OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; + tr->StartThread(tid, os_id, thread_type, &args); + + while (!thr->tctx->trace.parts.Empty()) thr->tctx->trace.parts.PopBack(); + +#if !SANITIZER_GO + if (ctx->after_multithreaded_fork) { + thr->ignore_interceptors++; + ThreadIgnoreBegin(thr, 0); + ThreadIgnoreSyncBegin(thr, 0); + } +#endif + +#if !SANITIZER_GO + // Don't imitate stack/TLS writes for the main thread, + // because its initialization is synchronized with all + // subsequent threads anyway. + if (tid != kMainTid) { + if (stk_addr && stk_size) { + const uptr pc = StackTrace::GetNextInstructionPc( + reinterpret_cast(__tsan_stack_initialization)); + MemoryRangeImitateWrite(thr, pc, stk_addr, stk_size); + } + + if (tls_addr && tls_size) + ImitateTlsWrite(thr, tls_addr, tls_size); + } +#endif +} + +void ThreadContext::OnStarted(void *arg) { + OnStartedArgs *args = static_cast(arg); + thr = args->thr; + // RoundUp so that one trace part does not contain events + // from different threads. + epoch0 = RoundUp(epoch1 + 1, kTracePartSize); + epoch1 = (u64)-1; + new (thr) + ThreadState(ctx, tid, unique_id, epoch0, reuse_count, args->stk_addr, + args->stk_size, args->tls_addr, args->tls_size); + if (common_flags()->detect_deadlocks) + thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id); + thr->fast_state.SetHistorySize(flags()->history_size); + // Commit switch to the new part of the trace. + // TraceAddEvent will reset stack0/mset0 in the new part for us. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + + thr->fast_synch_epoch = epoch0; + AcquireImpl(thr, 0, &sync); + sync.Reset(&thr->proc()->clock_cache); + thr->tctx = this; + thr->is_inited = true; + DPrintf( + "#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " + "tls_addr=%zx tls_size=%zx\n", + tid, (uptr)epoch0, args->stk_addr, args->stk_size, args->tls_addr, + args->tls_size); +} + +void ThreadFinish(ThreadState *thr) { + ThreadCheckIgnore(thr); + if (thr->stk_addr && thr->stk_size) + DontNeedShadowFor(thr->stk_addr, thr->stk_size); + if (thr->tls_addr && thr->tls_size) + DontNeedShadowFor(thr->tls_addr, thr->tls_size); + thr->is_dead = true; + thr->is_inited = false; +#if !SANITIZER_GO + thr->ignore_interceptors++; +#endif + ctx->thread_registry.FinishThread(thr->tid); +} + +void ThreadContext::OnFinished() { + if (!detached) { + thr->fast_state.IncrementEpoch(); + // Can't increment epoch w/o writing to the trace as well. + TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); + ReleaseImpl(thr, 0, &sync); + } + epoch1 = thr->fast_state.epoch(); + +#if !SANITIZER_GO + UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr)); +#else + Free(thr->shadow_stack); +#endif + thr->shadow_stack = nullptr; + thr->shadow_stack_pos = nullptr; + thr->shadow_stack_end = nullptr; + + if (common_flags()->detect_deadlocks) + ctx->dd->DestroyLogicalThread(thr->dd_lt); + thr->clock.ResetCached(&thr->proc()->clock_cache); +#if !SANITIZER_GO + thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); +#endif +#if !SANITIZER_GO + PlatformCleanUpThreadState(thr); +#endif + thr->~ThreadState(); + thr = 0; +} + +struct ConsumeThreadContext { + uptr uid; + ThreadContextBase *tctx; +}; + +Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) { + return ctx->thread_registry.ConsumeThreadUserId(uid); +} + +void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) { + CHECK_GT(tid, 0); + CHECK_LT(tid, kMaxTid); + DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid); + ctx->thread_registry.JoinThread(tid, thr); +} + +void ThreadContext::OnJoined(void *arg) { + ThreadState *caller_thr = static_cast(arg); + AcquireImpl(caller_thr, 0, &sync); + sync.Reset(&caller_thr->proc()->clock_cache); +} + +void ThreadContext::OnDead() { CHECK_EQ(sync.size(), 0); } + +void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) { + CHECK_GT(tid, 0); + CHECK_LT(tid, kMaxTid); + ctx->thread_registry.DetachThread(tid, thr); +} + +void ThreadContext::OnDetached(void *arg) { + ThreadState *thr1 = static_cast(arg); + sync.Reset(&thr1->proc()->clock_cache); +} + +void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) { + CHECK_GT(tid, 0); + CHECK_LT(tid, kMaxTid); + ctx->thread_registry.SetThreadUserId(tid, uid); +} + +void ThreadSetName(ThreadState *thr, const char *name) { + ctx->thread_registry.SetThreadName(thr->tid, name); +} + +#if !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = Alloc(sizeof(ThreadState)); + ThreadState *fiber = static_cast(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + Tid tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + Free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_shadow.h b/compiler-rt/lib/tsan/rtl-old/tsan_shadow.h new file mode 100644 index 000000000000..8b7bc341713e --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_shadow.h @@ -0,0 +1,233 @@ +//===-- tsan_shadow.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef TSAN_SHADOW_H +#define TSAN_SHADOW_H + +#include "tsan_defs.h" +#include "tsan_trace.h" + +namespace __tsan { + +// FastState (from most significant bit): +// ignore : 1 +// tid : kTidBits +// unused : - +// history_size : 3 +// epoch : kClkBits +class FastState { + public: + FastState(u64 tid, u64 epoch) { + x_ = tid << kTidShift; + x_ |= epoch; + DCHECK_EQ(tid, this->tid()); + DCHECK_EQ(epoch, this->epoch()); + DCHECK_EQ(GetIgnoreBit(), false); + } + + explicit FastState(u64 x) : x_(x) {} + + u64 raw() const { return x_; } + + u64 tid() const { + u64 res = (x_ & ~kIgnoreBit) >> kTidShift; + return res; + } + + u64 TidWithIgnore() const { + u64 res = x_ >> kTidShift; + return res; + } + + u64 epoch() const { + u64 res = x_ & ((1ull << kClkBits) - 1); + return res; + } + + void IncrementEpoch() { + u64 old_epoch = epoch(); + x_ += 1; + DCHECK_EQ(old_epoch + 1, epoch()); + (void)old_epoch; + } + + void SetIgnoreBit() { x_ |= kIgnoreBit; } + void ClearIgnoreBit() { x_ &= ~kIgnoreBit; } + bool GetIgnoreBit() const { return (s64)x_ < 0; } + + void SetHistorySize(int hs) { + CHECK_GE(hs, 0); + CHECK_LE(hs, 7); + x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift); + } + + ALWAYS_INLINE + int GetHistorySize() const { + return (int)((x_ >> kHistoryShift) & kHistoryMask); + } + + void ClearHistorySize() { SetHistorySize(0); } + + ALWAYS_INLINE + u64 GetTracePos() const { + const int hs = GetHistorySize(); + // When hs == 0, the trace consists of 2 parts. + const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1; + return epoch() & mask; + } + + private: + friend class Shadow; + static const int kTidShift = 64 - kTidBits - 1; + static const u64 kIgnoreBit = 1ull << 63; + static const u64 kFreedBit = 1ull << 63; + static const u64 kHistoryShift = kClkBits; + static const u64 kHistoryMask = 7; + u64 x_; +}; + +// Shadow (from most significant bit): +// freed : 1 +// tid : kTidBits +// is_atomic : 1 +// is_read : 1 +// size_log : 2 +// addr0 : 3 +// epoch : kClkBits +class Shadow : public FastState { + public: + explicit Shadow(u64 x) : FastState(x) {} + + explicit Shadow(const FastState &s) : FastState(s.x_) { ClearHistorySize(); } + + void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) { + DCHECK_EQ((x_ >> kClkBits) & 31, 0); + DCHECK_LE(addr0, 7); + DCHECK_LE(kAccessSizeLog, 3); + x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits; + DCHECK_EQ(kAccessSizeLog, size_log()); + DCHECK_EQ(addr0, this->addr0()); + } + + void SetWrite(unsigned kAccessIsWrite) { + DCHECK_EQ(x_ & kReadBit, 0); + if (!kAccessIsWrite) + x_ |= kReadBit; + DCHECK_EQ(kAccessIsWrite, IsWrite()); + } + + void SetAtomic(bool kIsAtomic) { + DCHECK(!IsAtomic()); + if (kIsAtomic) + x_ |= kAtomicBit; + DCHECK_EQ(IsAtomic(), kIsAtomic); + } + + bool IsAtomic() const { return x_ & kAtomicBit; } + + bool IsZero() const { return x_ == 0; } + + static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) { + u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift; + DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore()); + return shifted_xor == 0; + } + + static ALWAYS_INLINE bool Addr0AndSizeAreEqual(const Shadow s1, + const Shadow s2) { + u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31; + return masked_xor == 0; + } + + static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2, + unsigned kS2AccessSize) { + bool res = false; + u64 diff = s1.addr0() - s2.addr0(); + if ((s64)diff < 0) { // s1.addr0 < s2.addr0 + // if (s1.addr0() + size1) > s2.addr0()) return true; + if (s1.size() > -diff) + res = true; + } else { + // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true; + if (kS2AccessSize > diff) + res = true; + } + DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2)); + DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1)); + return res; + } + + u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; } + u64 ALWAYS_INLINE size() const { return 1ull << size_log(); } + bool ALWAYS_INLINE IsWrite() const { return !IsRead(); } + bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; } + + // The idea behind the freed bit is as follows. + // When the memory is freed (or otherwise unaccessible) we write to the shadow + // values with tid/epoch related to the free and the freed bit set. + // During memory accesses processing the freed bit is considered + // as msb of tid. So any access races with shadow with freed bit set + // (it is as if write from a thread with which we never synchronized before). + // This allows us to detect accesses to freed memory w/o additional + // overheads in memory access processing and at the same time restore + // tid/epoch of free. + void MarkAsFreed() { x_ |= kFreedBit; } + + bool IsFreed() const { return x_ & kFreedBit; } + + bool GetFreedAndReset() { + bool res = x_ & kFreedBit; + x_ &= ~kFreedBit; + return res; + } + + bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const { + bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift) | + (u64(kIsAtomic) << kAtomicShift)); + DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic)); + return v; + } + + bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const { + bool v = ((x_ >> kReadShift) & 3) <= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); + DCHECK_EQ(v, (IsAtomic() < kIsAtomic) || + (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite)); + return v; + } + + bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const { + bool v = ((x_ >> kReadShift) & 3) >= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); + DCHECK_EQ(v, (IsAtomic() > kIsAtomic) || + (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite)); + return v; + } + + private: + static const u64 kReadShift = 5 + kClkBits; + static const u64 kReadBit = 1ull << kReadShift; + static const u64 kAtomicShift = 6 + kClkBits; + static const u64 kAtomicBit = 1ull << kAtomicShift; + + u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; } + + static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) { + if (s1.addr0() == s2.addr0()) + return true; + if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0()) + return true; + if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0()) + return true; + return false; + } +}; + +const RawShadow kShadowRodata = (RawShadow)-1; // .rodata shadow marker + +} // namespace __tsan + +#endif diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp new file mode 100644 index 000000000000..9bbaafb3a85f --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp @@ -0,0 +1,57 @@ +//===-- tsan_stack_trace.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_stack_trace.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" + +namespace __tsan { + +VarSizeStackTrace::VarSizeStackTrace() + : StackTrace(nullptr, 0), trace_buffer(nullptr) {} + +VarSizeStackTrace::~VarSizeStackTrace() { + ResizeBuffer(0); +} + +void VarSizeStackTrace::ResizeBuffer(uptr new_size) { + Free(trace_buffer); + trace_buffer = (new_size > 0) + ? (uptr *)Alloc(new_size * sizeof(trace_buffer[0])) + : nullptr; + trace = trace_buffer; + size = new_size; +} + +void VarSizeStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) { + ResizeBuffer(cnt + !!extra_top_pc); + internal_memcpy(trace_buffer, pcs, cnt * sizeof(trace_buffer[0])); + if (extra_top_pc) + trace_buffer[cnt] = extra_top_pc; +} + +void VarSizeStackTrace::ReverseOrder() { + for (u32 i = 0; i < (size >> 1); i++) + Swap(trace_buffer[i], trace_buffer[size - 1 - i]); +} + +} // namespace __tsan + +#if !SANITIZER_GO +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + uptr top = 0; + uptr bottom = 0; + GetThreadStackTopAndBottom(false, &top, &bottom); + bool fast = StackTrace::WillUseFastUnwind(request_fast); + Unwind(max_depth, pc, bp, context, top, bottom, fast); +} +#endif // SANITIZER_GO diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h new file mode 100644 index 000000000000..3eb8ce156e83 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h @@ -0,0 +1,42 @@ +//===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_STACK_TRACE_H +#define TSAN_STACK_TRACE_H + +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "tsan_defs.h" + +namespace __tsan { + +// StackTrace which calls malloc/free to allocate the buffer for +// addresses in stack traces. +struct VarSizeStackTrace : public StackTrace { + uptr *trace_buffer; // Owned. + + VarSizeStackTrace(); + ~VarSizeStackTrace(); + void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0); + + // Reverses the current stack trace order, the top frame goes to the bottom, + // the last frame goes to the top. + void ReverseOrder(); + + private: + void ResizeBuffer(uptr new_size); + + VarSizeStackTrace(const VarSizeStackTrace &); + void operator=(const VarSizeStackTrace &); +}; + +} // namespace __tsan + +#endif // TSAN_STACK_TRACE_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp new file mode 100644 index 000000000000..a1c1bf81bf67 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp @@ -0,0 +1,161 @@ +//===-- tsan_suppressions.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_suppressions.h" +#include "tsan_suppressions.h" +#include "tsan_rtl.h" +#include "tsan_flags.h" +#include "tsan_mman.h" +#include "tsan_platform.h" + +#if !SANITIZER_GO +// Suppressions for true/false positives in standard libraries. +static const char *const std_suppressions = +// Libstdc++ 4.4 has data races in std::string. +// See http://crbug.com/181502 for an example. +"race:^_M_rep$\n" +"race:^_M_is_leaked$\n" +// False positive when using std . +// Happens because we miss atomic synchronization in libstdc++. +// See http://llvm.org/bugs/show_bug.cgi?id=17066 for details. +"race:std::_Sp_counted_ptr_inplaceParseFromFile(flags()->suppressions); +#if !SANITIZER_GO + suppression_ctx->Parse(__tsan_default_suppressions()); + suppression_ctx->Parse(std_suppressions); +#endif +} + +SuppressionContext *Suppressions() { + CHECK(suppression_ctx); + return suppression_ctx; +} + +static const char *conv(ReportType typ) { + switch (typ) { + case ReportTypeRace: + case ReportTypeVptrRace: + case ReportTypeUseAfterFree: + case ReportTypeVptrUseAfterFree: + case ReportTypeExternalRace: + return kSuppressionRace; + case ReportTypeThreadLeak: + return kSuppressionThread; + case ReportTypeMutexDestroyLocked: + case ReportTypeMutexDoubleLock: + case ReportTypeMutexInvalidAccess: + case ReportTypeMutexBadUnlock: + case ReportTypeMutexBadReadLock: + case ReportTypeMutexBadReadUnlock: + return kSuppressionMutex; + case ReportTypeSignalUnsafe: + case ReportTypeErrnoInSignal: + return kSuppressionSignal; + case ReportTypeDeadlock: + return kSuppressionDeadlock; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); +} + +static uptr IsSuppressed(const char *stype, const AddressInfo &info, + Suppression **sp) { + if (suppression_ctx->Match(info.function, stype, sp) || + suppression_ctx->Match(info.file, stype, sp) || + suppression_ctx->Match(info.module, stype, sp)) { + VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", (*sp)->templ); + atomic_fetch_add(&(*sp)->hit_count, 1, memory_order_relaxed); + return info.address; + } + return 0; +} + +uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp) { + CHECK(suppression_ctx); + if (!suppression_ctx->SuppressionCount() || stack == 0 || + !stack->suppressable) + return 0; + const char *stype = conv(typ); + if (0 == internal_strcmp(stype, kSuppressionNone)) + return 0; + for (const SymbolizedStack *frame = stack->frames; frame; + frame = frame->next) { + uptr pc = IsSuppressed(stype, frame->info, sp); + if (pc != 0) + return pc; + } + if (0 == internal_strcmp(stype, kSuppressionRace) && stack->frames != nullptr) + return IsSuppressed(kSuppressionRaceTop, stack->frames->info, sp); + return 0; +} + +uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) { + CHECK(suppression_ctx); + if (!suppression_ctx->SuppressionCount() || loc == 0 || + loc->type != ReportLocationGlobal || !loc->suppressable) + return 0; + const char *stype = conv(typ); + if (0 == internal_strcmp(stype, kSuppressionNone)) + return 0; + Suppression *s; + const DataInfo &global = loc->global; + if (suppression_ctx->Match(global.name, stype, &s) || + suppression_ctx->Match(global.module, stype, &s)) { + VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", s->templ); + atomic_fetch_add(&s->hit_count, 1, memory_order_relaxed); + *sp = s; + return global.start; + } + return 0; +} + +void PrintMatchedSuppressions() { + InternalMmapVector matched; + CHECK(suppression_ctx); + suppression_ctx->GetMatched(&matched); + if (!matched.size()) + return; + int hit_count = 0; + for (uptr i = 0; i < matched.size(); i++) + hit_count += atomic_load_relaxed(&matched[i]->hit_count); + Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count, + (int)internal_getpid()); + for (uptr i = 0; i < matched.size(); i++) { + Printf("%d %s:%s\n", atomic_load_relaxed(&matched[i]->hit_count), + matched[i]->type, matched[i]->templ); + } +} +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h new file mode 100644 index 000000000000..f430aeb6c4cf --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h @@ -0,0 +1,37 @@ +//===-- tsan_suppressions.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_SUPPRESSIONS_H +#define TSAN_SUPPRESSIONS_H + +#include "sanitizer_common/sanitizer_suppressions.h" +#include "tsan_report.h" + +namespace __tsan { + +const char kSuppressionNone[] = "none"; +const char kSuppressionRace[] = "race"; +const char kSuppressionRaceTop[] = "race_top"; +const char kSuppressionMutex[] = "mutex"; +const char kSuppressionThread[] = "thread"; +const char kSuppressionSignal[] = "signal"; +const char kSuppressionLib[] = "called_from_lib"; +const char kSuppressionDeadlock[] = "deadlock"; + +void InitializeSuppressions(); +SuppressionContext *Suppressions(); +void PrintMatchedSuppressions(); +uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp); +uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp); + +} // namespace __tsan + +#endif // TSAN_SUPPRESSIONS_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp new file mode 100644 index 000000000000..2e2744d2eae7 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp @@ -0,0 +1,123 @@ +//===-- tsan_symbolize.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "tsan_symbolize.h" + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_symbolizer.h" +#include "tsan_flags.h" +#include "tsan_report.h" +#include "tsan_rtl.h" + +namespace __tsan { + +void EnterSymbolizer() { + ThreadState *thr = cur_thread(); + CHECK(!thr->in_symbolizer); + thr->in_symbolizer = true; + thr->ignore_interceptors++; +} + +void ExitSymbolizer() { + ThreadState *thr = cur_thread(); + CHECK(thr->in_symbolizer); + thr->in_symbolizer = false; + thr->ignore_interceptors--; +} + +// Legacy API. +// May be overriden by JIT/JAVA/etc, +// whatever produces PCs marked with kExternalPCBit. +SANITIZER_WEAK_DEFAULT_IMPL +bool __tsan_symbolize_external(uptr pc, char *func_buf, uptr func_siz, + char *file_buf, uptr file_siz, int *line, + int *col) { + return false; +} + +// New API: call __tsan_symbolize_external_ex only when it exists. +// Once old clients are gone, provide dummy implementation. +SANITIZER_WEAK_DEFAULT_IMPL +void __tsan_symbolize_external_ex(uptr pc, + void (*add_frame)(void *, const char *, + const char *, int, int), + void *ctx) {} + +struct SymbolizedStackBuilder { + SymbolizedStack *head; + SymbolizedStack *tail; + uptr addr; +}; + +static void AddFrame(void *ctx, const char *function_name, const char *file, + int line, int column) { + SymbolizedStackBuilder *ssb = (struct SymbolizedStackBuilder *)ctx; + if (ssb->tail) { + ssb->tail->next = SymbolizedStack::New(ssb->addr); + ssb->tail = ssb->tail->next; + } else { + ssb->head = ssb->tail = SymbolizedStack::New(ssb->addr); + } + AddressInfo *info = &ssb->tail->info; + if (function_name) { + info->function = internal_strdup(function_name); + } + if (file) { + info->file = internal_strdup(file); + } + info->line = line; + info->column = column; +} + +SymbolizedStack *SymbolizeCode(uptr addr) { + // Check if PC comes from non-native land. + if (addr & kExternalPCBit) { + SymbolizedStackBuilder ssb = {nullptr, nullptr, addr}; + __tsan_symbolize_external_ex(addr, AddFrame, &ssb); + if (ssb.head) + return ssb.head; + // Legacy code: remove along with the declaration above + // once all clients using this API are gone. + // Declare static to not consume too much stack space. + // We symbolize reports in a single thread, so this is fine. + static char func_buf[1024]; + static char file_buf[1024]; + int line, col; + SymbolizedStack *frame = SymbolizedStack::New(addr); + if (__tsan_symbolize_external(addr, func_buf, sizeof(func_buf), file_buf, + sizeof(file_buf), &line, &col)) { + frame->info.function = internal_strdup(func_buf); + frame->info.file = internal_strdup(file_buf); + frame->info.line = line; + frame->info.column = col; + } + return frame; + } + return Symbolizer::GetOrInit()->SymbolizePC(addr); +} + +ReportLocation *SymbolizeData(uptr addr) { + DataInfo info; + if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) + return 0; + auto *ent = New(); + ent->type = ReportLocationGlobal; + internal_memcpy(&ent->global, &info, sizeof(info)); + return ent; +} + +void SymbolizeFlush() { + Symbolizer::GetOrInit()->Flush(); +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h new file mode 100644 index 000000000000..7adaa04dc273 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h @@ -0,0 +1,30 @@ +//===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_SYMBOLIZE_H +#define TSAN_SYMBOLIZE_H + +#include "tsan_defs.h" +#include "tsan_report.h" + +namespace __tsan { + +void EnterSymbolizer(); +void ExitSymbolizer(); +SymbolizedStack *SymbolizeCode(uptr addr); +ReportLocation *SymbolizeData(uptr addr); +void SymbolizeFlush(); + +ReportStack *NewReportStackEntry(uptr addr); + +} // namespace __tsan + +#endif // TSAN_SYMBOLIZE_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp new file mode 100644 index 000000000000..f042abab74e5 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp @@ -0,0 +1,279 @@ +//===-- tsan_sync.cpp -----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_sync.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" + +namespace __tsan { + +void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s); + +SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); } + +void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, + bool save_stack) { + this->addr = addr; + this->uid = uid; + this->next = 0; + + creation_stack_id = kInvalidStackID; + if (save_stack && !SANITIZER_GO) // Go does not use them + creation_stack_id = CurrentStackId(thr, pc); + if (common_flags()->detect_deadlocks) + DDMutexInit(thr, pc, this); +} + +void SyncVar::Reset(Processor *proc) { + uid = 0; + creation_stack_id = kInvalidStackID; + owner_tid = kInvalidTid; + last_lock = 0; + recursion = 0; + atomic_store_relaxed(&flags, 0); + + if (proc == 0) { + CHECK_EQ(clock.size(), 0); + CHECK_EQ(read_clock.size(), 0); + } else { + clock.Reset(&proc->clock_cache); + read_clock.Reset(&proc->clock_cache); + } +} + +MetaMap::MetaMap() + : block_alloc_(LINKER_INITIALIZED, "heap block allocator"), + sync_alloc_(LINKER_INITIALIZED, "sync allocator") { + atomic_store(&uid_gen_, 0, memory_order_relaxed); +} + +void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) { + u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache); + MBlock *b = block_alloc_.Map(idx); + b->siz = sz; + b->tag = 0; + b->tid = thr->tid; + b->stk = CurrentStackId(thr, pc); + u32 *meta = MemToMeta(p); + DCHECK_EQ(*meta, 0); + *meta = idx | kFlagBlock; +} + +uptr MetaMap::FreeBlock(Processor *proc, uptr p) { + MBlock* b = GetBlock(p); + if (b == 0) + return 0; + uptr sz = RoundUpTo(b->siz, kMetaShadowCell); + FreeRange(proc, p, sz); + return sz; +} + +bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) { + bool has_something = false; + u32 *meta = MemToMeta(p); + u32 *end = MemToMeta(p + sz); + if (end == meta) + end++; + for (; meta < end; meta++) { + u32 idx = *meta; + if (idx == 0) { + // Note: don't write to meta in this case -- the block can be huge. + continue; + } + *meta = 0; + has_something = true; + while (idx != 0) { + if (idx & kFlagBlock) { + block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask); + break; + } else if (idx & kFlagSync) { + DCHECK(idx & kFlagSync); + SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask); + u32 next = s->next; + s->Reset(proc); + sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask); + idx = next; + } else { + CHECK(0); + } + } + } + return has_something; +} + +// ResetRange removes all meta objects from the range. +// It is called for large mmap-ed regions. The function is best-effort wrt +// freeing of meta objects, because we don't want to page in the whole range +// which can be huge. The function probes pages one-by-one until it finds a page +// without meta objects, at this point it stops freeing meta objects. Because +// thread stacks grow top-down, we do the same starting from end as well. +void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) { + if (SANITIZER_GO) { + // UnmapOrDie/MmapFixedNoReserve does not work on Windows, + // so we do the optimization only for C/C++. + FreeRange(proc, p, sz); + return; + } + const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize; + const uptr kPageSize = GetPageSizeCached() * kMetaRatio; + if (sz <= 4 * kPageSize) { + // If the range is small, just do the normal free procedure. + FreeRange(proc, p, sz); + return; + } + // First, round both ends of the range to page size. + uptr diff = RoundUp(p, kPageSize) - p; + if (diff != 0) { + FreeRange(proc, p, diff); + p += diff; + sz -= diff; + } + diff = p + sz - RoundDown(p + sz, kPageSize); + if (diff != 0) { + FreeRange(proc, p + sz - diff, diff); + sz -= diff; + } + // Now we must have a non-empty page-aligned range. + CHECK_GT(sz, 0); + CHECK_EQ(p, RoundUp(p, kPageSize)); + CHECK_EQ(sz, RoundUp(sz, kPageSize)); + const uptr p0 = p; + const uptr sz0 = sz; + // Probe start of the range. + for (uptr checked = 0; sz > 0; checked += kPageSize) { + bool has_something = FreeRange(proc, p, kPageSize); + p += kPageSize; + sz -= kPageSize; + if (!has_something && checked > (128 << 10)) + break; + } + // Probe end of the range. + for (uptr checked = 0; sz > 0; checked += kPageSize) { + bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize); + sz -= kPageSize; + // Stacks grow down, so sync object are most likely at the end of the region + // (if it is a stack). The very end of the stack is TLS and tsan increases + // TLS by at least 256K, so check at least 512K. + if (!has_something && checked > (512 << 10)) + break; + } + // Finally, page out the whole range (including the parts that we've just + // freed). Note: we can't simply madvise, because we need to leave a zeroed + // range (otherwise __tsan_java_move can crash if it encounters a left-over + // meta objects in java heap). + uptr metap = (uptr)MemToMeta(p0); + uptr metasz = sz0 / kMetaRatio; + UnmapOrDie((void*)metap, metasz); + if (!MmapFixedSuperNoReserve(metap, metasz)) + Die(); +} + +MBlock* MetaMap::GetBlock(uptr p) { + u32 *meta = MemToMeta(p); + u32 idx = *meta; + for (;;) { + if (idx == 0) + return 0; + if (idx & kFlagBlock) + return block_alloc_.Map(idx & ~kFlagMask); + DCHECK(idx & kFlagSync); + SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask); + idx = s->next; + } +} + +SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create, + bool save_stack) { + u32 *meta = MemToMeta(addr); + u32 idx0 = *meta; + u32 myidx = 0; + SyncVar *mys = nullptr; + for (;;) { + for (u32 idx = idx0; idx && !(idx & kFlagBlock);) { + DCHECK(idx & kFlagSync); + SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask); + if (LIKELY(s->addr == addr)) { + if (UNLIKELY(myidx != 0)) { + mys->Reset(thr->proc()); + sync_alloc_.Free(&thr->proc()->sync_cache, myidx); + } + return s; + } + idx = s->next; + } + if (!create) + return nullptr; + if (UNLIKELY(*meta != idx0)) { + idx0 = *meta; + continue; + } + + if (LIKELY(myidx == 0)) { + const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed); + myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache); + mys = sync_alloc_.Map(myidx); + mys->Init(thr, pc, addr, uid, save_stack); + } + mys->next = idx0; + if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0, + myidx | kFlagSync, memory_order_release)) { + return mys; + } + } +} + +void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) { + // src and dst can overlap, + // there are no concurrent accesses to the regions (e.g. stop-the-world). + CHECK_NE(src, dst); + CHECK_NE(sz, 0); + uptr diff = dst - src; + u32 *src_meta = MemToMeta(src); + u32 *dst_meta = MemToMeta(dst); + u32 *src_meta_end = MemToMeta(src + sz); + uptr inc = 1; + if (dst > src) { + src_meta = MemToMeta(src + sz) - 1; + dst_meta = MemToMeta(dst + sz) - 1; + src_meta_end = MemToMeta(src) - 1; + inc = -1; + } + for (; src_meta != src_meta_end; src_meta += inc, dst_meta += inc) { + CHECK_EQ(*dst_meta, 0); + u32 idx = *src_meta; + *src_meta = 0; + *dst_meta = idx; + // Patch the addresses in sync objects. + while (idx != 0) { + if (idx & kFlagBlock) + break; + CHECK(idx & kFlagSync); + SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask); + s->addr += diff; + idx = s->next; + } + } +} + +void MetaMap::OnProcIdle(Processor *proc) { + block_alloc_.FlushCache(&proc->block_cache); + sync_alloc_.FlushCache(&proc->sync_cache); +} + +MetaMap::MemoryStats MetaMap::GetMemoryStats() const { + MemoryStats stats; + stats.mem_block = block_alloc_.AllocatedMemory(); + stats.sync_obj = sync_alloc_.AllocatedMemory(); + return stats; +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_sync.h b/compiler-rt/lib/tsan/rtl-old/tsan_sync.h new file mode 100644 index 000000000000..fc8fa288a841 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_sync.h @@ -0,0 +1,153 @@ +//===-- tsan_sync.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_SYNC_H +#define TSAN_SYNC_H + +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" +#include "tsan_defs.h" +#include "tsan_clock.h" +#include "tsan_dense_alloc.h" + +namespace __tsan { + +// These need to match __tsan_mutex_* flags defined in tsan_interface.h. +// See documentation there as well. +enum MutexFlags { + MutexFlagLinkerInit = 1 << 0, // __tsan_mutex_linker_init + MutexFlagWriteReentrant = 1 << 1, // __tsan_mutex_write_reentrant + MutexFlagReadReentrant = 1 << 2, // __tsan_mutex_read_reentrant + MutexFlagReadLock = 1 << 3, // __tsan_mutex_read_lock + MutexFlagTryLock = 1 << 4, // __tsan_mutex_try_lock + MutexFlagTryLockFailed = 1 << 5, // __tsan_mutex_try_lock_failed + MutexFlagRecursiveLock = 1 << 6, // __tsan_mutex_recursive_lock + MutexFlagRecursiveUnlock = 1 << 7, // __tsan_mutex_recursive_unlock + MutexFlagNotStatic = 1 << 8, // __tsan_mutex_not_static + + // The following flags are runtime private. + // Mutex API misuse was detected, so don't report any more. + MutexFlagBroken = 1 << 30, + // We did not intercept pre lock event, so handle it on post lock. + MutexFlagDoPreLockOnPostLock = 1 << 29, + // Must list all mutex creation flags. + MutexCreationFlagMask = MutexFlagLinkerInit | + MutexFlagWriteReentrant | + MutexFlagReadReentrant | + MutexFlagNotStatic, +}; + +// SyncVar is a descriptor of a user synchronization object +// (mutex or an atomic variable). +struct SyncVar { + SyncVar(); + + uptr addr; // overwritten by DenseSlabAlloc freelist + Mutex mtx; + u64 uid; // Globally unique id. + StackID creation_stack_id; + Tid owner_tid; // Set only by exclusive owners. + u64 last_lock; + int recursion; + atomic_uint32_t flags; + u32 next; // in MetaMap + DDMutex dd; + SyncClock read_clock; // Used for rw mutexes only. + // The clock is placed last, so that it is situated on a different cache line + // with the mtx. This reduces contention for hot sync objects. + SyncClock clock; + + void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, bool save_stack); + void Reset(Processor *proc); + + u64 GetId() const { + // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits. + return GetLsb((u64)addr | (uid << 48), 60); + } + bool CheckId(u64 uid) const { + CHECK_EQ(uid, GetLsb(uid, 14)); + return GetLsb(this->uid, 14) == uid; + } + static uptr SplitId(u64 id, u64 *uid) { + *uid = id >> 48; + return (uptr)GetLsb(id, 48); + } + + bool IsFlagSet(u32 f) const { + return atomic_load_relaxed(&flags) & f; + } + + void SetFlags(u32 f) { + atomic_store_relaxed(&flags, atomic_load_relaxed(&flags) | f); + } + + void UpdateFlags(u32 flagz) { + // Filter out operation flags. + if (!(flagz & MutexCreationFlagMask)) + return; + u32 current = atomic_load_relaxed(&flags); + if (current & MutexCreationFlagMask) + return; + // Note: this can be called from MutexPostReadLock which holds only read + // lock on the SyncVar. + atomic_store_relaxed(&flags, current | (flagz & MutexCreationFlagMask)); + } +}; + +// MetaMap maps app addresses to heap block (MBlock) and sync var (SyncVar) +// descriptors. It uses 1/2 direct shadow, see tsan_platform.h for the mapping. +class MetaMap { + public: + MetaMap(); + + void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz); + uptr FreeBlock(Processor *proc, uptr p); + bool FreeRange(Processor *proc, uptr p, uptr sz); + void ResetRange(Processor *proc, uptr p, uptr sz); + MBlock* GetBlock(uptr p); + + SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr, + bool save_stack) { + return GetSync(thr, pc, addr, true, save_stack); + } + SyncVar *GetSyncIfExists(uptr addr) { + return GetSync(nullptr, 0, addr, false, false); + } + + void MoveMemory(uptr src, uptr dst, uptr sz); + + void OnProcIdle(Processor *proc); + + struct MemoryStats { + uptr mem_block; + uptr sync_obj; + }; + + MemoryStats GetMemoryStats() const; + + private: + static const u32 kFlagMask = 3u << 30; + static const u32 kFlagBlock = 1u << 30; + static const u32 kFlagSync = 2u << 30; + typedef DenseSlabAlloc BlockAlloc; + typedef DenseSlabAlloc SyncAlloc; + BlockAlloc block_alloc_; + SyncAlloc sync_alloc_; + atomic_uint64_t uid_gen_; + + SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create, + bool save_stack); +}; + +} // namespace __tsan + +#endif // TSAN_SYNC_H diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_trace.h b/compiler-rt/lib/tsan/rtl-old/tsan_trace.h new file mode 100644 index 000000000000..ffc8c991ece0 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_trace.h @@ -0,0 +1,252 @@ +//===-- tsan_trace.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_TRACE_H +#define TSAN_TRACE_H + +#include "tsan_defs.h" +#include "tsan_ilist.h" +#include "tsan_mutexset.h" +#include "tsan_stack_trace.h" + +namespace __tsan { + +const int kTracePartSizeBits = 13; +const int kTracePartSize = 1 << kTracePartSizeBits; +const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize; +const int kTraceSize = kTracePartSize * kTraceParts; + +// Must fit into 3 bits. +enum EventType { + EventTypeMop, + EventTypeFuncEnter, + EventTypeFuncExit, + EventTypeLock, + EventTypeUnlock, + EventTypeRLock, + EventTypeRUnlock +}; + +// Represents a thread event (from most significant bit): +// u64 typ : 3; // EventType. +// u64 addr : 61; // Associated pc. +typedef u64 Event; + +const uptr kEventPCBits = 61; + +struct TraceHeader { +#if !SANITIZER_GO + BufferedStackTrace stack0; // Start stack for the trace. +#else + VarSizeStackTrace stack0; +#endif + u64 epoch0; // Start epoch for the trace. + MutexSet mset0; + + TraceHeader() : stack0(), epoch0() {} +}; + +struct Trace { + Mutex mtx; +#if !SANITIZER_GO + // Must be last to catch overflow as paging fault. + // Go shadow stack is dynamically allocated. + uptr shadow_stack[kShadowStackSize]; +#endif + // Must be the last field, because we unmap the unused part in + // CreateThreadContext. + TraceHeader headers[kTraceParts]; + + Trace() : mtx(MutexTypeTrace) {} +}; + +namespace v3 { + +enum class EventType : u64 { + kAccessExt, + kAccessRange, + kLock, + kRLock, + kUnlock, + kTime, +}; + +// "Base" type for all events for type dispatch. +struct Event { + // We use variable-length type encoding to give more bits to some event + // types that need them. If is_access is set, this is EventAccess. + // Otherwise, if is_func is set, this is EventFunc. + // Otherwise type denotes the type. + u64 is_access : 1; + u64 is_func : 1; + EventType type : 3; + u64 _ : 59; +}; +static_assert(sizeof(Event) == 8, "bad Event size"); + +// Nop event used as padding and does not affect state during replay. +static constexpr Event NopEvent = {1, 0, EventType::kAccessExt, 0}; + +// Compressed memory access can represent only some events with PCs +// close enough to each other. Otherwise we fall back to EventAccessExt. +struct EventAccess { + static constexpr uptr kPCBits = 15; + static_assert(kPCBits + kCompressedAddrBits + 5 == 64, + "unused bits in EventAccess"); + + u64 is_access : 1; // = 1 + u64 is_read : 1; + u64 is_atomic : 1; + u64 size_log : 2; + u64 pc_delta : kPCBits; // signed delta from the previous memory access PC + u64 addr : kCompressedAddrBits; +}; +static_assert(sizeof(EventAccess) == 8, "bad EventAccess size"); + +// Function entry (pc != 0) or exit (pc == 0). +struct EventFunc { + u64 is_access : 1; // = 0 + u64 is_func : 1; // = 1 + u64 pc : 62; +}; +static_assert(sizeof(EventFunc) == 8, "bad EventFunc size"); + +// Extended memory access with full PC. +struct EventAccessExt { + // Note: precisely specifying the unused parts of the bitfield is critical for + // performance. If we don't specify them, compiler will generate code to load + // the old value and shuffle it to extract the unused bits to apply to the new + // value. If we specify the unused part and store 0 in there, all that + // unnecessary code goes away (store of the 0 const is combined with other + // constant parts). + static constexpr uptr kUnusedBits = 11; + static_assert(kCompressedAddrBits + kUnusedBits + 9 == 64, + "unused bits in EventAccessExt"); + + u64 is_access : 1; // = 0 + u64 is_func : 1; // = 0 + EventType type : 3; // = EventType::kAccessExt + u64 is_read : 1; + u64 is_atomic : 1; + u64 size_log : 2; + u64 _ : kUnusedBits; + u64 addr : kCompressedAddrBits; + u64 pc; +}; +static_assert(sizeof(EventAccessExt) == 16, "bad EventAccessExt size"); + +// Access to a memory range. +struct EventAccessRange { + static constexpr uptr kSizeLoBits = 13; + static_assert(kCompressedAddrBits + kSizeLoBits + 7 == 64, + "unused bits in EventAccessRange"); + + u64 is_access : 1; // = 0 + u64 is_func : 1; // = 0 + EventType type : 3; // = EventType::kAccessRange + u64 is_read : 1; + u64 is_free : 1; + u64 size_lo : kSizeLoBits; + u64 pc : kCompressedAddrBits; + u64 addr : kCompressedAddrBits; + u64 size_hi : 64 - kCompressedAddrBits; +}; +static_assert(sizeof(EventAccessRange) == 16, "bad EventAccessRange size"); + +// Mutex lock. +struct EventLock { + static constexpr uptr kStackIDLoBits = 15; + static constexpr uptr kStackIDHiBits = + sizeof(StackID) * kByteBits - kStackIDLoBits; + static constexpr uptr kUnusedBits = 3; + static_assert(kCompressedAddrBits + kStackIDLoBits + 5 == 64, + "unused bits in EventLock"); + static_assert(kCompressedAddrBits + kStackIDHiBits + kUnusedBits == 64, + "unused bits in EventLock"); + + u64 is_access : 1; // = 0 + u64 is_func : 1; // = 0 + EventType type : 3; // = EventType::kLock or EventType::kRLock + u64 pc : kCompressedAddrBits; + u64 stack_lo : kStackIDLoBits; + u64 stack_hi : sizeof(StackID) * kByteBits - kStackIDLoBits; + u64 _ : kUnusedBits; + u64 addr : kCompressedAddrBits; +}; +static_assert(sizeof(EventLock) == 16, "bad EventLock size"); + +// Mutex unlock. +struct EventUnlock { + static constexpr uptr kUnusedBits = 15; + static_assert(kCompressedAddrBits + kUnusedBits + 5 == 64, + "unused bits in EventUnlock"); + + u64 is_access : 1; // = 0 + u64 is_func : 1; // = 0 + EventType type : 3; // = EventType::kUnlock + u64 _ : kUnusedBits; + u64 addr : kCompressedAddrBits; +}; +static_assert(sizeof(EventUnlock) == 8, "bad EventUnlock size"); + +// Time change event. +struct EventTime { + static constexpr uptr kUnusedBits = 37; + static_assert(kUnusedBits + sizeof(Sid) * kByteBits + kEpochBits + 5 == 64, + "unused bits in EventTime"); + + u64 is_access : 1; // = 0 + u64 is_func : 1; // = 0 + EventType type : 3; // = EventType::kTime + u64 sid : sizeof(Sid) * kByteBits; + u64 epoch : kEpochBits; + u64 _ : kUnusedBits; +}; +static_assert(sizeof(EventTime) == 8, "bad EventTime size"); + +struct Trace; + +struct TraceHeader { + Trace* trace = nullptr; // back-pointer to Trace containing this part + INode trace_parts; // in Trace::parts +}; + +struct TracePart : TraceHeader { + // There are a lot of goroutines in Go, so we use smaller parts. + static constexpr uptr kByteSize = (SANITIZER_GO ? 128 : 256) << 10; + static constexpr uptr kSize = + (kByteSize - sizeof(TraceHeader)) / sizeof(Event); + // TraceAcquire does a fast event pointer overflow check by comparing + // pointer into TracePart::events with kAlignment mask. Since TracePart's + // are allocated page-aligned, this check detects end of the array + // (it also have false positives in the middle that are filtered separately). + // This also requires events to be the last field. + static constexpr uptr kAlignment = 0xff0; + Event events[kSize]; + + TracePart() {} +}; +static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size"); + +struct Trace { + Mutex mtx; + IList parts; + Event* final_pos = + nullptr; // final position in the last part for finished threads + + Trace() : mtx(MutexTypeTrace) {} +}; + +} // namespace v3 + +} // namespace __tsan + +#endif // TSAN_TRACE_H diff --git a/compiler-rt/lib/tsan/rtl/tsan_update_shadow_word.inc b/compiler-rt/lib/tsan/rtl-old/tsan_update_shadow_word.inc similarity index 100% rename from compiler-rt/lib/tsan/rtl/tsan_update_shadow_word.inc rename to compiler-rt/lib/tsan/rtl-old/tsan_update_shadow_word.inc diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp new file mode 100644 index 000000000000..278298565d3f --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp @@ -0,0 +1,126 @@ +//===-- tsan_vector_clock.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#include "tsan_vector_clock.h" + +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_mman.h" + +namespace __tsan { + +#if TSAN_VECTORIZE +const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128); +#endif + +VectorClock::VectorClock() { Reset(); } + +void VectorClock::Reset() { +#if !TSAN_VECTORIZE + for (uptr i = 0; i < kThreadSlotCount; i++) + clk_[i] = kEpochZero; +#else + m128 z = _mm_setzero_si128(); + m128* vclk = reinterpret_cast(clk_); + for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z); +#endif +} + +void VectorClock::Acquire(const VectorClock* src) { + if (!src) + return; +#if !TSAN_VECTORIZE + for (uptr i = 0; i < kThreadSlotCount; i++) + clk_[i] = max(clk_[i], src->clk_[i]); +#else + m128* __restrict vdst = reinterpret_cast(clk_); + m128 const* __restrict vsrc = reinterpret_cast(src->clk_); + for (uptr i = 0; i < kVectorClockSize; i++) { + m128 s = _mm_load_si128(&vsrc[i]); + m128 d = _mm_load_si128(&vdst[i]); + m128 m = _mm_max_epu16(s, d); + _mm_store_si128(&vdst[i], m); + } +#endif +} + +static VectorClock* AllocClock(VectorClock** dstp) { + if (UNLIKELY(!*dstp)) + *dstp = New(); + return *dstp; +} + +void VectorClock::Release(VectorClock** dstp) const { + VectorClock* dst = AllocClock(dstp); + dst->Acquire(this); +} + +void VectorClock::ReleaseStore(VectorClock** dstp) const { + VectorClock* dst = AllocClock(dstp); + *dst = *this; +} + +VectorClock& VectorClock::operator=(const VectorClock& other) { +#if !TSAN_VECTORIZE + for (uptr i = 0; i < kThreadSlotCount; i++) + clk_[i] = other.clk_[i]; +#else + m128* __restrict vdst = reinterpret_cast(clk_); + m128 const* __restrict vsrc = reinterpret_cast(other.clk_); + for (uptr i = 0; i < kVectorClockSize; i++) { + m128 s = _mm_load_si128(&vsrc[i]); + _mm_store_si128(&vdst[i], s); + } +#endif + return *this; +} + +void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) { + VectorClock* dst = AllocClock(dstp); +#if !TSAN_VECTORIZE + for (uptr i = 0; i < kThreadSlotCount; i++) { + Epoch tmp = dst->clk_[i]; + dst->clk_[i] = clk_[i]; + clk_[i] = max(clk_[i], tmp); + } +#else + m128* __restrict vdst = reinterpret_cast(dst->clk_); + m128* __restrict vclk = reinterpret_cast(clk_); + for (uptr i = 0; i < kVectorClockSize; i++) { + m128 t = _mm_load_si128(&vdst[i]); + m128 c = _mm_load_si128(&vclk[i]); + m128 m = _mm_max_epu16(c, t); + _mm_store_si128(&vdst[i], c); + _mm_store_si128(&vclk[i], m); + } +#endif +} + +void VectorClock::ReleaseAcquire(VectorClock** dstp) { + VectorClock* dst = AllocClock(dstp); +#if !TSAN_VECTORIZE + for (uptr i = 0; i < kThreadSlotCount; i++) { + dst->clk_[i] = max(dst->clk_[i], clk_[i]); + clk_[i] = dst->clk_[i]; + } +#else + m128* __restrict vdst = reinterpret_cast(dst->clk_); + m128* __restrict vclk = reinterpret_cast(clk_); + for (uptr i = 0; i < kVectorClockSize; i++) { + m128 c = _mm_load_si128(&vclk[i]); + m128 d = _mm_load_si128(&vdst[i]); + m128 m = _mm_max_epu16(c, d); + _mm_store_si128(&vdst[i], m); + _mm_store_si128(&vclk[i], m); + } +#endif +} + +} // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h new file mode 100644 index 000000000000..63b206302190 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h @@ -0,0 +1,51 @@ +//===-- tsan_vector_clock.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// +#ifndef TSAN_VECTOR_CLOCK_H +#define TSAN_VECTOR_CLOCK_H + +#include "tsan_defs.h" + +namespace __tsan { + +// Fixed-size vector clock, used both for threads and sync objects. +class VectorClock { + public: + VectorClock(); + + Epoch Get(Sid sid) const; + void Set(Sid sid, Epoch v); + + void Reset(); + void Acquire(const VectorClock* src); + void Release(VectorClock** dstp) const; + void ReleaseStore(VectorClock** dstp) const; + void ReleaseStoreAcquire(VectorClock** dstp); + void ReleaseAcquire(VectorClock** dstp); + + VectorClock& operator=(const VectorClock& other); + + private: + Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED; +}; + +ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const { + return clk_[static_cast(sid)]; +} + +ALWAYS_INLINE void VectorClock::Set(Sid sid, Epoch v) { + DCHECK_GE(v, clk_[static_cast(sid)]); + clk_[static_cast(sid)] = v; +} + +} // namespace __tsan + +#endif // TSAN_VECTOR_CLOCK_H diff --git a/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp b/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp index 1d3c3849a446..1e61c31c5a97 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp @@ -157,7 +157,7 @@ int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr, ReportMutex *mutex = rep->mutexes[idx]; *mutex_id = mutex->id; *addr = (void *)mutex->addr; - *destroyed = mutex->destroyed; + *destroyed = false; if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size); return 1; } diff --git a/compiler-rt/lib/tsan/rtl/tsan_defs.h b/compiler-rt/lib/tsan/rtl/tsan_defs.h index 4712c2be1813..2e13e0e5486b 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_defs.h +++ b/compiler-rt/lib/tsan/rtl/tsan_defs.h @@ -63,41 +63,14 @@ enum class Epoch : u16 {}; constexpr uptr kEpochBits = 14; constexpr Epoch kEpochZero = static_cast(0); constexpr Epoch kEpochOver = static_cast(1 << kEpochBits); +constexpr Epoch kEpochLast = static_cast((1 << kEpochBits) - 1); -const int kClkBits = 42; -const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1; +inline Epoch EpochInc(Epoch epoch) { + return static_cast(static_cast(epoch) + 1); +} -struct ClockElem { - u64 epoch : kClkBits; - u64 reused : 64 - kClkBits; // tid reuse count -}; +inline bool EpochOverflow(Epoch epoch) { return epoch == kEpochOver; } -struct ClockBlock { - static const uptr kSize = 512; - static const uptr kTableSize = kSize / sizeof(u32); - static const uptr kClockCount = kSize / sizeof(ClockElem); - static const uptr kRefIdx = kTableSize - 1; - static const uptr kBlockIdx = kTableSize - 2; - - union { - u32 table[kTableSize]; - ClockElem clock[kClockCount]; - }; - - ClockBlock() { - } -}; - -const int kTidBits = 13; -// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is -// occupied by reference counter, so total number of elements we can store -// in SyncClock is kClockCount * (kTableSize - 1). -const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount; -#if !SANITIZER_GO -const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit. -#else -const unsigned kMaxTidInClock = kMaxTid; // Go does not track freed memory. -#endif const uptr kShadowStackSize = 64 * 1024; // Count of shadow values in a shadow cell. @@ -107,7 +80,7 @@ const uptr kShadowCnt = 4; const uptr kShadowCell = 8; // Single shadow value. -typedef u64 RawShadow; +enum class RawShadow : u32 {}; const uptr kShadowSize = sizeof(RawShadow); // Shadow memory is kShadowMultiplier times larger than user memory. @@ -184,10 +157,13 @@ MD5Hash md5_hash(const void *data, uptr size); struct Processor; struct ThreadState; class ThreadContext; +struct TidSlot; struct Context; struct ReportStack; class ReportDesc; class RegionAlloc; +struct Trace; +struct TracePart; typedef uptr AccessType; @@ -198,6 +174,8 @@ enum : AccessType { kAccessVptr = 1 << 2, // read or write of an object virtual table pointer kAccessFree = 1 << 3, // synthetic memory access during memory freeing kAccessExternalPC = 1 << 4, // access PC can have kExternalPCBit set + kAccessCheckOnly = 1 << 5, // check for races, but don't store + kAccessNoRodata = 1 << 6, // don't check for .rodata marker }; // Descriptor of user's memory block. @@ -219,9 +197,8 @@ enum ExternalTag : uptr { // as 16-bit values, see tsan_defs.h. }; -enum MutexType { - MutexTypeTrace = MutexLastCommon, - MutexTypeReport, +enum { + MutexTypeReport = MutexLastCommon, MutexTypeSyncVar, MutexTypeAnnotations, MutexTypeAtExit, @@ -229,6 +206,9 @@ enum MutexType { MutexTypeRacy, MutexTypeGlobalProc, MutexTypeInternalAlloc, + MutexTypeTrace, + MutexTypeSlot, + MutexTypeSlots, }; } // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h index 9e15f74a0615..7a39a39d51de 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h +++ b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h @@ -104,6 +104,15 @@ class DenseSlabAlloc { return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T); } + template + void ForEach(Func func) { + SpinMutexLock lock(&mtx_); + uptr fillpos = atomic_load_relaxed(&fillpos_); + for (uptr l1 = 0; l1 < fillpos; l1++) { + for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]); + } + } + private: T *map_[kL1Size]; SpinMutex mtx_; diff --git a/compiler-rt/lib/tsan/rtl/tsan_fd.cpp b/compiler-rt/lib/tsan/rtl/tsan_fd.cpp index 255ffa8daf76..9a6400c2e9f9 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_fd.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_fd.cpp @@ -11,9 +11,12 @@ //===----------------------------------------------------------------------===// #include "tsan_fd.h" -#include "tsan_rtl.h" + #include +#include "tsan_interceptors.h" +#include "tsan_rtl.h" + namespace __tsan { const int kTableSizeL1 = 1024; @@ -192,19 +195,21 @@ void FdClose(ThreadState *thr, uptr pc, int fd, bool write) { if (bogusfd(fd)) return; FdDesc *d = fddesc(thr, pc, fd); - if (write) { - // To catch races between fd usage and close. - MemoryAccess(thr, pc, (uptr)d, 8, kAccessWrite); - } else { - // This path is used only by dup2/dup3 calls. - // We do read instead of write because there is a number of legitimate - // cases where write would lead to false positives: - // 1. Some software dups a closed pipe in place of a socket before closing - // the socket (to prevent races actually). - // 2. Some daemons dup /dev/null in place of stdin/stdout. - // On the other hand we have not seen cases when write here catches real - // bugs. - MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); + if (!MustIgnoreInterceptor(thr)) { + if (write) { + // To catch races between fd usage and close. + MemoryAccess(thr, pc, (uptr)d, 8, kAccessWrite); + } else { + // This path is used only by dup2/dup3 calls. + // We do read instead of write because there is a number of legitimate + // cases where write would lead to false positives: + // 1. Some software dups a closed pipe in place of a socket before closing + // the socket (to prevent races actually). + // 2. Some daemons dup /dev/null in place of stdin/stdout. + // On the other hand we have not seen cases when write here catches real + // bugs. + MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead); + } } // We need to clear it, because if we do not intercept any call out there // that creates fd, we will hit false postives. diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp index ee89862d17bd..54bed9f9a6be 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp @@ -110,12 +110,6 @@ void InitializeFlags(Flags *f, const char *env, const char *env_option_name) { if (common_flags()->help) parser.PrintFlagDescriptions(); - if (f->history_size < 0 || f->history_size > 7) { - Printf("ThreadSanitizer: incorrect value for history_size" - " (must be [0..7])\n"); - Die(); - } - if (f->io_sync < 0 || f->io_sync > 2) { Printf("ThreadSanitizer: incorrect value for io_sync" " (must be [0..2])\n"); diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.inc b/compiler-rt/lib/tsan/rtl/tsan_flags.inc index 7954a4307fa1..b1691452d022 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_flags.inc +++ b/compiler-rt/lib/tsan/rtl/tsan_flags.inc @@ -43,6 +43,9 @@ TSAN_FLAG( bool, force_seq_cst_atomics, false, "If set, all atomics are effectively sequentially consistent (seq_cst), " "regardless of what user actually specified.") +TSAN_FLAG(bool, force_background_thread, false, + "If set, eagerly launch a background thread for memory reclamation " + "instead of waiting for a user call to pthread_create.") TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.") TSAN_FLAG(int, atexit_sleep_ms, 1000, "Sleep in main thread before exiting for that many ms " @@ -59,14 +62,10 @@ TSAN_FLAG(bool, stop_on_start, false, "Stops on start until __tsan_resume() is called (for debugging).") TSAN_FLAG(bool, running_on_valgrind, false, "Controls whether RunningOnValgrind() returns true or false.") -// There are a lot of goroutines in Go, so we use smaller history. TSAN_FLAG( - int, history_size, SANITIZER_GO ? 1 : 3, - "Per-thread history size, controls how many previous memory accesses " - "are remembered per thread. Possible values are [0..7]. " - "history_size=0 amounts to 32K memory accesses. Each next value doubles " - "the amount of memory accesses, up to history_size=7 that amounts to " - "4M memory accesses. The default value is 2 (128K memory accesses).") + uptr, history_size, 0, + "Per-thread history size," + " controls how many extra previous memory accesses are remembered per thread.") TSAN_FLAG(int, io_sync, 1, "Controls level of synchronization implied by IO operations. " "0 - no synchronization " diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index 61dbb81ffec4..88a54b554421 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -36,6 +36,10 @@ inline bool in_symbolizer() { } #endif +inline bool MustIgnoreInterceptor(ThreadState *thr) { + return !thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib; +} + } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ @@ -60,10 +64,10 @@ inline bool in_symbolizer() { # define CHECK_REAL_FUNC(func) DCHECK(REAL(func)) #endif -#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ - SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ - CHECK_REAL_FUNC(func); \ - if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \ +#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ + SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ + CHECK_REAL_FUNC(func); \ + if (MustIgnoreInterceptor(thr)) \ return REAL(func)(__VA_ARGS__); #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \ diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index cf3dc90d96a1..c4f43d8171ab 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -1681,11 +1681,10 @@ TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { - SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); - if (fd >= 0) - FdClose(thr, pc, fd); + SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags); + FdClose(thr, pc, fd); fd = REAL(signalfd)(fd, mask, flags); - if (fd >= 0) + if (!MustIgnoreInterceptor(thr)) FdSignalCreate(thr, pc, fd); return fd; } @@ -1762,17 +1761,15 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { } TSAN_INTERCEPTOR(int, close, int fd) { - SCOPED_TSAN_INTERCEPTOR(close, fd); - if (fd >= 0) - FdClose(thr, pc, fd); + SCOPED_INTERCEPTOR_RAW(close, fd); + FdClose(thr, pc, fd); return REAL(close)(fd); } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, __close, int fd) { - SCOPED_TSAN_INTERCEPTOR(__close, fd); - if (fd >= 0) - FdClose(thr, pc, fd); + SCOPED_INTERCEPTOR_RAW(__close, fd); + FdClose(thr, pc, fd); return REAL(__close)(fd); } #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) @@ -1783,13 +1780,10 @@ TSAN_INTERCEPTOR(int, __close, int fd) { // glibc guts #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { - SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); + SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr); int fds[64]; int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); - for (int i = 0; i < cnt; i++) { - if (fds[i] > 0) - FdClose(thr, pc, fds[i]); - } + for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]); REAL(__res_iclose)(state, free_addr); } #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) @@ -1870,7 +1864,7 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) { } TSAN_INTERCEPTOR(int, closedir, void *dirp) { - SCOPED_TSAN_INTERCEPTOR(closedir, dirp); + SCOPED_INTERCEPTOR_RAW(closedir, dirp); if (dirp) { int fd = dirfd(dirp); FdClose(thr, pc, fd); @@ -1981,6 +1975,7 @@ static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) { static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, int sig, __sanitizer_siginfo *info, void *uctx) { + CHECK(thr->slot); __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; if (acquire) Acquire(thr, 0, (uptr)&sigactions[sig]); @@ -2268,7 +2263,7 @@ struct dl_iterate_phdr_data { }; static bool IsAppNotRodata(uptr addr) { - return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata; + return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata; } static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, @@ -2374,7 +2369,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ if (file) { \ int fd = fileno_unlocked(file); \ - if (fd >= 0) FdClose(thr, pc, fd); \ + FdClose(thr, pc, fd); \ } #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ @@ -2581,7 +2576,7 @@ static USED void syscall_release(uptr pc, uptr addr) { } static void syscall_fd_close(uptr pc, int fd) { - TSAN_SYSCALL(); + auto *thr = cur_thread(); FdClose(thr, pc, fd); } diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface.cpp index 048715185151..e6c4bf2e60a7 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interface.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interface.cpp @@ -26,20 +26,6 @@ void __tsan_flush_memory() { FlushShadowMemory(); } -void __tsan_read16(void *addr) { - uptr pc = CALLERPC; - ThreadState *thr = cur_thread(); - MemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead); - MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead); -} - -void __tsan_write16(void *addr) { - uptr pc = CALLERPC; - ThreadState *thr = cur_thread(); - MemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite); - MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite); -} - void __tsan_read16_pc(void *addr, void *pc) { uptr pc_no_pac = STRIP_PAC_PC(pc); ThreadState *thr = cur_thread(); diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.inc b/compiler-rt/lib/tsan/rtl/tsan_interface.inc index 0031800e851f..b0a424ff9c25 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interface.inc +++ b/compiler-rt/lib/tsan/rtl/tsan_interface.inc @@ -34,6 +34,10 @@ void __tsan_read8(void *addr) { MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead); } +void __tsan_read16(void *addr) { + MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead); +} + void __tsan_write1(void *addr) { MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite); } @@ -50,6 +54,10 @@ void __tsan_write8(void *addr) { MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite); } +void __tsan_write16(void *addr) { + MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite); +} + void __tsan_read1_pc(void *addr, void *pc) { MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC); } diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp index 24ba3bb1f65d..f794a2fcdd0d 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp @@ -235,8 +235,9 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { T v = NoTsanAtomicLoad(a, mo); SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); if (s) { - ReadLock l(&s->mtx); - AcquireImpl(thr, pc, &s->clock); + SlotLocker locker(thr); + ReadLock lock(&s->mtx); + thr->clock.Acquire(s->clock); // Re-read under sync mutex because we need a consistent snapshot // of the value and the clock we acquire. v = NoTsanAtomicLoad(a, mo); @@ -270,14 +271,14 @@ static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, NoTsanAtomicStore(a, v, mo); return; } - __sync_synchronize(); - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - ReleaseStoreImpl(thr, pc, &s->clock); - NoTsanAtomicStore(a, v, mo); + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + Lock lock(&s->mtx); + thr->clock.ReleaseStore(&s->clock); + NoTsanAtomicStore(a, v, mo); + } + IncrementEpoch(thr); } template @@ -285,18 +286,21 @@ static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessWrite | kAccessAtomic); if (LIKELY(mo == mo_relaxed)) return F(a, v); - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - if (IsAcqRelOrder(mo)) - AcquireReleaseImpl(thr, pc, &s->clock); - else if (IsReleaseOrder(mo)) - ReleaseImpl(thr, pc, &s->clock); - else if (IsAcquireOrder(mo)) - AcquireImpl(thr, pc, &s->clock); - return F(a, v); + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + RWLock lock(&s->mtx, IsReleaseOrder(mo)); + if (IsAcqRelOrder(mo)) + thr->clock.ReleaseAcquire(&s->clock); + else if (IsReleaseOrder(mo)) + thr->clock.Release(&s->clock); + else if (IsAcquireOrder(mo)) + thr->clock.Acquire(s->clock); + v = F(a, v); + } + if (IsReleaseOrder(mo)) + IncrementEpoch(thr); + return v; } template @@ -416,27 +420,28 @@ static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, *c = pr; return false; } - + SlotLocker locker(thr); bool release = IsReleaseOrder(mo); - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - RWLock l(&s->mtx, release); - T cc = *c; - T pr = func_cas(a, cc, v); - bool success = pr == cc; - if (!success) { - *c = pr; - mo = fmo; + bool success; + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + RWLock lock(&s->mtx, release); + T cc = *c; + T pr = func_cas(a, cc, v); + success = pr == cc; + if (!success) { + *c = pr; + mo = fmo; + } + if (success && IsAcqRelOrder(mo)) + thr->clock.ReleaseAcquire(&s->clock); + else if (success && IsReleaseOrder(mo)) + thr->clock.Release(&s->clock); + else if (IsAcquireOrder(mo)) + thr->clock.Acquire(s->clock); } - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - - if (success && IsAcqRelOrder(mo)) - AcquireReleaseImpl(thr, pc, &s->clock); - else if (success && IsReleaseOrder(mo)) - ReleaseImpl(thr, pc, &s->clock); - else if (IsAcquireOrder(mo)) - AcquireImpl(thr, pc, &s->clock); + if (success && release) + IncrementEpoch(thr); return success; } diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp index c090c1f08cbe..7c15a1638826 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp @@ -106,7 +106,7 @@ void __tsan_java_free(jptr ptr, jptr size) { DCHECK_GE(ptr, jctx->heap_begin); DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size); - ctx->metamap.FreeRange(thr->proc(), ptr, size); + ctx->metamap.FreeRange(thr->proc(), ptr, size, false); } void __tsan_java_move(jptr src, jptr dst, jptr size) { @@ -133,7 +133,7 @@ void __tsan_java_move(jptr src, jptr dst, jptr size) { // support that anymore as it contains addresses of accesses. RawShadow *d = MemToShadow(dst); RawShadow *dend = MemToShadow(dst + size); - internal_memset(d, 0, (dend - d) * sizeof(*d)); + ShadowSet(d, dend, Shadow::kEmpty); } jptr __tsan_java_find(jptr *from_ptr, jptr to) { diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp index a31bebcb6ba9..7a72efb12263 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp @@ -125,7 +125,6 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() { } void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS { - global_proc()->mtx.Lock(); global_proc()->internal_alloc_mtx.Lock(); InternalAllocatorLock(); } @@ -133,6 +132,13 @@ void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS { void AllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS { InternalAllocatorUnlock(); global_proc()->internal_alloc_mtx.Unlock(); +} + +void GlobalProcessorLock() NO_THREAD_SAFETY_ANALYSIS { + global_proc()->mtx.Lock(); +} + +void GlobalProcessorUnlock() NO_THREAD_SAFETY_ANALYSIS { global_proc()->mtx.Unlock(); } @@ -192,6 +198,12 @@ void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align, GET_STACK_TRACE_FATAL(thr, pc); ReportAllocationSizeTooBig(sz, malloc_limit, &stack); } + if (UNLIKELY(IsRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + GET_STACK_TRACE_FATAL(thr, pc); + ReportRssLimitExceeded(&stack); + } void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align); if (UNLIKELY(!p)) { SetAllocatorOutOfMemory(); @@ -245,8 +257,17 @@ void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p); + // Note: this can run before thread initialization/after finalization. + // As a result this is not necessarily synchronized with DoReset, + // which iterates over and resets all sync objects, + // but it is fine to create new MBlocks in this context. ctx->metamap.AllocBlock(thr, pc, p, sz); - if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited) + // If this runs before thread initialization/after finalization + // and we don't have trace initialized, we can't imitate writes. + // In such case just reset the shadow range, it is fine since + // it affects only a small fraction of special objects. + if (write && thr->ignore_reads_and_writes == 0 && + atomic_load_relaxed(&thr->trace_pos)) MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); else MemoryResetRange(thr, pc, (uptr)p, sz); @@ -254,9 +275,16 @@ void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { CHECK_NE(p, (void*)0); - uptr sz = ctx->metamap.FreeBlock(thr->proc(), p); + if (!thr->slot) { + // Very early/late in thread lifetime, or during fork. + UNUSED uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, false); + DPrintf("#%d: free(0x%zx, %zu) (no slot)\n", thr->tid, p, sz); + return; + } + SlotLocker locker(thr); + uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, true); DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz); - if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited) + if (write && thr->ignore_reads_and_writes == 0) MemoryRangeFreed(thr, pc, (uptr)p, sz); } @@ -336,7 +364,7 @@ void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) { } uptr user_alloc_usable_size(const void *p) { - if (p == 0) + if (p == 0 || !IsAppMem((uptr)p)) return 0; MBlock *b = ctx->metamap.GetBlock((uptr)p); if (!b) @@ -421,8 +449,6 @@ uptr __sanitizer_get_allocated_size(const void *p) { void __tsan_on_thread_idle() { ThreadState *thr = cur_thread(); - thr->clock.ResetCached(&thr->proc()->clock_cache); - thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); allocator()->SwallowCache(&thr->proc()->alloc_cache); internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache); ctx->metamap.OnProcIdle(thr->proc()); diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.h b/compiler-rt/lib/tsan/rtl/tsan_mman.h index db8488eabbe2..2095f28c0253 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mman.h +++ b/compiler-rt/lib/tsan/rtl/tsan_mman.h @@ -26,6 +26,8 @@ void AllocatorProcFinish(Processor *proc); void AllocatorPrintStats(); void AllocatorLock(); void AllocatorUnlock(); +void GlobalProcessorLock(); +void GlobalProcessorUnlock(); // For user allocations. void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, diff --git a/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp b/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp index 735179686ba9..3a75b80ac30f 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp @@ -19,57 +19,7 @@ namespace __tsan { MutexSet::MutexSet() { } -void MutexSet::Add(u64 id, bool write, u64 epoch) { - // Look up existing mutex with the same id. - for (uptr i = 0; i < size_; i++) { - if (descs_[i].id == id) { - descs_[i].count++; - descs_[i].epoch = epoch; - return; - } - } - // On overflow, find the oldest mutex and drop it. - if (size_ == kMaxSize) { - u64 minepoch = (u64)-1; - u64 mini = (u64)-1; - for (uptr i = 0; i < size_; i++) { - if (descs_[i].epoch < minepoch) { - minepoch = descs_[i].epoch; - mini = i; - } - } - RemovePos(mini); - CHECK_EQ(size_, kMaxSize - 1); - } - // Add new mutex descriptor. - descs_[size_].addr = 0; - descs_[size_].stack_id = kInvalidStackID; - descs_[size_].id = id; - descs_[size_].write = write; - descs_[size_].epoch = epoch; - descs_[size_].seq = seq_++; - descs_[size_].count = 1; - size_++; -} - -void MutexSet::Del(u64 id, bool write) { - for (uptr i = 0; i < size_; i++) { - if (descs_[i].id == id) { - if (--descs_[i].count == 0) - RemovePos(i); - return; - } - } -} - -void MutexSet::Remove(u64 id) { - for (uptr i = 0; i < size_; i++) { - if (descs_[i].id == id) { - RemovePos(i); - return; - } - } -} +void MutexSet::Reset() { internal_memset(this, 0, sizeof(*this)); } void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) { // Look up existing mutex with the same id. @@ -93,9 +43,7 @@ void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) { // Add new mutex descriptor. descs_[size_].addr = addr; descs_[size_].stack_id = stack_id; - descs_[size_].id = 0; descs_[size_].write = write; - descs_[size_].epoch = 0; descs_[size_].seq = seq_++; descs_[size_].count = 1; size_++; diff --git a/compiler-rt/lib/tsan/rtl/tsan_mutexset.h b/compiler-rt/lib/tsan/rtl/tsan_mutexset.h index 93776a664135..aabd361e6afd 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mutexset.h +++ b/compiler-rt/lib/tsan/rtl/tsan_mutexset.h @@ -25,8 +25,6 @@ class MutexSet { struct Desc { uptr addr; StackID stack_id; - u64 id; - u64 epoch; u32 seq; u32 count; bool write; @@ -40,10 +38,7 @@ class MutexSet { }; MutexSet(); - // The 'id' is obtained from SyncVar::GetId(). - void Add(u64 id, bool write, u64 epoch); - void Del(u64 id, bool write); - void Remove(u64 id); // Removes the mutex completely (if it's destroyed). + void Reset(); void AddAddr(uptr addr, StackID stack_id, bool write); void DelAddr(uptr addr, bool destroy = false); uptr Size() const; @@ -82,9 +77,7 @@ class DynamicMutexSet { // in different goroutine). #if SANITIZER_GO MutexSet::MutexSet() {} -void MutexSet::Add(u64 id, bool write, u64 epoch) {} -void MutexSet::Del(u64 id, bool write) {} -void MutexSet::Remove(u64 id) {} +void MutexSet::Reset() {} void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {} void MutexSet::DelAddr(uptr addr, bool destroy) {} uptr MutexSet::Size() const { return 0; } diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h index 7ff0acace8f6..233bf0a39df0 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -18,8 +18,8 @@ # error "Only 64-bit is supported" #endif +#include "sanitizer_common/sanitizer_common.h" #include "tsan_defs.h" -#include "tsan_trace.h" namespace __tsan { @@ -40,14 +40,12 @@ enum { C/C++ on linux/x86_64 and freebsd/x86_64 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB) 0040 0000 0000 - 0100 0000 0000: - -0100 0000 0000 - 2000 0000 0000: shadow -2000 0000 0000 - 3000 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 3000 0000 0000: - 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 4000 0000 0000 - 5500 0000 0000: - 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels -5680 0000 0000 - 6000 0000 0000: - -6000 0000 0000 - 6200 0000 0000: traces -6200 0000 0000 - 7d00 0000 0000: - +5680 0000 0000 - 7d00 0000 0000: - 7b00 0000 0000 - 7c00 0000 0000: heap 7c00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack @@ -67,10 +65,8 @@ C/C++ on netbsd/amd64 can reuse the same mapping: struct Mapping48AddressSpace { static const uptr kMetaShadowBeg = 0x300000000000ull; static const uptr kMetaShadowEnd = 0x340000000000ull; - static const uptr kTraceMemBeg = 0x600000000000ull; - static const uptr kTraceMemEnd = 0x620000000000ull; static const uptr kShadowBeg = 0x010000000000ull; - static const uptr kShadowEnd = 0x200000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; static const uptr kHeapMemBeg = 0x7b0000000000ull; static const uptr kHeapMemEnd = 0x7c0000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; @@ -89,14 +85,13 @@ struct Mapping48AddressSpace { C/C++ on linux/mips64 (40-bit VMA) 0000 0000 00 - 0100 0000 00: - (4 GB) 0100 0000 00 - 0200 0000 00: main binary (4 GB) -0200 0000 00 - 2000 0000 00: - (120 GB) -2000 0000 00 - 4000 0000 00: shadow (128 GB) +0200 0000 00 - 1200 0000 00: - (64 GB) +1200 0000 00 - 2200 0000 00: shadow (64 GB) +2200 0000 00 - 4000 0000 00: - (120 GB) 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB) 5000 0000 00 - aa00 0000 00: - (360 GB) aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB) -ab00 0000 00 - b000 0000 00: - (20 GB) -b000 0000 00 - b200 0000 00: traces (8 GB) -b200 0000 00 - fe00 0000 00: - (304 GB) +ab00 0000 00 - fe00 0000 00: - (332 GB) fe00 0000 00 - ff00 0000 00: heap (4 GB) ff00 0000 00 - ff80 0000 00: - (2 GB) ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB) @@ -104,10 +99,8 @@ ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB) struct MappingMips64_40 { static const uptr kMetaShadowBeg = 0x4000000000ull; static const uptr kMetaShadowEnd = 0x5000000000ull; - static const uptr kTraceMemBeg = 0xb000000000ull; - static const uptr kTraceMemEnd = 0xb200000000ull; - static const uptr kShadowBeg = 0x2000000000ull; - static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kShadowBeg = 0x1200000000ull; + static const uptr kShadowEnd = 0x2200000000ull; static const uptr kHeapMemBeg = 0xfe00000000ull; static const uptr kHeapMemEnd = 0xff00000000ull; static const uptr kLoAppMemBeg = 0x0100000000ull; @@ -128,12 +121,10 @@ C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) 0200 0000 00 - 0300 0000 00: heap (4 GB) 0300 0000 00 - 0400 0000 00: - (4 GB) -0400 0000 00 - 0c00 0000 00: shadow memory (32 GB) -0c00 0000 00 - 0d00 0000 00: - (4 GB) +0400 0000 00 - 0800 0000 00: shadow memory (16 GB) +0800 0000 00 - 0d00 0000 00: - (20 GB) 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) -0e00 0000 00 - 0f00 0000 00: - (4 GB) -0f00 0000 00 - 0fc0 0000 00: traces (3 GB) -0fc0 0000 00 - 1000 0000 00: - +0e00 0000 00 - 1000 0000 00: - */ struct MappingAppleAarch64 { static const uptr kLoAppMemBeg = 0x0100000000ull; @@ -141,16 +132,14 @@ struct MappingAppleAarch64 { static const uptr kHeapMemBeg = 0x0200000000ull; static const uptr kHeapMemEnd = 0x0300000000ull; static const uptr kShadowBeg = 0x0400000000ull; - static const uptr kShadowEnd = 0x0c00000000ull; + static const uptr kShadowEnd = 0x0800000000ull; static const uptr kMetaShadowBeg = 0x0d00000000ull; static const uptr kMetaShadowEnd = 0x0e00000000ull; - static const uptr kTraceMemBeg = 0x0f00000000ull; - static const uptr kTraceMemEnd = 0x0fc0000000ull; static const uptr kHiAppMemBeg = 0x0fc0000000ull; static const uptr kHiAppMemEnd = 0x0fc0000000ull; static const uptr kShadowMsk = 0x0ull; static const uptr kShadowXor = 0x0ull; - static const uptr kShadowAdd = 0x0ull; + static const uptr kShadowAdd = 0x0200000000ull; static const uptr kVdsoBeg = 0x7000000000000000ull; static const uptr kMidAppMemBeg = 0; static const uptr kMidAppMemEnd = 0; @@ -159,29 +148,25 @@ struct MappingAppleAarch64 { /* C/C++ on linux/aarch64 (39-bit VMA) 0000 0010 00 - 0100 0000 00: main binary -0100 0000 00 - 0800 0000 00: - -0800 0000 00 - 2000 0000 00: shadow memory +0100 0000 00 - 0400 0000 00: - +0400 0000 00 - 1000 0000 00: shadow memory 2000 0000 00 - 3100 0000 00: - 3100 0000 00 - 3400 0000 00: metainfo 3400 0000 00 - 5500 0000 00: - 5500 0000 00 - 5600 0000 00: main binary (PIE) -5600 0000 00 - 6000 0000 00: - -6000 0000 00 - 6200 0000 00: traces -6200 0000 00 - 7d00 0000 00: - +5600 0000 00 - 7c00 0000 00: - 7c00 0000 00 - 7d00 0000 00: heap 7d00 0000 00 - 7fff ffff ff: modules and main thread stack */ struct MappingAarch64_39 { static const uptr kLoAppMemBeg = 0x0000001000ull; static const uptr kLoAppMemEnd = 0x0100000000ull; - static const uptr kShadowBeg = 0x0800000000ull; - static const uptr kShadowEnd = 0x2000000000ull; + static const uptr kShadowBeg = 0x0400000000ull; + static const uptr kShadowEnd = 0x1000000000ull; static const uptr kMetaShadowBeg = 0x3100000000ull; static const uptr kMetaShadowEnd = 0x3400000000ull; static const uptr kMidAppMemBeg = 0x5500000000ull; - static const uptr kMidAppMemEnd = 0x5600000000ull; - static const uptr kTraceMemBeg = 0x6000000000ull; - static const uptr kTraceMemEnd = 0x6200000000ull; + static const uptr kMidAppMemEnd = 0x5600000000ull; static const uptr kHeapMemBeg = 0x7c00000000ull; static const uptr kHeapMemEnd = 0x7d00000000ull; static const uptr kHiAppMemBeg = 0x7e00000000ull; @@ -195,15 +180,13 @@ struct MappingAarch64_39 { /* C/C++ on linux/aarch64 (42-bit VMA) 00000 0010 00 - 01000 0000 00: main binary -01000 0000 00 - 10000 0000 00: - -10000 0000 00 - 20000 0000 00: shadow memory -20000 0000 00 - 26000 0000 00: - +01000 0000 00 - 08000 0000 00: - +08000 0000 00 - 10000 0000 00: shadow memory +10000 0000 00 - 26000 0000 00: - 26000 0000 00 - 28000 0000 00: metainfo 28000 0000 00 - 2aa00 0000 00: - 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) -2ab00 0000 00 - 36200 0000 00: - -36200 0000 00 - 36240 0000 00: traces -36240 0000 00 - 3e000 0000 00: - +2ab00 0000 00 - 3e000 0000 00: - 3e000 0000 00 - 3f000 0000 00: heap 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack */ @@ -211,14 +194,12 @@ struct MappingAarch64_42 { static const uptr kBroken = kBrokenReverseMapping; static const uptr kLoAppMemBeg = 0x00000001000ull; static const uptr kLoAppMemEnd = 0x01000000000ull; - static const uptr kShadowBeg = 0x10000000000ull; - static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kShadowBeg = 0x08000000000ull; + static const uptr kShadowEnd = 0x10000000000ull; static const uptr kMetaShadowBeg = 0x26000000000ull; static const uptr kMetaShadowEnd = 0x28000000000ull; static const uptr kMidAppMemBeg = 0x2aa00000000ull; - static const uptr kMidAppMemEnd = 0x2ab00000000ull; - static const uptr kTraceMemBeg = 0x36200000000ull; - static const uptr kTraceMemEnd = 0x36400000000ull; + static const uptr kMidAppMemEnd = 0x2ab00000000ull; static const uptr kHeapMemBeg = 0x3e000000000ull; static const uptr kHeapMemEnd = 0x3f000000000ull; static const uptr kHiAppMemBeg = 0x3f000000000ull; @@ -232,14 +213,12 @@ struct MappingAarch64_42 { struct MappingAarch64_48 { static const uptr kLoAppMemBeg = 0x0000000001000ull; static const uptr kLoAppMemEnd = 0x0000200000000ull; - static const uptr kShadowBeg = 0x0002000000000ull; - static const uptr kShadowEnd = 0x0004000000000ull; + static const uptr kShadowBeg = 0x0001000000000ull; + static const uptr kShadowEnd = 0x0002000000000ull; static const uptr kMetaShadowBeg = 0x0005000000000ull; static const uptr kMetaShadowEnd = 0x0006000000000ull; static const uptr kMidAppMemBeg = 0x0aaaa00000000ull; - static const uptr kMidAppMemEnd = 0x0aaaf00000000ull; - static const uptr kTraceMemBeg = 0x0f06000000000ull; - static const uptr kTraceMemEnd = 0x0f06200000000ull; + static const uptr kMidAppMemEnd = 0x0aaaf00000000ull; static const uptr kHeapMemBeg = 0x0ffff00000000ull; static const uptr kHeapMemEnd = 0x0ffff00000000ull; static const uptr kHiAppMemBeg = 0x0ffff00000000ull; @@ -257,9 +236,7 @@ C/C++ on linux/powerpc64 (44-bit VMA) 0001 0000 0000 - 0b00 0000 0000: shadow 0b00 0000 0000 - 0b00 0000 0000: - 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) -0d00 0000 0000 - 0d00 0000 0000: - -0d00 0000 0000 - 0f00 0000 0000: traces -0f00 0000 0000 - 0f00 0000 0000: - +0d00 0000 0000 - 0f00 0000 0000: - 0f00 0000 0000 - 0f50 0000 0000: heap 0f50 0000 0000 - 0f60 0000 0000: - 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack @@ -269,8 +246,6 @@ struct MappingPPC64_44 { kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity; static const uptr kMetaShadowBeg = 0x0b0000000000ull; static const uptr kMetaShadowEnd = 0x0d0000000000ull; - static const uptr kTraceMemBeg = 0x0d0000000000ull; - static const uptr kTraceMemEnd = 0x0f0000000000ull; static const uptr kShadowBeg = 0x000100000000ull; static const uptr kShadowEnd = 0x0b0000000000ull; static const uptr kLoAppMemBeg = 0x000000000100ull; @@ -291,23 +266,19 @@ struct MappingPPC64_44 { C/C++ on linux/powerpc64 (46-bit VMA) 0000 0000 1000 - 0100 0000 0000: main binary 0100 0000 0000 - 0200 0000 0000: - -0100 0000 0000 - 1000 0000 0000: shadow -1000 0000 0000 - 1000 0000 0000: - -1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) -2000 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 2200 0000 0000: traces -2200 0000 0000 - 3d00 0000 0000: - +0100 0000 0000 - 0800 0000 0000: shadow +0800 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) +1200 0000 0000 - 3d00 0000 0000: - 3d00 0000 0000 - 3e00 0000 0000: heap 3e00 0000 0000 - 3e80 0000 0000: - 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack */ struct MappingPPC64_46 { static const uptr kMetaShadowBeg = 0x100000000000ull; - static const uptr kMetaShadowEnd = 0x200000000000ull; - static const uptr kTraceMemBeg = 0x200000000000ull; - static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kMetaShadowEnd = 0x120000000000ull; static const uptr kShadowBeg = 0x010000000000ull; - static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kShadowEnd = 0x080000000000ull; static const uptr kHeapMemBeg = 0x3d0000000000ull; static const uptr kHeapMemEnd = 0x3e0000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; @@ -326,23 +297,19 @@ struct MappingPPC64_46 { C/C++ on linux/powerpc64 (47-bit VMA) 0000 0000 1000 - 0100 0000 0000: main binary 0100 0000 0000 - 0200 0000 0000: - -0100 0000 0000 - 1000 0000 0000: shadow -1000 0000 0000 - 1000 0000 0000: - -1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) -2000 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 2200 0000 0000: traces -2200 0000 0000 - 7d00 0000 0000: - +0100 0000 0000 - 0800 0000 0000: shadow +0800 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) +1200 0000 0000 - 7d00 0000 0000: - 7d00 0000 0000 - 7e00 0000 0000: heap 7e00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack */ struct MappingPPC64_47 { static const uptr kMetaShadowBeg = 0x100000000000ull; - static const uptr kMetaShadowEnd = 0x200000000000ull; - static const uptr kTraceMemBeg = 0x200000000000ull; - static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kMetaShadowEnd = 0x120000000000ull; static const uptr kShadowBeg = 0x010000000000ull; - static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kShadowEnd = 0x080000000000ull; static const uptr kHeapMemBeg = 0x7d0000000000ull; static const uptr kHeapMemEnd = 0x7e0000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; @@ -362,22 +329,18 @@ C/C++ on linux/s390x While the kernel provides a 64-bit address space, we have to restrict ourselves to 48 bits due to how e.g. SyncVar::GetId() works. 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB -0e00 0000 0000 - 4000 0000 0000: - -4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app) -8000 0000 0000 - 9000 0000 0000: - +0e00 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app) +4000 0000 0000 - 9000 0000 0000: - 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) -9800 0000 0000 - a000 0000 0000: - -a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads) -b000 0000 0000 - be00 0000 0000: - +9800 0000 0000 - be00 0000 0000: - be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator) */ struct MappingS390x { static const uptr kMetaShadowBeg = 0x900000000000ull; static const uptr kMetaShadowEnd = 0x980000000000ull; - static const uptr kTraceMemBeg = 0xa00000000000ull; - static const uptr kTraceMemEnd = 0xb00000000000ull; - static const uptr kShadowBeg = 0x400000000000ull; - static const uptr kShadowEnd = 0x800000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x400000000000ull; static const uptr kHeapMemBeg = 0xbe0000000000ull; static const uptr kHeapMemEnd = 0xc00000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; @@ -397,21 +360,17 @@ struct MappingS390x { 0000 1000 0000 - 00c0 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 2380 0000 0000: shadow -2380 0000 0000 - 3000 0000 0000: - +2000 0000 0000 - 21c0 0000 0000: shadow +21c0 0000 0000 - 3000 0000 0000: - 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) -4000 0000 0000 - 6000 0000 0000: - -6000 0000 0000 - 6200 0000 0000: traces -6200 0000 0000 - 8000 0000 0000: - +4000 0000 0000 - 8000 0000 0000: - */ struct MappingGo48 { static const uptr kMetaShadowBeg = 0x300000000000ull; static const uptr kMetaShadowEnd = 0x400000000000ull; - static const uptr kTraceMemBeg = 0x600000000000ull; - static const uptr kTraceMemEnd = 0x620000000000ull; static const uptr kShadowBeg = 0x200000000000ull; - static const uptr kShadowEnd = 0x238000000000ull; + static const uptr kShadowEnd = 0x21c000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x00e000000000ull; static const uptr kMidAppMemBeg = 0; @@ -431,8 +390,8 @@ struct MappingGo48 { 0000 1000 0000 - 00f8 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 0100 0000 0000: - -0100 0000 0000 - 0500 0000 0000: shadow -0500 0000 0000 - 0700 0000 0000: traces +0100 0000 0000 - 0300 0000 0000: shadow +0300 0000 0000 - 0700 0000 0000: - 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects) 07d0 0000 0000 - 8000 0000 0000: - */ @@ -440,10 +399,8 @@ struct MappingGo48 { struct MappingGoWindows { static const uptr kMetaShadowBeg = 0x070000000000ull; static const uptr kMetaShadowEnd = 0x077000000000ull; - static const uptr kTraceMemBeg = 0x050000000000ull; - static const uptr kTraceMemEnd = 0x070000000000ull; static const uptr kShadowBeg = 0x010000000000ull; - static const uptr kShadowEnd = 0x050000000000ull; + static const uptr kShadowEnd = 0x030000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x00e000000000ull; static const uptr kMidAppMemBeg = 0; @@ -463,21 +420,17 @@ struct MappingGoWindows { 0000 1000 0000 - 00c0 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 2380 0000 0000: shadow -2380 0000 0000 - 2400 0000 0000: - -2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects) -3400 0000 0000 - 3600 0000 0000: - -3600 0000 0000 - 3800 0000 0000: traces -3800 0000 0000 - 4000 0000 0000: - +2000 0000 0000 - 21c0 0000 0000: shadow +21c0 0000 0000 - 2400 0000 0000: - +2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects) +2470 0000 0000 - 4000 0000 0000: - */ struct MappingGoPPC64_46 { static const uptr kMetaShadowBeg = 0x240000000000ull; - static const uptr kMetaShadowEnd = 0x340000000000ull; - static const uptr kTraceMemBeg = 0x360000000000ull; - static const uptr kTraceMemEnd = 0x380000000000ull; + static const uptr kMetaShadowEnd = 0x247000000000ull; static const uptr kShadowBeg = 0x200000000000ull; - static const uptr kShadowEnd = 0x238000000000ull; + static const uptr kShadowEnd = 0x21c000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x00e000000000ull; static const uptr kMidAppMemBeg = 0; @@ -497,21 +450,17 @@ struct MappingGoPPC64_46 { 0000 1000 0000 - 00c0 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 3000 0000 0000: shadow -3000 0000 0000 - 3000 0000 0000: - -3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) -4000 0000 0000 - 6000 0000 0000: - -6000 0000 0000 - 6200 0000 0000: traces -6200 0000 0000 - 8000 0000 0000: - +2000 0000 0000 - 2800 0000 0000: shadow +2800 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) +3200 0000 0000 - 8000 0000 0000: - */ struct MappingGoPPC64_47 { static const uptr kMetaShadowBeg = 0x300000000000ull; - static const uptr kMetaShadowEnd = 0x400000000000ull; - static const uptr kTraceMemBeg = 0x600000000000ull; - static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kMetaShadowEnd = 0x320000000000ull; static const uptr kShadowBeg = 0x200000000000ull; - static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kShadowEnd = 0x280000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x00e000000000ull; static const uptr kMidAppMemBeg = 0; @@ -531,20 +480,16 @@ struct MappingGoPPC64_47 { 0000 1000 0000 - 00c0 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 3000 0000 0000: shadow -3000 0000 0000 - 3000 0000 0000: - -3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) -4000 0000 0000 - 6000 0000 0000: - -6000 0000 0000 - 6200 0000 0000: traces -6200 0000 0000 - 8000 0000 0000: - +2000 0000 0000 - 2800 0000 0000: shadow +2800 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) +3200 0000 0000 - 8000 0000 0000: - */ struct MappingGoAarch64 { static const uptr kMetaShadowBeg = 0x300000000000ull; - static const uptr kMetaShadowEnd = 0x400000000000ull; - static const uptr kTraceMemBeg = 0x600000000000ull; - static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kMetaShadowEnd = 0x320000000000ull; static const uptr kShadowBeg = 0x200000000000ull; - static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kShadowEnd = 0x280000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x00e000000000ull; static const uptr kMidAppMemBeg = 0; @@ -565,20 +510,16 @@ Go on linux/mips64 (47-bit VMA) 0000 1000 0000 - 00c0 0000 0000: - 00c0 0000 0000 - 00e0 0000 0000: heap 00e0 0000 0000 - 2000 0000 0000: - -2000 0000 0000 - 3000 0000 0000: shadow -3000 0000 0000 - 3000 0000 0000: - -3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) -4000 0000 0000 - 6000 0000 0000: - -6000 0000 0000 - 6200 0000 0000: traces -6200 0000 0000 - 8000 0000 0000: - +2000 0000 0000 - 2800 0000 0000: shadow +2800 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) +3200 0000 0000 - 8000 0000 0000: - */ struct MappingGoMips64_47 { static const uptr kMetaShadowBeg = 0x300000000000ull; - static const uptr kMetaShadowEnd = 0x400000000000ull; - static const uptr kTraceMemBeg = 0x600000000000ull; - static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kMetaShadowEnd = 0x320000000000ull; static const uptr kShadowBeg = 0x200000000000ull; - static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kShadowEnd = 0x280000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x00e000000000ull; static const uptr kMidAppMemBeg = 0; @@ -597,19 +538,15 @@ struct MappingGoMips64_47 { Go on linux/s390x 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB 1000 0000 0000 - 4000 0000 0000: - -4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app) -8000 0000 0000 - 9000 0000 0000: - +4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app) +6000 0000 0000 - 9000 0000 0000: - 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) -9800 0000 0000 - a000 0000 0000: - -a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads) */ struct MappingGoS390x { static const uptr kMetaShadowBeg = 0x900000000000ull; static const uptr kMetaShadowEnd = 0x980000000000ull; - static const uptr kTraceMemBeg = 0xa00000000000ull; - static const uptr kTraceMemEnd = 0xb00000000000ull; static const uptr kShadowBeg = 0x400000000000ull; - static const uptr kShadowEnd = 0x800000000000ull; + static const uptr kShadowEnd = 0x600000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; static const uptr kLoAppMemEnd = 0x100000000000ull; static const uptr kMidAppMemBeg = 0; @@ -648,11 +585,11 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) { return Func::template Apply(arg); # endif #else // SANITIZER_GO -# if defined(__x86_64__) || SANITIZER_IOSSIM || SANITIZER_MAC && !SANITIZER_IOS - return Func::template Apply(arg); -# elif defined(__aarch64__) && defined(__APPLE__) +# if SANITIZER_IOS && !SANITIZER_IOSSIM return Func::template Apply(arg); -# elif defined(__aarch64__) && !defined(__APPLE__) +# elif defined(__x86_64__) || SANITIZER_MAC + return Func::template Apply(arg); +# elif defined(__aarch64__) switch (vmaSize) { case 39: return Func::template Apply(arg); @@ -715,8 +652,6 @@ enum MappingType { kShadowEnd, kMetaShadowBeg, kMetaShadowEnd, - kTraceMemBeg, - kTraceMemEnd, kVdsoBeg, }; @@ -750,10 +685,6 @@ struct MappingField { return Mapping::kMetaShadowBeg; case kMetaShadowEnd: return Mapping::kMetaShadowEnd; - case kTraceMemBeg: - return Mapping::kTraceMemBeg; - case kTraceMemEnd: - return Mapping::kTraceMemEnd; } Die(); } @@ -792,11 +723,6 @@ uptr MetaShadowBeg(void) { return SelectMapping(kMetaShadowBeg); } ALWAYS_INLINE uptr MetaShadowEnd(void) { return SelectMapping(kMetaShadowEnd); } -ALWAYS_INLINE -uptr TraceMemBeg(void) { return SelectMapping(kTraceMemBeg); } -ALWAYS_INLINE -uptr TraceMemEnd(void) { return SelectMapping(kTraceMemEnd); } - struct IsAppMemImpl { template static bool Apply(uptr mem) { @@ -934,43 +860,10 @@ inline uptr RestoreAddr(uptr addr) { return SelectMapping(addr); } -// The additional page is to catch shadow stack overflow as paging fault. -// Windows wants 64K alignment for mmaps. -const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) - + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); - -struct GetThreadTraceImpl { - template - static uptr Apply(uptr tid) { - uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize; - DCHECK_LT(p, Mapping::kTraceMemEnd); - return p; - } -}; - -ALWAYS_INLINE -uptr GetThreadTrace(int tid) { return SelectMapping(tid); } - -struct GetThreadTraceHeaderImpl { - template - static uptr Apply(uptr tid) { - uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize + - kTraceSize * sizeof(Event); - DCHECK_LT(p, Mapping::kTraceMemEnd); - return p; - } -}; - -ALWAYS_INLINE -uptr GetThreadTraceHeader(int tid) { - return SelectMapping(tid); -} - void InitializePlatform(); void InitializePlatformEarly(); void CheckAndProtect(); void InitializeShadowMemoryPlatform(); -void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns); int ExtractResolvFDs(void *state, int *fds, int nfd); int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp index 73ec14892d28..17dbdff8a539 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp @@ -94,7 +94,6 @@ enum { MemMeta, MemFile, MemMmap, - MemTrace, MemHeap, MemOther, MemCount, @@ -112,8 +111,6 @@ void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) { mem[file ? MemFile : MemMmap] += rss; else if (p >= HeapMemBeg() && p < HeapMemEnd()) mem[MemHeap] += rss; - else if (p >= TraceMemBeg() && p < TraceMemEnd()) - mem[MemTrace] += rss; else mem[MemOther] += rss; } @@ -126,42 +123,33 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { StackDepotStats stacks = StackDepotGetStats(); uptr nthread, nlive; ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive); + uptr trace_mem; + { + Lock l(&ctx->slot_mtx); + trace_mem = ctx->trace_part_total_allocated * sizeof(TracePart); + } uptr internal_stats[AllocatorStatCount]; internal_allocator()->GetStats(internal_stats); // All these are allocated from the common mmap region. - mem[MemMmap] -= meta.mem_block + meta.sync_obj + stacks.allocated + - internal_stats[AllocatorStatMapped]; + mem[MemMmap] -= meta.mem_block + meta.sync_obj + trace_mem + + stacks.allocated + internal_stats[AllocatorStatMapped]; if (s64(mem[MemMmap]) < 0) mem[MemMmap] = 0; internal_snprintf( buf, buf_size, - "%llus: RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd" - " trace:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu" - " stacks=%zd[%zd] nthr=%zd/%zd\n", - uptime_ns / (1000 * 1000 * 1000), mem[MemTotal] >> 20, - mem[MemShadow] >> 20, mem[MemMeta] >> 20, mem[MemFile] >> 20, - mem[MemMmap] >> 20, mem[MemTrace] >> 20, mem[MemHeap] >> 20, + "==%zu== %llus [%zu]: RSS %zd MB: shadow:%zd meta:%zd file:%zd" + " mmap:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu" + " trace:%zu stacks=%zd threads=%zu/%zu\n", + internal_getpid(), uptime_ns / (1000 * 1000 * 1000), ctx->global_epoch, + mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20, + mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemHeap] >> 20, mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20, - meta.mem_block >> 20, meta.sync_obj >> 20, stacks.allocated >> 20, - stacks.n_uniq_ids, nlive, nthread); -} - -# if SANITIZER_LINUX -void FlushShadowMemoryCallback( - const SuspendedThreadsList &suspended_threads_list, - void *argument) { - ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd()); -} -#endif - -void FlushShadowMemory() { -#if SANITIZER_LINUX - StopTheWorld(FlushShadowMemoryCallback, 0); -#endif + meta.mem_block >> 20, meta.sync_obj >> 20, trace_mem >> 20, + stacks.allocated >> 20, nlive, nthread); } #if !SANITIZER_GO -// Mark shadow for .rodata sections with the special kShadowRodata marker. +// Mark shadow for .rodata sections with the special Shadow::kRodata marker. // Accesses to .rodata can't race, so this saves time, memory and trace space. static void MapRodata() { // First create temp file. @@ -182,13 +170,13 @@ static void MapRodata() { return; internal_unlink(name); // Unlink it now, so that we can reuse the buffer. fd_t fd = openrv; - // Fill the file with kShadowRodata. + // Fill the file with Shadow::kRodata. const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow); InternalMmapVector marker(kMarkerSize); // volatile to prevent insertion of memset for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize; p++) - *p = kShadowRodata; + *p = Shadow::kRodata; internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow)); // Map the file into memory. uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE, diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp index 1465f9953c19..44b98d46cfbc 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp @@ -126,9 +126,6 @@ void cur_thread_finalize() { } #endif -void FlushShadowMemory() { -} - static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) { vm_address_t address = start; vm_address_t end_address = end; @@ -156,12 +153,10 @@ static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) { void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { uptr shadow_res, shadow_dirty; uptr meta_res, meta_dirty; - uptr trace_res, trace_dirty; RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty); RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty); - RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty); -#if !SANITIZER_GO +# if !SANITIZER_GO uptr low_res, low_dirty; uptr high_res, high_dirty; uptr heap_res, heap_dirty; @@ -180,7 +175,6 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { buf, buf_size, "shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" "meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" - "traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" # if !SANITIZER_GO "low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n" @@ -193,7 +187,6 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { "------------------------------\n", ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024, MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024, - TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024, # if !SANITIZER_GO LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024, HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024, diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp index 763ac444377e..71874aad8dc5 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp @@ -110,27 +110,23 @@ void CheckAndProtect() { Die(); } -# if defined(__aarch64__) && defined(__APPLE__) && SANITIZER_IOS +# if SANITIZER_IOS && !SANITIZER_IOSSIM ProtectRange(HeapMemEnd(), ShadowBeg()); ProtectRange(ShadowEnd(), MetaShadowBeg()); - ProtectRange(MetaShadowEnd(), TraceMemBeg()); -#else + ProtectRange(MetaShadowEnd(), HiAppMemBeg()); +# else ProtectRange(LoAppMemEnd(), ShadowBeg()); ProtectRange(ShadowEnd(), MetaShadowBeg()); if (MidAppMemBeg()) { ProtectRange(MetaShadowEnd(), MidAppMemBeg()); - ProtectRange(MidAppMemEnd(), TraceMemBeg()); + ProtectRange(MidAppMemEnd(), HeapMemBeg()); } else { - ProtectRange(MetaShadowEnd(), TraceMemBeg()); + ProtectRange(MetaShadowEnd(), HeapMemBeg()); } - // Memory for traces is mapped lazily in MapThreadTrace. - // Protect the whole range for now, so that user does not map something here. - ProtectRange(TraceMemBeg(), TraceMemEnd()); - ProtectRange(TraceMemEnd(), HeapMemBeg()); ProtectRange(HeapEnd(), HiAppMemBeg()); -#endif +# endif -#if defined(__s390x__) +# if defined(__s390x__) // Protect the rest of the address space. const uptr user_addr_max_l4 = 0x0020000000000000ull; const uptr user_addr_max_l5 = 0xfffffffffffff000ull; diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp index fea893768c79..eb8f354742f4 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp @@ -20,9 +20,6 @@ namespace __tsan { -void FlushShadowMemory() { -} - void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {} void InitializePlatformEarly() { diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp index a926c3761ccf..10d9c761b8ee 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp @@ -126,7 +126,7 @@ static void PrintMutexSet(Vector const& mset) { if (i == 0) Printf(" (mutexes:"); const ReportMopMutex m = mset[i]; - Printf(" %s M%llu", m.write ? "write" : "read", m.id); + Printf(" %s M%u", m.write ? "write" : "read", m.id); Printf(i == mset.Size() - 1 ? ")" : ","); } } @@ -211,29 +211,23 @@ static void PrintLocation(const ReportLocation *loc) { static void PrintMutexShort(const ReportMutex *rm, const char *after) { Decorator d; - Printf("%sM%lld%s%s", d.Mutex(), rm->id, d.Default(), after); + Printf("%sM%d%s%s", d.Mutex(), rm->id, d.Default(), after); } static void PrintMutexShortWithAddress(const ReportMutex *rm, const char *after) { Decorator d; - Printf("%sM%lld (%p)%s%s", d.Mutex(), rm->id, + Printf("%sM%d (%p)%s%s", d.Mutex(), rm->id, reinterpret_cast(rm->addr), d.Default(), after); } static void PrintMutex(const ReportMutex *rm) { Decorator d; - if (rm->destroyed) { - Printf("%s", d.Mutex()); - Printf(" Mutex M%llu is already destroyed.\n\n", rm->id); - Printf("%s", d.Default()); - } else { - Printf("%s", d.Mutex()); - Printf(" Mutex M%llu (%p) created at:\n", rm->id, - reinterpret_cast(rm->addr)); - Printf("%s", d.Default()); - PrintStack(rm->stack); - } + Printf("%s", d.Mutex()); + Printf(" Mutex M%u (%p) created at:\n", rm->id, + reinterpret_cast(rm->addr)); + Printf("%s", d.Default()); + PrintStack(rm->stack); } static void PrintThread(const ReportThread *rt) { @@ -460,12 +454,12 @@ void PrintReport(const ReportDesc *rep) { } else if (rep->typ == ReportTypeDeadlock) { Printf("WARNING: DEADLOCK\n"); for (uptr i = 0; i < rep->mutexes.Size(); i++) { - Printf("Goroutine %d lock mutex %llu while holding mutex %llu:\n", 999, + Printf("Goroutine %d lock mutex %u while holding mutex %u:\n", 999, rep->mutexes[i]->id, rep->mutexes[(i + 1) % rep->mutexes.Size()]->id); PrintStack(rep->stacks[2*i]); Printf("\n"); - Printf("Mutex %llu was previously locked here:\n", + Printf("Mutex %u was previously locked here:\n", rep->mutexes[(i + 1) % rep->mutexes.Size()]->id); PrintStack(rep->stacks[2*i + 1]); Printf("\n"); diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.h b/compiler-rt/lib/tsan/rtl/tsan_report.h index d68c2db88828..3b367f38e266 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.h +++ b/compiler-rt/lib/tsan/rtl/tsan_report.h @@ -43,7 +43,7 @@ struct ReportStack { }; struct ReportMopMutex { - u64 id; + int id; bool write; }; @@ -91,9 +91,8 @@ struct ReportThread { }; struct ReportMutex { - u64 id; + int id; uptr addr; - bool destroyed; ReportStack *stack; }; diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp index c14af9788e32..ed60e250cff8 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp @@ -57,110 +57,352 @@ Context *ctx; bool OnFinalize(bool failed); void OnInitialize(); #else -#include SANITIZER_WEAK_CXX_DEFAULT_IMPL bool OnFinalize(bool failed) { -#if !SANITIZER_GO +# if !SANITIZER_GO if (on_finalize) return on_finalize(failed); -#endif +# endif return failed; } + SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnInitialize() { -#if !SANITIZER_GO +# if !SANITIZER_GO if (on_initialize) on_initialize(); -#endif +# endif } #endif -static ThreadContextBase *CreateThreadContext(Tid tid) { - // Map thread trace when context is created. - char name[50]; - internal_snprintf(name, sizeof(name), "trace %u", tid); - MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name); - const uptr hdr = GetThreadTraceHeader(tid); - internal_snprintf(name, sizeof(name), "trace header %u", tid); - MapThreadTrace(hdr, sizeof(Trace), name); - new((void*)hdr) Trace(); - // We are going to use only a small part of the trace with the default - // value of history_size. However, the constructor writes to the whole trace. - // Release the unused part. - uptr hdr_end = hdr + sizeof(Trace); - hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts()); - hdr_end = RoundUp(hdr_end, GetPageSizeCached()); - if (hdr_end < hdr + sizeof(Trace)) { - ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace)); - uptr unused = hdr + sizeof(Trace) - hdr_end; - if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) { - Report("ThreadSanitizer: failed to mprotect [0x%zx-0x%zx) \n", hdr_end, - unused); - CHECK("unable to mprotect" && 0); +static TracePart* TracePartAlloc(ThreadState* thr) { + TracePart* part = nullptr; + { + Lock lock(&ctx->slot_mtx); + uptr max_parts = Trace::kMinParts + flags()->history_size; + Trace* trace = &thr->tctx->trace; + if (trace->parts_allocated == max_parts || + ctx->trace_part_finished_excess) { + part = ctx->trace_part_recycle.PopFront(); + DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part); + if (part && part->trace) { + Trace* trace1 = part->trace; + Lock trace_lock(&trace1->mtx); + part->trace = nullptr; + TracePart* part1 = trace1->parts.PopFront(); + CHECK_EQ(part, part1); + if (trace1->parts_allocated > trace1->parts.Size()) { + ctx->trace_part_finished_excess += + trace1->parts_allocated - trace1->parts.Size(); + trace1->parts_allocated = trace1->parts.Size(); + } + } + } + if (trace->parts_allocated < max_parts) { + trace->parts_allocated++; + if (ctx->trace_part_finished_excess) + ctx->trace_part_finished_excess--; + } + if (!part) + ctx->trace_part_total_allocated++; + else if (ctx->trace_part_recycle_finished) + ctx->trace_part_recycle_finished--; + } + if (!part) + part = new (MmapOrDie(sizeof(*part), "TracePart")) TracePart(); + return part; +} + +static void TracePartFree(TracePart* part) REQUIRES(ctx->slot_mtx) { + DCHECK(part->trace); + part->trace = nullptr; + ctx->trace_part_recycle.PushFront(part); +} + +void TraceResetForTesting() { + Lock lock(&ctx->slot_mtx); + while (auto* part = ctx->trace_part_recycle.PopFront()) { + if (auto trace = part->trace) + CHECK_EQ(trace->parts.PopFront(), part); + UnmapOrDie(part, sizeof(*part)); + } + ctx->trace_part_total_allocated = 0; + ctx->trace_part_recycle_finished = 0; + ctx->trace_part_finished_excess = 0; +} + +static void DoResetImpl(uptr epoch) { + ThreadRegistryLock lock0(&ctx->thread_registry); + Lock lock1(&ctx->slot_mtx); + CHECK_EQ(ctx->global_epoch, epoch); + ctx->global_epoch++; + CHECK(!ctx->resetting); + ctx->resetting = true; + for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) { + ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked( + static_cast(i)); + // Potentially we could purge all ThreadStatusDead threads from the + // registry. Since we reset all shadow, they can't race with anything + // anymore. However, their tid's can still be stored in some aux places + // (e.g. tid of thread that created something). + auto trace = &tctx->trace; + Lock lock(&trace->mtx); + bool attached = tctx->thr && tctx->thr->slot; + auto parts = &trace->parts; + bool local = false; + while (!parts->Empty()) { + auto part = parts->Front(); + local = local || part == trace->local_head; + if (local) + CHECK(!ctx->trace_part_recycle.Queued(part)); + else + ctx->trace_part_recycle.Remove(part); + if (attached && parts->Size() == 1) { + // The thread is running and this is the last/current part. + // Set the trace position to the end of the current part + // to force the thread to call SwitchTracePart and re-attach + // to a new slot and allocate a new trace part. + // Note: the thread is concurrently modifying the position as well, + // so this is only best-effort. The thread can only modify position + // within this part, because switching parts is protected by + // slot/trace mutexes that we hold here. + atomic_store_relaxed( + &tctx->thr->trace_pos, + reinterpret_cast(&part->events[TracePart::kSize])); + break; + } + parts->Remove(part); + TracePartFree(part); + } + CHECK_LE(parts->Size(), 1); + trace->local_head = parts->Front(); + if (tctx->thr && !tctx->thr->slot) { + atomic_store_relaxed(&tctx->thr->trace_pos, 0); + tctx->thr->trace_prev_pc = 0; + } + if (trace->parts_allocated > trace->parts.Size()) { + ctx->trace_part_finished_excess += + trace->parts_allocated - trace->parts.Size(); + trace->parts_allocated = trace->parts.Size(); } } - return New(tid); + while (ctx->slot_queue.PopFront()) { + } + for (auto& slot : ctx->slots) { + slot.SetEpoch(kEpochZero); + slot.journal.Reset(); + slot.thr = nullptr; + ctx->slot_queue.PushBack(&slot); + } + + DPrintf("Resetting shadow...\n"); + if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), + "shadow")) { + Printf("failed to reset shadow memory\n"); + Die(); + } + DPrintf("Resetting meta shadow...\n"); + ctx->metamap.ResetClocks(); + ctx->resetting = false; } +// Clang does not understand locking all slots in the loop: +// error: expecting mutex 'slot.mtx' to be held at start of each loop +void DoReset(ThreadState* thr, uptr epoch) NO_THREAD_SAFETY_ANALYSIS { + { + for (auto& slot : ctx->slots) { + slot.mtx.Lock(); + if (UNLIKELY(epoch == 0)) + epoch = ctx->global_epoch; + if (UNLIKELY(epoch != ctx->global_epoch)) { + // Epoch can't change once we've locked the first slot. + CHECK_EQ(slot.sid, 0); + slot.mtx.Unlock(); + return; + } + } + } + DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch); + DoResetImpl(epoch); + for (auto& slot : ctx->slots) slot.mtx.Unlock(); +} + +void FlushShadowMemory() { DoReset(nullptr, 0); } + +static TidSlot* FindSlotAndLock(ThreadState* thr) + ACQUIRE(thr->slot->mtx) NO_THREAD_SAFETY_ANALYSIS { + CHECK(!thr->slot); + TidSlot* slot = nullptr; + for (;;) { + uptr epoch; + { + Lock lock(&ctx->slot_mtx); + epoch = ctx->global_epoch; + if (slot) { + // This is an exhausted slot from the previous iteration. + if (ctx->slot_queue.Queued(slot)) + ctx->slot_queue.Remove(slot); + thr->slot_locked = false; + slot->mtx.Unlock(); + } + for (;;) { + slot = ctx->slot_queue.PopFront(); + if (!slot) + break; + if (slot->epoch() != kEpochLast) { + ctx->slot_queue.PushBack(slot); + break; + } + } + } + if (!slot) { + DoReset(thr, epoch); + continue; + } + slot->mtx.Lock(); + CHECK(!thr->slot_locked); + thr->slot_locked = true; + if (slot->thr) { + DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid, + slot->thr->tid); + slot->SetEpoch(slot->thr->fast_state.epoch()); + slot->thr = nullptr; + } + if (slot->epoch() != kEpochLast) + return slot; + } +} + +void SlotAttachAndLock(ThreadState* thr) { + TidSlot* slot = FindSlotAndLock(thr); + DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast(slot->sid)); + CHECK(!slot->thr); + CHECK(!thr->slot); + slot->thr = thr; + thr->slot = slot; + Epoch epoch = EpochInc(slot->epoch()); + CHECK(!EpochOverflow(epoch)); + slot->SetEpoch(epoch); + thr->fast_state.SetSid(slot->sid); + thr->fast_state.SetEpoch(epoch); + if (thr->slot_epoch != ctx->global_epoch) { + thr->slot_epoch = ctx->global_epoch; + thr->clock.Reset(); #if !SANITIZER_GO -static const u32 kThreadQuarantineSize = 16; -#else -static const u32 kThreadQuarantineSize = 64; + thr->last_sleep_stack_id = kInvalidStackID; + thr->last_sleep_clock.Reset(); #endif + } + thr->clock.Set(slot->sid, epoch); + slot->journal.PushBack({thr->tid, epoch}); +} + +static void SlotDetachImpl(ThreadState* thr, bool exiting) { + TidSlot* slot = thr->slot; + thr->slot = nullptr; + if (thr != slot->thr) { + slot = nullptr; // we don't own the slot anymore + if (thr->slot_epoch != ctx->global_epoch) { + TracePart* part = nullptr; + auto* trace = &thr->tctx->trace; + { + Lock l(&trace->mtx); + auto* parts = &trace->parts; + // The trace can be completely empty in an unlikely event + // the thread is preempted right after it acquired the slot + // in ThreadStart and did not trace any events yet. + CHECK_LE(parts->Size(), 1); + part = parts->PopFront(); + thr->tctx->trace.local_head = nullptr; + atomic_store_relaxed(&thr->trace_pos, 0); + thr->trace_prev_pc = 0; + } + if (part) { + Lock l(&ctx->slot_mtx); + TracePartFree(part); + } + } + return; + } + CHECK(exiting || thr->fast_state.epoch() == kEpochLast); + slot->SetEpoch(thr->fast_state.epoch()); + slot->thr = nullptr; +} + +void SlotDetach(ThreadState* thr) { + Lock lock(&thr->slot->mtx); + SlotDetachImpl(thr, true); +} + +void SlotLock(ThreadState* thr) NO_THREAD_SAFETY_ANALYSIS { + DCHECK(!thr->slot_locked); +#if SANITIZER_DEBUG + // Check these mutexes are not locked. + // We can call DoReset from SlotAttachAndLock, which will lock + // these mutexes, but it happens only every once in a while. + { ThreadRegistryLock lock(&ctx->thread_registry); } + { Lock lock(&ctx->slot_mtx); } +#endif + TidSlot* slot = thr->slot; + slot->mtx.Lock(); + thr->slot_locked = true; + if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast)) + return; + SlotDetachImpl(thr, false); + thr->slot_locked = false; + slot->mtx.Unlock(); + SlotAttachAndLock(thr); +} + +void SlotUnlock(ThreadState* thr) { + DCHECK(thr->slot_locked); + thr->slot_locked = false; + thr->slot->mtx.Unlock(); +} Context::Context() : initialized(), report_mtx(MutexTypeReport), nreported(), - thread_registry(CreateThreadContext, kMaxTid, kThreadQuarantineSize, - kMaxTidReuse), + thread_registry([](Tid tid) -> ThreadContextBase* { + return new (Alloc(sizeof(ThreadContext))) ThreadContext(tid); + }), racy_mtx(MutexTypeRacy), racy_stacks(), racy_addresses(), fired_suppressions_mtx(MutexTypeFired), - clock_alloc(LINKER_INITIALIZED, "clock allocator") { + slot_mtx(MutexTypeSlots), + resetting() { fired_suppressions.reserve(8); + for (uptr i = 0; i < ARRAY_SIZE(slots); i++) { + TidSlot* slot = &slots[i]; + slot->sid = static_cast(i); + slot_queue.PushBack(slot); + } + global_epoch = 1; } +TidSlot::TidSlot() : mtx(MutexTypeSlot) {} + // The objects are allocated in TLS, so one may rely on zero-initialization. -ThreadState::ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch, - unsigned reuse_count, uptr stk_addr, uptr stk_size, - uptr tls_addr, uptr tls_size) - : fast_state(tid, epoch) - // Do not touch these, rely on zero initialization, - // they may be accessed before the ctor. - // , ignore_reads_and_writes() - // , ignore_interceptors() - , - clock(tid, reuse_count) -#if !SANITIZER_GO - , - jmp_bufs() -#endif - , - tid(tid), - unique_id(unique_id), - stk_addr(stk_addr), - stk_size(stk_size), - tls_addr(tls_addr), - tls_size(tls_size) -#if !SANITIZER_GO - , - last_sleep_clock(tid) -#endif -{ +ThreadState::ThreadState(Tid tid) + // Do not touch these, rely on zero initialization, + // they may be accessed before the ctor. + // ignore_reads_and_writes() + // ignore_interceptors() + : tid(tid) { CHECK_EQ(reinterpret_cast(this) % SANITIZER_CACHE_LINE_SIZE, 0); #if !SANITIZER_GO // C/C++ uses fixed size shadow stack. const int kInitStackSize = kShadowStackSize; - shadow_stack = static_cast( + shadow_stack = static_cast( MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack")); SetShadowRegionHugePageMode(reinterpret_cast(shadow_stack), kInitStackSize * sizeof(uptr)); #else // Go uses malloc-allocated shadow stack with dynamic size. const int kInitStackSize = 8; - shadow_stack = static_cast(Alloc(kInitStackSize * sizeof(uptr))); + shadow_stack = static_cast(Alloc(kInitStackSize * sizeof(uptr))); #endif shadow_stack_pos = shadow_stack; shadow_stack_end = shadow_stack + kInitStackSize; @@ -175,11 +417,11 @@ void MemoryProfiler(u64 uptime) { WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data())); } -void InitializeMemoryProfiler() { +static bool InitializeMemoryProfiler() { ctx->memprof_fd = kInvalidFd; const char *fname = flags()->profile_memory; if (!fname || !fname[0]) - return; + return false; if (internal_strcmp(fname, "stdout") == 0) { ctx->memprof_fd = 1; } else if (internal_strcmp(fname, "stderr") == 0) { @@ -191,11 +433,11 @@ void InitializeMemoryProfiler() { if (ctx->memprof_fd == kInvalidFd) { Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", filename.data()); - return; + return false; } } MemoryProfiler(0); - MaybeSpawnBackgroundThread(); + return true; } static void *BackgroundThread(void *arg) { @@ -207,33 +449,34 @@ static void *BackgroundThread(void *arg) { const u64 kMs2Ns = 1000 * 1000; const u64 start = NanoTime(); - u64 last_flush = NanoTime(); + u64 last_flush = start; uptr last_rss = 0; - for (int i = 0; - atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0; - i++) { + while (!atomic_load_relaxed(&ctx->stop_background_thread)) { SleepForMillis(100); u64 now = NanoTime(); // Flush memory if requested. if (flags()->flush_memory_ms > 0) { if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) { - VPrintf(1, "ThreadSanitizer: periodic memory flush\n"); + VReport(1, "ThreadSanitizer: periodic memory flush\n"); FlushShadowMemory(); - last_flush = NanoTime(); + now = last_flush = NanoTime(); } } if (flags()->memory_limit_mb > 0) { uptr rss = GetRSS(); uptr limit = uptr(flags()->memory_limit_mb) << 20; - VPrintf(1, "ThreadSanitizer: memory flush check" - " RSS=%llu LAST=%llu LIMIT=%llu\n", + VReport(1, + "ThreadSanitizer: memory flush check" + " RSS=%llu LAST=%llu LIMIT=%llu\n", (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20); if (2 * rss > limit + last_rss) { - VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n"); + VReport(1, "ThreadSanitizer: flushing memory due to RSS\n"); FlushShadowMemory(); rss = GetRSS(); - VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20); + now = NanoTime(); + VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n", + (u64)rss >> 20); } last_rss = rss; } @@ -309,7 +552,8 @@ void UnmapShadow(ThreadState *thr, uptr addr, uptr size) { return; DontNeedShadowFor(addr, size); ScopedGlobalProcessor sgp; - ctx->metamap.ResetRange(thr->proc(), addr, size); + SlotLocker locker(thr, true); + ctx->metamap.ResetRange(thr->proc(), addr, size, true); } #endif @@ -355,18 +599,6 @@ void MapShadow(uptr addr, uptr size) { addr + size, meta_begin, meta_end); } -void MapThreadTrace(uptr addr, uptr size, const char *name) { - DPrintf("#0: Mapping trace at 0x%zx-0x%zx(0x%zx)\n", addr, addr + size, size); - CHECK_GE(addr, TraceMemBeg()); - CHECK_LE(addr + size, TraceMemEnd()); - CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment - if (!MmapFixedSuperNoReserve(addr, size, name)) { - Printf("FATAL: ThreadSanitizer can not mmap thread trace (0x%zx/0x%zx)\n", - addr, size); - Die(); - } -} - #if !SANITIZER_GO static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { @@ -385,8 +617,11 @@ void CheckUnwind() { // since we are going to die soon. ScopedIgnoreInterceptors ignore; #if !SANITIZER_GO - cur_thread()->ignore_sync++; - cur_thread()->ignore_reads_and_writes++; + ThreadState* thr = cur_thread(); + thr->nomalloc = false; + thr->ignore_sync++; + thr->ignore_reads_and_writes++; + atomic_store_relaxed(&thr->in_signal_handler, 0); #endif PrintCurrentStackSlow(StackTrace::GetCurrentPc()); } @@ -441,22 +676,23 @@ void Initialize(ThreadState *thr) { Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); #endif - VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", + VPrintf(1, "***** Running under ThreadSanitizer v3 (pid %d) *****\n", (int)internal_getpid()); // Initialize thread 0. - Tid tid = ThreadCreate(thr, 0, 0, true); + Tid tid = ThreadCreate(nullptr, 0, 0, true); CHECK_EQ(tid, kMainTid); ThreadStart(thr, tid, GetTid(), ThreadType::Regular); #if TSAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif - ctx->initialized = true; #if !SANITIZER_GO Symbolizer::LateInitialize(); - InitializeMemoryProfiler(); + if (InitializeMemoryProfiler() || flags()->force_background_thread) + MaybeSpawnBackgroundThread(); #endif + ctx->initialized = true; if (flags()->stop_on_start) { Printf("ThreadSanitizer is suspended at startup (pid %d)." @@ -482,7 +718,6 @@ void MaybeSpawnBackgroundThread() { #endif } - int Finalize(ThreadState *thr) { bool failed = false; @@ -490,12 +725,12 @@ int Finalize(ThreadState *thr) { DumpProcessMap(); if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1) - SleepForMillis(flags()->atexit_sleep_ms); + internal_usleep(u64(flags()->atexit_sleep_ms) * 1000); - // Wait for pending reports. - ctx->report_mtx.Lock(); - { ScopedErrorReportLock l; } - ctx->report_mtx.Unlock(); + { + // Wait for pending reports. + ScopedErrorReportLock lock; + } #if !SANITIZER_GO if (Verbosity()) AllocatorPrintStats(); @@ -522,8 +757,13 @@ int Finalize(ThreadState *thr) { #if !SANITIZER_GO void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { + GlobalProcessorLock(); + // Detaching from the slot makes OnUserFree skip writing to the shadow. + // The slot will be locked so any attempts to use it will deadlock anyway. + SlotDetach(thr); + for (auto& slot : ctx->slots) slot.mtx.Lock(); ctx->thread_registry.Lock(); - ctx->report_mtx.Lock(); + ctx->slot_mtx.Lock(); ScopedErrorReportLock::Lock(); AllocatorLock(); // Suppress all reports in the pthread_atfork callbacks. @@ -543,30 +783,29 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { __tsan_test_only_on_fork(); } -void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { +static void ForkAfter(ThreadState* thr) NO_THREAD_SAFETY_ANALYSIS { thr->suppress_reports--; // Enabled in ForkBefore. thr->ignore_interceptors--; thr->ignore_reads_and_writes--; AllocatorUnlock(); ScopedErrorReportLock::Unlock(); - ctx->report_mtx.Unlock(); + ctx->slot_mtx.Unlock(); ctx->thread_registry.Unlock(); + for (auto& slot : ctx->slots) slot.mtx.Unlock(); + SlotAttachAndLock(thr); + SlotUnlock(thr); + GlobalProcessorUnlock(); } -void ForkChildAfter(ThreadState *thr, uptr pc, - bool start_thread) NO_THREAD_SAFETY_ANALYSIS { - thr->suppress_reports--; // Enabled in ForkBefore. - thr->ignore_interceptors--; - thr->ignore_reads_and_writes--; - AllocatorUnlock(); - ScopedErrorReportLock::Unlock(); - ctx->report_mtx.Unlock(); - ctx->thread_registry.Unlock(); +void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); } - uptr nthread = 0; - ctx->thread_registry.GetNumberOfThreads(0, 0, &nthread /* alive threads */); - VPrintf(1, "ThreadSanitizer: forked new process with pid %d," - " parent had %d threads\n", (int)internal_getpid(), (int)nthread); +void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) { + ForkAfter(thr); + u32 nthread = ctx->thread_registry.OnFork(thr->tid); + VPrintf(1, + "ThreadSanitizer: forked new process with pid %d," + " parent had %d threads\n", + (int)internal_getpid(), (int)nthread); if (nthread == 1) { if (start_thread) StartBackgroundThread(); @@ -576,6 +815,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc, // ignores for everything in the hope that we will exec soon. ctx->after_multithreaded_fork = true; thr->ignore_interceptors++; + thr->suppress_reports++; ThreadIgnoreBegin(thr, pc); ThreadIgnoreSyncBegin(thr, pc); } @@ -597,8 +837,10 @@ void GrowShadowStack(ThreadState *thr) { #endif StackID CurrentStackId(ThreadState *thr, uptr pc) { +#if !SANITIZER_GO if (!thr->is_inited) // May happen during bootstrap. return kInvalidStackID; +#endif if (pc != 0) { #if !SANITIZER_GO DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); @@ -616,53 +858,72 @@ StackID CurrentStackId(ThreadState *thr, uptr pc) { return id; } -namespace v3 { - -NOINLINE -void TraceSwitchPart(ThreadState *thr) { +static bool TraceSkipGap(ThreadState* thr) { Trace *trace = &thr->tctx->trace; Event *pos = reinterpret_cast(atomic_load_relaxed(&thr->trace_pos)); DCHECK_EQ(reinterpret_cast(pos + 1) & TracePart::kAlignment, 0); auto *part = trace->parts.Back(); - DPrintf("TraceSwitchPart part=%p pos=%p\n", part, pos); - if (part) { - // We can get here when we still have space in the current trace part. - // The fast-path check in TraceAcquire has false positives in the middle of - // the part. Check if we are indeed at the end of the current part or not, - // and fill any gaps with NopEvent's. - Event *end = &part->events[TracePart::kSize]; - DCHECK_GE(pos, &part->events[0]); - DCHECK_LE(pos, end); - if (pos + 1 < end) { - if ((reinterpret_cast(pos) & TracePart::kAlignment) == - TracePart::kAlignment) - *pos++ = NopEvent; + DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid, + trace, trace->parts.Front(), part, pos); + if (!part) + return false; + // We can get here when we still have space in the current trace part. + // The fast-path check in TraceAcquire has false positives in the middle of + // the part. Check if we are indeed at the end of the current part or not, + // and fill any gaps with NopEvent's. + Event* end = &part->events[TracePart::kSize]; + DCHECK_GE(pos, &part->events[0]); + DCHECK_LE(pos, end); + if (pos + 1 < end) { + if ((reinterpret_cast(pos) & TracePart::kAlignment) == + TracePart::kAlignment) *pos++ = NopEvent; - DCHECK_LE(pos + 2, end); - atomic_store_relaxed(&thr->trace_pos, reinterpret_cast(pos)); - // Ensure we setup trace so that the next TraceAcquire - // won't detect trace part end. - Event *ev; - CHECK(TraceAcquire(thr, &ev)); - return; - } - // We are indeed at the end. - for (; pos < end; pos++) *pos = NopEvent; + *pos++ = NopEvent; + DCHECK_LE(pos + 2, end); + atomic_store_relaxed(&thr->trace_pos, reinterpret_cast(pos)); + return true; } + // We are indeed at the end. + for (; pos < end; pos++) *pos = NopEvent; + return false; +} + +NOINLINE +void TraceSwitchPart(ThreadState* thr) { + if (TraceSkipGap(thr)) + return; #if !SANITIZER_GO if (ctx->after_multithreaded_fork) { // We just need to survive till exec. - CHECK(part); - atomic_store_relaxed(&thr->trace_pos, - reinterpret_cast(&part->events[0])); - return; + TracePart* part = thr->tctx->trace.parts.Back(); + if (part) { + atomic_store_relaxed(&thr->trace_pos, + reinterpret_cast(&part->events[0])); + return; + } } #endif - part = new (MmapOrDie(sizeof(TracePart), "TracePart")) TracePart(); + TraceSwitchPartImpl(thr); +} + +void TraceSwitchPartImpl(ThreadState* thr) { + SlotLocker locker(thr, true); + Trace* trace = &thr->tctx->trace; + TracePart* part = TracePartAlloc(thr); part->trace = trace; thr->trace_prev_pc = 0; + TracePart* recycle = nullptr; + // Keep roughly half of parts local to the thread + // (not queued into the recycle queue). + uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2; { Lock lock(&trace->mtx); + if (trace->parts.Empty()) + trace->local_head = part; + if (trace->parts.Size() >= local_parts) { + recycle = trace->local_head; + trace->local_head = trace->parts.Next(recycle); + } trace->parts.PushBack(part); atomic_store_relaxed(&thr->trace_pos, reinterpret_cast(&part->events[0])); @@ -670,60 +931,49 @@ void TraceSwitchPart(ThreadState *thr) { // Make this part self-sufficient by restoring the current stack // and mutex set in the beginning of the trace. TraceTime(thr); - for (uptr *pos = &thr->shadow_stack[0]; pos < thr->shadow_stack_pos; pos++) - CHECK(TryTraceFunc(thr, *pos)); + { + // Pathologically large stacks may not fit into the part. + // In these cases we log only fixed number of top frames. + const uptr kMaxFrames = 1000; + // Sanity check that kMaxFrames won't consume the whole part. + static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big"); + uptr* pos = Max(&thr->shadow_stack[0], thr->shadow_stack_pos - kMaxFrames); + for (; pos < thr->shadow_stack_pos; pos++) { + if (TryTraceFunc(thr, *pos)) + continue; + CHECK(TraceSkipGap(thr)); + CHECK(TryTraceFunc(thr, *pos)); + } + } for (uptr i = 0; i < thr->mset.Size(); i++) { MutexSet::Desc d = thr->mset.Get(i); - TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0, - d.addr, d.stack_id); + for (uptr i = 0; i < d.count; i++) + TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0, + d.addr, d.stack_id); } + { + Lock lock(&ctx->slot_mtx); + // There is a small chance that the slot may be not queued at this point. + // This can happen if the slot has kEpochLast epoch and another thread + // in FindSlotAndLock discovered that it's exhausted and removed it from + // the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart + // was called with the slot locked and epoch already at kEpochLast, + // or (2) if we've acquired a new slot in SlotLock in the beginning + // of the function and the slot was at kEpochLast - 1, so after increment + // in SlotAttachAndLock it become kEpochLast. + if (ctx->slot_queue.Queued(thr->slot)) { + ctx->slot_queue.Remove(thr->slot); + ctx->slot_queue.PushBack(thr->slot); + } + if (recycle) + ctx->trace_part_recycle.PushBack(recycle); + } + DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid, + trace->parts.Front(), trace->parts.Back(), + atomic_load_relaxed(&thr->trace_pos)); } -} // namespace v3 - -void TraceSwitch(ThreadState *thr) { -#if !SANITIZER_GO - if (ctx->after_multithreaded_fork) - return; -#endif - thr->nomalloc++; - Trace *thr_trace = ThreadTrace(thr->tid); - Lock l(&thr_trace->mtx); - unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts(); - TraceHeader *hdr = &thr_trace->headers[trace]; - hdr->epoch0 = thr->fast_state.epoch(); - ObtainCurrentStack(thr, 0, &hdr->stack0); - hdr->mset0 = thr->mset; - thr->nomalloc--; -} - -Trace *ThreadTrace(Tid tid) { return (Trace *)GetThreadTraceHeader(tid); } - -uptr TraceTopPC(ThreadState *thr) { - Event *events = (Event*)GetThreadTrace(thr->tid); - uptr pc = events[thr->fast_state.GetTracePos()]; - return pc; -} - -uptr TraceSize() { - return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1)); -} - -uptr TraceParts() { - return TraceSize() / kTracePartSize; -} - -#if !SANITIZER_GO -extern "C" void __tsan_trace_switch() { - TraceSwitch(cur_thread()); -} - -extern "C" void __tsan_report_race() { - ReportRace(cur_thread()); -} -#endif - -void ThreadIgnoreBegin(ThreadState *thr, uptr pc) { +void ThreadIgnoreBegin(ThreadState* thr, uptr pc) { DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid); thr->ignore_reads_and_writes++; CHECK_GT(thr->ignore_reads_and_writes, 0); @@ -783,7 +1033,6 @@ void build_consistency_debug() {} #else void build_consistency_release() {} #endif - } // namespace __tsan #if SANITIZER_CHECK_DEADLOCKS @@ -791,21 +1040,27 @@ namespace __sanitizer { using namespace __tsan; MutexMeta mutex_meta[] = { {MutexInvalid, "Invalid", {}}, - {MutexThreadRegistry, "ThreadRegistry", {}}, - {MutexTypeTrace, "Trace", {}}, - {MutexTypeReport, - "Report", - {MutexTypeSyncVar, MutexTypeGlobalProc, MutexTypeTrace}}, - {MutexTypeSyncVar, "SyncVar", {MutexTypeTrace}}, + {MutexThreadRegistry, + "ThreadRegistry", + {MutexTypeSlots, MutexTypeTrace, MutexTypeReport}}, + {MutexTypeReport, "Report", {MutexTypeTrace}}, + {MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}}, {MutexTypeAnnotations, "Annotations", {}}, - {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}}, + {MutexTypeAtExit, "AtExit", {}}, {MutexTypeFired, "Fired", {MutexLeaf}}, {MutexTypeRacy, "Racy", {MutexLeaf}}, - {MutexTypeGlobalProc, "GlobalProc", {}}, + {MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}}, {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}}, + {MutexTypeTrace, "Trace", {}}, + {MutexTypeSlot, + "Slot", + {MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry, + MutexTypeSlots}}, + {MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}}, {}, }; void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); } + } // namespace __sanitizer #endif diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/compiler-rt/lib/tsan/rtl/tsan_rtl.h index c71b27e1cbf5..d06358b462eb 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -34,10 +34,10 @@ #include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" -#include "tsan_clock.h" #include "tsan_defs.h" #include "tsan_flags.h" #include "tsan_ignoreset.h" +#include "tsan_ilist.h" #include "tsan_mman.h" #include "tsan_mutexset.h" #include "tsan_platform.h" @@ -46,6 +46,7 @@ #include "tsan_stack_trace.h" #include "tsan_sync.h" #include "tsan_trace.h" +#include "tsan_vector_clock.h" #if SANITIZER_WORDSIZE != 64 # error "ThreadSanitizer is supported only on 64-bit platforms" @@ -116,7 +117,6 @@ struct Processor { #endif DenseSlabAllocCache block_cache; DenseSlabAllocCache sync_cache; - DenseSlabAllocCache clock_cache; DDPhysicalThread *dd_pt; }; @@ -130,30 +130,56 @@ struct ScopedGlobalProcessor { }; #endif +struct TidEpoch { + Tid tid; + Epoch epoch; +}; + +struct TidSlot { + Mutex mtx; + Sid sid; + atomic_uint32_t raw_epoch; + ThreadState *thr; + Vector journal; + INode node; + + Epoch epoch() const { + return static_cast(atomic_load(&raw_epoch, memory_order_relaxed)); + } + + void SetEpoch(Epoch v) { + atomic_store(&raw_epoch, static_cast(v), memory_order_relaxed); + } + + TidSlot(); +} ALIGNED(SANITIZER_CACHE_LINE_SIZE); + // This struct is stored in TLS. struct ThreadState { FastState fast_state; - // Synch epoch represents the threads's epoch before the last synchronization - // action. It allows to reduce number of shadow state updates. - // For example, fast_synch_epoch=100, last write to addr X was at epoch=150, - // if we are processing write to X from the same thread at epoch=200, - // we do nothing, because both writes happen in the same 'synch epoch'. - // That is, if another memory access does not race with the former write, - // it does not race with the latter as well. - // QUESTION: can we can squeeze this into ThreadState::Fast? - // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are - // taken by epoch between synchs. - // This way we can save one load from tls. - u64 fast_synch_epoch; + int ignore_sync; +#if !SANITIZER_GO + int ignore_interceptors; +#endif + uptr *shadow_stack_pos; + + // Current position in tctx->trace.Back()->events (Event*). + atomic_uintptr_t trace_pos; + // PC of the last memory access, used to compute PC deltas in the trace. + uptr trace_prev_pc; + // Technically `current` should be a separate THREADLOCAL variable; // but it is placed here in order to share cache line with previous fields. ThreadState* current; + + atomic_sint32_t pending_signals; + + VectorClock clock; + // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. // We do not distinguish beteween ignoring reads and writes // for better performance. int ignore_reads_and_writes; - atomic_sint32_t pending_signals; - int ignore_sync; int suppress_reports; // Go does not support ignores. #if !SANITIZER_GO @@ -162,31 +188,27 @@ struct ThreadState { #endif uptr *shadow_stack; uptr *shadow_stack_end; - uptr *shadow_stack_pos; - RawShadow *racy_shadow_addr; - RawShadow racy_state[2]; - MutexSet mset; - ThreadClock clock; #if !SANITIZER_GO Vector jmp_bufs; - int ignore_interceptors; -#endif - const Tid tid; - const int unique_id; - bool in_symbolizer; + int in_symbolizer; bool in_ignored_lib; bool is_inited; +#endif + MutexSet mset; bool is_dead; - bool is_freeing; - bool is_vptr_access; - const uptr stk_addr; - const uptr stk_size; - const uptr tls_addr; - const uptr tls_size; + const Tid tid; + uptr stk_addr; + uptr stk_size; + uptr tls_addr; + uptr tls_size; ThreadContext *tctx; DDLogicalThread *dd_lt; + TidSlot *slot; + uptr slot_epoch; + bool slot_locked; + // Current wired Processor, or nullptr. Required to handle any events. Processor *proc1; #if !SANITIZER_GO @@ -200,7 +222,7 @@ struct ThreadState { #if !SANITIZER_GO StackID last_sleep_stack_id; - ThreadClock last_sleep_clock; + VectorClock last_sleep_clock; #endif // Set in regions of runtime that must be signal-safe and fork-safe. @@ -209,16 +231,7 @@ struct ThreadState { const ReportDesc *current_report; - // Current position in tctx->trace.Back()->events (Event*). - atomic_uintptr_t trace_pos; - // PC of the last memory access, used to compute PC deltas in the trace. - uptr trace_prev_pc; - Sid sid; - Epoch epoch; - - explicit ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch, - unsigned reuse_count, uptr stk_addr, uptr stk_size, - uptr tls_addr, uptr tls_size); + explicit ThreadState(Tid tid); } ALIGNED(SANITIZER_CACHE_LINE_SIZE); #if !SANITIZER_GO @@ -252,14 +265,9 @@ class ThreadContext final : public ThreadContextBase { ~ThreadContext(); ThreadState *thr; StackID creation_stack_id; - SyncClock sync; - // Epoch at which the thread had started. - // If we see an event from the thread stamped by an older epoch, - // the event is from a dead thread that shared tid with this thread. - u64 epoch0; - u64 epoch1; - - v3::Trace trace; + VectorClock *sync; + uptr sync_epoch; + Trace trace; // Override superclass callbacks. void OnDead() override; @@ -314,12 +322,22 @@ struct Context { InternalMmapVector fired_suppressions; DDetector *dd; - ClockAlloc clock_alloc; - Flags flags; fd_t memprof_fd; + // The last slot index (kFreeSid) is used to denote freed memory. + TidSlot slots[kThreadSlotCount - 1]; + + // Protects global_epoch, slot_queue, trace_part_recycle. Mutex slot_mtx; + uptr global_epoch; // guarded by slot_mtx and by all slot mutexes + bool resetting; // global reset is in progress + IList slot_queue GUARDED_BY(slot_mtx); + IList trace_part_recycle + GUARDED_BY(slot_mtx); + uptr trace_part_total_allocated GUARDED_BY(slot_mtx); + uptr trace_part_recycle_finished GUARDED_BY(slot_mtx); + uptr trace_part_finished_excess GUARDED_BY(slot_mtx); }; extern Context *ctx; // The one and the only global runtime context. @@ -348,14 +366,13 @@ uptr TagFromShadowStackFrame(uptr pc); class ScopedReportBase { public: - void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack, - const MutexSet *mset); + void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid, + StackTrace stack, const MutexSet *mset); void AddStack(StackTrace stack, bool suppressable = false); void AddThread(const ThreadContext *tctx, bool suppressable = false); - void AddThread(Tid unique_tid, bool suppressable = false); + void AddThread(Tid tid, bool suppressable = false); void AddUniqueTid(Tid unique_tid); - void AddMutex(const SyncVar *s); - u64 AddMutex(u64 id); + int AddMutex(uptr addr, StackID creation_stack_id); void AddLocation(uptr addr, uptr size); void AddSleep(StackID stack_id); void SetCount(int count); @@ -372,8 +389,6 @@ class ScopedReportBase { // at best it will cause deadlocks on internal mutexes. ScopedIgnoreInterceptors ignore_interceptors_; - void AddDeadMutex(u64 id); - ScopedReportBase(const ScopedReportBase &) = delete; void operator=(const ScopedReportBase &) = delete; }; @@ -389,8 +404,6 @@ class ScopedReport : public ScopedReportBase { bool ShouldReport(ThreadState *thr, ReportType typ); ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack); -void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk, - MutexSet *mset, uptr *tag = nullptr); // The stack could look like: // |
| | tag | @@ -438,7 +451,8 @@ void ForkBefore(ThreadState *thr, uptr pc); void ForkParentAfter(ThreadState *thr, uptr pc); void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread); -void ReportRace(ThreadState *thr); +void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, + AccessType typ); bool OutputReport(ThreadState *thr, const ScopedReport &srep); bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace); bool IsExpectedReport(uptr addr, uptr size); @@ -468,55 +482,28 @@ int Finalize(ThreadState *thr); void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write); -void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, - int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic); -void MemoryAccessImpl(ThreadState *thr, uptr addr, - int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, - u64 *shadow_mem, Shadow cur); -void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, - uptr size, bool is_write); +void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, + AccessType typ); void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, AccessType typ); - -const int kSizeLog1 = 0; -const int kSizeLog2 = 1; -const int kSizeLog4 = 2; -const int kSizeLog8 = 3; +// This creates 2 non-inlined specialized versions of MemoryAccessRange. +template +void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size); ALWAYS_INLINE -void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, - AccessType typ) { - int size_log; - switch (size) { - case 1: - size_log = kSizeLog1; - break; - case 2: - size_log = kSizeLog2; - break; - case 4: - size_log = kSizeLog4; - break; - default: - DCHECK_EQ(size, 8); - size_log = kSizeLog8; - break; - } - bool is_write = !(typ & kAccessRead); - bool is_atomic = typ & kAccessAtomic; - if (typ & kAccessVptr) - thr->is_vptr_access = true; - if (typ & kAccessFree) - thr->is_freeing = true; - MemoryAccess(thr, pc, addr, size_log, is_write, is_atomic); - if (typ & kAccessVptr) - thr->is_vptr_access = false; - if (typ & kAccessFree) - thr->is_freeing = false; +void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, + bool is_write) { + if (size == 0) + return; + if (is_write) + MemoryAccessRangeT(thr, pc, addr, size); + else + MemoryAccessRangeT(thr, pc, addr, size); } -void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); +void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v); void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); +void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); @@ -526,9 +513,6 @@ void ThreadIgnoreEnd(ThreadState *thr); void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc); void ThreadIgnoreSyncEnd(ThreadState *thr); -void FuncEntry(ThreadState *thr, uptr pc); -void FuncExit(ThreadState *thr); - Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, ThreadType thread_type); @@ -574,63 +558,7 @@ void Release(ThreadState *thr, uptr pc, uptr addr); void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr); void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); void AfterSleep(ThreadState *thr, uptr pc); -void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c); -void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); -void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c); -void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c); -void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); - -// The hacky call uses custom calling convention and an assembly thunk. -// It is considerably faster that a normal call for the caller -// if it is not executed (it is intended for slow paths from hot functions). -// The trick is that the call preserves all registers and the compiler -// does not treat it as a call. -// If it does not work for you, use normal call. -#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC -// The caller may not create the stack frame for itself at all, -// so we create a reserve stack frame for it (1024b must be enough). -#define HACKY_CALL(f) \ - __asm__ __volatile__("sub $1024, %%rsp;" \ - CFI_INL_ADJUST_CFA_OFFSET(1024) \ - ".hidden " #f "_thunk;" \ - "call " #f "_thunk;" \ - "add $1024, %%rsp;" \ - CFI_INL_ADJUST_CFA_OFFSET(-1024) \ - ::: "memory", "cc"); -#else -#define HACKY_CALL(f) f() -#endif - -void TraceSwitch(ThreadState *thr); -uptr TraceTopPC(ThreadState *thr); -uptr TraceSize(); -uptr TraceParts(); -Trace *ThreadTrace(Tid tid); - -extern "C" void __tsan_trace_switch(); -void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, - EventType typ, u64 addr) { - if (!kCollectHistory) - return; - // TraceSwitch accesses shadow_stack, but it's called infrequently, - // so we check it here proactively. - DCHECK(thr->shadow_stack); - DCHECK_GE((int)typ, 0); - DCHECK_LE((int)typ, 7); - DCHECK_EQ(GetLsb(addr, kEventPCBits), addr); - u64 pos = fs.GetTracePos(); - if (UNLIKELY((pos % kTracePartSize) == 0)) { -#if !SANITIZER_GO - HACKY_CALL(__tsan_trace_switch); -#else - TraceSwitch(thr); -#endif - } - Event *trace = (Event*)GetThreadTrace(fs.tid()); - Event *evp = &trace[pos]; - Event ev = (u64)addr | ((u64)typ << kEventPCBits); - *evp = ev; -} +void IncrementEpoch(ThreadState *thr); #if !SANITIZER_GO uptr ALWAYS_INLINE HeapEnd() { @@ -638,6 +566,13 @@ uptr ALWAYS_INLINE HeapEnd() { } #endif +void SlotAttachAndLock(ThreadState *thr) ACQUIRE(thr->slot->mtx); +void SlotDetach(ThreadState *thr); +void SlotLock(ThreadState *thr) ACQUIRE(thr->slot->mtx); +void SlotUnlock(ThreadState *thr) RELEASE(thr->slot->mtx); +void DoReset(ThreadState *thr, uptr epoch); +void FlushShadowMemory(); + ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); @@ -648,6 +583,43 @@ enum FiberSwitchFlags { FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync }; +class SlotLocker { + public: + ALWAYS_INLINE + SlotLocker(ThreadState *thr, bool recursive = false) + : thr_(thr), locked_(recursive ? thr->slot_locked : false) { + if (!locked_) + SlotLock(thr_); + } + + ALWAYS_INLINE + ~SlotLocker() { + if (!locked_) + SlotUnlock(thr_); + } + + private: + ThreadState *thr_; + bool locked_; +}; + +class SlotUnlocker { + public: + SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) { + if (locked_) + SlotUnlock(thr_); + } + + ~SlotUnlocker() { + if (locked_) + SlotLock(thr_); + } + + private: + ThreadState *thr_; + bool locked_; +}; + ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) { if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals))) ProcessPendingSignalsImpl(thr); @@ -666,16 +638,19 @@ void LazyInitialize(ThreadState *thr) { #endif } -namespace v3 { - +void TraceResetForTesting(); void TraceSwitchPart(ThreadState *thr); -bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, - uptr size, AccessType typ, VarSizeStackTrace *pstk, +void TraceSwitchPartImpl(ThreadState *thr); +bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size, + AccessType typ, Tid *ptid, VarSizeStackTrace *pstk, MutexSet *pmset, uptr *ptag); template ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr, EventT **ev) { + // TraceSwitchPart accesses shadow_stack, but it's called infrequently, + // so we check it here proactively. + DCHECK(thr->shadow_stack); Event *pos = reinterpret_cast(atomic_load_relaxed(&thr->trace_pos)); #if SANITIZER_DEBUG // TraceSwitch acquires these mutexes, @@ -746,20 +721,16 @@ void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, void TraceMutexUnlock(ThreadState *thr, uptr addr); void TraceTime(ThreadState *thr); -} // namespace v3 +void TraceRestartFuncExit(ThreadState *thr); +void TraceRestartFuncEntry(ThreadState *thr, uptr pc); void GrowShadowStack(ThreadState *thr); ALWAYS_INLINE void FuncEntry(ThreadState *thr, uptr pc) { - DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc); - if (kCollectHistory) { - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); - } - - // Shadow stack maintenance can be replaced with - // stack unwinding during trace switch (which presumably must be faster). + DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc); + if (UNLIKELY(!TryTraceFunc(thr, pc))) + return TraceRestartFuncEntry(thr, pc); DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack); #if !SANITIZER_GO DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); @@ -773,12 +744,9 @@ void FuncEntry(ThreadState *thr, uptr pc) { ALWAYS_INLINE void FuncExit(ThreadState *thr) { - DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); - if (kCollectHistory) { - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0); - } - + DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid()); + if (UNLIKELY(!TryTraceFunc(thr, 0))) + return TraceRestartFuncExit(thr); DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); #if !SANITIZER_GO DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); @@ -790,7 +758,6 @@ void FuncExit(ThreadState *thr) { extern void (*on_initialize)(void); extern int (*on_finalize)(int); #endif - } // namespace __tsan #endif // TSAN_RTL_H diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp index 7365fdaa3038..940c20fcfa1a 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp @@ -15,15 +15,13 @@ namespace __tsan { -namespace v3 { - -ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, +ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState* thr, uptr pc, uptr addr, uptr size, AccessType typ) { DCHECK(size == 1 || size == 2 || size == 4 || size == 8); if (!kCollectHistory) return true; - EventAccess *ev; + EventAccess* ev; if (UNLIKELY(!TraceAcquire(thr, &ev))) return false; u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3; @@ -40,25 +38,27 @@ ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, TraceRelease(thr, ev); return true; } - auto *evex = reinterpret_cast(ev); + auto* evex = reinterpret_cast(ev); evex->is_access = 0; evex->is_func = 0; evex->type = EventType::kAccessExt; evex->is_read = !!(typ & kAccessRead); evex->is_atomic = !!(typ & kAccessAtomic); evex->size_log = size_log; + // Note: this is important, see comment in EventAccessExt. + evex->_ = 0; evex->addr = CompressAddr(addr); evex->pc = pc; TraceRelease(thr, evex); return true; } -ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, - uptr addr, uptr size, - AccessType typ) { +ALWAYS_INLINE +bool TryTraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size, + AccessType typ) { if (!kCollectHistory) return true; - EventAccessRange *ev; + EventAccessRange* ev; if (UNLIKELY(!TraceAcquire(thr, &ev))) return false; thr->trace_prev_pc = pc; @@ -75,7 +75,7 @@ ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, return true; } -void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, +void TraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size, AccessType typ) { if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ))) return; @@ -84,7 +84,7 @@ void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, DCHECK(res); } -void TraceFunc(ThreadState *thr, uptr pc) { +void TraceFunc(ThreadState* thr, uptr pc) { if (LIKELY(TryTraceFunc(thr, pc))) return; TraceSwitchPart(thr); @@ -92,7 +92,17 @@ void TraceFunc(ThreadState *thr, uptr pc) { DCHECK(res); } -void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, +NOINLINE void TraceRestartFuncEntry(ThreadState* thr, uptr pc) { + TraceSwitchPart(thr); + FuncEntry(thr, pc); +} + +NOINLINE void TraceRestartFuncExit(ThreadState* thr) { + TraceSwitchPart(thr); + FuncExit(thr); +} + +void TraceMutexLock(ThreadState* thr, EventType type, uptr pc, uptr addr, StackID stk) { DCHECK(type == EventType::kLock || type == EventType::kRLock); if (!kCollectHistory) @@ -109,7 +119,7 @@ void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, TraceEvent(thr, ev); } -void TraceMutexUnlock(ThreadState *thr, uptr addr) { +void TraceMutexUnlock(ThreadState* thr, uptr addr) { if (!kCollectHistory) return; EventUnlock ev; @@ -121,396 +131,523 @@ void TraceMutexUnlock(ThreadState *thr, uptr addr) { TraceEvent(thr, ev); } -void TraceTime(ThreadState *thr) { +void TraceTime(ThreadState* thr) { if (!kCollectHistory) return; + FastState fast_state = thr->fast_state; EventTime ev; ev.is_access = 0; ev.is_func = 0; ev.type = EventType::kTime; - ev.sid = static_cast(thr->sid); - ev.epoch = static_cast(thr->epoch); + ev.sid = static_cast(fast_state.sid()); + ev.epoch = static_cast(fast_state.epoch()); ev._ = 0; TraceEvent(thr, ev); } -} // namespace v3 +ALWAYS_INLINE RawShadow LoadShadow(RawShadow* p) { + return static_cast( + atomic_load((atomic_uint32_t*)p, memory_order_relaxed)); +} +ALWAYS_INLINE void StoreShadow(RawShadow* sp, RawShadow s) { + atomic_store((atomic_uint32_t*)sp, static_cast(s), memory_order_relaxed); +} + +NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, + Shadow old, + AccessType typ) NO_THREAD_SAFETY_ANALYSIS { + // For the free shadow markers the first element (that contains kFreeSid) + // triggers the race, but the second element contains info about the freeing + // thread, take it. + if (old.sid() == kFreeSid) + old = Shadow(LoadShadow(&shadow_mem[1])); + // This prevents trapping on this address in future. + for (uptr i = 0; i < kShadowCnt; i++) + StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty); + // See the comment in MemoryRangeFreed as to why the slot is locked + // for free memory accesses. ReportRace must not be called with + // the slot locked because of the fork. But MemoryRangeFreed is not + // called during fork because fork sets ignore_reads_and_writes, + // so simply unlocking the slot should be fine. + if (typ & kAccessFree) + SlotUnlock(thr); + ReportRace(thr, shadow_mem, cur, Shadow(old), typ); + if (typ & kAccessFree) + SlotLock(thr); +} + +#if !TSAN_VECTORIZE ALWAYS_INLINE -Shadow LoadShadow(u64 *p) { - u64 raw = atomic_load((atomic_uint64_t *)p, memory_order_relaxed); - return Shadow(raw); -} - -ALWAYS_INLINE -void StoreShadow(u64 *sp, u64 s) { - atomic_store((atomic_uint64_t *)sp, s, memory_order_relaxed); -} - -ALWAYS_INLINE -void StoreIfNotYetStored(u64 *sp, u64 *s) { - StoreShadow(sp, *s); - *s = 0; -} - -extern "C" void __tsan_report_race(); - -ALWAYS_INLINE -void HandleRace(ThreadState *thr, u64 *shadow_mem, Shadow cur, Shadow old) { - thr->racy_state[0] = cur.raw(); - thr->racy_state[1] = old.raw(); - thr->racy_shadow_addr = shadow_mem; -#if !SANITIZER_GO - HACKY_CALL(__tsan_report_race); -#else - ReportRace(thr); -#endif -} - -static inline bool HappensBefore(Shadow old, ThreadState *thr) { - return thr->clock.get(old.TidWithIgnore()) >= old.epoch(); -} - -ALWAYS_INLINE -void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog, - bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem, - Shadow cur) { - // This potentially can live in an MMX/SSE scratch register. - // The required intrinsics are: - // __m128i _mm_move_epi64(__m128i*); - // _mm_storel_epi64(u64*, __m128i); - u64 store_word = cur.raw(); - bool stored = false; - - // scan all the shadow values and dispatch to 4 categories: - // same, replace, candidate and race (see comments below). - // we consider only 3 cases regarding access sizes: - // equal, intersect and not intersect. initially I considered - // larger and smaller as well, it allowed to replace some - // 'candidates' with 'same' or 'replace', but I think - // it's just not worth it (performance- and complexity-wise). - - Shadow old(0); - - // It release mode we manually unroll the loop, - // because empirically gcc generates better code this way. - // However, we can't afford unrolling in debug mode, because the function - // consumes almost 4K of stack. Gtest gives only 4K of stack to death test - // threads, which is not enough for the unrolled loop. -#if SANITIZER_DEBUG - for (int idx = 0; idx < 4; idx++) { -# include "tsan_update_shadow_word.inc" - } -#else - int idx = 0; -# include "tsan_update_shadow_word.inc" - idx = 1; - if (stored) { -# include "tsan_update_shadow_word.inc" - } else { -# include "tsan_update_shadow_word.inc" - } - idx = 2; - if (stored) { -# include "tsan_update_shadow_word.inc" - } else { -# include "tsan_update_shadow_word.inc" - } - idx = 3; - if (stored) { -# include "tsan_update_shadow_word.inc" - } else { -# include "tsan_update_shadow_word.inc" - } -#endif - - // we did not find any races and had already stored - // the current access info, so we are done - if (LIKELY(stored)) - return; - // choose a random candidate slot and replace it - StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); - return; -RACE: - HandleRace(thr, shadow_mem, cur, old); - return; -} - -void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, - AccessType typ) { - DCHECK(!(typ & kAccessAtomic)); - const bool kAccessIsWrite = !(typ & kAccessRead); - const bool kIsAtomic = false; - while (size) { - int size1 = 1; - int kAccessSizeLog = kSizeLog1; - if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) { - size1 = 8; - kAccessSizeLog = kSizeLog8; - } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) { - size1 = 4; - kAccessSizeLog = kSizeLog4; - } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) { - size1 = 2; - kAccessSizeLog = kSizeLog2; - } - MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic); - addr += size1; - size -= size1; - } -} - -ALWAYS_INLINE -bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) { - Shadow cur(a); +bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1, + AccessType typ) { for (uptr i = 0; i < kShadowCnt; i++) { - Shadow old(LoadShadow(&s[i])); - if (Shadow::Addr0AndSizeAreEqual(cur, old) && - old.TidWithIgnore() == cur.TidWithIgnore() && - old.epoch() > sync_epoch && old.IsAtomic() == cur.IsAtomic() && - old.IsRead() <= cur.IsRead()) + auto old = LoadShadow(&s[i]); + if (!(typ & kAccessRead)) { + if (old == cur.raw()) + return true; + continue; + } + auto masked = static_cast(static_cast(old) | + static_cast(Shadow::kRodata)); + if (masked == cur.raw()) return true; + if (!(typ & kAccessNoRodata) && !SANITIZER_GO) { + if (old == Shadow::kRodata) + return true; + } } return false; } -#if TSAN_VECTORIZE -# define SHUF(v0, v1, i0, i1, i2, i3) \ - _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), \ - _mm_castsi128_ps(v1), \ - (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64)) ALWAYS_INLINE -bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) { - // This is an optimized version of ContainsSameAccessSlow. - // load current access into access[0:63] - const m128 access = _mm_cvtsi64_si128(a); - // duplicate high part of access in addr0: - // addr0[0:31] = access[32:63] - // addr0[32:63] = access[32:63] - // addr0[64:95] = access[32:63] - // addr0[96:127] = access[32:63] - const m128 addr0 = SHUF(access, access, 1, 1, 1, 1); - // load 4 shadow slots - const m128 shadow0 = _mm_load_si128((__m128i *)s); - const m128 shadow1 = _mm_load_si128((__m128i *)s + 1); - // load high parts of 4 shadow slots into addr_vect: - // addr_vect[0:31] = shadow0[32:63] - // addr_vect[32:63] = shadow0[96:127] - // addr_vect[64:95] = shadow1[32:63] - // addr_vect[96:127] = shadow1[96:127] - m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3); - if (!is_write) { - // set IsRead bit in addr_vect - const m128 rw_mask1 = _mm_cvtsi64_si128(1 << 15); - const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0); - addr_vect = _mm_or_si128(addr_vect, rw_mask); +bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, + int unused0, int unused1, AccessType typ) { + bool stored = false; + for (uptr idx = 0; idx < kShadowCnt; idx++) { + RawShadow* sp = &shadow_mem[idx]; + Shadow old(LoadShadow(sp)); + if (LIKELY(old.raw() == Shadow::kEmpty)) { + if (!(typ & kAccessCheckOnly) && !stored) + StoreShadow(sp, cur.raw()); + return false; + } + if (LIKELY(!(cur.access() & old.access()))) + continue; + if (LIKELY(cur.sid() == old.sid())) { + if (!(typ & kAccessCheckOnly) && + LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) { + StoreShadow(sp, cur.raw()); + stored = true; + } + continue; + } + if (LIKELY(old.IsBothReadsOrAtomic(typ))) + continue; + if (LIKELY(thr->clock.Get(old.sid()) >= old.epoch())) + continue; + DoReportRace(thr, shadow_mem, cur, old, typ); + return true; } - // addr0 == addr_vect? - const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect); - // epoch1[0:63] = sync_epoch - const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch); - // epoch[0:31] = sync_epoch[0:31] - // epoch[32:63] = sync_epoch[0:31] - // epoch[64:95] = sync_epoch[0:31] - // epoch[96:127] = sync_epoch[0:31] - const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0); - // load low parts of shadow cell epochs into epoch_vect: - // epoch_vect[0:31] = shadow0[0:31] - // epoch_vect[32:63] = shadow0[64:95] - // epoch_vect[64:95] = shadow1[0:31] - // epoch_vect[96:127] = shadow1[64:95] - const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2); - // epoch_vect >= sync_epoch? - const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch); - // addr_res & epoch_res - const m128 res = _mm_and_si128(addr_res, epoch_res); - // mask[0] = res[7] - // mask[1] = res[15] - // ... - // mask[15] = res[127] - const int mask = _mm_movemask_epi8(res); - return mask != 0; -} -#endif - -ALWAYS_INLINE -bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) { -#if TSAN_VECTORIZE - bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write); - // NOTE: this check can fail if the shadow is concurrently mutated - // by other threads. But it still can be useful if you modify - // ContainsSameAccessFast and want to ensure that it's not completely broken. - // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write)); - return res; -#else - return ContainsSameAccessSlow(s, a, sync_epoch, is_write); -#endif + // We did not find any races and had already stored + // the current access info, so we are done. + if (LIKELY(stored)) + return false; + // Choose a random candidate slot and replace it. + uptr index = + atomic_load_relaxed(&thr->trace_pos) / sizeof(Event) % kShadowCnt; + StoreShadow(&shadow_mem[index], cur.raw()); + return false; } -ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, - int kAccessSizeLog, bool kAccessIsWrite, - bool kIsAtomic) { - RawShadow *shadow_mem = MemToShadow(addr); - DPrintf2( - "#%d: MemoryAccess: @%p %p size=%d" - " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", - (int)thr->fast_state.tid(), (void *)pc, (void *)addr, - (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, - (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2], - (uptr)shadow_mem[3]); -#if SANITIZER_DEBUG - if (!IsAppMem(addr)) { - Printf("Access to non app mem %zx\n", addr); - DCHECK(IsAppMem(addr)); +# define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0 + +#else /* !TSAN_VECTORIZE */ + +ALWAYS_INLINE +bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow, + m128 access, AccessType typ) { + // Note: we could check if there is a larger access of the same type, + // e.g. we just allocated/memset-ed a block (so it contains 8 byte writes) + // and now do smaller reads/writes, these can also be considered as "same + // access". However, it will make the check more expensive, so it's unclear + // if it's worth it. But this would conserve trace space, so it's useful + // besides potential speed up. + if (!(typ & kAccessRead)) { + const m128 same = _mm_cmpeq_epi32(shadow, access); + return _mm_movemask_epi8(same); } - if (!IsShadowMem(shadow_mem)) { - Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); - DCHECK(IsShadowMem(shadow_mem)); + // For reads we need to reset read bit in the shadow, + // because we need to match read with both reads and writes. + // Shadow::kRodata has only read bit set, so it does what we want. + // We also abuse it for rodata check to save few cycles + // since we already loaded Shadow::kRodata into a register. + // Reads from rodata can't race. + // Measurements show that they can be 10-20% of all memory accesses. + // Shadow::kRodata has epoch 0 which cannot appear in shadow normally + // (thread epochs start from 1). So the same read bit mask + // serves as rodata indicator. + const m128 read_mask = _mm_set1_epi32(static_cast(Shadow::kRodata)); + const m128 masked_shadow = _mm_or_si128(shadow, read_mask); + m128 same = _mm_cmpeq_epi32(masked_shadow, access); + // Range memory accesses check Shadow::kRodata before calling this, + // Shadow::kRodatas is not possible for free memory access + // and Go does not use Shadow::kRodata. + if (!(typ & kAccessNoRodata) && !SANITIZER_GO) { + const m128 ro = _mm_cmpeq_epi32(shadow, read_mask); + same = _mm_or_si128(ro, same); } + return _mm_movemask_epi8(same); +} + +NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, + u32 race_mask, m128 shadow, AccessType typ) { + // race_mask points which of the shadow elements raced with the current + // access. Extract that element. + CHECK_NE(race_mask, 0); + u32 old; + // Note: _mm_extract_epi32 index must be a constant value. + switch (__builtin_ffs(race_mask) / 4) { + case 0: + old = _mm_extract_epi32(shadow, 0); + break; + case 1: + old = _mm_extract_epi32(shadow, 1); + break; + case 2: + old = _mm_extract_epi32(shadow, 2); + break; + case 3: + old = _mm_extract_epi32(shadow, 3); + break; + } + Shadow prev(static_cast(old)); + // For the free shadow markers the first element (that contains kFreeSid) + // triggers the race, but the second element contains info about the freeing + // thread, take it. + if (prev.sid() == kFreeSid) + prev = Shadow(static_cast(_mm_extract_epi32(shadow, 1))); + DoReportRace(thr, shadow_mem, cur, prev, typ); +} + +ALWAYS_INLINE +bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, + m128 shadow, m128 access, AccessType typ) { + // Note: empty/zero slots don't intersect with any access. + const m128 zero = _mm_setzero_si128(); + const m128 mask_access = _mm_set1_epi32(0x000000ff); + const m128 mask_sid = _mm_set1_epi32(0x0000ff00); + const m128 mask_read_atomic = _mm_set1_epi32(0xc0000000); + const m128 access_and = _mm_and_si128(access, shadow); + const m128 access_xor = _mm_xor_si128(access, shadow); + const m128 intersect = _mm_and_si128(access_and, mask_access); + const m128 not_intersect = _mm_cmpeq_epi32(intersect, zero); + const m128 not_same_sid = _mm_and_si128(access_xor, mask_sid); + const m128 same_sid = _mm_cmpeq_epi32(not_same_sid, zero); + const m128 both_read_or_atomic = _mm_and_si128(access_and, mask_read_atomic); + const m128 no_race = + _mm_or_si128(_mm_or_si128(not_intersect, same_sid), both_read_or_atomic); + const int race_mask = _mm_movemask_epi8(_mm_cmpeq_epi32(no_race, zero)); + if (UNLIKELY(race_mask)) + goto SHARED; + +STORE : { + if (typ & kAccessCheckOnly) + return false; + // We could also replace different sid's if access is the same, + // rw weaker and happens before. However, just checking access below + // is not enough because we also need to check that !both_read_or_atomic + // (reads from different sids can be concurrent). + // Theoretically we could replace smaller accesses with larger accesses, + // but it's unclear if it's worth doing. + const m128 mask_access_sid = _mm_set1_epi32(0x0000ffff); + const m128 not_same_sid_access = _mm_and_si128(access_xor, mask_access_sid); + const m128 same_sid_access = _mm_cmpeq_epi32(not_same_sid_access, zero); + const m128 access_read_atomic = + _mm_set1_epi32((typ & (kAccessRead | kAccessAtomic)) << 30); + const m128 rw_weaker = + _mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow); + const m128 rewrite = _mm_and_si128(same_sid_access, rw_weaker); + const int rewrite_mask = _mm_movemask_epi8(rewrite); + int index = __builtin_ffs(rewrite_mask); + if (UNLIKELY(index == 0)) { + const m128 empty = _mm_cmpeq_epi32(shadow, zero); + const int empty_mask = _mm_movemask_epi8(empty); + index = __builtin_ffs(empty_mask); + if (UNLIKELY(index == 0)) + index = (atomic_load_relaxed(&thr->trace_pos) / 2) % 16; + } + StoreShadow(&shadow_mem[index / 4], cur.raw()); + // We could zero other slots determined by rewrite_mask. + // That would help other threads to evict better slots, + // but it's unclear if it's worth it. + return false; +} + +SHARED: + m128 thread_epochs = _mm_set1_epi32(0x7fffffff); + // Need to unwind this because _mm_extract_epi8/_mm_insert_epi32 + // indexes must be constants. +# define LOAD_EPOCH(idx) \ + if (LIKELY(race_mask & (1 << (idx * 4)))) { \ + u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1); \ + u16 epoch = static_cast(thr->clock.Get(static_cast(sid))); \ + thread_epochs = _mm_insert_epi32(thread_epochs, u32(epoch) << 16, idx); \ + } + LOAD_EPOCH(0); + LOAD_EPOCH(1); + LOAD_EPOCH(2); + LOAD_EPOCH(3); +# undef LOAD_EPOCH + const m128 mask_epoch = _mm_set1_epi32(0x3fff0000); + const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch); + const m128 concurrent = _mm_cmplt_epi32(thread_epochs, shadow_epochs); + const int concurrent_mask = _mm_movemask_epi8(concurrent); + if (LIKELY(concurrent_mask == 0)) + goto STORE; + + DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ); + return true; +} + +# define LOAD_CURRENT_SHADOW(cur, shadow_mem) \ + const m128 access = _mm_set1_epi32(static_cast((cur).raw())); \ + const m128 shadow = _mm_load_si128(reinterpret_cast(shadow_mem)) #endif - if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { - // Access to .rodata section, no races here. - // Measurements show that it can be 10-20% of all memory accesses. - return; +char* DumpShadow(char* buf, RawShadow raw) { + if (raw == Shadow::kEmpty) { + internal_snprintf(buf, 64, "0"); + return buf; } + Shadow s(raw); + AccessType typ; + s.GetAccess(nullptr, nullptr, &typ); + internal_snprintf(buf, 64, "{tid=%u@%u access=0x%x typ=%x}", + static_cast(s.sid()), static_cast(s.epoch()), + s.access(), static_cast(typ)); + return buf; +} + +// TryTrace* and TraceRestart* functions allow to turn memory access and func +// entry/exit callbacks into leaf functions with all associated performance +// benefits. These hottest callbacks do only 2 slow path calls: report a race +// and trace part switching. Race reporting is easy to turn into a tail call, we +// just always return from the runtime after reporting a race. But trace part +// switching is harder because it needs to be in the middle of callbacks. To +// turn it into a tail call we immidiately return after TraceRestart* functions, +// but TraceRestart* functions themselves recurse into the callback after +// switching trace part. As the result the hottest callbacks contain only tail +// calls, which effectively makes them leaf functions (can use all registers, +// no frame setup, etc). +NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr, + uptr size, AccessType typ) { + TraceSwitchPart(thr); + MemoryAccess(thr, pc, addr, size, typ); +} + +ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr, + uptr size, AccessType typ) { + RawShadow* shadow_mem = MemToShadow(addr); + UNUSED char memBuf[4][64]; + DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid, + static_cast(thr->fast_state.sid()), + static_cast(thr->fast_state.epoch()), (void*)addr, size, + static_cast(typ), DumpShadow(memBuf[0], shadow_mem[0]), + DumpShadow(memBuf[1], shadow_mem[1]), + DumpShadow(memBuf[2], shadow_mem[2]), + DumpShadow(memBuf[3], shadow_mem[3])); FastState fast_state = thr->fast_state; - if (UNLIKELY(fast_state.GetIgnoreBit())) { + Shadow cur(fast_state, addr, size, typ); + + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) return; - } - - Shadow cur(fast_state); - cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); - cur.SetWrite(kAccessIsWrite); - cur.SetAtomic(kIsAtomic); - - if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, - kAccessIsWrite))) { + if (UNLIKELY(fast_state.GetIgnoreBit())) return; - } - - if (kCollectHistory) { - fast_state.IncrementEpoch(); - thr->fast_state = fast_state; - TraceAddEvent(thr, fast_state, EventTypeMop, pc); - cur.IncrementEpoch(); - } - - MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, - shadow_mem, cur); + if (!TryTraceMemoryAccess(thr, pc, addr, size, typ)) + return TraceRestartMemoryAccess(thr, pc, addr, size, typ); + CheckRaces(thr, shadow_mem, cur, shadow, access, typ); } -// Called by MemoryAccessRange in tsan_rtl_thread.cpp -ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr, - int kAccessSizeLog, - bool kAccessIsWrite, bool kIsAtomic, - u64 *shadow_mem, Shadow cur) { - if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, - kAccessIsWrite))) { - return; - } +void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ); - MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, - shadow_mem, cur); +NOINLINE +void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr, + AccessType typ) { + TraceSwitchPart(thr); + MemoryAccess16(thr, pc, addr, typ); } -static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, - u64 val) { - (void)thr; - (void)pc; +ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, + AccessType typ) { + const uptr size = 16; + FastState fast_state = thr->fast_state; + if (UNLIKELY(fast_state.GetIgnoreBit())) + return; + Shadow cur(fast_state, 0, 8, typ); + RawShadow* shadow_mem = MemToShadow(addr); + bool traced = false; + { + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + goto SECOND; + if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartMemoryAccess16(thr, pc, addr, typ); + traced = true; + if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) + return; + } +SECOND: + shadow_mem += kShadowCnt; + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + return; + if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartMemoryAccess16(thr, pc, addr, typ); + CheckRaces(thr, shadow_mem, cur, shadow, access, typ); +} + +NOINLINE +void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr, + uptr size, AccessType typ) { + TraceSwitchPart(thr); + UnalignedMemoryAccess(thr, pc, addr, size, typ); +} + +ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc, + uptr addr, uptr size, + AccessType typ) { + DCHECK_LE(size, 8); + FastState fast_state = thr->fast_state; + if (UNLIKELY(fast_state.GetIgnoreBit())) + return; + RawShadow* shadow_mem = MemToShadow(addr); + bool traced = false; + uptr size1 = Min(size, RoundUp(addr + 1, kShadowCell) - addr); + { + Shadow cur(fast_state, addr, size1, typ); + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + goto SECOND; + if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ); + traced = true; + if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) + return; + } +SECOND: + uptr size2 = size - size1; + if (LIKELY(size2 == 0)) + return; + shadow_mem += kShadowCnt; + Shadow cur(fast_state, 0, size2, typ); + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + return; + if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ); + CheckRaces(thr, shadow_mem, cur, shadow, access, typ); +} + +void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) { + DCHECK_LE(p, end); + DCHECK(IsShadowMem(p)); + DCHECK(IsShadowMem(end)); + UNUSED const uptr kAlign = kShadowCnt * kShadowSize; + DCHECK_EQ(reinterpret_cast(p) % kAlign, 0); + DCHECK_EQ(reinterpret_cast(end) % kAlign, 0); +#if !TSAN_VECTORIZE + for (; p < end; p += kShadowCnt) { + p[0] = v; + for (uptr i = 1; i < kShadowCnt; i++) p[i] = Shadow::kEmpty; + } +#else + m128 vv = _mm_setr_epi32( + static_cast(v), static_cast(Shadow::kEmpty), + static_cast(Shadow::kEmpty), static_cast(Shadow::kEmpty)); + m128* vp = reinterpret_cast(p); + m128* vend = reinterpret_cast(end); + for (; vp < vend; vp++) _mm_store_si128(vp, vv); +#endif +} + +static void MemoryRangeSet(uptr addr, uptr size, RawShadow val) { if (size == 0) return; - // FIXME: fix me. - uptr offset = addr % kShadowCell; - if (offset) { - offset = kShadowCell - offset; - if (size <= offset) - return; - addr += offset; - size -= offset; - } - DCHECK_EQ(addr % 8, 0); + DCHECK_EQ(addr % kShadowCell, 0); + DCHECK_EQ(size % kShadowCell, 0); // If a user passes some insane arguments (memset(0)), // let it just crash as usual. if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) return; + RawShadow* begin = MemToShadow(addr); + RawShadow* end = begin + size / kShadowCell * kShadowCnt; // Don't want to touch lots of shadow memory. // If a program maps 10MB stack, there is no need reset the whole range. - size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); // UnmapOrDie/MmapFixedNoReserve does not work on Windows. - if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) { - RawShadow *p = MemToShadow(addr); - CHECK(IsShadowMem(p)); - CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1)); - // FIXME: may overwrite a part outside the region - for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) { - p[i++] = val; - for (uptr j = 1; j < kShadowCnt; j++) p[i++] = 0; - } - } else { - // The region is big, reset only beginning and end. - const uptr kPageSize = GetPageSizeCached(); - RawShadow *begin = MemToShadow(addr); - RawShadow *end = begin + size / kShadowCell * kShadowCnt; - RawShadow *p = begin; - // Set at least first kPageSize/2 to page boundary. - while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) { - *p++ = val; - for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0; - } - // Reset middle part. - RawShadow *p1 = p; - p = RoundDown(end, kPageSize); - if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1)) - Die(); - // Set the ending. - while (p < end) { - *p++ = val; - for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0; - } + if (SANITIZER_WINDOWS || + size <= common_flags()->clear_shadow_mmap_threshold) { + ShadowSet(begin, end, val); + return; } + // The region is big, reset only beginning and end. + const uptr kPageSize = GetPageSizeCached(); + // Set at least first kPageSize/2 to page boundary. + RawShadow* mid1 = + Min(end, reinterpret_cast(RoundUp( + reinterpret_cast(begin) + kPageSize / 2, kPageSize))); + ShadowSet(begin, mid1, val); + // Reset middle part. + RawShadow* mid2 = RoundDown(end, kPageSize); + if (mid2 > mid1) { + if (!MmapFixedSuperNoReserve((uptr)mid1, (uptr)mid2 - (uptr)mid1)) + Die(); + } + // Set the ending. + ShadowSet(mid2, end, val); } -void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { - MemoryRangeSet(thr, pc, addr, size, 0); +void MemoryResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) { + uptr addr1 = RoundDown(addr, kShadowCell); + uptr size1 = RoundUp(size + addr - addr1, kShadowCell); + MemoryRangeSet(addr1, size1, Shadow::kEmpty); } -void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { - // Processing more than 1k (4k of shadow) is expensive, +void MemoryRangeFreed(ThreadState* thr, uptr pc, uptr addr, uptr size) { + // Callers must lock the slot to ensure synchronization with the reset. + // The problem with "freed" memory is that it's not "monotonic" + // with respect to bug detection: freed memory is bad to access, + // but then if the heap block is reallocated later, it's good to access. + // As the result a garbage "freed" shadow can lead to a false positive + // if it happens to match a real free in the thread trace, + // but the heap block was reallocated before the current memory access, + // so it's still good to access. It's not the case with data races. + DCHECK(thr->slot_locked); + DCHECK_EQ(addr % kShadowCell, 0); + size = RoundUp(size, kShadowCell); + // Processing more than 1k (2k of shadow) is expensive, // can cause excessive memory consumption (user does not necessary touch // the whole range) and most likely unnecessary. - if (size > 1024) - size = 1024; - CHECK_EQ(thr->is_freeing, false); - thr->is_freeing = true; - MemoryAccessRange(thr, pc, addr, size, true); - thr->is_freeing = false; - if (kCollectHistory) { - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + size = Min(size, 1024); + const AccessType typ = + kAccessWrite | kAccessFree | kAccessCheckOnly | kAccessNoRodata; + TraceMemoryAccessRange(thr, pc, addr, size, typ); + RawShadow* shadow_mem = MemToShadow(addr); + Shadow cur(thr->fast_state, 0, kShadowCell, typ); +#if TSAN_VECTORIZE + const m128 access = _mm_set1_epi32(static_cast(cur.raw())); + const m128 freed = _mm_setr_epi32( + static_cast(Shadow::FreedMarker()), + static_cast(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0); + for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) { + const m128 shadow = _mm_load_si128((m128*)shadow_mem); + if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) + return; + _mm_store_si128((m128*)shadow_mem, freed); } - Shadow s(thr->fast_state); - s.ClearIgnoreBit(); - s.MarkAsFreed(); - s.SetWrite(true); - s.SetAddr0AndSizeLog(0, 3); - MemoryRangeSet(thr, pc, addr, size, s.raw()); +#else + for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) { + if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ))) + return; + StoreShadow(&shadow_mem[0], Shadow::FreedMarker()); + StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch())); + StoreShadow(&shadow_mem[2], Shadow::kEmpty); + StoreShadow(&shadow_mem[3], Shadow::kEmpty); + } +#endif } -void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { - if (kCollectHistory) { - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); - } - Shadow s(thr->fast_state); - s.ClearIgnoreBit(); - s.SetWrite(true); - s.SetAddr0AndSizeLog(0, 3); - MemoryRangeSet(thr, pc, addr, size, s.raw()); +void MemoryRangeImitateWrite(ThreadState* thr, uptr pc, uptr addr, uptr size) { + DCHECK_EQ(addr % kShadowCell, 0); + size = RoundUp(size, kShadowCell); + TraceMemoryAccessRange(thr, pc, addr, size, kAccessWrite); + Shadow cur(thr->fast_state, 0, 8, kAccessWrite); + MemoryRangeSet(addr, size, cur.raw()); } -void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, +void MemoryRangeImitateWriteOrResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) { if (thr->ignore_reads_and_writes == 0) MemoryRangeImitateWrite(thr, pc, addr, size); @@ -518,14 +655,29 @@ void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, MemoryResetRange(thr, pc, addr, size); } -void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, - bool is_write) { - if (size == 0) - return; +ALWAYS_INLINE +bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, + AccessType typ) { + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + return false; + return CheckRaces(thr, shadow_mem, cur, shadow, access, typ); +} - RawShadow *shadow_mem = MemToShadow(addr); - DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n", thr->tid, - (void *)pc, (void *)addr, (int)size, is_write); +template +NOINLINE void RestartMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, + uptr size) { + TraceSwitchPart(thr); + MemoryAccessRangeT(thr, pc, addr, size); +} + +template +void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) { + const AccessType typ = + (is_read ? kAccessRead : kAccessWrite) | kAccessNoRodata; + RawShadow* shadow_mem = MemToShadow(addr); + DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid, + (void*)pc, (void*)addr, (int)size, is_read); #if SANITIZER_DEBUG if (!IsAppMem(addr)) { @@ -537,65 +689,62 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, DCHECK(IsAppMem(addr + size - 1)); } if (!IsShadowMem(shadow_mem)) { - Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); + Printf("Bad shadow addr %p (%zx)\n", static_cast(shadow_mem), addr); DCHECK(IsShadowMem(shadow_mem)); } - if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) { - Printf("Bad shadow addr %p (%zx)\n", shadow_mem + size * kShadowCnt / 8 - 1, + if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) { + Printf("Bad shadow addr %p (%zx)\n", + static_cast(shadow_mem + size * kShadowCnt - 1), addr + size - 1); - DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)); + DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1)); } #endif - if (*shadow_mem == kShadowRodata) { - DCHECK(!is_write); - // Access to .rodata section, no races here. - // Measurements show that it can be 10-20% of all memory accesses. + // Access to .rodata section, no races here. + // Measurements show that it can be 10-20% of all memory accesses. + // Check here once to not check for every access separately. + // Note: we could (and should) do this only for the is_read case + // (writes shouldn't go to .rodata). But it happens in Chromium tests: + // https://bugs.chromium.org/p/chromium/issues/detail?id=1275581#c19 + // Details are unknown since it happens only on CI machines. + if (*shadow_mem == Shadow::kRodata) return; - } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) + if (UNLIKELY(fast_state.GetIgnoreBit())) return; - fast_state.IncrementEpoch(); - thr->fast_state = fast_state; - TraceAddEvent(thr, fast_state, EventTypeMop, pc); + if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartMemoryAccessRange(thr, pc, addr, size); - bool unaligned = (addr % kShadowCell) != 0; - - // Handle unaligned beginning, if any. - for (; addr % kShadowCell && size; addr++, size--) { - int const kAccessSizeLog = 0; - Shadow cur(fast_state); - cur.SetWrite(is_write); - cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog); - MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem, - cur); + if (UNLIKELY(addr % kShadowCell)) { + // Handle unaligned beginning, if any. + uptr size1 = Min(size, RoundUp(addr, kShadowCell) - addr); + size -= size1; + Shadow cur(fast_state, addr, size1, typ); + if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) + return; + shadow_mem += kShadowCnt; } - if (unaligned) - shadow_mem += kShadowCnt; // Handle middle part, if any. - for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) { - int const kAccessSizeLog = 3; - Shadow cur(fast_state); - cur.SetWrite(is_write); - cur.SetAddr0AndSizeLog(0, kAccessSizeLog); - MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem, - cur); - shadow_mem += kShadowCnt; + Shadow cur(fast_state, 0, kShadowCell, typ); + for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) { + if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) + return; } // Handle ending, if any. - for (; size; addr++, size--) { - int const kAccessSizeLog = 0; - Shadow cur(fast_state); - cur.SetWrite(is_write); - cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog); - MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem, - cur); + if (UNLIKELY(size)) { + Shadow cur(fast_state, 0, size, typ); + if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) + return; } } +template void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, + uptr size); +template void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, + uptr size); + } // namespace __tsan #if !SANITIZER_GO diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S index 632b19d18158..f848be9dd46c 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S @@ -9,242 +9,6 @@ .section __TEXT,__text #endif -ASM_HIDDEN(__tsan_trace_switch) -.globl ASM_SYMBOL(__tsan_trace_switch_thunk) -ASM_SYMBOL(__tsan_trace_switch_thunk): - CFI_STARTPROC - _CET_ENDBR - # Save scratch registers. - push %rax - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rax, 0) - push %rcx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rcx, 0) - push %rdx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdx, 0) - push %rsi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rsi, 0) - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - push %r8 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r8, 0) - push %r9 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r9, 0) - push %r10 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r10, 0) - push %r11 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r11, 0) - # All XMM registers are caller-saved. - sub $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(0x100) - vmovdqu %xmm0, 0x0(%rsp) - vmovdqu %xmm1, 0x10(%rsp) - vmovdqu %xmm2, 0x20(%rsp) - vmovdqu %xmm3, 0x30(%rsp) - vmovdqu %xmm4, 0x40(%rsp) - vmovdqu %xmm5, 0x50(%rsp) - vmovdqu %xmm6, 0x60(%rsp) - vmovdqu %xmm7, 0x70(%rsp) - vmovdqu %xmm8, 0x80(%rsp) - vmovdqu %xmm9, 0x90(%rsp) - vmovdqu %xmm10, 0xa0(%rsp) - vmovdqu %xmm11, 0xb0(%rsp) - vmovdqu %xmm12, 0xc0(%rsp) - vmovdqu %xmm13, 0xd0(%rsp) - vmovdqu %xmm14, 0xe0(%rsp) - vmovdqu %xmm15, 0xf0(%rsp) - # Align stack frame. - push %rbx # non-scratch - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rbx, 0) - mov %rsp, %rbx # save current rsp - CFI_DEF_CFA_REGISTER(%rbx) - shr $4, %rsp # clear 4 lsb, align to 16 - shl $4, %rsp - - call ASM_SYMBOL(__tsan_trace_switch) - - # Unalign stack frame back. - mov %rbx, %rsp # restore the original rsp - CFI_DEF_CFA_REGISTER(%rsp) - pop %rbx - CFI_ADJUST_CFA_OFFSET(-8) - # Restore scratch registers. - vmovdqu 0x0(%rsp), %xmm0 - vmovdqu 0x10(%rsp), %xmm1 - vmovdqu 0x20(%rsp), %xmm2 - vmovdqu 0x30(%rsp), %xmm3 - vmovdqu 0x40(%rsp), %xmm4 - vmovdqu 0x50(%rsp), %xmm5 - vmovdqu 0x60(%rsp), %xmm6 - vmovdqu 0x70(%rsp), %xmm7 - vmovdqu 0x80(%rsp), %xmm8 - vmovdqu 0x90(%rsp), %xmm9 - vmovdqu 0xa0(%rsp), %xmm10 - vmovdqu 0xb0(%rsp), %xmm11 - vmovdqu 0xc0(%rsp), %xmm12 - vmovdqu 0xd0(%rsp), %xmm13 - vmovdqu 0xe0(%rsp), %xmm14 - vmovdqu 0xf0(%rsp), %xmm15 - add $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(-0x100) - pop %r11 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r10 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r9 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r8 - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rsi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rcx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rax - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rax) - CFI_RESTORE(%rbx) - CFI_RESTORE(%rcx) - CFI_RESTORE(%rdx) - CFI_RESTORE(%rsi) - CFI_RESTORE(%rdi) - CFI_RESTORE(%r8) - CFI_RESTORE(%r9) - CFI_RESTORE(%r10) - CFI_RESTORE(%r11) - ret - CFI_ENDPROC - -ASM_HIDDEN(__tsan_report_race) -.globl ASM_SYMBOL(__tsan_report_race_thunk) -ASM_SYMBOL(__tsan_report_race_thunk): - CFI_STARTPROC - _CET_ENDBR - # Save scratch registers. - push %rax - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rax, 0) - push %rcx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rcx, 0) - push %rdx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdx, 0) - push %rsi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rsi, 0) - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - push %r8 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r8, 0) - push %r9 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r9, 0) - push %r10 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r10, 0) - push %r11 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r11, 0) - # All XMM registers are caller-saved. - sub $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(0x100) - vmovdqu %xmm0, 0x0(%rsp) - vmovdqu %xmm1, 0x10(%rsp) - vmovdqu %xmm2, 0x20(%rsp) - vmovdqu %xmm3, 0x30(%rsp) - vmovdqu %xmm4, 0x40(%rsp) - vmovdqu %xmm5, 0x50(%rsp) - vmovdqu %xmm6, 0x60(%rsp) - vmovdqu %xmm7, 0x70(%rsp) - vmovdqu %xmm8, 0x80(%rsp) - vmovdqu %xmm9, 0x90(%rsp) - vmovdqu %xmm10, 0xa0(%rsp) - vmovdqu %xmm11, 0xb0(%rsp) - vmovdqu %xmm12, 0xc0(%rsp) - vmovdqu %xmm13, 0xd0(%rsp) - vmovdqu %xmm14, 0xe0(%rsp) - vmovdqu %xmm15, 0xf0(%rsp) - # Align stack frame. - push %rbx # non-scratch - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rbx, 0) - mov %rsp, %rbx # save current rsp - CFI_DEF_CFA_REGISTER(%rbx) - shr $4, %rsp # clear 4 lsb, align to 16 - shl $4, %rsp - - call ASM_SYMBOL(__tsan_report_race) - - # Unalign stack frame back. - mov %rbx, %rsp # restore the original rsp - CFI_DEF_CFA_REGISTER(%rsp) - pop %rbx - CFI_ADJUST_CFA_OFFSET(-8) - # Restore scratch registers. - vmovdqu 0x0(%rsp), %xmm0 - vmovdqu 0x10(%rsp), %xmm1 - vmovdqu 0x20(%rsp), %xmm2 - vmovdqu 0x30(%rsp), %xmm3 - vmovdqu 0x40(%rsp), %xmm4 - vmovdqu 0x50(%rsp), %xmm5 - vmovdqu 0x60(%rsp), %xmm6 - vmovdqu 0x70(%rsp), %xmm7 - vmovdqu 0x80(%rsp), %xmm8 - vmovdqu 0x90(%rsp), %xmm9 - vmovdqu 0xa0(%rsp), %xmm10 - vmovdqu 0xb0(%rsp), %xmm11 - vmovdqu 0xc0(%rsp), %xmm12 - vmovdqu 0xd0(%rsp), %xmm13 - vmovdqu 0xe0(%rsp), %xmm14 - vmovdqu 0xf0(%rsp), %xmm15 - add $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(-0x100) - pop %r11 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r10 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r9 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r8 - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rsi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rcx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rax - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rax) - CFI_RESTORE(%rbx) - CFI_RESTORE(%rcx) - CFI_RESTORE(%rdx) - CFI_RESTORE(%rsi) - CFI_RESTORE(%rdi) - CFI_RESTORE(%r8) - CFI_RESTORE(%r9) - CFI_RESTORE(%r10) - CFI_RESTORE(%r11) - ret - CFI_ENDPROC - ASM_HIDDEN(__tsan_setjmp) #if defined(__NetBSD__) .comm _ZN14__interception15real___setjmp14E,8,8 diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp index 7d6b41116aa6..5d31005c2af0 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp @@ -23,6 +23,8 @@ namespace __tsan { void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r); +void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr, + FastState last_lock, StackID creation_stack_id); struct Callback final : public DDCallback { ThreadState *thr; @@ -36,17 +38,17 @@ struct Callback final : public DDCallback { } StackID Unwind() override { return CurrentStackId(thr, pc); } - int UniqueTid() override { return thr->unique_id; } + int UniqueTid() override { return thr->tid; } }; void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) { Callback cb(thr, pc); ctx->dd->MutexInit(&cb, &s->dd); - s->dd.ctx = s->GetId(); + s->dd.ctx = s->addr; } static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, - uptr addr, u64 mid) { + uptr addr, StackID creation_stack_id) { // In Go, these misuses are either impossible, or detected by std lib, // or false positives (e.g. unlock in a different thread). if (SANITIZER_GO) @@ -55,7 +57,7 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, return; ThreadRegistryLock l(&ctx->thread_registry); ScopedReport rep(typ); - rep.AddMutex(mid); + rep.AddMutex(addr, creation_stack_id); VarSizeStackTrace trace; ObtainCurrentStack(thr, pc, &trace); rep.AddStack(trace, true); @@ -63,95 +65,93 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, OutputReport(thr, rep); } +static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr, + StackID stack_id, bool write) { + auto typ = write ? EventType::kLock : EventType::kRLock; + // Note: it's important to trace before modifying mutex set + // because tracing can switch trace part and we write the current + // mutex set in the beginning of each part. + // If we do it in the opposite order, we will write already reduced + // mutex set in the beginning of the part and then trace unlock again. + TraceMutexLock(thr, typ, pc, addr, stack_id); + thr->mset.AddAddr(addr, stack_id, write); +} + +static void RecordMutexUnlock(ThreadState *thr, uptr addr) { + // See the comment in RecordMutexLock re order of operations. + TraceMutexUnlock(thr, addr); + thr->mset.DelAddr(addr); +} + void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz); - if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) { - CHECK(!thr->is_freeing); - thr->is_freeing = true; + if (!(flagz & MutexFlagLinkerInit) && pc && IsAppMem(addr)) MemoryAccess(thr, pc, addr, 1, kAccessWrite); - thr->is_freeing = false; - } - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - Lock l(&s->mtx); + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); s->SetFlags(flagz & MutexCreationFlagMask); // Save stack in the case the sync object was created before as atomic. - if (!SANITIZER_GO && s->creation_stack_id == 0) + if (!SANITIZER_GO && s->creation_stack_id == kInvalidStackID) s->creation_stack_id = CurrentStackId(thr, pc); } void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr); bool unlock_locked = false; - u64 mid = 0; - u64 last_lock = 0; + StackID creation_stack_id; + FastState last_lock; { - SyncVar *s = ctx->metamap.GetSyncIfExists(addr); - if (s == 0) - return; - Lock l(&s->mtx); - if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) || - ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) { - // Destroy is no-op for linker-initialized mutexes. + auto s = ctx->metamap.GetSyncIfExists(addr); + if (!s) return; + SlotLocker locker(thr); + { + Lock lock(&s->mtx); + creation_stack_id = s->creation_stack_id; + last_lock = s->last_lock; + if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) || + ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) { + // Destroy is no-op for linker-initialized mutexes. + return; + } + if (common_flags()->detect_deadlocks) { + Callback cb(thr, pc); + ctx->dd->MutexDestroy(&cb, &s->dd); + ctx->dd->MutexInit(&cb, &s->dd); + } + if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid && + !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + unlock_locked = true; + } + s->Reset(); } - if (common_flags()->detect_deadlocks) { - Callback cb(thr, pc); - ctx->dd->MutexDestroy(&cb, &s->dd); - ctx->dd->MutexInit(&cb, &s->dd); - } - if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid && - !s->IsFlagSet(MutexFlagBroken)) { - s->SetFlags(MutexFlagBroken); - unlock_locked = true; - } - mid = s->GetId(); - last_lock = s->last_lock; - if (!unlock_locked) - s->Reset(thr->proc()); // must not reset it before the report is printed + // Imitate a memory write to catch unlock-destroy races. + if (pc && IsAppMem(addr)) + MemoryAccess(thr, pc, addr, 1, kAccessWrite | kAccessFree); } - if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) { - ThreadRegistryLock l(&ctx->thread_registry); - ScopedReport rep(ReportTypeMutexDestroyLocked); - rep.AddMutex(mid); - VarSizeStackTrace trace; - ObtainCurrentStack(thr, pc, &trace); - rep.AddStack(trace, true); - FastState last(last_lock); - RestoreStack(last.tid(), last.epoch(), &trace, 0); - rep.AddStack(trace, true); - rep.AddLocation(addr, 1); - OutputReport(thr, rep); - - SyncVar *s = ctx->metamap.GetSyncIfExists(addr); - if (s != 0) { - Lock l(&s->mtx); - s->Reset(thr->proc()); - } - } - thr->mset.Remove(mid); - // Imitate a memory write to catch unlock-destroy races. - // Do this outside of sync mutex, because it can report a race which locks - // sync mutexes. - if (IsAppMem(addr)) - MemoryAccess(thr, pc, addr, 1, kAccessWrite | kAccessFree); + if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) + ReportDestroyLocked(thr, pc, addr, last_lock, creation_stack_id); + thr->mset.DelAddr(addr, true); // s will be destroyed and freed in MetaMap::FreeBlock. } void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz); - if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - { - ReadLock l(&s->mtx); - s->UpdateFlags(flagz); - if (s->owner_tid != thr->tid) { - Callback cb(thr, pc); - ctx->dd->MutexBeforeLock(&cb, &s->dd, true); - } - } - Callback cb(thr, pc); - ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + if (flagz & MutexFlagTryLock) + return; + if (!common_flags()->detect_deadlocks) + return; + Callback cb(thr, pc); + { + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + ReadLock lock(&s->mtx); + s->UpdateFlags(flagz); + if (s->owner_tid != thr->tid) + ctx->dd->MutexBeforeLock(&cb, &s->dd, true); } + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); } void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) { @@ -161,48 +161,51 @@ void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) { CHECK_GT(rec, 0); else rec = 1; - if (IsAppMem(addr)) + if (pc && IsAppMem(addr)) MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); - u64 mid = 0; + bool report_double_lock = false; bool pre_lock = false; bool first = false; - bool report_double_lock = false; + StackID creation_stack_id = kInvalidStackID; { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - Lock l(&s->mtx); - s->UpdateFlags(flagz); - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId()); - if (s->owner_tid == kInvalidTid) { - CHECK_EQ(s->recursion, 0); - s->owner_tid = thr->tid; - s->last_lock = thr->fast_state.raw(); - } else if (s->owner_tid == thr->tid) { - CHECK_GT(s->recursion, 0); - } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { - s->SetFlags(MutexFlagBroken); - report_double_lock = true; + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + creation_stack_id = s->creation_stack_id; + RecordMutexLock(thr, pc, addr, creation_stack_id, true); + { + Lock lock(&s->mtx); + first = s->recursion == 0; + s->UpdateFlags(flagz); + if (s->owner_tid == kInvalidTid) { + CHECK_EQ(s->recursion, 0); + s->owner_tid = thr->tid; + s->last_lock = thr->fast_state; + } else if (s->owner_tid == thr->tid) { + CHECK_GT(s->recursion, 0); + } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_double_lock = true; + } + s->recursion += rec; + if (first) { + if (!thr->ignore_sync) { + thr->clock.Acquire(s->clock); + thr->clock.Acquire(s->read_clock); + } + } + if (first && common_flags()->detect_deadlocks) { + pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) && + !(flagz & MutexFlagTryLock); + Callback cb(thr, pc); + if (pre_lock) + ctx->dd->MutexBeforeLock(&cb, &s->dd, true); + ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock); + } } - first = s->recursion == 0; - s->recursion += rec; - if (first) { - AcquireImpl(thr, pc, &s->clock); - AcquireImpl(thr, pc, &s->read_clock); - } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) { - } - thr->mset.Add(s->GetId(), true, thr->fast_state.epoch()); - if (first && common_flags()->detect_deadlocks) { - pre_lock = - (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock); - Callback cb(thr, pc); - if (pre_lock) - ctx->dd->MutexBeforeLock(&cb, &s->dd, true); - ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock); - } - mid = s->GetId(); } if (report_double_lock) - ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid); + ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, + creation_stack_id); if (first && pre_lock && common_flags()->detect_deadlocks) { Callback cb(thr, pc); ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); @@ -211,40 +214,47 @@ void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) { int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz); - if (IsAppMem(addr)) + if (pc && IsAppMem(addr)) MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); - u64 mid = 0; + StackID creation_stack_id; + RecordMutexUnlock(thr, addr); bool report_bad_unlock = false; int rec = 0; { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); - if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) { - if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { - s->SetFlags(MutexFlagBroken); - report_bad_unlock = true; - } - } else { - rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1; - s->recursion -= rec; - if (s->recursion == 0) { - s->owner_tid = kInvalidTid; - ReleaseStoreImpl(thr, pc, &s->clock); + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + bool released = false; + { + Lock lock(&s->mtx); + creation_stack_id = s->creation_stack_id; + if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) { + if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_unlock = true; + } } else { + rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1; + s->recursion -= rec; + if (s->recursion == 0) { + s->owner_tid = kInvalidTid; + if (!thr->ignore_sync) { + thr->clock.ReleaseStore(&s->clock); + released = true; + } + } + } + if (common_flags()->detect_deadlocks && s->recursion == 0 && + !report_bad_unlock) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true); } } - thr->mset.Del(s->GetId(), true); - if (common_flags()->detect_deadlocks && s->recursion == 0 && - !report_bad_unlock) { - Callback cb(thr, pc); - ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true); - } - mid = s->GetId(); + if (released) + IncrementEpoch(thr); } if (report_bad_unlock) - ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); + ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, + creation_stack_id); if (common_flags()->detect_deadlocks && !report_bad_unlock) { Callback cb(thr, pc); ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); @@ -254,53 +264,56 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz); - if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) { - { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - ReadLock l(&s->mtx); - s->UpdateFlags(flagz); - Callback cb(thr, pc); - ctx->dd->MutexBeforeLock(&cb, &s->dd, false); - } - Callback cb(thr, pc); - ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + if ((flagz & MutexFlagTryLock) || !common_flags()->detect_deadlocks) + return; + Callback cb(thr, pc); + { + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + ReadLock lock(&s->mtx); + s->UpdateFlags(flagz); + ctx->dd->MutexBeforeLock(&cb, &s->dd, false); } + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); } void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz); - if (IsAppMem(addr)) + if (pc && IsAppMem(addr)) MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); - u64 mid = 0; bool report_bad_lock = false; bool pre_lock = false; + StackID creation_stack_id = kInvalidStackID; { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - ReadLock l(&s->mtx); - s->UpdateFlags(flagz); - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId()); - if (s->owner_tid != kInvalidTid) { - if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { - s->SetFlags(MutexFlagBroken); - report_bad_lock = true; + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + creation_stack_id = s->creation_stack_id; + RecordMutexLock(thr, pc, addr, creation_stack_id, false); + { + ReadLock lock(&s->mtx); + s->UpdateFlags(flagz); + if (s->owner_tid != kInvalidTid) { + if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_lock = true; + } + } + if (!thr->ignore_sync) + thr->clock.Acquire(s->clock); + s->last_lock = thr->fast_state; + if (common_flags()->detect_deadlocks) { + pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) && + !(flagz & MutexFlagTryLock); + Callback cb(thr, pc); + if (pre_lock) + ctx->dd->MutexBeforeLock(&cb, &s->dd, false); + ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock); } } - AcquireImpl(thr, pc, &s->clock); - s->last_lock = thr->fast_state.raw(); - thr->mset.Add(s->GetId(), false, thr->fast_state.epoch()); - if (common_flags()->detect_deadlocks) { - pre_lock = - (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock); - Callback cb(thr, pc); - if (pre_lock) - ctx->dd->MutexBeforeLock(&cb, &s->dd, false); - ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock); - } - mid = s->GetId(); } if (report_bad_lock) - ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid); + ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, + creation_stack_id); if (pre_lock && common_flags()->detect_deadlocks) { Callback cb(thr, pc); ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); @@ -309,31 +322,39 @@ void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr); - if (IsAppMem(addr)) + if (pc && IsAppMem(addr)) MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); - u64 mid = 0; + RecordMutexUnlock(thr, addr); + StackID creation_stack_id; bool report_bad_unlock = false; { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); - if (s->owner_tid != kInvalidTid) { - if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { - s->SetFlags(MutexFlagBroken); - report_bad_unlock = true; + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + bool released = false; + { + Lock lock(&s->mtx); + creation_stack_id = s->creation_stack_id; + if (s->owner_tid != kInvalidTid) { + if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_unlock = true; + } + } + if (!thr->ignore_sync) { + thr->clock.Release(&s->read_clock); + released = true; + } + if (common_flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false); } } - ReleaseImpl(thr, pc, &s->read_clock); - if (common_flags()->detect_deadlocks && s->recursion == 0) { - Callback cb(thr, pc); - ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false); - } - mid = s->GetId(); + if (released) + IncrementEpoch(thr); } - thr->mset.Del(mid, false); if (report_bad_unlock) - ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid); + ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, + creation_stack_id); if (common_flags()->detect_deadlocks) { Callback cb(thr, pc); ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); @@ -342,44 +363,52 @@ void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr); - if (IsAppMem(addr)) + if (pc && IsAppMem(addr)) MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic); - u64 mid = 0; + RecordMutexUnlock(thr, addr); + StackID creation_stack_id; bool report_bad_unlock = false; + bool write = true; { - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - Lock l(&s->mtx); - bool write = true; - if (s->owner_tid == kInvalidTid) { - // Seems to be read unlock. - write = false; - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); - ReleaseImpl(thr, pc, &s->read_clock); - } else if (s->owner_tid == thr->tid) { - // Seems to be write unlock. - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); - CHECK_GT(s->recursion, 0); - s->recursion--; - if (s->recursion == 0) { - s->owner_tid = kInvalidTid; - ReleaseStoreImpl(thr, pc, &s->clock); - } else { + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + bool released = false; + { + Lock lock(&s->mtx); + creation_stack_id = s->creation_stack_id; + if (s->owner_tid == kInvalidTid) { + // Seems to be read unlock. + write = false; + if (!thr->ignore_sync) { + thr->clock.Release(&s->read_clock); + released = true; + } + } else if (s->owner_tid == thr->tid) { + // Seems to be write unlock. + CHECK_GT(s->recursion, 0); + s->recursion--; + if (s->recursion == 0) { + s->owner_tid = kInvalidTid; + if (!thr->ignore_sync) { + thr->clock.ReleaseStore(&s->clock); + released = true; + } + } + } else if (!s->IsFlagSet(MutexFlagBroken)) { + s->SetFlags(MutexFlagBroken); + report_bad_unlock = true; + } + if (common_flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write); } - } else if (!s->IsFlagSet(MutexFlagBroken)) { - s->SetFlags(MutexFlagBroken); - report_bad_unlock = true; } - thr->mset.Del(s->GetId(), write); - if (common_flags()->detect_deadlocks && s->recursion == 0) { - Callback cb(thr, pc); - ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write); - } - mid = s->GetId(); + if (released) + IncrementEpoch(thr); } if (report_bad_unlock) - ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); + ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, + creation_stack_id); if (common_flags()->detect_deadlocks) { Callback cb(thr, pc); ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); @@ -388,151 +417,120 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { void MutexRepair(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr); - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - Lock l(&s->mtx); + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + Lock lock(&s->mtx); s->owner_tid = kInvalidTid; s->recursion = 0; } void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr); - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); - ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, s->GetId()); + StackID creation_stack_id = kInvalidStackID; + { + SlotLocker locker(thr); + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true); + if (s) + creation_stack_id = s->creation_stack_id; + } + ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, + creation_stack_id); } void Acquire(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: Acquire %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = ctx->metamap.GetSyncIfExists(addr); + auto s = ctx->metamap.GetSyncIfExists(addr); if (!s) return; - ReadLock l(&s->mtx); - AcquireImpl(thr, pc, &s->clock); -} - -static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) { - ThreadState *thr = reinterpret_cast(arg); - ThreadContext *tctx = static_cast(tctx_base); - u64 epoch = tctx->epoch1; - if (tctx->status == ThreadStatusRunning) { - epoch = tctx->thr->fast_state.epoch(); - tctx->thr->clock.NoteGlobalAcquire(epoch); - } - thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); + SlotLocker locker(thr); + if (!s->clock) + return; + ReadLock lock(&s->mtx); + thr->clock.Acquire(s->clock); } void AcquireGlobal(ThreadState *thr) { DPrintf("#%d: AcquireGlobal\n", thr->tid); if (thr->ignore_sync) return; - ThreadRegistryLock l(&ctx->thread_registry); - ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateClockCallback, thr); -} - -void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) { - DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr); - if (thr->ignore_sync) - return; - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - ReleaseStoreAcquireImpl(thr, pc, &s->clock); + SlotLocker locker(thr); + for (auto &slot : ctx->slots) thr->clock.Set(slot.sid, slot.epoch()); } void Release(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: Release %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - ReleaseImpl(thr, pc, &s->clock); + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); + Lock lock(&s->mtx); + thr->clock.Release(&s->clock); + } + IncrementEpoch(thr); } void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); - Lock l(&s->mtx); - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - ReleaseStoreImpl(thr, pc, &s->clock); + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); + Lock lock(&s->mtx); + thr->clock.ReleaseStore(&s->clock); + } + IncrementEpoch(thr); +} + +void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr); + if (thr->ignore_sync) + return; + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false); + Lock lock(&s->mtx); + thr->clock.ReleaseStoreAcquire(&s->clock); + } + IncrementEpoch(thr); +} + +void IncrementEpoch(ThreadState *thr) { + DCHECK(!thr->ignore_sync); + DCHECK(thr->slot_locked); + Epoch epoch = EpochInc(thr->fast_state.epoch()); + if (!EpochOverflow(epoch)) { + Sid sid = thr->fast_state.sid(); + thr->clock.Set(sid, epoch); + thr->fast_state.SetEpoch(epoch); + thr->slot->SetEpoch(epoch); + TraceTime(thr); + } } #if !SANITIZER_GO -static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { - ThreadState *thr = reinterpret_cast(arg); - ThreadContext *tctx = static_cast(tctx_base); - u64 epoch = tctx->epoch1; - if (tctx->status == ThreadStatusRunning) - epoch = tctx->thr->fast_state.epoch(); - thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); -} - void AfterSleep(ThreadState *thr, uptr pc) { DPrintf("#%d: AfterSleep\n", thr->tid); if (thr->ignore_sync) return; thr->last_sleep_stack_id = CurrentStackId(thr, pc); - ThreadRegistryLock l(&ctx->thread_registry); - ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateSleepClockCallback, - thr); + thr->last_sleep_clock.Reset(); + SlotLocker locker(thr); + for (auto &slot : ctx->slots) + thr->last_sleep_clock.Set(slot.sid, slot.epoch()); } #endif -void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { - if (thr->ignore_sync) - return; - thr->clock.set(thr->fast_state.epoch()); - thr->clock.acquire(&thr->proc()->clock_cache, c); -} - -void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { - if (thr->ignore_sync) - return; - thr->clock.set(thr->fast_state.epoch()); - thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c); -} - -void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { - if (thr->ignore_sync) - return; - thr->clock.set(thr->fast_state.epoch()); - thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.release(&thr->proc()->clock_cache, c); -} - -void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { - if (thr->ignore_sync) - return; - thr->clock.set(thr->fast_state.epoch()); - thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.ReleaseStore(&thr->proc()->clock_cache, c); -} - -void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { - if (thr->ignore_sync) - return; - thr->clock.set(thr->fast_state.epoch()); - thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.acq_rel(&thr->proc()->clock_cache, c); -} - void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock)) return; ThreadRegistryLock l(&ctx->thread_registry); ScopedReport rep(ReportTypeDeadlock); for (int i = 0; i < r->n; i++) { - rep.AddMutex(r->loop[i].mtx_ctx0); + rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]); rep.AddUniqueTid((int)r->loop[i].thr_ctx); rep.AddThread((int)r->loop[i].thr_ctx); } @@ -540,7 +538,7 @@ void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { for (int i = 0; i < r->n; i++) { for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) { u32 stk = r->loop[i].stk[j]; - if (stk && stk != 0xffffffff) { + if (stk && stk != kInvalidStackID) { rep.AddStack(StackDepotGet(stk), true); } else { // Sometimes we fail to extract the stack trace (FIXME: investigate), @@ -552,4 +550,28 @@ void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { OutputReport(thr, rep); } +void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr, + FastState last_lock, StackID creation_stack_id) { + // We need to lock the slot during RestoreStack because it protects + // the slot journal. + Lock slot_lock(&ctx->slots[static_cast(last_lock.sid())].mtx); + ThreadRegistryLock l0(&ctx->thread_registry); + Lock slots_lock(&ctx->slot_mtx); + ScopedReport rep(ReportTypeMutexDestroyLocked); + rep.AddMutex(addr, creation_stack_id); + VarSizeStackTrace trace; + ObtainCurrentStack(thr, pc, &trace); + rep.AddStack(trace, true); + + Tid tid; + DynamicMutexSet mset; + uptr tag; + if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr, + 0, kAccessWrite, &tid, &trace, mset, &tag)) + return; + rep.AddStack(trace, true); + rep.AddLocation(addr, 1); + OutputReport(thr, rep); +} + } // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp index def61cca14d5..5acc3967208e 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp @@ -35,7 +35,6 @@ void ProcDestroy(Processor *proc) { #if !SANITIZER_GO AllocatorProcFinish(proc); #endif - ctx->clock_alloc.FlushCache(&proc->clock_cache); ctx->metamap.OnProcIdle(proc); if (common_flags()->detect_deadlocks) ctx->dd->DestroyPhysicalThread(proc->dd_pt); diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp index f332a6a8d1d8..58949ead07b3 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp @@ -175,22 +175,26 @@ void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) { } void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, - StackTrace stack, const MutexSet *mset) { + Tid tid, StackTrace stack, + const MutexSet *mset) { + uptr addr0, size; + AccessType typ; + s.GetAccess(&addr0, &size, &typ); auto *mop = New(); rep_->mops.PushBack(mop); - mop->tid = s.tid(); - mop->addr = addr + s.addr0(); - mop->size = s.size(); - mop->write = s.IsWrite(); - mop->atomic = s.IsAtomic(); + mop->tid = tid; + mop->addr = addr + addr0; + mop->size = size; + mop->write = !(typ & kAccessRead); + mop->atomic = typ & kAccessAtomic; mop->stack = SymbolizeStack(stack); mop->external_tag = external_tag; if (mop->stack) mop->stack->suppressable = true; for (uptr i = 0; i < mset->Size(); i++) { MutexSet::Desc d = mset->Get(i); - u64 mid = this->AddMutex(d.id); - ReportMopMutex mtx = {mid, d.write}; + int id = this->AddMutex(d.addr, d.stack_id); + ReportMopMutex mtx = {id, d.write}; mop->mset.PushBack(mtx); } } @@ -219,18 +223,6 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) { } #if !SANITIZER_GO -static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) { - int unique_id = *(int *)arg; - return tctx->unique_id == (u32)unique_id; -} - -static ThreadContext *FindThreadByUidLocked(Tid unique_id) { - ctx->thread_registry.CheckLocked(); - return static_cast( - ctx->thread_registry.FindThreadContextLocked( - FindThreadByUidLockedCallback, &unique_id)); -} - static ThreadContext *FindThreadByTidLocked(Tid tid) { ctx->thread_registry.CheckLocked(); return static_cast( @@ -262,55 +254,24 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { } #endif -void ScopedReportBase::AddThread(Tid unique_tid, bool suppressable) { +void ScopedReportBase::AddThread(Tid tid, bool suppressable) { #if !SANITIZER_GO - if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid)) + if (const ThreadContext *tctx = FindThreadByTidLocked(tid)) AddThread(tctx, suppressable); #endif } -void ScopedReportBase::AddMutex(const SyncVar *s) { +int ScopedReportBase::AddMutex(uptr addr, StackID creation_stack_id) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { - if (rep_->mutexes[i]->id == s->uid) - return; + if (rep_->mutexes[i]->addr == addr) + return rep_->mutexes[i]->id; } auto *rm = New(); rep_->mutexes.PushBack(rm); - rm->id = s->uid; - rm->addr = s->addr; - rm->destroyed = false; - rm->stack = SymbolizeStackId(s->creation_stack_id); -} - -u64 ScopedReportBase::AddMutex(u64 id) { - u64 uid = 0; - u64 mid = id; - uptr addr = SyncVar::SplitId(id, &uid); - SyncVar *s = ctx->metamap.GetSyncIfExists(addr); - // Check that the mutex is still alive. - // Another mutex can be created at the same address, - // so check uid as well. - if (s && s->CheckId(uid)) { - Lock l(&s->mtx); - mid = s->uid; - AddMutex(s); - } else { - AddDeadMutex(id); - } - return mid; -} - -void ScopedReportBase::AddDeadMutex(u64 id) { - for (uptr i = 0; i < rep_->mutexes.Size(); i++) { - if (rep_->mutexes[i]->id == id) - return; - } - auto *rm = New(); - rep_->mutexes.PushBack(rm); - rm->id = id; - rm->addr = 0; - rm->destroyed = true; - rm->stack = 0; + rm->id = rep_->mutexes.Size() - 1; + rm->addr = addr; + rm->stack = SymbolizeStackId(creation_stack_id); + return rm->id; } void ScopedReportBase::AddLocation(uptr addr, uptr size) { @@ -327,7 +288,7 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) { loc->tid = creat_tid; loc->stack = SymbolizeStackId(creat_stack); rep_->locs.PushBack(loc); - ThreadContext *tctx = FindThreadByUidLocked(creat_tid); + ThreadContext *tctx = FindThreadByTidLocked(creat_tid); if (tctx) AddThread(tctx); return; @@ -343,16 +304,15 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) { if (!b) b = JavaHeapBlock(addr, &block_begin); if (b != 0) { - ThreadContext *tctx = FindThreadByTidLocked(b->tid); auto *loc = New(); loc->type = ReportLocationHeap; loc->heap_chunk_start = block_begin; loc->heap_chunk_size = b->siz; loc->external_tag = b->tag; - loc->tid = tctx ? tctx->tid : b->tid; + loc->tid = b->tid; loc->stack = SymbolizeStackId(b->stk); rep_->locs.PushBack(loc); - if (tctx) + if (ThreadContext *tctx = FindThreadByTidLocked(b->tid)) AddThread(tctx); return; } @@ -387,71 +347,6 @@ ScopedReport::ScopedReport(ReportType typ, uptr tag) ScopedReport::~ScopedReport() {} -void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk, - MutexSet *mset, uptr *tag) { - // This function restores stack trace and mutex set for the thread/epoch. - // It does so by getting stack trace and mutex set at the beginning of - // trace part, and then replaying the trace till the given epoch. - Trace* trace = ThreadTrace(tid); - ReadLock l(&trace->mtx); - const int partidx = (epoch / kTracePartSize) % TraceParts(); - TraceHeader* hdr = &trace->headers[partidx]; - if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize) - return; - CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0); - const u64 epoch0 = RoundDown(epoch, TraceSize()); - const u64 eend = epoch % TraceSize(); - const u64 ebegin = RoundDown(eend, kTracePartSize); - DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n", - tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx); - Vector stack; - stack.Resize(hdr->stack0.size + 64); - for (uptr i = 0; i < hdr->stack0.size; i++) { - stack[i] = hdr->stack0.trace[i]; - DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]); - } - if (mset) - *mset = hdr->mset0; - uptr pos = hdr->stack0.size; - Event *events = (Event*)GetThreadTrace(tid); - for (uptr i = ebegin; i <= eend; i++) { - Event ev = events[i]; - EventType typ = (EventType)(ev >> kEventPCBits); - uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1)); - DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc); - if (typ == EventTypeMop) { - stack[pos] = pc; - } else if (typ == EventTypeFuncEnter) { - if (stack.Size() < pos + 2) - stack.Resize(pos + 2); - stack[pos++] = pc; - } else if (typ == EventTypeFuncExit) { - if (pos > 0) - pos--; - } - if (mset) { - if (typ == EventTypeLock) { - mset->Add(pc, true, epoch0 + i); - } else if (typ == EventTypeUnlock) { - mset->Del(pc, true); - } else if (typ == EventTypeRLock) { - mset->Add(pc, false, epoch0 + i); - } else if (typ == EventTypeRUnlock) { - mset->Del(pc, false); - } - } - for (uptr j = 0; j <= pos; j++) - DPrintf2(" #%zu: %zx\n", j, stack[j]); - } - if (pos == 0 && stack[0] == 0) - return; - pos++; - stk->Init(&stack[0], pos); - ExtractTagFromStack(stk, tag); -} - -namespace v3 { - // Replays the trace up to last_pos position in the last part // or up to the provided epoch/sid (whichever is earlier) // and calls the provided function f for each event. @@ -469,6 +364,7 @@ void TraceReplay(Trace *trace, TracePart *last, Event *last_pos, Sid sid, Event *end = &part->events[TracePart::kSize - 1]; if (part == last) end = last_pos; + f(kFreeSid, kEpochOver, nullptr); // notify about part start for (Event *evp = &part->events[0]; evp < end; evp++) { Event *evp0 = evp; if (!evp->is_access && !evp->is_func) { @@ -528,21 +424,36 @@ static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2, return addr1 >= addr2 && addr1 + size1 <= addr2 + size2; } -// Replays the trace of thread tid up to the target event identified -// by sid/epoch/addr/size/typ and restores and returns stack, mutex set +// Replays the trace of slot sid up to the target event identified +// by epoch/addr/size/typ and restores and returns tid, stack, mutex set // and tag for that event. If there are multiple such events, it returns // the last one. Returns false if the event is not present in the trace. -bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, - uptr size, AccessType typ, VarSizeStackTrace *pstk, +bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size, + AccessType typ, Tid *ptid, VarSizeStackTrace *pstk, MutexSet *pmset, uptr *ptag) { // This function restores stack trace and mutex set for the thread/epoch. // It does so by getting stack trace and mutex set at the beginning of // trace part, and then replaying the trace till the given epoch. - DPrintf2("RestoreStack: tid=%u sid=%u@%u addr=0x%zx/%zu typ=%x\n", tid, + DPrintf2("RestoreStack: sid=%u@%u addr=0x%zx/%zu typ=%x\n", static_cast(sid), static_cast(epoch), addr, size, static_cast(typ)); ctx->slot_mtx.CheckLocked(); // needed to prevent trace part recycling ctx->thread_registry.CheckLocked(); + TidSlot *slot = &ctx->slots[static_cast(sid)]; + Tid tid = kInvalidTid; + // Need to lock the slot mutex as it protects slot->journal. + slot->mtx.CheckLocked(); + for (uptr i = 0; i < slot->journal.Size(); i++) { + DPrintf2(" journal: epoch=%d tid=%d\n", + static_cast(slot->journal[i].epoch), slot->journal[i].tid); + if (i == slot->journal.Size() - 1 || slot->journal[i + 1].epoch > epoch) { + tid = slot->journal[i].tid; + break; + } + } + if (tid == kInvalidTid) + return false; + *ptid = tid; ThreadContext *tctx = static_cast(ctx->thread_registry.GetThreadLocked(tid)); Trace *trace = &tctx->trace; @@ -553,8 +464,10 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, { Lock lock(&trace->mtx); first_part = trace->parts.Front(); - if (!first_part) + if (!first_part) { + DPrintf2("RestoreStack: tid=%d trace=%p no trace parts\n", tid, trace); return false; + } last_part = trace->parts.Back(); last_pos = trace->final_pos; if (tctx->thr) @@ -567,9 +480,18 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, bool is_read = typ & kAccessRead; bool is_atomic = typ & kAccessAtomic; bool is_free = typ & kAccessFree; + DPrintf2("RestoreStack: tid=%d parts=[%p-%p] last_pos=%p\n", tid, + trace->parts.Front(), last_part, last_pos); TraceReplay( trace, last_part, last_pos, sid, epoch, [&](Sid ev_sid, Epoch ev_epoch, Event *evp) { + if (evp == nullptr) { + // Each trace part is self-consistent, so we reset state. + stack.Resize(0); + mset->Reset(); + prev_pc = 0; + return; + } bool match = ev_sid == sid && ev_epoch == epoch; if (evp->is_access) { if (evp->is_func == 0 && evp->type == EventType::kAccessExt && @@ -592,12 +514,15 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, if (evp->is_func) { auto *ev = reinterpret_cast(evp); if (ev->pc) { - DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc); + DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc); stack.PushBack(ev->pc); } else { - DPrintf2(" FuncExit\n"); - CHECK(stack.Size()); - stack.PopBack(); + DPrintf2(" FuncExit\n"); + // We don't log pathologically large stacks in each part, + // if the stack was truncated we can have more func exits than + // entries. + if (stack.Size()) + stack.PopBack(); } return; } @@ -666,8 +591,6 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr, return found; } -} // namespace v3 - bool RacyStacks::operator==(const RacyStacks &other) const { if (hash[0] == other.hash[0] && hash[1] == other.hash[1]) return true; @@ -758,10 +681,7 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) { ctx->fired_suppressions.push_back(s); } { - bool old_is_freeing = thr->is_freeing; - thr->is_freeing = false; bool suppressed = OnReport(rep, pc_or_addr != 0); - thr->is_freeing = old_is_freeing; if (suppressed) { thr->current_report = nullptr; return false; @@ -808,97 +728,72 @@ static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) { return false; } -static bool RaceBetweenAtomicAndFree(ThreadState *thr) { - Shadow s0(thr->racy_state[0]); - Shadow s1(thr->racy_state[1]); - CHECK(!(s0.IsAtomic() && s1.IsAtomic())); - if (!s0.IsAtomic() && !s1.IsAtomic()) - return true; - if (s0.IsAtomic() && s1.IsFreed()) - return true; - if (s1.IsAtomic() && thr->is_freeing) - return true; - return false; -} - -void ReportRace(ThreadState *thr) { +void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, + AccessType typ0) { CheckedMutex::CheckNoLocks(); // Symbolizer makes lots of intercepted calls. If we try to process them, // at best it will cause deadlocks on internal mutexes. ScopedIgnoreInterceptors ignore; + uptr addr = ShadowToMem(shadow_mem); + DPrintf("#%d: ReportRace %p\n", thr->tid, (void *)addr); if (!ShouldReport(thr, ReportTypeRace)) return; - if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr)) - return; - - bool freed = false; - { - Shadow s(thr->racy_state[1]); - freed = s.GetFreedAndReset(); - thr->racy_state[1] = s.raw(); - } - - uptr addr = ShadowToMem(thr->racy_shadow_addr); - uptr addr_min = 0; - uptr addr_max = 0; - { - uptr a0 = addr + Shadow(thr->racy_state[0]).addr0(); - uptr a1 = addr + Shadow(thr->racy_state[1]).addr0(); - uptr e0 = a0 + Shadow(thr->racy_state[0]).size(); - uptr e1 = a1 + Shadow(thr->racy_state[1]).size(); - addr_min = min(a0, a1); - addr_max = max(e0, e1); - if (IsExpectedReport(addr_min, addr_max - addr_min)) - return; - } - if (HandleRacyAddress(thr, addr_min, addr_max)) - return; - - ReportType typ = ReportTypeRace; - if (thr->is_vptr_access && freed) - typ = ReportTypeVptrUseAfterFree; - else if (thr->is_vptr_access) - typ = ReportTypeVptrRace; - else if (freed) - typ = ReportTypeUseAfterFree; - - if (IsFiredSuppression(ctx, typ, addr)) + uptr addr_off0, size0; + cur.GetAccess(&addr_off0, &size0, nullptr); + uptr addr_off1, size1, typ1; + old.GetAccess(&addr_off1, &size1, &typ1); + if (!flags()->report_atomic_races && + ((typ0 & kAccessAtomic) || (typ1 & kAccessAtomic)) && + !(typ0 & kAccessFree) && !(typ1 & kAccessFree)) return; const uptr kMop = 2; - VarSizeStackTrace traces[kMop]; - uptr tags[kMop] = {kExternalTagNone}; - uptr toppc = TraceTopPC(thr); - if (toppc >> kEventPCBits) { - // This is a work-around for a known issue. - // The scenario where this happens is rather elaborate and requires - // an instrumented __sanitizer_report_error_summary callback and - // a __tsan_symbolize_external callback and a race during a range memory - // access larger than 8 bytes. MemoryAccessRange adds the current PC to - // the trace and starts processing memory accesses. A first memory access - // triggers a race, we report it and call the instrumented - // __sanitizer_report_error_summary, which adds more stuff to the trace - // since it is intrumented. Then a second memory access in MemoryAccessRange - // also triggers a race and we get here and call TraceTopPC to get the - // current PC, however now it contains some unrelated events from the - // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit - // event. Later we subtract -1 from it (in GetPreviousInstructionPc) - // and the resulting PC has kExternalPCBit set, so we pass it to - // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its - // rights to crash since the PC is completely bogus. - // test/tsan/double_race.cpp contains a test case for this. - toppc = 0; - } - ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]); - if (IsFiredSuppression(ctx, typ, traces[0])) + Shadow s[kMop] = {cur, old}; + uptr addr0 = addr + addr_off0; + uptr addr1 = addr + addr_off1; + uptr end0 = addr0 + size0; + uptr end1 = addr1 + size1; + uptr addr_min = min(addr0, addr1); + uptr addr_max = max(end0, end1); + if (IsExpectedReport(addr_min, addr_max - addr_min)) + return; + if (HandleRacyAddress(thr, addr_min, addr_max)) return; - DynamicMutexSet mset2; - Shadow s2(thr->racy_state[1]); - RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]); - if (IsFiredSuppression(ctx, typ, traces[1])) + ReportType rep_typ = ReportTypeRace; + if ((typ0 & kAccessVptr) && (typ1 & kAccessFree)) + rep_typ = ReportTypeVptrUseAfterFree; + else if (typ0 & kAccessVptr) + rep_typ = ReportTypeVptrRace; + else if (typ1 & kAccessFree) + rep_typ = ReportTypeUseAfterFree; + + if (IsFiredSuppression(ctx, rep_typ, addr)) + return; + + VarSizeStackTrace traces[kMop]; + Tid tids[kMop] = {thr->tid, kInvalidTid}; + uptr tags[kMop] = {kExternalTagNone, kExternalTagNone}; + + ObtainCurrentStack(thr, thr->trace_prev_pc, &traces[0], &tags[0]); + if (IsFiredSuppression(ctx, rep_typ, traces[0])) + return; + + DynamicMutexSet mset1; + MutexSet *mset[kMop] = {&thr->mset, mset1}; + + // We need to lock the slot during RestoreStack because it protects + // the slot journal. + Lock slot_lock(&ctx->slots[static_cast(s[1].sid())].mtx); + ThreadRegistryLock l0(&ctx->thread_registry); + Lock slots_lock(&ctx->slot_mtx); + if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1, + size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) + return; + + if (IsFiredSuppression(ctx, rep_typ, traces[1])) return; if (HandleRacyStacks(thr, traces)) @@ -908,39 +803,29 @@ void ReportRace(ThreadState *thr) { uptr tag = kExternalTagNone; for (uptr i = 0; i < kMop; i++) { if (tags[i] != kExternalTagNone) { - typ = ReportTypeExternalRace; + rep_typ = ReportTypeExternalRace; tag = tags[i]; break; } } - ThreadRegistryLock l0(&ctx->thread_registry); - ScopedReport rep(typ, tag); - for (uptr i = 0; i < kMop; i++) { - Shadow s(thr->racy_state[i]); - rep.AddMemoryAccess(addr, tags[i], s, traces[i], - i == 0 ? &thr->mset : mset2); - } + ScopedReport rep(rep_typ, tag); + for (uptr i = 0; i < kMop; i++) + rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]); for (uptr i = 0; i < kMop; i++) { - FastState s(thr->racy_state[i]); ThreadContext *tctx = static_cast( - ctx->thread_registry.GetThreadLocked(s.tid())); - if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1) - continue; + ctx->thread_registry.GetThreadLocked(tids[i])); rep.AddThread(tctx); } rep.AddLocation(addr_min, addr_max - addr_min); #if !SANITIZER_GO - { - Shadow s(thr->racy_state[1]); - if (s.epoch() <= thr->last_sleep_clock.get(s.tid())) - rep.AddSleep(thr->last_sleep_stack_id); - } + if (!((typ0 | typ1) & kAccessFree) && + s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid())) + rep.AddSleep(thr->last_sleep_stack_id); #endif - OutputReport(thr, rep); } diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp index c8f7124c009d..86c8b3764cc7 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp @@ -21,20 +21,14 @@ namespace __tsan { // ThreadContext implementation. -ThreadContext::ThreadContext(Tid tid) - : ThreadContextBase(tid), thr(), sync(), epoch0(), epoch1() {} +ThreadContext::ThreadContext(Tid tid) : ThreadContextBase(tid), thr(), sync() {} #if !SANITIZER_GO ThreadContext::~ThreadContext() { } #endif -void ThreadContext::OnReset() { - CHECK_EQ(sync.size(), 0); - uptr trace_p = GetThreadTrace(tid); - ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event)); - //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace)); -} +void ThreadContext::OnReset() { CHECK(!sync); } #if !SANITIZER_GO struct ThreadLeak { @@ -57,7 +51,9 @@ static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) { } #endif -#if !SANITIZER_GO +// Disabled on Mac because lldb test TestTsanBasic fails: +// https://reviews.llvm.org/D112603#3163158 +#if !SANITIZER_GO && !SANITIZER_MAC static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) { if (tctx->tid == kMainTid) { Printf("ThreadSanitizer: main thread finished with ignores enabled\n"); @@ -112,30 +108,35 @@ int ThreadCount(ThreadState *thr) { } struct OnCreatedArgs { - ThreadState *thr; - uptr pc; + VectorClock *sync; + uptr sync_epoch; + StackID stack; }; Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { - OnCreatedArgs args = { thr, pc }; - u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers. - Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent_tid, &args); - DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid); + // The main thread and GCD workers don't have a parent thread. + Tid parent = kInvalidTid; + OnCreatedArgs arg = {nullptr, 0, kInvalidStackID}; + if (thr) { + parent = thr->tid; + arg.stack = CurrentStackId(thr, pc); + if (!thr->ignore_sync) { + SlotLocker locker(thr); + thr->clock.ReleaseStore(&arg.sync); + arg.sync_epoch = ctx->global_epoch; + IncrementEpoch(thr); + } + } + Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent, &arg); + DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent, tid, uid); return tid; } void ThreadContext::OnCreated(void *arg) { - thr = 0; - if (tid == kMainTid) - return; OnCreatedArgs *args = static_cast(arg); - if (!args->thr) // GCD workers don't have a parent thread. - return; - args->thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); - ReleaseImpl(args->thr, 0, &sync); - creation_stack_id = CurrentStackId(args->thr, args->pc); + sync = args->sync; + sync_epoch = args->sync_epoch; + creation_stack_id = args->stack; } extern "C" void __tsan_stack_initialization() {} @@ -150,6 +151,15 @@ struct OnStartedArgs { void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, ThreadType thread_type) { + ctx->thread_registry.StartThread(tid, os_id, thread_type, thr); + if (!thr->ignore_sync) { + SlotAttachAndLock(thr); + if (thr->tctx->sync_epoch == ctx->global_epoch) + thr->clock.Acquire(thr->tctx->sync); + SlotUnlock(thr); + } + Free(thr->tctx->sync); + uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; @@ -159,12 +169,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr, &tls_size); #endif - - ThreadRegistry *tr = &ctx->thread_registry; - OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; - tr->StartThread(tid, os_id, thread_type, &args); - - while (!thr->tctx->trace.parts.Empty()) thr->tctx->trace.parts.PopBack(); + thr->stk_addr = stk_addr; + thr->stk_size = stk_size; + thr->tls_addr = tls_addr; + thr->tls_size = tls_size; #if !SANITIZER_GO if (ctx->after_multithreaded_fork) { @@ -192,57 +200,41 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, } void ThreadContext::OnStarted(void *arg) { - OnStartedArgs *args = static_cast(arg); - thr = args->thr; - // RoundUp so that one trace part does not contain events - // from different threads. - epoch0 = RoundUp(epoch1 + 1, kTracePartSize); - epoch1 = (u64)-1; - new (thr) - ThreadState(ctx, tid, unique_id, epoch0, reuse_count, args->stk_addr, - args->stk_size, args->tls_addr, args->tls_size); + thr = static_cast(arg); + DPrintf("#%d: ThreadStart\n", tid); + new (thr) ThreadState(tid); if (common_flags()->detect_deadlocks) - thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id); - thr->fast_state.SetHistorySize(flags()->history_size); - // Commit switch to the new part of the trace. - // TraceAddEvent will reset stack0/mset0 in the new part for us. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - - thr->fast_synch_epoch = epoch0; - AcquireImpl(thr, 0, &sync); - sync.Reset(&thr->proc()->clock_cache); + thr->dd_lt = ctx->dd->CreateLogicalThread(tid); thr->tctx = this; +#if !SANITIZER_GO thr->is_inited = true; - DPrintf( - "#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " - "tls_addr=%zx tls_size=%zx\n", - tid, (uptr)epoch0, args->stk_addr, args->stk_size, args->tls_addr, - args->tls_size); +#endif } void ThreadFinish(ThreadState *thr) { + DPrintf("#%d: ThreadFinish\n", thr->tid); ThreadCheckIgnore(thr); if (thr->stk_addr && thr->stk_size) DontNeedShadowFor(thr->stk_addr, thr->stk_size); if (thr->tls_addr && thr->tls_size) DontNeedShadowFor(thr->tls_addr, thr->tls_size); thr->is_dead = true; - thr->is_inited = false; #if !SANITIZER_GO + thr->is_inited = false; thr->ignore_interceptors++; + PlatformCleanUpThreadState(thr); #endif - ctx->thread_registry.FinishThread(thr->tid); -} - -void ThreadContext::OnFinished() { - if (!detached) { - thr->fast_state.IncrementEpoch(); - // Can't increment epoch w/o writing to the trace as well. - TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - ReleaseImpl(thr, 0, &sync); + if (!thr->ignore_sync) { + SlotLocker locker(thr); + ThreadRegistryLock lock(&ctx->thread_registry); + // Note: detached is protected by the thread registry mutex, + // the thread may be detaching concurrently in another thread. + if (!thr->tctx->detached) { + thr->clock.ReleaseStore(&thr->tctx->sync); + thr->tctx->sync_epoch = ctx->global_epoch; + IncrementEpoch(thr); + } } - epoch1 = thr->fast_state.epoch(); - #if !SANITIZER_GO UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr)); #else @@ -251,18 +243,37 @@ void ThreadContext::OnFinished() { thr->shadow_stack = nullptr; thr->shadow_stack_pos = nullptr; thr->shadow_stack_end = nullptr; - if (common_flags()->detect_deadlocks) ctx->dd->DestroyLogicalThread(thr->dd_lt); - thr->clock.ResetCached(&thr->proc()->clock_cache); -#if !SANITIZER_GO - thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); -#endif -#if !SANITIZER_GO - PlatformCleanUpThreadState(thr); -#endif + SlotDetach(thr); + ctx->thread_registry.FinishThread(thr->tid); thr->~ThreadState(); - thr = 0; +} + +void ThreadContext::OnFinished() { + Lock lock(&ctx->slot_mtx); + Lock lock1(&trace.mtx); + // Queue all trace parts into the global recycle queue. + auto parts = &trace.parts; + while (trace.local_head) { + CHECK(parts->Queued(trace.local_head)); + ctx->trace_part_recycle.PushBack(trace.local_head); + trace.local_head = parts->Next(trace.local_head); + } + ctx->trace_part_recycle_finished += parts->Size(); + if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadHi) { + ctx->trace_part_finished_excess += parts->Size(); + trace.parts_allocated = 0; + } else if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadLo && + parts->Size() > 1) { + ctx->trace_part_finished_excess += parts->Size() - 1; + trace.parts_allocated = 1; + } + // From now on replay will use trace->final_pos. + trace.final_pos = (Event *)atomic_load_relaxed(&thr->trace_pos); + atomic_store_relaxed(&thr->trace_pos, 0); + thr->tctx = nullptr; + thr = nullptr; } struct ConsumeThreadContext { @@ -274,35 +285,43 @@ Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) { return ctx->thread_registry.ConsumeThreadUserId(uid); } +struct JoinArg { + VectorClock *sync; + uptr sync_epoch; +}; + void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) { CHECK_GT(tid, 0); - CHECK_LT(tid, kMaxTid); DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid); - ctx->thread_registry.JoinThread(tid, thr); + JoinArg arg = {}; + ctx->thread_registry.JoinThread(tid, &arg); + if (!thr->ignore_sync) { + SlotLocker locker(thr); + if (arg.sync_epoch == ctx->global_epoch) + thr->clock.Acquire(arg.sync); + } + Free(arg.sync); } -void ThreadContext::OnJoined(void *arg) { - ThreadState *caller_thr = static_cast(arg); - AcquireImpl(caller_thr, 0, &sync); - sync.Reset(&caller_thr->proc()->clock_cache); +void ThreadContext::OnJoined(void *ptr) { + auto arg = static_cast(ptr); + arg->sync = sync; + arg->sync_epoch = sync_epoch; + sync = nullptr; + sync_epoch = 0; } -void ThreadContext::OnDead() { CHECK_EQ(sync.size(), 0); } +void ThreadContext::OnDead() { CHECK_EQ(sync, nullptr); } void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) { CHECK_GT(tid, 0); - CHECK_LT(tid, kMaxTid); ctx->thread_registry.DetachThread(tid, thr); } -void ThreadContext::OnDetached(void *arg) { - ThreadState *thr1 = static_cast(arg); - sync.Reset(&thr1->proc()->clock_cache); -} +void ThreadContext::OnDetached(void *arg) { Free(sync); } void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) { CHECK_GT(tid, 0); - CHECK_LT(tid, kMaxTid); ctx->thread_registry.SetThreadUserId(tid, uid); } diff --git a/compiler-rt/lib/tsan/rtl/tsan_shadow.h b/compiler-rt/lib/tsan/rtl/tsan_shadow.h index 8b7bc341713e..843573ecf5d3 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_shadow.h +++ b/compiler-rt/lib/tsan/rtl/tsan_shadow.h @@ -10,223 +10,170 @@ #define TSAN_SHADOW_H #include "tsan_defs.h" -#include "tsan_trace.h" namespace __tsan { -// FastState (from most significant bit): -// ignore : 1 -// tid : kTidBits -// unused : - -// history_size : 3 -// epoch : kClkBits class FastState { public: - FastState(u64 tid, u64 epoch) { - x_ = tid << kTidShift; - x_ |= epoch; - DCHECK_EQ(tid, this->tid()); - DCHECK_EQ(epoch, this->epoch()); - DCHECK_EQ(GetIgnoreBit(), false); + FastState() { Reset(); } + + void Reset() { + part_.unused0_ = 0; + part_.sid_ = static_cast(kFreeSid); + part_.epoch_ = static_cast(kEpochLast); + part_.unused1_ = 0; + part_.ignore_accesses_ = false; } - explicit FastState(u64 x) : x_(x) {} + void SetSid(Sid sid) { part_.sid_ = static_cast(sid); } - u64 raw() const { return x_; } + Sid sid() const { return static_cast(part_.sid_); } - u64 tid() const { - u64 res = (x_ & ~kIgnoreBit) >> kTidShift; - return res; - } + Epoch epoch() const { return static_cast(part_.epoch_); } - u64 TidWithIgnore() const { - u64 res = x_ >> kTidShift; - return res; - } + void SetEpoch(Epoch epoch) { part_.epoch_ = static_cast(epoch); } - u64 epoch() const { - u64 res = x_ & ((1ull << kClkBits) - 1); - return res; - } - - void IncrementEpoch() { - u64 old_epoch = epoch(); - x_ += 1; - DCHECK_EQ(old_epoch + 1, epoch()); - (void)old_epoch; - } - - void SetIgnoreBit() { x_ |= kIgnoreBit; } - void ClearIgnoreBit() { x_ &= ~kIgnoreBit; } - bool GetIgnoreBit() const { return (s64)x_ < 0; } - - void SetHistorySize(int hs) { - CHECK_GE(hs, 0); - CHECK_LE(hs, 7); - x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift); - } - - ALWAYS_INLINE - int GetHistorySize() const { - return (int)((x_ >> kHistoryShift) & kHistoryMask); - } - - void ClearHistorySize() { SetHistorySize(0); } - - ALWAYS_INLINE - u64 GetTracePos() const { - const int hs = GetHistorySize(); - // When hs == 0, the trace consists of 2 parts. - const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1; - return epoch() & mask; - } + void SetIgnoreBit() { part_.ignore_accesses_ = 1; } + void ClearIgnoreBit() { part_.ignore_accesses_ = 0; } + bool GetIgnoreBit() const { return part_.ignore_accesses_; } private: friend class Shadow; - static const int kTidShift = 64 - kTidBits - 1; - static const u64 kIgnoreBit = 1ull << 63; - static const u64 kFreedBit = 1ull << 63; - static const u64 kHistoryShift = kClkBits; - static const u64 kHistoryMask = 7; - u64 x_; + struct Parts { + u32 unused0_ : 8; + u32 sid_ : 8; + u32 epoch_ : kEpochBits; + u32 unused1_ : 1; + u32 ignore_accesses_ : 1; + }; + union { + Parts part_; + u32 raw_; + }; }; -// Shadow (from most significant bit): -// freed : 1 -// tid : kTidBits -// is_atomic : 1 -// is_read : 1 -// size_log : 2 -// addr0 : 3 -// epoch : kClkBits -class Shadow : public FastState { +static_assert(sizeof(FastState) == kShadowSize, "bad FastState size"); + +class Shadow { public: - explicit Shadow(u64 x) : FastState(x) {} + static constexpr RawShadow kEmpty = static_cast(0); - explicit Shadow(const FastState &s) : FastState(s.x_) { ClearHistorySize(); } - - void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) { - DCHECK_EQ((x_ >> kClkBits) & 31, 0); - DCHECK_LE(addr0, 7); - DCHECK_LE(kAccessSizeLog, 3); - x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits; - DCHECK_EQ(kAccessSizeLog, size_log()); - DCHECK_EQ(addr0, this->addr0()); + Shadow(FastState state, u32 addr, u32 size, AccessType typ) { + raw_ = state.raw_; + DCHECK_GT(size, 0); + DCHECK_LE(size, 8); + UNUSED Sid sid0 = part_.sid_; + UNUSED u16 epoch0 = part_.epoch_; + raw_ |= (!!(typ & kAccessAtomic) << kIsAtomicShift) | + (!!(typ & kAccessRead) << kIsReadShift) | + (((((1u << size) - 1) << (addr & 0x7)) & 0xff) << kAccessShift); + // Note: we don't check kAccessAtomic because it overlaps with + // FastState::ignore_accesses_ and it may be set spuriously. + DCHECK_EQ(part_.is_read_, !!(typ & kAccessRead)); + DCHECK_EQ(sid(), sid0); + DCHECK_EQ(epoch(), epoch0); } - void SetWrite(unsigned kAccessIsWrite) { - DCHECK_EQ(x_ & kReadBit, 0); - if (!kAccessIsWrite) - x_ |= kReadBit; - DCHECK_EQ(kAccessIsWrite, IsWrite()); + explicit Shadow(RawShadow x = Shadow::kEmpty) { raw_ = static_cast(x); } + + RawShadow raw() const { return static_cast(raw_); } + Sid sid() const { return part_.sid_; } + Epoch epoch() const { return static_cast(part_.epoch_); } + u8 access() const { return part_.access_; } + + void GetAccess(uptr *addr, uptr *size, AccessType *typ) const { + DCHECK(part_.access_ != 0 || raw_ == static_cast(Shadow::kRodata)); + if (addr) + *addr = part_.access_ ? __builtin_ffs(part_.access_) - 1 : 0; + if (size) + *size = part_.access_ == kFreeAccess ? kShadowCell + : __builtin_popcount(part_.access_); + if (typ) + *typ = (part_.is_read_ ? kAccessRead : kAccessWrite) | + (part_.is_atomic_ ? kAccessAtomic : 0) | + (part_.access_ == kFreeAccess ? kAccessFree : 0); } - void SetAtomic(bool kIsAtomic) { - DCHECK(!IsAtomic()); - if (kIsAtomic) - x_ |= kAtomicBit; - DCHECK_EQ(IsAtomic(), kIsAtomic); - } - - bool IsAtomic() const { return x_ & kAtomicBit; } - - bool IsZero() const { return x_ == 0; } - - static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) { - u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift; - DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore()); - return shifted_xor == 0; - } - - static ALWAYS_INLINE bool Addr0AndSizeAreEqual(const Shadow s1, - const Shadow s2) { - u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31; - return masked_xor == 0; - } - - static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2, - unsigned kS2AccessSize) { - bool res = false; - u64 diff = s1.addr0() - s2.addr0(); - if ((s64)diff < 0) { // s1.addr0 < s2.addr0 - // if (s1.addr0() + size1) > s2.addr0()) return true; - if (s1.size() > -diff) - res = true; - } else { - // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true; - if (kS2AccessSize > diff) - res = true; - } - DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2)); - DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1)); + ALWAYS_INLINE + bool IsBothReadsOrAtomic(AccessType typ) const { + u32 is_read = !!(typ & kAccessRead); + u32 is_atomic = !!(typ & kAccessAtomic); + bool res = + raw_ & ((is_atomic << kIsAtomicShift) | (is_read << kIsReadShift)); + DCHECK_EQ(res, + (part_.is_read_ && is_read) || (part_.is_atomic_ && is_atomic)); return res; } - u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; } - u64 ALWAYS_INLINE size() const { return 1ull << size_log(); } - bool ALWAYS_INLINE IsWrite() const { return !IsRead(); } - bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; } + ALWAYS_INLINE + bool IsRWWeakerOrEqual(AccessType typ) const { + u32 is_read = !!(typ & kAccessRead); + u32 is_atomic = !!(typ & kAccessAtomic); + UNUSED u32 res0 = + (part_.is_atomic_ > is_atomic) || + (part_.is_atomic_ == is_atomic && part_.is_read_ >= is_read); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + const u32 kAtomicReadMask = (1 << kIsAtomicShift) | (1 << kIsReadShift); + bool res = (raw_ & kAtomicReadMask) >= + ((is_atomic << kIsAtomicShift) | (is_read << kIsReadShift)); - // The idea behind the freed bit is as follows. - // When the memory is freed (or otherwise unaccessible) we write to the shadow - // values with tid/epoch related to the free and the freed bit set. - // During memory accesses processing the freed bit is considered - // as msb of tid. So any access races with shadow with freed bit set - // (it is as if write from a thread with which we never synchronized before). - // This allows us to detect accesses to freed memory w/o additional - // overheads in memory access processing and at the same time restore - // tid/epoch of free. - void MarkAsFreed() { x_ |= kFreedBit; } - - bool IsFreed() const { return x_ & kFreedBit; } - - bool GetFreedAndReset() { - bool res = x_ & kFreedBit; - x_ &= ~kFreedBit; + DCHECK_EQ(res, res0); return res; +#else + return res0; +#endif } - bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const { - bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift) | - (u64(kIsAtomic) << kAtomicShift)); - DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic)); - return v; + // The FreedMarker must not pass "the same access check" so that we don't + // return from the race detection algorithm early. + static RawShadow FreedMarker() { + FastState fs; + fs.SetSid(kFreeSid); + fs.SetEpoch(kEpochLast); + Shadow s(fs, 0, 8, kAccessWrite); + return s.raw(); } - bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const { - bool v = ((x_ >> kReadShift) & 3) <= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); - DCHECK_EQ(v, (IsAtomic() < kIsAtomic) || - (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite)); - return v; - } - - bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const { - bool v = ((x_ >> kReadShift) & 3) >= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); - DCHECK_EQ(v, (IsAtomic() > kIsAtomic) || - (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite)); - return v; + static RawShadow FreedInfo(Sid sid, Epoch epoch) { + Shadow s; + s.part_.sid_ = sid; + s.part_.epoch_ = static_cast(epoch); + s.part_.access_ = kFreeAccess; + return s.raw(); } private: - static const u64 kReadShift = 5 + kClkBits; - static const u64 kReadBit = 1ull << kReadShift; - static const u64 kAtomicShift = 6 + kClkBits; - static const u64 kAtomicBit = 1ull << kAtomicShift; + struct Parts { + u8 access_; + Sid sid_; + u16 epoch_ : kEpochBits; + u16 is_read_ : 1; + u16 is_atomic_ : 1; + }; + union { + Parts part_; + u32 raw_; + }; - u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; } + static constexpr u8 kFreeAccess = 0x81; - static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) { - if (s1.addr0() == s2.addr0()) - return true; - if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0()) - return true; - if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0()) - return true; - return false; - } +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + static constexpr uptr kAccessShift = 0; + static constexpr uptr kIsReadShift = 30; + static constexpr uptr kIsAtomicShift = 31; +#else + static constexpr uptr kAccessShift = 24; + static constexpr uptr kIsReadShift = 1; + static constexpr uptr kIsAtomicShift = 0; +#endif + + public: + // .rodata shadow marker, see MapRodata and ContainsSameAccessFast. + static constexpr RawShadow kRodata = + static_cast(1 << kIsReadShift); }; -const RawShadow kShadowRodata = (RawShadow)-1; // .rodata shadow marker +static_assert(sizeof(Shadow) == kShadowSize, "bad Shadow size"); } // namespace __tsan diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp index f042abab74e5..09d41780d188 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp @@ -18,43 +18,31 @@ namespace __tsan { void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s); -SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); } +SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); } -void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, - bool save_stack) { +void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) { + Reset(); this->addr = addr; - this->uid = uid; - this->next = 0; - - creation_stack_id = kInvalidStackID; + next = 0; if (save_stack && !SANITIZER_GO) // Go does not use them creation_stack_id = CurrentStackId(thr, pc); if (common_flags()->detect_deadlocks) DDMutexInit(thr, pc, this); } -void SyncVar::Reset(Processor *proc) { - uid = 0; +void SyncVar::Reset() { + CHECK(!ctx->resetting); creation_stack_id = kInvalidStackID; owner_tid = kInvalidTid; - last_lock = 0; + last_lock.Reset(); recursion = 0; atomic_store_relaxed(&flags, 0); - - if (proc == 0) { - CHECK_EQ(clock.size(), 0); - CHECK_EQ(read_clock.size(), 0); - } else { - clock.Reset(&proc->clock_cache); - read_clock.Reset(&proc->clock_cache); - } + Free(clock); + Free(read_clock); } MetaMap::MetaMap() - : block_alloc_(LINKER_INITIALIZED, "heap block allocator"), - sync_alloc_(LINKER_INITIALIZED, "sync allocator") { - atomic_store(&uid_gen_, 0, memory_order_relaxed); -} + : block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {} void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) { u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache); @@ -68,16 +56,16 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) { *meta = idx | kFlagBlock; } -uptr MetaMap::FreeBlock(Processor *proc, uptr p) { +uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) { MBlock* b = GetBlock(p); if (b == 0) return 0; uptr sz = RoundUpTo(b->siz, kMetaShadowCell); - FreeRange(proc, p, sz); + FreeRange(proc, p, sz, reset); return sz; } -bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) { +bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) { bool has_something = false; u32 *meta = MemToMeta(p); u32 *end = MemToMeta(p + sz); @@ -99,7 +87,8 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) { DCHECK(idx & kFlagSync); SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask); u32 next = s->next; - s->Reset(proc); + if (reset) + s->Reset(); sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask); idx = next; } else { @@ -116,30 +105,30 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) { // which can be huge. The function probes pages one-by-one until it finds a page // without meta objects, at this point it stops freeing meta objects. Because // thread stacks grow top-down, we do the same starting from end as well. -void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) { +void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) { if (SANITIZER_GO) { // UnmapOrDie/MmapFixedNoReserve does not work on Windows, // so we do the optimization only for C/C++. - FreeRange(proc, p, sz); + FreeRange(proc, p, sz, reset); return; } const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize; const uptr kPageSize = GetPageSizeCached() * kMetaRatio; if (sz <= 4 * kPageSize) { // If the range is small, just do the normal free procedure. - FreeRange(proc, p, sz); + FreeRange(proc, p, sz, reset); return; } // First, round both ends of the range to page size. uptr diff = RoundUp(p, kPageSize) - p; if (diff != 0) { - FreeRange(proc, p, diff); + FreeRange(proc, p, diff, reset); p += diff; sz -= diff; } diff = p + sz - RoundDown(p + sz, kPageSize); if (diff != 0) { - FreeRange(proc, p + sz - diff, diff); + FreeRange(proc, p + sz - diff, diff, reset); sz -= diff; } // Now we must have a non-empty page-aligned range. @@ -150,7 +139,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) { const uptr sz0 = sz; // Probe start of the range. for (uptr checked = 0; sz > 0; checked += kPageSize) { - bool has_something = FreeRange(proc, p, kPageSize); + bool has_something = FreeRange(proc, p, kPageSize, reset); p += kPageSize; sz -= kPageSize; if (!has_something && checked > (128 << 10)) @@ -158,7 +147,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) { } // Probe end of the range. for (uptr checked = 0; sz > 0; checked += kPageSize) { - bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize); + bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset); sz -= kPageSize; // Stacks grow down, so sync object are most likely at the end of the region // (if it is a stack). The very end of the stack is TLS and tsan increases @@ -177,6 +166,27 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) { Die(); } +void MetaMap::ResetClocks() { + // This can be called from the background thread + // which does not have proc/cache. + // The cache is too large for stack. + static InternalAllocatorCache cache; + internal_memset(&cache, 0, sizeof(cache)); + internal_allocator()->InitCache(&cache); + sync_alloc_.ForEach([&](SyncVar *s) { + if (s->clock) { + InternalFree(s->clock, &cache); + s->clock = nullptr; + } + if (s->read_clock) { + InternalFree(s->read_clock, &cache); + s->read_clock = nullptr; + } + s->last_lock.Reset(); + }); + internal_allocator()->DestroyCache(&cache); +} + MBlock* MetaMap::GetBlock(uptr p) { u32 *meta = MemToMeta(p); u32 idx = *meta; @@ -193,6 +203,7 @@ MBlock* MetaMap::GetBlock(uptr p) { SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create, bool save_stack) { + DCHECK(!create || thr->slot_locked); u32 *meta = MemToMeta(addr); u32 idx0 = *meta; u32 myidx = 0; @@ -203,7 +214,7 @@ SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create, SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask); if (LIKELY(s->addr == addr)) { if (UNLIKELY(myidx != 0)) { - mys->Reset(thr->proc()); + mys->Reset(); sync_alloc_.Free(&thr->proc()->sync_cache, myidx); } return s; @@ -218,10 +229,9 @@ SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create, } if (LIKELY(myidx == 0)) { - const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed); myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache); mys = sync_alloc_.Map(myidx); - mys->Init(thr, pc, addr, uid, save_stack); + mys->Init(thr, pc, addr, save_stack); } mys->next = idx0; if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0, diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.h b/compiler-rt/lib/tsan/rtl/tsan_sync.h index fc8fa288a841..67d3c0b5e7dd 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_sync.h +++ b/compiler-rt/lib/tsan/rtl/tsan_sync.h @@ -16,8 +16,9 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_deadlock_detector_interface.h" #include "tsan_defs.h" -#include "tsan_clock.h" #include "tsan_dense_alloc.h" +#include "tsan_shadow.h" +#include "tsan_vector_clock.h" namespace __tsan { @@ -53,34 +54,18 @@ struct SyncVar { uptr addr; // overwritten by DenseSlabAlloc freelist Mutex mtx; - u64 uid; // Globally unique id. StackID creation_stack_id; Tid owner_tid; // Set only by exclusive owners. - u64 last_lock; + FastState last_lock; int recursion; atomic_uint32_t flags; u32 next; // in MetaMap DDMutex dd; - SyncClock read_clock; // Used for rw mutexes only. - // The clock is placed last, so that it is situated on a different cache line - // with the mtx. This reduces contention for hot sync objects. - SyncClock clock; + VectorClock *read_clock; // Used for rw mutexes only. + VectorClock *clock; - void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, bool save_stack); - void Reset(Processor *proc); - - u64 GetId() const { - // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits. - return GetLsb((u64)addr | (uid << 48), 60); - } - bool CheckId(u64 uid) const { - CHECK_EQ(uid, GetLsb(uid, 14)); - return GetLsb(this->uid, 14) == uid; - } - static uptr SplitId(u64 id, u64 *uid) { - *uid = id >> 48; - return (uptr)GetLsb(id, 48); - } + void Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack); + void Reset(); bool IsFlagSet(u32 f) const { return atomic_load_relaxed(&flags) & f; @@ -110,9 +95,20 @@ class MetaMap { MetaMap(); void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz); - uptr FreeBlock(Processor *proc, uptr p); - bool FreeRange(Processor *proc, uptr p, uptr sz); - void ResetRange(Processor *proc, uptr p, uptr sz); + + // FreeBlock resets all sync objects in the range if reset=true and must not + // run concurrently with ResetClocks which resets all sync objects + // w/o any synchronization (as part of DoReset). + // If we don't have a thread slot (very early/late in thread lifetime or + // Go/Java callbacks) or the slot is not locked, then reset must be set to + // false. In such case sync object clocks will be reset later (when it's + // reused or during the next ResetClocks). + uptr FreeBlock(Processor *proc, uptr p, bool reset); + bool FreeRange(Processor *proc, uptr p, uptr sz, bool reset); + void ResetRange(Processor *proc, uptr p, uptr sz, bool reset); + // Reset vector clocks of all sync objects. + // Must be called when no other threads access sync objects. + void ResetClocks(); MBlock* GetBlock(uptr p); SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr, @@ -142,7 +138,6 @@ class MetaMap { typedef DenseSlabAlloc SyncAlloc; BlockAlloc block_alloc_; SyncAlloc sync_alloc_; - atomic_uint64_t uid_gen_; SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create, bool save_stack); diff --git a/compiler-rt/lib/tsan/rtl/tsan_trace.h b/compiler-rt/lib/tsan/rtl/tsan_trace.h index ffc8c991ece0..01bb7b34f43a 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_trace.h +++ b/compiler-rt/lib/tsan/rtl/tsan_trace.h @@ -19,57 +19,6 @@ namespace __tsan { -const int kTracePartSizeBits = 13; -const int kTracePartSize = 1 << kTracePartSizeBits; -const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize; -const int kTraceSize = kTracePartSize * kTraceParts; - -// Must fit into 3 bits. -enum EventType { - EventTypeMop, - EventTypeFuncEnter, - EventTypeFuncExit, - EventTypeLock, - EventTypeUnlock, - EventTypeRLock, - EventTypeRUnlock -}; - -// Represents a thread event (from most significant bit): -// u64 typ : 3; // EventType. -// u64 addr : 61; // Associated pc. -typedef u64 Event; - -const uptr kEventPCBits = 61; - -struct TraceHeader { -#if !SANITIZER_GO - BufferedStackTrace stack0; // Start stack for the trace. -#else - VarSizeStackTrace stack0; -#endif - u64 epoch0; // Start epoch for the trace. - MutexSet mset0; - - TraceHeader() : stack0(), epoch0() {} -}; - -struct Trace { - Mutex mtx; -#if !SANITIZER_GO - // Must be last to catch overflow as paging fault. - // Go shadow stack is dynamically allocated. - uptr shadow_stack[kShadowStackSize]; -#endif - // Must be the last field, because we unmap the unused part in - // CreateThreadContext. - TraceHeader headers[kTraceParts]; - - Trace() : mtx(MutexTypeTrace) {} -}; - -namespace v3 { - enum class EventType : u64 { kAccessExt, kAccessRange, @@ -217,6 +166,7 @@ struct Trace; struct TraceHeader { Trace* trace = nullptr; // back-pointer to Trace containing this part INode trace_parts; // in Trace::parts + INode global; // in Contex::trace_part_recycle }; struct TracePart : TraceHeader { @@ -239,13 +189,26 @@ static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size"); struct Trace { Mutex mtx; IList parts; - Event* final_pos = - nullptr; // final position in the last part for finished threads + // First node non-queued into ctx->trace_part_recycle. + TracePart* local_head; + // Final position in the last part for finished threads. + Event* final_pos = nullptr; + // Number of trace parts allocated on behalf of this trace specifically. + // Total number of parts in this trace can be larger if we retake some + // parts from other traces. + uptr parts_allocated = 0; Trace() : mtx(MutexTypeTrace) {} -}; -} // namespace v3 + // We need at least 3 parts per thread, because we want to keep at last + // 2 parts per thread that are not queued into ctx->trace_part_recycle + // (the current one being filled and one full part that ensures that + // we always have at least one part worth of previous memory accesses). + static constexpr uptr kMinParts = 3; + + static constexpr uptr kFinishedThreadLo = 16; + static constexpr uptr kFinishedThreadHi = 64; +}; } // namespace __tsan diff --git a/compiler-rt/lib/xray/xray_allocator.h b/compiler-rt/lib/xray/xray_allocator.h index 4b42c473261d..0284f4299fb1 100644 --- a/compiler-rt/lib/xray/xray_allocator.h +++ b/compiler-rt/lib/xray/xray_allocator.h @@ -65,9 +65,9 @@ template T *allocate() XRAY_NEVER_INSTRUMENT { int ErrNo = 0; if (UNLIKELY(internal_iserror(B, &ErrNo))) { if (Verbosity()) - Report( - "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", - RoundedSize, B); + Report("XRay Profiling: Failed to allocate memory of size %zu; Error = " + "%zu\n", + RoundedSize, B); return nullptr; } #endif @@ -114,9 +114,9 @@ T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT { int ErrNo = 0; if (UNLIKELY(internal_iserror(B, &ErrNo))) { if (Verbosity()) - Report( - "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", - RoundedSize, B); + Report("XRay Profiling: Failed to allocate memory of size %zu; Error = " + "%zu\n", + RoundedSize, B); return nullptr; } #endif @@ -183,7 +183,7 @@ template struct Allocator { BackingStore = allocateBuffer(MaxMemory); if (BackingStore == nullptr) { if (Verbosity()) - Report("XRay Profiling: Failed to allocate memory for allocator.\n"); + Report("XRay Profiling: Failed to allocate memory for allocator\n"); return nullptr; } @@ -198,7 +198,7 @@ template struct Allocator { AlignedNextBlock = BackingStore = nullptr; if (Verbosity()) Report("XRay Profiling: Cannot obtain enough memory from " - "preallocated region.\n"); + "preallocated region\n"); return nullptr; } diff --git a/compiler-rt/lib/xray/xray_basic_logging.cpp b/compiler-rt/lib/xray/xray_basic_logging.cpp index a58ae9b5e267..6e83252a0516 100644 --- a/compiler-rt/lib/xray/xray_basic_logging.cpp +++ b/compiler-rt/lib/xray/xray_basic_logging.cpp @@ -345,12 +345,12 @@ static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT { if (TLD.ShadowStack) InternalFree(TLD.ShadowStack); if (Verbosity()) - Report("Cleaned up log for TID: %d\n", GetTid()); + Report("Cleaned up log for TID: %llu\n", GetTid()); }); if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) { if (Verbosity()) - Report("Skipping buffer for TID: %d; Offset = %llu\n", GetTid(), + Report("Skipping buffer for TID: %llu; Offset = %zu\n", GetTid(), TLD.BufferOffset); return; } diff --git a/compiler-rt/lib/xray/xray_hexagon.cpp b/compiler-rt/lib/xray/xray_hexagon.cpp new file mode 100644 index 000000000000..7f127b2b499c --- /dev/null +++ b/compiler-rt/lib/xray/xray_hexagon.cpp @@ -0,0 +1,168 @@ +//===-- xray_hexagon.cpp --------------------------------------*- C++ ---*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// Implementation of hexagon-specific routines (32-bit). +// +//===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_common.h" +#include "xray_defs.h" +#include "xray_interface_internal.h" +#include +#include + +namespace __xray { + +// The machine codes for some instructions used in runtime patching. +enum PatchOpcodes : uint32_t { + PO_JUMPI_14 = 0x5800c00a, // jump #0x014 (PC + 0x014) + PO_CALLR_R6 = 0x50a6c000, // indirect call: callr r6 + PO_TFR_IMM = 0x78000000, // transfer immed + // ICLASS 0x7 - S2-type A-type + PO_IMMEXT = 0x00000000, // constant extender +}; + +enum PacketWordParseBits : uint32_t { + PP_DUPLEX = 0x00 << 14, + PP_NOT_END = 0x01 << 14, + PP_PACKET_END = 0x03 << 14, +}; + +enum RegNum : uint32_t { + RN_R6 = 0x6, + RN_R7 = 0x7, +}; + +inline static uint32_t +encodeExtendedTransferImmediate(uint32_t Imm, RegNum DestReg, + bool PacketEnd = false) XRAY_NEVER_INSTRUMENT { + static const uint32_t REG_MASK = 0x1f; + assert((DestReg & (~REG_MASK)) == 0); + // The constant-extended register transfer encodes the 6 least + // significant bits of the effective constant: + Imm = Imm & 0x03f; + const PacketWordParseBits ParseBits = PacketEnd ? PP_PACKET_END : PP_NOT_END; + + return PO_TFR_IMM | ParseBits | (Imm << 5) | (DestReg & REG_MASK); +} + +inline static uint32_t +encodeConstantExtender(uint32_t Imm) XRAY_NEVER_INSTRUMENT { + // Bits Name Description + // ----- ------- ------------------------------------------ + // 31:28 ICLASS Instruction class = 0000 + // 27:16 high High 12 bits of 26-bit constant extension + // 15:14 Parse Parse bits + // 13:0 low Low 14 bits of 26-bit constant extension + static const uint32_t IMM_MASK_LOW = 0x03fff; + static const uint32_t IMM_MASK_HIGH = 0x00fff << 14; + + // The extender encodes the 26 most significant bits of the effective + // constant: + Imm = Imm >> 6; + + const uint32_t high = (Imm & IMM_MASK_HIGH) << 16; + const uint32_t low = Imm & IMM_MASK_LOW; + + return PO_IMMEXT | high | PP_NOT_END | low; +} + +static void WriteInstFlushCache(void *Addr, uint32_t NewInstruction) { + asm volatile("icinva(%[inst_addr])\n\t" + "isync\n\t" + "memw(%[inst_addr]) = %[new_inst]\n\t" + "dccleaninva(%[inst_addr])\n\t" + "syncht\n\t" + : + : [ inst_addr ] "r"(Addr), [ new_inst ] "r"(NewInstruction) + : "memory"); +} + +inline static bool patchSled(const bool Enable, const uint32_t FuncId, + const XRaySledEntry &Sled, + void (*TracingHook)()) XRAY_NEVER_INSTRUMENT { + // When |Enable| == true, + // We replace the following compile-time stub (sled): + // + // .L_xray_sled_N: + // : + // { jump .Ltmp0 } + // { nop + // nop + // nop + // nop } + // .Ltmp0: + + // With the following runtime patch: + // + // xray_sled_n (32-bit): + // + // : + // { immext(#...) // upper 26-bits of func id + // r7 = ##... // lower 6-bits of func id + // immext(#...) // upper 26-bits of trampoline + // r6 = ##... } // lower 6 bits of trampoline + // { callr r6 } + // + // When |Enable|==false, we set back the first instruction in the sled to be + // { jump .Ltmp0 } + + uint32_t *FirstAddress = reinterpret_cast(Sled.address()); + if (Enable) { + uint32_t *CurAddress = FirstAddress + 1; + *CurAddress = encodeExtendedTransferImmediate(FuncId, RN_R7); + CurAddress++; + *CurAddress = encodeConstantExtender(reinterpret_cast(TracingHook)); + CurAddress++; + *CurAddress = + encodeExtendedTransferImmediate(reinterpret_cast(TracingHook), RN_R6, true); + CurAddress++; + + *CurAddress = uint32_t(PO_CALLR_R6); + + WriteInstFlushCache(FirstAddress, uint32_t(encodeConstantExtender(FuncId))); + } else { + WriteInstFlushCache(FirstAddress, uint32_t(PatchOpcodes::PO_JUMPI_14)); + } + return true; +} + +bool patchFunctionEntry(const bool Enable, const uint32_t FuncId, + const XRaySledEntry &Sled, + void (*Trampoline)()) XRAY_NEVER_INSTRUMENT { + return patchSled(Enable, FuncId, Sled, Trampoline); +} + +bool patchFunctionExit(const bool Enable, const uint32_t FuncId, + const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { + return patchSled(Enable, FuncId, Sled, __xray_FunctionExit); +} + +bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId, + const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { + return patchSled(Enable, FuncId, Sled, __xray_FunctionExit); +} + +bool patchCustomEvent(const bool Enable, const uint32_t FuncId, + const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { + // FIXME: Implement in hexagon? + return false; +} + +bool patchTypedEvent(const bool Enable, const uint32_t FuncId, + const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { + // FIXME: Implement in hexagon? + return false; +} + +} // namespace __xray + +extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT { + // FIXME: this will have to be implemented in the trampoline assembly file +} diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp index ddf184c9b857..73e67618c9d5 100644 --- a/compiler-rt/lib/xray/xray_interface.cpp +++ b/compiler-rt/lib/xray/xray_interface.cpp @@ -14,7 +14,7 @@ #include "xray_interface_internal.h" -#include +#include #include #include #include @@ -52,6 +52,8 @@ static const int16_t cSledLength = 48; static const int16_t cSledLength = 64; #elif defined(__powerpc64__) static const int16_t cSledLength = 8; +#elif defined(__hexagon__) +static const int16_t cSledLength = 20; #else #error "Unsupported CPU Architecture" #endif /* CPU architecture */ @@ -169,7 +171,8 @@ bool patchSled(const XRaySledEntry &Sled, bool Enable, Success = patchTypedEvent(Enable, FuncId, Sled); break; default: - Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind)); + Report("Unsupported sled kind '%" PRIu64 "' @%04x\n", Sled.Address, + int(Sled.Kind)); return false; } return Success; @@ -305,7 +308,7 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { ? flags()->xray_page_size_override : GetPageSizeCached(); if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { - Report("System page size is not a power of two: %lld\n", PageSize); + Report("System page size is not a power of two: %zu\n", PageSize); return XRayPatchingStatus::FAILED; } @@ -356,7 +359,7 @@ XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, ? flags()->xray_page_size_override : GetPageSizeCached(); if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { - Report("Provided page size is not a power of two: %lld\n", PageSize); + Report("Provided page size is not a power of two: %zu\n", PageSize); return XRayPatchingStatus::FAILED; } diff --git a/compiler-rt/lib/xray/xray_trampoline_hexagon.S b/compiler-rt/lib/xray/xray_trampoline_hexagon.S new file mode 100644 index 000000000000..c87ec4bed1f9 --- /dev/null +++ b/compiler-rt/lib/xray/xray_trampoline_hexagon.S @@ -0,0 +1,99 @@ +//===-- xray_trampoline_hexagon.s -------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// This implements the hexagon-specific assembler for the trampolines. +// +//===----------------------------------------------------------------------===// + +#include "../builtins/assembly.h" +#include "../sanitizer_common/sanitizer_asm.h" + +.macro SAVE_REGISTERS +memw(sp+#0)=r0 +memw(sp+#4)=r1 +memw(sp+#8)=r2 +memw(sp+#12)=r3 +memw(sp+#16)=r4 +.endm +.macro RESTORE_REGISTERS +r0=memw(sp+#0) +r1=memw(sp+#4) +r2=memw(sp+#8) +r3=memw(sp+#12) +r4=memw(sp+#16) +.endm + +.macro CALL_PATCHED_FUNC entry_type + // if (xray::XRayPatchedFunctionE != NULL) + // xray::XRayPatchedFunctionE(FuncType); + + r8 = #ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE) + + // The patched sled puts the function type + // into r6. Move it into r0 to pass it to + // the patched function. + { r0 = r6 + r1 = \entry_type + p0 = !cmp.eq(r8, #0) + if (p0) callr r8 } +.endm + + .text + .globl ASM_SYMBOL(__xray_FunctionEntry) + ASM_HIDDEN(__xray_FunctionEntry) + ASM_TYPE_FUNCTION(__xray_FunctionEntry) +# LLVM-MCA-BEGIN __xray_FunctionEntry +ASM_SYMBOL(__xray_FunctionEntry): + CFI_STARTPROC + SAVE_REGISTERS + + CALL_PATCHED_FUNC #0 // XRayEntryType::ENTRY +.Ltmp0: + RESTORE_REGISTERS + // return +# LLVM-MCA-END + ASM_SIZE(__xray_FunctionEntry) + CFI_ENDPROC + + + .globl ASM_SYMBOL(__xray_FunctionExit) + ASM_HIDDEN(__xray_FunctionExit) + ASM_TYPE_FUNCTION(__xray_FunctionExit) +# LLVM-MCA-BEGIN __xray_FunctionExit +ASM_SYMBOL(__xray_FunctionExit): + CFI_STARTPROC + SAVE_REGISTERS + + CALL_PATCHED_FUNC #1 // XRayEntryType::EXIT +.Ltmp1: + RESTORE_REGISTERS + // return + jumpr r31 +# LLVM-MCA-END + ASM_SIZE(__xray_FunctionExit) + CFI_ENDPROC + + + .globl ASM_SYMBOL(__xray_FunctionTailExit) + ASM_HIDDEN(__xray_FunctionTailExit) + ASM_TYPE_FUNCTION(__xray_FunctionTailExit) +# LLVM-MCA-BEGIN __xray_FunctionTailExit +ASM_SYMBOL(__xray_FunctionTailExit): + CFI_STARTPROC + SAVE_REGISTERS + + CALL_PATCHED_FUNC #2 // XRayEntryType::TAIL +.Ltmp2: + RESTORE_REGISTERS + // return + jumpr r31 +# LLVM-MCA-END + ASM_SIZE(__xray_FunctionTailExit) + CFI_ENDPROC diff --git a/compiler-rt/lib/xray/xray_tsc.h b/compiler-rt/lib/xray/xray_tsc.h index bd7e1911abb3..58347dca5f7a 100644 --- a/compiler-rt/lib/xray/xray_tsc.h +++ b/compiler-rt/lib/xray/xray_tsc.h @@ -42,7 +42,8 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { #include "xray_x86_64.inc" #elif defined(__powerpc64__) #include "xray_powerpc64.inc" -#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) +#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__hexagon__) // Emulated TSC. // There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does // not have a constant frequency like TSC on x86(_64), it may go faster diff --git a/libcxx/CREDITS.TXT b/libcxx/CREDITS.TXT index fc442f4db1a1..cd5bc08a60fc 100644 --- a/libcxx/CREDITS.TXT +++ b/libcxx/CREDITS.TXT @@ -12,6 +12,9 @@ N: Saleem Abdulrasool E: compnerd@compnerd.org D: Minor patches and Linux fixes. +N: Ulf Adams +D: Invented the Ryu and Ryu Printf algorithms used in floating-point to_chars, and wrote the initial code. + N: Muiez Ahmed E: muiez@ibm.com D: z/OS port. @@ -28,6 +31,9 @@ N: Holger Arnold E: holgerar@gmail.com D: Minor fix. +N: Jorg Brown +D: Ported floating-point to_chars from MSVC to libc++. + N: David Chisnall E: theraven at theravensnest dot org D: FreeBSD and Solaris ports, libcxxrt support, some atomics work. @@ -81,6 +87,14 @@ N: Argyrios Kyrtzidis E: kyrtzidis@apple.com D: Bug fixes. +N: Stephan T. Lavavej +E: stl@microsoft.com +E: stl@nuwen.net +D: Implemented floating-point to_chars. + +N: Microsoft Corporation +D: Contributed floating-point to_chars. + N: Bruce Mitchener, Jr. E: bruce.mitchener@gmail.com D: Emscripten-related changes. @@ -152,6 +166,7 @@ D: Minor bug fix. N: Mark de Wever E: koraq at xs4all dot nl D: Format library support. +D: Finalized the porting of MSVC's to_chars to libc++. N: Zhang Xiongpang E: zhangxiongpang@gmail.com diff --git a/libcxx/include/__availability b/libcxx/include/__availability index 87d43ed414bf..4652a6fd91b4 100644 --- a/libcxx/include/__availability +++ b/libcxx/include/__availability @@ -129,6 +129,10 @@ // This controls the availability of std::to_chars. # define _LIBCPP_AVAILABILITY_TO_CHARS + // This controls the availability of floating-point std::to_chars functions. + // These overloads were added later than the integer overloads. +# define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT + // This controls the availability of the C++20 synchronization library, // which requires shared library support for various operations // (see libcxx/src/atomic.cpp). @@ -222,6 +226,9 @@ # define _LIBCPP_AVAILABILITY_TO_CHARS \ _LIBCPP_AVAILABILITY_FILESYSTEM +# define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT \ + __attribute__((unavailable)) + # define _LIBCPP_AVAILABILITY_SYNC \ __attribute__((availability(macosx,strict,introduced=11.0))) \ __attribute__((availability(ios,strict,introduced=14.0))) \ diff --git a/libcxx/include/__compare/strong_order.h b/libcxx/include/__compare/strong_order.h index e49b2d45de45..42f060387d59 100644 --- a/libcxx/include/__compare/strong_order.h +++ b/libcxx/include/__compare/strong_order.h @@ -86,11 +86,11 @@ namespace __strong_order { bool __u_is_nan = _VSTD::isnan(__u); bool __t_is_negative = _VSTD::signbit(__t); bool __u_is_negative = _VSTD::signbit(__u); - using _IntType = std::conditional_t< - sizeof(__t) == sizeof(int32_t), int32_t, std::conditional_t< + using _IntType = conditional_t< + sizeof(__t) == sizeof(int32_t), int32_t, conditional_t< sizeof(__t) == sizeof(int64_t), int64_t, void> >; - if constexpr (std::is_same_v<_IntType, void>) { + if constexpr (is_same_v<_IntType, void>) { static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type"); } else if (__t_is_nan && __u_is_nan) { // Order by sign bit, then by "payload bits" (we'll just use bit_cast). diff --git a/libcxx/include/__compare/weak_order.h b/libcxx/include/__compare/weak_order.h index f67416ed3ebe..ce914b232108 100644 --- a/libcxx/include/__compare/weak_order.h +++ b/libcxx/include/__compare/weak_order.h @@ -42,13 +42,13 @@ namespace __weak_order { _LIBCPP_HIDE_FROM_ABI static constexpr weak_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept { - std::partial_ordering __po = (__t <=> __u); - if (__po == std::partial_ordering::less) { - return std::weak_ordering::less; - } else if (__po == std::partial_ordering::equivalent) { - return std::weak_ordering::equivalent; - } else if (__po == std::partial_ordering::greater) { - return std::weak_ordering::greater; + partial_ordering __po = (__t <=> __u); + if (__po == partial_ordering::less) { + return weak_ordering::less; + } else if (__po == partial_ordering::equivalent) { + return weak_ordering::equivalent; + } else if (__po == partial_ordering::greater) { + return weak_ordering::greater; } else { // Otherwise, at least one of them is a NaN. bool __t_is_nan = _VSTD::isnan(__t); diff --git a/libcxx/include/__config b/libcxx/include/__config index da03e877f753..720e12eac0dd 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -333,18 +333,43 @@ # define _LIBCPP_SHORT_WCHAR 1 #endif +// Libc++ supports various implementations of std::random_device. +// +// _LIBCPP_USING_DEV_RANDOM +// Read entropy from the given file, by default `/dev/urandom`. +// If a token is provided, it is assumed to be the path to a file +// to read entropy from. This is the default behavior if nothing +// else is specified. This implementation requires storing state +// inside `std::random_device`. +// +// _LIBCPP_USING_ARC4_RANDOM +// Use arc4random(). This allows obtaining random data even when +// using sandboxing mechanisms. On some platforms like Apple, this +// is the recommended source of entropy for user-space programs. +// When this option is used, the token passed to `std::random_device`'s +// constructor *must* be "/dev/urandom" -- anything else is an error. +// +// _LIBCPP_USING_GETENTROPY +// Use getentropy(). +// When this option is used, the token passed to `std::random_device`'s +// constructor *must* be "/dev/urandom" -- anything else is an error. +// +// _LIBCPP_USING_NACL_RANDOM +// NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access, +// including accesses to the special files under `/dev`. This implementation +// uses the NaCL syscall `nacl_secure_random_init()` to get entropy. +// When this option is used, the token passed to `std::random_device`'s +// constructor *must* be "/dev/urandom" -- anything else is an error. +// +// _LIBCPP_USING_WIN32_RANDOM +// Use rand_s(), for use on Windows. +// When this option is used, the token passed to `std::random_device`'s +// constructor *must* be "/dev/urandom" -- anything else is an error. #if defined(__OpenBSD__) - // Certain architectures provide arc4random(). Prefer using - // arc4random() over /dev/{u,}random to make it possible to obtain - // random data even when using sandboxing mechanisms such as chroots, - // Capsicum, etc. # define _LIBCPP_USING_ARC4_RANDOM #elif defined(__Fuchsia__) || defined(__wasi__) # define _LIBCPP_USING_GETENTROPY #elif defined(__native_client__) - // NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access, - // including accesses to the special files under /dev. C++11's - // std::random_device is instead exposed through a NaCl syscall. # define _LIBCPP_USING_NACL_RANDOM #elif defined(_LIBCPP_WIN32API) # define _LIBCPP_USING_WIN32_RANDOM @@ -837,12 +862,6 @@ typedef unsigned int char32_t; #define _LIBCPP_HAS_NO_RANGES #endif -#ifdef _LIBCPP_CXX03_LANG -# define _LIBCPP_DEFAULT {} -#else -# define _LIBCPP_DEFAULT = default; -#endif - #ifdef __GNUC__ # define _LIBCPP_NOALIAS __attribute__((__malloc__)) #else diff --git a/libcxx/include/__debug b/libcxx/include/__debug index e25039c088c6..42f6cef4c07f 100644 --- a/libcxx/include/__debug +++ b/libcxx/include/__debug @@ -34,7 +34,7 @@ # define _LIBCPP_DEBUG_ASSERT(x, m) ((void)0) # define _LIBCPP_ASSERT_IMPL(x, m) ((x) ? (void)0 : _VSTD::__libcpp_debug_function(_VSTD::__libcpp_debug_info(__FILE__, __LINE__, #x, m))) #elif _LIBCPP_DEBUG_LEVEL == 2 -# define _LIBCPP_DEBUG_ASSERT(x, m) _LIBCPP_ASSERT(x, m) +# define _LIBCPP_DEBUG_ASSERT(x, m) _LIBCPP_ASSERT(__libcpp_is_constant_evaluated() || (x), m) # define _LIBCPP_ASSERT_IMPL(x, m) ((x) ? (void)0 : _VSTD::__libcpp_debug_function(_VSTD::__libcpp_debug_info(__FILE__, __LINE__, #x, m))) #else # error _LIBCPP_DEBUG_LEVEL must be one of 0, 1, 2 diff --git a/libcxx/include/__filesystem/copy_options.h b/libcxx/include/__filesystem/copy_options.h new file mode 100644 index 000000000000..c0140d45717b --- /dev/null +++ b/libcxx/include/__filesystem/copy_options.h @@ -0,0 +1,80 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_COPY_OPTIONS_H +#define _LIBCPP___FILESYSTEM_COPY_OPTIONS_H + +#include <__availability> +#include <__config> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +enum class _LIBCPP_ENUM_VIS copy_options : unsigned short { + none = 0, + skip_existing = 1, + overwrite_existing = 2, + update_existing = 4, + recursive = 8, + copy_symlinks = 16, + skip_symlinks = 32, + directories_only = 64, + create_symlinks = 128, + create_hard_links = 256, + __in_recursive_copy = 512, +}; + +_LIBCPP_INLINE_VISIBILITY +inline constexpr copy_options operator&(copy_options _LHS, copy_options _RHS) { + return static_cast(static_cast(_LHS) & + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr copy_options operator|(copy_options _LHS, copy_options _RHS) { + return static_cast(static_cast(_LHS) | + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr copy_options operator^(copy_options _LHS, copy_options _RHS) { + return static_cast(static_cast(_LHS) ^ + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr copy_options operator~(copy_options _LHS) { + return static_cast(~static_cast(_LHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline copy_options& operator&=(copy_options& _LHS, copy_options _RHS) { + return _LHS = _LHS & _RHS; +} + +_LIBCPP_INLINE_VISIBILITY +inline copy_options& operator|=(copy_options& _LHS, copy_options _RHS) { + return _LHS = _LHS | _RHS; +} + +_LIBCPP_INLINE_VISIBILITY +inline copy_options& operator^=(copy_options& _LHS, copy_options _RHS) { + return _LHS = _LHS ^ _RHS; +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_COPY_OPTIONS_H diff --git a/libcxx/include/__filesystem/directory_entry.h b/libcxx/include/__filesystem/directory_entry.h new file mode 100644 index 000000000000..9efe19465428 --- /dev/null +++ b/libcxx/include/__filesystem/directory_entry.h @@ -0,0 +1,504 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H +#define _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H + +#include <__availability> +#include <__config> +#include <__filesystem/path.h> +#include <__filesystem/file_time_type.h> +#include <__filesystem/filesystem_error.h> +#include <__filesystem/file_status.h> +#include <__filesystem/file_type.h> +#include <__filesystem/operations.h> +#include <__filesystem/perms.h> +#include <__errc> +#include +#include +#include +#include + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + + +class directory_entry { + typedef _VSTD_FS::path _Path; + +public: + // constructors and destructors + directory_entry() noexcept = default; + directory_entry(directory_entry const&) = default; + directory_entry(directory_entry&&) noexcept = default; + + _LIBCPP_INLINE_VISIBILITY + explicit directory_entry(_Path const& __p) : __p_(__p) { + error_code __ec; + __refresh(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + directory_entry(_Path const& __p, error_code& __ec) : __p_(__p) { + __refresh(&__ec); + } + + ~directory_entry() {} + + directory_entry& operator=(directory_entry const&) = default; + directory_entry& operator=(directory_entry&&) noexcept = default; + + _LIBCPP_INLINE_VISIBILITY + void assign(_Path const& __p) { + __p_ = __p; + error_code __ec; + __refresh(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + void assign(_Path const& __p, error_code& __ec) { + __p_ = __p; + __refresh(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + void replace_filename(_Path const& __p) { + __p_.replace_filename(__p); + error_code __ec; + __refresh(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + void replace_filename(_Path const& __p, error_code& __ec) { + __p_ = __p_.parent_path() / __p; + __refresh(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + void refresh() { __refresh(); } + + _LIBCPP_INLINE_VISIBILITY + void refresh(error_code& __ec) noexcept { __refresh(&__ec); } + + _LIBCPP_INLINE_VISIBILITY + _Path const& path() const noexcept { return __p_; } + + _LIBCPP_INLINE_VISIBILITY + operator const _Path&() const noexcept { return __p_; } + + _LIBCPP_INLINE_VISIBILITY + bool exists() const { return _VSTD_FS::exists(file_status{__get_ft()}); } + + _LIBCPP_INLINE_VISIBILITY + bool exists(error_code& __ec) const noexcept { + return _VSTD_FS::exists(file_status{__get_ft(&__ec)}); + } + + _LIBCPP_INLINE_VISIBILITY + bool is_block_file() const { return __get_ft() == file_type::block; } + + _LIBCPP_INLINE_VISIBILITY + bool is_block_file(error_code& __ec) const noexcept { + return __get_ft(&__ec) == file_type::block; + } + + _LIBCPP_INLINE_VISIBILITY + bool is_character_file() const { return __get_ft() == file_type::character; } + + _LIBCPP_INLINE_VISIBILITY + bool is_character_file(error_code& __ec) const noexcept { + return __get_ft(&__ec) == file_type::character; + } + + _LIBCPP_INLINE_VISIBILITY + bool is_directory() const { return __get_ft() == file_type::directory; } + + _LIBCPP_INLINE_VISIBILITY + bool is_directory(error_code& __ec) const noexcept { + return __get_ft(&__ec) == file_type::directory; + } + + _LIBCPP_INLINE_VISIBILITY + bool is_fifo() const { return __get_ft() == file_type::fifo; } + + _LIBCPP_INLINE_VISIBILITY + bool is_fifo(error_code& __ec) const noexcept { + return __get_ft(&__ec) == file_type::fifo; + } + + _LIBCPP_INLINE_VISIBILITY + bool is_other() const { return _VSTD_FS::is_other(file_status{__get_ft()}); } + + _LIBCPP_INLINE_VISIBILITY + bool is_other(error_code& __ec) const noexcept { + return _VSTD_FS::is_other(file_status{__get_ft(&__ec)}); + } + + _LIBCPP_INLINE_VISIBILITY + bool is_regular_file() const { return __get_ft() == file_type::regular; } + + _LIBCPP_INLINE_VISIBILITY + bool is_regular_file(error_code& __ec) const noexcept { + return __get_ft(&__ec) == file_type::regular; + } + + _LIBCPP_INLINE_VISIBILITY + bool is_socket() const { return __get_ft() == file_type::socket; } + + _LIBCPP_INLINE_VISIBILITY + bool is_socket(error_code& __ec) const noexcept { + return __get_ft(&__ec) == file_type::socket; + } + + _LIBCPP_INLINE_VISIBILITY + bool is_symlink() const { return __get_sym_ft() == file_type::symlink; } + + _LIBCPP_INLINE_VISIBILITY + bool is_symlink(error_code& __ec) const noexcept { + return __get_sym_ft(&__ec) == file_type::symlink; + } + _LIBCPP_INLINE_VISIBILITY + uintmax_t file_size() const { return __get_size(); } + + _LIBCPP_INLINE_VISIBILITY + uintmax_t file_size(error_code& __ec) const noexcept { + return __get_size(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + uintmax_t hard_link_count() const { return __get_nlink(); } + + _LIBCPP_INLINE_VISIBILITY + uintmax_t hard_link_count(error_code& __ec) const noexcept { + return __get_nlink(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + file_time_type last_write_time() const { return __get_write_time(); } + + _LIBCPP_INLINE_VISIBILITY + file_time_type last_write_time(error_code& __ec) const noexcept { + return __get_write_time(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + file_status status() const { return __get_status(); } + + _LIBCPP_INLINE_VISIBILITY + file_status status(error_code& __ec) const noexcept { + return __get_status(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + file_status symlink_status() const { return __get_symlink_status(); } + + _LIBCPP_INLINE_VISIBILITY + file_status symlink_status(error_code& __ec) const noexcept { + return __get_symlink_status(&__ec); + } + + _LIBCPP_INLINE_VISIBILITY + bool operator<(directory_entry const& __rhs) const noexcept { + return __p_ < __rhs.__p_; + } + + _LIBCPP_INLINE_VISIBILITY + bool operator==(directory_entry const& __rhs) const noexcept { + return __p_ == __rhs.__p_; + } + + _LIBCPP_INLINE_VISIBILITY + bool operator!=(directory_entry const& __rhs) const noexcept { + return __p_ != __rhs.__p_; + } + + _LIBCPP_INLINE_VISIBILITY + bool operator<=(directory_entry const& __rhs) const noexcept { + return __p_ <= __rhs.__p_; + } + + _LIBCPP_INLINE_VISIBILITY + bool operator>(directory_entry const& __rhs) const noexcept { + return __p_ > __rhs.__p_; + } + + _LIBCPP_INLINE_VISIBILITY + bool operator>=(directory_entry const& __rhs) const noexcept { + return __p_ >= __rhs.__p_; + } + +private: + friend class directory_iterator; + friend class recursive_directory_iterator; + friend class __dir_stream; + + enum _CacheType : unsigned char { + _Empty, + _IterSymlink, + _IterNonSymlink, + _RefreshSymlink, + _RefreshSymlinkUnresolved, + _RefreshNonSymlink + }; + + struct __cached_data { + uintmax_t __size_; + uintmax_t __nlink_; + file_time_type __write_time_; + perms __sym_perms_; + perms __non_sym_perms_; + file_type __type_; + _CacheType __cache_type_; + + _LIBCPP_INLINE_VISIBILITY + __cached_data() noexcept { __reset(); } + + _LIBCPP_INLINE_VISIBILITY + void __reset() { + __cache_type_ = _Empty; + __type_ = file_type::none; + __sym_perms_ = __non_sym_perms_ = perms::unknown; + __size_ = __nlink_ = uintmax_t(-1); + __write_time_ = file_time_type::min(); + } + }; + + _LIBCPP_INLINE_VISIBILITY + static __cached_data __create_iter_result(file_type __ft) { + __cached_data __data; + __data.__type_ = __ft; + __data.__cache_type_ = [&]() { + switch (__ft) { + case file_type::none: + return _Empty; + case file_type::symlink: + return _IterSymlink; + default: + return _IterNonSymlink; + } + }(); + return __data; + } + + _LIBCPP_INLINE_VISIBILITY + void __assign_iter_entry(_Path&& __p, __cached_data __dt) { + __p_ = _VSTD::move(__p); + __data_ = __dt; + } + + _LIBCPP_FUNC_VIS + error_code __do_refresh() noexcept; + + _LIBCPP_INLINE_VISIBILITY + static bool __is_dne_error(error_code const& __ec) { + if (!__ec) + return true; + switch (static_cast(__ec.value())) { + case errc::no_such_file_or_directory: + case errc::not_a_directory: + return true; + default: + return false; + } + } + + _LIBCPP_INLINE_VISIBILITY + void __handle_error(const char* __msg, error_code* __dest_ec, + error_code const& __ec, bool __allow_dne = false) const { + if (__dest_ec) { + *__dest_ec = __ec; + return; + } + if (__ec && (!__allow_dne || !__is_dne_error(__ec))) + __throw_filesystem_error(__msg, __p_, __ec); + } + + _LIBCPP_INLINE_VISIBILITY + void __refresh(error_code* __ec = nullptr) { + __handle_error("in directory_entry::refresh", __ec, __do_refresh(), + /*allow_dne*/ true); + } + + _LIBCPP_INLINE_VISIBILITY + file_type __get_sym_ft(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + return __symlink_status(__p_, __ec).type(); + case _IterSymlink: + case _RefreshSymlink: + case _RefreshSymlinkUnresolved: + if (__ec) + __ec->clear(); + return file_type::symlink; + case _IterNonSymlink: + case _RefreshNonSymlink: + file_status __st(__data_.__type_); + if (__ec && !_VSTD_FS::exists(__st)) + *__ec = make_error_code(errc::no_such_file_or_directory); + else if (__ec) + __ec->clear(); + return __data_.__type_; + } + _LIBCPP_UNREACHABLE(); + } + + _LIBCPP_INLINE_VISIBILITY + file_type __get_ft(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + case _IterSymlink: + case _RefreshSymlinkUnresolved: + return __status(__p_, __ec).type(); + case _IterNonSymlink: + case _RefreshNonSymlink: + case _RefreshSymlink: { + file_status __st(__data_.__type_); + if (__ec && !_VSTD_FS::exists(__st)) + *__ec = make_error_code(errc::no_such_file_or_directory); + else if (__ec) + __ec->clear(); + return __data_.__type_; + } + } + _LIBCPP_UNREACHABLE(); + } + + _LIBCPP_INLINE_VISIBILITY + file_status __get_status(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + case _IterNonSymlink: + case _IterSymlink: + case _RefreshSymlinkUnresolved: + return __status(__p_, __ec); + case _RefreshNonSymlink: + case _RefreshSymlink: + return file_status(__get_ft(__ec), __data_.__non_sym_perms_); + } + _LIBCPP_UNREACHABLE(); + } + + _LIBCPP_INLINE_VISIBILITY + file_status __get_symlink_status(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + case _IterNonSymlink: + case _IterSymlink: + return __symlink_status(__p_, __ec); + case _RefreshNonSymlink: + return file_status(__get_sym_ft(__ec), __data_.__non_sym_perms_); + case _RefreshSymlink: + case _RefreshSymlinkUnresolved: + return file_status(__get_sym_ft(__ec), __data_.__sym_perms_); + } + _LIBCPP_UNREACHABLE(); + } + + _LIBCPP_INLINE_VISIBILITY + uintmax_t __get_size(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + case _IterNonSymlink: + case _IterSymlink: + case _RefreshSymlinkUnresolved: + return _VSTD_FS::__file_size(__p_, __ec); + case _RefreshSymlink: + case _RefreshNonSymlink: { + error_code __m_ec; + file_status __st(__get_ft(&__m_ec)); + __handle_error("in directory_entry::file_size", __ec, __m_ec); + if (_VSTD_FS::exists(__st) && !_VSTD_FS::is_regular_file(__st)) { + errc __err_kind = _VSTD_FS::is_directory(__st) ? errc::is_a_directory + : errc::not_supported; + __handle_error("in directory_entry::file_size", __ec, + make_error_code(__err_kind)); + } + return __data_.__size_; + } + } + _LIBCPP_UNREACHABLE(); + } + + _LIBCPP_INLINE_VISIBILITY + uintmax_t __get_nlink(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + case _IterNonSymlink: + case _IterSymlink: + case _RefreshSymlinkUnresolved: + return _VSTD_FS::__hard_link_count(__p_, __ec); + case _RefreshSymlink: + case _RefreshNonSymlink: { + error_code __m_ec; + (void)__get_ft(&__m_ec); + __handle_error("in directory_entry::hard_link_count", __ec, __m_ec); + return __data_.__nlink_; + } + } + _LIBCPP_UNREACHABLE(); + } + + _LIBCPP_INLINE_VISIBILITY + file_time_type __get_write_time(error_code* __ec = nullptr) const { + switch (__data_.__cache_type_) { + case _Empty: + case _IterNonSymlink: + case _IterSymlink: + case _RefreshSymlinkUnresolved: + return _VSTD_FS::__last_write_time(__p_, __ec); + case _RefreshSymlink: + case _RefreshNonSymlink: { + error_code __m_ec; + file_status __st(__get_ft(&__m_ec)); + __handle_error("in directory_entry::last_write_time", __ec, __m_ec); + if (_VSTD_FS::exists(__st) && + __data_.__write_time_ == file_time_type::min()) + __handle_error("in directory_entry::last_write_time", __ec, + make_error_code(errc::value_too_large)); + return __data_.__write_time_; + } + } + _LIBCPP_UNREACHABLE(); + } + +private: + _Path __p_; + __cached_data __data_; +}; + +class __dir_element_proxy { +public: + inline _LIBCPP_INLINE_VISIBILITY directory_entry operator*() { + return _VSTD::move(__elem_); + } + +private: + friend class directory_iterator; + friend class recursive_directory_iterator; + explicit __dir_element_proxy(directory_entry const& __e) : __elem_(__e) {} + __dir_element_proxy(__dir_element_proxy&& __o) + : __elem_(_VSTD::move(__o.__elem_)) {} + directory_entry __elem_; +}; + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H diff --git a/libcxx/include/__filesystem/directory_iterator.h b/libcxx/include/__filesystem/directory_iterator.h new file mode 100644 index 000000000000..be958e0eb8de --- /dev/null +++ b/libcxx/include/__filesystem/directory_iterator.h @@ -0,0 +1,150 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H +#define _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H + +#include <__availability> +#include <__config> +#include <__filesystem/directory_entry.h> +#include <__filesystem/directory_options.h> +#include <__filesystem/path.h> +#include <__iterator/iterator_traits.h> +#include <__memory/shared_ptr.h> +#include <__debug> +#include <__ranges/enable_borrowed_range.h> +#include <__ranges/enable_view.h> +#include +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +class _LIBCPP_HIDDEN __dir_stream; +class directory_iterator { +public: + typedef directory_entry value_type; + typedef ptrdiff_t difference_type; + typedef value_type const* pointer; + typedef value_type const& reference; + typedef input_iterator_tag iterator_category; + +public: + //ctor & dtor + directory_iterator() noexcept {} + + explicit directory_iterator(const path& __p) + : directory_iterator(__p, nullptr) {} + + directory_iterator(const path& __p, directory_options __opts) + : directory_iterator(__p, nullptr, __opts) {} + + directory_iterator(const path& __p, error_code& __ec) + : directory_iterator(__p, &__ec) {} + + directory_iterator(const path& __p, directory_options __opts, + error_code& __ec) + : directory_iterator(__p, &__ec, __opts) {} + + directory_iterator(const directory_iterator&) = default; + directory_iterator(directory_iterator&&) = default; + directory_iterator& operator=(const directory_iterator&) = default; + + directory_iterator& operator=(directory_iterator&& __o) noexcept { + // non-default implementation provided to support self-move assign. + if (this != &__o) { + __imp_ = _VSTD::move(__o.__imp_); + } + return *this; + } + + ~directory_iterator() = default; + + const directory_entry& operator*() const { + _LIBCPP_ASSERT(__imp_, "The end iterator cannot be dereferenced"); + return __dereference(); + } + + const directory_entry* operator->() const { return &**this; } + + directory_iterator& operator++() { return __increment(); } + + __dir_element_proxy operator++(int) { + __dir_element_proxy __p(**this); + __increment(); + return __p; + } + + directory_iterator& increment(error_code& __ec) { return __increment(&__ec); } + +private: + inline _LIBCPP_INLINE_VISIBILITY friend bool + operator==(const directory_iterator& __lhs, + const directory_iterator& __rhs) noexcept; + + // construct the dir_stream + _LIBCPP_FUNC_VIS + directory_iterator(const path&, error_code*, + directory_options = directory_options::none); + + _LIBCPP_FUNC_VIS + directory_iterator& __increment(error_code* __ec = nullptr); + + _LIBCPP_FUNC_VIS + const directory_entry& __dereference() const; + +private: + shared_ptr<__dir_stream> __imp_; +}; + +inline _LIBCPP_INLINE_VISIBILITY bool +operator==(const directory_iterator& __lhs, + const directory_iterator& __rhs) noexcept { + return __lhs.__imp_ == __rhs.__imp_; +} + +inline _LIBCPP_INLINE_VISIBILITY bool +operator!=(const directory_iterator& __lhs, + const directory_iterator& __rhs) noexcept { + return !(__lhs == __rhs); +} + +// enable directory_iterator range-based for statements +inline _LIBCPP_INLINE_VISIBILITY directory_iterator +begin(directory_iterator __iter) noexcept { + return __iter; +} + +inline _LIBCPP_INLINE_VISIBILITY directory_iterator +end(directory_iterator) noexcept { + return directory_iterator(); +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#if !defined(_LIBCPP_HAS_NO_RANGES) + +template <> +_LIBCPP_AVAILABILITY_FILESYSTEM +inline constexpr bool _VSTD::ranges::enable_borrowed_range<_VSTD_FS::directory_iterator> = true; + +template <> +_LIBCPP_AVAILABILITY_FILESYSTEM +inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::directory_iterator> = true; + +#endif + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H diff --git a/libcxx/include/__filesystem/directory_options.h b/libcxx/include/__filesystem/directory_options.h new file mode 100644 index 000000000000..79c0c2cbaa55 --- /dev/null +++ b/libcxx/include/__filesystem/directory_options.h @@ -0,0 +1,78 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H +#define _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H + +#include <__availability> +#include <__config> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +enum class _LIBCPP_ENUM_VIS directory_options : unsigned char { + none = 0, + follow_directory_symlink = 1, + skip_permission_denied = 2 +}; + +_LIBCPP_INLINE_VISIBILITY +inline constexpr directory_options operator&(directory_options _LHS, + directory_options _RHS) { + return static_cast(static_cast(_LHS) & + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr directory_options operator|(directory_options _LHS, + directory_options _RHS) { + return static_cast(static_cast(_LHS) | + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr directory_options operator^(directory_options _LHS, + directory_options _RHS) { + return static_cast(static_cast(_LHS) ^ + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr directory_options operator~(directory_options _LHS) { + return static_cast(~static_cast(_LHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline directory_options& operator&=(directory_options& _LHS, + directory_options _RHS) { + return _LHS = _LHS & _RHS; +} + +_LIBCPP_INLINE_VISIBILITY +inline directory_options& operator|=(directory_options& _LHS, + directory_options _RHS) { + return _LHS = _LHS | _RHS; +} + +_LIBCPP_INLINE_VISIBILITY +inline directory_options& operator^=(directory_options& _LHS, + directory_options _RHS) { + return _LHS = _LHS ^ _RHS; +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H diff --git a/libcxx/include/__filesystem/file_status.h b/libcxx/include/__filesystem/file_status.h new file mode 100644 index 000000000000..a8f653ab44fc --- /dev/null +++ b/libcxx/include/__filesystem/file_status.h @@ -0,0 +1,68 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_FILE_STATUS_H +#define _LIBCPP___FILESYSTEM_FILE_STATUS_H + +#include <__availability> +#include <__config> +#include <__filesystem/file_type.h> +#include <__filesystem/perms.h> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +class _LIBCPP_TYPE_VIS file_status { +public: + // constructors + _LIBCPP_INLINE_VISIBILITY + file_status() noexcept : file_status(file_type::none) {} + _LIBCPP_INLINE_VISIBILITY + explicit file_status(file_type __ft, perms __prms = perms::unknown) noexcept + : __ft_(__ft), + __prms_(__prms) {} + + file_status(const file_status&) noexcept = default; + file_status(file_status&&) noexcept = default; + + _LIBCPP_INLINE_VISIBILITY + ~file_status() {} + + file_status& operator=(const file_status&) noexcept = default; + file_status& operator=(file_status&&) noexcept = default; + + // observers + _LIBCPP_INLINE_VISIBILITY + file_type type() const noexcept { return __ft_; } + + _LIBCPP_INLINE_VISIBILITY + perms permissions() const noexcept { return __prms_; } + + // modifiers + _LIBCPP_INLINE_VISIBILITY + void type(file_type __ft) noexcept { __ft_ = __ft; } + + _LIBCPP_INLINE_VISIBILITY + void permissions(perms __p) noexcept { __prms_ = __p; } + +private: + file_type __ft_; + perms __prms_; +}; + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_FILE_STATUS_H diff --git a/libcxx/include/__filesystem/file_time_type.h b/libcxx/include/__filesystem/file_time_type.h new file mode 100644 index 000000000000..590146a06600 --- /dev/null +++ b/libcxx/include/__filesystem/file_time_type.h @@ -0,0 +1,27 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H +#define _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H + +#include <__availability> +#include <__config> +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +typedef chrono::time_point<_FilesystemClock> file_time_type; + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H diff --git a/libcxx/include/__filesystem/file_type.h b/libcxx/include/__filesystem/file_type.h new file mode 100644 index 000000000000..93bee86ad635 --- /dev/null +++ b/libcxx/include/__filesystem/file_type.h @@ -0,0 +1,39 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_FILE_TYPE_H +#define _LIBCPP___FILESYSTEM_FILE_TYPE_H + +#include <__availability> +#include <__config> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +// On Windows, the library never identifies files as block, character, fifo +// or socket. +enum class _LIBCPP_ENUM_VIS file_type : signed char { + none = 0, + not_found = -1, + regular = 1, + directory = 2, + symlink = 3, + block = 4, + character = 5, + fifo = 6, + socket = 7, + unknown = 8 +}; + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_FILE_TYPE_H diff --git a/libcxx/include/__filesystem/filesystem_error.h b/libcxx/include/__filesystem/filesystem_error.h new file mode 100644 index 000000000000..a8c6977b48e9 --- /dev/null +++ b/libcxx/include/__filesystem/filesystem_error.h @@ -0,0 +1,99 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H +#define _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H + +#include <__availability> +#include <__config> +#include <__filesystem/path.h> +#include <__memory/shared_ptr.h> +#include +#include +#include +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +class _LIBCPP_AVAILABILITY_FILESYSTEM _LIBCPP_EXCEPTION_ABI filesystem_error : public system_error { +public: + _LIBCPP_INLINE_VISIBILITY + filesystem_error(const string& __what, error_code __ec) + : system_error(__ec, __what), + __storage_(make_shared<_Storage>(path(), path())) { + __create_what(0); + } + + _LIBCPP_INLINE_VISIBILITY + filesystem_error(const string& __what, const path& __p1, error_code __ec) + : system_error(__ec, __what), + __storage_(make_shared<_Storage>(__p1, path())) { + __create_what(1); + } + + _LIBCPP_INLINE_VISIBILITY + filesystem_error(const string& __what, const path& __p1, const path& __p2, + error_code __ec) + : system_error(__ec, __what), + __storage_(make_shared<_Storage>(__p1, __p2)) { + __create_what(2); + } + + _LIBCPP_INLINE_VISIBILITY + const path& path1() const noexcept { return __storage_->__p1_; } + + _LIBCPP_INLINE_VISIBILITY + const path& path2() const noexcept { return __storage_->__p2_; } + + filesystem_error(const filesystem_error&) = default; + ~filesystem_error() override; // key function + + _LIBCPP_INLINE_VISIBILITY + const char* what() const noexcept override { + return __storage_->__what_.c_str(); + } + + void __create_what(int __num_paths); + +private: + struct _LIBCPP_HIDDEN _Storage { + _LIBCPP_INLINE_VISIBILITY + _Storage(const path& __p1, const path& __p2) : __p1_(__p1), __p2_(__p2) {} + + path __p1_; + path __p2_; + string __what_; + }; + shared_ptr<_Storage> __storage_; +}; + +// TODO(ldionne): We need to pop the pragma and push it again after +// filesystem_error to work around PR41078. +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +template +_LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY +#ifndef _LIBCPP_NO_EXCEPTIONS +void __throw_filesystem_error(_Args&&... __args) { + throw filesystem_error(_VSTD::forward<_Args>(__args)...); +} +#else +void __throw_filesystem_error(_Args&&...) { + _VSTD::abort(); +} +#endif +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H diff --git a/libcxx/include/__filesystem/operations.h b/libcxx/include/__filesystem/operations.h new file mode 100644 index 000000000000..19d6c2d437f9 --- /dev/null +++ b/libcxx/include/__filesystem/operations.h @@ -0,0 +1,599 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_OPERATIONS_H +#define _LIBCPP___FILESYSTEM_OPERATIONS_H + +#include <__availability> +#include <__config> +#include <__filesystem/copy_options.h> +#include <__filesystem/file_status.h> +#include <__filesystem/file_time_type.h> +#include <__filesystem/file_type.h> +#include <__filesystem/path.h> +#include <__filesystem/perm_options.h> +#include <__filesystem/perms.h> +#include <__filesystem/space_info.h> +#include +#include +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +_LIBCPP_FUNC_VIS +path __absolute(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +path __canonical(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +bool __copy_file(const path& __from, const path& __to, copy_options __opt, + error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +void __copy_symlink(const path& __existing_symlink, const path& __new_symlink, + error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +void __copy(const path& __from, const path& __to, copy_options __opt, + error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +bool __create_directories(const path& p, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +void __create_directory_symlink(const path& __to, const path& __new_symlink, + error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +bool __create_directory(const path& p, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +bool __create_directory(const path& p, const path& attributes, + error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +void __create_hard_link(const path& __to, const path& __new_hard_link, + error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +void __create_symlink(const path& __to, const path& __new_symlink, + error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +path __current_path(error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +void __current_path(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +bool __equivalent(const path&, const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +file_status __status(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +uintmax_t __file_size(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +uintmax_t __hard_link_count(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +file_status __symlink_status(const path&, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +file_time_type __last_write_time(const path& p, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +void __last_write_time(const path& p, file_time_type new_time, + error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +path __weakly_canonical(path const& __p, error_code* __ec = nullptr); +_LIBCPP_FUNC_VIS +path __read_symlink(const path& p, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +uintmax_t __remove_all(const path& p, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +bool __remove(const path& p, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +void __rename(const path& from, const path& to, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +void __resize_file(const path& p, uintmax_t size, error_code* ec = nullptr); +_LIBCPP_FUNC_VIS +path __temp_directory_path(error_code* __ec = nullptr); + +inline _LIBCPP_INLINE_VISIBILITY path absolute(const path& __p) { + return __absolute(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY path absolute(const path& __p, + error_code& __ec) { + return __absolute(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path canonical(const path& __p) { + return __canonical(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY path canonical(const path& __p, + error_code& __ec) { + return __canonical(__p, &__ec); +} + + +inline _LIBCPP_INLINE_VISIBILITY bool copy_file(const path& __from, + const path& __to) { + return __copy_file(__from, __to, copy_options::none); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +copy_file(const path& __from, const path& __to, error_code& __ec) { + return __copy_file(__from, __to, copy_options::none, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +copy_file(const path& __from, const path& __to, copy_options __opt) { + return __copy_file(__from, __to, __opt); +} + +inline _LIBCPP_INLINE_VISIBILITY bool copy_file(const path& __from, + const path& __to, + copy_options __opt, + error_code& __ec) { + return __copy_file(__from, __to, __opt, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void copy_symlink(const path& __existing, + const path& __new) { + __copy_symlink(__existing, __new); +} + +inline _LIBCPP_INLINE_VISIBILITY void +copy_symlink(const path& __ext, const path& __new, error_code& __ec) noexcept { + __copy_symlink(__ext, __new, &__ec); +} + + +inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, + const path& __to) { + __copy(__from, __to, copy_options::none); +} + +inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to, + error_code& __ec) { + __copy(__from, __to, copy_options::none, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to, + copy_options __opt) { + __copy(__from, __to, __opt); +} + +inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to, + copy_options __opt, + error_code& __ec) { + __copy(__from, __to, __opt, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool create_directories(const path& __p) { + return __create_directories(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY bool create_directories(const path& __p, + error_code& __ec) { + return __create_directories(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void +create_directory_symlink(const path& __to, const path& __new) { + __create_directory_symlink(__to, __new); +} + +inline _LIBCPP_INLINE_VISIBILITY void +create_directory_symlink(const path& __to, const path& __new, + error_code& __ec) noexcept { + __create_directory_symlink(__to, __new, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool create_directory(const path& __p) { + return __create_directory(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +create_directory(const path& __p, error_code& __ec) noexcept { + return __create_directory(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool create_directory(const path& __p, + const path& __attrs) { + return __create_directory(__p, __attrs); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +create_directory(const path& __p, const path& __attrs, + error_code& __ec) noexcept { + return __create_directory(__p, __attrs, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void create_hard_link(const path& __to, + const path& __new) { + __create_hard_link(__to, __new); +} + +inline _LIBCPP_INLINE_VISIBILITY void +create_hard_link(const path& __to, const path& __new, + error_code& __ec) noexcept { + __create_hard_link(__to, __new, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void create_symlink(const path& __to, + const path& __new) { + __create_symlink(__to, __new); +} + +inline _LIBCPP_INLINE_VISIBILITY void +create_symlink(const path& __to, const path& __new, error_code& __ec) noexcept { + return __create_symlink(__to, __new, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path current_path() { + return __current_path(); +} + +inline _LIBCPP_INLINE_VISIBILITY path current_path(error_code& __ec) { + return __current_path(&__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void current_path(const path& __p) { + __current_path(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY void current_path(const path& __p, + error_code& __ec) noexcept { + __current_path(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool equivalent(const path& __p1, + const path& __p2) { + return __equivalent(__p1, __p2); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept { + return __equivalent(__p1, __p2, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool status_known(file_status __s) noexcept { + return __s.type() != file_type::none; +} + +inline _LIBCPP_INLINE_VISIBILITY bool exists(file_status __s) noexcept { + return status_known(__s) && __s.type() != file_type::not_found; +} + +inline _LIBCPP_INLINE_VISIBILITY bool exists(const path& __p) { + return exists(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool exists(const path& __p, + error_code& __ec) noexcept { + auto __s = __status(__p, &__ec); + if (status_known(__s)) + __ec.clear(); + return exists(__s); +} + +inline _LIBCPP_INLINE_VISIBILITY uintmax_t file_size(const path& __p) { + return __file_size(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY uintmax_t +file_size(const path& __p, error_code& __ec) noexcept { + return __file_size(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY uintmax_t hard_link_count(const path& __p) { + return __hard_link_count(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY uintmax_t +hard_link_count(const path& __p, error_code& __ec) noexcept { + return __hard_link_count(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(file_status __s) noexcept { + return __s.type() == file_type::block; +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(const path& __p) { + return is_block_file(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(const path& __p, + error_code& __ec) noexcept { + return is_block_file(__status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +is_character_file(file_status __s) noexcept { + return __s.type() == file_type::character; +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_character_file(const path& __p) { + return is_character_file(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +is_character_file(const path& __p, error_code& __ec) noexcept { + return is_character_file(__status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_directory(file_status __s) noexcept { + return __s.type() == file_type::directory; +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_directory(const path& __p) { + return is_directory(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_directory(const path& __p, + error_code& __ec) noexcept { + return is_directory(__status(__p, &__ec)); +} + +_LIBCPP_FUNC_VIS +bool __fs_is_empty(const path& p, error_code* ec = nullptr); + +inline _LIBCPP_INLINE_VISIBILITY bool is_empty(const path& __p) { + return __fs_is_empty(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_empty(const path& __p, + error_code& __ec) { + return __fs_is_empty(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(file_status __s) noexcept { + return __s.type() == file_type::fifo; +} +inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(const path& __p) { + return is_fifo(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(const path& __p, + error_code& __ec) noexcept { + return is_fifo(__status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +is_regular_file(file_status __s) noexcept { + return __s.type() == file_type::regular; +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_regular_file(const path& __p) { + return is_regular_file(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool +is_regular_file(const path& __p, error_code& __ec) noexcept { + return is_regular_file(__status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(file_status __s) noexcept { + return __s.type() == file_type::symlink; +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(const path& __p) { + return is_symlink(__symlink_status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(const path& __p, + error_code& __ec) noexcept { + return is_symlink(__symlink_status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_other(file_status __s) noexcept { + return exists(__s) && !is_regular_file(__s) && !is_directory(__s) && + !is_symlink(__s); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_other(const path& __p) { + return is_other(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_other(const path& __p, + error_code& __ec) noexcept { + return is_other(__status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_socket(file_status __s) noexcept { + return __s.type() == file_type::socket; +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_socket(const path& __p) { + return is_socket(__status(__p)); +} + +inline _LIBCPP_INLINE_VISIBILITY bool is_socket(const path& __p, + error_code& __ec) noexcept { + return is_socket(__status(__p, &__ec)); +} + +inline _LIBCPP_INLINE_VISIBILITY file_time_type +last_write_time(const path& __p) { + return __last_write_time(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY file_time_type +last_write_time(const path& __p, error_code& __ec) noexcept { + return __last_write_time(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void last_write_time(const path& __p, + file_time_type __t) { + __last_write_time(__p, __t); +} + +inline _LIBCPP_INLINE_VISIBILITY void +last_write_time(const path& __p, file_time_type __t, + error_code& __ec) noexcept { + __last_write_time(__p, __t, &__ec); +} + +_LIBCPP_FUNC_VIS +void __permissions(const path&, perms, perm_options, error_code* = nullptr); + +inline _LIBCPP_INLINE_VISIBILITY void +permissions(const path& __p, perms __prms, + perm_options __opts = perm_options::replace) { + __permissions(__p, __prms, __opts); +} + +inline _LIBCPP_INLINE_VISIBILITY void permissions(const path& __p, perms __prms, + error_code& __ec) noexcept { + __permissions(__p, __prms, perm_options::replace, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void permissions(const path& __p, perms __prms, + perm_options __opts, + error_code& __ec) { + __permissions(__p, __prms, __opts, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path proximate(const path& __p, + const path& __base, + error_code& __ec) { + path __tmp = __weakly_canonical(__p, &__ec); + if (__ec) + return {}; + path __tmp_base = __weakly_canonical(__base, &__ec); + if (__ec) + return {}; + return __tmp.lexically_proximate(__tmp_base); +} + +inline _LIBCPP_INLINE_VISIBILITY path proximate(const path& __p, + error_code& __ec) { + return proximate(__p, current_path(), __ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path +proximate(const path& __p, const path& __base = current_path()) { + return __weakly_canonical(__p).lexically_proximate( + __weakly_canonical(__base)); +} + +inline _LIBCPP_INLINE_VISIBILITY path read_symlink(const path& __p) { + return __read_symlink(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY path read_symlink(const path& __p, + error_code& __ec) { + return __read_symlink(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path relative(const path& __p, + const path& __base, + error_code& __ec) { + path __tmp = __weakly_canonical(__p, &__ec); + if (__ec) + return path(); + path __tmpbase = __weakly_canonical(__base, &__ec); + if (__ec) + return path(); + return __tmp.lexically_relative(__tmpbase); +} + +inline _LIBCPP_INLINE_VISIBILITY path relative(const path& __p, + error_code& __ec) { + return relative(__p, current_path(), __ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path +relative(const path& __p, const path& __base = current_path()) { + return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base)); +} + +inline _LIBCPP_INLINE_VISIBILITY uintmax_t remove_all(const path& __p) { + return __remove_all(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY uintmax_t remove_all(const path& __p, + error_code& __ec) { + return __remove_all(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY bool remove(const path& __p) { + return __remove(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY bool remove(const path& __p, + error_code& __ec) noexcept { + return __remove(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void rename(const path& __from, + const path& __to) { + return __rename(__from, __to); +} + +inline _LIBCPP_INLINE_VISIBILITY void +rename(const path& __from, const path& __to, error_code& __ec) noexcept { + return __rename(__from, __to, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY void resize_file(const path& __p, + uintmax_t __ns) { + return __resize_file(__p, __ns); +} + +inline _LIBCPP_INLINE_VISIBILITY void +resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept { + return __resize_file(__p, __ns, &__ec); +} + +_LIBCPP_FUNC_VIS +space_info __space(const path&, error_code* __ec = nullptr); + +inline _LIBCPP_INLINE_VISIBILITY space_info space(const path& __p) { + return __space(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY space_info space(const path& __p, + error_code& __ec) noexcept { + return __space(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY file_status status(const path& __p) { + return __status(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY file_status status(const path& __p, + error_code& __ec) noexcept { + return __status(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY file_status symlink_status(const path& __p) { + return __symlink_status(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY file_status +symlink_status(const path& __p, error_code& __ec) noexcept { + return __symlink_status(__p, &__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path temp_directory_path() { + return __temp_directory_path(); +} + +inline _LIBCPP_INLINE_VISIBILITY path temp_directory_path(error_code& __ec) { + return __temp_directory_path(&__ec); +} + +inline _LIBCPP_INLINE_VISIBILITY path weakly_canonical(path const& __p) { + return __weakly_canonical(__p); +} + +inline _LIBCPP_INLINE_VISIBILITY path weakly_canonical(path const& __p, + error_code& __ec) { + return __weakly_canonical(__p, &__ec); +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_OPERATIONS_H diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h new file mode 100644 index 000000000000..a6d1ee997d91 --- /dev/null +++ b/libcxx/include/__filesystem/path.h @@ -0,0 +1,1018 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_PATH_H +#define _LIBCPP___FILESYSTEM_PATH_H + +#include <__availability> +#include <__config> +#include +#include +#include <__iterator/back_insert_iterator.h> +#include <__iterator/iterator_traits.h> +#include +#include + +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +# include // for quoted +#endif + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +template +struct __can_convert_char { + static const bool value = false; +}; +template +struct __can_convert_char : public __can_convert_char<_Tp> {}; +template <> +struct __can_convert_char { + static const bool value = true; + using __char_type = char; +}; +template <> +struct __can_convert_char { + static const bool value = true; + using __char_type = wchar_t; +}; +#ifndef _LIBCPP_HAS_NO_CHAR8_T +template <> +struct __can_convert_char { + static const bool value = true; + using __char_type = char8_t; +}; +#endif +template <> +struct __can_convert_char { + static const bool value = true; + using __char_type = char16_t; +}; +template <> +struct __can_convert_char { + static const bool value = true; + using __char_type = char32_t; +}; + +template +typename enable_if<__can_convert_char<_ECharT>::value, bool>::type +__is_separator(_ECharT __e) { +#if defined(_LIBCPP_WIN32API) + return __e == _ECharT('/') || __e == _ECharT('\\'); +#else + return __e == _ECharT('/'); +#endif +} + +#ifndef _LIBCPP_HAS_NO_CHAR8_T +typedef u8string __u8_string; +#else +typedef string __u8_string; +#endif + +struct _NullSentinel {}; + +template +using _Void = void; + +template +struct __is_pathable_string : public false_type {}; + +template +struct __is_pathable_string< + basic_string<_ECharT, _Traits, _Alloc>, + _Void::__char_type> > + : public __can_convert_char<_ECharT> { + using _Str = basic_string<_ECharT, _Traits, _Alloc>; + using _Base = __can_convert_char<_ECharT>; + static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); } + static _ECharT const* __range_end(_Str const& __s) { + return __s.data() + __s.length(); + } + static _ECharT __first_or_null(_Str const& __s) { + return __s.empty() ? _ECharT{} : __s[0]; + } +}; + +template +struct __is_pathable_string< + basic_string_view<_ECharT, _Traits>, + _Void::__char_type> > + : public __can_convert_char<_ECharT> { + using _Str = basic_string_view<_ECharT, _Traits>; + using _Base = __can_convert_char<_ECharT>; + static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); } + static _ECharT const* __range_end(_Str const& __s) { + return __s.data() + __s.length(); + } + static _ECharT __first_or_null(_Str const& __s) { + return __s.empty() ? _ECharT{} : __s[0]; + } +}; + +template ::type, + class _UnqualPtrType = + typename remove_const::type>::type, + bool _IsCharPtr = is_pointer<_DS>::value&& + __can_convert_char<_UnqualPtrType>::value> +struct __is_pathable_char_array : false_type {}; + +template +struct __is_pathable_char_array<_Source, _ECharT*, _UPtr, true> + : __can_convert_char::type> { + using _Base = __can_convert_char::type>; + + static _ECharT const* __range_begin(const _ECharT* __b) { return __b; } + static _ECharT const* __range_end(const _ECharT* __b) { + using _Iter = const _ECharT*; + const _ECharT __sentinel = _ECharT{}; + _Iter __e = __b; + for (; *__e != __sentinel; ++__e) + ; + return __e; + } + + static _ECharT __first_or_null(const _ECharT* __b) { return *__b; } +}; + +template ::value, + class = void> +struct __is_pathable_iter : false_type {}; + +template +struct __is_pathable_iter< + _Iter, true, + _Void::value_type>::__char_type> > + : __can_convert_char::value_type> { + using _ECharT = typename iterator_traits<_Iter>::value_type; + using _Base = __can_convert_char<_ECharT>; + + static _Iter __range_begin(_Iter __b) { return __b; } + static _NullSentinel __range_end(_Iter) { return _NullSentinel{}; } + + static _ECharT __first_or_null(_Iter __b) { return *__b; } +}; + +template ::value, + bool _IsCharIterT = __is_pathable_char_array<_Tp>::value, + bool _IsIterT = !_IsCharIterT && __is_pathable_iter<_Tp>::value> +struct __is_pathable : false_type { + static_assert(!_IsStringT && !_IsCharIterT && !_IsIterT, "Must all be false"); +}; + +template +struct __is_pathable<_Tp, true, false, false> : __is_pathable_string<_Tp> {}; + +template +struct __is_pathable<_Tp, false, true, false> : __is_pathable_char_array<_Tp> { +}; + +template +struct __is_pathable<_Tp, false, false, true> : __is_pathable_iter<_Tp> {}; + +#if defined(_LIBCPP_WIN32API) +typedef wstring __path_string; +typedef wchar_t __path_value; +#else +typedef string __path_string; +typedef char __path_value; +#endif + +#if defined(_LIBCPP_WIN32API) +_LIBCPP_FUNC_VIS +size_t __wide_to_char(const wstring&, char*, size_t); +_LIBCPP_FUNC_VIS +size_t __char_to_wide(const string&, wchar_t*, size_t); +#endif + +template +struct _PathCVT; + +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +template +struct _PathCVT { + static_assert(__can_convert_char<_ECharT>::value, + "Char type not convertible"); + + typedef __narrow_to_utf8 _Narrower; +#if defined(_LIBCPP_WIN32API) + typedef __widen_from_utf8 _Widener; +#endif + + static void __append_range(__path_string& __dest, _ECharT const* __b, + _ECharT const* __e) { +#if defined(_LIBCPP_WIN32API) + string __utf8; + _Narrower()(back_inserter(__utf8), __b, __e); + _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); +#else + _Narrower()(back_inserter(__dest), __b, __e); +#endif + } + + template + static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload"); + if (__b == __e) + return; + basic_string<_ECharT> __tmp(__b, __e); +#if defined(_LIBCPP_WIN32API) + string __utf8; + _Narrower()(back_inserter(__utf8), __tmp.data(), + __tmp.data() + __tmp.length()); + _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); +#else + _Narrower()(back_inserter(__dest), __tmp.data(), + __tmp.data() + __tmp.length()); +#endif + } + + template + static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { + static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload"); + const _ECharT __sentinel = _ECharT{}; + if (*__b == __sentinel) + return; + basic_string<_ECharT> __tmp; + for (; *__b != __sentinel; ++__b) + __tmp.push_back(*__b); +#if defined(_LIBCPP_WIN32API) + string __utf8; + _Narrower()(back_inserter(__utf8), __tmp.data(), + __tmp.data() + __tmp.length()); + _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); +#else + _Narrower()(back_inserter(__dest), __tmp.data(), + __tmp.data() + __tmp.length()); +#endif + } + + template + static void __append_source(__path_string& __dest, _Source const& __s) { + using _Traits = __is_pathable<_Source>; + __append_range(__dest, _Traits::__range_begin(__s), + _Traits::__range_end(__s)); + } +}; +#endif // !_LIBCPP_HAS_NO_LOCALIZATION + +template <> +struct _PathCVT<__path_value> { + + template + static typename enable_if<__is_exactly_cpp17_input_iterator<_Iter>::value>::type + __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + for (; __b != __e; ++__b) + __dest.push_back(*__b); + } + + template + static typename enable_if<__is_cpp17_forward_iterator<_Iter>::value>::type + __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + __dest.append(__b, __e); + } + + template + static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { + const char __sentinel = char{}; + for (; *__b != __sentinel; ++__b) + __dest.push_back(*__b); + } + + template + static void __append_source(__path_string& __dest, _Source const& __s) { + using _Traits = __is_pathable<_Source>; + __append_range(__dest, _Traits::__range_begin(__s), + _Traits::__range_end(__s)); + } +}; + +#if defined(_LIBCPP_WIN32API) +template <> +struct _PathCVT { + + static void + __append_string(__path_string& __dest, const basic_string &__str) { + size_t __size = __char_to_wide(__str, nullptr, 0); + size_t __pos = __dest.size(); + __dest.resize(__pos + __size); + __char_to_wide(__str, const_cast<__path_value*>(__dest.data()) + __pos, __size); + } + + template + static typename enable_if<__is_exactly_cpp17_input_iterator<_Iter>::value>::type + __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + basic_string __tmp(__b, __e); + __append_string(__dest, __tmp); + } + + template + static typename enable_if<__is_cpp17_forward_iterator<_Iter>::value>::type + __append_range(__path_string& __dest, _Iter __b, _Iter __e) { + basic_string __tmp(__b, __e); + __append_string(__dest, __tmp); + } + + template + static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) { + const char __sentinel = char{}; + basic_string __tmp; + for (; *__b != __sentinel; ++__b) + __tmp.push_back(*__b); + __append_string(__dest, __tmp); + } + + template + static void __append_source(__path_string& __dest, _Source const& __s) { + using _Traits = __is_pathable<_Source>; + __append_range(__dest, _Traits::__range_begin(__s), + _Traits::__range_end(__s)); + } +}; + +template +struct _PathExport { + typedef __narrow_to_utf8 _Narrower; + typedef __widen_from_utf8 _Widener; + + template + static void __append(_Str& __dest, const __path_string& __src) { + string __utf8; + _Narrower()(back_inserter(__utf8), __src.data(), __src.data() + __src.size()); + _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size()); + } +}; + +template <> +struct _PathExport { + template + static void __append(_Str& __dest, const __path_string& __src) { + size_t __size = __wide_to_char(__src, nullptr, 0); + size_t __pos = __dest.size(); + __dest.resize(__size); + __wide_to_char(__src, const_cast(__dest.data()) + __pos, __size); + } +}; + +template <> +struct _PathExport { + template + static void __append(_Str& __dest, const __path_string& __src) { + __dest.append(__src.begin(), __src.end()); + } +}; + +template <> +struct _PathExport { + template + static void __append(_Str& __dest, const __path_string& __src) { + __dest.append(__src.begin(), __src.end()); + } +}; + +#ifndef _LIBCPP_HAS_NO_CHAR8_T +template <> +struct _PathExport { + typedef __narrow_to_utf8 _Narrower; + + template + static void __append(_Str& __dest, const __path_string& __src) { + _Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size()); + } +}; +#endif /* !_LIBCPP_HAS_NO_CHAR8_T */ +#endif /* _LIBCPP_WIN32API */ + +class _LIBCPP_TYPE_VIS path { + template + using _EnableIfPathable = + typename enable_if<__is_pathable<_SourceOrIter>::value, _Tp>::type; + + template + using _SourceChar = typename __is_pathable<_Tp>::__char_type; + + template + using _SourceCVT = _PathCVT<_SourceChar<_Tp> >; + +public: +#if defined(_LIBCPP_WIN32API) + typedef wchar_t value_type; + static constexpr value_type preferred_separator = L'\\'; +#else + typedef char value_type; + static constexpr value_type preferred_separator = '/'; +#endif + typedef basic_string string_type; + typedef basic_string_view __string_view; + + enum _LIBCPP_ENUM_VIS format : unsigned char { + auto_format, + native_format, + generic_format + }; + + // constructors and destructor + _LIBCPP_INLINE_VISIBILITY path() noexcept {} + _LIBCPP_INLINE_VISIBILITY path(const path& __p) : __pn_(__p.__pn_) {} + _LIBCPP_INLINE_VISIBILITY path(path&& __p) noexcept + : __pn_(_VSTD::move(__p.__pn_)) {} + + _LIBCPP_INLINE_VISIBILITY + path(string_type&& __s, format = format::auto_format) noexcept + : __pn_(_VSTD::move(__s)) {} + + template > + path(const _Source& __src, format = format::auto_format) { + _SourceCVT<_Source>::__append_source(__pn_, __src); + } + + template + path(_InputIt __first, _InputIt __last, format = format::auto_format) { + typedef typename iterator_traits<_InputIt>::value_type _ItVal; + _PathCVT<_ItVal>::__append_range(__pn_, __first, __last); + } + +/* +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + // TODO Implement locale conversions. + template > + path(const _Source& __src, const locale& __loc, format = format::auto_format); + template + path(_InputIt __first, _InputIt _last, const locale& __loc, + format = format::auto_format); +#endif +*/ + + _LIBCPP_INLINE_VISIBILITY + ~path() = default; + + // assignments + _LIBCPP_INLINE_VISIBILITY + path& operator=(const path& __p) { + __pn_ = __p.__pn_; + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& operator=(path&& __p) noexcept { + __pn_ = _VSTD::move(__p.__pn_); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& operator=(string_type&& __s) noexcept { + __pn_ = _VSTD::move(__s); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& assign(string_type&& __s) noexcept { + __pn_ = _VSTD::move(__s); + return *this; + } + + template + _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source> + operator=(const _Source& __src) { + return this->assign(__src); + } + + template + _EnableIfPathable<_Source> assign(const _Source& __src) { + __pn_.clear(); + _SourceCVT<_Source>::__append_source(__pn_, __src); + return *this; + } + + template + path& assign(_InputIt __first, _InputIt __last) { + typedef typename iterator_traits<_InputIt>::value_type _ItVal; + __pn_.clear(); + _PathCVT<_ItVal>::__append_range(__pn_, __first, __last); + return *this; + } + +public: + // appends +#if defined(_LIBCPP_WIN32API) + path& operator/=(const path& __p) { + auto __p_root_name = __p.__root_name(); + auto __p_root_name_size = __p_root_name.size(); + if (__p.is_absolute() || + (!__p_root_name.empty() && __p_root_name != __string_view(root_name().__pn_))) { + __pn_ = __p.__pn_; + return *this; + } + if (__p.has_root_directory()) { + path __root_name_str = root_name(); + __pn_ = __root_name_str.native(); + __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size); + return *this; + } + if (has_filename() || (!has_root_directory() && is_absolute())) + __pn_ += preferred_separator; + __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size); + return *this; + } + template + _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source> + operator/=(const _Source& __src) { + return operator/=(path(__src)); + } + + template + _EnableIfPathable<_Source> append(const _Source& __src) { + return operator/=(path(__src)); + } + + template + path& append(_InputIt __first, _InputIt __last) { + return operator/=(path(__first, __last)); + } +#else + path& operator/=(const path& __p) { + if (__p.is_absolute()) { + __pn_ = __p.__pn_; + return *this; + } + if (has_filename()) + __pn_ += preferred_separator; + __pn_ += __p.native(); + return *this; + } + + // FIXME: Use _LIBCPP_DIAGNOSE_WARNING to produce a diagnostic when __src + // is known at compile time to be "/' since the user almost certainly intended + // to append a separator instead of overwriting the path with "/" + template + _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source> + operator/=(const _Source& __src) { + return this->append(__src); + } + + template + _EnableIfPathable<_Source> append(const _Source& __src) { + using _Traits = __is_pathable<_Source>; + using _CVT = _PathCVT<_SourceChar<_Source> >; + bool __source_is_absolute = __is_separator(_Traits::__first_or_null(__src)); + if (__source_is_absolute) + __pn_.clear(); + else if (has_filename()) + __pn_ += preferred_separator; + _CVT::__append_source(__pn_, __src); + return *this; + } + + template + path& append(_InputIt __first, _InputIt __last) { + typedef typename iterator_traits<_InputIt>::value_type _ItVal; + static_assert(__can_convert_char<_ItVal>::value, "Must convertible"); + using _CVT = _PathCVT<_ItVal>; + if (__first != __last && __is_separator(*__first)) + __pn_.clear(); + else if (has_filename()) + __pn_ += preferred_separator; + _CVT::__append_range(__pn_, __first, __last); + return *this; + } +#endif + + // concatenation + _LIBCPP_INLINE_VISIBILITY + path& operator+=(const path& __x) { + __pn_ += __x.__pn_; + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& operator+=(const string_type& __x) { + __pn_ += __x; + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& operator+=(__string_view __x) { + __pn_ += __x; + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& operator+=(const value_type* __x) { + __pn_ += __x; + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& operator+=(value_type __x) { + __pn_ += __x; + return *this; + } + + template + typename enable_if<__can_convert_char<_ECharT>::value, path&>::type + operator+=(_ECharT __x) { + _PathCVT<_ECharT>::__append_source(__pn_, + basic_string_view<_ECharT>(&__x, 1)); + return *this; + } + + template + _EnableIfPathable<_Source> operator+=(const _Source& __x) { + return this->concat(__x); + } + + template + _EnableIfPathable<_Source> concat(const _Source& __x) { + _SourceCVT<_Source>::__append_source(__pn_, __x); + return *this; + } + + template + path& concat(_InputIt __first, _InputIt __last) { + typedef typename iterator_traits<_InputIt>::value_type _ItVal; + _PathCVT<_ItVal>::__append_range(__pn_, __first, __last); + return *this; + } + + // modifiers + _LIBCPP_INLINE_VISIBILITY + void clear() noexcept { __pn_.clear(); } + + path& make_preferred() { +#if defined(_LIBCPP_WIN32API) + _VSTD::replace(__pn_.begin(), __pn_.end(), L'/', L'\\'); +#endif + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + path& remove_filename() { + auto __fname = __filename(); + if (!__fname.empty()) + __pn_.erase(__fname.data() - __pn_.data()); + return *this; + } + + path& replace_filename(const path& __replacement) { + remove_filename(); + return (*this /= __replacement); + } + + path& replace_extension(const path& __replacement = path()); + + _LIBCPP_INLINE_VISIBILITY + void swap(path& __rhs) noexcept { __pn_.swap(__rhs.__pn_); } + + // private helper to allow reserving memory in the path + _LIBCPP_INLINE_VISIBILITY + void __reserve(size_t __s) { __pn_.reserve(__s); } + + // native format observers + _LIBCPP_INLINE_VISIBILITY + const string_type& native() const noexcept { return __pn_; } + + _LIBCPP_INLINE_VISIBILITY + const value_type* c_str() const noexcept { return __pn_.c_str(); } + + _LIBCPP_INLINE_VISIBILITY operator string_type() const { return __pn_; } + +#if defined(_LIBCPP_WIN32API) + _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const { return __pn_; } + + _VSTD::wstring generic_wstring() const { + _VSTD::wstring __s; + __s.resize(__pn_.size()); + _VSTD::replace_copy(__pn_.begin(), __pn_.end(), __s.begin(), '\\', '/'); + return __s; + } + +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template , + class _Allocator = allocator<_ECharT> > + basic_string<_ECharT, _Traits, _Allocator> + string(const _Allocator& __a = _Allocator()) const { + using _Str = basic_string<_ECharT, _Traits, _Allocator>; + _Str __s(__a); + __s.reserve(__pn_.size()); + _PathExport<_ECharT>::__append(__s, __pn_); + return __s; + } + + _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const { + return string(); + } + _LIBCPP_INLINE_VISIBILITY __u8_string u8string() const { + using _CVT = __narrow_to_utf8; + __u8_string __s; + __s.reserve(__pn_.size()); + _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size()); + return __s; + } + + _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const { + return string(); + } + _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const { + return string(); + } + + // generic format observers + template , + class _Allocator = allocator<_ECharT> > + basic_string<_ECharT, _Traits, _Allocator> + generic_string(const _Allocator& __a = _Allocator()) const { + using _Str = basic_string<_ECharT, _Traits, _Allocator>; + _Str __s = string<_ECharT, _Traits, _Allocator>(__a); + // Note: This (and generic_u8string below) is slightly suboptimal as + // it iterates twice over the string; once to convert it to the right + // character type, and once to replace path delimiters. + _VSTD::replace(__s.begin(), __s.end(), + static_cast<_ECharT>('\\'), static_cast<_ECharT>('/')); + return __s; + } + + _VSTD::string generic_string() const { return generic_string(); } + _VSTD::u16string generic_u16string() const { return generic_string(); } + _VSTD::u32string generic_u32string() const { return generic_string(); } + __u8_string generic_u8string() const { + __u8_string __s = u8string(); + _VSTD::replace(__s.begin(), __s.end(), '\\', '/'); + return __s; + } +#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ +#else /* _LIBCPP_WIN32API */ + + _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const { return __pn_; } +#ifndef _LIBCPP_HAS_NO_CHAR8_T + _LIBCPP_INLINE_VISIBILITY _VSTD::u8string u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); } +#else + _LIBCPP_INLINE_VISIBILITY _VSTD::string u8string() const { return __pn_; } +#endif + +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template , + class _Allocator = allocator<_ECharT> > + basic_string<_ECharT, _Traits, _Allocator> + string(const _Allocator& __a = _Allocator()) const { + using _CVT = __widen_from_utf8; + using _Str = basic_string<_ECharT, _Traits, _Allocator>; + _Str __s(__a); + __s.reserve(__pn_.size()); + _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size()); + return __s; + } + +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const { + return string(); + } +#endif + _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const { + return string(); + } + _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const { + return string(); + } +#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ + + // generic format observers + _VSTD::string generic_string() const { return __pn_; } +#ifndef _LIBCPP_HAS_NO_CHAR8_T + _VSTD::u8string generic_u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); } +#else + _VSTD::string generic_u8string() const { return __pn_; } +#endif + +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template , + class _Allocator = allocator<_ECharT> > + basic_string<_ECharT, _Traits, _Allocator> + generic_string(const _Allocator& __a = _Allocator()) const { + return string<_ECharT, _Traits, _Allocator>(__a); + } + +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + _VSTD::wstring generic_wstring() const { return string(); } +#endif + _VSTD::u16string generic_u16string() const { return string(); } + _VSTD::u32string generic_u32string() const { return string(); } +#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */ +#endif /* !_LIBCPP_WIN32API */ + +private: + int __compare(__string_view) const; + __string_view __root_name() const; + __string_view __root_directory() const; + __string_view __root_path_raw() const; + __string_view __relative_path() const; + __string_view __parent_path() const; + __string_view __filename() const; + __string_view __stem() const; + __string_view __extension() const; + +public: + // compare + _LIBCPP_INLINE_VISIBILITY int compare(const path& __p) const noexcept { + return __compare(__p.__pn_); + } + _LIBCPP_INLINE_VISIBILITY int compare(const string_type& __s) const { + return __compare(__s); + } + _LIBCPP_INLINE_VISIBILITY int compare(__string_view __s) const { + return __compare(__s); + } + _LIBCPP_INLINE_VISIBILITY int compare(const value_type* __s) const { + return __compare(__s); + } + + // decomposition + _LIBCPP_INLINE_VISIBILITY path root_name() const { + return string_type(__root_name()); + } + _LIBCPP_INLINE_VISIBILITY path root_directory() const { + return string_type(__root_directory()); + } + _LIBCPP_INLINE_VISIBILITY path root_path() const { +#if defined(_LIBCPP_WIN32API) + return string_type(__root_path_raw()); +#else + return root_name().append(string_type(__root_directory())); +#endif + } + _LIBCPP_INLINE_VISIBILITY path relative_path() const { + return string_type(__relative_path()); + } + _LIBCPP_INLINE_VISIBILITY path parent_path() const { + return string_type(__parent_path()); + } + _LIBCPP_INLINE_VISIBILITY path filename() const { + return string_type(__filename()); + } + _LIBCPP_INLINE_VISIBILITY path stem() const { return string_type(__stem()); } + _LIBCPP_INLINE_VISIBILITY path extension() const { + return string_type(__extension()); + } + + // query + _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool + empty() const noexcept { + return __pn_.empty(); + } + + _LIBCPP_INLINE_VISIBILITY bool has_root_name() const { + return !__root_name().empty(); + } + _LIBCPP_INLINE_VISIBILITY bool has_root_directory() const { + return !__root_directory().empty(); + } + _LIBCPP_INLINE_VISIBILITY bool has_root_path() const { + return !__root_path_raw().empty(); + } + _LIBCPP_INLINE_VISIBILITY bool has_relative_path() const { + return !__relative_path().empty(); + } + _LIBCPP_INLINE_VISIBILITY bool has_parent_path() const { + return !__parent_path().empty(); + } + _LIBCPP_INLINE_VISIBILITY bool has_filename() const { + return !__filename().empty(); + } + _LIBCPP_INLINE_VISIBILITY bool has_stem() const { return !__stem().empty(); } + _LIBCPP_INLINE_VISIBILITY bool has_extension() const { + return !__extension().empty(); + } + + _LIBCPP_INLINE_VISIBILITY bool is_absolute() const { +#if defined(_LIBCPP_WIN32API) + __string_view __root_name_str = __root_name(); + __string_view __root_dir = __root_directory(); + if (__root_name_str.size() == 2 && __root_name_str[1] == ':') { + // A drive letter with no root directory is relative, e.g. x:example. + return !__root_dir.empty(); + } + // If no root name, it's relative, e.g. \example is relative to the current drive + if (__root_name_str.empty()) + return false; + if (__root_name_str.size() < 3) + return false; + // A server root name, like \\server, is always absolute + if (__root_name_str[0] != '/' && __root_name_str[0] != '\\') + return false; + if (__root_name_str[1] != '/' && __root_name_str[1] != '\\') + return false; + // Seems to be a server root name + return true; +#else + return has_root_directory(); +#endif + } + _LIBCPP_INLINE_VISIBILITY bool is_relative() const { return !is_absolute(); } + + // relative paths + path lexically_normal() const; + path lexically_relative(const path& __base) const; + + _LIBCPP_INLINE_VISIBILITY path lexically_proximate(const path& __base) const { + path __result = this->lexically_relative(__base); + if (__result.native().empty()) + return *this; + return __result; + } + + // iterators + class _LIBCPP_TYPE_VIS iterator; + typedef iterator const_iterator; + + iterator begin() const; + iterator end() const; + +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) + template + _LIBCPP_INLINE_VISIBILITY friend + typename enable_if::value && + is_same<_Traits, char_traits >::value, + basic_ostream<_CharT, _Traits>&>::type + operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) { + __os << _VSTD::__quoted(__p.native()); + return __os; + } + + template + _LIBCPP_INLINE_VISIBILITY friend + typename enable_if::value || + !is_same<_Traits, char_traits >::value, + basic_ostream<_CharT, _Traits>&>::type + operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) { + __os << _VSTD::__quoted(__p.string<_CharT, _Traits>()); + return __os; + } + + template + _LIBCPP_INLINE_VISIBILITY friend basic_istream<_CharT, _Traits>& + operator>>(basic_istream<_CharT, _Traits>& __is, path& __p) { + basic_string<_CharT, _Traits> __tmp; + __is >> __quoted(__tmp); + __p = __tmp; + return __is; + } +#endif // !_LIBCPP_HAS_NO_LOCALIZATION + + friend _LIBCPP_INLINE_VISIBILITY bool operator==(const path& __lhs, const path& __rhs) noexcept { + return __lhs.__compare(__rhs.__pn_) == 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator!=(const path& __lhs, const path& __rhs) noexcept { + return __lhs.__compare(__rhs.__pn_) != 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator<(const path& __lhs, const path& __rhs) noexcept { + return __lhs.__compare(__rhs.__pn_) < 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator<=(const path& __lhs, const path& __rhs) noexcept { + return __lhs.__compare(__rhs.__pn_) <= 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator>(const path& __lhs, const path& __rhs) noexcept { + return __lhs.__compare(__rhs.__pn_) > 0; + } + friend _LIBCPP_INLINE_VISIBILITY bool operator>=(const path& __lhs, const path& __rhs) noexcept { + return __lhs.__compare(__rhs.__pn_) >= 0; + } + + friend _LIBCPP_INLINE_VISIBILITY path operator/(const path& __lhs, + const path& __rhs) { + path __result(__lhs); + __result /= __rhs; + return __result; + } +private: + inline _LIBCPP_INLINE_VISIBILITY path& + __assign_view(__string_view const& __s) noexcept { + __pn_ = string_type(__s); + return *this; + } + string_type __pn_; +}; + +inline _LIBCPP_INLINE_VISIBILITY void swap(path& __lhs, path& __rhs) noexcept { + __lhs.swap(__rhs); +} + +_LIBCPP_FUNC_VIS +size_t hash_value(const path& __p) noexcept; + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_PATH_H diff --git a/libcxx/include/__filesystem/path_iterator.h b/libcxx/include/__filesystem/path_iterator.h new file mode 100644 index 000000000000..62f8dc6fd357 --- /dev/null +++ b/libcxx/include/__filesystem/path_iterator.h @@ -0,0 +1,132 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_PATH_ITERATOR_H +#define _LIBCPP___FILESYSTEM_PATH_ITERATOR_H + +#include <__availability> +#include <__config> +#include <__filesystem/path.h> +#include <__iterator/iterator_traits.h> +#include <__debug> +#include +#include +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +class _LIBCPP_TYPE_VIS path::iterator { +public: + enum _ParserState : unsigned char { + _Singular, + _BeforeBegin, + _InRootName, + _InRootDir, + _InFilenames, + _InTrailingSep, + _AtEnd + }; + +public: + typedef bidirectional_iterator_tag iterator_category; + + typedef path value_type; + typedef ptrdiff_t difference_type; + typedef const path* pointer; + typedef const path& reference; + + typedef void + __stashing_iterator_tag; // See reverse_iterator and __is_stashing_iterator + +public: + _LIBCPP_INLINE_VISIBILITY + iterator() + : __stashed_elem_(), __path_ptr_(nullptr), __entry_(), + __state_(_Singular) {} + + iterator(const iterator&) = default; + ~iterator() = default; + + iterator& operator=(const iterator&) = default; + + _LIBCPP_INLINE_VISIBILITY + reference operator*() const { return __stashed_elem_; } + + _LIBCPP_INLINE_VISIBILITY + pointer operator->() const { return &__stashed_elem_; } + + _LIBCPP_INLINE_VISIBILITY + iterator& operator++() { + _LIBCPP_ASSERT(__state_ != _Singular, + "attempting to increment a singular iterator"); + _LIBCPP_ASSERT(__state_ != _AtEnd, + "attempting to increment the end iterator"); + return __increment(); + } + + _LIBCPP_INLINE_VISIBILITY + iterator operator++(int) { + iterator __it(*this); + this->operator++(); + return __it; + } + + _LIBCPP_INLINE_VISIBILITY + iterator& operator--() { + _LIBCPP_ASSERT(__state_ != _Singular, + "attempting to decrement a singular iterator"); + _LIBCPP_ASSERT(__entry_.data() != __path_ptr_->native().data(), + "attempting to decrement the begin iterator"); + return __decrement(); + } + + _LIBCPP_INLINE_VISIBILITY + iterator operator--(int) { + iterator __it(*this); + this->operator--(); + return __it; + } + +private: + friend class path; + + inline _LIBCPP_INLINE_VISIBILITY friend bool operator==(const iterator&, + const iterator&); + + iterator& __increment(); + iterator& __decrement(); + + path __stashed_elem_; + const path* __path_ptr_; + path::__string_view __entry_; + _ParserState __state_; +}; + +inline _LIBCPP_INLINE_VISIBILITY bool operator==(const path::iterator& __lhs, + const path::iterator& __rhs) { + return __lhs.__path_ptr_ == __rhs.__path_ptr_ && + __lhs.__entry_.data() == __rhs.__entry_.data(); +} + +inline _LIBCPP_INLINE_VISIBILITY bool operator!=(const path::iterator& __lhs, + const path::iterator& __rhs) { + return !(__lhs == __rhs); +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_PATH_ITERATOR_H diff --git a/libcxx/include/__filesystem/perm_options.h b/libcxx/include/__filesystem/perm_options.h new file mode 100644 index 000000000000..62cd8f575650 --- /dev/null +++ b/libcxx/include/__filesystem/perm_options.h @@ -0,0 +1,73 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_PERM_OPTIONS_H +#define _LIBCPP___FILESYSTEM_PERM_OPTIONS_H + +#include <__availability> +#include <__config> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +enum class _LIBCPP_ENUM_VIS perm_options : unsigned char { + replace = 1, + add = 2, + remove = 4, + nofollow = 8 +}; + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perm_options operator&(perm_options _LHS, perm_options _RHS) { + return static_cast(static_cast(_LHS) & + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perm_options operator|(perm_options _LHS, perm_options _RHS) { + return static_cast(static_cast(_LHS) | + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perm_options operator^(perm_options _LHS, perm_options _RHS) { + return static_cast(static_cast(_LHS) ^ + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perm_options operator~(perm_options _LHS) { + return static_cast(~static_cast(_LHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline perm_options& operator&=(perm_options& _LHS, perm_options _RHS) { + return _LHS = _LHS & _RHS; +} + +_LIBCPP_INLINE_VISIBILITY +inline perm_options& operator|=(perm_options& _LHS, perm_options _RHS) { + return _LHS = _LHS | _RHS; +} + +_LIBCPP_INLINE_VISIBILITY +inline perm_options& operator^=(perm_options& _LHS, perm_options _RHS) { + return _LHS = _LHS ^ _RHS; +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_PERM_OPTIONS_H diff --git a/libcxx/include/__filesystem/perms.h b/libcxx/include/__filesystem/perms.h new file mode 100644 index 000000000000..832f8b07e55c --- /dev/null +++ b/libcxx/include/__filesystem/perms.h @@ -0,0 +1,91 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_PERMS_H +#define _LIBCPP___FILESYSTEM_PERMS_H + +#include <__availability> +#include <__config> + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +// On Windows, these permission bits map to one single readonly flag per +// file, and the executable bit is always returned as set. When setting +// permissions, as long as the write bit is set for either owner, group or +// others, the readonly flag is cleared. +enum class _LIBCPP_ENUM_VIS perms : unsigned { + none = 0, + + owner_read = 0400, + owner_write = 0200, + owner_exec = 0100, + owner_all = 0700, + + group_read = 040, + group_write = 020, + group_exec = 010, + group_all = 070, + + others_read = 04, + others_write = 02, + others_exec = 01, + others_all = 07, + + all = 0777, + + set_uid = 04000, + set_gid = 02000, + sticky_bit = 01000, + mask = 07777, + unknown = 0xFFFF, +}; + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perms operator&(perms _LHS, perms _RHS) { + return static_cast(static_cast(_LHS) & + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perms operator|(perms _LHS, perms _RHS) { + return static_cast(static_cast(_LHS) | + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perms operator^(perms _LHS, perms _RHS) { + return static_cast(static_cast(_LHS) ^ + static_cast(_RHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline constexpr perms operator~(perms _LHS) { + return static_cast(~static_cast(_LHS)); +} + +_LIBCPP_INLINE_VISIBILITY +inline perms& operator&=(perms& _LHS, perms _RHS) { return _LHS = _LHS & _RHS; } + +_LIBCPP_INLINE_VISIBILITY +inline perms& operator|=(perms& _LHS, perms _RHS) { return _LHS = _LHS | _RHS; } + +_LIBCPP_INLINE_VISIBILITY +inline perms& operator^=(perms& _LHS, perms _RHS) { return _LHS = _LHS ^ _RHS; } + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_PERMS_H diff --git a/libcxx/include/__filesystem/recursive_directory_iterator.h b/libcxx/include/__filesystem/recursive_directory_iterator.h new file mode 100644 index 000000000000..db7e793e8530 --- /dev/null +++ b/libcxx/include/__filesystem/recursive_directory_iterator.h @@ -0,0 +1,181 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H +#define _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H + +#include <__availability> +#include <__config> +#include <__filesystem/directory_entry.h> +#include <__filesystem/directory_options.h> +#include <__filesystem/path.h> +#include <__iterator/iterator_traits.h> +#include <__memory/shared_ptr.h> +#include <__ranges/enable_borrowed_range.h> +#include <__ranges/enable_view.h> +#include +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +class recursive_directory_iterator { +public: + using value_type = directory_entry; + using difference_type = ptrdiff_t; + using pointer = directory_entry const*; + using reference = directory_entry const&; + using iterator_category = input_iterator_tag; + +public: + // constructors and destructor + _LIBCPP_INLINE_VISIBILITY + recursive_directory_iterator() noexcept : __rec_(false) {} + + _LIBCPP_INLINE_VISIBILITY + explicit recursive_directory_iterator( + const path& __p, directory_options __xoptions = directory_options::none) + : recursive_directory_iterator(__p, __xoptions, nullptr) {} + + _LIBCPP_INLINE_VISIBILITY + recursive_directory_iterator(const path& __p, directory_options __xoptions, + error_code& __ec) + : recursive_directory_iterator(__p, __xoptions, &__ec) {} + + _LIBCPP_INLINE_VISIBILITY + recursive_directory_iterator(const path& __p, error_code& __ec) + : recursive_directory_iterator(__p, directory_options::none, &__ec) {} + + recursive_directory_iterator(const recursive_directory_iterator&) = default; + recursive_directory_iterator(recursive_directory_iterator&&) = default; + + recursive_directory_iterator& + operator=(const recursive_directory_iterator&) = default; + + _LIBCPP_INLINE_VISIBILITY + recursive_directory_iterator& + operator=(recursive_directory_iterator&& __o) noexcept { + // non-default implementation provided to support self-move assign. + if (this != &__o) { + __imp_ = _VSTD::move(__o.__imp_); + __rec_ = __o.__rec_; + } + return *this; + } + + ~recursive_directory_iterator() = default; + + _LIBCPP_INLINE_VISIBILITY + const directory_entry& operator*() const { return __dereference(); } + + _LIBCPP_INLINE_VISIBILITY + const directory_entry* operator->() const { return &__dereference(); } + + recursive_directory_iterator& operator++() { return __increment(); } + + _LIBCPP_INLINE_VISIBILITY + __dir_element_proxy operator++(int) { + __dir_element_proxy __p(**this); + __increment(); + return __p; + } + + _LIBCPP_INLINE_VISIBILITY + recursive_directory_iterator& increment(error_code& __ec) { + return __increment(&__ec); + } + + _LIBCPP_FUNC_VIS directory_options options() const; + _LIBCPP_FUNC_VIS int depth() const; + + _LIBCPP_INLINE_VISIBILITY + void pop() { __pop(); } + + _LIBCPP_INLINE_VISIBILITY + void pop(error_code& __ec) { __pop(&__ec); } + + _LIBCPP_INLINE_VISIBILITY + bool recursion_pending() const { return __rec_; } + + _LIBCPP_INLINE_VISIBILITY + void disable_recursion_pending() { __rec_ = false; } + +private: + _LIBCPP_FUNC_VIS + recursive_directory_iterator(const path& __p, directory_options __opt, + error_code* __ec); + + _LIBCPP_FUNC_VIS + const directory_entry& __dereference() const; + + _LIBCPP_FUNC_VIS + bool __try_recursion(error_code* __ec); + + _LIBCPP_FUNC_VIS + void __advance(error_code* __ec = nullptr); + + _LIBCPP_FUNC_VIS + recursive_directory_iterator& __increment(error_code* __ec = nullptr); + + _LIBCPP_FUNC_VIS + void __pop(error_code* __ec = nullptr); + + inline _LIBCPP_INLINE_VISIBILITY friend bool + operator==(const recursive_directory_iterator&, + const recursive_directory_iterator&) noexcept; + + struct _LIBCPP_HIDDEN __shared_imp; + shared_ptr<__shared_imp> __imp_; + bool __rec_; +}; // class recursive_directory_iterator + +inline _LIBCPP_INLINE_VISIBILITY bool +operator==(const recursive_directory_iterator& __lhs, + const recursive_directory_iterator& __rhs) noexcept { + return __lhs.__imp_ == __rhs.__imp_; +} + +_LIBCPP_INLINE_VISIBILITY +inline bool operator!=(const recursive_directory_iterator& __lhs, + const recursive_directory_iterator& __rhs) noexcept { + return !(__lhs == __rhs); +} +// enable recursive_directory_iterator range-based for statements +inline _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator +begin(recursive_directory_iterator __iter) noexcept { + return __iter; +} + +inline _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator +end(recursive_directory_iterator) noexcept { + return recursive_directory_iterator(); +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#if !defined(_LIBCPP_HAS_NO_RANGES) + +template <> +_LIBCPP_AVAILABILITY_FILESYSTEM +inline constexpr bool _VSTD::ranges::enable_borrowed_range<_VSTD_FS::recursive_directory_iterator> = true; + +template <> +_LIBCPP_AVAILABILITY_FILESYSTEM +inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::recursive_directory_iterator> = true; + +#endif + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H diff --git a/libcxx/include/__filesystem/space_info.h b/libcxx/include/__filesystem/space_info.h new file mode 100644 index 000000000000..098f085678e4 --- /dev/null +++ b/libcxx/include/__filesystem/space_info.h @@ -0,0 +1,35 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_SPACE_INFO_H +#define _LIBCPP___FILESYSTEM_SPACE_INFO_H + +#include <__availability> +#include <__config> +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +struct _LIBCPP_TYPE_VIS space_info { + uintmax_t capacity; + uintmax_t free; + uintmax_t available; +}; + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_SPACE_INFO_H diff --git a/libcxx/include/__filesystem/u8path.h b/libcxx/include/__filesystem/u8path.h new file mode 100644 index 000000000000..dca3b0c5028b --- /dev/null +++ b/libcxx/include/__filesystem/u8path.h @@ -0,0 +1,96 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FILESYSTEM_U8PATH_H +#define _LIBCPP___FILESYSTEM_U8PATH_H + +#include <__availability> +#include <__config> +#include <__filesystem/path.h> +#include + +#ifndef _LIBCPP_CXX03_LANG + +_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM + +_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH + +template +_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T + typename enable_if<__is_pathable<_InputIt>::value, path>::type + u8path(_InputIt __f, _InputIt __l) { + static_assert( +#ifndef _LIBCPP_HAS_NO_CHAR8_T + is_same::__char_type, char8_t>::value || +#endif + is_same::__char_type, char>::value, + "u8path(Iter, Iter) requires Iter have a value_type of type 'char'" + " or 'char8_t'"); +#if defined(_LIBCPP_WIN32API) + string __tmp(__f, __l); + using _CVT = __widen_from_utf8; + _VSTD::wstring __w; + __w.reserve(__tmp.size()); + _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size()); + return path(__w); +#else + return path(__f, __l); +#endif /* !_LIBCPP_WIN32API */ +} + +#if defined(_LIBCPP_WIN32API) +template +_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T + typename enable_if<__is_pathable<_InputIt>::value, path>::type + u8path(_InputIt __f, _NullSentinel) { + static_assert( +#ifndef _LIBCPP_HAS_NO_CHAR8_T + is_same::__char_type, char8_t>::value || +#endif + is_same::__char_type, char>::value, + "u8path(Iter, Iter) requires Iter have a value_type of type 'char'" + " or 'char8_t'"); + string __tmp; + const char __sentinel = char{}; + for (; *__f != __sentinel; ++__f) + __tmp.push_back(*__f); + using _CVT = __widen_from_utf8; + _VSTD::wstring __w; + __w.reserve(__tmp.size()); + _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size()); + return path(__w); +} +#endif /* _LIBCPP_WIN32API */ + +template +_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T + typename enable_if<__is_pathable<_Source>::value, path>::type + u8path(const _Source& __s) { + static_assert( +#ifndef _LIBCPP_HAS_NO_CHAR8_T + is_same::__char_type, char8_t>::value || +#endif + is_same::__char_type, char>::value, + "u8path(Source const&) requires Source have a character type of type " + "'char' or 'char8_t'"); +#if defined(_LIBCPP_WIN32API) + using _Traits = __is_pathable<_Source>; + return u8path(_VSTD::__unwrap_iter(_Traits::__range_begin(__s)), _VSTD::__unwrap_iter(_Traits::__range_end(__s))); +#else + return path(__s); +#endif +} + +_LIBCPP_AVAILABILITY_FILESYSTEM_POP + +_LIBCPP_END_NAMESPACE_FILESYSTEM + +#endif // _LIBCPP_CXX03_LANG + +#endif // _LIBCPP___FILESYSTEM_U8PATH_H diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h index a9a8c1f0da03..59429c13d415 100644 --- a/libcxx/include/__format/format_arg.h +++ b/libcxx/include/__format/format_arg.h @@ -37,21 +37,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_CONCEPTS) namespace __format { -/** The type stored in @ref basic_format_arg. */ +/// The type stored in @ref basic_format_arg. +/// +/// @note The 128-bit types are unconditionally in the list to avoid the values +/// of the enums to depend on the availability of 128-bit integers. enum class _LIBCPP_ENUM_VIS __arg_t : uint8_t { __none, __boolean, __char_type, __int, __long_long, -#ifndef _LIBCPP_HAS_NO_INT128 __i128, -#endif __unsigned, __unsigned_long_long, -#ifndef _LIBCPP_HAS_NO_INT128 __u128, -#endif __float, __double, __long_double, @@ -75,18 +74,22 @@ visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) { return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__int); case __format::__arg_t::__long_long: return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__long_long); -#ifndef _LIBCPP_HAS_NO_INT128 case __format::__arg_t::__i128: +#ifndef _LIBCPP_HAS_NO_INT128 return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__i128); +#else + _LIBCPP_UNREACHABLE(); #endif case __format::__arg_t::__unsigned: return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__unsigned); case __format::__arg_t::__unsigned_long_long: return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__unsigned_long_long); -#ifndef _LIBCPP_HAS_NO_INT128 case __format::__arg_t::__u128: +#ifndef _LIBCPP_HAS_NO_INT128 return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__u128); +#else + _LIBCPP_UNREACHABLE(); #endif case __format::__arg_t::__float: return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__float); diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h index b4fe5cc7b12c..f8ec7c8eb001 100644 --- a/libcxx/include/__format/format_context.h +++ b/libcxx/include/__format/format_context.h @@ -14,10 +14,9 @@ #include <__config> #include <__format/format_args.h> #include <__format/format_fwd.h> +#include <__iterator/back_insert_iterator.h> #include <__iterator/concepts.h> #include -#include -#include #ifndef _LIBCPP_HAS_NO_LOCALIZATION #include diff --git a/libcxx/include/__format/formatter_string.h b/libcxx/include/__format/formatter_string.h index 2be36a1ba947..75a81f5184a0 100644 --- a/libcxx/include/__format/formatter_string.h +++ b/libcxx/include/__format/formatter_string.h @@ -16,7 +16,6 @@ #include <__format/format_string.h> #include <__format/formatter.h> #include <__format/parser_std_format_spec.h> -#include #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h index 0b74d91b7746..0eb95c824664 100644 --- a/libcxx/include/__functional/bind.h +++ b/libcxx/include/__functional/bind.h @@ -70,7 +70,7 @@ _LIBCPP_FUNC_VIS extern const __ph<10> _10; /* inline */ constexpr __ph<10> _10{}; #endif // defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY) -} // placeholders +} // namespace placeholders template struct __is_placeholder > diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h index 83fd7c12de2f..8336d85adf2e 100644 --- a/libcxx/include/__functional/function.h +++ b/libcxx/include/__functional/function.h @@ -946,7 +946,7 @@ class __func<_Rp1(^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> #endif // _LIBCPP_HAS_EXTENSION_BLOCKS && !_LIBCPP_HAS_OBJC_ARC -} // __function +} // namespace __function template class _LIBCPP_TEMPLATE_VIS function<_Rp(_ArgTypes...)> @@ -1717,13 +1717,11 @@ class _LIBCPP_TEMPLATE_VIS function<_Rp()> // 20.7.16.2.3, function capacity: _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;} -private: - // deleted overloads close possible hole in the type system template - bool operator==(const function<_R2()>&) const;// = delete; + bool operator==(const function<_R2()>&) const = delete; template - bool operator!=(const function<_R2()>&) const;// = delete; -public: + bool operator!=(const function<_R2()>&) const = delete; + // 20.7.16.2.4, function invocation: _Rp operator()() const; @@ -1997,13 +1995,11 @@ class _LIBCPP_TEMPLATE_VIS function<_Rp(_A0)> // 20.7.16.2.3, function capacity: _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;} -private: - // deleted overloads close possible hole in the type system template - bool operator==(const function<_R2(_B0)>&) const;// = delete; + bool operator==(const function<_R2(_B0)>&) const = delete; template - bool operator!=(const function<_R2(_B0)>&) const;// = delete; -public: + bool operator!=(const function<_R2(_B0)>&) const = delete; + // 20.7.16.2.4, function invocation: _Rp operator()(_A0) const; @@ -2277,13 +2273,11 @@ class _LIBCPP_TEMPLATE_VIS function<_Rp(_A0, _A1)> // 20.7.16.2.3, function capacity: _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;} -private: - // deleted overloads close possible hole in the type system template - bool operator==(const function<_R2(_B0, _B1)>&) const;// = delete; + bool operator==(const function<_R2(_B0, _B1)>&) const = delete; template - bool operator!=(const function<_R2(_B0, _B1)>&) const;// = delete; -public: + bool operator!=(const function<_R2(_B0, _B1)>&) const = delete; + // 20.7.16.2.4, function invocation: _Rp operator()(_A0, _A1) const; @@ -2556,13 +2550,11 @@ class _LIBCPP_TEMPLATE_VIS function<_Rp(_A0, _A1, _A2)> // 20.7.16.2.3, function capacity: _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;} -private: - // deleted overloads close possible hole in the type system template - bool operator==(const function<_R2(_B0, _B1, _B2)>&) const;// = delete; + bool operator==(const function<_R2(_B0, _B1, _B2)>&) const = delete; template - bool operator!=(const function<_R2(_B0, _B1, _B2)>&) const;// = delete; -public: + bool operator!=(const function<_R2(_B0, _B1, _B2)>&) const = delete; + // 20.7.16.2.4, function invocation: _Rp operator()(_A0, _A1, _A2) const; diff --git a/libcxx/include/__iterator/advance.h b/libcxx/include/__iterator/advance.h index a60052a08f0d..2236a57936cc 100644 --- a/libcxx/include/__iterator/advance.h +++ b/libcxx/include/__iterator/advance.h @@ -69,6 +69,7 @@ void advance(_InputIter& __i, _Distance __orig_n) { namespace ranges { // [range.iter.op.advance] +// TODO(varconst): rename `__advance_fn` to `__fn`. struct __advance_fn final : private __function_like { private: template diff --git a/libcxx/include/__iterator/concepts.h b/libcxx/include/__iterator/concepts.h index 531acdf0a5b2..d7a666743afb 100644 --- a/libcxx/include/__iterator/concepts.h +++ b/libcxx/include/__iterator/concepts.h @@ -28,8 +28,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_RANGES) -// clang-format off - // [iterator.concept.readable] template concept __indirectly_readable_impl = @@ -259,8 +257,6 @@ concept indirectly_movable_storable = // Note: indirectly_swappable is located in iter_swap.h to prevent a dependency cycle // (both iter_swap and indirectly_swappable require indirectly_readable). -// clang-format on - #endif // !defined(_LIBCPP_HAS_NO_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__iterator/next.h b/libcxx/include/__iterator/next.h index 0464708607d4..d332abfa8e0e 100644 --- a/libcxx/include/__iterator/next.h +++ b/libcxx/include/__iterator/next.h @@ -39,6 +39,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 #if !defined(_LIBCPP_HAS_NO_RANGES) namespace ranges { +// TODO(varconst): rename `__next_fn` to `__fn`. struct __next_fn final : private __function_like { _LIBCPP_HIDE_FROM_ABI constexpr explicit __next_fn(__tag __x) noexcept : __function_like(__x) {} @@ -79,4 +80,4 @@ inline constexpr auto next = __next_fn(__function_like::__tag()); _LIBCPP_END_NAMESPACE_STD -#endif // _LIBCPP___ITERATOR_PRIMITIVES_H +#endif // _LIBCPP___ITERATOR_NEXT_H diff --git a/libcxx/include/__iterator/prev.h b/libcxx/include/__iterator/prev.h index cbe1e8759af3..57f2d04a1325 100644 --- a/libcxx/include/__iterator/prev.h +++ b/libcxx/include/__iterator/prev.h @@ -38,6 +38,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14 #if !defined(_LIBCPP_HAS_NO_RANGES) namespace ranges { +// TODO(varconst): rename `__prev_fn` to `__fn`. struct __prev_fn final : private __function_like { _LIBCPP_HIDE_FROM_ABI constexpr explicit __prev_fn(__tag __x) noexcept : __function_like(__x) {} diff --git a/libcxx/include/__locale b/libcxx/include/__locale index 4296adbbd8e9..2582a90bb578 100644 --- a/libcxx/include/__locale +++ b/libcxx/include/__locale @@ -208,10 +208,11 @@ class _LIBCPP_TYPE_VIS locale::id static int32_t __next_id; public: _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR id() :__id_(0) {} + void operator=(const id&) = delete; + id(const id&) = delete; + private: void __init(); - void operator=(const id&); // = delete; - id(const id&); // = delete; public: // only needed for tests long __get(); diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h index 283212fb703d..708bd82b02d2 100644 --- a/libcxx/include/__memory/allocator.h +++ b/libcxx/include/__memory/allocator.h @@ -89,7 +89,7 @@ class _LIBCPP_TEMPLATE_VIS allocator typedef true_type is_always_equal; _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 - allocator() _NOEXCEPT _LIBCPP_DEFAULT + allocator() _NOEXCEPT = default; template _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 @@ -171,7 +171,7 @@ class _LIBCPP_TEMPLATE_VIS allocator typedef true_type is_always_equal; _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 - allocator() _NOEXCEPT _LIBCPP_DEFAULT + allocator() _NOEXCEPT = default; template _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h index fd1fcbe5bf39..fd6d7109f8eb 100644 --- a/libcxx/include/__memory/compressed_pair.h +++ b/libcxx/include/__memory/compressed_pair.h @@ -58,10 +58,8 @@ struct __compressed_pair_elem { : __value_(_VSTD::forward<_Args>(_VSTD::get<_Indexes>(__args))...) {} #endif - - _LIBCPP_INLINE_VISIBILITY reference __get() _NOEXCEPT { return __value_; } - _LIBCPP_INLINE_VISIBILITY - const_reference __get() const _NOEXCEPT { return __value_; } + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 reference __get() _NOEXCEPT { return __value_; } + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return __value_; } private: _Tp __value_; @@ -97,9 +95,8 @@ struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp { : __value_type(_VSTD::forward<_Args>(_VSTD::get<_Indexes>(__args))...) {} #endif - _LIBCPP_INLINE_VISIBILITY reference __get() _NOEXCEPT { return *this; } - _LIBCPP_INLINE_VISIBILITY - const_reference __get() const _NOEXCEPT { return *this; } + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 reference __get() _NOEXCEPT { return *this; } + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return *this; } }; template @@ -143,23 +140,19 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>, typename __make_tuple_indices::type()) {} #endif - _LIBCPP_INLINE_VISIBILITY - typename _Base1::reference first() _NOEXCEPT { + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 typename _Base1::reference first() _NOEXCEPT { return static_cast<_Base1&>(*this).__get(); } - _LIBCPP_INLINE_VISIBILITY - typename _Base1::const_reference first() const _NOEXCEPT { + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR typename _Base1::const_reference first() const _NOEXCEPT { return static_cast<_Base1 const&>(*this).__get(); } - _LIBCPP_INLINE_VISIBILITY - typename _Base2::reference second() _NOEXCEPT { + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 typename _Base2::reference second() _NOEXCEPT { return static_cast<_Base2&>(*this).__get(); } - _LIBCPP_INLINE_VISIBILITY - typename _Base2::const_reference second() const _NOEXCEPT { + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR typename _Base2::const_reference second() const _NOEXCEPT { return static_cast<_Base2 const&>(*this).__get(); } @@ -172,11 +165,8 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>, return static_cast<_Base2*>(__pair); } - _LIBCPP_INLINE_VISIBILITY - void swap(__compressed_pair& __x) - _NOEXCEPT_(__is_nothrow_swappable<_T1>::value && - __is_nothrow_swappable<_T2>::value) - { + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 void swap(__compressed_pair& __x) + _NOEXCEPT_(__is_nothrow_swappable<_T1>::value && __is_nothrow_swappable<_T2>::value) { using _VSTD::swap; swap(first(), __x.first()); swap(second(), __x.second()); @@ -184,10 +174,9 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>, }; template -inline _LIBCPP_INLINE_VISIBILITY +inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 void swap(__compressed_pair<_T1, _T2>& __x, __compressed_pair<_T1, _T2>& __y) - _NOEXCEPT_(__is_nothrow_swappable<_T1>::value && - __is_nothrow_swappable<_T2>::value) { + _NOEXCEPT_(__is_nothrow_swappable<_T1>::value && __is_nothrow_swappable<_T2>::value) { __x.swap(__y); } diff --git a/libcxx/include/__memory/concepts.h b/libcxx/include/__memory/concepts.h new file mode 100644 index 000000000000..4029b590fe8c --- /dev/null +++ b/libcxx/include/__memory/concepts.h @@ -0,0 +1,66 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_CONCEPTS_H +#define _LIBCPP___MEMORY_CONCEPTS_H + +#include <__config> +#include <__iterator/concepts.h> +#include <__iterator/iterator_traits.h> +#include <__iterator/readable_traits.h> +#include <__ranges/access.h> +#include <__ranges/concepts.h> +#include +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +#pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if !defined(_LIBCPP_HAS_NO_RANGES) +namespace ranges { + +// [special.mem.concepts] + +// This concept ensures that uninitialized algorithms can construct an object +// at the address pointed-to by the iterator, which requires an lvalue. +template +concept __nothrow_input_iterator = + input_iterator<_Ip> && + is_lvalue_reference_v> && + same_as>, iter_value_t<_Ip>>; + +template +concept __nothrow_sentinel_for = sentinel_for<_Sp, _Ip>; + +template +concept __nothrow_input_range = + range<_Rp> && + __nothrow_input_iterator> && + __nothrow_sentinel_for, iterator_t<_Rp>>; + +template +concept __nothrow_forward_iterator = + __nothrow_input_iterator<_Ip> && + forward_iterator<_Ip> && + __nothrow_sentinel_for<_Ip, _Ip>; + +template +concept __nothrow_forward_range = + __nothrow_input_range<_Rp> && + __nothrow_forward_iterator>; + +} // namespace ranges +#endif // !defined(_LIBCPP_HAS_NO_RANGES) + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___MEMORY_CONCEPTS_H diff --git a/libcxx/include/__memory/construct_at.h b/libcxx/include/__memory/construct_at.h index 789677d7a613..3b58451c5009 100644 --- a/libcxx/include/__memory/construct_at.h +++ b/libcxx/include/__memory/construct_at.h @@ -41,44 +41,67 @@ constexpr _Tp* construct_at(_Tp* __location, _Args&& ...__args) { // destroy_at -#if _LIBCPP_STD_VER > 14 +// The internal functions are available regardless of the language version (with the exception of the `__destroy_at` +// taking an array). template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 -void destroy(_ForwardIterator, _ForwardIterator); +void __destroy(_ForwardIterator, _ForwardIterator); + +template ::value, int>::type = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 +void __destroy_at(_Tp* __loc) { + _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at"); + __loc->~_Tp(); +} + +#if _LIBCPP_STD_VER > 17 +template ::value, int>::type = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 +void __destroy_at(_Tp* __loc) { + _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at"); + _VSTD::__destroy(_VSTD::begin(*__loc), _VSTD::end(*__loc)); +} +#endif + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 +void __destroy(_ForwardIterator __first, _ForwardIterator __last) { + for (; __first != __last; ++__first) + _VSTD::__destroy_at(_VSTD::addressof(*__first)); +} + +#if _LIBCPP_STD_VER > 14 template , int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 void destroy_at(_Tp* __loc) { - _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at"); - __loc->~_Tp(); + _VSTD::__destroy_at(__loc); } #if _LIBCPP_STD_VER > 17 template , int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 void destroy_at(_Tp* __loc) { - _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at"); - _VSTD::destroy(_VSTD::begin(*__loc), _VSTD::end(*__loc)); + _VSTD::__destroy_at(__loc); } #endif template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 void destroy(_ForwardIterator __first, _ForwardIterator __last) { - for (; __first != __last; ++__first) - _VSTD::destroy_at(_VSTD::addressof(*__first)); + _VSTD::__destroy(_VSTD::move(__first), _VSTD::move(__last)); } template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _ForwardIterator destroy_n(_ForwardIterator __first, _Size __n) { for (; __n > 0; (void)++__first, --__n) - _VSTD::destroy_at(_VSTD::addressof(*__first)); + _VSTD::__destroy_at(_VSTD::addressof(*__first)); return __first; } -#endif +#endif // _LIBCPP_STD_VER > 14 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__memory/ranges_uninitialized_algorithms.h b/libcxx/include/__memory/ranges_uninitialized_algorithms.h new file mode 100644 index 000000000000..6ec803806c05 --- /dev/null +++ b/libcxx/include/__memory/ranges_uninitialized_algorithms.h @@ -0,0 +1,212 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H +#define _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H + +#include <__concepts/constructible.h> +#include <__config> +#include <__function_like.h> +#include <__iterator/incrementable_traits.h> +#include <__iterator/iterator_traits.h> +#include <__iterator/readable_traits.h> +#include <__memory/concepts.h> +#include <__memory/uninitialized_algorithms.h> +#include <__ranges/access.h> +#include <__ranges/concepts.h> +#include <__ranges/dangling.h> +#include <__utility/move.h> +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +#pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if !defined(_LIBCPP_HAS_NO_RANGES) +namespace ranges { + +// uninitialized_default_construct + +namespace __uninitialized_default_construct { + +struct __fn final : private __function_like { + + constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {} + + template <__nothrow_forward_iterator _ForwardIterator, + __nothrow_sentinel_for<_ForwardIterator> _Sentinel> + requires default_initializable> + _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last) const { + using _ValueType = remove_reference_t>; + return _VSTD::__uninitialized_default_construct<_ValueType>( + _VSTD::move(__first), _VSTD::move(__last)); + } + + template <__nothrow_forward_range _ForwardRange> + requires default_initializable> + borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range) const { + return (*this)(ranges::begin(__range), ranges::end(__range)); + } + +}; + +} // namespace __uninitialized_default_construct + +inline namespace __cpo { +inline constexpr auto uninitialized_default_construct = + __uninitialized_default_construct::__fn(__function_like::__tag()); +} // namespace __cpo + +// uninitialized_default_construct_n + +namespace __uninitialized_default_construct_n { + +struct __fn final : private __function_like { + + constexpr explicit __fn(__tag __x) noexcept : + __function_like(__x) {} + + template <__nothrow_forward_iterator _ForwardIterator> + requires default_initializable> + _ForwardIterator operator()(_ForwardIterator __first, + iter_difference_t<_ForwardIterator> __n) const { + using _ValueType = remove_reference_t>; + return _VSTD::__uninitialized_default_construct_n<_ValueType>(_VSTD::move(__first), __n); + } + +}; + +} // namespace __uninitialized_default_construct_n + +inline namespace __cpo { +inline constexpr auto uninitialized_default_construct_n = + __uninitialized_default_construct_n::__fn(__function_like::__tag()); +} // namespace __cpo + +// uninitialized_value_construct + +namespace __uninitialized_value_construct { + +struct __fn final : private __function_like { + + constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {} + + template <__nothrow_forward_iterator _ForwardIterator, + __nothrow_sentinel_for<_ForwardIterator> _Sentinel> + requires default_initializable> + _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last) const { + using _ValueType = remove_reference_t>; + return _VSTD::__uninitialized_value_construct<_ValueType>( + _VSTD::move(__first), _VSTD::move(__last)); + } + + template <__nothrow_forward_range _ForwardRange> + requires default_initializable> + borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range) const { + return (*this)(ranges::begin(__range), ranges::end(__range)); + } + +}; + +} // namespace __uninitialized_value_construct + +inline namespace __cpo { +inline constexpr auto uninitialized_value_construct = + __uninitialized_value_construct::__fn(__function_like::__tag()); +} // namespace __cpo + +// uninitialized_value_construct_n + +namespace __uninitialized_value_construct_n { + +struct __fn final : private __function_like { + + constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {} + + template <__nothrow_forward_iterator _ForwardIterator> + requires default_initializable> + _ForwardIterator operator()(_ForwardIterator __first, + iter_difference_t<_ForwardIterator> __n) const { + using _ValueType = remove_reference_t>; + return _VSTD::__uninitialized_value_construct_n<_ValueType>(_VSTD::move(__first), __n); + } + +}; + +} // namespace __uninitialized_value_construct_n + +inline namespace __cpo { +inline constexpr auto uninitialized_value_construct_n = + __uninitialized_value_construct_n::__fn(__function_like::__tag()); +} // namespace __cpo + +// uninitialized_fill + +namespace __uninitialized_fill { + +struct __fn final : private __function_like { + + constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {} + + template <__nothrow_forward_iterator _ForwardIterator, + __nothrow_sentinel_for<_ForwardIterator> _Sentinel, + class _Tp> + requires constructible_from, const _Tp&> + _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) const { + using _ValueType = remove_reference_t>; + return _VSTD::__uninitialized_fill<_ValueType>(_VSTD::move(__first), _VSTD::move(__last), __x); + } + + template <__nothrow_forward_range _ForwardRange, class _Tp> + requires constructible_from, const _Tp&> + borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range, const _Tp& __x) const { + return (*this)(ranges::begin(__range), ranges::end(__range), __x); + } + +}; + +} // namespace __uninitialized_fil + +inline namespace __cpo { +inline constexpr auto uninitialized_fill = __uninitialized_fill::__fn(__function_like::__tag()); +} // namespace __cpo + +// uninitialized_fill_n + +namespace __uninitialized_fill_n { + +struct __fn final : private __function_like { + + constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {} + + template <__nothrow_forward_iterator _ForwardIterator, class _Tp> + requires constructible_from, const _Tp&> + _ForwardIterator operator()(_ForwardIterator __first, + iter_difference_t<_ForwardIterator> __n, + const _Tp& __x) const { + using _ValueType = remove_reference_t>; + return _VSTD::__uninitialized_fill_n<_ValueType>(_VSTD::move(__first), __n, __x); + } + +}; + +} // namespace __uninitialized_fill_n + +inline namespace __cpo { +inline constexpr auto uninitialized_fill_n = __uninitialized_fill_n::__fn(__function_like::__tag()); +} // namespace __cpo + +} // namespace ranges +#endif // !defined(_LIBCPP_HAS_NO_RANGES) + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h index e83d62e0db08..69132633a48e 100644 --- a/libcxx/include/__memory/uninitialized_algorithms.h +++ b/libcxx/include/__memory/uninitialized_algorithms.h @@ -13,6 +13,7 @@ #include <__config> #include <__memory/addressof.h> #include <__memory/construct_at.h> +#include <__memory/voidify.h> #include #include @@ -70,130 +71,187 @@ uninitialized_copy_n(_InputIterator __f, _Size __n, _ForwardIterator __r) return __r; } -template -void -uninitialized_fill(_ForwardIterator __f, _ForwardIterator __l, const _Tp& __x) +// uninitialized_fill + +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator __uninitialized_fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) { - typedef typename iterator_traits<_ForwardIterator>::value_type value_type; + _ForwardIterator __idx = __first; #ifndef _LIBCPP_NO_EXCEPTIONS - _ForwardIterator __s = __f; try { #endif - for (; __f != __l; ++__f) - ::new ((void*)_VSTD::addressof(*__f)) value_type(__x); + for (; __idx != __last; ++__idx) + ::new (_VSTD::__voidify(*__idx)) _ValueType(__x); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - for (; __s != __f; ++__s) - __s->~value_type(); + _VSTD::__destroy(__first, __idx); throw; } #endif + + return __idx; +} + +template +inline _LIBCPP_HIDE_FROM_ABI +void uninitialized_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __x) +{ + typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType; + (void)_VSTD::__uninitialized_fill<_ValueType>(__first, __last, __x); +} + +// uninitialized_fill_n + +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator __uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) +{ + _ForwardIterator __idx = __first; +#ifndef _LIBCPP_NO_EXCEPTIONS + try + { +#endif + for (; __n > 0; ++__idx, (void) --__n) + ::new (_VSTD::__voidify(*__idx)) _ValueType(__x); +#ifndef _LIBCPP_NO_EXCEPTIONS + } + catch (...) + { + _VSTD::__destroy(__first, __idx); + throw; + } +#endif + + return __idx; } template -_ForwardIterator -uninitialized_fill_n(_ForwardIterator __f, _Size __n, const _Tp& __x) +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) { - typedef typename iterator_traits<_ForwardIterator>::value_type value_type; -#ifndef _LIBCPP_NO_EXCEPTIONS - _ForwardIterator __s = __f; - try - { -#endif - for (; __n > 0; ++__f, (void) --__n) - ::new ((void*)_VSTD::addressof(*__f)) value_type(__x); -#ifndef _LIBCPP_NO_EXCEPTIONS - } - catch (...) - { - for (; __s != __f; ++__s) - __s->~value_type(); - throw; - } -#endif - return __f; + typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType; + return _VSTD::__uninitialized_fill_n<_ValueType>(__first, __n, __x); } #if _LIBCPP_STD_VER > 14 +// uninitialized_default_construct + +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator __uninitialized_default_construct(_ForwardIterator __first, _Sentinel __last) { + auto __idx = __first; +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif + for (; __idx != __last; ++__idx) + ::new (_VSTD::__voidify(*__idx)) _ValueType; +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + _VSTD::__destroy(__first, __idx); + throw; + } +#endif + + return __idx; +} + template -inline _LIBCPP_INLINE_VISIBILITY +inline _LIBCPP_HIDE_FROM_ABI void uninitialized_default_construct(_ForwardIterator __first, _ForwardIterator __last) { - using _Vt = typename iterator_traits<_ForwardIterator>::value_type; - auto __idx = __first; -#ifndef _LIBCPP_NO_EXCEPTIONS - try { -#endif - for (; __idx != __last; ++__idx) - ::new ((void*)_VSTD::addressof(*__idx)) _Vt; -#ifndef _LIBCPP_NO_EXCEPTIONS - } catch (...) { - _VSTD::destroy(__first, __idx); - throw; - } -#endif + using _ValueType = typename iterator_traits<_ForwardIterator>::value_type; + (void)_VSTD::__uninitialized_default_construct<_ValueType>( + _VSTD::move(__first), _VSTD::move(__last)); } -template -inline _LIBCPP_INLINE_VISIBILITY -_ForwardIterator uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) { - using _Vt = typename iterator_traits<_ForwardIterator>::value_type; +// uninitialized_default_construct_n + +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator __uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) { auto __idx = __first; #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif for (; __n > 0; ++__idx, (void) --__n) - ::new ((void*)_VSTD::addressof(*__idx)) _Vt; - return __idx; + ::new (_VSTD::__voidify(*__idx)) _ValueType; #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - _VSTD::destroy(__first, __idx); + _VSTD::__destroy(__first, __idx); throw; } #endif + + return __idx; } +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) { + using _ValueType = typename iterator_traits<_ForwardIterator>::value_type; + return _VSTD::__uninitialized_default_construct_n<_ValueType>(_VSTD::move(__first), __n); +} + +// uninitialized_value_construct + +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator __uninitialized_value_construct(_ForwardIterator __first, _Sentinel __last) { + auto __idx = __first; +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif + for (; __idx != __last; ++__idx) + ::new (_VSTD::__voidify(*__idx)) _ValueType(); +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + _VSTD::__destroy(__first, __idx); + throw; + } +#endif + + return __idx; +} template -inline _LIBCPP_INLINE_VISIBILITY +inline _LIBCPP_HIDE_FROM_ABI void uninitialized_value_construct(_ForwardIterator __first, _ForwardIterator __last) { - using _Vt = typename iterator_traits<_ForwardIterator>::value_type; - auto __idx = __first; -#ifndef _LIBCPP_NO_EXCEPTIONS - try { -#endif - for (; __idx != __last; ++__idx) - ::new ((void*)_VSTD::addressof(*__idx)) _Vt(); -#ifndef _LIBCPP_NO_EXCEPTIONS - } catch (...) { - _VSTD::destroy(__first, __idx); - throw; - } -#endif + using _ValueType = typename iterator_traits<_ForwardIterator>::value_type; + (void)_VSTD::__uninitialized_value_construct<_ValueType>( + _VSTD::move(__first), _VSTD::move(__last)); } -template -inline _LIBCPP_INLINE_VISIBILITY -_ForwardIterator uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) { - using _Vt = typename iterator_traits<_ForwardIterator>::value_type; +// uninitialized_value_construct_n + +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator __uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) { auto __idx = __first; #ifndef _LIBCPP_NO_EXCEPTIONS try { #endif for (; __n > 0; ++__idx, (void) --__n) - ::new ((void*)_VSTD::addressof(*__idx)) _Vt(); - return __idx; + ::new (_VSTD::__voidify(*__idx)) _ValueType(); #ifndef _LIBCPP_NO_EXCEPTIONS } catch (...) { - _VSTD::destroy(__first, __idx); + _VSTD::__destroy(__first, __idx); throw; } #endif + + return __idx; } +template +inline _LIBCPP_HIDE_FROM_ABI +_ForwardIterator uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) { + using _ValueType = typename iterator_traits<_ForwardIterator>::value_type; + return __uninitialized_value_construct_n<_ValueType>(_VSTD::move(__first), __n); +} template inline _LIBCPP_INLINE_VISIBILITY diff --git a/libcxx/include/__memory/voidify.h b/libcxx/include/__memory/voidify.h new file mode 100644 index 000000000000..3a65c0e83fb7 --- /dev/null +++ b/libcxx/include/__memory/voidify.h @@ -0,0 +1,30 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_VOIDIFY_H +#define _LIBCPP___MEMORY_VOIDIFY_H + +#include <__config> +#include <__memory/addressof.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template +_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 void* __voidify(_Tp& __from) { + // Cast away cv-qualifiers to allow modifying elements of a range through const iterators. + return const_cast(static_cast(_VSTD::addressof(__from))); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___MEMORY_VOIDIFY_H diff --git a/libcxx/include/__mutex_base b/libcxx/include/__mutex_base index da2967164a68..21e2075603b5 100644 --- a/libcxx/include/__mutex_base +++ b/libcxx/include/__mutex_base @@ -140,11 +140,9 @@ public: __m_->unlock(); } -private: - unique_lock(unique_lock const&); // = delete; - unique_lock& operator=(unique_lock const&); // = delete; + unique_lock(unique_lock const&) = delete; + unique_lock& operator=(unique_lock const&) = delete; -public: _LIBCPP_INLINE_VISIBILITY unique_lock(unique_lock&& __u) _NOEXCEPT : __m_(__u.__m_), __owns_(__u.__owns_) diff --git a/libcxx/include/__nullptr b/libcxx/include/__nullptr index d02be215ef1d..c6645cd0150e 100644 --- a/libcxx/include/__nullptr +++ b/libcxx/include/__nullptr @@ -54,7 +54,7 @@ _LIBCPP_END_NAMESPACE_STD namespace std { typedef decltype(nullptr) nullptr_t; -} +} // namespace std #endif // _LIBCPP_HAS_NO_NULLPTR diff --git a/libcxx/include/__random/clamp_to_integral.h b/libcxx/include/__random/clamp_to_integral.h new file mode 100644 index 000000000000..dd5d2b0186e4 --- /dev/null +++ b/libcxx/include/__random/clamp_to_integral.h @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H +#define _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H + +#include <__config> +#include +#include +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +#pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +template ::digits > numeric_limits<_IntT>::digits), + int _Bits = (numeric_limits<_IntT>::digits - numeric_limits<_FloatT>::digits)> +_LIBCPP_INLINE_VISIBILITY +_LIBCPP_CONSTEXPR _IntT __max_representable_int_for_float() _NOEXCEPT { + static_assert(is_floating_point<_FloatT>::value, "must be a floating point type"); + static_assert(is_integral<_IntT>::value, "must be an integral type"); + static_assert(numeric_limits<_FloatT>::radix == 2, "FloatT has incorrect radix"); + static_assert((_IsSame<_FloatT, float>::value || _IsSame<_FloatT, double>::value + || _IsSame<_FloatT,long double>::value), "unsupported floating point type"); + return _FloatBigger ? numeric_limits<_IntT>::max() : (numeric_limits<_IntT>::max() >> _Bits << _Bits); +} + +// Convert a floating point number to the specified integral type after +// clamping to the integral type's representable range. +// +// The behavior is undefined if `__r` is NaN. +template +_LIBCPP_INLINE_VISIBILITY +_IntT __clamp_to_integral(_RealT __r) _NOEXCEPT { + using _Lim = numeric_limits<_IntT>; + const _IntT _MaxVal = __max_representable_int_for_float<_IntT, _RealT>(); + if (__r >= ::nextafter(static_cast<_RealT>(_MaxVal), INFINITY)) { + return _Lim::max(); + } else if (__r <= _Lim::lowest()) { + return _Lim::min(); + } + return static_cast<_IntT>(__r); +} + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H diff --git a/libcxx/include/__random/poisson_distribution.h b/libcxx/include/__random/poisson_distribution.h index fb213b0103ad..d157e8f230ef 100644 --- a/libcxx/include/__random/poisson_distribution.h +++ b/libcxx/include/__random/poisson_distribution.h @@ -10,6 +10,7 @@ #define _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H #include <__config> +#include <__random/clamp_to_integral.h> #include <__random/exponential_distribution.h> #include <__random/normal_distribution.h> #include <__random/uniform_real_distribution.h> diff --git a/libcxx/include/__random/random_device.h b/libcxx/include/__random/random_device.h index f62f7a3d269b..835f726fdbcc 100644 --- a/libcxx/include/__random/random_device.h +++ b/libcxx/include/__random/random_device.h @@ -27,7 +27,7 @@ class _LIBCPP_TYPE_VIS random_device { #ifdef _LIBCPP_USING_DEV_RANDOM int __f_; -#endif // defined(_LIBCPP_USING_DEV_RANDOM) +#endif public: // types typedef unsigned result_type; @@ -56,10 +56,8 @@ class _LIBCPP_TYPE_VIS random_device // property functions double entropy() const _NOEXCEPT; -private: - // no copy functions - random_device(const random_device&); // = delete; - random_device& operator=(const random_device&); // = delete; + random_device(const random_device&) = delete; + void operator=(const random_device&) = delete; }; #endif // !_LIBCPP_HAS_NO_RANDOM_DEVICE diff --git a/libcxx/include/__random/seed_seq.h b/libcxx/include/__random/seed_seq.h index 97bc88d0d4d1..bf27af6627a5 100644 --- a/libcxx/include/__random/seed_seq.h +++ b/libcxx/include/__random/seed_seq.h @@ -63,10 +63,8 @@ class _LIBCPP_TEMPLATE_VIS seed_seq void param(_OutputIterator __dest) const {_VSTD::copy(__v_.begin(), __v_.end(), __dest);} -private: - // no copy functions - seed_seq(const seed_seq&); // = delete; - void operator=(const seed_seq&); // = delete; + seed_seq(const seed_seq&) = delete; + void operator=(const seed_seq&) = delete; _LIBCPP_INLINE_VISIBILITY static result_type _Tp(result_type __x) {return __x ^ (__x >> 27);} diff --git a/libcxx/include/__ranges/access.h b/libcxx/include/__ranges/access.h index b0b89c0eeea3..91dc3055c86d 100644 --- a/libcxx/include/__ranges/access.h +++ b/libcxx/include/__ranges/access.h @@ -14,8 +14,7 @@ #include <__iterator/readable_traits.h> #include <__ranges/enable_borrowed_range.h> #include <__utility/as_const.h> -#include <__utility/decay_copy.h> -#include <__utility/forward.h> +#include <__utility/auto_cast.h> #include #include @@ -27,24 +26,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_RANGES) -// clang-format off - namespace ranges { template concept __can_borrow = - is_lvalue_reference_v<_Tp> || enable_borrowed_range >; - - template - concept __is_complete = requires { sizeof(_Tp); }; + is_lvalue_reference_v<_Tp> || enable_borrowed_range>; } // namespace ranges // [range.access.begin] -namespace ranges::__begin { + +namespace ranges { +namespace __begin { template concept __member_begin = __can_borrow<_Tp> && requires(_Tp&& __t) { - { _VSTD::__decay_copy(__t.begin()) } -> input_or_output_iterator; + { _LIBCPP_AUTO_CAST(__t.begin()) } -> input_or_output_iterator; }; void begin(auto&) = delete; @@ -54,61 +50,61 @@ namespace ranges::__begin { concept __unqualified_begin = !__member_begin<_Tp> && __can_borrow<_Tp> && - __class_or_enum > && + __class_or_enum> && requires(_Tp && __t) { - { _VSTD::__decay_copy(begin(__t)) } -> input_or_output_iterator; + { _LIBCPP_AUTO_CAST(begin(__t)) } -> input_or_output_iterator; }; struct __fn { template - requires is_array_v> - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const noexcept { - constexpr bool __complete = __is_complete >; - if constexpr (__complete) { // used to disable cryptic diagnostic - return __t + 0; - } - else { - static_assert(__complete, "`std::ranges::begin` is SFINAE-unfriendly on arrays of an incomplete type."); - } + requires is_array_v> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const noexcept + { + return __t; } template - requires __member_begin<_Tp> + requires __member_begin<_Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(_VSTD::__decay_copy(__t.begin()))) + noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.begin()))) { - return __t.begin(); + return _LIBCPP_AUTO_CAST(__t.begin()); } template - requires __unqualified_begin<_Tp> + requires __unqualified_begin<_Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(_VSTD::__decay_copy(begin(__t)))) + noexcept(noexcept(_LIBCPP_AUTO_CAST(begin(__t)))) { - return begin(__t); + return _LIBCPP_AUTO_CAST(begin(__t)); } void operator()(auto&&) const = delete; }; -} // namespace ranges::__begin +} + +inline namespace __cpo { + inline constexpr auto begin = __begin::__fn{}; +} // namespace __cpo +} // namespace ranges + +// [range.range] namespace ranges { - inline namespace __cpo { - inline constexpr auto begin = __begin::__fn{}; - } // namespace __cpo - template using iterator_t = decltype(ranges::begin(declval<_Tp&>())); } // namespace ranges // [range.access.end] -namespace ranges::__end { + +namespace ranges { +namespace __end { template concept __member_end = __can_borrow<_Tp> && requires(_Tp&& __t) { typename iterator_t<_Tp>; - { _VSTD::__decay_copy(_VSTD::forward<_Tp>(__t).end()) } -> sentinel_for >; + { _LIBCPP_AUTO_CAST(__t.end()) } -> sentinel_for>; }; void end(auto&) = delete; @@ -118,98 +114,101 @@ namespace ranges::__end { concept __unqualified_end = !__member_end<_Tp> && __can_borrow<_Tp> && - __class_or_enum > && + __class_or_enum> && requires(_Tp && __t) { typename iterator_t<_Tp>; - { _VSTD::__decay_copy(end(_VSTD::forward<_Tp>(__t))) } -> sentinel_for >; + { _LIBCPP_AUTO_CAST(end(__t)) } -> sentinel_for>; }; class __fn { public: template - [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[_Np]) const noexcept { - constexpr bool __complete = __is_complete >; - if constexpr (__complete) { // used to disable cryptic diagnostic - return __t + _Np; - } - else { - static_assert(__complete, "`std::ranges::end` is SFINAE-unfriendly on arrays of an incomplete type."); - } + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[_Np]) const noexcept + requires (sizeof(*__t) != 0) // Disallow incomplete element types. + { + return __t + _Np; } template - requires __member_end<_Tp> + requires __member_end<_Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(_VSTD::__decay_copy(__t.end()))) + noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.end()))) { - return _VSTD::forward<_Tp>(__t).end(); + return _LIBCPP_AUTO_CAST(__t.end()); } template - requires __unqualified_end<_Tp> + requires __unqualified_end<_Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(_VSTD::__decay_copy(end(__t)))) + noexcept(noexcept(_LIBCPP_AUTO_CAST(end(__t)))) { - return end(__t); + return _LIBCPP_AUTO_CAST(end(__t)); } void operator()(auto&&) const = delete; }; -} // namespace ranges::__end +} -namespace ranges::inline __cpo { +inline namespace __cpo { inline constexpr auto end = __end::__fn{}; -} // namespace ranges::__cpo +} // namespace __cpo +} // namespace ranges -namespace ranges::__cbegin { +// [range.access.cbegin] + +namespace ranges { +namespace __cbegin { struct __fn { template - requires invocable + requires invocable [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const - noexcept(noexcept(ranges::begin(_VSTD::as_const(__t)))) + noexcept(noexcept(ranges::begin(_VSTD::as_const(__t)))) { return ranges::begin(_VSTD::as_const(__t)); } template - requires is_rvalue_reference_v<_Tp> && invocable + requires is_rvalue_reference_v<_Tp> && invocable [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(ranges::begin(static_cast<_Tp const&&>(__t)))) + noexcept(noexcept(ranges::begin(static_cast<_Tp const&&>(__t)))) { return ranges::begin(static_cast<_Tp const&&>(__t)); } }; -} // namespace ranges::__cbegin +} -namespace ranges::inline __cpo { +inline namespace __cpo { inline constexpr auto cbegin = __cbegin::__fn{}; -} // namespace ranges::__cpo +} // namespace __cpo +} // namespace ranges -namespace ranges::__cend { +// [range.access.cend] + +namespace ranges { +namespace __cend { struct __fn { template - requires invocable + requires invocable [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const - noexcept(noexcept(ranges::end(_VSTD::as_const(__t)))) + noexcept(noexcept(ranges::end(_VSTD::as_const(__t)))) { return ranges::end(_VSTD::as_const(__t)); } template - requires is_rvalue_reference_v<_Tp> && invocable + requires is_rvalue_reference_v<_Tp> && invocable [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(ranges::end(static_cast<_Tp const&&>(__t)))) + noexcept(noexcept(ranges::end(static_cast<_Tp const&&>(__t)))) { return ranges::end(static_cast<_Tp const&&>(__t)); } }; -} // namespace ranges::__cend +} -namespace ranges::inline __cpo { +inline namespace __cpo { inline constexpr auto cend = __cend::__fn{}; -} // namespace ranges::__cpo - -// clang-format off +} // namespace __cpo +} // namespace ranges #endif // !defined(_LIBCPP_HAS_NO_RANGES) diff --git a/libcxx/include/__ranges/all.h b/libcxx/include/__ranges/all.h index affe13ee0862..ccc77258ba10 100644 --- a/libcxx/include/__ranges/all.h +++ b/libcxx/include/__ranges/all.h @@ -17,7 +17,7 @@ #include <__ranges/range_adaptor.h> #include <__ranges/ref_view.h> #include <__ranges/subrange.h> -#include <__utility/decay_copy.h> +#include <__utility/auto_cast.h> #include <__utility/declval.h> #include <__utility/forward.h> #include @@ -38,9 +38,9 @@ namespace __all { requires ranges::view> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const - noexcept(noexcept(_VSTD::__decay_copy(_VSTD::forward<_Tp>(__t)))) + noexcept(noexcept(_LIBCPP_AUTO_CAST(_VSTD::forward<_Tp>(__t)))) { - return _VSTD::forward<_Tp>(__t); + return _LIBCPP_AUTO_CAST(_VSTD::forward<_Tp>(__t)); } template diff --git a/libcxx/include/__ranges/concepts.h b/libcxx/include/__ranges/concepts.h index 6a8364006beb..bad23c8c4bfb 100644 --- a/libcxx/include/__ranges/concepts.h +++ b/libcxx/include/__ranges/concepts.h @@ -29,8 +29,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD -// clang-format off - #if !defined(_LIBCPP_HAS_NO_RANGES) namespace ranges { @@ -126,8 +124,6 @@ namespace ranges { #endif // !defined(_LIBCPP_HAS_NO_RANGES) -// clang-format on - _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP___RANGES_CONCEPTS_H diff --git a/libcxx/include/__ranges/counted.h b/libcxx/include/__ranges/counted.h index d292bcbb1849..cb9784092420 100644 --- a/libcxx/include/__ranges/counted.h +++ b/libcxx/include/__ranges/counted.h @@ -9,6 +9,7 @@ #ifndef _LIBCPP___RANGES_COUNTED_H #define _LIBCPP___RANGES_COUNTED_H +#include <__concepts/convertible_to.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/counted_iterator.h> @@ -16,10 +17,7 @@ #include <__iterator/incrementable_traits.h> #include <__iterator/iterator_traits.h> #include <__memory/pointer_traits.h> -#include <__ranges/concepts.h> #include <__ranges/subrange.h> -#include <__utility/decay_copy.h> -#include <__utility/declval.h> #include <__utility/forward.h> #include <__utility/move.h> #include @@ -36,50 +34,39 @@ _LIBCPP_BEGIN_NAMESPACE_STD namespace ranges::views { namespace __counted { - template - concept __explicitly_convertible = requires { - _To(_From{}); - }; struct __fn { - template - requires contiguous_iterator> && - __explicitly_convertible<_Diff, iter_difference_t<_Iter>> + template _LIBCPP_HIDE_FROM_ABI - constexpr auto operator()(_Iter&& __it, _Diff __c) const - noexcept(noexcept( - span(_VSTD::to_address(__it), static_cast>(__c)) - )) - { - return span(_VSTD::to_address(__it), static_cast>(__c)); - } + static constexpr auto __go(_It __it, iter_difference_t<_It> __count) + noexcept(noexcept(span(_VSTD::to_address(__it), static_cast(__count)))) + // Deliberately omit return-type SFINAE, because to_address is not SFINAE-friendly + { return span(_VSTD::to_address(__it), static_cast(__count)); } - template - requires random_access_iterator> && - __explicitly_convertible<_Diff, iter_difference_t<_Iter>> + template _LIBCPP_HIDE_FROM_ABI - constexpr auto operator()(_Iter&& __it, _Diff __c) const - noexcept( - noexcept(__it + static_cast>(__c)) && - noexcept(ranges::subrange(_VSTD::forward<_Iter>(__it), _VSTD::__decay_copy(__it))) - ) - { - auto __last = __it + static_cast>(__c); - return ranges::subrange(_VSTD::forward<_Iter>(__it), _VSTD::move(__last)); - } + static constexpr auto __go(_It __it, iter_difference_t<_It> __count) + noexcept(noexcept(subrange(__it, __it + __count))) + -> decltype( subrange(__it, __it + __count)) + { return subrange(__it, __it + __count); } - template - requires __explicitly_convertible<_Diff, iter_difference_t<_Iter>> + template _LIBCPP_HIDE_FROM_ABI - constexpr auto operator()(_Iter&& __it, _Diff __c) const - noexcept(noexcept( - ranges::subrange(counted_iterator(_VSTD::forward<_Iter>(__it), __c), default_sentinel) - )) - { - return ranges::subrange(counted_iterator(_VSTD::forward<_Iter>(__it), __c), default_sentinel); - } + static constexpr auto __go(_It __it, iter_difference_t<_It> __count) + noexcept(noexcept(subrange(counted_iterator(_VSTD::move(__it), __count), default_sentinel))) + -> decltype( subrange(counted_iterator(_VSTD::move(__it), __count), default_sentinel)) + { return subrange(counted_iterator(_VSTD::move(__it), __count), default_sentinel); } + + template> _Diff> + requires input_or_output_iterator> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI + constexpr auto operator()(_It&& __it, _Diff&& __count) const + noexcept(noexcept(__go(_VSTD::forward<_It>(__it), _VSTD::forward<_Diff>(__count)))) + -> decltype( __go(_VSTD::forward<_It>(__it), _VSTD::forward<_Diff>(__count))) + { return __go(_VSTD::forward<_It>(__it), _VSTD::forward<_Diff>(__count)); } }; -} + +} // namespace __counted inline namespace __cpo { inline constexpr auto counted = __counted::__fn{}; diff --git a/libcxx/include/__ranges/data.h b/libcxx/include/__ranges/data.h index 7eade494cceb..cc151c59f3d7 100644 --- a/libcxx/include/__ranges/data.h +++ b/libcxx/include/__ranges/data.h @@ -26,9 +26,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_RANGES) -// clang-format off -namespace ranges { // [range.prim.data] + +namespace ranges { namespace __data { template concept __ptr_to_object = is_pointer_v<_Tp> && is_object_v>; @@ -65,15 +65,13 @@ namespace __data { return _VSTD::to_address(ranges::begin(_VSTD::forward<_Tp>(__t))); } }; -} // end namespace __data +} inline namespace __cpo { inline constexpr auto data = __data::__fn{}; } // namespace __cpo } // namespace ranges -// clang-format off - #endif // !defined(_LIBCPP_HAS_NO_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/empty.h b/libcxx/include/__ranges/empty.h index fc6a938fd86e..e8a8aabf4aed 100644 --- a/libcxx/include/__ranges/empty.h +++ b/libcxx/include/__ranges/empty.h @@ -13,7 +13,6 @@ #include <__iterator/concepts.h> #include <__ranges/access.h> #include <__ranges/size.h> -#include <__utility/forward.h> #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -24,19 +23,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_RANGES) -// clang-format off -namespace ranges { // [range.prim.empty] + +namespace ranges { namespace __empty { template concept __member_empty = requires(_Tp&& __t) { - bool(_VSTD::forward<_Tp>(__t).empty()); + bool(__t.empty()); }; template concept __can_invoke_size = !__member_empty<_Tp> && - requires(_Tp&& __t) { ranges::size(_VSTD::forward<_Tp>(__t)); }; + requires(_Tp&& __t) { ranges::size(__t); }; template concept __can_compare_begin_end = @@ -51,13 +50,13 @@ namespace __empty { template <__member_empty _Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const noexcept(noexcept(bool(__t.empty()))) { - return __t.empty(); + return bool(__t.empty()); } template <__can_invoke_size _Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const - noexcept(noexcept(ranges::size(_VSTD::forward<_Tp>(__t)))) { - return ranges::size(_VSTD::forward<_Tp>(__t)) == 0; + noexcept(noexcept(ranges::size(__t))) { + return ranges::size(__t) == 0; } template<__can_compare_begin_end _Tp> @@ -72,7 +71,6 @@ inline namespace __cpo { inline constexpr auto empty = __empty::__fn{}; } // namespace __cpo } // namespace ranges -// clang-format off #endif // !defined(_LIBCPP_HAS_NO_RANGES) diff --git a/libcxx/include/__ranges/size.h b/libcxx/include/__ranges/size.h index af0a8479f2ec..fc6641cf4887 100644 --- a/libcxx/include/__ranges/size.h +++ b/libcxx/include/__ranges/size.h @@ -13,8 +13,7 @@ #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__ranges/access.h> -#include <__utility/decay_copy.h> -#include <__utility/forward.h> +#include <__utility/auto_cast.h> #include #include @@ -26,12 +25,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if !defined(_LIBCPP_HAS_NO_RANGES) -// clang-format off namespace ranges { -template -inline constexpr bool disable_sized_range = false; + template + inline constexpr bool disable_sized_range = false; +} // [range.prim.size] + +namespace ranges { namespace __size { void size(auto&) = delete; void size(const auto&) = delete; @@ -41,7 +42,7 @@ namespace __size { template concept __member_size = __size_enabled<_Tp> && requires(_Tp&& __t) { - { _VSTD::__decay_copy(_VSTD::forward<_Tp>(__t).size()) } -> __integer_like; + { _LIBCPP_AUTO_CAST(__t.size()) } -> __integer_like; }; template @@ -50,7 +51,7 @@ namespace __size { !__member_size<_Tp> && __class_or_enum> && requires(_Tp&& __t) { - { _VSTD::__decay_copy(size(_VSTD::forward<_Tp>(__t))) } -> __integer_like; + { _LIBCPP_AUTO_CAST(size(__t)) } -> __integer_like; }; template @@ -76,14 +77,14 @@ namespace __size { template <__member_size _Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const - noexcept(noexcept(_VSTD::forward<_Tp>(__t).size())) { - return _VSTD::forward<_Tp>(__t).size(); + noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.size()))) { + return _LIBCPP_AUTO_CAST(__t.size()); } template <__unqualified_size _Tp> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const - noexcept(noexcept(size(_VSTD::forward<_Tp>(__t)))) { - return size(_VSTD::forward<_Tp>(__t)); + noexcept(noexcept(_LIBCPP_AUTO_CAST(size(__t)))) { + return _LIBCPP_AUTO_CAST(size(__t)); } template<__difference _Tp> @@ -92,18 +93,23 @@ namespace __size { return _VSTD::__to_unsigned_like(ranges::end(__t) - ranges::begin(__t)); } }; -} // end namespace __size +} inline namespace __cpo { inline constexpr auto size = __size::__fn{}; } // namespace __cpo +} // namespace ranges +// [range.prim.ssize] + +namespace ranges { namespace __ssize { struct __fn { template requires requires (_Tp&& __t) { ranges::size(__t); } [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr integral auto operator()(_Tp&& __t) const - noexcept(noexcept(ranges::size(__t))) { + noexcept(noexcept(ranges::size(__t))) + { using _Signed = make_signed_t; if constexpr (sizeof(ptrdiff_t) > sizeof(_Signed)) return static_cast(ranges::size(__t)); @@ -118,8 +124,6 @@ inline namespace __cpo { } // namespace __cpo } // namespace ranges -// clang-format off - #endif // !defined(_LIBCPP_HAS_NO_RANGES) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__ranges/subrange.h b/libcxx/include/__ranges/subrange.h index af4e27600625..8e984f2bf06c 100644 --- a/libcxx/include/__ranges/subrange.h +++ b/libcxx/include/__ranges/subrange.h @@ -91,14 +91,14 @@ namespace ranges { _LIBCPP_HIDE_FROM_ABI constexpr subrange(__convertible_to_non_slicing<_Iter> auto __iter, _Sent __sent) requires _MustProvideSizeAtConstruction - : __begin_(_VSTD::move(__iter)), __end_(std::move(__sent)) + : __begin_(_VSTD::move(__iter)), __end_(_VSTD::move(__sent)) { } _LIBCPP_HIDE_FROM_ABI constexpr subrange(__convertible_to_non_slicing<_Iter> auto __iter, _Sent __sent, make_unsigned_t> __n) requires (_Kind == subrange_kind::sized) - : __begin_(_VSTD::move(__iter)), __end_(std::move(__sent)), __size_(__n) + : __begin_(_VSTD::move(__iter)), __end_(_VSTD::move(__sent)), __size_(__n) { if constexpr (sized_sentinel_for<_Sent, _Iter>) _LIBCPP_ASSERT((__end_ - __begin_) == static_cast>(__n), diff --git a/libcxx/include/__ranges/transform_view.h b/libcxx/include/__ranges/transform_view.h index 208a9a22694a..0f53fbaa7e68 100644 --- a/libcxx/include/__ranges/transform_view.h +++ b/libcxx/include/__ranges/transform_view.h @@ -46,11 +46,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD namespace ranges { +template +concept __regular_invocable_with_range_ref = + regular_invocable<_Fn, range_reference_t<_View>>; + template concept __transform_view_constraints = - view<_View> && is_object_v<_Fn> && - regular_invocable<_Fn&, range_reference_t<_View>> && - __referenceable>>; + view<_View> && is_object_v<_Fn> && + regular_invocable<_Fn&, range_reference_t<_View>> && + __referenceable>>; template requires __transform_view_constraints<_View, _Fn> @@ -82,7 +86,7 @@ class transform_view : public view_interface> { _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const requires range && - regular_invocable> + __regular_invocable_with_range_ref { return __iterator(*this, ranges::begin(__base_)); } @@ -100,14 +104,14 @@ class transform_view : public view_interface> { _LIBCPP_HIDE_FROM_ABI constexpr __sentinel end() const requires range && - regular_invocable> + __regular_invocable_with_range_ref { return __sentinel(ranges::end(__base_)); } _LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const requires common_range && - regular_invocable> + __regular_invocable_with_range_ref { return __iterator(*this, ranges::end(__base_)); } diff --git a/libcxx/include/__string b/libcxx/include/__string index 890fb21dd3f1..13ff7b35e47d 100644 --- a/libcxx/include/__string +++ b/libcxx/include/__string @@ -42,9 +42,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -// The the extern template ABI lists are kept outside of to improve the -// readability of that header. - // The extern template ABI lists are kept outside of to improve the // readability of that header. We maintain 2 ABI lists: // - _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST @@ -293,31 +290,34 @@ char_traits<_CharT>::assign(char_type* __s, size_t __n, char_type __a) template static inline _LIBCPP_CONSTEXPR_AFTER_CXX17 -_CharT* __move_constexpr(_CharT* __s1, const _CharT* __s2, size_t __n) _NOEXCEPT +_CharT* __copy_constexpr(_CharT* __dest, const _CharT* __source, size_t __n) _NOEXCEPT { - if (__n == 0) return __s1; - if (__s1 < __s2) { - _VSTD::copy(__s2, __s2 + __n, __s1); - } else if (__s2 < __s1) { - _VSTD::copy_backward(__s2, __s2 + __n, __s1 + __n); - } - return __s1; + _LIBCPP_ASSERT(__libcpp_is_constant_evaluated(), "__copy_constexpr() should always be constant evaluated"); + _VSTD::copy_n(__source, __n, __dest); + return __dest; } template static inline _LIBCPP_CONSTEXPR_AFTER_CXX17 -_CharT* __copy_constexpr(_CharT* __s1, const _CharT* __s2, size_t __n) _NOEXCEPT +_CharT* __move_constexpr(_CharT* __dest, const _CharT* __source, size_t __n) _NOEXCEPT { - _VSTD::copy_n(__s2, __n, __s1); - return __s1; + _LIBCPP_ASSERT(__libcpp_is_constant_evaluated(), "__move_constexpr() should always be constant evaluated"); + if (__n == 0) + return __dest; + _CharT* __allocation = new _CharT[__n]; + _VSTD::__copy_constexpr(__allocation, __source, __n); + _VSTD::__copy_constexpr(__dest, static_cast(__allocation), __n); + delete[] __allocation; + return __dest; } template static inline _LIBCPP_CONSTEXPR_AFTER_CXX17 _CharT* __assign_constexpr(_CharT* __s, size_t __n, _CharT __a) _NOEXCEPT { - _VSTD::fill_n(__s, __n, __a); - return __s; + _LIBCPP_ASSERT(__libcpp_is_constant_evaluated(), "__assign_constexpr() should always be constant evaluated"); + _VSTD::fill_n(__s, __n, __a); + return __s; } // char_traits @@ -340,8 +340,21 @@ struct _LIBCPP_TEMPLATE_VIS char_traits static _LIBCPP_CONSTEXPR_AFTER_CXX14 int compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT; - static inline size_t _LIBCPP_CONSTEXPR_AFTER_CXX14 - length(const char_type* __s) _NOEXCEPT {return __builtin_strlen(__s);} + + static inline size_t _LIBCPP_CONSTEXPR_AFTER_CXX14 length(const char_type* __s) _NOEXCEPT { + // GCC currently does not support __builtin_strlen during constant evaluation. + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70816 +#ifdef _LIBCPP_COMPILER_GCC + if (__libcpp_is_constant_evaluated()) { + size_t __i = 0; + for (; __s[__i] != char_type('\0'); ++__i) + ; + return __i; + } +#endif + return __builtin_strlen(__s); + } + static _LIBCPP_CONSTEXPR_AFTER_CXX14 const char_type* find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT; static inline _LIBCPP_CONSTEXPR_AFTER_CXX17 diff --git a/libcxx/include/__threading_support b/libcxx/include/__threading_support index 0094fca2fb38..68f381a62183 100644 --- a/libcxx/include/__threading_support +++ b/libcxx/include/__threading_support @@ -274,7 +274,8 @@ struct __libcpp_timed_backoff_policy { namespace __thread_detail { -inline __libcpp_timespec_t __convert_to_timespec(const chrono::nanoseconds& __ns) +_LIBCPP_HIDE_FROM_ABI inline +__libcpp_timespec_t __convert_to_timespec(const chrono::nanoseconds& __ns) { using namespace chrono; seconds __s = duration_cast(__ns); @@ -296,10 +297,11 @@ inline __libcpp_timespec_t __convert_to_timespec(const chrono::nanoseconds& __ns return __ts; } -} +} // namespace __thread_detail #if defined(_LIBCPP_HAS_THREAD_API_PTHREAD) +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m) { pthread_mutexattr_t attr; @@ -324,74 +326,88 @@ int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m) return 0; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_lock(__m); } +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_trylock(__m) == 0; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_unlock(__m); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t *__m) { return pthread_mutex_destroy(__m); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_lock(__libcpp_mutex_t *__m) { return pthread_mutex_lock(__m); } +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_mutex_trylock(__libcpp_mutex_t *__m) { return pthread_mutex_trylock(__m) == 0; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_unlock(__libcpp_mutex_t *__m) { return pthread_mutex_unlock(__m); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_destroy(__libcpp_mutex_t *__m) { return pthread_mutex_destroy(__m); } // Condition Variable +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_signal(__libcpp_condvar_t *__cv) { return pthread_cond_signal(__cv); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_broadcast(__libcpp_condvar_t *__cv) { return pthread_cond_broadcast(__cv); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_wait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m) { return pthread_cond_wait(__cv, __m); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m, __libcpp_timespec_t *__ts) { return pthread_cond_timedwait(__cv, __m, __ts); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv) { return pthread_cond_destroy(__cv); } // Execute once +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_execute_once(__libcpp_exec_once_flag *flag, void (*init_routine)()) { return pthread_once(flag, init_routine); @@ -399,34 +415,40 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag, // Thread id // Returns non-zero if the thread ids are equal, otherwise 0 +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2) { return t1 == t2; } // Returns non-zero if t1 < t2, otherwise 0 +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2) { return t1 < t2; } // Thread +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) { return __libcpp_thread_get_id(__t) == 0; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *), void *__arg) { return pthread_create(__t, nullptr, __func, __arg); } +_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_current_id() { const __libcpp_thread_t thread = pthread_self(); return __libcpp_thread_get_id(&thread); } +_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t) { #if defined(__MVS__) @@ -436,21 +458,25 @@ __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t) #endif } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_join(__libcpp_thread_t *__t) { return pthread_join(*__t, nullptr); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_detach(__libcpp_thread_t *__t) { return pthread_detach(*__t); } +_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_yield() { sched_yield(); } +_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) { __libcpp_timespec_t __ts = __thread_detail::__convert_to_timespec(__ns); @@ -458,16 +484,19 @@ void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) } // Thread local storage +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_create(__libcpp_tls_key *__key, void (*__at_exit)(void *)) { return pthread_key_create(__key, __at_exit); } +_LIBCPP_HIDE_FROM_ABI inline void *__libcpp_tls_get(__libcpp_tls_key __key) { return pthread_getspecific(__key); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_set(__libcpp_tls_key __key, void *__p) { return pthread_setspecific(__key, __p); @@ -475,47 +504,56 @@ int __libcpp_tls_set(__libcpp_tls_key __key, void *__p) #elif defined(_LIBCPP_HAS_THREAD_API_C11) +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m) { return mtx_init(__m, mtx_plain | mtx_recursive) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t *__m) { return mtx_lock(__m) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t *__m) { return mtx_trylock(__m) == thrd_success; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t *__m) { return mtx_unlock(__m) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t *__m) { mtx_destroy(__m); return 0; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_lock(__libcpp_mutex_t *__m) { return mtx_lock(__m) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_mutex_trylock(__libcpp_mutex_t *__m) { return mtx_trylock(__m) == thrd_success; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_unlock(__libcpp_mutex_t *__m) { return mtx_unlock(__m) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_mutex_destroy(__libcpp_mutex_t *__m) { mtx_destroy(__m); @@ -523,21 +561,25 @@ int __libcpp_mutex_destroy(__libcpp_mutex_t *__m) } // Condition Variable +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_signal(__libcpp_condvar_t *__cv) { return cnd_signal(__cv) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_broadcast(__libcpp_condvar_t *__cv) { return cnd_broadcast(__cv) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_wait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m) { return cnd_wait(__cv, __m) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m, timespec *__ts) { @@ -545,6 +587,7 @@ int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m, return __ec == thrd_timedout ? ETIMEDOUT : __ec; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv) { cnd_destroy(__cv); @@ -552,6 +595,7 @@ int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv) } // Execute once +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_execute_once(__libcpp_exec_once_flag *flag, void (*init_routine)(void)) { ::call_once(flag, init_routine); @@ -560,22 +604,26 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag, // Thread id // Returns non-zero if the thread ids are equal, otherwise 0 +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2) { return thrd_equal(t1, t2) != 0; } // Returns non-zero if t1 < t2, otherwise 0 +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2) { return t1 < t2; } // Thread +_LIBCPP_HIDE_FROM_ABI inline bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) { return __libcpp_thread_get_id(__t) == 0; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *), void *__arg) { @@ -583,31 +631,37 @@ int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *), return __ec == thrd_nomem ? ENOMEM : __ec; } +_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_current_id() { return thrd_current(); } +_LIBCPP_HIDE_FROM_ABI inline __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t) { return *__t; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_join(__libcpp_thread_t *__t) { return thrd_join(*__t, nullptr) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_thread_detach(__libcpp_thread_t *__t) { return thrd_detach(*__t) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_yield() { thrd_yield(); } +_LIBCPP_HIDE_FROM_ABI inline void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) { __libcpp_timespec_t __ts = __thread_detail::__convert_to_timespec(__ns); @@ -615,16 +669,19 @@ void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns) } // Thread local storage +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_create(__libcpp_tls_key *__key, void (*__at_exit)(void *)) { return tss_create(__key, __at_exit) == thrd_success ? 0 : EINVAL; } +_LIBCPP_HIDE_FROM_ABI inline void *__libcpp_tls_get(__libcpp_tls_key __key) { return tss_get(__key); } +_LIBCPP_HIDE_FROM_ABI inline int __libcpp_tls_set(__libcpp_tls_key __key, void *__p) { return tss_set(__key, __p) == thrd_success ? 0 : EINVAL; @@ -643,7 +700,7 @@ namespace this_thread _LIBCPP_INLINE_VISIBILITY __thread_id get_id() _NOEXCEPT; -} // this_thread +} // namespace this_thread template<> struct hash<__thread_id>; @@ -713,7 +770,7 @@ get_id() _NOEXCEPT return __libcpp_thread_get_current_id(); } -} // this_thread +} // namespace this_thread #endif // !_LIBCPP_HAS_NO_THREADS diff --git a/libcxx/include/__tuple b/libcxx/include/__tuple index 11fbba260238..e8eb0b3aaf0c 100644 --- a/libcxx/include/__tuple +++ b/libcxx/include/__tuple @@ -387,7 +387,7 @@ template struct __all_dummy; template -using __all = _IsSame<__all_dummy<_Pred...>, __all_dummy<((void)_Pred, true)...>>; +struct __all : _IsSame<__all_dummy<_Pred...>, __all_dummy<((void)_Pred, true)...>> {}; struct __tuple_sfinae_base { template