diff options
Diffstat (limited to 'compiler/rustc_llvm/llvm-wrapper')
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp | 24 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h | 1 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 124 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 55 |
4 files changed, 176 insertions, 28 deletions
diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index 25badc3f4e1..e97d96e3a4e 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -8,6 +8,17 @@ using namespace llvm; +struct LLVMRustCounterMappingRegion { + coverage::Counter Count; + uint32_t FileID; + uint32_t ExpandedFileID; + uint32_t LineStart; + uint32_t ColumnStart; + uint32_t LineEnd; + uint32_t ColumnEnd; + coverage::CounterMappingRegion::RegionKind Kind; +}; + extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( const char* const Filenames[], size_t FilenamesLen, @@ -27,13 +38,22 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer( unsigned NumVirtualFileMappingIDs, const coverage::CounterExpression *Expressions, unsigned NumExpressions, - coverage::CounterMappingRegion *MappingRegions, + LLVMRustCounterMappingRegion *RustMappingRegions, unsigned NumMappingRegions, RustStringRef BufferOut) { + // Convert from FFI representation to LLVM representation. + SmallVector<coverage::CounterMappingRegion, 0> MappingRegions; + MappingRegions.reserve(NumMappingRegions); + for (const auto &Region : makeArrayRef(RustMappingRegions, NumMappingRegions)) { + MappingRegions.emplace_back( + Region.Count, Region.FileID, Region.ExpandedFileID, + Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, + Region.Kind); + } auto CoverageMappingWriter = coverage::CoverageMappingWriter( makeArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs), makeArrayRef(Expressions, NumExpressions), - makeMutableArrayRef(MappingRegions, NumMappingRegions)); + MappingRegions); RawRustStringOstream OS(BufferOut); CoverageMappingWriter.write(OS); } diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h index 0e3bf5615af..f67e06706ea 100644 --- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h +++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h @@ -86,6 +86,7 @@ enum LLVMRustAttribute { ReadNone = 26, InaccessibleMemOnly = 27, SanitizeHWAddress = 28, + WillReturn = 29, }; typedef struct OpaqueRustString *RustStringRef; diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 5263d5dcf3e..2f28162f908 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -5,6 +5,7 @@ #include "LLVMWrapper.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -683,6 +684,25 @@ void LLVMSelfProfileInitializeCallbacks( PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { +#if LLVM_VERSION_GE(12, 0) + PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterPassCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, + const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerAfterPassInvalidatedCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); +#else PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback]( StringRef Pass, llvm::Any Ir) { std::string PassName = Pass.str(); @@ -700,6 +720,7 @@ void LLVMSelfProfileInitializeCallbacks( [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) { AfterPassCallback(LlvmSelfProfiler); }); +#endif PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( StringRef Pass, llvm::Any Ir) { @@ -760,8 +781,15 @@ LLVMRustOptimizeWithNewPassManager( PTO.LoopVectorization = LoopVectorize; PTO.SLPVectorization = SLPVectorize; + // FIXME: We may want to expose this as an option. + bool DebugPassManager = false; + PassInstrumentationCallbacks PIC; +#if LLVM_VERSION_GE(12, 0) + StandardInstrumentations SI(DebugPassManager); +#else StandardInstrumentations SI; +#endif SI.registerCallbacks(PIC); if (LlvmSelfProfiler){ @@ -777,10 +805,12 @@ LLVMRustOptimizeWithNewPassManager( PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse); } +#if LLVM_VERSION_GE(12, 0) + PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); +#else PassBuilder PB(TM, PTO, PGOOpt, &PIC); +#endif - // FIXME: We may want to expose this as an option. - bool DebugPassManager = false; LoopAnalysisManager LAM(DebugPassManager); FunctionAnalysisManager FAM(DebugPassManager); CGSCCAnalysisManager CGAM(DebugPassManager); @@ -802,7 +832,8 @@ LLVMRustOptimizeWithNewPassManager( // We manually collect pipeline callbacks so we can apply them at O0, where the // PassBuilder does not create a pipeline. - std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks; + std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>> + PipelineStartEPCallbacks; #if LLVM_VERSION_GE(11, 0) std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>> OptimizerLastEPCallbacks; @@ -812,9 +843,11 @@ LLVMRustOptimizeWithNewPassManager( #endif if (VerifyIR) { - PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) { + PipelineStartEPCallbacks.push_back( + [VerifyIR](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { MPM.addPass(VerifierPass()); - }); + } + ); } if (SanitizerOptions) { @@ -832,9 +865,11 @@ LLVMRustOptimizeWithNewPassManager( ); #else #if LLVM_VERSION_GE(10, 0) - PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) { - MPM.addPass(MemorySanitizerPass(Options)); - }); + PipelineStartEPCallbacks.push_back( + [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(MemorySanitizerPass(Options)); + } + ); #endif OptimizerLastEPCallbacks.push_back( [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { @@ -854,9 +889,11 @@ LLVMRustOptimizeWithNewPassManager( ); #else #if LLVM_VERSION_GE(10, 0) - PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) { - MPM.addPass(ThreadSanitizerPass()); - }); + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(ThreadSanitizerPass()); + } + ); #endif OptimizerLastEPCallbacks.push_back( [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { @@ -879,9 +916,11 @@ LLVMRustOptimizeWithNewPassManager( } ); #else - PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) { - MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>()); - }); + PipelineStartEPCallbacks.push_back( + [&](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>()); + } + ); OptimizerLastEPCallbacks.push_back( [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { FPM.addPass(AddressSanitizerPass( @@ -890,7 +929,7 @@ LLVMRustOptimizeWithNewPassManager( } ); PipelineStartEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM) { + [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { MPM.addPass(ModuleAddressSanitizerPass( /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); } @@ -907,7 +946,7 @@ LLVMRustOptimizeWithNewPassManager( ); #else PipelineStartEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM) { + [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { MPM.addPass(HWAddressSanitizerPass( /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); } @@ -917,35 +956,53 @@ LLVMRustOptimizeWithNewPassManager( } ModulePassManager MPM(DebugPassManager); + bool NeedThinLTOBufferPasses = UseThinLTOBuffers; if (!NoPrepopulatePasses) { if (OptLevel == PassBuilder::OptimizationLevel::O0) { +#if LLVM_VERSION_GE(12, 0) for (const auto &C : PipelineStartEPCallbacks) - C(MPM); + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); -#if LLVM_VERSION_GE(11, 0) + // Pass false as we manually schedule ThinLTOBufferPasses below. + MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); +#else + for (const auto &C : PipelineStartEPCallbacks) + C(MPM, OptLevel); + +# if LLVM_VERSION_GE(11, 0) for (const auto &C : OptimizerLastEPCallbacks) C(MPM, OptLevel); -#else +# else if (!OptimizerLastEPCallbacks.empty()) { FunctionPassManager FPM(DebugPassManager); for (const auto &C : OptimizerLastEPCallbacks) C(FPM, OptLevel); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } -#endif +# endif MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers)); -#if LLVM_VERSION_GE(10, 0) +# if LLVM_VERSION_GE(10, 0) if (PGOOpt) { PB.addPGOInstrPassesForO0( MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr, /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); } +# endif #endif } else { +#if LLVM_VERSION_GE(12, 0) for (const auto &C : PipelineStartEPCallbacks) PB.registerPipelineStartEPCallback(C); +#else + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback([C, OptLevel](ModulePassManager &MPM) { + C(MPM, OptLevel); + }); +#endif if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { for (const auto &C : OptimizerLastEPCallbacks) PB.registerOptimizerLastEPCallback(C); @@ -956,7 +1013,17 @@ LLVMRustOptimizeWithNewPassManager( MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); break; case LLVMRustOptStage::PreLinkThinLTO: +#if LLVM_VERSION_GE(12, 0) + MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); + // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback + // passes may still run afterwards. This means we need to run the buffer passes again. + // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks + // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. + if (OptimizerLastEPCallbacks.empty()) + NeedThinLTOBufferPasses = false; +#else MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); +#endif #if LLVM_VERSION_GE(11, 0) for (const auto &C : OptimizerLastEPCallbacks) C(MPM, OptLevel); @@ -970,21 +1037,34 @@ LLVMRustOptimizeWithNewPassManager( #endif break; case LLVMRustOptStage::PreLinkFatLTO: +#if LLVM_VERSION_GE(12, 0) + MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); + NeedThinLTOBufferPasses = false; +#else MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); +#endif break; case LLVMRustOptStage::ThinLTO: // FIXME: Does it make sense to pass the ModuleSummaryIndex? // It only seems to be needed for C++ specific optimizations. +#if LLVM_VERSION_GE(12, 0) + MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); +#else MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); +#endif break; case LLVMRustOptStage::FatLTO: +#if LLVM_VERSION_GE(12, 0) + MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); +#else MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); +#endif break; } } } - if (UseThinLTOBuffers) { + if (NeedThinLTOBufferPasses) { MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 45835990cec..a8536595404 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -207,6 +207,8 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { return Attribute::InaccessibleMemOnly; case SanitizeHWAddress: return Attribute::SanitizeHWAddress; + case WillReturn: + return Attribute::WillReturn; } report_fatal_error("bad AttributeKind"); } @@ -263,6 +265,17 @@ extern "C" void LLVMRustAddByValCallSiteAttr(LLVMValueRef Instr, unsigned Index, Call->addAttribute(Index, Attr); } +extern "C" void LLVMRustAddStructRetCallSiteAttr(LLVMValueRef Instr, unsigned Index, + LLVMTypeRef Ty) { + CallBase *Call = unwrap<CallBase>(Instr); +#if LLVM_VERSION_GE(12, 0) + Attribute Attr = Attribute::getWithStructRetType(Call->getContext(), unwrap(Ty)); +#else + Attribute Attr = Attribute::get(Call->getContext(), Attribute::StructRet); +#endif + Call->addAttribute(Index, Attr); +} + extern "C" void LLVMRustAddFunctionAttribute(LLVMValueRef Fn, unsigned Index, LLVMRustAttribute RustAttr) { Function *A = unwrap<Function>(Fn); @@ -304,6 +317,17 @@ extern "C" void LLVMRustAddByValAttr(LLVMValueRef Fn, unsigned Index, F->addAttribute(Index, Attr); } +extern "C" void LLVMRustAddStructRetAttr(LLVMValueRef Fn, unsigned Index, + LLVMTypeRef Ty) { + Function *F = unwrap<Function>(Fn); +#if LLVM_VERSION_GE(12, 0) + Attribute Attr = Attribute::getWithStructRetType(F->getContext(), unwrap(Ty)); +#else + Attribute Attr = Attribute::get(F->getContext(), Attribute::StructRet); +#endif + F->addAttribute(Index, Attr); +} + extern "C" void LLVMRustAddFunctionAttrStringValue(LLVMValueRef Fn, unsigned Index, const char *Name, @@ -1007,12 +1031,19 @@ LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, - LLVMMetadataRef Scope, + LLVMMetadataRef ScopeRef, LLVMMetadataRef InlinedAt) { - DebugLoc debug_loc = DebugLoc::get(Line, Column, unwrapDIPtr<MDNode>(Scope), +#if LLVM_VERSION_GE(12, 0) + MDNode *Scope = unwrapDIPtr<MDNode>(ScopeRef); + DILocation *Loc = DILocation::get( + Scope->getContext(), Line, Column, Scope, + unwrapDIPtr<MDNode>(InlinedAt)); + return wrap(Loc); +#else + DebugLoc debug_loc = DebugLoc::get(Line, Column, unwrapDIPtr<MDNode>(ScopeRef), unwrapDIPtr<MDNode>(InlinedAt)); - return wrap(debug_loc.getAsMDNode()); +#endif } extern "C" int64_t LLVMRustDIBuilderCreateOpDeref() { @@ -1263,6 +1294,10 @@ extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { case Type::BFloatTyID: return LLVMBFloatTypeKind; #endif +#if LLVM_VERSION_GE(12, 0) + case Type::X86_AMXTyID: + return LLVMX86_AMXTypeKind; +#endif } report_fatal_error("Unhandled TypeID."); } @@ -1708,11 +1743,23 @@ LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) } extern "C" LLVMValueRef LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { - return wrap(unwrap(B)->CreateFPMinReduce(unwrap(Src), NoNaN)); +#if LLVM_VERSION_GE(12, 0) + Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +#else + return wrap(unwrap(B)->CreateFPMinReduce(unwrap(Src), NoNaN)); +#endif } extern "C" LLVMValueRef LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { +#if LLVM_VERSION_GE(12, 0) + Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +#else return wrap(unwrap(B)->CreateFPMaxReduce(unwrap(Src), NoNaN)); +#endif } extern "C" LLVMValueRef |