diff options
Diffstat (limited to 'compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp')
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 698 |
1 files changed, 357 insertions, 341 deletions
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 067374c0261..20859b167bc 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1,58 +1,53 @@ -#include <stdio.h> - -#include <cstddef> -#include <iomanip> -#include <vector> -#include <set> - #include "LLVMWrapper.h" -#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm-c/Core.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/Lint.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/CodeGen/CommandFlags.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/AssemblyAnnotationWriter.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" +#include "llvm/LTO/LTO.h" +#include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Object/IRObjectFile.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/TimeProfiler.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/TargetParser/Host.h" #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" -#include "llvm/Transforms/Utils/AddDiscriminators.h" -#include "llvm/Transforms/Utils/FunctionImportUtils.h" -#include "llvm/LTO/LTO.h" -#include "llvm/Bitcode/BitcodeWriter.h" -#if LLVM_VERSION_GE(18, 0) -#include "llvm/TargetParser/Host.h" -#endif -#include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h" -#include "llvm/Support/TimeProfiler.h" -#if LLVM_VERSION_GE(19, 0) -#include "llvm/Support/PGOOptions.h" -#endif -#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" -#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" -#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" -#include "llvm/Transforms/Utils.h" +#include <set> +#include <string> +#include <vector> + +// Conditional includes prevent clang-format from fully sorting the list, +// so keep them separate. +#if LLVM_VERSION_GE(19, 0) +#include "llvm/Support/PGOOptions.h" +#endif using namespace llvm; @@ -74,7 +69,7 @@ extern "C" void LLVMRustTimeTraceProfilerFinishThread() { timeTraceProfilerFinishThread(); } -extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { +extern "C" void LLVMRustTimeTraceProfilerFinish(const char *FileName) { auto FN = StringRef(FileName); std::error_code EC; auto OS = raw_fd_ostream(FN, EC, sys::fs::CD_CreateAlways); @@ -155,6 +150,12 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { #define SUBTARGET_SPARC #endif +#ifdef LLVM_COMPONENT_XTENSA +#define SUBTARGET_XTENSA SUBTARGET(XTENSA) +#else +#define SUBTARGET_XTENSA +#endif + #ifdef LLVM_COMPONENT_HEXAGON #define SUBTARGET_HEXAGON SUBTARGET(Hexagon) #else @@ -180,8 +181,9 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { SUBTARGET_MSP430 \ SUBTARGET_SPARC \ SUBTARGET_HEXAGON \ + SUBTARGET_XTENSA \ SUBTARGET_RISCV \ - SUBTARGET_LOONGARCH \ + SUBTARGET_LOONGARCH #define SUBTARGET(x) \ namespace llvm { \ @@ -208,8 +210,7 @@ enum class LLVMRustCodeModel { None, }; -static std::optional<CodeModel::Model> -fromRust(LLVMRustCodeModel Model) { +static std::optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) { switch (Model) { case LLVMRustCodeModel::Tiny: return CodeModel::Tiny; @@ -235,11 +236,7 @@ enum class LLVMRustCodeGenOptLevel { Aggressive, }; -#if LLVM_VERSION_GE(18, 0) - using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; -#else - using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level; -#endif +using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) { switch (Level) { @@ -312,83 +309,49 @@ static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { } /// getLongestEntryLength - Return the length of the longest entry in the table. -template<typename KV> -static size_t getLongestEntryLength(ArrayRef<KV> Table) { +template <typename KV> static size_t getLongestEntryLength(ArrayRef<KV> Table) { size_t MaxLen = 0; for (auto &I : Table) MaxLen = std::max(MaxLen, std::strlen(I.Key)); return MaxLen; } -using PrintBackendInfo = void(void*, const char* Data, size_t Len); - extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, - const char* TargetCPU, - PrintBackendInfo Print, - void* Out) { - const TargetMachine *Target = unwrap(TM); - const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch(); - const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); - - std::ostringstream Buf; + RustStringRef OutStr) { + ArrayRef<SubtargetSubTypeKV> CPUTable = + unwrap(TM)->getMCSubtargetInfo()->getAllProcessorDescriptions(); + auto OS = RawRustStringOstream(OutStr); - const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getAllProcessorDescriptions(); - unsigned MaxCPULen = getLongestEntryLength(CPUTable); - - Buf << "Available CPUs for this target:\n"; - // Don't print the "native" entry when the user specifies --target with a - // different arch since that could be wrong or misleading. - if (HostArch == TargetArch) { - MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native")); - const StringRef HostCPU = sys::getHostCPUName(); - Buf << " " << std::left << std::setw(MaxCPULen) << "native" - << " - Select the CPU of the current host " - "(currently " << HostCPU.str() << ").\n"; - } + // Just print a bare list of target CPU names, and let Rust-side code handle + // the full formatting of `--print=target-cpus`. for (auto &CPU : CPUTable) { - // Compare cpu against current target to label the default - if (strcmp(CPU.Key, TargetCPU) == 0) { - Buf << " " << std::left << std::setw(MaxCPULen) << CPU.Key - << " - This is the default target CPU for the current build target " - "(currently " << Target->getTargetTriple().str() << ")."; - } - else { - Buf << " " << CPU.Key; - } - Buf << "\n"; + OS << CPU.Key << "\n"; } - - const auto &BufString = Buf.str(); - Print(Out, BufString.data(), BufString.size()); } extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { -#if LLVM_VERSION_GE(18, 0) const TargetMachine *Target = unwrap(TM); const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getAllProcessorFeatures(); + const ArrayRef<SubtargetFeatureKV> FeatTable = + MCInfo->getAllProcessorFeatures(); return FeatTable.size(); -#else - return 0; -#endif } extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, - const char** Feature, const char** Desc) { -#if LLVM_VERSION_GE(18, 0) + const char **Feature, + const char **Desc) { const TargetMachine *Target = unwrap(TM); const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getAllProcessorFeatures(); + const ArrayRef<SubtargetFeatureKV> FeatTable = + MCInfo->getAllProcessorFeatures(); const SubtargetFeatureKV Feat = FeatTable[Index]; *Feature = Feat.Key; *Desc = Feat.Desc; -#endif } -extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { +extern "C" const char *LLVMRustGetHostCPUName(size_t *OutLen) { StringRef Name = sys::getHostCPUName(); - *len = Name.size(); + *OutLen = Name.size(); return Name.data(); } @@ -396,19 +359,11 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( const char *TripleStr, const char *CPU, const char *Feature, const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, - bool FunctionSections, - bool DataSections, - bool UniqueSectionNames, - bool TrapUnreachable, - bool Singlethread, - bool AsmComments, - bool EmitStackSizeSection, - bool RelaxELFRelocations, - bool UseInitArray, - const char *SplitDwarfFile, - const char *OutputObjFile, - const char *DebugInfoCompression, - bool UseEmulatedTls, + bool FunctionSections, bool DataSections, bool UniqueSectionNames, + bool TrapUnreachable, bool Singlethread, bool VerboseAsm, + bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, + const char *SplitDwarfFile, const char *OutputObjFile, + const char *DebugInfoCompression, bool UseEmulatedTls, const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { auto OptLevel = fromRust(RustOptLevel); @@ -433,22 +388,25 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; Options.UniqueSectionNames = UniqueSectionNames; - Options.MCOptions.AsmVerbose = AsmComments; - Options.MCOptions.PreserveAsmComments = AsmComments; + Options.MCOptions.AsmVerbose = VerboseAsm; + // Always preserve comments that were written by the user + Options.MCOptions.PreserveAsmComments = true; Options.MCOptions.ABIName = ABIStr; if (SplitDwarfFile) { - Options.MCOptions.SplitDwarfFile = SplitDwarfFile; + Options.MCOptions.SplitDwarfFile = SplitDwarfFile; } if (OutputObjFile) { - Options.ObjectFilenameForDebug = OutputObjFile; + Options.ObjectFilenameForDebug = OutputObjFile; } - if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { + if (!strcmp("zlib", DebugInfoCompression) && + llvm::compression::zlib::isAvailable()) { #if LLVM_VERSION_GE(19, 0) Options.MCOptions.CompressDebugSections = DebugCompressionType::Zlib; #else Options.CompressDebugSections = DebugCompressionType::Zlib; #endif - } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { + } else if (!strcmp("zstd", DebugInfoCompression) && + llvm::compression::zstd::isAvailable()) { #if LLVM_VERSION_GE(19, 0) Options.MCOptions.CompressDebugSections = DebugCompressionType::Zstd; #else @@ -492,24 +450,37 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.EmitStackSizeSection = EmitStackSizeSection; - - if (ArgsCstrBuff != nullptr) - { + if (ArgsCstrBuff != nullptr) { +#if LLVM_VERSION_GE(20, 0) + int buffer_offset = 0; + assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); + auto Arg0 = std::string(ArgsCstrBuff); + buffer_offset = Arg0.size() + 1; + auto ArgsCppStr = std::string(ArgsCstrBuff + buffer_offset, + ArgsCstrBuffLen - buffer_offset); + auto i = 0; + while (i != std::string::npos) { + i = ArgsCppStr.find('\0', i + 1); + if (i != std::string::npos) + ArgsCppStr.replace(i, 1, " "); + } + Options.MCOptions.Argv0 = Arg0; + Options.MCOptions.CommandlineArgs = ArgsCppStr; +#else int buffer_offset = 0; assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); const size_t arg0_len = std::strlen(ArgsCstrBuff); - char* arg0 = new char[arg0_len + 1]; + char *arg0 = new char[arg0_len + 1]; memcpy(arg0, ArgsCstrBuff, arg0_len); arg0[arg0_len] = '\0'; buffer_offset += arg0_len + 1; - const int num_cmd_arg_strings = - std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); + const int num_cmd_arg_strings = std::count( + &ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); - std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings]; - for (int i = 0; i < num_cmd_arg_strings; ++i) - { + std::string *cmd_arg_strings = new std::string[num_cmd_arg_strings]; + for (int i = 0; i < num_cmd_arg_strings; ++i) { assert(buffer_offset < ArgsCstrBuffLen); const int len = std::strlen(ArgsCstrBuff + buffer_offset); cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); @@ -520,7 +491,8 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.MCOptions.Argv0 = arg0; Options.MCOptions.CommandLineArgs = - llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings); + llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings); +#endif } TargetMachine *TM = TheTarget->createTargetMachine( @@ -529,10 +501,11 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( } extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { - - MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions; +#if LLVM_VERSION_LT(20, 0) + MCTargetOptions &MCOptions = unwrap(TM)->Options.MCOptions; delete[] MCOptions.Argv0; delete[] MCOptions.CommandLineArgs.data(); +#endif delete unwrap(TM); } @@ -568,17 +541,9 @@ enum class LLVMRustFileType { static CodeGenFileType fromRust(LLVMRustFileType Type) { switch (Type) { case LLVMRustFileType::AssemblyFile: -#if LLVM_VERSION_GE(18, 0) return CodeGenFileType::AssemblyFile; -#else - return CGFT_AssemblyFile; -#endif case LLVMRustFileType::ObjectFile: -#if LLVM_VERSION_GE(18, 0) return CodeGenFileType::ObjectFile; -#else - return CGFT_ObjectFile; -#endif default: report_fatal_error("Bad FileType."); } @@ -587,7 +552,7 @@ static CodeGenFileType fromRust(LLVMRustFileType Type) { extern "C" LLVMRustResult LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, LLVMModuleRef M, const char *Path, const char *DwoPath, - LLVMRustFileType RustFileType) { + LLVMRustFileType RustFileType, bool VerifyIR) { llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); auto FileType = fromRust(RustFileType); @@ -606,16 +571,16 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, auto DOS = raw_fd_ostream(DwoPath, EC, sys::fs::OF_None); EC.clear(); if (EC) - ErrorInfo = EC.message(); + ErrorInfo = EC.message(); if (ErrorInfo != "") { LLVMRustSetLastError(ErrorInfo.c_str()); return LLVMRustResult::Failure; } auto DBOS = buffer_ostream(DOS); - unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, !VerifyIR); PM->run(*unwrap(M)); } else { - unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, !VerifyIR); PM->run(*unwrap(M)); } @@ -626,10 +591,12 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, return LLVMRustResult::Success; } -extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler - const char*, // pass name - const char*); // IR name -extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler +extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)( + void *, // LlvmSelfProfiler + const char *, // pass name + const char *); // IR name +extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)( + void *); // LlvmSelfProfiler std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { if (const auto *Cast = any_cast<const Module *>(&WrappedIr)) @@ -643,35 +610,35 @@ std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { return "<UNKNOWN>"; } - void LLVMSelfProfileInitializeCallbacks( - PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, + PassInstrumentationCallbacks &PIC, void *LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { - PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( - StringRef Pass, llvm::Any Ir) { - std::string PassName = Pass.str(); - std::string IrName = LLVMRustwrappedIrGetName(Ir); - BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); - }); + PIC.registerBeforeNonSkippedPassCallback( + [LlvmSelfProfiler, BeforePassCallback](StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); PIC.registerAfterPassCallback( - [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, - const PreservedAnalyses &Preserved) { + [LlvmSelfProfiler, AfterPassCallback]( + StringRef Pass, llvm::Any IR, const PreservedAnalyses &Preserved) { AfterPassCallback(LlvmSelfProfiler); }); PIC.registerAfterPassInvalidatedCallback( - [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { + [LlvmSelfProfiler, + AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { AfterPassCallback(LlvmSelfProfiler); }); - PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( - StringRef Pass, llvm::Any Ir) { - std::string PassName = Pass.str(); - std::string IrName = LLVMRustwrappedIrGetName(Ir); - BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); - }); + PIC.registerBeforeAnalysisCallback( + [LlvmSelfProfiler, BeforePassCallback](StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); PIC.registerAfterAnalysisCallback( [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { @@ -697,7 +664,7 @@ struct LLVMRustSanitizerOptions { bool SanitizeKCFI; bool SanitizeMemory; bool SanitizeMemoryRecover; - int SanitizeMemoryTrackOrigins; + int SanitizeMemoryTrackOrigins; bool SanitizeThread; bool SanitizeHWAddress; bool SanitizeHWAddressRecover; @@ -705,31 +672,24 @@ struct LLVMRustSanitizerOptions { bool SanitizeKernelAddressRecover; }; -extern "C" LLVMRustResult -LLVMRustOptimize( - LLVMModuleRef ModuleRef, - LLVMTargetMachineRef TMRef, - LLVMRustPassBuilderOptLevel OptLevelRust, - LLVMRustOptStage OptStage, - bool IsLinkerPluginLTO, - bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, - bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, - bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, - LLVMRustSanitizerOptions *SanitizerOptions, - const char *PGOGenPath, const char *PGOUsePath, - bool InstrumentCoverage, const char *InstrProfileOutput, - bool InstrumentGCOV, - const char *PGOSampleUsePath, bool DebugInfoForProfiling, - void* LlvmSelfProfiler, +extern "C" LLVMRustResult LLVMRustOptimize( + LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, + LLVMRustPassBuilderOptLevel OptLevelRust, LLVMRustOptStage OptStage, + bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, + bool LintIR, bool UseThinLTOBuffers, bool MergeFunctions, bool UnrollLoops, + bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, + bool EmitLifetimeMarkers, LLVMRustSanitizerOptions *SanitizerOptions, + const char *PGOGenPath, const char *PGOUsePath, bool InstrumentCoverage, + const char *InstrProfileOutput, const char *PGOSampleUsePath, + bool DebugInfoForProfiling, void *LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, LLVMRustSelfProfileAfterPassCallback AfterPassCallback, - const char *ExtraPasses, size_t ExtraPassesLen, - const char *LLVMPlugins, size_t LLVMPluginsLen) { + const char *ExtraPasses, size_t ExtraPassesLen, const char *LLVMPlugins, + size_t LLVMPluginsLen) { Module *TheModule = unwrap(ModuleRef); TargetMachine *TM = unwrap(TMRef); OptimizationLevel OptLevel = fromRust(OptLevelRust); - PipelineTuningOptions PTO; PTO.LoopUnrolling = UnrollLoops; PTO.LoopInterleaving = UnrollLoops; @@ -737,45 +697,41 @@ LLVMRustOptimize( PTO.SLPVectorization = SLPVectorize; PTO.MergeFunctions = MergeFunctions; - // FIXME: We may want to expose this as an option. - bool DebugPassManager = false; - PassInstrumentationCallbacks PIC; - StandardInstrumentations SI(TheModule->getContext(), DebugPassManager); - SI.registerCallbacks(PIC); - if (LlvmSelfProfiler){ - LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); + if (LlvmSelfProfiler) { + LLVMSelfProfileInitializeCallbacks(PIC, LlvmSelfProfiler, + BeforePassCallback, AfterPassCallback); } std::optional<PGOOptions> PGOOpt; auto FS = vfs::getRealFileSystem(); if (PGOGenPath) { assert(!PGOUsePath && !PGOSampleUsePath); - PGOOpt = PGOOptions(PGOGenPath, "", "", "", FS, - PGOOptions::IRInstr, PGOOptions::NoCSAction, + PGOOpt = PGOOptions(PGOGenPath, "", "", "", FS, PGOOptions::IRInstr, + PGOOptions::NoCSAction, #if LLVM_VERSION_GE(19, 0) PGOOptions::ColdFuncOpt::Default, #endif DebugInfoForProfiling); } else if (PGOUsePath) { assert(!PGOSampleUsePath); - PGOOpt = PGOOptions(PGOUsePath, "", "", "", FS, - PGOOptions::IRUse, PGOOptions::NoCSAction, + PGOOpt = PGOOptions(PGOUsePath, "", "", "", FS, PGOOptions::IRUse, + PGOOptions::NoCSAction, #if LLVM_VERSION_GE(19, 0) PGOOptions::ColdFuncOpt::Default, #endif DebugInfoForProfiling); } else if (PGOSampleUsePath) { - PGOOpt = PGOOptions(PGOSampleUsePath, "", "", "", FS, - PGOOptions::SampleUse, PGOOptions::NoCSAction, + PGOOpt = PGOOptions(PGOSampleUsePath, "", "", "", FS, PGOOptions::SampleUse, + PGOOptions::NoCSAction, #if LLVM_VERSION_GE(19, 0) PGOOptions::ColdFuncOpt::Default, #endif DebugInfoForProfiling); } else if (DebugInfoForProfiling) { - PGOOpt = PGOOptions("", "", "", "", FS, - PGOOptions::NoAction, PGOOptions::NoCSAction, + PGOOpt = PGOOptions("", "", "", "", FS, PGOOptions::NoAction, + PGOOptions::NoCSAction, #if LLVM_VERSION_GE(19, 0) PGOOptions::ColdFuncOpt::Default, #endif @@ -788,11 +744,15 @@ LLVMRustOptimize( CGSCCAnalysisManager CGAM; ModuleAnalysisManager MAM; + StandardInstrumentations SI(TheModule->getContext(), + /*DebugLogging=*/false); + SI.registerCallbacks(PIC, &MAM); + if (LLVMPluginsLen) { auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); SmallVector<StringRef> Plugins; PluginsStr.split(Plugins, ',', -1, false); - for (auto PluginPath: Plugins) { + for (auto PluginPath : Plugins) { auto Plugin = PassPlugin::Load(PluginPath.str()); if (!Plugin) { auto Err = Plugin.takeError(); @@ -807,7 +767,8 @@ LLVMRustOptimize( FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); Triple TargetTriple(TheModule->getTargetTriple()); - std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple)); + std::unique_ptr<TargetLibraryInfoImpl> TLII( + new TargetLibraryInfoImpl(TargetTriple)); if (DisableSimplifyLibCalls) TLII->disableAllFunctions(); FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); @@ -818,58 +779,55 @@ LLVMRustOptimize( PB.registerLoopAnalyses(LAM); PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - // We manually collect pipeline callbacks so we can apply them at O0, where the - // PassBuilder does not create a pipeline. + // We manually collect pipeline callbacks so we can apply them at O0, where + // the PassBuilder does not create a pipeline. std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> PipelineStartEPCallbacks; +#if LLVM_VERSION_GE(20, 0) + std::vector<std::function<void(ModulePassManager &, OptimizationLevel, + ThinOrFullLTOPhase)>> + OptimizerLastEPCallbacks; +#else std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> OptimizerLastEPCallbacks; +#endif - if (!IsLinkerPluginLTO - && SanitizerOptions && SanitizerOptions->SanitizeCFI - && !NoPrepopulatePasses) { + if (!IsLinkerPluginLTO && SanitizerOptions && SanitizerOptions->SanitizeCFI && + !NoPrepopulatePasses) { PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, - /*ImportSummary=*/nullptr, - /*DropTypeTests=*/false)); - } - ); + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(LowerTypeTestsPass( + /*ExportSummary=*/nullptr, + /*ImportSummary=*/nullptr)); + }); } if (VerifyIR) { PipelineStartEPCallbacks.push_back( - [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(VerifierPass()); - } - ); + [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(VerifierPass()); + }); } - if (InstrumentGCOV) { + if (LintIR) { PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); - } - ); + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(createModuleToFunctionPassAdaptor(LintPass())); + }); } if (InstrumentCoverage) { PipelineStartEPCallbacks.push_back( - [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { - InstrProfOptions Options; - if (InstrProfileOutput) { - Options.InstrProfileOutput = InstrProfileOutput; - } - // cargo run tests in multhreading mode by default - // so use atomics for coverage counters - Options.Atomic = true; -#if LLVM_VERSION_GE(18, 0) - MPM.addPass(InstrProfilingLoweringPass(Options, false)); -#else - MPM.addPass(InstrProfiling(Options, false)); -#endif - } - ); + [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { + InstrProfOptions Options; + if (InstrProfileOutput) { + Options.InstrProfileOutput = InstrProfileOutput; + } + // cargo run tests in multhreading mode by default + // so use atomics for coverage counters + Options.Atomic = true; + MPM.addPass(InstrProfilingLoweringPass(Options, false)); + }); } if (SanitizerOptions) { @@ -879,10 +837,14 @@ LLVMRustOptimize( SanitizerOptions->SanitizeDataFlowABIList + SanitizerOptions->SanitizeDataFlowABIListLen); OptimizerLastEPCallbacks.push_back( - [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(DataFlowSanitizerPass(ABIListFiles)); - } - ); +#if LLVM_VERSION_GE(20, 0) + [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(DataFlowSanitizerPass(ABIListFiles)); + }); } if (SanitizerOptions->SanitizeMemory) { @@ -892,62 +854,91 @@ LLVMRustOptimize( /*CompileKernel=*/false, /*EagerChecks=*/true); OptimizerLastEPCallbacks.push_back( - [Options](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(MemorySanitizerPass(Options)); - } - ); +#if LLVM_VERSION_GE(20, 0) + [Options](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [Options](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(MemorySanitizerPass(Options)); + }); } if (SanitizerOptions->SanitizeThread) { OptimizerLastEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(ModuleThreadSanitizerPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); - } - ); +#if LLVM_VERSION_GE(20, 0) + [](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(ModuleThreadSanitizerPass()); + MPM.addPass( + createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); + }); } - if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) { + if (SanitizerOptions->SanitizeAddress || + SanitizerOptions->SanitizeKernelAddress) { OptimizerLastEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { - auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; - AddressSanitizerOptions opts = AddressSanitizerOptions{ - CompileKernel, - SanitizerOptions->SanitizeAddressRecover - || SanitizerOptions->SanitizeKernelAddressRecover, - /*UseAfterScope=*/true, - AsanDetectStackUseAfterReturnMode::Runtime, - }; - MPM.addPass(AddressSanitizerPass(opts)); - } - ); +#if LLVM_VERSION_GE(20, 0) + [SanitizerOptions, TM](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [SanitizerOptions, TM](ModulePassManager &MPM, + OptimizationLevel Level) { +#endif + auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; + AddressSanitizerOptions opts = AddressSanitizerOptions{ + CompileKernel, + SanitizerOptions->SanitizeAddressRecover || + SanitizerOptions->SanitizeKernelAddressRecover, + /*UseAfterScope=*/true, + AsanDetectStackUseAfterReturnMode::Runtime, + }; + MPM.addPass(AddressSanitizerPass( + opts, + /*UseGlobalGC*/ true, + // UseOdrIndicator should be false on windows machines + // https://reviews.llvm.org/D137227 + !TM->getTargetTriple().isOSWindows())); + }); } if (SanitizerOptions->SanitizeHWAddress) { OptimizerLastEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { - HWAddressSanitizerOptions opts( - /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, - /*DisableOptimization=*/false); - MPM.addPass(HWAddressSanitizerPass(opts)); - } - ); +#if LLVM_VERSION_GE(20, 0) + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + HWAddressSanitizerOptions opts( + /*CompileKernel=*/false, + SanitizerOptions->SanitizeHWAddressRecover, + /*DisableOptimization=*/false); + MPM.addPass(HWAddressSanitizerPass(opts)); + }); } } ModulePassManager MPM; bool NeedThinLTOBufferPasses = UseThinLTOBuffers; if (!NoPrepopulatePasses) { - // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead. - // At the same time, the LTO pipelines do support O0 and using them is required. - bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; + // The pre-link pipelines don't support O0 and require using + // buildO0DefaultPipeline() instead. At the same time, the LTO pipelines do + // support O0 and using them is required. + bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || + OptStage == LLVMRustOptStage::FatLTO; if (OptLevel == OptimizationLevel::O0 && !IsLTO) { for (const auto &C : PipelineStartEPCallbacks) PB.registerPipelineStartEPCallback(C); for (const auto &C : OptimizerLastEPCallbacks) PB.registerOptimizerLastEPCallback(C); - // Pass false as we manually schedule ThinLTOBufferPasses below. - MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); + // We manually schedule ThinLTOBufferPasses below, so don't pass the value + // to enable it here. + MPM = PB.buildO0DefaultPipeline(OptLevel); } else { for (const auto &C : PipelineStartEPCallbacks) PB.registerPipelineStartEPCallback(C); @@ -956,7 +947,7 @@ LLVMRustOptimize( switch (OptStage) { case LLVMRustOptStage::PreLinkNoLTO: - MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); + MPM = PB.buildPerModuleDefaultPipeline(OptLevel); break; case LLVMRustOptStage::PreLinkThinLTO: MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); @@ -982,11 +973,16 @@ LLVMRustOptimize( for (const auto &C : PipelineStartEPCallbacks) C(MPM, OptLevel); for (const auto &C : OptimizerLastEPCallbacks) +#if LLVM_VERSION_GE(20, 0) + C(MPM, OptLevel, ThinOrFullLTOPhase::None); +#else C(MPM, OptLevel); +#endif } if (ExtraPassesLen) { - if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { + if (auto Err = + PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { std::string ErrMsg = toString(std::move(Err)); LLVMRustSetLastError(ErrMsg.c_str()); return LLVMRustResult::Failure; @@ -1013,8 +1009,7 @@ LLVMRustOptimize( // * output buffer // * output buffer len // Returns len of demangled string, or 0 if demangle failed. -typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); - +typedef size_t (*DemangleFn)(const char *, size_t, char *, size_t); namespace { @@ -1057,7 +1052,7 @@ public: formatted_raw_ostream &OS) override { StringRef Demangled = CallDemangle(F->getName()); if (Demangled.empty()) { - return; + return; } OS << "; " << Demangled << "\n"; @@ -1070,7 +1065,7 @@ public: if (const CallInst *CI = dyn_cast<CallInst>(I)) { Name = "call"; Value = CI->getCalledOperand(); - } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { + } else if (const InvokeInst *II = dyn_cast<InvokeInst>(I)) { Name = "invoke"; Value = II->getCalledOperand(); } else { @@ -1094,8 +1089,8 @@ public: } // namespace -extern "C" LLVMRustResult -LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { +extern "C" LLVMRustResult LLVMRustPrintModule(LLVMModuleRef M, const char *Path, + DemangleFn Demangle) { std::string ErrorInfo; std::error_code EC; auto OS = raw_fd_ostream(Path, EC, sys::fs::OF_None); @@ -1210,15 +1205,13 @@ struct LLVMRustThinLTOData { // Not 100% sure what these are, but they impact what's internalized and // what's inlined across modules, I believe. -#if LLVM_VERSION_GE(18, 0) +#if LLVM_VERSION_GE(20, 0) + FunctionImporter::ImportListsTy ImportLists; +#else DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists; +#endif DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists; DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries; -#else - StringMap<FunctionImporter::ImportMapTy> ImportLists; - StringMap<FunctionImporter::ExportSetTy> ExportLists; - StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; -#endif StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} @@ -1257,37 +1250,32 @@ getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { // The main entry point for creating the global ThinLTO analysis. The structure // here is basically the same as before threads are spawned in the `run` // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. -extern "C" LLVMRustThinLTOData* -LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, - int num_modules, - const char **preserved_symbols, - int num_symbols) { +extern "C" LLVMRustThinLTOData * +LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, size_t num_modules, + const char **preserved_symbols, size_t num_symbols) { auto Ret = std::make_unique<LLVMRustThinLTOData>(); // Load each module's summary and merge it into one combined index - for (int i = 0; i < num_modules; i++) { + for (size_t i = 0; i < num_modules; i++) { auto module = &modules[i]; auto buffer = StringRef(module->data, module->len); auto mem_buffer = MemoryBufferRef(buffer, module->identifier); Ret->ModuleMap[module->identifier] = mem_buffer; -#if LLVM_VERSION_GE(18, 0) if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { -#else - if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { -#endif LLVMRustSetLastError(toString(std::move(Err)).c_str()); return nullptr; } } // Collect for each module the list of function it defines (GUID -> Summary) - Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); + Ret->Index.collectDefinedGVSummariesPerModule( + Ret->ModuleToDefinedGVSummaries); // Convert the preserved symbols set from string to GUID, this is then needed // for internalization. - for (int i = 0; i < num_symbols; i++) { + for (size_t i = 0; i < num_symbols; i++) { auto GUID = GlobalValue::getGUID(preserved_symbols[i]); Ret->GUIDPreservedSymbols.insert(GUID); } @@ -1303,7 +1291,8 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, // crate, so we need `ImportEnabled = false` to limit internalization. // Otherwise, we sometimes lose `static` values -- see #60184. computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, - deadIsPrevailing, /* ImportEnabled = */ false); + deadIsPrevailing, + /* ImportEnabled = */ false); // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it // impacts the caching. // @@ -1312,7 +1301,8 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; for (auto &I : Ret->Index) { if (I.second.SummaryList.size() > 1) - PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); + PrevailingCopy[I.first] = + getFirstDefinitionForLinker(I.second.SummaryList); } auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { const auto &Prevailing = PrevailingCopy.find(GUID); @@ -1320,13 +1310,8 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, return true; return Prevailing->second == S; }; - ComputeCrossModuleImport( - Ret->Index, - Ret->ModuleToDefinedGVSummaries, - isPrevailing, - Ret->ImportLists, - Ret->ExportLists - ); + ComputeCrossModuleImport(Ret->Index, Ret->ModuleToDefinedGVSummaries, + isPrevailing, Ret->ImportLists, Ret->ExportLists); auto recordNewLinkage = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID, @@ -1338,8 +1323,8 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, // formats. We probably could and should use ELF visibility scheme for many of // our targets, however. lto::Config conf; - thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, - Ret->GUIDPreservedSymbols); + thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, + recordNewLinkage, Ret->GUIDPreservedSymbols); // Here we calculate an `ExportedGUIDs` set for use in the `isExported` // callback below. This callback below will dictate the linkage for all @@ -1348,7 +1333,7 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, // linkage will stay as external, and internal will stay as internal. std::set<GlobalValue::GUID> ExportedGUIDs; for (auto &List : Ret->Index) { - for (auto &GVS: List.second.SummaryList) { + for (auto &GVS : List.second.SummaryList) { if (GlobalValue::isLocalLinkage(GVS->linkage())) continue; auto GUID = GVS->getOriginalName(); @@ -1359,16 +1344,15 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); return (ExportList != Ret->ExportLists.end() && - ExportList->second.count(VI)) || - ExportedGUIDs.count(VI.getGUID()); + ExportList->second.count(VI)) || + ExportedGUIDs.count(VI.getGUID()); }; thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); return Ret.release(); } -extern "C" void -LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { +extern "C" void LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { delete Data; } @@ -1380,20 +1364,18 @@ LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { // `ProcessThinLTOModule` function. Here they're split up into separate steps // so rustc can save off the intermediate bytecode between each step. -static bool -clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { +static bool clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { // When linking an ELF shared object, dso_local should be dropped. We // conservatively do this for -fpic. - bool ClearDSOLocalOnDeclarations = - TM.getTargetTriple().isOSBinFormatELF() && - TM.getRelocationModel() != Reloc::Static && - Mod.getPIELevel() == PIELevel::Default; + bool ClearDSOLocalOnDeclarations = TM.getTargetTriple().isOSBinFormatELF() && + TM.getRelocationModel() != Reloc::Static && + Mod.getPIELevel() == PIELevel::Default; return ClearDSOLocalOnDeclarations; } -extern "C" bool -LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, - LLVMTargetMachineRef TM) { +extern "C" bool LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, + LLVMModuleRef M, + LLVMTargetMachineRef TM) { Module &Mod = *unwrap(M); TargetMachine &Target = *unwrap(TM); @@ -1408,24 +1390,28 @@ LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, } extern "C" bool -LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { +LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, + LLVMModuleRef M) { Module &Mod = *unwrap(M); - const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + const auto &DefinedGlobals = + Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); return true; } extern "C" bool -LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { +LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, + LLVMModuleRef M) { Module &Mod = *unwrap(M); - const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + const auto &DefinedGlobals = + Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); thinLTOInternalizeModule(Mod, DefinedGlobals); return true; } -extern "C" bool -LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, - LLVMTargetMachineRef TM) { +extern "C" bool LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, + LLVMModuleRef M, + LLVMTargetMachineRef TM) { Module &Mod = *unwrap(M); TargetMachine &Target = *unwrap(TM); @@ -1457,7 +1443,8 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, return Ret; } - auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); + auto *WasmCustomSections = + (*MOrErr)->getNamedMetadata("wasm.custom_sections"); if (WasmCustomSections) WasmCustomSections->eraseFromParent(); @@ -1488,13 +1475,15 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, // a ThinLTO summary attached. struct LLVMRustThinLTOBuffer { std::string data; + std::string thin_link_data; }; -extern "C" LLVMRustThinLTOBuffer* -LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { +extern "C" LLVMRustThinLTOBuffer * +LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin, bool emit_summary) { auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); { auto OS = raw_string_ostream(Ret->data); + auto ThinLinkOS = raw_string_ostream(Ret->thin_link_data); { if (is_thin) { PassBuilder PB; @@ -1508,7 +1497,11 @@ LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { PB.registerLoopAnalyses(LAM); PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); ModulePassManager MPM; - MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr)); + // We only pass ThinLinkOS to be filled in if we want the summary, + // because otherwise LLVM does extra work and may double-emit some + // errors or warnings. + MPM.addPass( + ThinLTOBitcodeWriterPass(OS, emit_summary ? &ThinLinkOS : nullptr)); MPM.run(*unwrap(M), MAM); } else { WriteBitcodeToFile(*unwrap(M), OS); @@ -1518,12 +1511,11 @@ LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { return Ret.release(); } -extern "C" void -LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { +extern "C" void LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { delete Buffer; } -extern "C" const void* +extern "C" const void * LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { return Buffer->data.data(); } @@ -1533,14 +1525,23 @@ LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { return Buffer->data.length(); } +extern "C" const void * +LLVMRustThinLTOBufferThinLinkDataPtr(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->thin_link_data.data(); +} + +extern "C" size_t +LLVMRustThinLTOBufferThinLinkDataLen(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->thin_link_data.length(); +} + // This is what we used to parse upstream bitcode for actual ThinLTO // processing. We'll call this once per module optimized through ThinLTO, and // it'll be called concurrently on many threads. -extern "C" LLVMModuleRef -LLVMRustParseBitcodeForLTO(LLVMContextRef Context, - const char *data, - size_t len, - const char *identifier) { +extern "C" LLVMModuleRef LLVMRustParseBitcodeForLTO(LLVMContextRef Context, + const char *data, + size_t len, + const char *identifier) { auto Data = StringRef(data, len); auto Buffer = MemoryBufferRef(Data, identifier); unwrap(Context)->enableDebugTypeODRUniquing(); @@ -1558,8 +1559,10 @@ LLVMRustParseBitcodeForLTO(LLVMContextRef Context, extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, size_t len, const char *name, + size_t name_len, size_t *out_len) { *out_len = 0; + auto Name = StringRef(name, name_len); auto Data = StringRef(data, len); auto Buffer = MemoryBufferRef(Data, ""); // The id is unused. file_magic Type = identify_magic(Buffer.getBuffer()); @@ -1570,8 +1573,8 @@ extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, return nullptr; } for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { - Expected<StringRef> Name = Sec.getName(); - if (Name && *Name == name) { + Expected<StringRef> SecName = Sec.getName(); + if (SecName && *SecName == Name) { Expected<StringRef> SectionOrError = Sec.getContents(); if (!SectionOrError) { LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); @@ -1592,16 +1595,22 @@ extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, // of access globals, etc). // The precise details are determined by LLVM in `computeLTOCacheKey`, which is // used during the normal linker-plugin incremental thin-LTO process. -extern "C" void -LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { +extern "C" void LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, + const char *ModId, + LLVMRustThinLTOData *Data) { SmallString<40> Key; llvm::lto::Config conf; const auto &ImportList = Data->ImportLists.lookup(ModId); const auto &ExportList = Data->ExportLists.lookup(ModId); const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); +#if LLVM_VERSION_GE(20, 0) + DenseSet<GlobalValue::GUID> CfiFunctionDefs; + DenseSet<GlobalValue::GUID> CfiFunctionDecls; +#else std::set<GlobalValue::GUID> CfiFunctionDefs; std::set<GlobalValue::GUID> CfiFunctionDecls; +#endif // Based on the 'InProcessThinBackend' constructor in LLVM for (auto &Name : Data->Index.cfiFunctionDefs()) @@ -1611,9 +1620,16 @@ LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThin CfiFunctionDecls.insert( GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); - llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, - ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls - ); +#if LLVM_VERSION_GE(20, 0) + Key = llvm::computeLTOCacheKey(conf, Data->Index, ModId, ImportList, + ExportList, ResolvedODR, DefinedGlobals, + CfiFunctionDefs, CfiFunctionDecls); +#else + llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, ImportList, + ExportList, ResolvedODR, DefinedGlobals, + CfiFunctionDefs, CfiFunctionDecls); +#endif - LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); + auto OS = RawRustStringOstream(KeyOut); + OS << Key.str(); } |