Merge branch 'master' of github.com:rust-lang/rust into add-tvSO-target

1 files changed, 358 insertions, 2 deletions

diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp
index 6698e5d58be..65071c3ed86 100644
--- a/src/rustllvm/PassWrapper.cpp
+++ b/src/rustllvm/PassWrapper.cpp
@@ -8,9 +8,15 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/AssemblyAnnotationWriter.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#if LLVM_VERSION_GE(9, 0)
+#include "llvm/Passes/StandardInstrumentations.h"
+#endif
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -25,14 +31,18 @@
 #include "llvm/Transforms/Instrumentation.h"
 #if LLVM_VERSION_GE(9, 0)
 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
+#include "llvm/Support/TimeProfiler.h"
 #endif
 #if LLVM_VERSION_GE(8, 0)
 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 #endif
+#if LLVM_VERSION_GE(9, 0)
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+#endif
+#include "llvm/Transforms/Utils/NameAnonGlobals.h"
 
 using namespace llvm;
-using namespace llvm::legacy;
 
 typedef struct LLVMOpaquePass *LLVMPassRef;
 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
@@ -56,6 +66,23 @@ extern "C" void LLVMInitializePasses() {
   initializeTarget(Registry);
 }
 
+extern "C" void LLVMTimeTraceProfilerInitialize() {
+#if LLVM_VERSION_GE(9, 0)
+  timeTraceProfilerInitialize();
+#endif
+}
+
+extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
+#if LLVM_VERSION_GE(9, 0)
+  StringRef FN(FileName);
+  std::error_code EC;
+  raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
+
+  timeTraceProfilerWrite(OS);
+  timeTraceProfilerCleanup();
+#endif
+}
+
 enum class LLVMRustPassKind {
   Other,
   Function,
@@ -86,8 +113,9 @@ extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
 
 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
   const bool CompileKernel = false;
+  const bool UseAfterScope = true;
 
-  return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover));
+  return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
 }
 
 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
@@ -294,6 +322,34 @@ static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
   }
 }
 
+enum class LLVMRustPassBuilderOptLevel {
+  O0,
+  O1,
+  O2,
+  O3,
+  Os,
+  Oz,
+};
+
+static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
+  switch (Level) {
+  case LLVMRustPassBuilderOptLevel::O0:
+    return PassBuilder::O0;
+  case LLVMRustPassBuilderOptLevel::O1:
+    return PassBuilder::O1;
+  case LLVMRustPassBuilderOptLevel::O2:
+    return PassBuilder::O2;
+  case LLVMRustPassBuilderOptLevel::O3:
+    return PassBuilder::O3;
+  case LLVMRustPassBuilderOptLevel::Os:
+    return PassBuilder::Os;
+  case LLVMRustPassBuilderOptLevel::Oz:
+    return PassBuilder::Oz;
+  default:
+    report_fatal_error("Bad PassBuilderOptLevel.");
+  }
+}
+
 enum class LLVMRustRelocMode {
   Default,
   Static,
@@ -532,6 +588,18 @@ enum class LLVMRustFileType {
   ObjectFile,
 };
 
+#if LLVM_VERSION_GE(10, 0)
+static CodeGenFileType fromRust(LLVMRustFileType Type) {
+  switch (Type) {
+  case LLVMRustFileType::AssemblyFile:
+    return CGFT_AssemblyFile;
+  case LLVMRustFileType::ObjectFile:
+    return CGFT_ObjectFile;
+  default:
+    report_fatal_error("Bad FileType.");
+  }
+}
+#else
 static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
   switch (Type) {
   case LLVMRustFileType::AssemblyFile:
@@ -542,6 +610,7 @@ static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
     report_fatal_error("Bad FileType.");
   }
 }
+#endif
 
 extern "C" LLVMRustResult
 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
@@ -571,6 +640,275 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
   return LLVMRustResult::Success;
 }
 
+extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
+                                                      const char*,      // pass name
+                                                      const char*);     // IR name
+extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
+
+#if LLVM_VERSION_GE(9, 0)
+
+std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
+  if (any_isa<const Module *>(WrappedIr))
+    return any_cast<const Module *>(WrappedIr)->getName().str();
+  if (any_isa<const Function *>(WrappedIr))
+    return any_cast<const Function *>(WrappedIr)->getName().str();
+  if (any_isa<const Loop *>(WrappedIr))
+    return any_cast<const Loop *>(WrappedIr)->getName().str();
+  if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
+    return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
+  return "<UNKNOWN>";
+}
+
+
+void LLVMSelfProfileInitializeCallbacks(
+    PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
+    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
+  PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
+                                     StringRef Pass, llvm::Any Ir) {
+    std::string PassName = Pass.str();
+    std::string IrName = LLVMRustwrappedIrGetName(Ir);
+    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+    return true;
+  });
+
+  PIC.registerAfterPassCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+
+  PIC.registerAfterPassInvalidatedCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+
+  PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
+                                         StringRef Pass, llvm::Any Ir) {
+    std::string PassName = Pass.str();
+    std::string IrName = LLVMRustwrappedIrGetName(Ir);
+    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+  });
+
+  PIC.registerAfterAnalysisCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+}
+#endif
+
+enum class LLVMRustOptStage {
+  PreLinkNoLTO,
+  PreLinkThinLTO,
+  PreLinkFatLTO,
+  ThinLTO,
+  FatLTO,
+};
+
+struct LLVMRustSanitizerOptions {
+  bool SanitizeMemory;
+  bool SanitizeThread;
+  bool SanitizeAddress;
+  bool SanitizeRecover;
+  int SanitizeMemoryTrackOrigins;
+};
+
+extern "C" void
+LLVMRustOptimizeWithNewPassManager(
+    LLVMModuleRef ModuleRef,
+    LLVMTargetMachineRef TMRef,
+    LLVMRustPassBuilderOptLevel OptLevelRust,
+    LLVMRustOptStage OptStage,
+    bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
+    bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
+    bool DisableSimplifyLibCalls,
+    LLVMRustSanitizerOptions *SanitizerOptions,
+    const char *PGOGenPath, const char *PGOUsePath,
+    void* LlvmSelfProfiler,
+    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
+#if LLVM_VERSION_GE(9, 0)
+  Module *TheModule = unwrap(ModuleRef);
+  TargetMachine *TM = unwrap(TMRef);
+  PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
+
+  // FIXME: MergeFunctions is not supported by NewPM yet.
+  (void) MergeFunctions;
+
+  PipelineTuningOptions PTO;
+  PTO.LoopUnrolling = UnrollLoops;
+  PTO.LoopInterleaving = UnrollLoops;
+  PTO.LoopVectorization = LoopVectorize;
+  PTO.SLPVectorization = SLPVectorize;
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI;
+  SI.registerCallbacks(PIC);
+
+  if (LlvmSelfProfiler){
+    LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
+  }
+
+  Optional<PGOOptions> PGOOpt;
+  if (PGOGenPath) {
+    assert(!PGOUsePath);
+    PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
+  } else if (PGOUsePath) {
+    assert(!PGOGenPath);
+    PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
+  }
+
+  PassBuilder PB(TM, PTO, PGOOpt, &PIC);
+
+  // FIXME: We may want to expose this as an option.
+  bool DebugPassManager = false;
+  LoopAnalysisManager LAM(DebugPassManager);
+  FunctionAnalysisManager FAM(DebugPassManager);
+  CGSCCAnalysisManager CGAM(DebugPassManager);
+  ModuleAnalysisManager MAM(DebugPassManager);
+
+  FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
+
+  Triple TargetTriple(TheModule->getTargetTriple());
+  std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
+  if (DisableSimplifyLibCalls)
+    TLII->disableAllFunctions();
+  FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
+
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+
+  // We manually collect pipeline callbacks so we can apply them at O0, where the
+  // PassBuilder does not create a pipeline.
+  std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks;
+  std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
+      OptimizerLastEPCallbacks;
+
+  if (VerifyIR) {
+    PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) {
+        MPM.addPass(VerifierPass());
+    });
+  }
+
+  if (SanitizerOptions) {
+    if (SanitizerOptions->SanitizeMemory) {
+      MemorySanitizerOptions Options(
+          SanitizerOptions->SanitizeMemoryTrackOrigins,
+          SanitizerOptions->SanitizeRecover,
+          /*CompileKernel=*/false);
+#if LLVM_VERSION_GE(10, 0)
+      PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) {
+        MPM.addPass(MemorySanitizerPass(Options));
+      });
+#endif
+      OptimizerLastEPCallbacks.push_back(
+        [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
+          FPM.addPass(MemorySanitizerPass(Options));
+        }
+      );
+    }
+
+    if (SanitizerOptions->SanitizeThread) {
+#if LLVM_VERSION_GE(10, 0)
+      PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) {
+        MPM.addPass(ThreadSanitizerPass());
+      });
+#endif
+      OptimizerLastEPCallbacks.push_back(
+        [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
+          FPM.addPass(ThreadSanitizerPass());
+        }
+      );
+    }
+
+    if (SanitizerOptions->SanitizeAddress) {
+      PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) {
+        MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
+      });
+      OptimizerLastEPCallbacks.push_back(
+        [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
+          FPM.addPass(AddressSanitizerPass(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover,
+              /*UseAfterScope=*/true));
+        }
+      );
+      PipelineStartEPCallbacks.push_back(
+        [SanitizerOptions](ModulePassManager &MPM) {
+          MPM.addPass(ModuleAddressSanitizerPass(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover));
+        }
+      );
+    }
+  }
+
+  ModulePassManager MPM(DebugPassManager);
+  if (!NoPrepopulatePasses) {
+    if (OptLevel == PassBuilder::O0) {
+      for (const auto &C : PipelineStartEPCallbacks)
+        C(MPM);
+
+      if (!OptimizerLastEPCallbacks.empty()) {
+        FunctionPassManager FPM(DebugPassManager);
+        for (const auto &C : OptimizerLastEPCallbacks)
+          C(FPM, OptLevel);
+        MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+      }
+
+      MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false));
+
+#if LLVM_VERSION_GE(10, 0)
+      if (PGOOpt) {
+        PB.addPGOInstrPassesForO0(
+            MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
+            /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
+      }
+#endif
+    } else {
+      for (const auto &C : PipelineStartEPCallbacks)
+        PB.registerPipelineStartEPCallback(C);
+      for (const auto &C : OptimizerLastEPCallbacks)
+        PB.registerOptimizerLastEPCallback(C);
+
+      switch (OptStage) {
+      case LLVMRustOptStage::PreLinkNoLTO:
+        MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::PreLinkThinLTO:
+        MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::PreLinkFatLTO:
+        MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::ThinLTO:
+        // FIXME: Does it make sense to pass the ModuleSummaryIndex?
+        // It only seems to be needed for C++ specific optimizations.
+        MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
+        break;
+      case LLVMRustOptStage::FatLTO:
+        MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
+        break;
+      }
+    }
+  }
+
+  if (UseThinLTOBuffers) {
+    MPM.addPass(CanonicalizeAliasesPass());
+    MPM.addPass(NameAnonGlobalPass());
+  }
+
+  // Upgrade all calls to old intrinsics first.
+  for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
+    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
+
+  MPM.run(*TheModule, MAM);
+#else
+  // The new pass manager has been available for a long time,
+  // but we don't bother supporting it on old LLVM versions.
+  report_fatal_error("New pass manager only supported since LLVM 9");
+#endif
+}
 
 // Callback to demangle function name
 // Parameters:
@@ -849,7 +1187,11 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
                           int num_modules,
                           const char **preserved_symbols,
                           int num_symbols) {
+#if LLVM_VERSION_GE(10, 0)
+  auto Ret = std::make_unique<LLVMRustThinLTOData>();
+#else
   auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
+#endif
 
   // Load each module's summary and merge it into one combined index
   for (int i = 0; i < num_modules; i++) {
@@ -944,6 +1286,15 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
         ExportedGUIDs.insert(GUID);
     }
   }
+#if LLVM_VERSION_GE(10, 0)
+  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
+    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
+    return (ExportList != Ret->ExportLists.end() &&
+      ExportList->second.count(VI)) ||
+      ExportedGUIDs.count(VI.getGUID());
+  };
+  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
+#else
   auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
     const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
     return (ExportList != Ret->ExportLists.end() &&
@@ -951,6 +1302,7 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
       ExportedGUIDs.count(GUID);
   };
   thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
+#endif
 
   return Ret.release();
 }
@@ -1081,7 +1433,11 @@ struct LLVMRustThinLTOBuffer {
 
 extern "C" LLVMRustThinLTOBuffer*
 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
+#if LLVM_VERSION_GE(10, 0)
+  auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
+#else
   auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
+#endif
   {
     raw_string_ostream OS(Ret->data);
     {