10 files changed, 4340 insertions, 0 deletions

diff --git a/compiler/rustc_llvm/llvm-wrapper/.editorconfig b/compiler/rustc_llvm/llvm-wrapper/.editorconfig
new file mode 100644
index 00000000000..865cd45f708
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/.editorconfig
@@ -0,0 +1,6 @@
+[*.{h,cpp}]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = space
+indent_size = 2
diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp
new file mode 100644
index 00000000000..54fdc84c77d
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp
@@ -0,0 +1,217 @@
+#include "LLVMWrapper.h"
+
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Support/Path.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+struct RustArchiveMember {
+  const char *Filename;
+  const char *Name;
+  Archive::Child Child;
+
+  RustArchiveMember()
+      : Filename(nullptr), Name(nullptr),
+        Child(nullptr, nullptr, nullptr)
+  {
+  }
+  ~RustArchiveMember() {}
+};
+
+struct RustArchiveIterator {
+  bool First;
+  Archive::child_iterator Cur;
+  Archive::child_iterator End;
+  std::unique_ptr<Error> Err;
+
+  RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End,
+      std::unique_ptr<Error> Err)
+    : First(true),
+      Cur(Cur),
+      End(End),
+      Err(std::move(Err)) {}
+};
+
+enum class LLVMRustArchiveKind {
+  GNU,
+  BSD,
+  DARWIN,
+  COFF,
+  AIX_BIG,
+};
+
+static Archive::Kind fromRust(LLVMRustArchiveKind Kind) {
+  switch (Kind) {
+  case LLVMRustArchiveKind::GNU:
+    return Archive::K_GNU;
+  case LLVMRustArchiveKind::BSD:
+    return Archive::K_BSD;
+  case LLVMRustArchiveKind::DARWIN:
+    return Archive::K_DARWIN;
+  case LLVMRustArchiveKind::COFF:
+    return Archive::K_COFF;
+  case LLVMRustArchiveKind::AIX_BIG:
+    return Archive::K_AIXBIG;
+  default:
+    report_fatal_error("Bad ArchiveKind.");
+  }
+}
+
+typedef OwningBinary<Archive> *LLVMRustArchiveRef;
+typedef RustArchiveMember *LLVMRustArchiveMemberRef;
+typedef Archive::Child *LLVMRustArchiveChildRef;
+typedef Archive::Child const *LLVMRustArchiveChildConstRef;
+typedef RustArchiveIterator *LLVMRustArchiveIteratorRef;
+
+extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BufOr =
+      MemoryBuffer::getFile(Path, -1, false);
+  if (!BufOr) {
+    LLVMRustSetLastError(BufOr.getError().message().c_str());
+    return nullptr;
+  }
+
+  Expected<std::unique_ptr<Archive>> ArchiveOr =
+      Archive::create(BufOr.get()->getMemBufferRef());
+
+  if (!ArchiveOr) {
+    LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str());
+    return nullptr;
+  }
+
+  OwningBinary<Archive> *Ret = new OwningBinary<Archive>(
+      std::move(ArchiveOr.get()), std::move(BufOr.get()));
+
+  return Ret;
+}
+
+extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) {
+  delete RustArchive;
+}
+
+extern "C" LLVMRustArchiveIteratorRef
+LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) {
+  Archive *Archive = RustArchive->getBinary();
+  std::unique_ptr<Error> Err = std::make_unique<Error>(Error::success());
+  auto Cur = Archive->child_begin(*Err);
+  if (*Err) {
+    LLVMRustSetLastError(toString(std::move(*Err)).c_str());
+    return nullptr;
+  }
+  auto End = Archive->child_end();
+  return new RustArchiveIterator(Cur, End, std::move(Err));
+}
+
+extern "C" LLVMRustArchiveChildConstRef
+LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) {
+  if (RAI->Cur == RAI->End)
+    return nullptr;
+
+  // Advancing the iterator validates the next child, and this can
+  // uncover an error. LLVM requires that we check all Errors,
+  // so we only advance the iterator if we actually need to fetch
+  // the next child.
+  // This means we must not advance the iterator in the *first* call,
+  // but instead advance it *before* fetching the child in all later calls.
+  if (!RAI->First) {
+    ++RAI->Cur;
+    if (*RAI->Err) {
+      LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str());
+      return nullptr;
+    }
+  } else {
+    RAI->First = false;
+  }
+
+  if (RAI->Cur == RAI->End)
+    return nullptr;
+
+  const Archive::Child &Child = *RAI->Cur.operator->();
+  Archive::Child *Ret = new Archive::Child(Child);
+
+  return Ret;
+}
+
+extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) {
+  delete Child;
+}
+
+extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) {
+  delete RAI;
+}
+
+extern "C" const char *
+LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) {
+  Expected<StringRef> NameOrErr = Child->getName();
+  if (!NameOrErr) {
+    // rustc_codegen_llvm currently doesn't use this error string, but it might be
+    // useful in the future, and in the mean time this tells LLVM that the
+    // error was not ignored and that it shouldn't abort the process.
+    LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str());
+    return nullptr;
+  }
+  StringRef Name = NameOrErr.get();
+  *Size = Name.size();
+  return Name.data();
+}
+
+extern "C" LLVMRustArchiveMemberRef
+LLVMRustArchiveMemberNew(char *Filename, char *Name,
+                         LLVMRustArchiveChildRef Child) {
+  RustArchiveMember *Member = new RustArchiveMember;
+  Member->Filename = Filename;
+  Member->Name = Name;
+  if (Child)
+    Member->Child = *Child;
+  return Member;
+}
+
+extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) {
+  delete Member;
+}
+
+extern "C" LLVMRustResult
+LLVMRustWriteArchive(char *Dst, size_t NumMembers,
+                     const LLVMRustArchiveMemberRef *NewMembers,
+                     bool WriteSymbtab, LLVMRustArchiveKind RustKind) {
+
+  std::vector<NewArchiveMember> Members;
+  auto Kind = fromRust(RustKind);
+
+  for (size_t I = 0; I < NumMembers; I++) {
+    auto Member = NewMembers[I];
+    assert(Member->Name);
+    if (Member->Filename) {
+      Expected<NewArchiveMember> MOrErr =
+          NewArchiveMember::getFile(Member->Filename, true);
+      if (!MOrErr) {
+        LLVMRustSetLastError(toString(MOrErr.takeError()).c_str());
+        return LLVMRustResult::Failure;
+      }
+      MOrErr->MemberName = sys::path::filename(MOrErr->MemberName);
+      Members.push_back(std::move(*MOrErr));
+    } else {
+      Expected<NewArchiveMember> MOrErr =
+          NewArchiveMember::getOldMember(Member->Child, true);
+      if (!MOrErr) {
+        LLVMRustSetLastError(toString(MOrErr.takeError()).c_str());
+        return LLVMRustResult::Failure;
+      }
+      Members.push_back(std::move(*MOrErr));
+    }
+  }
+
+#if LLVM_VERSION_LT(18, 0)
+  auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false);
+#else
+  auto SymtabMode = WriteSymbtab ? SymtabWritingMode::NormalSymtab : SymtabWritingMode::NoSymtab;
+  auto Result = writeArchive(Dst, Members, SymtabMode, Kind, true, false);
+#endif
+  if (!Result)
+    return LLVMRustResult::Success;
+  LLVMRustSetLastError(toString(std::move(Result)).c_str());
+
+  return LLVMRustResult::Failure;
+}
diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
new file mode 100644
index 00000000000..d61ec0b641c
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
@@ -0,0 +1,206 @@
+#include "LLVMWrapper.h"
+#include "llvm/ProfileData/Coverage/CoverageMapping.h"
+#include "llvm/ProfileData/Coverage/CoverageMappingWriter.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ADT/ArrayRef.h"
+
+#include <iostream>
+
+using namespace llvm;
+
+// FFI equivalent of enum `llvm::coverage::Counter::CounterKind`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L97-L99
+enum class LLVMRustCounterKind {
+  Zero = 0,
+  CounterValueReference = 1,
+  Expression = 2,
+};
+
+// FFI equivalent of struct `llvm::coverage::Counter`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L94-L149
+struct LLVMRustCounter {
+  LLVMRustCounterKind CounterKind;
+  uint32_t ID;
+};
+
+static coverage::Counter fromRust(LLVMRustCounter Counter) {
+  switch (Counter.CounterKind) {
+  case LLVMRustCounterKind::Zero:
+    return coverage::Counter::getZero();
+  case LLVMRustCounterKind::CounterValueReference:
+    return coverage::Counter::getCounter(Counter.ID);
+  case LLVMRustCounterKind::Expression:
+    return coverage::Counter::getExpression(Counter.ID);
+  }
+  report_fatal_error("Bad LLVMRustCounterKind!");
+}
+
+// FFI equivalent of enum `llvm::coverage::CounterMappingRegion::RegionKind`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L213-L234
+enum class LLVMRustCounterMappingRegionKind {
+  CodeRegion = 0,
+  ExpansionRegion = 1,
+  SkippedRegion = 2,
+  GapRegion = 3,
+  BranchRegion = 4,
+};
+
+static coverage::CounterMappingRegion::RegionKind
+fromRust(LLVMRustCounterMappingRegionKind Kind) {
+  switch (Kind) {
+  case LLVMRustCounterMappingRegionKind::CodeRegion:
+    return coverage::CounterMappingRegion::CodeRegion;
+  case LLVMRustCounterMappingRegionKind::ExpansionRegion:
+    return coverage::CounterMappingRegion::ExpansionRegion;
+  case LLVMRustCounterMappingRegionKind::SkippedRegion:
+    return coverage::CounterMappingRegion::SkippedRegion;
+  case LLVMRustCounterMappingRegionKind::GapRegion:
+    return coverage::CounterMappingRegion::GapRegion;
+  case LLVMRustCounterMappingRegionKind::BranchRegion:
+    return coverage::CounterMappingRegion::BranchRegion;
+  }
+  report_fatal_error("Bad LLVMRustCounterMappingRegionKind!");
+}
+
+// FFI equivalent of struct `llvm::coverage::CounterMappingRegion`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304
+struct LLVMRustCounterMappingRegion {
+  LLVMRustCounter Count;
+  LLVMRustCounter FalseCount;
+  uint32_t FileID;
+  uint32_t ExpandedFileID;
+  uint32_t LineStart;
+  uint32_t ColumnStart;
+  uint32_t LineEnd;
+  uint32_t ColumnEnd;
+  LLVMRustCounterMappingRegionKind Kind;
+};
+
+// FFI equivalent of enum `llvm::coverage::CounterExpression::ExprKind`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L154
+enum class LLVMRustCounterExprKind {
+  Subtract = 0,
+  Add = 1,
+};
+
+// FFI equivalent of struct `llvm::coverage::CounterExpression`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L160
+struct LLVMRustCounterExpression {
+  LLVMRustCounterExprKind Kind;
+  LLVMRustCounter LHS;
+  LLVMRustCounter RHS;
+};
+
+static coverage::CounterExpression::ExprKind
+fromRust(LLVMRustCounterExprKind Kind) {
+  switch (Kind) {
+  case LLVMRustCounterExprKind::Subtract:
+    return coverage::CounterExpression::Subtract;
+  case LLVMRustCounterExprKind::Add:
+    return coverage::CounterExpression::Add;
+  }
+  report_fatal_error("Bad LLVMRustCounterExprKind!");
+}
+
+extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer(
+    const char *const Filenames[],
+    size_t FilenamesLen,
+    const size_t *const Lengths,
+    size_t LengthsLen,
+    RustStringRef BufferOut) {
+  if (FilenamesLen != LengthsLen) {
+    report_fatal_error(
+        "Mismatched lengths in LLVMRustCoverageWriteFilenamesSectionToBuffer");
+  }
+
+  SmallVector<std::string,32> FilenameRefs;
+  FilenameRefs.reserve(FilenamesLen);
+  for (size_t i = 0; i < FilenamesLen; i++) {
+    FilenameRefs.emplace_back(Filenames[i], Lengths[i]);
+  }
+  auto FilenamesWriter =
+      coverage::CoverageFilenamesSectionWriter(ArrayRef<std::string>(FilenameRefs));
+  RawRustStringOstream OS(BufferOut);
+  FilenamesWriter.write(OS);
+}
+
+extern "C" void LLVMRustCoverageWriteMappingToBuffer(
+    const unsigned *VirtualFileMappingIDs,
+    unsigned NumVirtualFileMappingIDs,
+    const LLVMRustCounterExpression *RustExpressions,
+    unsigned NumExpressions,
+    const LLVMRustCounterMappingRegion *RustMappingRegions,
+    unsigned NumMappingRegions,
+    RustStringRef BufferOut) {
+  // Convert from FFI representation to LLVM representation.
+  SmallVector<coverage::CounterMappingRegion, 0> MappingRegions;
+  MappingRegions.reserve(NumMappingRegions);
+  for (const auto &Region : ArrayRef<LLVMRustCounterMappingRegion>(
+           RustMappingRegions, NumMappingRegions)) {
+    MappingRegions.emplace_back(
+        fromRust(Region.Count), fromRust(Region.FalseCount),
+        Region.FileID, Region.ExpandedFileID,
+        Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd,
+        fromRust(Region.Kind));
+  }
+
+  std::vector<coverage::CounterExpression> Expressions;
+  Expressions.reserve(NumExpressions);
+  for (const auto &Expression :
+       ArrayRef<LLVMRustCounterExpression>(RustExpressions, NumExpressions)) {
+    Expressions.emplace_back(fromRust(Expression.Kind),
+                             fromRust(Expression.LHS),
+                             fromRust(Expression.RHS));
+  }
+
+  auto CoverageMappingWriter = coverage::CoverageMappingWriter(
+      ArrayRef<unsigned>(VirtualFileMappingIDs, NumVirtualFileMappingIDs),
+      Expressions,
+      MappingRegions);
+  RawRustStringOstream OS(BufferOut);
+  CoverageMappingWriter.write(OS);
+}
+
+extern "C" LLVMValueRef LLVMRustCoverageCreatePGOFuncNameVar(
+    LLVMValueRef F,
+    const char *FuncName,
+    size_t FuncNameLen) {
+  StringRef FuncNameRef(FuncName, FuncNameLen);
+  return wrap(createPGOFuncNameVar(*cast<Function>(unwrap(F)), FuncNameRef));
+}
+
+extern "C" uint64_t LLVMRustCoverageHashByteArray(
+    const char *Bytes,
+    size_t NumBytes) {
+  StringRef StrRef(Bytes, NumBytes);
+  return IndexedInstrProf::ComputeHash(StrRef);
+}
+
+static void WriteSectionNameToString(LLVMModuleRef M,
+                                     InstrProfSectKind SK,
+                                     RustStringRef Str) {
+  Triple TargetTriple(unwrap(M)->getTargetTriple());
+  auto name = getInstrProfSectionName(SK, TargetTriple.getObjectFormat());
+  RawRustStringOstream OS(Str);
+  OS << name;
+}
+
+extern "C" void LLVMRustCoverageWriteMapSectionNameToString(LLVMModuleRef M,
+                                                            RustStringRef Str) {
+  WriteSectionNameToString(M, IPSK_covmap, Str);
+}
+
+extern "C" void LLVMRustCoverageWriteFuncSectionNameToString(LLVMModuleRef M,
+                                                             RustStringRef Str) {
+  WriteSectionNameToString(M, IPSK_covfun, Str);
+}
+
+extern "C" void LLVMRustCoverageWriteMappingVarNameToString(RustStringRef Str) {
+  auto name = getCoverageMappingVarName();
+  RawRustStringOstream OS(Str);
+  OS << name;
+}
+
+extern "C" uint32_t LLVMRustCoverageMappingVersion() {
+  return coverage::CovMapVersion::Version6;
+}
diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
new file mode 100644
index 00000000000..5bfffc5d911
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
@@ -0,0 +1,122 @@
+#include "SuppressLLVMWarnings.h"
+
+#include "llvm-c/BitReader.h"
+#include "llvm-c/Core.h"
+#include "llvm-c/Object.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/Lint.h"
+#include "llvm/Analysis/Passes.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/Timer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Scalar.h"
+
+#define LLVM_VERSION_GE(major, minor)                                          \
+  (LLVM_VERSION_MAJOR > (major) ||                                             \
+   LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR >= (minor))
+
+#define LLVM_VERSION_LT(major, minor) (!LLVM_VERSION_GE((major), (minor)))
+
+#include "llvm/IR/LegacyPassManager.h"
+
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+
+#include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/Linker/Linker.h"
+
+#include "llvm/TargetParser/Triple.h"
+
+extern "C" void LLVMRustSetLastError(const char *);
+
+enum class LLVMRustResult { Success, Failure };
+
+enum LLVMRustAttribute {
+  AlwaysInline = 0,
+  ByVal = 1,
+  Cold = 2,
+  InlineHint = 3,
+  MinSize = 4,
+  Naked = 5,
+  NoAlias = 6,
+  NoCapture = 7,
+  NoInline = 8,
+  NonNull = 9,
+  NoRedZone = 10,
+  NoReturn = 11,
+  NoUnwind = 12,
+  OptimizeForSize = 13,
+  ReadOnly = 14,
+  SExt = 15,
+  StructRet = 16,
+  UWTable = 17,
+  ZExt = 18,
+  InReg = 19,
+  SanitizeThread = 20,
+  SanitizeAddress = 21,
+  SanitizeMemory = 22,
+  NonLazyBind = 23,
+  OptimizeNone = 24,
+  ReturnsTwice = 25,
+  ReadNone = 26,
+  SanitizeHWAddress = 28,
+  WillReturn = 29,
+  StackProtectReq = 30,
+  StackProtectStrong = 31,
+  StackProtect = 32,
+  NoUndef = 33,
+  SanitizeMemTag = 34,
+  NoCfCheck = 35,
+  ShadowCallStack = 36,
+  AllocSize = 37,
+  AllocatedPointer = 38,
+  AllocAlign = 39,
+  SanitizeSafeStack = 40,
+  FnRetThunkExtern = 41,
+};
+
+typedef struct OpaqueRustString *RustStringRef;
+typedef struct LLVMOpaqueTwine *LLVMTwineRef;
+typedef struct LLVMOpaqueSMDiagnostic *LLVMSMDiagnosticRef;
+
+extern "C" void LLVMRustStringWriteImpl(RustStringRef Str, const char *Ptr,
+                                        size_t Size);
+
+class RawRustStringOstream : public llvm::raw_ostream {
+  RustStringRef Str;
+  uint64_t Pos;
+
+  void write_impl(const char *Ptr, size_t Size) override {
+    LLVMRustStringWriteImpl(Str, Ptr, Size);
+    Pos += Size;
+  }
+
+  uint64_t current_pos() const override { return Pos; }
+
+public:
+  explicit RawRustStringOstream(RustStringRef Str) : Str(Str), Pos(0) {}
+
+  ~RawRustStringOstream() {
+    // LLVM requires this.
+    flush();
+  }
+};
diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
new file mode 100644
index 00000000000..533df0f75f8
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
@@ -0,0 +1,49 @@
+#include "SuppressLLVMWarnings.h"
+#include "llvm/Linker/Linker.h"
+
+#include "LLVMWrapper.h"
+
+using namespace llvm;
+
+struct RustLinker {
+  Linker L;
+  LLVMContext &Ctx;
+
+  RustLinker(Module &M) :
+    L(M),
+    Ctx(M.getContext())
+  {}
+};
+
+extern "C" RustLinker*
+LLVMRustLinkerNew(LLVMModuleRef DstRef) {
+  Module *Dst = unwrap(DstRef);
+
+  return new RustLinker(*Dst);
+}
+
+extern "C" void
+LLVMRustLinkerFree(RustLinker *L) {
+  delete L;
+}
+
+extern "C" bool
+LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) {
+  std::unique_ptr<MemoryBuffer> Buf =
+      MemoryBuffer::getMemBufferCopy(StringRef(BC, Len));
+
+  Expected<std::unique_ptr<Module>> SrcOrError =
+      llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx);
+  if (!SrcOrError) {
+    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
+    return false;
+  }
+
+  auto Src = std::move(*SrcOrError);
+
+  if (L->L.linkInModule(std::move(Src))) {
+    LLVMRustSetLastError("");
+    return false;
+  }
+  return true;
+}
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
new file mode 100644
index 00000000000..55e1c84c8a2
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -0,0 +1,1592 @@
+#include <stdio.h>
+
+#include <cstddef>
+#include <iomanip>
+#include <vector>
+#include <set>
+
+#include "LLVMWrapper.h"
+
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/AutoUpgrade.h"
+#include "llvm/IR/AssemblyAnnotationWriter.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/IRObjectFile.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/PassPlugin.h"
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/FileSystem.h"
+#if LLVM_VERSION_GE(17, 0)
+#include "llvm/Support/VirtualFileSystem.h"
+#endif
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/IPO/AlwaysInliner.h"
+#include "llvm/Transforms/IPO/FunctionImport.h"
+#include "llvm/Transforms/IPO/Internalize.h"
+#include "llvm/Transforms/IPO/LowerTypeTests.h"
+#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
+#include "llvm/Transforms/Utils/AddDiscriminators.h"
+#include "llvm/Transforms/Utils/FunctionImportUtils.h"
+#include "llvm/LTO/LTO.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
+#include "llvm/Support/TimeProfiler.h"
+#include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
+#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
+#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
+#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
+#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+#include "llvm/Transforms/Utils/NameAnonGlobals.h"
+#include "llvm/Transforms/Utils.h"
+
+using namespace llvm;
+
+static codegen::RegisterCodeGenFlags CGF;
+
+typedef struct LLVMOpaquePass *LLVMPassRef;
+typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
+
+DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
+DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
+
+extern "C" void LLVMRustTimeTraceProfilerInitialize() {
+  timeTraceProfilerInitialize(
+      /* TimeTraceGranularity */ 0,
+      /* ProcName */ "rustc");
+}
+
+extern "C" void LLVMRustTimeTraceProfilerFinishThread() {
+  timeTraceProfilerFinishThread();
+}
+
+extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) {
+  StringRef FN(FileName);
+  std::error_code EC;
+  raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
+
+  timeTraceProfilerWrite(OS);
+  timeTraceProfilerCleanup();
+}
+
+#ifdef LLVM_COMPONENT_X86
+#define SUBTARGET_X86 SUBTARGET(X86)
+#else
+#define SUBTARGET_X86
+#endif
+
+#ifdef LLVM_COMPONENT_ARM
+#define SUBTARGET_ARM SUBTARGET(ARM)
+#else
+#define SUBTARGET_ARM
+#endif
+
+#ifdef LLVM_COMPONENT_AARCH64
+#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
+#else
+#define SUBTARGET_AARCH64
+#endif
+
+#ifdef LLVM_COMPONENT_AVR
+#define SUBTARGET_AVR SUBTARGET(AVR)
+#else
+#define SUBTARGET_AVR
+#endif
+
+#ifdef LLVM_COMPONENT_M68k
+#define SUBTARGET_M68K SUBTARGET(M68k)
+#else
+#define SUBTARGET_M68K
+#endif
+
+#ifdef LLVM_COMPONENT_CSKY
+#define SUBTARGET_CSKY SUBTARGET(CSKY)
+#else
+#define SUBTARGET_CSKY
+#endif
+
+#ifdef LLVM_COMPONENT_MIPS
+#define SUBTARGET_MIPS SUBTARGET(Mips)
+#else
+#define SUBTARGET_MIPS
+#endif
+
+#ifdef LLVM_COMPONENT_POWERPC
+#define SUBTARGET_PPC SUBTARGET(PPC)
+#else
+#define SUBTARGET_PPC
+#endif
+
+#ifdef LLVM_COMPONENT_SYSTEMZ
+#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
+#else
+#define SUBTARGET_SYSTEMZ
+#endif
+
+#ifdef LLVM_COMPONENT_MSP430
+#define SUBTARGET_MSP430 SUBTARGET(MSP430)
+#else
+#define SUBTARGET_MSP430
+#endif
+
+#ifdef LLVM_COMPONENT_RISCV
+#define SUBTARGET_RISCV SUBTARGET(RISCV)
+#else
+#define SUBTARGET_RISCV
+#endif
+
+#ifdef LLVM_COMPONENT_SPARC
+#define SUBTARGET_SPARC SUBTARGET(Sparc)
+#else
+#define SUBTARGET_SPARC
+#endif
+
+#ifdef LLVM_COMPONENT_HEXAGON
+#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
+#else
+#define SUBTARGET_HEXAGON
+#endif
+
+#ifdef LLVM_COMPONENT_LOONGARCH
+#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch)
+#else
+#define SUBTARGET_LOONGARCH
+#endif
+
+#define GEN_SUBTARGETS                                                         \
+  SUBTARGET_X86                                                                \
+  SUBTARGET_ARM                                                                \
+  SUBTARGET_AARCH64                                                            \
+  SUBTARGET_AVR                                                                \
+  SUBTARGET_M68K                                                               \
+  SUBTARGET_CSKY                                                               \
+  SUBTARGET_MIPS                                                               \
+  SUBTARGET_PPC                                                                \
+  SUBTARGET_SYSTEMZ                                                            \
+  SUBTARGET_MSP430                                                             \
+  SUBTARGET_SPARC                                                              \
+  SUBTARGET_HEXAGON                                                            \
+  SUBTARGET_RISCV                                                              \
+  SUBTARGET_LOONGARCH                                                          \
+
+#define SUBTARGET(x)                                                           \
+  namespace llvm {                                                             \
+  extern const SubtargetFeatureKV x##FeatureKV[];                              \
+  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
+  }
+
+GEN_SUBTARGETS
+#undef SUBTARGET
+
+extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
+                                   const char *Feature) {
+  TargetMachine *Target = unwrap(TM);
+  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
+  return MCInfo->checkFeatures(std::string("+") + Feature);
+}
+
+enum class LLVMRustCodeModel {
+  Tiny,
+  Small,
+  Kernel,
+  Medium,
+  Large,
+  None,
+};
+
+static std::optional<CodeModel::Model>
+fromRust(LLVMRustCodeModel Model) {
+  switch (Model) {
+  case LLVMRustCodeModel::Tiny:
+    return CodeModel::Tiny;
+  case LLVMRustCodeModel::Small:
+    return CodeModel::Small;
+  case LLVMRustCodeModel::Kernel:
+    return CodeModel::Kernel;
+  case LLVMRustCodeModel::Medium:
+    return CodeModel::Medium;
+  case LLVMRustCodeModel::Large:
+    return CodeModel::Large;
+  case LLVMRustCodeModel::None:
+    return std::nullopt;
+  default:
+    report_fatal_error("Bad CodeModel.");
+  }
+}
+
+enum class LLVMRustCodeGenOptLevel {
+  None,
+  Less,
+  Default,
+  Aggressive,
+};
+
+#if LLVM_VERSION_GE(18, 0)
+  using CodeGenOptLevelEnum = llvm::CodeGenOptLevel;
+#else
+  using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level;
+#endif
+
+static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) {
+  switch (Level) {
+  case LLVMRustCodeGenOptLevel::None:
+    return CodeGenOptLevelEnum::None;
+  case LLVMRustCodeGenOptLevel::Less:
+    return CodeGenOptLevelEnum::Less;
+  case LLVMRustCodeGenOptLevel::Default:
+    return CodeGenOptLevelEnum::Default;
+  case LLVMRustCodeGenOptLevel::Aggressive:
+    return CodeGenOptLevelEnum::Aggressive;
+  default:
+    report_fatal_error("Bad CodeGenOptLevel.");
+  }
+}
+
+enum class LLVMRustPassBuilderOptLevel {
+  O0,
+  O1,
+  O2,
+  O3,
+  Os,
+  Oz,
+};
+
+static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
+  switch (Level) {
+  case LLVMRustPassBuilderOptLevel::O0:
+    return OptimizationLevel::O0;
+  case LLVMRustPassBuilderOptLevel::O1:
+    return OptimizationLevel::O1;
+  case LLVMRustPassBuilderOptLevel::O2:
+    return OptimizationLevel::O2;
+  case LLVMRustPassBuilderOptLevel::O3:
+    return OptimizationLevel::O3;
+  case LLVMRustPassBuilderOptLevel::Os:
+    return OptimizationLevel::Os;
+  case LLVMRustPassBuilderOptLevel::Oz:
+    return OptimizationLevel::Oz;
+  default:
+    report_fatal_error("Bad PassBuilderOptLevel.");
+  }
+}
+
+enum class LLVMRustRelocModel {
+  Static,
+  PIC,
+  DynamicNoPic,
+  ROPI,
+  RWPI,
+  ROPIRWPI,
+};
+
+static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
+  switch (RustReloc) {
+  case LLVMRustRelocModel::Static:
+    return Reloc::Static;
+  case LLVMRustRelocModel::PIC:
+    return Reloc::PIC_;
+  case LLVMRustRelocModel::DynamicNoPic:
+    return Reloc::DynamicNoPIC;
+  case LLVMRustRelocModel::ROPI:
+    return Reloc::ROPI;
+  case LLVMRustRelocModel::RWPI:
+    return Reloc::RWPI;
+  case LLVMRustRelocModel::ROPIRWPI:
+    return Reloc::ROPI_RWPI;
+  }
+  report_fatal_error("Bad RelocModel.");
+}
+
+/// getLongestEntryLength - Return the length of the longest entry in the table.
+template<typename KV>
+static size_t getLongestEntryLength(ArrayRef<KV> Table) {
+  size_t MaxLen = 0;
+  for (auto &I : Table)
+    MaxLen = std::max(MaxLen, std::strlen(I.Key));
+  return MaxLen;
+}
+
+using PrintBackendInfo = void(void*, const char* Data, size_t Len);
+
+extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM,
+                                        const char* TargetCPU,
+                                        PrintBackendInfo Print,
+                                        void* Out) {
+  const TargetMachine *Target = unwrap(TM);
+  const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch();
+  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
+
+  std::ostringstream Buf;
+
+#if LLVM_VERSION_GE(17, 0)
+  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
+  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getAllProcessorDescriptions();
+#else
+  Buf << "Full target CPU help is not supported by this LLVM version.\n\n";
+  SubtargetSubTypeKV TargetCPUKV = { TargetCPU, {{}}, {{}} };
+  const ArrayRef<SubtargetSubTypeKV> CPUTable = TargetCPUKV;
+#endif
+  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
+
+  Buf << "Available CPUs for this target:\n";
+  // Don't print the "native" entry when the user specifies --target with a
+  // different arch since that could be wrong or misleading.
+  if (HostArch == TargetArch) {
+    MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native"));
+    const StringRef HostCPU = sys::getHostCPUName();
+    Buf << "    " << std::left << std::setw(MaxCPULen) << "native"
+        << " - Select the CPU of the current host "
+           "(currently " << HostCPU.str() << ").\n";
+  }
+  for (auto &CPU : CPUTable) {
+    // Compare cpu against current target to label the default
+    if (strcmp(CPU.Key, TargetCPU) == 0) {
+      Buf << "    " << std::left << std::setw(MaxCPULen) << CPU.Key
+          << " - This is the default target CPU for the current build target "
+             "(currently " << Target->getTargetTriple().str() << ").";
+    }
+    else {
+      Buf << "    " << CPU.Key;
+    }
+    Buf << "\n";
+  }
+
+  const auto &BufString = Buf.str();
+  Print(Out, BufString.data(), BufString.size());
+}
+
+extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
+#ifdef LLVM_RUSTLLVM
+  const TargetMachine *Target = unwrap(TM);
+  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
+  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
+  return FeatTable.size();
+#else
+  return 0;
+#endif
+}
+
+extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
+                                         const char** Feature, const char** Desc) {
+#ifdef LLVM_RUSTLLVM
+  const TargetMachine *Target = unwrap(TM);
+  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
+  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
+  const SubtargetFeatureKV Feat = FeatTable[Index];
+  *Feature = Feat.Key;
+  *Desc = Feat.Desc;
+#endif
+}
+
+extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
+  StringRef Name = sys::getHostCPUName();
+  *len = Name.size();
+  return Name.data();
+}
+
+extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
+    const char *TripleStr, const char *CPU, const char *Feature,
+    const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
+    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
+    bool FunctionSections,
+    bool DataSections,
+    bool UniqueSectionNames,
+    bool TrapUnreachable,
+    bool Singlethread,
+    bool AsmComments,
+    bool EmitStackSizeSection,
+    bool RelaxELFRelocations,
+    bool UseInitArray,
+    const char *SplitDwarfFile,
+    const char *OutputObjFile,
+    const char *DebugInfoCompression,
+    bool UseEmulatedTls,
+    const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
+
+  auto OptLevel = fromRust(RustOptLevel);
+  auto RM = fromRust(RustReloc);
+  auto CM = fromRust(RustCM);
+
+  std::string Error;
+  Triple Trip(Triple::normalize(TripleStr));
+  const llvm::Target *TheTarget =
+      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
+  if (TheTarget == nullptr) {
+    LLVMRustSetLastError(Error.c_str());
+    return nullptr;
+  }
+
+  TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip);
+
+  Options.FloatABIType = FloatABI::Default;
+  if (UseSoftFloat) {
+    Options.FloatABIType = FloatABI::Soft;
+  }
+  Options.DataSections = DataSections;
+  Options.FunctionSections = FunctionSections;
+  Options.UniqueSectionNames = UniqueSectionNames;
+  Options.MCOptions.AsmVerbose = AsmComments;
+  Options.MCOptions.PreserveAsmComments = AsmComments;
+  Options.MCOptions.ABIName = ABIStr;
+  if (SplitDwarfFile) {
+      Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
+  }
+  if (OutputObjFile) {
+      Options.ObjectFilenameForDebug = OutputObjFile;
+  }
+  if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) {
+    Options.CompressDebugSections = DebugCompressionType::Zlib;
+  } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) {
+    Options.CompressDebugSections = DebugCompressionType::Zstd;
+  } else if (!strcmp("none", DebugInfoCompression)) {
+    Options.CompressDebugSections = DebugCompressionType::None;
+  }
+
+  Options.RelaxELFRelocations = RelaxELFRelocations;
+  Options.UseInitArray = UseInitArray;
+
+#if LLVM_VERSION_LT(17, 0)
+  Options.ExplicitEmulatedTLS = true;
+#endif
+  Options.EmulatedTLS = UseEmulatedTls;
+
+  if (TrapUnreachable) {
+    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
+    // This limits the extent of possible undefined behavior in some cases, as
+    // it prevents control flow from "falling through" into whatever code
+    // happens to be laid out next in memory.
+    Options.TrapUnreachable = true;
+  }
+
+  if (Singlethread) {
+    Options.ThreadModel = ThreadModel::Single;
+  }
+
+  Options.EmitStackSizeSection = EmitStackSizeSection;
+
+
+  if (ArgsCstrBuff != nullptr)
+  {
+    int buffer_offset = 0;
+    assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0');
+
+    const size_t arg0_len = std::strlen(ArgsCstrBuff);
+    char* arg0 = new char[arg0_len + 1];
+    memcpy(arg0, ArgsCstrBuff, arg0_len);
+    arg0[arg0_len] = '\0';
+    buffer_offset += arg0_len + 1;
+
+    const int num_cmd_arg_strings =
+      std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0');
+
+    std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings];
+    for (int i = 0; i < num_cmd_arg_strings; ++i)
+    {
+      assert(buffer_offset < ArgsCstrBuffLen);
+      const int len = std::strlen(ArgsCstrBuff + buffer_offset);
+      cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len);
+      buffer_offset += len + 1;
+    }
+
+    assert(buffer_offset == ArgsCstrBuffLen);
+
+    Options.MCOptions.Argv0 = arg0;
+    Options.MCOptions.CommandLineArgs =
+      llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings);
+  }
+
+  TargetMachine *TM = TheTarget->createTargetMachine(
+      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
+  return wrap(TM);
+}
+
+extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
+
+  MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions;
+  delete[] MCOptions.Argv0;
+  delete[] MCOptions.CommandLineArgs.data();
+
+  delete unwrap(TM);
+}
+
+// Unfortunately, the LLVM C API doesn't provide a way to create the
+// TargetLibraryInfo pass, so we use this method to do so.
+extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M) {
+  Triple TargetTriple(unwrap(M)->getTargetTriple());
+  TargetLibraryInfoImpl TLII(TargetTriple);
+  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
+}
+
+extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
+  // Initializing the command-line options more than once is not allowed. So,
+  // check if they've already been initialized. (This could happen if we're
+  // being called from rustpkg, for example). If the arguments change, then
+  // that's just kinda unfortunate.
+  static bool Initialized = false;
+  if (Initialized)
+    return;
+  Initialized = true;
+  cl::ParseCommandLineOptions(Argc, Argv);
+}
+
+enum class LLVMRustFileType {
+  AssemblyFile,
+  ObjectFile,
+};
+
+static CodeGenFileType fromRust(LLVMRustFileType Type) {
+  switch (Type) {
+  case LLVMRustFileType::AssemblyFile:
+#if LLVM_VERSION_GE(18, 0)
+    return CodeGenFileType::AssemblyFile;
+#else
+    return CGFT_AssemblyFile;
+#endif
+  case LLVMRustFileType::ObjectFile:
+#if LLVM_VERSION_GE(18, 0)
+    return CodeGenFileType::ObjectFile;
+#else
+    return CGFT_ObjectFile;
+#endif
+  default:
+    report_fatal_error("Bad FileType.");
+  }
+}
+
+extern "C" LLVMRustResult
+LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
+                        LLVMModuleRef M, const char *Path, const char *DwoPath,
+                        LLVMRustFileType RustFileType) {
+  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
+  auto FileType = fromRust(RustFileType);
+
+  std::string ErrorInfo;
+  std::error_code EC;
+  raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
+  if (EC)
+    ErrorInfo = EC.message();
+  if (ErrorInfo != "") {
+    LLVMRustSetLastError(ErrorInfo.c_str());
+    return LLVMRustResult::Failure;
+  }
+
+  buffer_ostream BOS(OS);
+  if (DwoPath) {
+    raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
+    EC.clear();
+    if (EC)
+        ErrorInfo = EC.message();
+    if (ErrorInfo != "") {
+      LLVMRustSetLastError(ErrorInfo.c_str());
+      return LLVMRustResult::Failure;
+    }
+    buffer_ostream DBOS(DOS);
+    unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
+    PM->run(*unwrap(M));
+  } else {
+    unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
+    PM->run(*unwrap(M));
+  }
+
+  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
+  // stream (OS), so the only real safe place to delete this is here? Don't we
+  // wish this was written in Rust?
+  LLVMDisposePassManager(PMR);
+  return LLVMRustResult::Success;
+}
+
+extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
+                                                      const char*,      // pass name
+                                                      const char*);     // IR name
+extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
+
+std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
+  if (const auto *Cast = any_cast<const Module *>(&WrappedIr))
+    return (*Cast)->getName().str();
+  if (const auto *Cast = any_cast<const Function *>(&WrappedIr))
+    return (*Cast)->getName().str();
+  if (const auto *Cast = any_cast<const Loop *>(&WrappedIr))
+    return (*Cast)->getName().str();
+  if (const auto *Cast = any_cast<const LazyCallGraph::SCC *>(&WrappedIr))
+    return (*Cast)->getName();
+  return "<UNKNOWN>";
+}
+
+
+void LLVMSelfProfileInitializeCallbacks(
+    PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
+    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
+  PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
+                                           StringRef Pass, llvm::Any Ir) {
+    std::string PassName = Pass.str();
+    std::string IrName = LLVMRustwrappedIrGetName(Ir);
+    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+  });
+
+  PIC.registerAfterPassCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
+                                            const PreservedAnalyses &Preserved) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+
+  PIC.registerAfterPassInvalidatedCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+
+  PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
+                                         StringRef Pass, llvm::Any Ir) {
+    std::string PassName = Pass.str();
+    std::string IrName = LLVMRustwrappedIrGetName(Ir);
+    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+  });
+
+  PIC.registerAfterAnalysisCallback(
+      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+        AfterPassCallback(LlvmSelfProfiler);
+      });
+}
+
+enum class LLVMRustOptStage {
+  PreLinkNoLTO,
+  PreLinkThinLTO,
+  PreLinkFatLTO,
+  ThinLTO,
+  FatLTO,
+};
+
+struct LLVMRustSanitizerOptions {
+  bool SanitizeAddress;
+  bool SanitizeAddressRecover;
+  bool SanitizeCFI;
+  bool SanitizeKCFI;
+  bool SanitizeMemory;
+  bool SanitizeMemoryRecover;
+  int  SanitizeMemoryTrackOrigins;
+  bool SanitizeThread;
+  bool SanitizeHWAddress;
+  bool SanitizeHWAddressRecover;
+  bool SanitizeKernelAddress;
+  bool SanitizeKernelAddressRecover;
+};
+
+extern "C" LLVMRustResult
+LLVMRustOptimize(
+    LLVMModuleRef ModuleRef,
+    LLVMTargetMachineRef TMRef,
+    LLVMRustPassBuilderOptLevel OptLevelRust,
+    LLVMRustOptStage OptStage,
+    bool IsLinkerPluginLTO,
+    bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
+    bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
+    bool EmitLifetimeMarkers,
+    LLVMRustSanitizerOptions *SanitizerOptions,
+    const char *PGOGenPath, const char *PGOUsePath,
+    bool InstrumentCoverage, const char *InstrProfileOutput,
+    bool InstrumentGCOV,
+    const char *PGOSampleUsePath, bool DebugInfoForProfiling,
+    void* LlvmSelfProfiler,
+    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+    LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
+    const char *ExtraPasses, size_t ExtraPassesLen,
+    const char *LLVMPlugins, size_t LLVMPluginsLen) {
+  Module *TheModule = unwrap(ModuleRef);
+  TargetMachine *TM = unwrap(TMRef);
+  OptimizationLevel OptLevel = fromRust(OptLevelRust);
+
+
+  PipelineTuningOptions PTO;
+  PTO.LoopUnrolling = UnrollLoops;
+  PTO.LoopInterleaving = UnrollLoops;
+  PTO.LoopVectorization = LoopVectorize;
+  PTO.SLPVectorization = SLPVectorize;
+  PTO.MergeFunctions = MergeFunctions;
+
+  // FIXME: We may want to expose this as an option.
+  bool DebugPassManager = false;
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI(TheModule->getContext(), DebugPassManager);
+  SI.registerCallbacks(PIC);
+
+  if (LlvmSelfProfiler){
+    LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
+  }
+
+  std::optional<PGOOptions> PGOOpt;
+#if LLVM_VERSION_GE(17, 0)
+  auto FS = vfs::getRealFileSystem();
+#endif
+  if (PGOGenPath) {
+    assert(!PGOUsePath && !PGOSampleUsePath);
+    PGOOpt = PGOOptions(PGOGenPath, "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        "",
+                        FS,
+#endif
+                        PGOOptions::IRInstr, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
+  } else if (PGOUsePath) {
+    assert(!PGOSampleUsePath);
+    PGOOpt = PGOOptions(PGOUsePath, "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        "",
+                        FS,
+#endif
+                        PGOOptions::IRUse, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
+  } else if (PGOSampleUsePath) {
+    PGOOpt = PGOOptions(PGOSampleUsePath, "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        "",
+                        FS,
+#endif
+                        PGOOptions::SampleUse, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
+  } else if (DebugInfoForProfiling) {
+    PGOOpt = PGOOptions("", "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        "",
+                        FS,
+#endif
+                        PGOOptions::NoAction, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
+  }
+
+  PassBuilder PB(TM, PTO, PGOOpt, &PIC);
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+
+  if (LLVMPluginsLen) {
+    auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
+    SmallVector<StringRef> Plugins;
+    PluginsStr.split(Plugins, ',', -1, false);
+    for (auto PluginPath: Plugins) {
+      auto Plugin = PassPlugin::Load(PluginPath.str());
+      if (!Plugin) {
+        LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
+        return LLVMRustResult::Failure;
+      }
+      Plugin->registerPassBuilderCallbacks(PB);
+    }
+  }
+
+  FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
+
+  Triple TargetTriple(TheModule->getTargetTriple());
+  std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
+  FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
+
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+
+  // We manually collect pipeline callbacks so we can apply them at O0, where the
+  // PassBuilder does not create a pipeline.
+  std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
+      PipelineStartEPCallbacks;
+  std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
+      OptimizerLastEPCallbacks;
+
+  if (!IsLinkerPluginLTO
+      && SanitizerOptions && SanitizerOptions->SanitizeCFI
+      && !NoPrepopulatePasses) {
+    PipelineStartEPCallbacks.push_back(
+      [](ModulePassManager &MPM, OptimizationLevel Level) {
+        MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr,
+                                       /*ImportSummary=*/nullptr,
+                                       /*DropTypeTests=*/false));
+      }
+    );
+  }
+
+  if (VerifyIR) {
+    PipelineStartEPCallbacks.push_back(
+      [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
+        MPM.addPass(VerifierPass());
+      }
+    );
+  }
+
+  if (InstrumentGCOV) {
+    PipelineStartEPCallbacks.push_back(
+      [](ModulePassManager &MPM, OptimizationLevel Level) {
+        MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
+      }
+    );
+  }
+
+  if (InstrumentCoverage) {
+    PipelineStartEPCallbacks.push_back(
+      [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
+        InstrProfOptions Options;
+        if (InstrProfileOutput) {
+          Options.InstrProfileOutput = InstrProfileOutput;
+        }
+        // cargo run tests in multhreading mode by default
+        // so use atomics for coverage counters
+        Options.Atomic = true;
+        MPM.addPass(InstrProfiling(Options, false));
+      }
+    );
+  }
+
+  if (SanitizerOptions) {
+    if (SanitizerOptions->SanitizeMemory) {
+      MemorySanitizerOptions Options(
+          SanitizerOptions->SanitizeMemoryTrackOrigins,
+          SanitizerOptions->SanitizeMemoryRecover,
+          /*CompileKernel=*/false,
+          /*EagerChecks=*/true);
+      OptimizerLastEPCallbacks.push_back(
+        [Options](ModulePassManager &MPM, OptimizationLevel Level) {
+          MPM.addPass(MemorySanitizerPass(Options));
+        }
+      );
+    }
+
+    if (SanitizerOptions->SanitizeThread) {
+      OptimizerLastEPCallbacks.push_back(
+        [](ModulePassManager &MPM, OptimizationLevel Level) {
+          MPM.addPass(ModuleThreadSanitizerPass());
+          MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
+        }
+      );
+    }
+
+    if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) {
+      OptimizerLastEPCallbacks.push_back(
+        [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
+          auto CompileKernel = SanitizerOptions->SanitizeKernelAddress;
+          AddressSanitizerOptions opts = AddressSanitizerOptions{
+            CompileKernel,
+            SanitizerOptions->SanitizeAddressRecover
+              || SanitizerOptions->SanitizeKernelAddressRecover,
+            /*UseAfterScope=*/true,
+            AsanDetectStackUseAfterReturnMode::Runtime,
+          };
+          MPM.addPass(AddressSanitizerPass(opts));
+        }
+      );
+    }
+    if (SanitizerOptions->SanitizeHWAddress) {
+      OptimizerLastEPCallbacks.push_back(
+        [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
+          HWAddressSanitizerOptions opts(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
+              /*DisableOptimization=*/false);
+          MPM.addPass(HWAddressSanitizerPass(opts));
+        }
+      );
+    }
+  }
+
+  ModulePassManager MPM;
+  bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
+  if (!NoPrepopulatePasses) {
+    // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead.
+    // At the same time, the LTO pipelines do support O0 and using them is required.
+    bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
+    if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
+      for (const auto &C : PipelineStartEPCallbacks)
+        PB.registerPipelineStartEPCallback(C);
+      for (const auto &C : OptimizerLastEPCallbacks)
+        PB.registerOptimizerLastEPCallback(C);
+
+      // Pass false as we manually schedule ThinLTOBufferPasses below.
+      MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
+    } else {
+      for (const auto &C : PipelineStartEPCallbacks)
+        PB.registerPipelineStartEPCallback(C);
+      if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
+        for (const auto &C : OptimizerLastEPCallbacks)
+          PB.registerOptimizerLastEPCallback(C);
+      }
+
+      switch (OptStage) {
+      case LLVMRustOptStage::PreLinkNoLTO:
+        MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::PreLinkThinLTO:
+        MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
+        // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
+        // passes may still run afterwards. This means we need to run the buffer passes again.
+        // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
+        // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
+        if (OptimizerLastEPCallbacks.empty())
+          NeedThinLTOBufferPasses = false;
+        for (const auto &C : OptimizerLastEPCallbacks)
+          C(MPM, OptLevel);
+        break;
+      case LLVMRustOptStage::PreLinkFatLTO:
+        MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
+        NeedThinLTOBufferPasses = false;
+        break;
+      case LLVMRustOptStage::ThinLTO:
+        // FIXME: Does it make sense to pass the ModuleSummaryIndex?
+        // It only seems to be needed for C++ specific optimizations.
+        MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
+        break;
+      case LLVMRustOptStage::FatLTO:
+        MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
+        break;
+      }
+    }
+  } else {
+    // We're not building any of the default pipelines but we still want to
+    // add the verifier, instrumentation, etc passes if they were requested
+    for (const auto &C : PipelineStartEPCallbacks)
+      C(MPM, OptLevel);
+    for (const auto &C : OptimizerLastEPCallbacks)
+      C(MPM, OptLevel);
+  }
+
+  if (ExtraPassesLen) {
+    if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
+      std::string ErrMsg = toString(std::move(Err));
+      LLVMRustSetLastError(ErrMsg.c_str());
+      return LLVMRustResult::Failure;
+    }
+  }
+
+  if (NeedThinLTOBufferPasses) {
+    MPM.addPass(CanonicalizeAliasesPass());
+    MPM.addPass(NameAnonGlobalPass());
+  }
+
+  // Upgrade all calls to old intrinsics first.
+  for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
+    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
+
+  MPM.run(*TheModule, MAM);
+  return LLVMRustResult::Success;
+}
+
+// Callback to demangle function name
+// Parameters:
+// * name to be demangled
+// * name len
+// * output buffer
+// * output buffer len
+// Returns len of demangled string, or 0 if demangle failed.
+typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
+
+
+namespace {
+
+class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
+  DemangleFn Demangle;
+  std::vector<char> Buf;
+
+public:
+  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
+
+  // Return empty string if demangle failed
+  // or if name does not need to be demangled
+  StringRef CallDemangle(StringRef name) {
+    if (!Demangle) {
+      return StringRef();
+    }
+
+    if (Buf.size() < name.size() * 2) {
+      // Semangled name usually shorter than mangled,
+      // but allocate twice as much memory just in case
+      Buf.resize(name.size() * 2);
+    }
+
+    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
+    if (!R) {
+      // Demangle failed.
+      return StringRef();
+    }
+
+    auto Demangled = StringRef(Buf.data(), R);
+    if (Demangled == name) {
+      // Do not print anything if demangled name is equal to mangled.
+      return StringRef();
+    }
+
+    return Demangled;
+  }
+
+  void emitFunctionAnnot(const Function *F,
+                         formatted_raw_ostream &OS) override {
+    StringRef Demangled = CallDemangle(F->getName());
+    if (Demangled.empty()) {
+        return;
+    }
+
+    OS << "; " << Demangled << "\n";
+  }
+
+  void emitInstructionAnnot(const Instruction *I,
+                            formatted_raw_ostream &OS) override {
+    const char *Name;
+    const Value *Value;
+    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
+      Name = "call";
+      Value = CI->getCalledOperand();
+    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
+      Name = "invoke";
+      Value = II->getCalledOperand();
+    } else {
+      // Could demangle more operations, e. g.
+      // `store %place, @function`.
+      return;
+    }
+
+    if (!Value->hasName()) {
+      return;
+    }
+
+    StringRef Demangled = CallDemangle(Value->getName());
+    if (Demangled.empty()) {
+      return;
+    }
+
+    OS << "; " << Name << " " << Demangled << "\n";
+  }
+};
+
+} // namespace
+
+extern "C" LLVMRustResult
+LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
+  std::string ErrorInfo;
+  std::error_code EC;
+  raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
+  if (EC)
+    ErrorInfo = EC.message();
+  if (ErrorInfo != "") {
+    LLVMRustSetLastError(ErrorInfo.c_str());
+    return LLVMRustResult::Failure;
+  }
+
+  RustAssemblyAnnotationWriter AAW(Demangle);
+  formatted_raw_ostream FOS(OS);
+  unwrap(M)->print(FOS, &AAW);
+
+  return LLVMRustResult::Success;
+}
+
+extern "C" void LLVMRustPrintPasses() {
+  PassBuilder PB;
+  PB.printPassNames(outs());
+}
+
+extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
+                                           size_t Len) {
+  auto PreserveFunctions = [=](const GlobalValue &GV) {
+    // Preserve LLVM-injected, ASAN-related symbols.
+    // See also https://github.com/rust-lang/rust/issues/113404.
+    if (GV.getName() == "___asan_globals_registered") {
+      return true;
+    }
+
+    // Preserve symbols exported from Rust modules.
+    for (size_t I = 0; I < Len; I++) {
+      if (GV.getName() == Symbols[I]) {
+        return true;
+      }
+    }
+    return false;
+  };
+
+  internalizeModule(*unwrap(M), PreserveFunctions);
+}
+
+extern "C" void
+LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
+                                       LLVMTargetMachineRef TMR) {
+  TargetMachine *Target = unwrap(TMR);
+  unwrap(Module)->setDataLayout(Target->createDataLayout());
+}
+
+extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
+  unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
+}
+
+extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
+  unwrap(M)->setPIELevel(PIELevel::Level::Large);
+}
+
+extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
+                                           LLVMRustCodeModel Model) {
+  auto CM = fromRust(Model);
+  if (!CM)
+    return;
+  unwrap(M)->setCodeModel(*CM);
+}
+
+// Here you'll find an implementation of ThinLTO as used by the Rust compiler
+// right now. This ThinLTO support is only enabled on "recent ish" versions of
+// LLVM, and otherwise it's just blanket rejected from other compilers.
+//
+// Most of this implementation is straight copied from LLVM. At the time of
+// this writing it wasn't *quite* suitable to reuse more code from upstream
+// for our purposes, but we should strive to upstream this support once it's
+// ready to go! I figure we may want a bit of testing locally first before
+// sending this upstream to LLVM. I hear though they're quite eager to receive
+// feedback like this!
+//
+// If you're reading this code and wondering "what in the world" or you're
+// working "good lord by LLVM upgrade is *still* failing due to these bindings"
+// then fear not! (ok maybe fear a little). All code here is mostly based
+// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
+//
+// You'll find that the general layout here roughly corresponds to the `run`
+// method in that file as well as `ProcessThinLTOModule`. Functions are
+// specifically commented below as well, but if you're updating this code
+// or otherwise trying to understand it, the LLVM source will be useful in
+// interpreting the mysteries within.
+//
+// Otherwise I'll apologize in advance, it probably requires a relatively
+// significant investment on your part to "truly understand" what's going on
+// here. Not saying I do myself, but it took me awhile staring at LLVM's source
+// and various online resources about ThinLTO to make heads or tails of all
+// this.
+
+// This is a shared data structure which *must* be threadsafe to share
+// read-only amongst threads. This also corresponds basically to the arguments
+// of the `ProcessThinLTOModule` function in the LLVM source.
+struct LLVMRustThinLTOData {
+  // The combined index that is the global analysis over all modules we're
+  // performing ThinLTO for. This is mostly managed by LLVM.
+  ModuleSummaryIndex Index;
+
+  // All modules we may look at, stored as in-memory serialized versions. This
+  // is later used when inlining to ensure we can extract any module to inline
+  // from.
+  StringMap<MemoryBufferRef> ModuleMap;
+
+  // A set that we manage of everything we *don't* want internalized. Note that
+  // this includes all transitive references right now as well, but it may not
+  // always!
+  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
+
+  // Not 100% sure what these are, but they impact what's internalized and
+  // what's inlined across modules, I believe.
+#if LLVM_VERSION_GE(18, 0)
+  DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists;
+  DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists;
+  DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
+#else
+  StringMap<FunctionImporter::ImportMapTy> ImportLists;
+  StringMap<FunctionImporter::ExportSetTy> ExportLists;
+  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
+#endif
+  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
+
+  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
+};
+
+// Just an argument to the `LLVMRustCreateThinLTOData` function below.
+struct LLVMRustThinLTOModule {
+  const char *identifier;
+  const char *data;
+  size_t len;
+};
+
+// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
+// does.
+static const GlobalValueSummary *
+getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
+  auto StrongDefForLinker = llvm::find_if(
+      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
+        auto Linkage = Summary->linkage();
+        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
+               !GlobalValue::isWeakForLinker(Linkage);
+      });
+  if (StrongDefForLinker != GVSummaryList.end())
+    return StrongDefForLinker->get();
+
+  auto FirstDefForLinker = llvm::find_if(
+      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
+        auto Linkage = Summary->linkage();
+        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
+      });
+  if (FirstDefForLinker == GVSummaryList.end())
+    return nullptr;
+  return FirstDefForLinker->get();
+}
+
+// The main entry point for creating the global ThinLTO analysis. The structure
+// here is basically the same as before threads are spawned in the `run`
+// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
+extern "C" LLVMRustThinLTOData*
+LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
+                          int num_modules,
+                          const char **preserved_symbols,
+                          int num_symbols) {
+  auto Ret = std::make_unique<LLVMRustThinLTOData>();
+
+  // Load each module's summary and merge it into one combined index
+  for (int i = 0; i < num_modules; i++) {
+    auto module = &modules[i];
+    StringRef buffer(module->data, module->len);
+    MemoryBufferRef mem_buffer(buffer, module->identifier);
+
+    Ret->ModuleMap[module->identifier] = mem_buffer;
+
+#if LLVM_VERSION_GE(18, 0)
+    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) {
+#else
+    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
+#endif
+      LLVMRustSetLastError(toString(std::move(Err)).c_str());
+      return nullptr;
+    }
+  }
+
+  // Collect for each module the list of function it defines (GUID -> Summary)
+  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
+
+  // Convert the preserved symbols set from string to GUID, this is then needed
+  // for internalization.
+  for (int i = 0; i < num_symbols; i++) {
+    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
+    Ret->GUIDPreservedSymbols.insert(GUID);
+  }
+
+  // Collect the import/export lists for all modules from the call-graph in the
+  // combined index
+  //
+  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
+  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
+    return PrevailingType::Unknown;
+  };
+  // We don't have a complete picture in our use of ThinLTO, just our immediate
+  // crate, so we need `ImportEnabled = false` to limit internalization.
+  // Otherwise, we sometimes lose `static` values -- see #60184.
+  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
+                                  deadIsPrevailing, /* ImportEnabled = */ false);
+  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
+  // impacts the caching.
+  //
+  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
+  // being lifted from `lib/LTO/LTO.cpp` as well
+  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
+  for (auto &I : Ret->Index) {
+    if (I.second.SummaryList.size() > 1)
+      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
+  }
+  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
+    const auto &Prevailing = PrevailingCopy.find(GUID);
+    if (Prevailing == PrevailingCopy.end())
+      return true;
+    return Prevailing->second == S;
+  };
+  ComputeCrossModuleImport(
+    Ret->Index,
+    Ret->ModuleToDefinedGVSummaries,
+#if LLVM_VERSION_GE(17, 0)
+    isPrevailing,
+#endif
+    Ret->ImportLists,
+    Ret->ExportLists
+  );
+
+  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
+                              GlobalValue::GUID GUID,
+                              GlobalValue::LinkageTypes NewLinkage) {
+    Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
+  };
+
+  // Uses FromPrevailing visibility scheme which works for many binary
+  // formats. We probably could and should use ELF visibility scheme for many of
+  // our targets, however.
+  lto::Config conf;
+  thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
+                                  Ret->GUIDPreservedSymbols);
+
+  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
+  // callback below. This callback below will dictate the linkage for all
+  // summaries in the index, and we basically just only want to ensure that dead
+  // symbols are internalized. Otherwise everything that's already external
+  // linkage will stay as external, and internal will stay as internal.
+  std::set<GlobalValue::GUID> ExportedGUIDs;
+  for (auto &List : Ret->Index) {
+    for (auto &GVS: List.second.SummaryList) {
+      if (GlobalValue::isLocalLinkage(GVS->linkage()))
+        continue;
+      auto GUID = GVS->getOriginalName();
+      if (GVS->flags().Live)
+        ExportedGUIDs.insert(GUID);
+    }
+  }
+  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
+    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
+    return (ExportList != Ret->ExportLists.end() &&
+      ExportList->second.count(VI)) ||
+      ExportedGUIDs.count(VI.getGUID());
+  };
+  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
+
+  return Ret.release();
+}
+
+extern "C" void
+LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
+  delete Data;
+}
+
+// Below are the various passes that happen *per module* when doing ThinLTO.
+//
+// In other words, these are the functions that are all run concurrently
+// with one another, one per module. The passes here correspond to the analysis
+// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
+// `ProcessThinLTOModule` function. Here they're split up into separate steps
+// so rustc can save off the intermediate bytecode between each step.
+
+static bool
+clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
+  // When linking an ELF shared object, dso_local should be dropped. We
+  // conservatively do this for -fpic.
+  bool ClearDSOLocalOnDeclarations =
+      TM.getTargetTriple().isOSBinFormatELF() &&
+      TM.getRelocationModel() != Reloc::Static &&
+      Mod.getPIELevel() == PIELevel::Default;
+  return ClearDSOLocalOnDeclarations;
+}
+
+extern "C" bool
+LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
+                             LLVMTargetMachineRef TM) {
+  Module &Mod = *unwrap(M);
+  TargetMachine &Target = *unwrap(TM);
+
+  bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
+  bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
+
+  if (error) {
+    LLVMRustSetLastError("renameModuleForThinLTO failed");
+    return false;
+  }
+  return true;
+}
+
+extern "C" bool
+LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
+  Module &Mod = *unwrap(M);
+  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
+  thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
+  return true;
+}
+
+extern "C" bool
+LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
+  Module &Mod = *unwrap(M);
+  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
+  thinLTOInternalizeModule(Mod, DefinedGlobals);
+  return true;
+}
+
+extern "C" bool
+LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
+                             LLVMTargetMachineRef TM) {
+  Module &Mod = *unwrap(M);
+  TargetMachine &Target = *unwrap(TM);
+
+  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
+  auto Loader = [&](StringRef Identifier) {
+    const auto &Memory = Data->ModuleMap.lookup(Identifier);
+    auto &Context = Mod.getContext();
+    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
+
+    if (!MOrErr)
+      return MOrErr;
+
+    // The rest of this closure is a workaround for
+    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
+    // we accidentally import wasm custom sections into different modules,
+    // duplicating them by in the final output artifact.
+    //
+    // The issue is worked around here by manually removing the
+    // `wasm.custom_sections` named metadata node from any imported module. This
+    // we know isn't used by any optimization pass so there's no need for it to
+    // be imported.
+    //
+    // Note that the metadata is currently lazily loaded, so we materialize it
+    // here before looking up if there's metadata inside. The `FunctionImporter`
+    // will immediately materialize metadata anyway after an import, so this
+    // shouldn't be a perf hit.
+    if (Error Err = (*MOrErr)->materializeMetadata()) {
+      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
+      return Ret;
+    }
+
+    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
+    if (WasmCustomSections)
+      WasmCustomSections->eraseFromParent();
+
+    // `llvm.ident` named metadata also gets duplicated.
+    auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident");
+    if (llvmIdent)
+      llvmIdent->eraseFromParent();
+
+    return MOrErr;
+  };
+  bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
+  FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
+  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
+  if (!Result) {
+    LLVMRustSetLastError(toString(Result.takeError()).c_str());
+    return false;
+  }
+  return true;
+}
+
+// This struct and various functions are sort of a hack right now, but the
+// problem is that we've got in-memory LLVM modules after we generate and
+// optimize all codegen-units for one compilation in rustc. To be compatible
+// with the LTO support above we need to serialize the modules plus their
+// ThinLTO summary into memory.
+//
+// This structure is basically an owned version of a serialize module, with
+// a ThinLTO summary attached.
+struct LLVMRustThinLTOBuffer {
+  std::string data;
+};
+
+extern "C" LLVMRustThinLTOBuffer*
+LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
+  auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
+  {
+    raw_string_ostream OS(Ret->data);
+    {
+      if (is_thin) {
+        PassBuilder PB;
+        LoopAnalysisManager LAM;
+        FunctionAnalysisManager FAM;
+        CGSCCAnalysisManager CGAM;
+        ModuleAnalysisManager MAM;
+        PB.registerModuleAnalyses(MAM);
+        PB.registerCGSCCAnalyses(CGAM);
+        PB.registerFunctionAnalyses(FAM);
+        PB.registerLoopAnalyses(LAM);
+        PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+        ModulePassManager MPM;
+        MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
+        MPM.run(*unwrap(M), MAM);
+      } else {
+        WriteBitcodeToFile(*unwrap(M), OS);
+      }
+    }
+  }
+  return Ret.release();
+}
+
+extern "C" void
+LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
+  delete Buffer;
+}
+
+extern "C" const void*
+LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
+  return Buffer->data.data();
+}
+
+extern "C" size_t
+LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
+  return Buffer->data.length();
+}
+
+// This is what we used to parse upstream bitcode for actual ThinLTO
+// processing. We'll call this once per module optimized through ThinLTO, and
+// it'll be called concurrently on many threads.
+extern "C" LLVMModuleRef
+LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
+                           const char *data,
+                           size_t len,
+                           const char *identifier) {
+  StringRef Data(data, len);
+  MemoryBufferRef Buffer(Data, identifier);
+  unwrap(Context)->enableDebugTypeODRUniquing();
+  Expected<std::unique_ptr<Module>> SrcOrError =
+      parseBitcodeFile(Buffer, *unwrap(Context));
+  if (!SrcOrError) {
+    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
+    return nullptr;
+  }
+  return wrap(std::move(*SrcOrError).release());
+}
+
+// Find a section of an object file by name. Fail if the section is missing or
+// empty.
+extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data,
+                                                            size_t len,
+                                                            const char *name,
+                                                            size_t *out_len) {
+  *out_len = 0;
+  StringRef Data(data, len);
+  MemoryBufferRef Buffer(Data, ""); // The id is unused.
+  file_magic Type = identify_magic(Buffer.getBuffer());
+  Expected<std::unique_ptr<object::ObjectFile>> ObjFileOrError =
+      object::ObjectFile::createObjectFile(Buffer, Type);
+  if (!ObjFileOrError) {
+    LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str());
+    return nullptr;
+  }
+  for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) {
+    Expected<StringRef> Name = Sec.getName();
+    if (Name && *Name == name) {
+      Expected<StringRef> SectionOrError = Sec.getContents();
+      if (!SectionOrError) {
+        LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str());
+        return nullptr;
+      }
+      *out_len = SectionOrError->size();
+      return SectionOrError->data();
+    }
+  }
+  LLVMRustSetLastError("could not find requested section");
+  return nullptr;
+}
+
+// Computes the LTO cache key for the provided 'ModId' in the given 'Data',
+// storing the result in 'KeyOut'.
+// Currently, this cache key is a SHA-1 hash of anything that could affect
+// the result of optimizing this module (e.g. module imports, exports, liveness
+// of access globals, etc).
+// The precise details are determined by LLVM in `computeLTOCacheKey`, which is
+// used during the normal linker-plugin incremental thin-LTO process.
+extern "C" void
+LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
+  SmallString<40> Key;
+  llvm::lto::Config conf;
+  const auto &ImportList = Data->ImportLists.lookup(ModId);
+  const auto &ExportList = Data->ExportLists.lookup(ModId);
+  const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
+  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
+  std::set<GlobalValue::GUID> CfiFunctionDefs;
+  std::set<GlobalValue::GUID> CfiFunctionDecls;
+
+  // Based on the 'InProcessThinBackend' constructor in LLVM
+  for (auto &Name : Data->Index.cfiFunctionDefs())
+    CfiFunctionDefs.insert(
+        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
+  for (auto &Name : Data->Index.cfiFunctionDecls())
+    CfiFunctionDecls.insert(
+        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
+
+  llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
+      ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
+  );
+
+  LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());
+}
diff --git a/compiler/rustc_llvm/llvm-wrapper/README b/compiler/rustc_llvm/llvm-wrapper/README
new file mode 100644
index 00000000000..e1c6dd07d2b
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/README
@@ -0,0 +1,16 @@
+This directory currently contains some LLVM support code. This will generally
+be sent upstream to LLVM in time; for now it lives here.
+
+NOTE: the LLVM C++ ABI is subject to between-version breakage and must *never*
+be exposed to Rust. To allow for easy auditing of that, all Rust-exposed types
+must be typedef-ed as "LLVMXyz", or "LLVMRustXyz" if they were defined here.
+
+Functions that return a failure status and leave the error in
+the LLVM last error should return an LLVMRustResult rather than an
+int or anything to avoid confusion.
+
+When translating enums, add a single `Other` variant as the first
+one to allow for new variants to be added. It should abort when used
+as an input.
+
+All other types must not be typedef-ed as such.
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
new file mode 100644
index 00000000000..b227dd76f02
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -0,0 +1,2022 @@
+#include "LLVMWrapper.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DiagnosticHandler.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsARM.h"
+#include "llvm/IR/LLVMRemarkStreamer.h"
+#include "llvm/IR/Mangler.h"
+#include "llvm/Remarks/RemarkStreamer.h"
+#include "llvm/Remarks/RemarkSerializer.h"
+#include "llvm/Remarks/RemarkFormat.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/ModRef.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/COFFImportFile.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Pass.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/Support/Signals.h"
+
+#include <iostream>
+
+//===----------------------------------------------------------------------===
+//
+// This file defines alternate interfaces to core functions that are more
+// readily callable by Rust's FFI.
+//
+//===----------------------------------------------------------------------===
+
+using namespace llvm;
+using namespace llvm::sys;
+using namespace llvm::object;
+
+// LLVMAtomicOrdering is already an enum - don't create another
+// one.
+static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) {
+  switch (Ordering) {
+  case LLVMAtomicOrderingNotAtomic:
+    return AtomicOrdering::NotAtomic;
+  case LLVMAtomicOrderingUnordered:
+    return AtomicOrdering::Unordered;
+  case LLVMAtomicOrderingMonotonic:
+    return AtomicOrdering::Monotonic;
+  case LLVMAtomicOrderingAcquire:
+    return AtomicOrdering::Acquire;
+  case LLVMAtomicOrderingRelease:
+    return AtomicOrdering::Release;
+  case LLVMAtomicOrderingAcquireRelease:
+    return AtomicOrdering::AcquireRelease;
+  case LLVMAtomicOrderingSequentiallyConsistent:
+    return AtomicOrdering::SequentiallyConsistent;
+  }
+
+  report_fatal_error("Invalid LLVMAtomicOrdering value!");
+}
+
+static LLVM_THREAD_LOCAL char *LastError;
+
+// Custom error handler for fatal LLVM errors.
+//
+// Notably it exits the process with code 101, unlike LLVM's default of 1.
+static void FatalErrorHandler(void *UserData,
+                              const char* Reason,
+                              bool GenCrashDiag) {
+  // Do the same thing that the default error handler does.
+  std::cerr << "LLVM ERROR: " << Reason << std::endl;
+
+  // Since this error handler exits the process, we have to run any cleanup that
+  // LLVM would run after handling the error. This might change with an LLVM
+  // upgrade.
+  sys::RunInterruptHandlers();
+
+  exit(101);
+}
+
+extern "C" void LLVMRustInstallFatalErrorHandler() {
+  install_fatal_error_handler(FatalErrorHandler);
+}
+
+extern "C" void LLVMRustDisableSystemDialogsOnCrash() {
+  sys::DisableSystemDialogsOnCrash();
+}
+
+extern "C" char *LLVMRustGetLastError(void) {
+  char *Ret = LastError;
+  LastError = nullptr;
+  return Ret;
+}
+
+extern "C" void LLVMRustSetLastError(const char *Err) {
+  free((void *)LastError);
+  LastError = strdup(Err);
+}
+
+extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) {
+  auto ctx = new LLVMContext();
+  ctx->setDiscardValueNames(shouldDiscardNames);
+  return wrap(ctx);
+}
+
+extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M,
+                                            const char *Triple) {
+  unwrap(M)->setTargetTriple(Triple::normalize(Triple));
+}
+
+extern "C" const char *LLVMRustPrintPassTimings(size_t *Len) {
+  std::string buf;
+  raw_string_ostream SS(buf);
+  TimerGroup::printAll(SS);
+  SS.flush();
+  *Len = buf.length();
+  char *CStr = (char *)malloc(*Len);
+  memcpy(CStr, buf.c_str(), *Len);
+  return CStr;
+}
+
+extern "C" const char *LLVMRustPrintStatistics(size_t *Len) {
+  std::string buf;
+  raw_string_ostream SS(buf);
+  llvm::PrintStatistics(SS);
+  SS.flush();
+  *Len = buf.length();
+  char *CStr = (char *)malloc(*Len);
+  memcpy(CStr, buf.c_str(), *Len);
+  return CStr;
+}
+
+extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name,
+                                              size_t NameLen) {
+  return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen)));
+}
+
+enum class LLVMRustTailCallKind {
+  None,
+  Tail,
+  MustTail,
+  NoTail,
+};
+
+static CallInst::TailCallKind fromRust(LLVMRustTailCallKind Kind) {
+  switch (Kind) {
+  case LLVMRustTailCallKind::None:
+    return CallInst::TailCallKind::TCK_None;
+  case LLVMRustTailCallKind::Tail:
+    return CallInst::TailCallKind::TCK_Tail;
+  case LLVMRustTailCallKind::MustTail:
+    return CallInst::TailCallKind::TCK_MustTail;
+  case LLVMRustTailCallKind::NoTail:
+    return CallInst::TailCallKind::TCK_NoTail;
+  default:
+    report_fatal_error("bad CallInst::TailCallKind.");
+  }
+}
+
+extern "C" void LLVMRustSetTailCallKind(LLVMValueRef Call, LLVMRustTailCallKind TCK) {
+  unwrap<CallInst>(Call)->setTailCallKind(fromRust(TCK));
+}
+
+extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M,
+                                                    const char *Name,
+                                                    size_t NameLen,
+                                                    LLVMTypeRef FunctionTy) {
+  return wrap(unwrap(M)
+                  ->getOrInsertFunction(StringRef(Name, NameLen),
+                                        unwrap<FunctionType>(FunctionTy))
+                  .getCallee()
+  );
+}
+
+extern "C" LLVMValueRef
+LLVMRustGetOrInsertGlobal(LLVMModuleRef M, const char *Name, size_t NameLen, LLVMTypeRef Ty) {
+  Module *Mod = unwrap(M);
+  StringRef NameRef(Name, NameLen);
+
+  // We don't use Module::getOrInsertGlobal because that returns a Constant*,
+  // which may either be the real GlobalVariable*, or a constant bitcast of it
+  // if our type doesn't match the original declaration. We always want the
+  // GlobalVariable* so we can access linkage, visibility, etc.
+  GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true);
+  if (!GV)
+    GV = new GlobalVariable(*Mod, unwrap(Ty), false,
+                            GlobalValue::ExternalLinkage, nullptr, NameRef);
+  return wrap(GV);
+}
+
+extern "C" LLVMValueRef
+LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) {
+  return wrap(new GlobalVariable(*unwrap(M),
+                                 unwrap(Ty),
+                                 false,
+                                 GlobalValue::PrivateLinkage,
+                                 nullptr));
+}
+
+static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
+  switch (Kind) {
+  case AlwaysInline:
+    return Attribute::AlwaysInline;
+  case ByVal:
+    return Attribute::ByVal;
+  case Cold:
+    return Attribute::Cold;
+  case InlineHint:
+    return Attribute::InlineHint;
+  case MinSize:
+    return Attribute::MinSize;
+  case Naked:
+    return Attribute::Naked;
+  case NoAlias:
+    return Attribute::NoAlias;
+  case NoCapture:
+    return Attribute::NoCapture;
+  case NoCfCheck:
+    return Attribute::NoCfCheck;
+  case NoInline:
+    return Attribute::NoInline;
+  case NonNull:
+    return Attribute::NonNull;
+  case NoRedZone:
+    return Attribute::NoRedZone;
+  case NoReturn:
+    return Attribute::NoReturn;
+  case NoUnwind:
+    return Attribute::NoUnwind;
+  case OptimizeForSize:
+    return Attribute::OptimizeForSize;
+  case ReadOnly:
+    return Attribute::ReadOnly;
+  case SExt:
+    return Attribute::SExt;
+  case StructRet:
+    return Attribute::StructRet;
+  case UWTable:
+    return Attribute::UWTable;
+  case ZExt:
+    return Attribute::ZExt;
+  case InReg:
+    return Attribute::InReg;
+  case SanitizeThread:
+    return Attribute::SanitizeThread;
+  case SanitizeAddress:
+    return Attribute::SanitizeAddress;
+  case SanitizeMemory:
+    return Attribute::SanitizeMemory;
+  case NonLazyBind:
+    return Attribute::NonLazyBind;
+  case OptimizeNone:
+    return Attribute::OptimizeNone;
+  case ReturnsTwice:
+    return Attribute::ReturnsTwice;
+  case ReadNone:
+    return Attribute::ReadNone;
+  case SanitizeHWAddress:
+    return Attribute::SanitizeHWAddress;
+  case WillReturn:
+    return Attribute::WillReturn;
+  case StackProtectReq:
+    return Attribute::StackProtectReq;
+  case StackProtectStrong:
+    return Attribute::StackProtectStrong;
+  case StackProtect:
+    return Attribute::StackProtect;
+  case NoUndef:
+    return Attribute::NoUndef;
+  case SanitizeMemTag:
+    return Attribute::SanitizeMemTag;
+  case ShadowCallStack:
+    return Attribute::ShadowCallStack;
+  case AllocSize:
+    return Attribute::AllocSize;
+  case AllocatedPointer:
+    return Attribute::AllocatedPointer;
+  case AllocAlign:
+    return Attribute::AllocAlign;
+  case SanitizeSafeStack:
+    return Attribute::SafeStack;
+  case FnRetThunkExtern:
+    return Attribute::FnRetThunkExtern;
+  }
+  report_fatal_error("bad AttributeKind");
+}
+
+template<typename T> static inline void AddAttributes(T *t, unsigned Index,
+                                                      LLVMAttributeRef *Attrs, size_t AttrsLen) {
+  AttributeList PAL = t->getAttributes();
+  AttrBuilder B(t->getContext());
+  for (LLVMAttributeRef Attr : ArrayRef<LLVMAttributeRef>(Attrs, AttrsLen))
+    B.addAttribute(unwrap(Attr));
+  AttributeList PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B);
+  t->setAttributes(PALNew);
+}
+
+extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index,
+                                              LLVMAttributeRef *Attrs, size_t AttrsLen) {
+  Function *F = unwrap<Function>(Fn);
+  AddAttributes(F, Index, Attrs, AttrsLen);
+}
+
+extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, unsigned Index,
+                                              LLVMAttributeRef *Attrs, size_t AttrsLen) {
+  CallBase *Call = unwrap<CallBase>(Instr);
+  AddAttributes(Call, Index, Attrs, AttrsLen);
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C,
+                                                      LLVMRustAttribute RustAttr) {
+  return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr)));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C,
+                                                        uint64_t Bytes) {
+  return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes)));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C,
+                                                              uint64_t Bytes) {
+  return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C,
+                                                                    uint64_t Bytes) {
+  return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, LLVMTypeRef Ty) {
+  return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty)));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) {
+  return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty)));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) {
+  return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty)));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) {
+  return wrap(Attribute::getWithUWTableKind(
+      *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync));
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) {
+  return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, std::nullopt));
+}
+
+// These values **must** match ffi::AllocKindFlags.
+// It _happens_ to match the LLVM values of llvm::AllocFnKind,
+// but that's happenstance and we do explicit conversions before
+// passing them to LLVM.
+enum class LLVMRustAllocKindFlags : uint64_t {
+  Unknown = 0,
+  Alloc = 1,
+  Realloc = 1 << 1,
+  Free = 1 << 2,
+  Uninitialized = 1 << 3,
+  Zeroed = 1 << 4,
+  Aligned = 1 << 5,
+};
+
+static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) {
+  return static_cast<LLVMRustAllocKindFlags>(static_cast<uint64_t>(A) &
+                                      static_cast<uint64_t>(B));
+}
+
+static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; }
+
+static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) {
+  llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown;
+  if (isSet(F & LLVMRustAllocKindFlags::Alloc)) {
+    AFK |= llvm::AllocFnKind::Alloc;
+  }
+  if (isSet(F & LLVMRustAllocKindFlags::Realloc)) {
+    AFK |= llvm::AllocFnKind::Realloc;
+  }
+  if (isSet(F & LLVMRustAllocKindFlags::Free)) {
+    AFK |= llvm::AllocFnKind::Free;
+  }
+  if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) {
+    AFK |= llvm::AllocFnKind::Uninitialized;
+  }
+  if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) {
+    AFK |= llvm::AllocFnKind::Zeroed;
+  }
+  if (isSet(F & LLVMRustAllocKindFlags::Aligned)) {
+    AFK |= llvm::AllocFnKind::Aligned;
+  }
+  return AFK;
+}
+
+extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) {
+  return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind,
+      static_cast<uint64_t>(allocKindFromRust(static_cast<LLVMRustAllocKindFlags>(AllocKindArg)))));
+}
+
+// Simplified representation of `MemoryEffects` across the FFI boundary.
+//
+// Each variant corresponds to one of the static factory methods on `MemoryEffects`.
+enum class LLVMRustMemoryEffects {
+  None,
+  ReadOnly,
+  InaccessibleMemOnly,
+};
+
+extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C,
+                                                            LLVMRustMemoryEffects Effects) {
+  switch (Effects) {
+    case LLVMRustMemoryEffects::None:
+      return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::none()));
+    case LLVMRustMemoryEffects::ReadOnly:
+      return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::readOnly()));
+    case LLVMRustMemoryEffects::InaccessibleMemOnly:
+      return wrap(Attribute::getWithMemoryEffects(*unwrap(C),
+                                                  MemoryEffects::inaccessibleMemOnly()));
+    default:
+      report_fatal_error("bad MemoryEffects.");
+  }
+}
+
+// Enable a fast-math flag
+//
+// https://llvm.org/docs/LangRef.html#fast-math-flags
+extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
+  if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
+    I->setFast(true);
+  }
+}
+
+extern "C" LLVMValueRef
+LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source,
+                        const char *Name, LLVMAtomicOrdering Order) {
+  Value *Ptr = unwrap(Source);
+  LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name);
+  LI->setAtomic(fromRust(Order));
+  return wrap(LI);
+}
+
+extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B,
+                                                 LLVMValueRef V,
+                                                 LLVMValueRef Target,
+                                                 LLVMAtomicOrdering Order) {
+  StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target));
+  SI->setAtomic(fromRust(Order));
+  return wrap(SI);
+}
+
+enum class LLVMRustAsmDialect {
+  Att,
+  Intel,
+};
+
+static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) {
+  switch (Dialect) {
+  case LLVMRustAsmDialect::Att:
+    return InlineAsm::AD_ATT;
+  case LLVMRustAsmDialect::Intel:
+    return InlineAsm::AD_Intel;
+  default:
+    report_fatal_error("bad AsmDialect.");
+  }
+}
+
+extern "C" LLVMValueRef
+LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen,
+                  char *Constraints, size_t ConstraintsLen,
+                  LLVMBool HasSideEffects, LLVMBool IsAlignStack,
+                  LLVMRustAsmDialect Dialect, LLVMBool CanThrow) {
+  return wrap(InlineAsm::get(unwrap<FunctionType>(Ty),
+                             StringRef(AsmString, AsmStringLen),
+                             StringRef(Constraints, ConstraintsLen),
+                             HasSideEffects, IsAlignStack,
+                             fromRust(Dialect), CanThrow));
+}
+
+extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints,
+                                        size_t ConstraintsLen) {
+  // llvm::Error converts to true if it is an error.
+  return !llvm::errorToBool(InlineAsm::verify(
+      unwrap<FunctionType>(Ty), StringRef(Constraints, ConstraintsLen)));
+}
+
+typedef DIBuilder *LLVMRustDIBuilderRef;
+
+template <typename DIT> DIT *unwrapDIPtr(LLVMMetadataRef Ref) {
+  return (DIT *)(Ref ? unwrap<MDNode>(Ref) : nullptr);
+}
+
+#define DIDescriptor DIScope
+#define DIArray DINodeArray
+#define unwrapDI unwrapDIPtr
+
+// These values **must** match debuginfo::DIFlags! They also *happen*
+// to match LLVM, but that isn't required as we do giant sets of
+// matching below. The value shouldn't be directly passed to LLVM.
+enum class LLVMRustDIFlags : uint32_t {
+  FlagZero = 0,
+  FlagPrivate = 1,
+  FlagProtected = 2,
+  FlagPublic = 3,
+  FlagFwdDecl = (1 << 2),
+  FlagAppleBlock = (1 << 3),
+  FlagBlockByrefStruct = (1 << 4),
+  FlagVirtual = (1 << 5),
+  FlagArtificial = (1 << 6),
+  FlagExplicit = (1 << 7),
+  FlagPrototyped = (1 << 8),
+  FlagObjcClassComplete = (1 << 9),
+  FlagObjectPointer = (1 << 10),
+  FlagVector = (1 << 11),
+  FlagStaticMember = (1 << 12),
+  FlagLValueReference = (1 << 13),
+  FlagRValueReference = (1 << 14),
+  FlagExternalTypeRef = (1 << 15),
+  FlagIntroducedVirtual = (1 << 18),
+  FlagBitField = (1 << 19),
+  FlagNoReturn = (1 << 20),
+  // Do not add values that are not supported by the minimum LLVM
+  // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def
+};
+
+inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) {
+  return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(A) &
+                                      static_cast<uint32_t>(B));
+}
+
+inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) {
+  return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(A) |
+                                      static_cast<uint32_t>(B));
+}
+
+inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) {
+  return A = A | B;
+}
+
+inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; }
+
+inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) {
+  return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(F) & 0x3);
+}
+
+static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) {
+  DINode::DIFlags Result = DINode::DIFlags::FlagZero;
+
+  switch (visibility(Flags)) {
+  case LLVMRustDIFlags::FlagPrivate:
+    Result |= DINode::DIFlags::FlagPrivate;
+    break;
+  case LLVMRustDIFlags::FlagProtected:
+    Result |= DINode::DIFlags::FlagProtected;
+    break;
+  case LLVMRustDIFlags::FlagPublic:
+    Result |= DINode::DIFlags::FlagPublic;
+    break;
+  default:
+    // The rest are handled below
+    break;
+  }
+
+  if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) {
+    Result |= DINode::DIFlags::FlagFwdDecl;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) {
+    Result |= DINode::DIFlags::FlagAppleBlock;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) {
+    Result |= DINode::DIFlags::FlagVirtual;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) {
+    Result |= DINode::DIFlags::FlagArtificial;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) {
+    Result |= DINode::DIFlags::FlagExplicit;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) {
+    Result |= DINode::DIFlags::FlagPrototyped;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) {
+    Result |= DINode::DIFlags::FlagObjcClassComplete;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) {
+    Result |= DINode::DIFlags::FlagObjectPointer;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagVector)) {
+    Result |= DINode::DIFlags::FlagVector;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) {
+    Result |= DINode::DIFlags::FlagStaticMember;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) {
+    Result |= DINode::DIFlags::FlagLValueReference;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) {
+    Result |= DINode::DIFlags::FlagRValueReference;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) {
+    Result |= DINode::DIFlags::FlagIntroducedVirtual;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) {
+    Result |= DINode::DIFlags::FlagBitField;
+  }
+  if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) {
+    Result |= DINode::DIFlags::FlagNoReturn;
+  }
+
+  return Result;
+}
+
+// These values **must** match debuginfo::DISPFlags! They also *happen*
+// to match LLVM, but that isn't required as we do giant sets of
+// matching below. The value shouldn't be directly passed to LLVM.
+enum class LLVMRustDISPFlags : uint32_t {
+  SPFlagZero = 0,
+  SPFlagVirtual = 1,
+  SPFlagPureVirtual = 2,
+  SPFlagLocalToUnit = (1 << 2),
+  SPFlagDefinition = (1 << 3),
+  SPFlagOptimized = (1 << 4),
+  SPFlagMainSubprogram = (1 << 5),
+  // Do not add values that are not supported by the minimum LLVM
+  // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def
+  // (In LLVM < 8, createFunction supported these as separate bool arguments.)
+};
+
+inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) {
+  return static_cast<LLVMRustDISPFlags>(static_cast<uint32_t>(A) &
+                                      static_cast<uint32_t>(B));
+}
+
+inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) {
+  return static_cast<LLVMRustDISPFlags>(static_cast<uint32_t>(A) |
+                                      static_cast<uint32_t>(B));
+}
+
+inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, LLVMRustDISPFlags B) {
+  return A = A | B;
+}
+
+inline bool isSet(LLVMRustDISPFlags F) { return F != LLVMRustDISPFlags::SPFlagZero; }
+
+inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) {
+  return static_cast<LLVMRustDISPFlags>(static_cast<uint32_t>(F) & 0x3);
+}
+
+static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) {
+  DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero;
+
+  switch (virtuality(SPFlags)) {
+  case LLVMRustDISPFlags::SPFlagVirtual:
+    Result |= DISubprogram::DISPFlags::SPFlagVirtual;
+    break;
+  case LLVMRustDISPFlags::SPFlagPureVirtual:
+    Result |= DISubprogram::DISPFlags::SPFlagPureVirtual;
+    break;
+  default:
+    // The rest are handled below
+    break;
+  }
+
+  if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) {
+    Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit;
+  }
+  if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) {
+    Result |= DISubprogram::DISPFlags::SPFlagDefinition;
+  }
+  if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) {
+    Result |= DISubprogram::DISPFlags::SPFlagOptimized;
+  }
+  if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) {
+    Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram;
+  }
+
+  return Result;
+}
+
+enum class LLVMRustDebugEmissionKind {
+  NoDebug,
+  FullDebug,
+  LineTablesOnly,
+  DebugDirectivesOnly,
+};
+
+static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) {
+  switch (Kind) {
+  case LLVMRustDebugEmissionKind::NoDebug:
+    return DICompileUnit::DebugEmissionKind::NoDebug;
+  case LLVMRustDebugEmissionKind::FullDebug:
+    return DICompileUnit::DebugEmissionKind::FullDebug;
+  case LLVMRustDebugEmissionKind::LineTablesOnly:
+    return DICompileUnit::DebugEmissionKind::LineTablesOnly;
+  case LLVMRustDebugEmissionKind::DebugDirectivesOnly:
+    return DICompileUnit::DebugEmissionKind::DebugDirectivesOnly;
+  default:
+    report_fatal_error("bad DebugEmissionKind.");
+  }
+}
+
+enum class LLVMRustChecksumKind {
+  None,
+  MD5,
+  SHA1,
+  SHA256,
+};
+
+static std::optional<DIFile::ChecksumKind> fromRust(LLVMRustChecksumKind Kind) {
+  switch (Kind) {
+  case LLVMRustChecksumKind::None:
+    return std::nullopt;
+  case LLVMRustChecksumKind::MD5:
+    return DIFile::ChecksumKind::CSK_MD5;
+  case LLVMRustChecksumKind::SHA1:
+    return DIFile::ChecksumKind::CSK_SHA1;
+  case LLVMRustChecksumKind::SHA256:
+    return DIFile::ChecksumKind::CSK_SHA256;
+  default:
+    report_fatal_error("bad ChecksumKind.");
+  }
+}
+
+extern "C" uint32_t LLVMRustDebugMetadataVersion() {
+  return DEBUG_METADATA_VERSION;
+}
+
+extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; }
+
+extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; }
+
+extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; }
+
+extern "C" void LLVMRustAddModuleFlag(
+    LLVMModuleRef M,
+    Module::ModFlagBehavior MergeBehavior,
+    const char *Name,
+    uint32_t Value) {
+  unwrap(M)->addModuleFlag(MergeBehavior, Name, Value);
+}
+
+extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name,
+                                      size_t Len) {
+  return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr;
+}
+
+extern "C" void LLVMRustGlobalAddMetadata(
+    LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) {
+  unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD));
+}
+
+extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) {
+  return new DIBuilder(*unwrap(M));
+}
+
+extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) {
+  delete Builder;
+}
+
+extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) {
+  Builder->finalize();
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit(
+    LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef,
+    const char *Producer, size_t ProducerLen, bool isOptimized,
+    const char *Flags, unsigned RuntimeVer,
+    const char *SplitName, size_t SplitNameLen,
+    LLVMRustDebugEmissionKind Kind,
+    uint64_t DWOId, bool SplitDebugInlining) {
+  auto *File = unwrapDI<DIFile>(FileRef);
+
+  return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen),
+                                         isOptimized, Flags, RuntimeVer,
+                                         StringRef(SplitName, SplitNameLen),
+                                         fromRust(Kind), DWOId, SplitDebugInlining));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile(
+    LLVMRustDIBuilderRef Builder,
+    const char *Filename, size_t FilenameLen,
+    const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind,
+    const char *Checksum, size_t ChecksumLen) {
+
+  std::optional<DIFile::ChecksumKind> llvmCSKind = fromRust(CSKind);
+  std::optional<DIFile::ChecksumInfo<StringRef>> CSInfo{};
+  if (llvmCSKind)
+    CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen});
+  return wrap(Builder->createFile(StringRef(Filename, FilenameLen),
+                                  StringRef(Directory, DirectoryLen),
+                                  CSInfo));
+}
+
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder,
+                                      LLVMMetadataRef ParameterTypes) {
+  return wrap(Builder->createSubroutineType(
+      DITypeRefArray(unwrap<MDTuple>(ParameterTypes))));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    const char *LinkageName, size_t LinkageNameLen,
+    LLVMMetadataRef File, unsigned LineNo,
+    LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags,
+    LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam,
+    LLVMMetadataRef Decl) {
+  DITemplateParameterArray TParams =
+      DITemplateParameterArray(unwrap<MDTuple>(TParam));
+  DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags);
+  DINode::DIFlags llvmFlags = fromRust(Flags);
+  DISubprogram *Sub = Builder->createFunction(
+      unwrapDI<DIScope>(Scope),
+      StringRef(Name, NameLen),
+      StringRef(LinkageName, LinkageNameLen),
+      unwrapDI<DIFile>(File), LineNo,
+      unwrapDI<DISubroutineType>(Ty), ScopeLine, llvmFlags,
+      llvmSPFlags, TParams, unwrapDIPtr<DISubprogram>(Decl));
+  if (MaybeFn)
+    unwrap<Function>(MaybeFn)->setSubprogram(Sub);
+  return wrap(Sub);
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    const char *LinkageName, size_t LinkageNameLen,
+    LLVMMetadataRef File, unsigned LineNo,
+    LLVMMetadataRef Ty, LLVMRustDIFlags Flags,
+    LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) {
+  DITemplateParameterArray TParams =
+      DITemplateParameterArray(unwrap<MDTuple>(TParam));
+  DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags);
+  DINode::DIFlags llvmFlags = fromRust(Flags);
+  DISubprogram *Sub = Builder->createMethod(
+      unwrapDI<DIScope>(Scope),
+      StringRef(Name, NameLen),
+      StringRef(LinkageName, LinkageNameLen),
+      unwrapDI<DIFile>(File), LineNo,
+      unwrapDI<DISubroutineType>(Ty),
+      0, 0, nullptr, // VTable params aren't used
+      llvmFlags, llvmSPFlags, TParams);
+  return wrap(Sub);
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType(
+    LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen,
+    uint64_t SizeInBits, unsigned Encoding) {
+  return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) {
+  return wrap(Builder->createTypedef(
+    unwrap<DIType>(Type), StringRef(Name, NameLen), unwrap<DIFile>(File),
+    LineNo, unwrapDIPtr<DIScope>(Scope)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy,
+    uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace,
+    const char *Name, size_t NameLen) {
+  return wrap(Builder->createPointerType(unwrapDI<DIType>(PointeeTy),
+                                         SizeInBits, AlignInBits,
+                                         AddressSpace,
+                                         StringRef(Name, NameLen)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
+    uint32_t AlignInBits, LLVMRustDIFlags Flags,
+    LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements,
+    unsigned RunTimeLang, LLVMMetadataRef VTableHolder,
+    const char *UniqueId, size_t UniqueIdLen) {
+  return wrap(Builder->createStructType(
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
+      unwrapDI<DIFile>(File), LineNumber,
+      SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIType>(DerivedFrom),
+      DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang,
+      unwrapDI<DIType>(VTableHolder), StringRef(UniqueId, UniqueIdLen)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
+    uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator,
+    LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) {
+  return wrap(Builder->createVariantPart(
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
+      unwrapDI<DIFile>(File), LineNumber,
+      SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIDerivedType>(Discriminator),
+      DINodeArray(unwrapDI<MDTuple>(Elements)), StringRef(UniqueId, UniqueIdLen)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits,
+    uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags,
+    LLVMMetadataRef Ty) {
+  return wrap(Builder->createMemberType(unwrapDI<DIDescriptor>(Scope),
+                                        StringRef(Name, NameLen),
+                                        unwrapDI<DIFile>(File), LineNo,
+                                        SizeInBits, AlignInBits, OffsetInBits,
+                                        fromRust(Flags), unwrapDI<DIType>(Ty)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo,
+    uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant,
+    LLVMRustDIFlags Flags, LLVMMetadataRef Ty) {
+  llvm::ConstantInt* D = nullptr;
+  if (Discriminant) {
+    D = unwrap<llvm::ConstantInt>(Discriminant);
+  }
+  return wrap(Builder->createVariantMemberType(unwrapDI<DIDescriptor>(Scope),
+                                               StringRef(Name, NameLen),
+                                               unwrapDI<DIFile>(File), LineNo,
+                                               SizeInBits, AlignInBits, OffsetInBits, D,
+                                               fromRust(Flags), unwrapDI<DIType>(Ty)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticMemberType(
+    LLVMRustDIBuilderRef Builder,
+    LLVMMetadataRef Scope,
+    const char *Name,
+    size_t NameLen,
+    LLVMMetadataRef File,
+    unsigned LineNo,
+    LLVMMetadataRef Ty,
+    LLVMRustDIFlags Flags,
+    LLVMValueRef val,
+    uint32_t AlignInBits
+) {
+  return wrap(Builder->createStaticMemberType(
+    unwrapDI<DIDescriptor>(Scope),
+    StringRef(Name, NameLen),
+    unwrapDI<DIFile>(File),
+    LineNo,
+    unwrapDI<DIType>(Ty),
+    fromRust(Flags),
+    unwrap<llvm::ConstantInt>(val),
+#if LLVM_VERSION_GE(18, 0)
+    llvm::dwarf::DW_TAG_member,
+#endif
+    AlignInBits
+  ));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    LLVMMetadataRef File, unsigned Line, unsigned Col) {
+  return wrap(Builder->createLexicalBlock(unwrapDI<DIDescriptor>(Scope),
+                                          unwrapDI<DIFile>(File), Line, Col));
+}
+
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder,
+                                        LLVMMetadataRef Scope,
+                                        LLVMMetadataRef File) {
+  return wrap(Builder->createLexicalBlockFile(unwrapDI<DIDescriptor>(Scope),
+                                              unwrapDI<DIFile>(File)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context,
+    const char *Name, size_t NameLen,
+    const char *LinkageName, size_t LinkageNameLen,
+    LLVMMetadataRef File, unsigned LineNo,
+    LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V,
+    LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0) {
+  llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V));
+
+  llvm::DIExpression *InitExpr = nullptr;
+  if (llvm::ConstantInt *IntVal = llvm::dyn_cast<llvm::ConstantInt>(InitVal)) {
+    InitExpr = Builder->createConstantValueExpression(
+        IntVal->getValue().getSExtValue());
+  } else if (llvm::ConstantFP *FPVal =
+                 llvm::dyn_cast<llvm::ConstantFP>(InitVal)) {
+    InitExpr = Builder->createConstantValueExpression(
+        FPVal->getValueAPF().bitcastToAPInt().getZExtValue());
+  }
+
+  llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression(
+      unwrapDI<DIDescriptor>(Context), StringRef(Name, NameLen),
+      StringRef(LinkageName, LinkageNameLen),
+      unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), IsLocalToUnit,
+      /* isDefined */ true,
+      InitExpr, unwrapDIPtr<MDNode>(Decl),
+      /* templateParams */ nullptr,
+      AlignInBits);
+
+  InitVal->setMetadata("dbg", VarExpr);
+
+  return wrap(VarExpr);
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable(
+    LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo,
+    LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags,
+    unsigned ArgNo, uint32_t AlignInBits) {
+  if (Tag == 0x100) { // DW_TAG_auto_variable
+    return wrap(Builder->createAutoVariable(
+        unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
+        unwrapDI<DIFile>(File), LineNo,
+        unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits));
+  } else {
+    return wrap(Builder->createParameterVariable(
+        unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), ArgNo,
+        unwrapDI<DIFile>(File), LineNo,
+        unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags)));
+  }
+}
+
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size,
+                                 uint32_t AlignInBits, LLVMMetadataRef Ty,
+                                 LLVMMetadataRef Subscripts) {
+  return wrap(
+      Builder->createArrayType(Size, AlignInBits, unwrapDI<DIType>(Ty),
+                               DINodeArray(unwrapDI<MDTuple>(Subscripts))));
+}
+
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo,
+                                     int64_t Count) {
+  return wrap(Builder->getOrCreateSubrange(Lo, Count));
+}
+
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder,
+                                  LLVMMetadataRef *Ptr, unsigned Count) {
+  Metadata **DataValue = unwrap(Ptr);
+  return wrap(
+      Builder->getOrCreateArray(ArrayRef<Metadata *>(DataValue, Count)).get());
+}
+
+extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd(
+    LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo,
+    uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL,
+    LLVMBasicBlockRef InsertAtEnd) {
+  return wrap(Builder->insertDeclare(
+      unwrap(V), unwrap<DILocalVariable>(VarInfo),
+      Builder->createExpression(llvm::ArrayRef<uint64_t>(AddrOps, AddrOpsCount)),
+      DebugLoc(cast<MDNode>(unwrap(DL))),
+      unwrap(InsertAtEnd)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator(
+    LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen,
+    const uint64_t Value[2], unsigned SizeInBits, bool IsUnsigned) {
+  return wrap(Builder->createEnumerator(StringRef(Name, NameLen),
+      APSInt(APInt(SizeInBits, ArrayRef<uint64_t>(Value, 2)), IsUnsigned)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
+    uint32_t AlignInBits, LLVMMetadataRef Elements,
+    LLVMMetadataRef ClassTy, bool IsScoped) {
+  return wrap(Builder->createEnumerationType(
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
+      unwrapDI<DIFile>(File), LineNumber,
+      SizeInBits, AlignInBits, DINodeArray(unwrapDI<MDTuple>(Elements)),
+      unwrapDI<DIType>(ClassTy),
+#if LLVM_VERSION_GE(18, 0)
+      /* RunTimeLang */ 0,
+#endif
+      "", IsScoped));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
+    uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements,
+    unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) {
+  return wrap(Builder->createUnionType(
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File),
+      LineNumber, SizeInBits, AlignInBits, fromRust(Flags),
+      DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang,
+      StringRef(UniqueId, UniqueIdLen)));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen, LLVMMetadataRef Ty) {
+  bool IsDefault = false; // FIXME: should we ever set this true?
+  return wrap(Builder->createTemplateTypeParameter(
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIType>(Ty), IsDefault));
+}
+
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateNameSpace(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen, bool ExportSymbols) {
+  return wrap(Builder->createNameSpace(
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), ExportSymbols
+  ));
+}
+
+extern "C" void
+LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder,
+                                     LLVMMetadataRef CompositeTy,
+                                     LLVMMetadataRef Elements,
+                                     LLVMMetadataRef Params) {
+  DICompositeType *Tmp = unwrapDI<DICompositeType>(CompositeTy);
+  Builder->replaceArrays(Tmp, DINodeArray(unwrap<MDTuple>(Elements)),
+                         DINodeArray(unwrap<MDTuple>(Params)));
+}
+
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column,
+                                     LLVMMetadataRef ScopeRef,
+                                     LLVMMetadataRef InlinedAt) {
+  MDNode *Scope = unwrapDIPtr<MDNode>(ScopeRef);
+  DILocation *Loc = DILocation::get(
+      Scope->getContext(), Line, Column, Scope,
+      unwrapDIPtr<MDNode>(InlinedAt));
+  return wrap(Loc);
+}
+
+extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() {
+  return dwarf::DW_OP_deref;
+}
+
+extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() {
+  return dwarf::DW_OP_plus_uconst;
+}
+
+extern "C" int64_t LLVMRustDIBuilderCreateOpLLVMFragment() {
+  return dwarf::DW_OP_LLVM_fragment;
+}
+
+extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) {
+  RawRustStringOstream OS(Str);
+  unwrap<llvm::Type>(Ty)->print(OS);
+}
+
+extern "C" void LLVMRustWriteValueToString(LLVMValueRef V,
+                                           RustStringRef Str) {
+  RawRustStringOstream OS(Str);
+  if (!V) {
+    OS << "(null)";
+  } else {
+    OS << "(";
+    unwrap<llvm::Value>(V)->getType()->print(OS);
+    OS << ":";
+    unwrap<llvm::Value>(V)->print(OS);
+    OS << ")";
+  }
+}
+
+// LLVMArrayType function does not support 64-bit ElementCount
+// FIXME: replace with LLVMArrayType2 when bumped minimal version to llvm-17
+// https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc
+extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy,
+                                         uint64_t ElementCount) {
+  return wrap(ArrayType::get(unwrap(ElementTy), ElementCount));
+}
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef)
+
+extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) {
+  RawRustStringOstream OS(Str);
+  unwrap(T)->print(OS);
+}
+
+extern "C" void LLVMRustUnpackOptimizationDiagnostic(
+    LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut,
+    LLVMValueRef *FunctionOut, unsigned* Line, unsigned* Column,
+    RustStringRef FilenameOut, RustStringRef MessageOut) {
+  // Undefined to call this not on an optimization diagnostic!
+  llvm::DiagnosticInfoOptimizationBase *Opt =
+      static_cast<llvm::DiagnosticInfoOptimizationBase *>(unwrap(DI));
+
+  RawRustStringOstream PassNameOS(PassNameOut);
+  PassNameOS << Opt->getPassName();
+  *FunctionOut = wrap(&Opt->getFunction());
+
+  RawRustStringOstream FilenameOS(FilenameOut);
+  DiagnosticLocation loc = Opt->getLocation();
+  if (loc.isValid()) {
+    *Line = loc.getLine();
+    *Column = loc.getColumn();
+    FilenameOS << loc.getAbsolutePath();
+  }
+
+  RawRustStringOstream MessageOS(MessageOut);
+  MessageOS << Opt->getMsg();
+}
+
+enum class LLVMRustDiagnosticLevel {
+    Error,
+    Warning,
+    Note,
+    Remark,
+};
+
+extern "C" void
+LLVMRustUnpackInlineAsmDiagnostic(LLVMDiagnosticInfoRef DI,
+                                  LLVMRustDiagnosticLevel *LevelOut,
+                                  unsigned *CookieOut,
+                                  LLVMTwineRef *MessageOut) {
+  // Undefined to call this not on an inline assembly diagnostic!
+  llvm::DiagnosticInfoInlineAsm *IA =
+      static_cast<llvm::DiagnosticInfoInlineAsm *>(unwrap(DI));
+
+  *CookieOut = IA->getLocCookie();
+  *MessageOut = wrap(&IA->getMsgStr());
+
+  switch (IA->getSeverity()) {
+    case DS_Error:
+      *LevelOut = LLVMRustDiagnosticLevel::Error;
+      break;
+    case DS_Warning:
+      *LevelOut = LLVMRustDiagnosticLevel::Warning;
+      break;
+    case DS_Note:
+      *LevelOut = LLVMRustDiagnosticLevel::Note;
+      break;
+    case DS_Remark:
+      *LevelOut = LLVMRustDiagnosticLevel::Remark;
+      break;
+    default:
+      report_fatal_error("Invalid LLVMRustDiagnosticLevel value!");
+  }
+}
+
+extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI,
+                                                    RustStringRef Str) {
+  RawRustStringOstream OS(Str);
+  DiagnosticPrinterRawOStream DP(OS);
+  unwrap(DI)->print(DP);
+}
+
+enum class LLVMRustDiagnosticKind {
+  Other,
+  InlineAsm,
+  StackSize,
+  DebugMetadataVersion,
+  SampleProfile,
+  OptimizationRemark,
+  OptimizationRemarkMissed,
+  OptimizationRemarkAnalysis,
+  OptimizationRemarkAnalysisFPCommute,
+  OptimizationRemarkAnalysisAliasing,
+  OptimizationRemarkOther,
+  OptimizationFailure,
+  PGOProfile,
+  Linker,
+  Unsupported,
+  SrcMgr,
+};
+
+static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) {
+  switch (Kind) {
+  case DK_InlineAsm:
+    return LLVMRustDiagnosticKind::InlineAsm;
+  case DK_StackSize:
+    return LLVMRustDiagnosticKind::StackSize;
+  case DK_DebugMetadataVersion:
+    return LLVMRustDiagnosticKind::DebugMetadataVersion;
+  case DK_SampleProfile:
+    return LLVMRustDiagnosticKind::SampleProfile;
+  case DK_OptimizationRemark:
+  case DK_MachineOptimizationRemark:
+    return LLVMRustDiagnosticKind::OptimizationRemark;
+  case DK_OptimizationRemarkMissed:
+  case DK_MachineOptimizationRemarkMissed:
+    return LLVMRustDiagnosticKind::OptimizationRemarkMissed;
+  case DK_OptimizationRemarkAnalysis:
+  case DK_MachineOptimizationRemarkAnalysis:
+    return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis;
+  case DK_OptimizationRemarkAnalysisFPCommute:
+    return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute;
+  case DK_OptimizationRemarkAnalysisAliasing:
+    return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing;
+  case DK_PGOProfile:
+    return LLVMRustDiagnosticKind::PGOProfile;
+  case DK_Linker:
+    return LLVMRustDiagnosticKind::Linker;
+  case DK_Unsupported:
+    return LLVMRustDiagnosticKind::Unsupported;
+  case DK_SrcMgr:
+    return LLVMRustDiagnosticKind::SrcMgr;
+  default:
+    return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark)
+               ? LLVMRustDiagnosticKind::OptimizationRemarkOther
+               : LLVMRustDiagnosticKind::Other;
+  }
+}
+
+extern "C" LLVMRustDiagnosticKind
+LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) {
+  return toRust((DiagnosticKind)unwrap(DI)->getKind());
+}
+
+// This is kept distinct from LLVMGetTypeKind, because when
+// a new type kind is added, the Rust-side enum must be
+// updated or UB will result.
+extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) {
+  switch (unwrap(Ty)->getTypeID()) {
+  case Type::VoidTyID:
+    return LLVMVoidTypeKind;
+  case Type::HalfTyID:
+    return LLVMHalfTypeKind;
+  case Type::FloatTyID:
+    return LLVMFloatTypeKind;
+  case Type::DoubleTyID:
+    return LLVMDoubleTypeKind;
+  case Type::X86_FP80TyID:
+    return LLVMX86_FP80TypeKind;
+  case Type::FP128TyID:
+    return LLVMFP128TypeKind;
+  case Type::PPC_FP128TyID:
+    return LLVMPPC_FP128TypeKind;
+  case Type::LabelTyID:
+    return LLVMLabelTypeKind;
+  case Type::MetadataTyID:
+    return LLVMMetadataTypeKind;
+  case Type::IntegerTyID:
+    return LLVMIntegerTypeKind;
+  case Type::FunctionTyID:
+    return LLVMFunctionTypeKind;
+  case Type::StructTyID:
+    return LLVMStructTypeKind;
+  case Type::ArrayTyID:
+    return LLVMArrayTypeKind;
+  case Type::PointerTyID:
+    return LLVMPointerTypeKind;
+  case Type::FixedVectorTyID:
+    return LLVMVectorTypeKind;
+  case Type::X86_MMXTyID:
+    return LLVMX86_MMXTypeKind;
+  case Type::TokenTyID:
+    return LLVMTokenTypeKind;
+  case Type::ScalableVectorTyID:
+    return LLVMScalableVectorTypeKind;
+  case Type::BFloatTyID:
+    return LLVMBFloatTypeKind;
+  case Type::X86_AMXTyID:
+    return LLVMX86_AMXTypeKind;
+  default:
+    {
+      std::string error;
+      llvm::raw_string_ostream stream(error);
+      stream << "Rust does not support the TypeID: " << unwrap(Ty)->getTypeID()
+             << " for the type: " << *unwrap(Ty);
+      stream.flush();
+      report_fatal_error(error.c_str());
+    }
+  }
+}
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef)
+
+extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic(
+    LLVMDiagnosticInfoRef DI, unsigned *Cookie) {
+  llvm::DiagnosticInfoSrcMgr *SM = static_cast<llvm::DiagnosticInfoSrcMgr *>(unwrap(DI));
+  *Cookie = SM->getLocCookie();
+  return wrap(&SM->getSMDiag());
+}
+
+extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef,
+                                           RustStringRef MessageOut,
+                                           RustStringRef BufferOut,
+                                           LLVMRustDiagnosticLevel* LevelOut,
+                                           unsigned* LocOut,
+                                           unsigned* RangesOut,
+                                           size_t* NumRanges) {
+  SMDiagnostic& D = *unwrap(DRef);
+  RawRustStringOstream MessageOS(MessageOut);
+  MessageOS << D.getMessage();
+
+  switch (D.getKind()) {
+    case SourceMgr::DK_Error:
+      *LevelOut = LLVMRustDiagnosticLevel::Error;
+      break;
+    case SourceMgr::DK_Warning:
+      *LevelOut = LLVMRustDiagnosticLevel::Warning;
+      break;
+    case SourceMgr::DK_Note:
+      *LevelOut = LLVMRustDiagnosticLevel::Note;
+      break;
+    case SourceMgr::DK_Remark:
+      *LevelOut = LLVMRustDiagnosticLevel::Remark;
+      break;
+    default:
+      report_fatal_error("Invalid LLVMRustDiagnosticLevel value!");
+  }
+
+  if (D.getLoc() == SMLoc())
+    return false;
+
+  const SourceMgr &LSM = *D.getSourceMgr();
+  const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc()));
+  LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), LBuf->getBufferSize());
+
+  *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart();
+
+  *NumRanges = std::min(*NumRanges, D.getRanges().size());
+  size_t LineStart = *LocOut - (size_t)D.getColumnNo();
+  for (size_t i = 0; i < *NumRanges; i++) {
+    RangesOut[i * 2] = LineStart + D.getRanges()[i].first;
+    RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second;
+  }
+
+  return true;
+}
+
+extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name,
+                                                           LLVMValueRef *Inputs,
+                                                           unsigned NumInputs) {
+  return new OperandBundleDef(Name, ArrayRef<Value*>(unwrap(Inputs), NumInputs));
+}
+
+extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) {
+  delete Bundle;
+}
+
+extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                                          LLVMValueRef *Args, unsigned NumArgs,
+                                          OperandBundleDef **OpBundles,
+                                          unsigned NumOpBundles) {
+  Value *Callee = unwrap(Fn);
+  FunctionType *FTy = unwrap<FunctionType>(Ty);
+  return wrap(unwrap(B)->CreateCall(
+      FTy, Callee, ArrayRef<Value*>(unwrap(Args), NumArgs),
+      ArrayRef<OperandBundleDef>(*OpBundles, NumOpBundles)));
+}
+
+extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
+  return wrap(llvm::Intrinsic::getDeclaration(unwrap(M),
+              (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment));
+}
+
+extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B,
+                                            LLVMValueRef Dst, unsigned DstAlign,
+                                            LLVMValueRef Src, unsigned SrcAlign,
+                                            LLVMValueRef Size, bool IsVolatile) {
+  return wrap(unwrap(B)->CreateMemCpy(
+      unwrap(Dst), MaybeAlign(DstAlign),
+      unwrap(Src), MaybeAlign(SrcAlign),
+      unwrap(Size), IsVolatile));
+}
+
+extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
+                                             LLVMValueRef Dst, unsigned DstAlign,
+                                             LLVMValueRef Src, unsigned SrcAlign,
+                                             LLVMValueRef Size, bool IsVolatile) {
+  return wrap(unwrap(B)->CreateMemMove(
+      unwrap(Dst), MaybeAlign(DstAlign),
+      unwrap(Src), MaybeAlign(SrcAlign),
+      unwrap(Size), IsVolatile));
+}
+
+extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B,
+                                            LLVMValueRef Dst, unsigned DstAlign,
+                                            LLVMValueRef Val,
+                                            LLVMValueRef Size, bool IsVolatile) {
+  return wrap(unwrap(B)->CreateMemSet(
+      unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile));
+}
+
+extern "C" LLVMValueRef
+LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                    LLVMValueRef *Args, unsigned NumArgs,
+                    LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
+                    OperandBundleDef **OpBundles, unsigned NumOpBundles,
+                    const char *Name) {
+  Value *Callee = unwrap(Fn);
+  FunctionType *FTy = unwrap<FunctionType>(Ty);
+  return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch),
+                                      ArrayRef<Value*>(unwrap(Args), NumArgs),
+                                      ArrayRef<OperandBundleDef>(*OpBundles, NumOpBundles),
+                                      Name));
+}
+
+extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B,
+                                               LLVMBasicBlockRef BB) {
+  auto Point = unwrap(BB)->getFirstInsertionPt();
+  unwrap(B)->SetInsertPoint(unwrap(BB), Point);
+}
+
+extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V,
+                                  const char *Name, size_t NameLen) {
+  Triple TargetTriple(unwrap(M)->getTargetTriple());
+  GlobalObject *GV = unwrap<GlobalObject>(V);
+  if (TargetTriple.supportsCOMDAT()) {
+    StringRef NameRef(Name, NameLen);
+    GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef));
+  }
+}
+
+enum class LLVMRustLinkage {
+  ExternalLinkage = 0,
+  AvailableExternallyLinkage = 1,
+  LinkOnceAnyLinkage = 2,
+  LinkOnceODRLinkage = 3,
+  WeakAnyLinkage = 4,
+  WeakODRLinkage = 5,
+  AppendingLinkage = 6,
+  InternalLinkage = 7,
+  PrivateLinkage = 8,
+  ExternalWeakLinkage = 9,
+  CommonLinkage = 10,
+};
+
+static LLVMRustLinkage toRust(LLVMLinkage Linkage) {
+  switch (Linkage) {
+  case LLVMExternalLinkage:
+    return LLVMRustLinkage::ExternalLinkage;
+  case LLVMAvailableExternallyLinkage:
+    return LLVMRustLinkage::AvailableExternallyLinkage;
+  case LLVMLinkOnceAnyLinkage:
+    return LLVMRustLinkage::LinkOnceAnyLinkage;
+  case LLVMLinkOnceODRLinkage:
+    return LLVMRustLinkage::LinkOnceODRLinkage;
+  case LLVMWeakAnyLinkage:
+    return LLVMRustLinkage::WeakAnyLinkage;
+  case LLVMWeakODRLinkage:
+    return LLVMRustLinkage::WeakODRLinkage;
+  case LLVMAppendingLinkage:
+    return LLVMRustLinkage::AppendingLinkage;
+  case LLVMInternalLinkage:
+    return LLVMRustLinkage::InternalLinkage;
+  case LLVMPrivateLinkage:
+    return LLVMRustLinkage::PrivateLinkage;
+  case LLVMExternalWeakLinkage:
+    return LLVMRustLinkage::ExternalWeakLinkage;
+  case LLVMCommonLinkage:
+    return LLVMRustLinkage::CommonLinkage;
+  default:
+    report_fatal_error("Invalid LLVMRustLinkage value!");
+  }
+}
+
+static LLVMLinkage fromRust(LLVMRustLinkage Linkage) {
+  switch (Linkage) {
+  case LLVMRustLinkage::ExternalLinkage:
+    return LLVMExternalLinkage;
+  case LLVMRustLinkage::AvailableExternallyLinkage:
+    return LLVMAvailableExternallyLinkage;
+  case LLVMRustLinkage::LinkOnceAnyLinkage:
+    return LLVMLinkOnceAnyLinkage;
+  case LLVMRustLinkage::LinkOnceODRLinkage:
+    return LLVMLinkOnceODRLinkage;
+  case LLVMRustLinkage::WeakAnyLinkage:
+    return LLVMWeakAnyLinkage;
+  case LLVMRustLinkage::WeakODRLinkage:
+    return LLVMWeakODRLinkage;
+  case LLVMRustLinkage::AppendingLinkage:
+    return LLVMAppendingLinkage;
+  case LLVMRustLinkage::InternalLinkage:
+    return LLVMInternalLinkage;
+  case LLVMRustLinkage::PrivateLinkage:
+    return LLVMPrivateLinkage;
+  case LLVMRustLinkage::ExternalWeakLinkage:
+    return LLVMExternalWeakLinkage;
+  case LLVMRustLinkage::CommonLinkage:
+    return LLVMCommonLinkage;
+  }
+  report_fatal_error("Invalid LLVMRustLinkage value!");
+}
+
+extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) {
+  return toRust(LLVMGetLinkage(V));
+}
+
+extern "C" void LLVMRustSetLinkage(LLVMValueRef V,
+                                   LLVMRustLinkage RustLinkage) {
+  LLVMSetLinkage(V, fromRust(RustLinkage));
+}
+
+extern "C" bool LLVMRustConstIntGetZExtValue(LLVMValueRef CV, uint64_t *value) {
+    auto C = unwrap<llvm::ConstantInt>(CV);
+    if (C->getBitWidth() > 64)
+      return false;
+    *value = C->getZExtValue();
+    return true;
+}
+
+// Returns true if both high and low were successfully set. Fails in case constant wasn’t any of
+// the common sizes (1, 8, 16, 32, 64, 128 bits)
+extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *high, uint64_t *low)
+{
+    auto C = unwrap<llvm::ConstantInt>(CV);
+    if (C->getBitWidth() > 128) { return false; }
+    APInt AP;
+    if (sext) {
+        AP = C->getValue().sext(128);
+    } else {
+        AP = C->getValue().zext(128);
+    }
+    *low = AP.getLoBits(64).getZExtValue();
+    *high = AP.getHiBits(64).getZExtValue();
+    return true;
+}
+
+enum class LLVMRustVisibility {
+  Default = 0,
+  Hidden = 1,
+  Protected = 2,
+};
+
+static LLVMRustVisibility toRust(LLVMVisibility Vis) {
+  switch (Vis) {
+  case LLVMDefaultVisibility:
+    return LLVMRustVisibility::Default;
+  case LLVMHiddenVisibility:
+    return LLVMRustVisibility::Hidden;
+  case LLVMProtectedVisibility:
+    return LLVMRustVisibility::Protected;
+  }
+  report_fatal_error("Invalid LLVMRustVisibility value!");
+}
+
+static LLVMVisibility fromRust(LLVMRustVisibility Vis) {
+  switch (Vis) {
+  case LLVMRustVisibility::Default:
+    return LLVMDefaultVisibility;
+  case LLVMRustVisibility::Hidden:
+    return LLVMHiddenVisibility;
+  case LLVMRustVisibility::Protected:
+    return LLVMProtectedVisibility;
+  }
+  report_fatal_error("Invalid LLVMRustVisibility value!");
+}
+
+extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) {
+  return toRust(LLVMGetVisibility(V));
+}
+
+extern "C" void LLVMRustSetVisibility(LLVMValueRef V,
+                                      LLVMRustVisibility RustVisibility) {
+  LLVMSetVisibility(V, fromRust(RustVisibility));
+}
+
+extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) {
+  unwrap<GlobalValue>(Global)->setDSOLocal(is_dso_local);
+}
+
+struct LLVMRustModuleBuffer {
+  std::string data;
+};
+
+extern "C" LLVMRustModuleBuffer*
+LLVMRustModuleBufferCreate(LLVMModuleRef M) {
+  auto Ret = std::make_unique<LLVMRustModuleBuffer>();
+  {
+    raw_string_ostream OS(Ret->data);
+    WriteBitcodeToFile(*unwrap(M), OS);
+  }
+  return Ret.release();
+}
+
+extern "C" void
+LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) {
+  delete Buffer;
+}
+
+extern "C" const void*
+LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) {
+  return Buffer->data.data();
+}
+
+extern "C" size_t
+LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) {
+  return Buffer->data.length();
+}
+
+extern "C" uint64_t
+LLVMRustModuleCost(LLVMModuleRef M) {
+  auto f = unwrap(M)->functions();
+  return std::distance(std::begin(f), std::end(f));
+}
+
+extern "C" void
+LLVMRustModuleInstructionStats(LLVMModuleRef M, RustStringRef Str)
+{
+  RawRustStringOstream OS(Str);
+  llvm::json::OStream JOS(OS);
+  auto Module = unwrap(M);
+
+  JOS.object([&] {
+    JOS.attribute("module", Module->getName());
+    JOS.attribute("total", Module->getInstructionCount());
+  });
+}
+
+// Vector reductions:
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc),unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateAddReduce(unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateMulReduce(unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateAndReduce(unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateOrReduce(unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, LLVMValueRef Src) {
+    return wrap(unwrap(B)->CreateXorReduce(unwrap(Src)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) {
+    return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) {
+    return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) {
+  Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src));
+  I->setHasNoNaNs(NoNaN);
+  return wrap(I);
+}
+extern "C" LLVMValueRef
+LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) {
+  Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src));
+  I->setHasNoNaNs(NoNaN);
+  return wrap(I);
+}
+
+extern "C" LLVMValueRef
+LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) {
+    return wrap(unwrap(B)->CreateMinNum(unwrap(LHS),unwrap(RHS)));
+}
+extern "C" LLVMValueRef
+LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) {
+    return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS)));
+}
+
+// This struct contains all necessary info about a symbol exported from a DLL.
+struct LLVMRustCOFFShortExport {
+  const char* name;
+  bool ordinal_present;
+  // The value of `ordinal` is only meaningful if `ordinal_present` is true.
+  uint16_t ordinal;
+};
+
+// Machine must be a COFF machine type, as defined in PE specs.
+extern "C" LLVMRustResult LLVMRustWriteImportLibrary(
+  const char* ImportName,
+  const char* Path,
+  const LLVMRustCOFFShortExport* Exports,
+  size_t NumExports,
+  uint16_t Machine,
+  bool MinGW)
+{
+  std::vector<llvm::object::COFFShortExport> ConvertedExports;
+  ConvertedExports.reserve(NumExports);
+
+  for (size_t i = 0; i < NumExports; ++i) {
+    bool ordinal_present = Exports[i].ordinal_present;
+    uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0;
+    ConvertedExports.push_back(llvm::object::COFFShortExport{
+      Exports[i].name,  // Name
+      std::string{},    // ExtName
+      std::string{},    // SymbolName
+      std::string{},    // AliasTarget
+      ordinal,          // Ordinal
+      ordinal_present,  // Noname
+      false,            // Data
+      false,            // Private
+      false             // Constant
+    });
+  }
+
+  auto Error = llvm::object::writeImportLibrary(
+    ImportName,
+    Path,
+    ConvertedExports,
+    static_cast<llvm::COFF::MachineTypes>(Machine),
+    MinGW);
+  if (Error) {
+    std::string errorString;
+    llvm::raw_string_ostream stream(errorString);
+    stream << Error;
+    stream.flush();
+    LLVMRustSetLastError(errorString.c_str());
+    return LLVMRustResult::Failure;
+  } else {
+    return LLVMRustResult::Success;
+  }
+}
+
+// Transfers ownership of DiagnosticHandler unique_ptr to the caller.
+extern "C" DiagnosticHandler *
+LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) {
+  std::unique_ptr<DiagnosticHandler> DH = unwrap(C)->getDiagnosticHandler();
+  return DH.release();
+}
+
+// Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic
+// handling. Ownership of the handler is moved to the LLVMContext.
+extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C,
+                                                    DiagnosticHandler *DH) {
+  unwrap(C)->setDiagnosticHandler(std::unique_ptr<DiagnosticHandler>(DH));
+}
+
+using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy;
+
+// Configures a diagnostic handler that invokes provided callback when a
+// backend needs to emit a diagnostic.
+//
+// When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise
+// the RemarkPasses array specifies individual passes for which remarks will be
+// enabled.
+//
+// If RemarkFilePath is not NULL, optimization remarks will be streamed directly into this file,
+// bypassing the diagnostics handler.
+extern "C" void LLVMRustContextConfigureDiagnosticHandler(
+    LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback,
+    void *DiagnosticHandlerContext, bool RemarkAllPasses,
+    const char * const * RemarkPasses, size_t RemarkPassesLen,
+    const char * RemarkFilePath,
+    bool PGOAvailable
+) {
+
+  class RustDiagnosticHandler final : public DiagnosticHandler {
+  public:
+    RustDiagnosticHandler(
+      LLVMDiagnosticHandlerTy DiagnosticHandlerCallback,
+      void *DiagnosticHandlerContext,
+      bool RemarkAllPasses,
+      std::vector<std::string> RemarkPasses,
+      std::unique_ptr<ToolOutputFile> RemarksFile,
+      std::unique_ptr<llvm::remarks::RemarkStreamer> RemarkStreamer,
+      std::unique_ptr<LLVMRemarkStreamer> LlvmRemarkStreamer
+    )
+        : DiagnosticHandlerCallback(DiagnosticHandlerCallback),
+          DiagnosticHandlerContext(DiagnosticHandlerContext),
+          RemarkAllPasses(RemarkAllPasses),
+          RemarkPasses(std::move(RemarkPasses)),
+          RemarksFile(std::move(RemarksFile)),
+          RemarkStreamer(std::move(RemarkStreamer)),
+          LlvmRemarkStreamer(std::move(LlvmRemarkStreamer)) {}
+
+    virtual bool handleDiagnostics(const DiagnosticInfo &DI) override {
+      // If this diagnostic is one of the optimization remark kinds, we can check if it's enabled
+      // before emitting it. This can avoid many short-lived allocations when unpacking the
+      // diagnostic and converting its various C++ strings into rust strings.
+      // FIXME: some diagnostic infos still allocate before we get here, and avoiding that would be
+      // good in the future. That will require changing a few call sites in LLVM.
+      if (auto *OptDiagBase = dyn_cast<DiagnosticInfoOptimizationBase>(&DI)) {
+        if (OptDiagBase->isEnabled()) {
+          if (this->LlvmRemarkStreamer) {
+            this->LlvmRemarkStreamer->emit(*OptDiagBase);
+            return true;
+          }
+        } else {
+          return true;
+        }
+      }
+      if (DiagnosticHandlerCallback) {
+        DiagnosticHandlerCallback(DI, DiagnosticHandlerContext);
+        return true;
+      }
+      return false;
+    }
+
+    bool isAnalysisRemarkEnabled(StringRef PassName) const override {
+      return isRemarkEnabled(PassName);
+    }
+
+    bool isMissedOptRemarkEnabled(StringRef PassName) const override {
+      return isRemarkEnabled(PassName);
+    }
+
+    bool isPassedOptRemarkEnabled(StringRef PassName) const override {
+      return isRemarkEnabled(PassName);
+    }
+
+    bool isAnyRemarkEnabled() const override {
+      return RemarkAllPasses || !RemarkPasses.empty();
+    }
+
+  private:
+    bool isRemarkEnabled(StringRef PassName) const {
+      if (RemarkAllPasses)
+        return true;
+
+      for (auto &Pass : RemarkPasses)
+        if (Pass == PassName)
+          return true;
+
+      return false;
+    }
+
+    LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr;
+    void *DiagnosticHandlerContext = nullptr;
+
+    bool RemarkAllPasses = false;
+    std::vector<std::string> RemarkPasses;
+
+    // Since LlvmRemarkStreamer contains a pointer to RemarkStreamer, the ordering of the three
+    // members below is important.
+    std::unique_ptr<ToolOutputFile> RemarksFile;
+    std::unique_ptr<llvm::remarks::RemarkStreamer> RemarkStreamer;
+    std::unique_ptr<LLVMRemarkStreamer> LlvmRemarkStreamer;
+  };
+
+  std::vector<std::string> Passes;
+  for (size_t I = 0; I != RemarkPassesLen; ++I)
+  {
+    Passes.push_back(RemarkPasses[I]);
+  }
+
+  // We need to hold onto both the streamers and the opened file
+  std::unique_ptr<ToolOutputFile> RemarkFile;
+  std::unique_ptr<llvm::remarks::RemarkStreamer> RemarkStreamer;
+  std::unique_ptr<LLVMRemarkStreamer> LlvmRemarkStreamer;
+
+  if (RemarkFilePath != nullptr) {
+    if (PGOAvailable) {
+      // Enable PGO hotness data for remarks, if available
+      unwrap(C)->setDiagnosticsHotnessRequested(true);
+    }
+
+    std::error_code EC;
+    RemarkFile = std::make_unique<ToolOutputFile>(
+      RemarkFilePath,
+      EC,
+      llvm::sys::fs::OF_TextWithCRLF
+    );
+    if (EC) {
+      std::string Error = std::string("Cannot create remark file: ") +
+              toString(errorCodeToError(EC));
+      report_fatal_error(Twine(Error));
+    }
+
+    // Do not delete the file after we gather remarks
+    RemarkFile->keep();
+
+    auto RemarkSerializer = remarks::createRemarkSerializer(
+      llvm::remarks::Format::YAML,
+      remarks::SerializerMode::Separate,
+      RemarkFile->os()
+    );
+    if (Error E = RemarkSerializer.takeError())
+    {
+      std::string Error = std::string("Cannot create remark serializer: ") + toString(std::move(E));
+      report_fatal_error(Twine(Error));
+    }
+    RemarkStreamer = std::make_unique<llvm::remarks::RemarkStreamer>(std::move(*RemarkSerializer));
+    LlvmRemarkStreamer = std::make_unique<LLVMRemarkStreamer>(*RemarkStreamer);
+  }
+
+  unwrap(C)->setDiagnosticHandler(std::make_unique<RustDiagnosticHandler>(
+    DiagnosticHandlerCallback,
+    DiagnosticHandlerContext,
+    RemarkAllPasses,
+    Passes,
+    std::move(RemarkFile),
+    std::move(RemarkStreamer),
+    std::move(LlvmRemarkStreamer)
+  ));
+}
+
+extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) {
+  RawRustStringOstream OS(Str);
+  GlobalValue *GV = unwrap<GlobalValue>(V);
+  Mangler().getNameWithPrefix(OS, GV, true);
+}
+
+extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) {
+    auto *CB = unwrap<CallBase>(CallSite);
+    switch (CB->getIntrinsicID()) {
+        case Intrinsic::arm_ldrex:
+            return 0;
+        case Intrinsic::arm_strex:
+            return 1;
+    }
+    return -1;
+}
+
+extern "C" bool LLVMRustIsBitcode(char *ptr, size_t len) {
+  return identify_magic(StringRef(ptr, len)) == file_magic::bitcode;
+}
+
+extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) {
+  if (unwrap<Value>(V)->getType()->isPointerTy()) {
+    if (auto *GV = dyn_cast<GlobalValue>(unwrap<Value>(V))) {
+      if (GV->getValueType()->isFunctionTy())
+        return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() {
+  return llvm::compression::zlib::isAvailable();
+}
+
+extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() {
+  return llvm::compression::zstd::isAvailable();
+}
diff --git a/compiler/rustc_llvm/llvm-wrapper/SuppressLLVMWarnings.h b/compiler/rustc_llvm/llvm-wrapper/SuppressLLVMWarnings.h
new file mode 100644
index 00000000000..56964e4eaa7
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/SuppressLLVMWarnings.h
@@ -0,0 +1,13 @@
+#ifndef _rustc_llvm_SuppressLLVMWarnings_h
+#define _rustc_llvm_SuppressLLVMWarnings_h
+
+// LLVM currently generates many warnings when compiled using MSVC. These warnings make it difficult
+// to diagnose real problems when working on C++ code, so we suppress them.
+
+#ifdef _MSC_VER
+#pragma warning(disable:4530) // C++ exception handler used, but unwind semantics are not enabled.
+#pragma warning(disable:4624) // 'xxx': destructor was implicitly defined as deleted
+#pragma warning(disable:4244) // conversion from 'xxx' to 'yyy', possible loss of data
+#endif
+
+#endif // _rustc_llvm_SuppressLLVMWarnings_h
diff --git a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
new file mode 100644
index 00000000000..91f84692df8
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
@@ -0,0 +1,97 @@
+// Derived from code in LLVM, which is:
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Derived from:
+// * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/include/llvm/Object/ArchiveWriter.h
+// * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/lib/Object/ArchiveWriter.cpp
+
+#include "SuppressLLVMWarnings.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Object/ObjectFile.h"
+
+using namespace llvm;
+using namespace llvm::sys;
+using namespace llvm::object;
+
+static bool isArchiveSymbol(const object::BasicSymbolRef &S) {
+  Expected<uint32_t> SymFlagsOrErr = S.getFlags();
+  if (!SymFlagsOrErr)
+    // FIXME: Actually report errors helpfully.
+    report_fatal_error(SymFlagsOrErr.takeError());
+  if (*SymFlagsOrErr & object::SymbolRef::SF_FormatSpecific)
+    return false;
+  if (!(*SymFlagsOrErr & object::SymbolRef::SF_Global))
+    return false;
+  if (*SymFlagsOrErr & object::SymbolRef::SF_Undefined)
+    return false;
+  return true;
+}
+
+typedef void *(*LLVMRustGetSymbolsCallback)(void *, const char *);
+typedef void *(*LLVMRustGetSymbolsErrorCallback)(const char *);
+
+// Note: This is implemented in C++ instead of using the C api from Rust as IRObjectFile doesn't
+// implement getSymbolName, only printSymbolName, which is inaccessible from the C api.
+extern "C" void *LLVMRustGetSymbols(
+  char *BufPtr, size_t BufLen, void *State, LLVMRustGetSymbolsCallback Callback,
+  LLVMRustGetSymbolsErrorCallback ErrorCallback) {
+  std::unique_ptr<MemoryBuffer> Buf =
+    MemoryBuffer::getMemBuffer(StringRef(BufPtr, BufLen), StringRef("LLVMRustGetSymbolsObject"),
+                               false);
+  SmallString<0> SymNameBuf;
+  raw_svector_ostream SymName(SymNameBuf);
+
+  // In the scenario when LLVMContext is populated SymbolicFile will contain a
+  // reference to it, thus SymbolicFile should be destroyed first.
+  LLVMContext Context;
+  std::unique_ptr<object::SymbolicFile> Obj;
+
+  const file_magic Type = identify_magic(Buf->getBuffer());
+  if (!object::SymbolicFile::isSymbolicFile(Type, &Context)) {
+    return 0;
+  }
+
+  if (Type == file_magic::bitcode) {
+    auto ObjOrErr = object::SymbolicFile::createSymbolicFile(
+      Buf->getMemBufferRef(), file_magic::bitcode, &Context);
+    if (!ObjOrErr) {
+      Error E = ObjOrErr.takeError();
+      SmallString<0> ErrorBuf;
+      raw_svector_ostream Error(ErrorBuf);
+      Error << E << '\0';
+      return ErrorCallback(Error.str().data());
+    }
+    Obj = std::move(*ObjOrErr);
+  } else {
+    auto ObjOrErr = object::SymbolicFile::createSymbolicFile(Buf->getMemBufferRef());
+    if (!ObjOrErr) {
+      Error E = ObjOrErr.takeError();
+      SmallString<0> ErrorBuf;
+      raw_svector_ostream Error(ErrorBuf);
+      Error << E << '\0';
+      return ErrorCallback(Error.str().data());
+    }
+    Obj = std::move(*ObjOrErr);
+  }
+
+
+  for (const object::BasicSymbolRef &S : Obj->symbols()) {
+    if (!isArchiveSymbol(S))
+      continue;
+    if (Error E = S.printName(SymName)) {
+      SmallString<0> ErrorBuf;
+      raw_svector_ostream Error(ErrorBuf);
+      Error << E << '\0';
+      return ErrorCallback(Error.str().data());
+    }
+    SymName << '\0';
+    if (void *E = Callback(State, SymNameBuf.str().data())) {
+      return E;
+    }
+    SymNameBuf.clear();
+  }
+  return 0;
+}