diff options
Diffstat (limited to 'compiler/rustc_llvm/llvm-wrapper')
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp | 244 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h | 103 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/Linker.cpp | 8 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 462 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 1165 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp | 4 |
6 files changed, 886 insertions, 1100 deletions
diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index cda81d4a9b5..4695de8ea09 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -1,5 +1,9 @@ #include "LLVMWrapper.h" + #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Module.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" #include "llvm/ProfileData/InstrProf.h" @@ -33,45 +37,6 @@ static coverage::Counter fromRust(LLVMRustCounter Counter) { report_fatal_error("Bad LLVMRustCounterKind!"); } -// FFI equivalent of enum `llvm::coverage::CounterMappingRegion::RegionKind` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L213-L234 -enum class LLVMRustCounterMappingRegionKind { - CodeRegion = 0, - ExpansionRegion = 1, - SkippedRegion = 2, - GapRegion = 3, - BranchRegion = 4, - MCDCDecisionRegion = 5, - MCDCBranchRegion = 6 -}; - -static coverage::CounterMappingRegion::RegionKind -fromRust(LLVMRustCounterMappingRegionKind Kind) { - switch (Kind) { - case LLVMRustCounterMappingRegionKind::CodeRegion: - return coverage::CounterMappingRegion::CodeRegion; - case LLVMRustCounterMappingRegionKind::ExpansionRegion: - return coverage::CounterMappingRegion::ExpansionRegion; - case LLVMRustCounterMappingRegionKind::SkippedRegion: - return coverage::CounterMappingRegion::SkippedRegion; - case LLVMRustCounterMappingRegionKind::GapRegion: - return coverage::CounterMappingRegion::GapRegion; - case LLVMRustCounterMappingRegionKind::BranchRegion: - return coverage::CounterMappingRegion::BranchRegion; - case LLVMRustCounterMappingRegionKind::MCDCDecisionRegion: - return coverage::CounterMappingRegion::MCDCDecisionRegion; - case LLVMRustCounterMappingRegionKind::MCDCBranchRegion: - return coverage::CounterMappingRegion::MCDCBranchRegion; - } - report_fatal_error("Bad LLVMRustCounterMappingRegionKind!"); -} - -enum LLVMRustMCDCParametersTag { - None = 0, - Decision = 1, - Branch = 2, -}; - struct LLVMRustMCDCDecisionParameters { uint32_t BitmapIdx; uint16_t NumConditions; @@ -82,47 +47,63 @@ struct LLVMRustMCDCBranchParameters { int16_t ConditionIDs[2]; }; -struct LLVMRustMCDCParameters { - LLVMRustMCDCParametersTag Tag; - LLVMRustMCDCDecisionParameters DecisionParameters; - LLVMRustMCDCBranchParameters BranchParameters; -}; +static coverage::mcdc::BranchParameters +fromRust(LLVMRustMCDCBranchParameters Params) { + return coverage::mcdc::BranchParameters( + Params.ConditionID, {Params.ConditionIDs[0], Params.ConditionIDs[1]}); +} -#if LLVM_VERSION_GE(19, 0) -static coverage::mcdc::Parameters fromRust(LLVMRustMCDCParameters Params) { - switch (Params.Tag) { - case LLVMRustMCDCParametersTag::None: - return std::monostate(); - case LLVMRustMCDCParametersTag::Decision: - return coverage::mcdc::DecisionParameters( - Params.DecisionParameters.BitmapIdx, - Params.DecisionParameters.NumConditions); - case LLVMRustMCDCParametersTag::Branch: - return coverage::mcdc::BranchParameters( - static_cast<coverage::mcdc::ConditionID>( - Params.BranchParameters.ConditionID), - {static_cast<coverage::mcdc::ConditionID>( - Params.BranchParameters.ConditionIDs[0]), - static_cast<coverage::mcdc::ConditionID>( - Params.BranchParameters.ConditionIDs[1])}); - } - report_fatal_error("Bad LLVMRustMCDCParametersTag!"); +static coverage::mcdc::DecisionParameters +fromRust(LLVMRustMCDCDecisionParameters Params) { + return coverage::mcdc::DecisionParameters(Params.BitmapIdx, + Params.NumConditions); } -#endif -// FFI equivalent of struct `llvm::coverage::CounterMappingRegion` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304 -struct LLVMRustCounterMappingRegion { - LLVMRustCounter Count; - LLVMRustCounter FalseCount; - LLVMRustMCDCParameters MCDCParameters; +// Must match the layout of +// `rustc_codegen_llvm::coverageinfo::ffi::CoverageSpan`. +struct LLVMRustCoverageSpan { uint32_t FileID; - uint32_t ExpandedFileID; uint32_t LineStart; uint32_t ColumnStart; uint32_t LineEnd; uint32_t ColumnEnd; - LLVMRustCounterMappingRegionKind Kind; +}; + +// Must match the layout of `rustc_codegen_llvm::coverageinfo::ffi::CodeRegion`. +struct LLVMRustCoverageCodeRegion { + LLVMRustCoverageSpan Span; + LLVMRustCounter Count; +}; + +// Must match the layout of +// `rustc_codegen_llvm::coverageinfo::ffi::ExpansionRegion`. +struct LLVMRustCoverageExpansionRegion { + LLVMRustCoverageSpan Span; + uint32_t ExpandedFileID; +}; + +// Must match the layout of +// `rustc_codegen_llvm::coverageinfo::ffi::BranchRegion`. +struct LLVMRustCoverageBranchRegion { + LLVMRustCoverageSpan Span; + LLVMRustCounter TrueCount; + LLVMRustCounter FalseCount; +}; + +// Must match the layout of +// `rustc_codegen_llvm::coverageinfo::ffi::MCDCBranchRegion`. +struct LLVMRustCoverageMCDCBranchRegion { + LLVMRustCoverageSpan Span; + LLVMRustCounter TrueCount; + LLVMRustCounter FalseCount; + LLVMRustMCDCBranchParameters MCDCBranchParams; +}; + +// Must match the layout of +// `rustc_codegen_llvm::coverageinfo::ffi::MCDCDecisionRegion`. +struct LLVMRustCoverageMCDCDecisionRegion { + LLVMRustCoverageSpan Span; + LLVMRustMCDCDecisionParameters MCDCDecisionParams; }; // FFI equivalent of enum `llvm::coverage::CounterExpression::ExprKind` @@ -151,13 +132,13 @@ fromRust(LLVMRustCounterExprKind Kind) { report_fatal_error("Bad LLVMRustCounterExprKind!"); } -extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( +extern "C" void LLVMRustCoverageWriteFilenamesToBuffer( const char *const Filenames[], size_t FilenamesLen, // String start pointers const size_t *const Lengths, size_t LengthsLen, // Corresponding lengths RustStringRef BufferOut) { if (FilenamesLen != LengthsLen) { report_fatal_error( - "Mismatched lengths in LLVMRustCoverageWriteFilenamesSectionToBuffer"); + "Mismatched lengths in LLVMRustCoverageWriteFilenamesToBuffer"); } SmallVector<std::string, 32> FilenameRefs; @@ -171,31 +152,20 @@ extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( FilenamesWriter.write(OS); } -extern "C" void LLVMRustCoverageWriteMappingToBuffer( - const unsigned *VirtualFileMappingIDs, unsigned NumVirtualFileMappingIDs, - const LLVMRustCounterExpression *RustExpressions, unsigned NumExpressions, - const LLVMRustCounterMappingRegion *RustMappingRegions, - unsigned NumMappingRegions, RustStringRef BufferOut) { +extern "C" void LLVMRustCoverageWriteFunctionMappingsToBuffer( + const unsigned *VirtualFileMappingIDs, size_t NumVirtualFileMappingIDs, + const LLVMRustCounterExpression *RustExpressions, size_t NumExpressions, + const LLVMRustCoverageCodeRegion *CodeRegions, size_t NumCodeRegions, + const LLVMRustCoverageExpansionRegion *ExpansionRegions, + size_t NumExpansionRegions, + const LLVMRustCoverageBranchRegion *BranchRegions, size_t NumBranchRegions, + const LLVMRustCoverageMCDCBranchRegion *MCDCBranchRegions, + size_t NumMCDCBranchRegions, + const LLVMRustCoverageMCDCDecisionRegion *MCDCDecisionRegions, + size_t NumMCDCDecisionRegions, RustStringRef BufferOut) { // Convert from FFI representation to LLVM representation. - SmallVector<coverage::CounterMappingRegion, 0> MappingRegions; - MappingRegions.reserve(NumMappingRegions); - for (const auto &Region : ArrayRef<LLVMRustCounterMappingRegion>( - RustMappingRegions, NumMappingRegions)) { - MappingRegions.emplace_back( - fromRust(Region.Count), fromRust(Region.FalseCount), -#if LLVM_VERSION_LT(19, 0) - coverage::CounterMappingRegion::MCDCParameters{}, -#endif - Region.FileID, Region.ExpandedFileID, // File IDs, then region info. - Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, - fromRust(Region.Kind) -#if LLVM_VERSION_GE(19, 0) - , - fromRust(Region.MCDCParameters) -#endif - ); - } + // Expressions: std::vector<coverage::CounterExpression> Expressions; Expressions.reserve(NumExpressions); for (const auto &Expression : @@ -205,6 +175,51 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer( fromRust(Expression.RHS)); } + std::vector<coverage::CounterMappingRegion> MappingRegions; + MappingRegions.reserve(NumCodeRegions + NumBranchRegions + + NumMCDCBranchRegions + NumMCDCDecisionRegions); + + // Code regions: + for (const auto &Region : ArrayRef(CodeRegions, NumCodeRegions)) { + MappingRegions.push_back(coverage::CounterMappingRegion::makeRegion( + fromRust(Region.Count), Region.Span.FileID, Region.Span.LineStart, + Region.Span.ColumnStart, Region.Span.LineEnd, Region.Span.ColumnEnd)); + } + + // Expansion regions: + for (const auto &Region : ArrayRef(ExpansionRegions, NumExpansionRegions)) { + MappingRegions.push_back(coverage::CounterMappingRegion::makeExpansion( + Region.Span.FileID, Region.ExpandedFileID, Region.Span.LineStart, + Region.Span.ColumnStart, Region.Span.LineEnd, Region.Span.ColumnEnd)); + } + + // Branch regions: + for (const auto &Region : ArrayRef(BranchRegions, NumBranchRegions)) { + MappingRegions.push_back(coverage::CounterMappingRegion::makeBranchRegion( + fromRust(Region.TrueCount), fromRust(Region.FalseCount), + Region.Span.FileID, Region.Span.LineStart, Region.Span.ColumnStart, + Region.Span.LineEnd, Region.Span.ColumnEnd)); + } + + // MC/DC branch regions: + for (const auto &Region : ArrayRef(MCDCBranchRegions, NumMCDCBranchRegions)) { + MappingRegions.push_back(coverage::CounterMappingRegion::makeBranchRegion( + fromRust(Region.TrueCount), fromRust(Region.FalseCount), + Region.Span.FileID, Region.Span.LineStart, Region.Span.ColumnStart, + Region.Span.LineEnd, Region.Span.ColumnEnd, + fromRust(Region.MCDCBranchParams))); + } + + // MC/DC decision regions: + for (const auto &Region : + ArrayRef(MCDCDecisionRegions, NumMCDCDecisionRegions)) { + MappingRegions.push_back(coverage::CounterMappingRegion::makeDecisionRegion( + fromRust(Region.MCDCDecisionParams), Region.Span.FileID, + Region.Span.LineStart, Region.Span.ColumnStart, Region.Span.LineEnd, + Region.Span.ColumnEnd)); + } + + // Write the converted expressions and mappings to a byte buffer. auto CoverageMappingWriter = coverage::CoverageMappingWriter( ArrayRef<unsigned>(VirtualFileMappingIDs, NumVirtualFileMappingIDs), Expressions, MappingRegions); @@ -219,34 +234,37 @@ LLVMRustCoverageCreatePGOFuncNameVar(LLVMValueRef F, const char *FuncName, return wrap(createPGOFuncNameVar(*cast<Function>(unwrap(F)), FuncNameRef)); } -extern "C" uint64_t LLVMRustCoverageHashByteArray(const char *Bytes, - size_t NumBytes) { - auto StrRef = StringRef(Bytes, NumBytes); - return IndexedInstrProf::ComputeHash(StrRef); +extern "C" uint64_t LLVMRustCoverageHashBytes(const char *Bytes, + size_t NumBytes) { + return IndexedInstrProf::ComputeHash(StringRef(Bytes, NumBytes)); } -static void WriteSectionNameToString(LLVMModuleRef M, InstrProfSectKind SK, - RustStringRef Str) { +// Private helper function for getting the covmap and covfun section names. +static void writeInstrProfSectionNameToString(LLVMModuleRef M, + InstrProfSectKind SectKind, + RustStringRef OutStr) { auto TargetTriple = Triple(unwrap(M)->getTargetTriple()); - auto name = getInstrProfSectionName(SK, TargetTriple.getObjectFormat()); - auto OS = RawRustStringOstream(Str); + auto name = getInstrProfSectionName(SectKind, TargetTriple.getObjectFormat()); + auto OS = RawRustStringOstream(OutStr); OS << name; } -extern "C" void LLVMRustCoverageWriteMapSectionNameToString(LLVMModuleRef M, - RustStringRef Str) { - WriteSectionNameToString(M, IPSK_covmap, Str); +extern "C" void +LLVMRustCoverageWriteCovmapSectionNameToString(LLVMModuleRef M, + RustStringRef OutStr) { + writeInstrProfSectionNameToString(M, IPSK_covmap, OutStr); } extern "C" void -LLVMRustCoverageWriteFuncSectionNameToString(LLVMModuleRef M, - RustStringRef Str) { - WriteSectionNameToString(M, IPSK_covfun, Str); +LLVMRustCoverageWriteCovfunSectionNameToString(LLVMModuleRef M, + RustStringRef OutStr) { + writeInstrProfSectionNameToString(M, IPSK_covfun, OutStr); } -extern "C" void LLVMRustCoverageWriteMappingVarNameToString(RustStringRef Str) { +extern "C" void +LLVMRustCoverageWriteCovmapVarNameToString(RustStringRef OutStr) { auto name = getCoverageMappingVarName(); - auto OS = RawRustStringOstream(Str); + auto OS = RawRustStringOstream(OutStr); OS << name; } diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h index 73bbc9de855..f6598f9faf5 100644 --- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h +++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h @@ -1,31 +1,12 @@ +#ifndef INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H +#define INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H + #include "SuppressLLVMWarnings.h" -#include "llvm-c/BitReader.h" -#include "llvm-c/Core.h" -#include "llvm-c/Object.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/Lint.h" -#include "llvm/Analysis/Passes.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/InlineAsm.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/JSON.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Transforms/IPO.h" -#include "llvm/Transforms/Scalar.h" +#include "llvm/Config/llvm-config.h" // LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR +#include "llvm/Support/raw_ostream.h" // llvm::raw_ostream +#include <cstddef> // size_t etc +#include <cstdint> // uint64_t etc #define LLVM_VERSION_GE(major, minor) \ (LLVM_VERSION_MAJOR > (major) || \ @@ -33,79 +14,17 @@ #define LLVM_VERSION_LT(major, minor) (!LLVM_VERSION_GE((major), (minor))) -#if LLVM_VERSION_GE(20, 0) -#include "llvm/Transforms/Utils/Instrumentation.h" -#else -#include "llvm/Transforms/Instrumentation.h" -#endif - -#include "llvm/IR/LegacyPassManager.h" - -#include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/Bitcode/BitcodeWriter.h" - -#include "llvm/IR/DIBuilder.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/IRPrintingPasses.h" -#include "llvm/Linker/Linker.h" - -#include "llvm/TargetParser/Triple.h" - extern "C" void LLVMRustSetLastError(const char *); enum class LLVMRustResult { Success, Failure }; -enum LLVMRustAttribute { - AlwaysInline = 0, - ByVal = 1, - Cold = 2, - InlineHint = 3, - MinSize = 4, - Naked = 5, - NoAlias = 6, - NoCapture = 7, - NoInline = 8, - NonNull = 9, - NoRedZone = 10, - NoReturn = 11, - NoUnwind = 12, - OptimizeForSize = 13, - ReadOnly = 14, - SExt = 15, - StructRet = 16, - UWTable = 17, - ZExt = 18, - InReg = 19, - SanitizeThread = 20, - SanitizeAddress = 21, - SanitizeMemory = 22, - NonLazyBind = 23, - OptimizeNone = 24, - ReadNone = 26, - SanitizeHWAddress = 28, - WillReturn = 29, - StackProtectReq = 30, - StackProtectStrong = 31, - StackProtect = 32, - NoUndef = 33, - SanitizeMemTag = 34, - NoCfCheck = 35, - ShadowCallStack = 36, - AllocSize = 37, - AllocatedPointer = 38, - AllocAlign = 39, - SanitizeSafeStack = 40, - FnRetThunkExtern = 41, - Writable = 42, - DeadOnUnwind = 43, -}; - typedef struct OpaqueRustString *RustStringRef; typedef struct LLVMOpaqueTwine *LLVMTwineRef; typedef struct LLVMOpaqueSMDiagnostic *LLVMSMDiagnosticRef; -extern "C" void LLVMRustStringWriteImpl(RustStringRef Str, const char *Ptr, - size_t Size); +extern "C" void LLVMRustStringWriteImpl(RustStringRef buf, + const char *slice_ptr, + size_t slice_len); class RawRustStringOstream : public llvm::raw_ostream { RustStringRef Str; @@ -126,3 +45,5 @@ public: flush(); } }; + +#endif // INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp index f43128ed550..fdf54e2d9b4 100644 --- a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp @@ -1,8 +1,10 @@ -#include "llvm/Linker/Linker.h" -#include "SuppressLLVMWarnings.h" - #include "LLVMWrapper.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/IR/Module.h" +#include "llvm/Linker/Linker.h" +#include "llvm/Support/MemoryBuffer.h" + using namespace llvm; struct RustLinker { diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 4b303511dbc..d4a05fbbbc5 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1,55 +1,54 @@ -#include <stdio.h> - -#include <cstddef> -#include <iomanip> -#include <set> -#include <vector> - #include "LLVMWrapper.h" -#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm-c/Core.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/Lint.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/CodeGen/CommandFlags.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/AssemblyAnnotationWriter.h" #include "llvm/IR/AutoUpgrade.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" +#include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/LTO/LTO.h" +#include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/TargetRegistry.h" -#include "llvm/Object/IRObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/Program.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Host.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h" -#include "llvm/Transforms/Utils/AddDiscriminators.h" -#include "llvm/Transforms/Utils/FunctionImportUtils.h" -#if LLVM_VERSION_GE(19, 0) -#include "llvm/Support/PGOOptions.h" -#endif -#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" -#include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Scalar/AnnotationRemarks.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" +#include <set> +#include <string> +#include <vector> + +// Conditional includes prevent clang-format from fully sorting the list, +// so if any are needed, keep them separate down here. using namespace llvm; @@ -196,6 +195,19 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char *FileName) { GEN_SUBTARGETS #undef SUBTARGET +// This struct and various functions are sort of a hack right now, but the +// problem is that we've got in-memory LLVM modules after we generate and +// optimize all codegen-units for one compilation in rustc. To be compatible +// with the LTO support above we need to serialize the modules plus their +// ThinLTO summary into memory. +// +// This structure is basically an owned version of a serialize module, with +// a ThinLTO summary attached. +struct LLVMRustThinLTOBuffer { + std::string data; + std::string thin_link_data; +}; + extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, const char *Feature) { TargetMachine *Target = unwrap(TM); @@ -310,6 +322,24 @@ static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { report_fatal_error("Bad RelocModel."); } +enum class LLVMRustFloatABI { + Default, + Soft, + Hard, +}; + +static FloatABI::ABIType fromRust(LLVMRustFloatABI RustFloatAbi) { + switch (RustFloatAbi) { + case LLVMRustFloatABI::Default: + return FloatABI::Default; + case LLVMRustFloatABI::Soft: + return FloatABI::Soft; + case LLVMRustFloatABI::Hard: + return FloatABI::Hard; + } + report_fatal_error("Bad FloatABI."); +} + /// getLongestEntryLength - Return the length of the longest entry in the table. template <typename KV> static size_t getLongestEntryLength(ArrayRef<KV> Table) { size_t MaxLen = 0; @@ -318,49 +348,17 @@ template <typename KV> static size_t getLongestEntryLength(ArrayRef<KV> Table) { return MaxLen; } -using PrintBackendInfo = void(void *, const char *Data, size_t Len); - extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, - const char *TargetCPU, - PrintBackendInfo Print, void *Out) { - const TargetMachine *Target = unwrap(TM); - const Triple::ArchType HostArch = - Triple(sys::getDefaultTargetTriple()).getArch(); - const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); + RustStringRef OutStr) { + ArrayRef<SubtargetSubTypeKV> CPUTable = + unwrap(TM)->getMCSubtargetInfo()->getAllProcessorDescriptions(); + auto OS = RawRustStringOstream(OutStr); - std::ostringstream Buf; - - const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef<SubtargetSubTypeKV> CPUTable = - MCInfo->getAllProcessorDescriptions(); - unsigned MaxCPULen = getLongestEntryLength(CPUTable); - - Buf << "Available CPUs for this target:\n"; - // Don't print the "native" entry when the user specifies --target with a - // different arch since that could be wrong or misleading. - if (HostArch == TargetArch) { - MaxCPULen = std::max(MaxCPULen, (unsigned)std::strlen("native")); - const StringRef HostCPU = sys::getHostCPUName(); - Buf << " " << std::left << std::setw(MaxCPULen) << "native" - << " - Select the CPU of the current host " - "(currently " - << HostCPU.str() << ").\n"; - } + // Just print a bare list of target CPU names, and let Rust-side code handle + // the full formatting of `--print=target-cpus`. for (auto &CPU : CPUTable) { - // Compare cpu against current target to label the default - if (strcmp(CPU.Key, TargetCPU) == 0) { - Buf << " " << std::left << std::setw(MaxCPULen) << CPU.Key - << " - This is the default target CPU for the current build target " - "(currently " - << Target->getTargetTriple().str() << ")."; - } else { - Buf << " " << CPU.Key; - } - Buf << "\n"; + OS << CPU.Key << "\n"; } - - const auto &BufString = Buf.str(); - Print(Out, BufString.data(), BufString.size()); } extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { @@ -383,16 +381,16 @@ extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, *Desc = Feat.Desc; } -extern "C" const char *LLVMRustGetHostCPUName(size_t *len) { +extern "C" const char *LLVMRustGetHostCPUName(size_t *OutLen) { StringRef Name = sys::getHostCPUName(); - *len = Name.size(); + *OutLen = Name.size(); return Name.data(); } extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( const char *TripleStr, const char *CPU, const char *Feature, const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, - LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, + LLVMRustCodeGenOptLevel RustOptLevel, LLVMRustFloatABI RustFloatABIType, bool FunctionSections, bool DataSections, bool UniqueSectionNames, bool TrapUnreachable, bool Singlethread, bool VerboseAsm, bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, @@ -403,6 +401,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( auto OptLevel = fromRust(RustOptLevel); auto RM = fromRust(RustReloc); auto CM = fromRust(RustCM); + auto FloatABIType = fromRust(RustFloatABIType); std::string Error; auto Trip = Triple(Triple::normalize(TripleStr)); @@ -415,10 +414,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); - Options.FloatABIType = FloatABI::Default; - if (UseSoftFloat) { - Options.FloatABIType = FloatABI::Soft; - } + Options.FloatABIType = FloatABIType; Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; Options.UniqueSectionNames = UniqueSectionNames; @@ -434,31 +430,15 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( } if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { -#if LLVM_VERSION_GE(19, 0) Options.MCOptions.CompressDebugSections = DebugCompressionType::Zlib; -#else - Options.CompressDebugSections = DebugCompressionType::Zlib; -#endif } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { -#if LLVM_VERSION_GE(19, 0) Options.MCOptions.CompressDebugSections = DebugCompressionType::Zstd; -#else - Options.CompressDebugSections = DebugCompressionType::Zstd; -#endif } else if (!strcmp("none", DebugInfoCompression)) { -#if LLVM_VERSION_GE(19, 0) Options.MCOptions.CompressDebugSections = DebugCompressionType::None; -#else - Options.CompressDebugSections = DebugCompressionType::None; -#endif } -#if LLVM_VERSION_GE(19, 0) Options.MCOptions.X86RelaxRelocations = RelaxELFRelocations; -#else - Options.RelaxELFRelocations = RelaxELFRelocations; -#endif Options.UseInitArray = UseInitArray; Options.EmulatedTLS = UseEmulatedTls; @@ -486,22 +466,25 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( if (ArgsCstrBuff != nullptr) { #if LLVM_VERSION_GE(20, 0) - int buffer_offset = 0; + size_t buffer_offset = 0; assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); auto Arg0 = std::string(ArgsCstrBuff); buffer_offset = Arg0.size() + 1; - auto ArgsCppStr = std::string(ArgsCstrBuff + buffer_offset, - ArgsCstrBuffLen - buffer_offset); - auto i = 0; - while (i != std::string::npos) { - i = ArgsCppStr.find('\0', i + 1); - if (i != std::string::npos) - ArgsCppStr.replace(i, 1, " "); + + std::string CommandlineArgs; + raw_string_ostream OS(CommandlineArgs); + ListSeparator LS(" "); + for (StringRef Arg : split(StringRef(ArgsCstrBuff + buffer_offset, + ArgsCstrBuffLen - buffer_offset), + '\0')) { + OS << LS; + sys::printArg(OS, Arg, /*Quote=*/true); } + OS.flush(); Options.MCOptions.Argv0 = Arg0; - Options.MCOptions.CommandlineArgs = ArgsCppStr; + Options.MCOptions.CommandlineArgs = CommandlineArgs; #else - int buffer_offset = 0; + size_t buffer_offset = 0; assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); const size_t arg0_len = std::strlen(ArgsCstrBuff); @@ -510,13 +493,13 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( arg0[arg0_len] = '\0'; buffer_offset += arg0_len + 1; - const int num_cmd_arg_strings = std::count( + const size_t num_cmd_arg_strings = std::count( &ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); std::string *cmd_arg_strings = new std::string[num_cmd_arg_strings]; - for (int i = 0; i < num_cmd_arg_strings; ++i) { + for (size_t i = 0; i < num_cmd_arg_strings; ++i) { assert(buffer_offset < ArgsCstrBuffLen); - const int len = std::strlen(ArgsCstrBuff + buffer_offset); + const size_t len = std::strlen(ArgsCstrBuff + buffer_offset); cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); buffer_offset += len + 1; } @@ -529,8 +512,13 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( #endif } +#if LLVM_VERSION_GE(21, 0) + TargetMachine *TM = TheTarget->createTargetMachine(Trip, CPU, Feature, + Options, RM, CM, OptLevel); +#else TargetMachine *TM = TheTarget->createTargetMachine( Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); +#endif return wrap(TM); } @@ -586,7 +574,7 @@ static CodeGenFileType fromRust(LLVMRustFileType Type) { extern "C" LLVMRustResult LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, LLVMModuleRef M, const char *Path, const char *DwoPath, - LLVMRustFileType RustFileType) { + LLVMRustFileType RustFileType, bool VerifyIR) { llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); auto FileType = fromRust(RustFileType); @@ -611,10 +599,10 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, return LLVMRustResult::Failure; } auto DBOS = buffer_ostream(DOS); - unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, !VerifyIR); PM->run(*unwrap(M)); } else { - unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, !VerifyIR); PM->run(*unwrap(M)); } @@ -706,17 +694,27 @@ struct LLVMRustSanitizerOptions { bool SanitizeKernelAddressRecover; }; +// This symbol won't be available or used when Enzyme is not enabled. +// Always set AugmentPassBuilder to true, since it registers optimizations which +// will improve the performance for Enzyme. +#ifdef ENZYME +extern "C" void registerEnzymeAndPassPipeline(llvm::PassBuilder &PB, + /* augmentPassBuilder */ bool); +#endif + extern "C" LLVMRustResult LLVMRustOptimize( LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, LLVMRustPassBuilderOptLevel OptLevelRust, LLVMRustOptStage OptStage, bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, - bool LintIR, bool UseThinLTOBuffers, bool MergeFunctions, bool UnrollLoops, + bool LintIR, LLVMRustThinLTOBuffer **ThinLTOBufferRef, bool EmitThinLTO, + bool EmitThinLTOSummary, bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, - bool EmitLifetimeMarkers, LLVMRustSanitizerOptions *SanitizerOptions, - const char *PGOGenPath, const char *PGOUsePath, bool InstrumentCoverage, - const char *InstrProfileOutput, bool InstrumentGCOV, - const char *PGOSampleUsePath, bool DebugInfoForProfiling, - void *LlvmSelfProfiler, + bool EmitLifetimeMarkers, bool RunEnzyme, bool PrintBeforeEnzyme, + bool PrintAfterEnzyme, bool PrintPasses, + LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, + const char *PGOUsePath, bool InstrumentCoverage, + const char *InstrProfileOutput, const char *PGOSampleUsePath, + bool DebugInfoForProfiling, void *LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, LLVMRustSelfProfileAfterPassCallback AfterPassCallback, const char *ExtraPasses, size_t ExtraPassesLen, const char *LLVMPlugins, @@ -743,34 +741,23 @@ extern "C" LLVMRustResult LLVMRustOptimize( auto FS = vfs::getRealFileSystem(); if (PGOGenPath) { assert(!PGOUsePath && !PGOSampleUsePath); - PGOOpt = PGOOptions(PGOGenPath, "", "", "", FS, PGOOptions::IRInstr, - PGOOptions::NoCSAction, -#if LLVM_VERSION_GE(19, 0) - PGOOptions::ColdFuncOpt::Default, -#endif - DebugInfoForProfiling); + PGOOpt = PGOOptions( + PGOGenPath, "", "", "", FS, PGOOptions::IRInstr, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, DebugInfoForProfiling); } else if (PGOUsePath) { assert(!PGOSampleUsePath); - PGOOpt = PGOOptions(PGOUsePath, "", "", "", FS, PGOOptions::IRUse, - PGOOptions::NoCSAction, -#if LLVM_VERSION_GE(19, 0) - PGOOptions::ColdFuncOpt::Default, -#endif - DebugInfoForProfiling); + PGOOpt = PGOOptions( + PGOUsePath, "", "", "", FS, PGOOptions::IRUse, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, DebugInfoForProfiling); } else if (PGOSampleUsePath) { - PGOOpt = PGOOptions(PGOSampleUsePath, "", "", "", FS, PGOOptions::SampleUse, - PGOOptions::NoCSAction, -#if LLVM_VERSION_GE(19, 0) - PGOOptions::ColdFuncOpt::Default, -#endif - DebugInfoForProfiling); + PGOOpt = + PGOOptions(PGOSampleUsePath, "", "", "", FS, PGOOptions::SampleUse, + PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default, + DebugInfoForProfiling); } else if (DebugInfoForProfiling) { - PGOOpt = PGOOptions("", "", "", "", FS, PGOOptions::NoAction, - PGOOptions::NoCSAction, -#if LLVM_VERSION_GE(19, 0) - PGOOptions::ColdFuncOpt::Default, -#endif - DebugInfoForProfiling); + PGOOpt = PGOOptions( + "", "", "", "", FS, PGOOptions::NoAction, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, DebugInfoForProfiling); } auto PB = PassBuilder(TM, PTO, PGOOpt, &PIC); @@ -779,10 +766,8 @@ extern "C" LLVMRustResult LLVMRustOptimize( CGSCCAnalysisManager CGAM; ModuleAnalysisManager MAM; - // FIXME: We may want to expose this as an option. - bool DebugPassManager = false; - - StandardInstrumentations SI(TheModule->getContext(), DebugPassManager); + StandardInstrumentations SI(TheModule->getContext(), + /*DebugLogging=*/false); SI.registerCallbacks(PIC, &MAM); if (LLVMPluginsLen) { @@ -820,16 +805,22 @@ extern "C" LLVMRustResult LLVMRustOptimize( // the PassBuilder does not create a pipeline. std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> PipelineStartEPCallbacks; +#if LLVM_VERSION_GE(20, 0) + std::vector<std::function<void(ModulePassManager &, OptimizationLevel, + ThinOrFullLTOPhase)>> + OptimizerLastEPCallbacks; +#else std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> OptimizerLastEPCallbacks; +#endif if (!IsLinkerPluginLTO && SanitizerOptions && SanitizerOptions->SanitizeCFI && !NoPrepopulatePasses) { PipelineStartEPCallbacks.push_back( [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, - /*ImportSummary=*/nullptr, - /*DropTypeTests=*/false)); + MPM.addPass(LowerTypeTestsPass( + /*ExportSummary=*/nullptr, + /*ImportSummary=*/nullptr)); }); } @@ -841,17 +832,15 @@ extern "C" LLVMRustResult LLVMRustOptimize( } if (LintIR) { - PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(createModuleToFunctionPassAdaptor(LintPass())); - }); - } - - if (InstrumentGCOV) { - PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); - }); + PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM, + OptimizationLevel Level) { +#if LLVM_VERSION_GE(21, 0) + MPM.addPass( + createModuleToFunctionPassAdaptor(LintPass(/*AbortOnError=*/true))); +#else + MPM.addPass(createModuleToFunctionPassAdaptor(LintPass())); +#endif + }); } if (InstrumentCoverage) { @@ -875,7 +864,12 @@ extern "C" LLVMRustResult LLVMRustOptimize( SanitizerOptions->SanitizeDataFlowABIList + SanitizerOptions->SanitizeDataFlowABIListLen); OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level) { +#endif MPM.addPass(DataFlowSanitizerPass(ABIListFiles)); }); } @@ -887,23 +881,41 @@ extern "C" LLVMRustResult LLVMRustOptimize( /*CompileKernel=*/false, /*EagerChecks=*/true); OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [Options](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else [Options](ModulePassManager &MPM, OptimizationLevel Level) { +#endif MPM.addPass(MemorySanitizerPass(Options)); }); } if (SanitizerOptions->SanitizeThread) { - OptimizerLastEPCallbacks.push_back([](ModulePassManager &MPM, - OptimizationLevel Level) { - MPM.addPass(ModuleThreadSanitizerPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); - }); + OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(ModuleThreadSanitizerPass()); + MPM.addPass( + createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); + }); } if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) { OptimizerLastEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_GE(20, 0) + [SanitizerOptions, TM](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [SanitizerOptions, TM](ModulePassManager &MPM, + OptimizationLevel Level) { +#endif auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; AddressSanitizerOptions opts = AddressSanitizerOptions{ CompileKernel, @@ -912,12 +924,22 @@ extern "C" LLVMRustResult LLVMRustOptimize( /*UseAfterScope=*/true, AsanDetectStackUseAfterReturnMode::Runtime, }; - MPM.addPass(AddressSanitizerPass(opts)); + MPM.addPass(AddressSanitizerPass( + opts, + /*UseGlobalGC*/ true, + // UseOdrIndicator should be false on windows machines + // https://reviews.llvm.org/D137227 + !TM->getTargetTriple().isOSWindows())); }); } if (SanitizerOptions->SanitizeHWAddress) { OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#endif HWAddressSanitizerOptions opts( /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, @@ -928,7 +950,10 @@ extern "C" LLVMRustResult LLVMRustOptimize( } ModulePassManager MPM; - bool NeedThinLTOBufferPasses = UseThinLTOBuffers; + bool NeedThinLTOBufferPasses = EmitThinLTO; + auto ThinLTOBuffer = std::make_unique<LLVMRustThinLTOBuffer>(); + raw_string_ostream ThinLTODataOS(ThinLTOBuffer->data); + raw_string_ostream ThinLinkDataOS(ThinLTOBuffer->thin_link_data); if (!NoPrepopulatePasses) { // The pre-link pipelines don't support O0 and require using // buildO0DefaultPipeline() instead. At the same time, the LTO pipelines do @@ -941,8 +966,9 @@ extern "C" LLVMRustResult LLVMRustOptimize( for (const auto &C : OptimizerLastEPCallbacks) PB.registerOptimizerLastEPCallback(C); - // Pass false as we manually schedule ThinLTOBufferPasses below. - MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); + // We manually schedule ThinLTOBufferPasses below, so don't pass the value + // to enable it here. + MPM = PB.buildO0DefaultPipeline(OptLevel); } else { for (const auto &C : PipelineStartEPCallbacks) PB.registerPipelineStartEPCallback(C); @@ -951,7 +977,25 @@ extern "C" LLVMRustResult LLVMRustOptimize( switch (OptStage) { case LLVMRustOptStage::PreLinkNoLTO: - MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); + if (ThinLTOBufferRef) { + // This is similar to LLVM's `buildFatLTODefaultPipeline`, where the + // bitcode for embedding is obtained after performing + // `ThinLTOPreLinkDefaultPipeline`. + MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(OptLevel)); + if (EmitThinLTO) { + MPM.addPass(ThinLTOBitcodeWriterPass( + ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr)); + } else { + MPM.addPass(BitcodeWriterPass(ThinLTODataOS)); + } + *ThinLTOBufferRef = ThinLTOBuffer.release(); + MPM.addPass(PB.buildModuleOptimizationPipeline( + OptLevel, ThinOrFullLTOPhase::None)); + MPM.addPass( + createModuleToFunctionPassAdaptor(AnnotationRemarksPass())); + } else { + MPM = PB.buildPerModuleDefaultPipeline(OptLevel); + } break; case LLVMRustOptStage::PreLinkThinLTO: MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); @@ -977,7 +1021,11 @@ extern "C" LLVMRustResult LLVMRustOptimize( for (const auto &C : PipelineStartEPCallbacks) C(MPM, OptLevel); for (const auto &C : OptimizerLastEPCallbacks) +#if LLVM_VERSION_GE(20, 0) + C(MPM, OptLevel, ThinOrFullLTOPhase::None); +#else C(MPM, OptLevel); +#endif } if (ExtraPassesLen) { @@ -993,6 +1041,52 @@ extern "C" LLVMRustResult LLVMRustOptimize( MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); } + // For `-Copt-level=0`, ThinLTO, or LTO. + if (ThinLTOBufferRef && *ThinLTOBufferRef == nullptr) { + if (EmitThinLTO) { + MPM.addPass(ThinLTOBitcodeWriterPass( + ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr)); + } else { + MPM.addPass(BitcodeWriterPass(ThinLTODataOS)); + } + *ThinLTOBufferRef = ThinLTOBuffer.release(); + } + + // now load "-enzyme" pass: +#ifdef ENZYME + if (RunEnzyme) { + + if (PrintBeforeEnzyme) { + // Handle the Rust flag `-Zautodiff=PrintModBefore`. + std::string Banner = "Module before EnzymeNewPM"; + MPM.addPass(PrintModulePass(outs(), Banner, true, false)); + } + + registerEnzymeAndPassPipeline(PB, false); + if (auto Err = PB.parsePassPipeline(MPM, "enzyme")) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + + if (PrintAfterEnzyme) { + // Handle the Rust flag `-Zautodiff=PrintModAfter`. + std::string Banner = "Module after EnzymeNewPM"; + MPM.addPass(PrintModulePass(outs(), Banner, true, false)); + } + } +#endif + if (PrintPasses) { + // Print all passes from the PM: + std::string Pipeline; + raw_string_ostream SOS(Pipeline); + MPM.printPipeline(SOS, [&PIC](StringRef ClassName) { + auto PassName = PIC.getPassNameForClassName(ClassName); + return PassName.empty() ? ClassName : PassName; + }); + outs() << Pipeline; + outs() << "\n"; + } // Upgrade all calls to old intrinsics first. for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) @@ -1251,12 +1345,12 @@ getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { // here is basically the same as before threads are spawned in the `run` // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. extern "C" LLVMRustThinLTOData * -LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, int num_modules, - const char **preserved_symbols, int num_symbols) { +LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, size_t num_modules, + const char **preserved_symbols, size_t num_symbols) { auto Ret = std::make_unique<LLVMRustThinLTOData>(); // Load each module's summary and merge it into one combined index - for (int i = 0; i < num_modules; i++) { + for (size_t i = 0; i < num_modules; i++) { auto module = &modules[i]; auto buffer = StringRef(module->data, module->len); auto mem_buffer = MemoryBufferRef(buffer, module->identifier); @@ -1275,8 +1369,13 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, int num_modules, // Convert the preserved symbols set from string to GUID, this is then needed // for internalization. - for (int i = 0; i < num_symbols; i++) { + for (size_t i = 0; i < num_symbols; i++) { +#if LLVM_VERSION_GE(21, 0) + auto GUID = + GlobalValue::getGUIDAssumingExternalLinkage(preserved_symbols[i]); +#else auto GUID = GlobalValue::getGUID(preserved_symbols[i]); +#endif Ret->GUIDPreservedSymbols.insert(GUID); } @@ -1373,20 +1472,14 @@ static bool clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { return ClearDSOLocalOnDeclarations; } -extern "C" bool LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, +extern "C" void LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, LLVMTargetMachineRef TM) { Module &Mod = *unwrap(M); TargetMachine &Target = *unwrap(TM); bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); - bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); - - if (error) { - LLVMRustSetLastError("renameModuleForThinLTO failed"); - return false; - } - return true; + renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); } extern "C" bool @@ -1465,19 +1558,6 @@ extern "C" bool LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, return true; } -// This struct and various functions are sort of a hack right now, but the -// problem is that we've got in-memory LLVM modules after we generate and -// optimize all codegen-units for one compilation in rustc. To be compatible -// with the LTO support above we need to serialize the modules plus their -// ThinLTO summary into memory. -// -// This structure is basically an owned version of a serialize module, with -// a ThinLTO summary attached. -struct LLVMRustThinLTOBuffer { - std::string data; - std::string thin_link_data; -}; - extern "C" LLVMRustThinLTOBuffer * LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin, bool emit_summary) { auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); @@ -1559,8 +1639,10 @@ extern "C" LLVMModuleRef LLVMRustParseBitcodeForLTO(LLVMContextRef Context, extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, size_t len, const char *name, + size_t name_len, size_t *out_len) { *out_len = 0; + auto Name = StringRef(name, name_len); auto Data = StringRef(data, len); auto Buffer = MemoryBufferRef(Data, ""); // The id is unused. file_magic Type = identify_magic(Buffer.getBuffer()); @@ -1571,8 +1653,8 @@ extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, return nullptr; } for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { - Expected<StringRef> Name = Sec.getName(); - if (Name && *Name == name) { + Expected<StringRef> SecName = Sec.getName(); + if (SecName && *SecName == Name) { Expected<StringRef> SectionOrError = Sec.getContents(); if (!SectionOrError) { LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); @@ -1611,12 +1693,21 @@ extern "C" void LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, #endif // Based on the 'InProcessThinBackend' constructor in LLVM +#if LLVM_VERSION_GE(21, 0) + for (auto &Name : Data->Index.cfiFunctionDefs().symbols()) + CfiFunctionDefs.insert(GlobalValue::getGUIDAssumingExternalLinkage( + GlobalValue::dropLLVMManglingEscape(Name))); + for (auto &Name : Data->Index.cfiFunctionDecls().symbols()) + CfiFunctionDecls.insert(GlobalValue::getGUIDAssumingExternalLinkage( + GlobalValue::dropLLVMManglingEscape(Name))); +#else for (auto &Name : Data->Index.cfiFunctionDefs()) CfiFunctionDefs.insert( GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); for (auto &Name : Data->Index.cfiFunctionDecls()) CfiFunctionDecls.insert( GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); +#endif #if LLVM_VERSION_GE(20, 0) Key = llvm::computeLTOCacheKey(conf, Data->Index, ModId, ImportList, @@ -1628,5 +1719,6 @@ extern "C" void LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, CfiFunctionDefs, CfiFunctionDecls); #endif - LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); + auto OS = RawRustStringOstream(KeyOut); + OS << Key.str(); } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 72b03fa0560..90aa9188c83 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1,28 +1,40 @@ #include "LLVMWrapper.h" + +#include "llvm-c/Analysis.h" +#include "llvm-c/Core.h" +#include "llvm-c/DebugInfo.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/Magic.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DiagnosticHandler.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsARM.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Value.h" -#include "llvm/Object/Archive.h" #include "llvm/Object/COFFImportFile.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Pass.h" #include "llvm/Remarks/RemarkFormat.h" #include "llvm/Remarks/RemarkSerializer.h" #include "llvm/Remarks/RemarkStreamer.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/ModRef.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/Timer.h" #include "llvm/Support/ToolOutputFile.h" - #include <iostream> // for raw `write` in the bad-alloc handler @@ -43,6 +55,10 @@ using namespace llvm; using namespace llvm::sys; using namespace llvm::object; +// This opcode is an LLVM detail that could hypothetically change (?), so +// verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM. +static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000); + // LLVMAtomicOrdering is already an enum - don't create another // one. static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { @@ -136,30 +152,22 @@ extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { } extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, - const char *Triple) { - unwrap(M)->setTargetTriple(Triple::normalize(Triple)); + const char *Target) { +#if LLVM_VERSION_GE(21, 0) + unwrap(M)->setTargetTriple(Triple(Triple::normalize(Target))); +#else + unwrap(M)->setTargetTriple(Triple::normalize(Target)); +#endif } -extern "C" const char *LLVMRustPrintPassTimings(size_t *Len) { - std::string buf; - auto SS = raw_string_ostream(buf); - TimerGroup::printAll(SS); - SS.flush(); - *Len = buf.length(); - char *CStr = (char *)malloc(*Len); - memcpy(CStr, buf.c_str(), *Len); - return CStr; +extern "C" void LLVMRustPrintPassTimings(RustStringRef OutBuf) { + auto OS = RawRustStringOstream(OutBuf); + TimerGroup::printAll(OS); } -extern "C" const char *LLVMRustPrintStatistics(size_t *Len) { - std::string buf; - auto SS = raw_string_ostream(buf); - llvm::PrintStatistics(SS); - SS.flush(); - *Len = buf.length(); - char *CStr = (char *)malloc(*Len); - memcpy(CStr, buf.c_str(), *Len); - return CStr; +extern "C" void LLVMRustPrintStatistics(RustStringRef OutBuf) { + auto OS = RawRustStringOstream(OutBuf); + llvm::PrintStatistics(OS); } extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, @@ -167,31 +175,28 @@ extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); } -enum class LLVMRustTailCallKind { - None, - Tail, - MustTail, - NoTail, +enum class LLVMRustVerifierFailureAction { + AbortProcessAction = 0, + PrintMessageAction = 1, + ReturnStatusAction = 2, }; -static CallInst::TailCallKind fromRust(LLVMRustTailCallKind Kind) { - switch (Kind) { - case LLVMRustTailCallKind::None: - return CallInst::TailCallKind::TCK_None; - case LLVMRustTailCallKind::Tail: - return CallInst::TailCallKind::TCK_Tail; - case LLVMRustTailCallKind::MustTail: - return CallInst::TailCallKind::TCK_MustTail; - case LLVMRustTailCallKind::NoTail: - return CallInst::TailCallKind::TCK_NoTail; - default: - report_fatal_error("bad CallInst::TailCallKind."); +static LLVMVerifierFailureAction +fromRust(LLVMRustVerifierFailureAction Action) { + switch (Action) { + case LLVMRustVerifierFailureAction::AbortProcessAction: + return LLVMAbortProcessAction; + case LLVMRustVerifierFailureAction::PrintMessageAction: + return LLVMPrintMessageAction; + case LLVMRustVerifierFailureAction::ReturnStatusAction: + return LLVMReturnStatusAction; } + report_fatal_error("Invalid LLVMVerifierFailureAction value!"); } -extern "C" void LLVMRustSetTailCallKind(LLVMValueRef Call, - LLVMRustTailCallKind TCK) { - unwrap<CallInst>(Call)->setTailCallKind(fromRust(TCK)); +extern "C" LLVMBool +LLVMRustVerifyFunction(LLVMValueRef Fn, LLVMRustVerifierFailureAction Action) { + return LLVMVerifyFunction(Fn, fromRust(Action)); } extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, @@ -228,94 +233,144 @@ extern "C" LLVMValueRef LLVMRustInsertPrivateGlobal(LLVMModuleRef M, GlobalValue::PrivateLinkage, nullptr)); } -static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { +// Must match the layout of `rustc_codegen_llvm::llvm::ffi::AttributeKind`. +enum class LLVMRustAttributeKind { + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + ReadOnly = 14, + SExt = 15, + StructRet = 16, + UWTable = 17, + ZExt = 18, + InReg = 19, + SanitizeThread = 20, + SanitizeAddress = 21, + SanitizeMemory = 22, + NonLazyBind = 23, + OptimizeNone = 24, + ReadNone = 26, + SanitizeHWAddress = 28, + WillReturn = 29, + StackProtectReq = 30, + StackProtectStrong = 31, + StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, + SanitizeSafeStack = 40, + FnRetThunkExtern = 41, + Writable = 42, + DeadOnUnwind = 43, +}; + +static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { switch (Kind) { - case AlwaysInline: + case LLVMRustAttributeKind::AlwaysInline: return Attribute::AlwaysInline; - case ByVal: + case LLVMRustAttributeKind::ByVal: return Attribute::ByVal; - case Cold: + case LLVMRustAttributeKind::Cold: return Attribute::Cold; - case InlineHint: + case LLVMRustAttributeKind::InlineHint: return Attribute::InlineHint; - case MinSize: + case LLVMRustAttributeKind::MinSize: return Attribute::MinSize; - case Naked: + case LLVMRustAttributeKind::Naked: return Attribute::Naked; - case NoAlias: + case LLVMRustAttributeKind::NoAlias: return Attribute::NoAlias; - case NoCapture: + case LLVMRustAttributeKind::NoCapture: +#if LLVM_VERSION_GE(21, 0) + report_fatal_error("NoCapture doesn't exist in LLVM 21"); +#else return Attribute::NoCapture; - case NoCfCheck: +#endif + case LLVMRustAttributeKind::NoCfCheck: return Attribute::NoCfCheck; - case NoInline: + case LLVMRustAttributeKind::NoInline: return Attribute::NoInline; - case NonNull: + case LLVMRustAttributeKind::NonNull: return Attribute::NonNull; - case NoRedZone: + case LLVMRustAttributeKind::NoRedZone: return Attribute::NoRedZone; - case NoReturn: + case LLVMRustAttributeKind::NoReturn: return Attribute::NoReturn; - case NoUnwind: + case LLVMRustAttributeKind::NoUnwind: return Attribute::NoUnwind; - case OptimizeForSize: + case LLVMRustAttributeKind::OptimizeForSize: return Attribute::OptimizeForSize; - case ReadOnly: + case LLVMRustAttributeKind::ReadOnly: return Attribute::ReadOnly; - case SExt: + case LLVMRustAttributeKind::SExt: return Attribute::SExt; - case StructRet: + case LLVMRustAttributeKind::StructRet: return Attribute::StructRet; - case UWTable: + case LLVMRustAttributeKind::UWTable: return Attribute::UWTable; - case ZExt: + case LLVMRustAttributeKind::ZExt: return Attribute::ZExt; - case InReg: + case LLVMRustAttributeKind::InReg: return Attribute::InReg; - case SanitizeThread: + case LLVMRustAttributeKind::SanitizeThread: return Attribute::SanitizeThread; - case SanitizeAddress: + case LLVMRustAttributeKind::SanitizeAddress: return Attribute::SanitizeAddress; - case SanitizeMemory: + case LLVMRustAttributeKind::SanitizeMemory: return Attribute::SanitizeMemory; - case NonLazyBind: + case LLVMRustAttributeKind::NonLazyBind: return Attribute::NonLazyBind; - case OptimizeNone: + case LLVMRustAttributeKind::OptimizeNone: return Attribute::OptimizeNone; - case ReadNone: + case LLVMRustAttributeKind::ReadNone: return Attribute::ReadNone; - case SanitizeHWAddress: + case LLVMRustAttributeKind::SanitizeHWAddress: return Attribute::SanitizeHWAddress; - case WillReturn: + case LLVMRustAttributeKind::WillReturn: return Attribute::WillReturn; - case StackProtectReq: + case LLVMRustAttributeKind::StackProtectReq: return Attribute::StackProtectReq; - case StackProtectStrong: + case LLVMRustAttributeKind::StackProtectStrong: return Attribute::StackProtectStrong; - case StackProtect: + case LLVMRustAttributeKind::StackProtect: return Attribute::StackProtect; - case NoUndef: + case LLVMRustAttributeKind::NoUndef: return Attribute::NoUndef; - case SanitizeMemTag: + case LLVMRustAttributeKind::SanitizeMemTag: return Attribute::SanitizeMemTag; - case ShadowCallStack: + case LLVMRustAttributeKind::ShadowCallStack: return Attribute::ShadowCallStack; - case AllocSize: + case LLVMRustAttributeKind::AllocSize: return Attribute::AllocSize; - case AllocatedPointer: + case LLVMRustAttributeKind::AllocatedPointer: return Attribute::AllocatedPointer; - case AllocAlign: + case LLVMRustAttributeKind::AllocAlign: return Attribute::AllocAlign; - case SanitizeSafeStack: + case LLVMRustAttributeKind::SanitizeSafeStack: return Attribute::SafeStack; - case FnRetThunkExtern: + case LLVMRustAttributeKind::FnRetThunkExtern: return Attribute::FnRetThunkExtern; - case Writable: + case LLVMRustAttributeKind::Writable: return Attribute::Writable; - case DeadOnUnwind: + case LLVMRustAttributeKind::DeadOnUnwind: return Attribute::DeadOnUnwind; } - report_fatal_error("bad AttributeKind"); + report_fatal_error("bad LLVMRustAttributeKind"); } template <typename T> @@ -329,6 +384,12 @@ static inline void AddAttributes(T *t, unsigned Index, LLVMAttributeRef *Attrs, t->setAttributes(PALNew); } +extern "C" bool LLVMRustHasAttributeAtIndex(LLVMValueRef Fn, unsigned Index, + LLVMRustAttributeKind RustAttr) { + Function *F = unwrap<Function>(Fn); + return F->hasParamAttribute(Index, fromRust(RustAttr)); +} + extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, LLVMAttributeRef *Attrs, size_t AttrsLen) { @@ -344,8 +405,25 @@ extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, AddAttributes(Call, Index, Attrs, AttrsLen); } +extern "C" LLVMValueRef LLVMRustGetTerminator(LLVMBasicBlockRef BB) { + Instruction *ret = unwrap(BB)->getTerminator(); + return wrap(ret); +} + +extern "C" void LLVMRustEraseInstFromParent(LLVMValueRef Instr) { + if (auto I = dyn_cast<Instruction>(unwrap<Value>(Instr))) { + I->eraseFromParent(); + } +} + extern "C" LLVMAttributeRef -LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttribute RustAttr) { +LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttributeKind RustAttr) { +#if LLVM_VERSION_GE(21, 0) + // LLVM 21 replaced the NoCapture attribute with Captures(none). + if (RustAttr == LLVMRustAttributeKind::NoCapture) { + return wrap(Attribute::getWithCaptureInfo(*unwrap(C), CaptureInfo::none())); + } +#endif return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); } @@ -395,12 +473,8 @@ extern "C" LLVMAttributeRef LLVMRustCreateRangeAttribute(LLVMContextRef C, unsigned NumBits, const uint64_t LowerWords[], const uint64_t UpperWords[]) { -#if LLVM_VERSION_GE(19, 0) return LLVMCreateConstantRangeAttribute(C, Attribute::Range, NumBits, LowerWords, UpperWords); -#else - report_fatal_error("LLVM 19.0 is required for Range Attribute"); -#endif } // These values **must** match ffi::AllocKindFlags. @@ -548,31 +622,8 @@ extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, return wrap(SI); } -enum class LLVMRustAsmDialect { - Att, - Intel, -}; - -static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { - switch (Dialect) { - case LLVMRustAsmDialect::Att: - return InlineAsm::AD_ATT; - case LLVMRustAsmDialect::Intel: - return InlineAsm::AD_Intel; - default: - report_fatal_error("bad AsmDialect."); - } -} - -extern "C" LLVMValueRef -LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, - char *Constraints, size_t ConstraintsLen, - LLVMBool HasSideEffects, LLVMBool IsAlignStack, - LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { - return wrap(InlineAsm::get( - unwrap<FunctionType>(Ty), StringRef(AsmString, AsmStringLen), - StringRef(Constraints, ConstraintsLen), HasSideEffects, IsAlignStack, - fromRust(Dialect), CanThrow)); +extern "C" uint64_t LLVMRustGetArrayNumElements(LLVMTypeRef Ty) { + return unwrap(Ty)->getArrayNumElements(); } extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, @@ -582,8 +633,6 @@ extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, unwrap<FunctionType>(Ty), StringRef(Constraints, ConstraintsLen))); } -typedef DIBuilder *LLVMRustDIBuilderRef; - template <typename DIT> DIT *unwrapDIPtr(LLVMMetadataRef Ref) { return (DIT *)(Ref ? unwrap<MDNode>(Ref) : nullptr); } @@ -592,120 +641,67 @@ template <typename DIT> DIT *unwrapDIPtr(LLVMMetadataRef Ref) { #define DIArray DINodeArray #define unwrapDI unwrapDIPtr -// These values **must** match debuginfo::DIFlags! They also *happen* -// to match LLVM, but that isn't required as we do giant sets of -// matching below. The value shouldn't be directly passed to LLVM. -enum class LLVMRustDIFlags : uint32_t { - FlagZero = 0, - FlagPrivate = 1, - FlagProtected = 2, - FlagPublic = 3, - FlagFwdDecl = (1 << 2), - FlagAppleBlock = (1 << 3), - FlagBlockByrefStruct = (1 << 4), - FlagVirtual = (1 << 5), - FlagArtificial = (1 << 6), - FlagExplicit = (1 << 7), - FlagPrototyped = (1 << 8), - FlagObjcClassComplete = (1 << 9), - FlagObjectPointer = (1 << 10), - FlagVector = (1 << 11), - FlagStaticMember = (1 << 12), - FlagLValueReference = (1 << 13), - FlagRValueReference = (1 << 14), - FlagExternalTypeRef = (1 << 15), - FlagIntroducedVirtual = (1 << 18), - FlagBitField = (1 << 19), - FlagNoReturn = (1 << 20), - // Do not add values that are not supported by the minimum LLVM - // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def -}; - -inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) { - return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(A) & - static_cast<uint32_t>(B)); -} - -inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) { - return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(A) | - static_cast<uint32_t>(B)); -} - -inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) { - return A = A | B; -} - -inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; } - -inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) { - return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(F) & 0x3); -} - -static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) { - DINode::DIFlags Result = DINode::DIFlags::FlagZero; - - switch (visibility(Flags)) { - case LLVMRustDIFlags::FlagPrivate: - Result |= DINode::DIFlags::FlagPrivate; - break; - case LLVMRustDIFlags::FlagProtected: - Result |= DINode::DIFlags::FlagProtected; - break; - case LLVMRustDIFlags::FlagPublic: - Result |= DINode::DIFlags::FlagPublic; - break; - default: - // The rest are handled below - break; - } - - if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) { - Result |= DINode::DIFlags::FlagFwdDecl; - } - if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) { - Result |= DINode::DIFlags::FlagAppleBlock; - } - if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) { - Result |= DINode::DIFlags::FlagVirtual; - } - if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) { - Result |= DINode::DIFlags::FlagArtificial; - } - if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) { - Result |= DINode::DIFlags::FlagExplicit; - } - if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) { - Result |= DINode::DIFlags::FlagPrototyped; - } - if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) { - Result |= DINode::DIFlags::FlagObjcClassComplete; - } - if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) { - Result |= DINode::DIFlags::FlagObjectPointer; - } - if (isSet(Flags & LLVMRustDIFlags::FlagVector)) { - Result |= DINode::DIFlags::FlagVector; - } - if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) { - Result |= DINode::DIFlags::FlagStaticMember; - } - if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) { - Result |= DINode::DIFlags::FlagLValueReference; - } - if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) { - Result |= DINode::DIFlags::FlagRValueReference; - } - if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) { - Result |= DINode::DIFlags::FlagIntroducedVirtual; - } - if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) { - Result |= DINode::DIFlags::FlagBitField; - } - if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) { - Result |= DINode::DIFlags::FlagNoReturn; - } - - return Result; +// Statically assert that `LLVMDIFlags` (C) and `DIFlags` (C++) have the same +// layout, at least for the flags we know about. This isn't guaranteed, but is +// likely to remain true, and as long as it is true it makes conversions easy. +#define ASSERT_DIFLAG_VALUE(FLAG, VALUE) \ + static_assert((LLVMDI##FLAG == (VALUE)) && (DINode::DIFlags::FLAG == (VALUE))) +ASSERT_DIFLAG_VALUE(FlagZero, 0); +ASSERT_DIFLAG_VALUE(FlagPrivate, 1); +ASSERT_DIFLAG_VALUE(FlagProtected, 2); +ASSERT_DIFLAG_VALUE(FlagPublic, 3); +// Bit (1 << 1) is part of the private/protected/public values above. +ASSERT_DIFLAG_VALUE(FlagFwdDecl, 1 << 2); +ASSERT_DIFLAG_VALUE(FlagAppleBlock, 1 << 3); +ASSERT_DIFLAG_VALUE(FlagReservedBit4, 1 << 4); +ASSERT_DIFLAG_VALUE(FlagVirtual, 1 << 5); +ASSERT_DIFLAG_VALUE(FlagArtificial, 1 << 6); +ASSERT_DIFLAG_VALUE(FlagExplicit, 1 << 7); +ASSERT_DIFLAG_VALUE(FlagPrototyped, 1 << 8); +ASSERT_DIFLAG_VALUE(FlagObjcClassComplete, 1 << 9); +ASSERT_DIFLAG_VALUE(FlagObjectPointer, 1 << 10); +ASSERT_DIFLAG_VALUE(FlagVector, 1 << 11); +ASSERT_DIFLAG_VALUE(FlagStaticMember, 1 << 12); +ASSERT_DIFLAG_VALUE(FlagLValueReference, 1 << 13); +ASSERT_DIFLAG_VALUE(FlagRValueReference, 1 << 14); +// Bit (1 << 15) has been recycled, but the C API value hasn't been renamed. +static_assert((LLVMDIFlagReserved == (1 << 15)) && + (DINode::DIFlags::FlagExportSymbols == (1 << 15))); +ASSERT_DIFLAG_VALUE(FlagSingleInheritance, 1 << 16); +ASSERT_DIFLAG_VALUE(FlagMultipleInheritance, 2 << 16); +ASSERT_DIFLAG_VALUE(FlagVirtualInheritance, 3 << 16); +// Bit (1 << 17) is part of the inheritance values above. +ASSERT_DIFLAG_VALUE(FlagIntroducedVirtual, 1 << 18); +ASSERT_DIFLAG_VALUE(FlagBitField, 1 << 19); +ASSERT_DIFLAG_VALUE(FlagNoReturn, 1 << 20); +// Bit (1 << 21) is unused, but was `LLVMDIFlagMainSubprogram`. +ASSERT_DIFLAG_VALUE(FlagTypePassByValue, 1 << 22); +ASSERT_DIFLAG_VALUE(FlagTypePassByReference, 1 << 23); +ASSERT_DIFLAG_VALUE(FlagEnumClass, 1 << 24); +ASSERT_DIFLAG_VALUE(FlagThunk, 1 << 25); +ASSERT_DIFLAG_VALUE(FlagNonTrivial, 1 << 26); +ASSERT_DIFLAG_VALUE(FlagBigEndian, 1 << 27); +ASSERT_DIFLAG_VALUE(FlagLittleEndian, 1 << 28); +ASSERT_DIFLAG_VALUE(FlagIndirectVirtualBase, (1 << 2) | (1 << 5)); +#undef ASSERT_DIFLAG_VALUE + +// There are two potential ways to convert `LLVMDIFlags` to `DIFlags`: +// - Check and copy every individual bit/subvalue from input to output. +// - Statically assert that both have the same layout, and cast. +// As long as the static assertions succeed, a cast is easier and faster. +// In the (hopefully) unlikely event that the assertions do fail someday, and +// LLVM doesn't expose its own conversion function, we'll have to switch over +// to copying each bit/subvalue. +static DINode::DIFlags fromRust(LLVMDIFlags Flags) { + // Check that all set bits are covered by the static assertions above. + const unsigned UNKNOWN_BITS = (1 << 31) | (1 << 30) | (1 << 29) | (1 << 21); + if (Flags & UNKNOWN_BITS) { + report_fatal_error("bad LLVMDIFlags"); + } + + // As long as the static assertions are satisfied and no unknown bits are + // present, we can convert from `LLVMDIFlags` to `DIFlags` with a cast. + return static_cast<DINode::DIFlags>(Flags); } // These values **must** match debuginfo::DISPFlags! They also *happen* @@ -853,23 +849,122 @@ extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } -extern "C" void LLVMRustAddModuleFlagU32(LLVMModuleRef M, - Module::ModFlagBehavior MergeBehavior, - const char *Name, uint32_t Value) { - unwrap(M)->addModuleFlag(MergeBehavior, Name, Value); +// FFI equivalent of LLVM's `llvm::Module::ModFlagBehavior`. +// Must match the layout of +// `rustc_codegen_llvm::llvm::ffi::ModuleFlagMergeBehavior`. +// +// There is a stable LLVM-C version of this enum (`LLVMModuleFlagBehavior`), +// but as of LLVM 19 it does not support all of the enum values in the unstable +// C++ API. +enum class LLVMRustModuleFlagMergeBehavior { + Error = 1, + Warning = 2, + Require = 3, + Override = 4, + Append = 5, + AppendUnique = 6, + Max = 7, + Min = 8, +}; + +static Module::ModFlagBehavior +fromRust(LLVMRustModuleFlagMergeBehavior Behavior) { + switch (Behavior) { + case LLVMRustModuleFlagMergeBehavior::Error: + return Module::ModFlagBehavior::Error; + case LLVMRustModuleFlagMergeBehavior::Warning: + return Module::ModFlagBehavior::Warning; + case LLVMRustModuleFlagMergeBehavior::Require: + return Module::ModFlagBehavior::Require; + case LLVMRustModuleFlagMergeBehavior::Override: + return Module::ModFlagBehavior::Override; + case LLVMRustModuleFlagMergeBehavior::Append: + return Module::ModFlagBehavior::Append; + case LLVMRustModuleFlagMergeBehavior::AppendUnique: + return Module::ModFlagBehavior::AppendUnique; + case LLVMRustModuleFlagMergeBehavior::Max: + return Module::ModFlagBehavior::Max; + case LLVMRustModuleFlagMergeBehavior::Min: + return Module::ModFlagBehavior::Min; + } + report_fatal_error("bad LLVMRustModuleFlagMergeBehavior"); +} + +extern "C" void +LLVMRustAddModuleFlagU32(LLVMModuleRef M, + LLVMRustModuleFlagMergeBehavior MergeBehavior, + const char *Name, size_t NameLen, uint32_t Value) { + unwrap(M)->addModuleFlag(fromRust(MergeBehavior), StringRef(Name, NameLen), + Value); } extern "C" void LLVMRustAddModuleFlagString( - LLVMModuleRef M, Module::ModFlagBehavior MergeBehavior, const char *Name, - const char *Value, size_t ValueLen) { + LLVMModuleRef M, LLVMRustModuleFlagMergeBehavior MergeBehavior, + const char *Name, size_t NameLen, const char *Value, size_t ValueLen) { unwrap(M)->addModuleFlag( - MergeBehavior, Name, + fromRust(MergeBehavior), StringRef(Name, NameLen), MDString::get(unwrap(M)->getContext(), StringRef(Value, ValueLen))); } -extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, - size_t Len) { - return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; +extern "C" LLVMValueRef LLVMRustGetLastInstruction(LLVMBasicBlockRef BB) { + auto Point = unwrap(BB)->rbegin(); + if (Point != unwrap(BB)->rend()) + return wrap(&*Point); + return nullptr; +} + +extern "C" void LLVMRustEraseInstUntilInclusive(LLVMBasicBlockRef bb, + LLVMValueRef I) { + auto &BB = *unwrap(bb); + auto &Inst = *unwrap<Instruction>(I); + auto It = BB.begin(); + while (&*It != &Inst) + ++It; + // Make sure we found the Instruction. + assert(It != BB.end()); + // Delete in rev order to ensure no dangling references. + while (It != BB.begin()) { + auto Prev = std::prev(It); + It->eraseFromParent(); + It = Prev; + } + It->eraseFromParent(); +} + +extern "C" bool LLVMRustHasMetadata(LLVMValueRef inst, unsigned kindID) { + if (auto *I = dyn_cast<Instruction>(unwrap<Value>(inst))) { + return I->hasMetadata(kindID); + } + return false; +} + +extern "C" LLVMMetadataRef LLVMRustDIGetInstMetadata(LLVMValueRef x) { + if (auto *I = dyn_cast<Instruction>(unwrap<Value>(x))) { + auto *MD = I->getDebugLoc().getAsMDNode(); + return wrap(MD); + } + return nullptr; +} + +extern "C" void +LLVMRustRemoveEnumAttributeAtIndex(LLVMValueRef F, size_t index, + LLVMRustAttributeKind RustAttr) { + LLVMRemoveEnumAttributeAtIndex(F, index, fromRust(RustAttr)); +} + +extern "C" bool LLVMRustHasFnAttribute(LLVMValueRef F, const char *Name, + size_t NameLen) { + if (auto *Fn = dyn_cast<Function>(unwrap<Value>(F))) { + return Fn->hasFnAttribute(StringRef(Name, NameLen)); + } + return false; +} + +extern "C" void LLVMRustRemoveFnAttribute(LLVMValueRef Fn, const char *Name, + size_t NameLen) { + if (auto *F = dyn_cast<Function>(unwrap<Value>(Fn))) { + F->removeFnAttr(StringRef(Name, NameLen)); + } } extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind, @@ -877,34 +972,30 @@ extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind, unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD)); } -extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { - return new DIBuilder(*unwrap(M)); +extern "C" LLVMDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { + return wrap(new DIBuilder(*unwrap(M))); } -extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) { - delete Builder; -} - -extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) { - Builder->finalize(); +extern "C" void LLVMRustDIBuilderDispose(LLVMDIBuilderRef Builder) { + delete unwrap(Builder); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( - LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, + LLVMDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, const char *Producer, size_t ProducerLen, bool isOptimized, const char *Flags, unsigned RuntimeVer, const char *SplitName, size_t SplitNameLen, LLVMRustDebugEmissionKind Kind, uint64_t DWOId, bool SplitDebugInlining, LLVMRustDebugNameTableKind TableKind) { auto *File = unwrapDI<DIFile>(FileRef); - return wrap(Builder->createCompileUnit( + return wrap(unwrap(Builder)->createCompileUnit( Lang, File, StringRef(Producer, ProducerLen), isOptimized, Flags, RuntimeVer, StringRef(SplitName, SplitNameLen), fromRust(Kind), DWOId, SplitDebugInlining, false, fromRust(TableKind))); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateFile(LLVMRustDIBuilderRef Builder, const char *Filename, +LLVMRustDIBuilderCreateFile(LLVMDIBuilderRef Builder, const char *Filename, size_t FilenameLen, const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, const char *Checksum, size_t ChecksumLen, @@ -917,29 +1008,29 @@ LLVMRustDIBuilderCreateFile(LLVMRustDIBuilderRef Builder, const char *Filename, std::optional<StringRef> oSource{}; if (Source) oSource = StringRef(Source, SourceLen); - return wrap(Builder->createFile(StringRef(Filename, FilenameLen), - StringRef(Directory, DirectoryLen), CSInfo, - oSource)); + return wrap(unwrap(Builder)->createFile(StringRef(Filename, FilenameLen), + StringRef(Directory, DirectoryLen), + CSInfo, oSource)); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder, +LLVMRustDIBuilderCreateSubroutineType(LLVMDIBuilderRef Builder, LLVMMetadataRef ParameterTypes) { - return wrap(Builder->createSubroutineType( + return wrap(unwrap(Builder)->createSubroutineType( DITypeRefArray(unwrap<MDTuple>(ParameterTypes)))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, const char *LinkageName, size_t LinkageNameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, - unsigned ScopeLine, LLVMRustDIFlags Flags, LLVMRustDISPFlags SPFlags, + unsigned ScopeLine, LLVMDIFlags Flags, LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, LLVMMetadataRef Decl) { DITemplateParameterArray TParams = DITemplateParameterArray(unwrap<MDTuple>(TParam)); DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); DINode::DIFlags llvmFlags = fromRust(Flags); - DISubprogram *Sub = Builder->createFunction( + DISubprogram *Sub = unwrap(Builder)->createFunction( unwrapDI<DIScope>(Scope), StringRef(Name, NameLen), StringRef(LinkageName, LinkageNameLen), unwrapDI<DIFile>(File), LineNo, unwrapDI<DISubroutineType>(Ty), ScopeLine, llvmFlags, llvmSPFlags, @@ -950,15 +1041,15 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, const char *LinkageName, size_t LinkageNameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, - LLVMRustDIFlags Flags, LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) { + LLVMDIFlags Flags, LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) { DITemplateParameterArray TParams = DITemplateParameterArray(unwrap<MDTuple>(TParam)); DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); DINode::DIFlags llvmFlags = fromRust(Flags); - DISubprogram *Sub = Builder->createMethod( + DISubprogram *Sub = unwrap(Builder)->createMethod( unwrapDI<DIScope>(Scope), StringRef(Name, NameLen), StringRef(LinkageName, LinkageNameLen), unwrapDI<DIFile>(File), LineNo, unwrapDI<DISubroutineType>(Ty), 0, 0, @@ -968,39 +1059,39 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod( } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateBasicType(LLVMRustDIBuilderRef Builder, const char *Name, +LLVMRustDIBuilderCreateBasicType(LLVMDIBuilderRef Builder, const char *Name, size_t NameLen, uint64_t SizeInBits, unsigned Encoding) { - return wrap( - Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding)); + return wrap(unwrap(Builder)->createBasicType(StringRef(Name, NameLen), + SizeInBits, Encoding)); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateTypedef(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef Type, const char *Name, - size_t NameLen, LLVMMetadataRef File, - unsigned LineNo, LLVMMetadataRef Scope) { - return wrap(Builder->createTypedef( +LLVMRustDIBuilderCreateTypedef(LLVMDIBuilderRef Builder, LLVMMetadataRef Type, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Scope) { + return wrap(unwrap(Builder)->createTypedef( unwrap<DIType>(Type), StringRef(Name, NameLen), unwrap<DIFile>(File), LineNo, unwrapDIPtr<DIScope>(Scope))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy, - uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace, - const char *Name, size_t NameLen) { - return wrap(Builder->createPointerType(unwrapDI<DIType>(PointeeTy), - SizeInBits, AlignInBits, AddressSpace, - StringRef(Name, NameLen))); + LLVMDIBuilderRef Builder, LLVMMetadataRef PointeeTy, uint64_t SizeInBits, + uint32_t AlignInBits, unsigned AddressSpace, const char *Name, + size_t NameLen) { + return wrap(unwrap(Builder)->createPointerType( + unwrapDI<DIType>(PointeeTy), SizeInBits, AlignInBits, AddressSpace, + StringRef(Name, NameLen))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, - uint64_t SizeInBits, uint32_t AlignInBits, LLVMRustDIFlags Flags, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags, LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, unsigned RunTimeLang, LLVMMetadataRef VTableHolder, const char *UniqueId, size_t UniqueIdLen) { - return wrap(Builder->createStructType( + return wrap(unwrap(Builder)->createStructType( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber, SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIType>(DerivedFrom), @@ -1009,12 +1100,12 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, - uint64_t SizeInBits, uint32_t AlignInBits, LLVMRustDIFlags Flags, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags, LLVMMetadataRef Discriminator, LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) { - return wrap(Builder->createVariantPart( + return wrap(unwrap(Builder)->createVariantPart( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber, SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIDerivedType>(Discriminator), @@ -1023,57 +1114,50 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, - uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags, + uint32_t AlignInBits, uint64_t OffsetInBits, LLVMDIFlags Flags, LLVMMetadataRef Ty) { - return wrap(Builder->createMemberType( + return wrap(unwrap(Builder)->createMemberType( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNo, SizeInBits, AlignInBits, OffsetInBits, fromRust(Flags), unwrapDI<DIType>(Ty))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, - LLVMRustDIFlags Flags, LLVMMetadataRef Ty) { + LLVMDIFlags Flags, LLVMMetadataRef Ty) { llvm::ConstantInt *D = nullptr; if (Discriminant) { D = unwrap<llvm::ConstantInt>(Discriminant); } - return wrap(Builder->createVariantMemberType( + return wrap(unwrap(Builder)->createVariantMemberType( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNo, SizeInBits, AlignInBits, OffsetInBits, D, fromRust(Flags), unwrapDI<DIType>(Ty))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticMemberType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, - LLVMRustDIFlags Flags, LLVMValueRef val, uint32_t AlignInBits) { - return wrap(Builder->createStaticMemberType( + LLVMDIFlags Flags, LLVMValueRef val, uint32_t AlignInBits) { + return wrap(unwrap(Builder)->createStaticMemberType( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), fromRust(Flags), unwrap<llvm::ConstantInt>(val), llvm::dwarf::DW_TAG_member, AlignInBits)); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateLexicalBlock(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef Scope, LLVMMetadataRef File, - unsigned Line, unsigned Col) { - return wrap(Builder->createLexicalBlock(unwrapDI<DIDescriptor>(Scope), - unwrapDI<DIFile>(File), Line, Col)); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlockFile( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef File) { - return wrap(Builder->createLexicalBlockFile(unwrapDI<DIDescriptor>(Scope), - unwrapDI<DIFile>(File))); +LLVMRustDIBuilderCreateQualifiedType(LLVMDIBuilderRef Builder, unsigned Tag, + LLVMMetadataRef Type) { + return wrap( + unwrap(Builder)->createQualifiedType(Tag, unwrapDI<DIType>(Type))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Context, const char *Name, size_t NameLen, const char *LinkageName, size_t LinkageNameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V, LLVMMetadataRef Decl = nullptr, @@ -1082,16 +1166,16 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( llvm::DIExpression *InitExpr = nullptr; if (llvm::ConstantInt *IntVal = llvm::dyn_cast<llvm::ConstantInt>(InitVal)) { - InitExpr = Builder->createConstantValueExpression( + InitExpr = unwrap(Builder)->createConstantValueExpression( IntVal->getValue().getSExtValue()); } else if (llvm::ConstantFP *FPVal = llvm::dyn_cast<llvm::ConstantFP>(InitVal)) { - InitExpr = Builder->createConstantValueExpression( + InitExpr = unwrap(Builder)->createConstantValueExpression( FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); } llvm::DIGlobalVariableExpression *VarExpr = - Builder->createGlobalVariableExpression( + unwrap(Builder)->createGlobalVariableExpression( unwrapDI<DIDescriptor>(Context), StringRef(Name, NameLen), StringRef(LinkageName, LinkageNameLen), unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), IsLocalToUnit, @@ -1104,17 +1188,17 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( - LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, + LLVMDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, - LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags, - unsigned ArgNo, uint32_t AlignInBits) { + LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMDIFlags Flags, unsigned ArgNo, + uint32_t AlignInBits) { if (Tag == 0x100) { // DW_TAG_auto_variable - return wrap(Builder->createAutoVariable( + return wrap(unwrap(Builder)->createAutoVariable( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits)); } else { - return wrap(Builder->createParameterVariable( + return wrap(unwrap(Builder)->createParameterVariable( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), ArgNo, unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags))); @@ -1122,53 +1206,56 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size, +LLVMRustDIBuilderCreateArrayType(LLVMDIBuilderRef Builder, uint64_t Size, uint32_t AlignInBits, LLVMMetadataRef Ty, LLVMMetadataRef Subscripts) { - return wrap( - Builder->createArrayType(Size, AlignInBits, unwrapDI<DIType>(Ty), - DINodeArray(unwrapDI<MDTuple>(Subscripts)))); + return wrap(unwrap(Builder)->createArrayType( + Size, AlignInBits, unwrapDI<DIType>(Ty), + DINodeArray(unwrapDI<MDTuple>(Subscripts)))); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo, +LLVMRustDIBuilderGetOrCreateSubrange(LLVMDIBuilderRef Builder, int64_t Lo, int64_t Count) { - return wrap(Builder->getOrCreateSubrange(Lo, Count)); + return wrap(unwrap(Builder)->getOrCreateSubrange(Lo, Count)); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, +LLVMRustDIBuilderGetOrCreateArray(LLVMDIBuilderRef Builder, LLVMMetadataRef *Ptr, unsigned Count) { Metadata **DataValue = unwrap(Ptr); - return wrap( - Builder->getOrCreateArray(ArrayRef<Metadata *>(DataValue, Count)).get()); + return wrap(unwrap(Builder) + ->getOrCreateArray(ArrayRef<Metadata *>(DataValue, Count)) + .get()); } -extern "C" void LLVMRustDIBuilderInsertDeclareAtEnd( - LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, - uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, - LLVMBasicBlockRef InsertAtEnd) { - Builder->insertDeclare(unwrap(V), unwrap<DILocalVariable>(VarInfo), - Builder->createExpression( - llvm::ArrayRef<uint64_t>(AddrOps, AddrOpsCount)), - DebugLoc(cast<MDNode>(unwrap(DL))), - unwrap(InsertAtEnd)); +extern "C" void +LLVMRustDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef V, + LLVMMetadataRef VarInfo, uint64_t *AddrOps, + unsigned AddrOpsCount, LLVMMetadataRef DL, + LLVMBasicBlockRef InsertAtEnd) { + unwrap(Builder)->insertDeclare( + unwrap(V), unwrap<DILocalVariable>(VarInfo), + unwrap(Builder)->createExpression( + llvm::ArrayRef<uint64_t>(AddrOps, AddrOpsCount)), + DebugLoc(cast<MDNode>(unwrap(DL))), unwrap(InsertAtEnd)); } -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( - LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, - const uint64_t Value[2], unsigned SizeInBits, bool IsUnsigned) { - return wrap(Builder->createEnumerator( +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder, const char *Name, + size_t NameLen, const uint64_t Value[2], + unsigned SizeInBits, bool IsUnsigned) { + return wrap(unwrap(Builder)->createEnumerator( StringRef(Name, NameLen), APSInt(APInt(SizeInBits, ArrayRef<uint64_t>(Value, 2)), IsUnsigned))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, uint32_t AlignInBits, LLVMMetadataRef Elements, LLVMMetadataRef ClassTy, bool IsScoped) { - return wrap(Builder->createEnumerationType( + return wrap(unwrap(Builder)->createEnumerationType( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber, SizeInBits, AlignInBits, DINodeArray(unwrapDI<MDTuple>(Elements)), unwrapDI<DIType>(ClassTy), @@ -1176,12 +1263,12 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, - uint64_t SizeInBits, uint32_t AlignInBits, LLVMRustDIFlags Flags, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags, LLVMMetadataRef Elements, unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) { - return wrap(Builder->createUnionType( + return wrap(unwrap(Builder)->createUnionType( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber, SizeInBits, AlignInBits, fromRust(Flags), DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang, @@ -1189,50 +1276,28 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen, LLVMMetadataRef Ty) { bool IsDefault = false; // FIXME: should we ever set this true? - return wrap(Builder->createTemplateTypeParameter( + return wrap(unwrap(Builder)->createTemplateTypeParameter( unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIType>(Ty), IsDefault)); } -extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateNameSpace(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef Scope, const char *Name, - size_t NameLen, bool ExportSymbols) { - return wrap(Builder->createNameSpace( - unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), ExportSymbols)); -} - extern "C" void LLVMRustDICompositeTypeReplaceArrays( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef CompositeTy, + LLVMDIBuilderRef Builder, LLVMMetadataRef CompositeTy, LLVMMetadataRef Elements, LLVMMetadataRef Params) { DICompositeType *Tmp = unwrapDI<DICompositeType>(CompositeTy); - Builder->replaceArrays(Tmp, DINodeArray(unwrap<MDTuple>(Elements)), - DINodeArray(unwrap<MDTuple>(Params))); + unwrap(Builder)->replaceArrays(Tmp, DINodeArray(unwrap<MDTuple>(Elements)), + DINodeArray(unwrap<MDTuple>(Params))); } extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, - LLVMMetadataRef ScopeRef, - LLVMMetadataRef InlinedAt) { - MDNode *Scope = unwrapDIPtr<MDNode>(ScopeRef); - DILocation *Loc = DILocation::get(Scope->getContext(), Line, Column, Scope, - unwrapDIPtr<MDNode>(InlinedAt)); - return wrap(Loc); -} - -extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() { - return dwarf::DW_OP_deref; -} - -extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { - return dwarf::DW_OP_plus_uconst; -} - -extern "C" int64_t LLVMRustDIBuilderCreateOpLLVMFragment() { - return dwarf::DW_OP_LLVM_fragment; +LLVMRustDILocationCloneWithBaseDiscriminator(LLVMMetadataRef Location, + unsigned BD) { + DILocation *Loc = unwrapDIPtr<DILocation>(Location); + auto NewLoc = Loc->cloneWithBaseDiscriminator(BD); + return wrap(NewLoc.has_value() ? NewLoc.value() : nullptr); } extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { @@ -1445,7 +1510,7 @@ extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic(LLVMDiagnosticInfoRef DI, - unsigned *Cookie) { + uint64_t *Cookie) { llvm::DiagnosticInfoSrcMgr *SM = static_cast<llvm::DiagnosticInfoSrcMgr *>(unwrap(DI)); *Cookie = SM->getLocCookie(); @@ -1484,8 +1549,8 @@ LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, RustStringRef MessageOut, const SourceMgr &LSM = *D.getSourceMgr(); const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); - LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), - LBuf->getBufferSize()); + auto BufferOS = RawRustStringOstream(BufferOut); + BufferOS << LBuf->getBuffer(); *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); @@ -1499,77 +1564,6 @@ LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, RustStringRef MessageOut, return true; } -extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, - LLVMValueRef *Inputs, - unsigned NumInputs) { - return new OperandBundleDef(Name, - ArrayRef<Value *>(unwrap(Inputs), NumInputs)); -} - -extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { - delete Bundle; -} - -// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). -extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, - LLVMValueRef Fn, LLVMValueRef *Args, - unsigned NumArgs, - OperandBundleDef **OpBundlesIndirect, - unsigned NumOpBundles) { - Value *Callee = unwrap(Fn); - FunctionType *FTy = unwrap<FunctionType>(Ty); - - // FIXME: Is there a way around this? - SmallVector<OperandBundleDef> OpBundles; - OpBundles.reserve(NumOpBundles); - for (unsigned i = 0; i < NumOpBundles; ++i) { - OpBundles.push_back(*OpBundlesIndirect[i]); - } - - return wrap(unwrap(B)->CreateCall(FTy, Callee, - ArrayRef<Value *>(unwrap(Args), NumArgs), - ArrayRef<OperandBundleDef>(OpBundles))); -} - -extern "C" LLVMValueRef -LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { -#if LLVM_VERSION_GE(20, 0) - return wrap(llvm::Intrinsic::getOrInsertDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_increment)); -#else - return wrap(llvm::Intrinsic::getDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_increment)); -#endif -} - -extern "C" LLVMValueRef -LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) { -#if LLVM_VERSION_LT(19, 0) - report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions"); -#endif -#if LLVM_VERSION_GE(20, 0) - return wrap(llvm::Intrinsic::getOrInsertDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters)); -#else - return wrap(llvm::Intrinsic::getDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters)); -#endif -} - -extern "C" LLVMValueRef -LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(LLVMModuleRef M) { -#if LLVM_VERSION_LT(19, 0) - report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions"); -#endif -#if LLVM_VERSION_GE(20, 0) - return wrap(llvm::Intrinsic::getOrInsertDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update)); -#else - return wrap(llvm::Intrinsic::getDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update)); -#endif -} - extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, LLVMValueRef Src, unsigned SrcAlign, @@ -1597,157 +1591,12 @@ extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, LLVMValueRef Dst, MaybeAlign(DstAlign), IsVolatile)); } -// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). -extern "C" LLVMValueRef -LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, - LLVMValueRef *Args, unsigned NumArgs, - LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, - OperandBundleDef **OpBundlesIndirect, unsigned NumOpBundles, - const char *Name) { - Value *Callee = unwrap(Fn); - FunctionType *FTy = unwrap<FunctionType>(Ty); - - // FIXME: Is there a way around this? - SmallVector<OperandBundleDef> OpBundles; - OpBundles.reserve(NumOpBundles); - for (unsigned i = 0; i < NumOpBundles; ++i) { - OpBundles.push_back(*OpBundlesIndirect[i]); - } - - return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), - ArrayRef<Value *>(unwrap(Args), NumArgs), - ArrayRef<OperandBundleDef>(OpBundles), - Name)); -} - -// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). -extern "C" LLVMValueRef -LLVMRustBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, - LLVMBasicBlockRef DefaultDest, - LLVMBasicBlockRef *IndirectDests, unsigned NumIndirectDests, - LLVMValueRef *Args, unsigned NumArgs, - OperandBundleDef **OpBundlesIndirect, unsigned NumOpBundles, - const char *Name) { - Value *Callee = unwrap(Fn); - FunctionType *FTy = unwrap<FunctionType>(Ty); - - // FIXME: Is there a way around this? - std::vector<BasicBlock *> IndirectDestsUnwrapped; - IndirectDestsUnwrapped.reserve(NumIndirectDests); - for (unsigned i = 0; i < NumIndirectDests; ++i) { - IndirectDestsUnwrapped.push_back(unwrap(IndirectDests[i])); - } - - // FIXME: Is there a way around this? - SmallVector<OperandBundleDef> OpBundles; - OpBundles.reserve(NumOpBundles); - for (unsigned i = 0; i < NumOpBundles; ++i) { - OpBundles.push_back(*OpBundlesIndirect[i]); - } - - return wrap( - unwrap(B)->CreateCallBr(FTy, Callee, unwrap(DefaultDest), - ArrayRef<BasicBlock *>(IndirectDestsUnwrapped), - ArrayRef<Value *>(unwrap(Args), NumArgs), - ArrayRef<OperandBundleDef>(OpBundles), Name)); -} - extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, LLVMBasicBlockRef BB) { auto Point = unwrap(BB)->getFirstInsertionPt(); unwrap(B)->SetInsertPoint(unwrap(BB), Point); } -extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V, - const char *Name, size_t NameLen) { - Triple TargetTriple = Triple(unwrap(M)->getTargetTriple()); - GlobalObject *GV = unwrap<GlobalObject>(V); - if (TargetTriple.supportsCOMDAT()) { - StringRef NameRef(Name, NameLen); - GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef)); - } -} - -enum class LLVMRustLinkage { - ExternalLinkage = 0, - AvailableExternallyLinkage = 1, - LinkOnceAnyLinkage = 2, - LinkOnceODRLinkage = 3, - WeakAnyLinkage = 4, - WeakODRLinkage = 5, - AppendingLinkage = 6, - InternalLinkage = 7, - PrivateLinkage = 8, - ExternalWeakLinkage = 9, - CommonLinkage = 10, -}; - -static LLVMRustLinkage toRust(LLVMLinkage Linkage) { - switch (Linkage) { - case LLVMExternalLinkage: - return LLVMRustLinkage::ExternalLinkage; - case LLVMAvailableExternallyLinkage: - return LLVMRustLinkage::AvailableExternallyLinkage; - case LLVMLinkOnceAnyLinkage: - return LLVMRustLinkage::LinkOnceAnyLinkage; - case LLVMLinkOnceODRLinkage: - return LLVMRustLinkage::LinkOnceODRLinkage; - case LLVMWeakAnyLinkage: - return LLVMRustLinkage::WeakAnyLinkage; - case LLVMWeakODRLinkage: - return LLVMRustLinkage::WeakODRLinkage; - case LLVMAppendingLinkage: - return LLVMRustLinkage::AppendingLinkage; - case LLVMInternalLinkage: - return LLVMRustLinkage::InternalLinkage; - case LLVMPrivateLinkage: - return LLVMRustLinkage::PrivateLinkage; - case LLVMExternalWeakLinkage: - return LLVMRustLinkage::ExternalWeakLinkage; - case LLVMCommonLinkage: - return LLVMRustLinkage::CommonLinkage; - default: - report_fatal_error("Invalid LLVMRustLinkage value!"); - } -} - -static LLVMLinkage fromRust(LLVMRustLinkage Linkage) { - switch (Linkage) { - case LLVMRustLinkage::ExternalLinkage: - return LLVMExternalLinkage; - case LLVMRustLinkage::AvailableExternallyLinkage: - return LLVMAvailableExternallyLinkage; - case LLVMRustLinkage::LinkOnceAnyLinkage: - return LLVMLinkOnceAnyLinkage; - case LLVMRustLinkage::LinkOnceODRLinkage: - return LLVMLinkOnceODRLinkage; - case LLVMRustLinkage::WeakAnyLinkage: - return LLVMWeakAnyLinkage; - case LLVMRustLinkage::WeakODRLinkage: - return LLVMWeakODRLinkage; - case LLVMRustLinkage::AppendingLinkage: - return LLVMAppendingLinkage; - case LLVMRustLinkage::InternalLinkage: - return LLVMInternalLinkage; - case LLVMRustLinkage::PrivateLinkage: - return LLVMPrivateLinkage; - case LLVMRustLinkage::ExternalWeakLinkage: - return LLVMExternalWeakLinkage; - case LLVMRustLinkage::CommonLinkage: - return LLVMCommonLinkage; - } - report_fatal_error("Invalid LLVMRustLinkage value!"); -} - -extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { - return toRust(LLVMGetLinkage(V)); -} - -extern "C" void LLVMRustSetLinkage(LLVMValueRef V, - LLVMRustLinkage RustLinkage) { - LLVMSetLinkage(V, fromRust(RustLinkage)); -} - extern "C" bool LLVMRustConstIntGetZExtValue(LLVMValueRef CV, uint64_t *value) { auto C = unwrap<llvm::ConstantInt>(CV); if (C->getBitWidth() > 64) @@ -1775,45 +1624,6 @@ extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, return true; } -enum class LLVMRustVisibility { - Default = 0, - Hidden = 1, - Protected = 2, -}; - -static LLVMRustVisibility toRust(LLVMVisibility Vis) { - switch (Vis) { - case LLVMDefaultVisibility: - return LLVMRustVisibility::Default; - case LLVMHiddenVisibility: - return LLVMRustVisibility::Hidden; - case LLVMProtectedVisibility: - return LLVMRustVisibility::Protected; - } - report_fatal_error("Invalid LLVMRustVisibility value!"); -} - -static LLVMVisibility fromRust(LLVMRustVisibility Vis) { - switch (Vis) { - case LLVMRustVisibility::Default: - return LLVMDefaultVisibility; - case LLVMRustVisibility::Hidden: - return LLVMHiddenVisibility; - case LLVMRustVisibility::Protected: - return LLVMProtectedVisibility; - } - report_fatal_error("Invalid LLVMRustVisibility value!"); -} - -extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { - return toRust(LLVMGetVisibility(V)); -} - -extern "C" void LLVMRustSetVisibility(LLVMValueRef V, - LLVMRustVisibility RustVisibility) { - LLVMSetVisibility(V, fromRust(RustVisibility)); -} - extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { unwrap<GlobalValue>(Global)->setDSOLocal(is_dso_local); } @@ -1924,56 +1734,6 @@ extern "C" LLVMValueRef LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS), unwrap(RHS))); } -// This struct contains all necessary info about a symbol exported from a DLL. -struct LLVMRustCOFFShortExport { - const char *name; - bool ordinal_present; - // The value of `ordinal` is only meaningful if `ordinal_present` is true. - uint16_t ordinal; -}; - -// Machine must be a COFF machine type, as defined in PE specs. -extern "C" LLVMRustResult -LLVMRustWriteImportLibrary(const char *ImportName, const char *Path, - const LLVMRustCOFFShortExport *Exports, - size_t NumExports, uint16_t Machine, bool MinGW) { - std::vector<llvm::object::COFFShortExport> ConvertedExports; - ConvertedExports.reserve(NumExports); - - for (size_t i = 0; i < NumExports; ++i) { - bool ordinal_present = Exports[i].ordinal_present; - uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0; - ConvertedExports.push_back(llvm::object::COFFShortExport{ - Exports[i].name, // Name - std::string{}, // ExtName - std::string{}, // SymbolName - std::string{}, // AliasTarget -#if LLVM_VERSION_GE(19, 0) - std::string{}, // ExportAs -#endif - ordinal, // Ordinal - ordinal_present, // Noname - false, // Data - false, // Private - false // Constant - }); - } - - auto Error = llvm::object::writeImportLibrary( - ImportName, Path, ConvertedExports, - static_cast<llvm::COFF::MachineTypes>(Machine), MinGW); - if (Error) { - std::string errorString; - auto stream = llvm::raw_string_ostream(errorString); - stream << Error; - stream.flush(); - LLVMRustSetLastError(errorString.c_str()); - return LLVMRustResult::Failure; - } else { - return LLVMRustResult::Success; - } -} - // Transfers ownership of DiagnosticHandler unique_ptr to the caller. extern "C" DiagnosticHandler * LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { @@ -2041,11 +1801,7 @@ extern "C" void LLVMRustContextConfigureDiagnosticHandler( } } if (DiagnosticHandlerCallback) { -#if LLVM_VERSION_GE(19, 0) DiagnosticHandlerCallback(&DI, DiagnosticHandlerContext); -#else - DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); -#endif return true; } return false; @@ -2156,10 +1912,6 @@ extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { return -1; } -extern "C" bool LLVMRustIsBitcode(char *ptr, size_t len) { - return identify_magic(StringRef(ptr, len)) == file_magic::bitcode; -} - extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) { if (unwrap<Value>(V)->getType()->isPointerTy()) { if (auto *GV = dyn_cast<GlobalValue>(unwrap<Value>(V))) { @@ -2179,20 +1931,21 @@ extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() { return llvm::compression::zstd::isAvailable(); } -// Operations on composite constants. -// These are clones of LLVM api functions that will become available in future -// releases. They can be removed once Rust's minimum supported LLVM version -// supports them. See https://github.com/rust-lang/rust/issues/121868 See -// https://llvm.org/doxygen/group__LLVMCCoreValueConstantComposite.html +extern "C" void LLVMRustSetNoSanitizeAddress(LLVMValueRef Global) { + GlobalValue &GV = *unwrap<GlobalValue>(Global); + GlobalValue::SanitizerMetadata MD; + if (GV.hasSanitizerMetadata()) + MD = GV.getSanitizerMetadata(); + MD.NoAddress = true; + MD.IsDynInit = false; + GV.setSanitizerMetadata(MD); +} -// FIXME: Remove when Rust's minimum supported LLVM version reaches 19. -// https://github.com/llvm/llvm-project/commit/e1405e4f71c899420ebf8262d5e9745598419df8 -#if LLVM_VERSION_LT(19, 0) -extern "C" LLVMValueRef LLVMConstStringInContext2(LLVMContextRef C, - const char *Str, - size_t Length, - bool DontNullTerminate) { - return wrap(ConstantDataArray::getString(*unwrap(C), StringRef(Str, Length), - !DontNullTerminate)); +extern "C" void LLVMRustSetNoSanitizeHWAddress(LLVMValueRef Global) { + GlobalValue &GV = *unwrap<GlobalValue>(Global); + GlobalValue::SanitizerMetadata MD; + if (GV.hasSanitizerMetadata()) + MD = GV.getSanitizerMetadata(); + MD.NoHWAddress = true; + GV.setSanitizerMetadata(MD); } -#endif diff --git a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp index 54ee79dc290..a910e78d489 100644 --- a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp @@ -8,14 +8,14 @@ // * https://github.com/llvm/llvm-project/blob/ef6d1ec07c693352c4a60dd58db08d2d8558f6ea/llvm/lib/Object/ArchiveWriter.cpp #include "LLVMWrapper.h" -#include "SuppressLLVMWarnings.h" + #include "llvm/ADT/SmallString.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Object/COFF.h" #include "llvm/Object/COFFImportFile.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Object/ObjectFile.h" -#include <llvm/Support/raw_ostream.h> +#include "llvm/Support/raw_ostream.h" using namespace llvm; using namespace llvm::sys; |