diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
33 files changed, 1189 insertions, 1076 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 31ee0eeca11..28f423efc21 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -4,8 +4,7 @@ use std::cmp; use libc::c_uint; use rustc_abi as abi; pub(crate) use rustc_abi::ExternAbi; -use rustc_abi::Primitive::Int; -use rustc_abi::{HasDataLayout, Size}; +use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind, Size}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; @@ -440,7 +439,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| { if cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() >= (19, 0, 0) - && matches!(scalar.primitive(), Int(..)) + && matches!(scalar.primitive(), Primitive::Int(..)) // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected // by the LLVM verifier. @@ -448,11 +447,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // LLVM also rejects full range. && !scalar.is_always_valid(cx) { - attributes::apply_to_llfn(llfn, idx, &[llvm::CreateRangeAttr( - cx.llcx, - scalar.size(cx), - scalar.valid_range(cx), - )]); + attributes::apply_to_llfn( + llfn, + idx, + &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))], + ); } }; @@ -472,10 +471,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); if cx.sess().opts.optimize != config::OptLevel::No { - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[ - llvm::AttributeKind::Writable.create_attr(cx.llcx), - llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx), - ]); + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[ + llvm::AttributeKind::Writable.create_attr(cx.llcx), + llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx), + ], + ); } } PassMode::Cast { cast, pad_i32: _ } => { @@ -574,7 +577,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { if bx.cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() < (19, 0, 0) && let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr - && matches!(scalar.primitive(), Int(..)) + && matches!(scalar.primitive(), Primitive::Int(..)) // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected // by the LLVM verifier. @@ -593,9 +596,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { bx.cx.llcx, bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()), ); - attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[ - byval, - ]); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(i), + &[byval], + ); } PassMode::Direct(attrs) | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { @@ -627,9 +632,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // This will probably get ignored on all targets but those supporting the TrustZone-M // extension (thumbv8m targets). let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call"); - attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &[ - cmse_nonsecure_call, - ]); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Function, + &[cmse_nonsecure_call], + ); } // Some intrinsics require that an elementtype attribute (with the pointee type of a @@ -657,7 +664,7 @@ impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } impl llvm::CallConv { - pub fn from_conv(conv: Conv, arch: &str) -> Self { + pub(crate) fn from_conv(conv: Conv, arch: &str) -> Self { match conv { Conv::C | Conv::Rust @@ -680,7 +687,6 @@ impl llvm::CallConv { Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, Conv::ArmAapcs => llvm::ArmAapcsCallConv, Conv::Msp430Intr => llvm::Msp430Intr, - Conv::PtxKernel => llvm::PtxKernel, Conv::X86Fastcall => llvm::X86FastcallCallConv, Conv::X86Intr => llvm::X86_Intr, Conv::X86Stdcall => llvm::X86StdcallCallConv, diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 149ded28356..66723cbf882 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -81,13 +81,13 @@ pub(crate) unsafe fn codegen( llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); - llvm::LLVMSetInitializer(ll_g, llval); + llvm::set_initializer(ll_g, llval); let name = NO_ALLOC_SHIM_IS_UNSTABLE; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let llval = llvm::LLVMConstInt(i8, 0, False); - llvm::LLVMSetInitializer(ll_g, llval); + llvm::set_initializer(ll_g, llval); } if tcx.sess.opts.debuginfo != DebugInfo::None { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 3722d4350a2..be5673eddf9 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -286,7 +286,9 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::M68k => { constraints.push("~{ccr}".to_string()); } - InlineAsmArch::CSKY => {} + InlineAsmArch::CSKY => { + constraints.push("~{psr}".to_string()); + } } } if !options.contains(InlineAsmOptions::NOMEM) { diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 33a956e552f..93553f3f364 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -11,7 +11,7 @@ use rustc_codegen_ssa::back::archive::{ use rustc_session::Session; use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind}; +use crate::llvm::{self, ArchiveKind, last_error}; /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] @@ -169,6 +169,8 @@ impl<'a> LlvmArchiveBuilder<'a> { .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); let mut additions = mem::take(&mut self.additions); + // Values in the `members` list below will contain pointers to the strings allocated here. + // So they need to get dropped after all elements of `members` get freed. let mut strings = Vec::new(); let mut members = Vec::new(); @@ -229,12 +231,7 @@ impl<'a> LlvmArchiveBuilder<'a> { self.sess.target.arch == "arm64ec", ); let ret = if r.into_result().is_err() { - let err = llvm::LLVMRustGetLastError(); - let msg = if err.is_null() { - "failed to write archive".into() - } else { - String::from_utf8_lossy(CStr::from_ptr(err).to_bytes()) - }; + let msg = last_error().unwrap_or_else(|| "failed to write archive".into()); Err(io::Error::new(io::ErrorKind::Other, msg)) } else { Ok(!members.is_empty()) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 78c759bbe8c..7262fce4911 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -606,10 +606,31 @@ pub(crate) fn run_pass_manager( // If this rustc version was build with enzyme/autodiff enabled, and if users applied the // `#[autodiff]` macro at least once, then we will later call llvm_optimize a second time. - let first_run = true; debug!("running llvm pm opt pipeline"); unsafe { - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, first_run)?; + write::llvm_optimize( + cgcx, + dcx, + module, + config, + opt_level, + opt_stage, + write::AutodiffStage::DuringAD, + )?; + } + // FIXME(ZuseZ4): Make this more granular + if cfg!(llvm_enzyme) && !thin { + unsafe { + write::llvm_optimize( + cgcx, + dcx, + module, + config, + opt_level, + llvm::OptStage::FatLTO, + write::AutodiffStage::PostAD, + )?; + } } debug!("lto done"); Ok(()) @@ -621,7 +642,7 @@ unsafe impl Send for ModuleBuffer {} unsafe impl Sync for ModuleBuffer {} impl ModuleBuffer { - pub fn new(m: &llvm::Module) -> ModuleBuffer { + pub(crate) fn new(m: &llvm::Module) -> ModuleBuffer { ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) } } @@ -663,7 +684,7 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { + pub(crate) fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { unsafe { let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); ThinBuffer(buffer) diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 4cbd49aa44d..f075f332462 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -17,7 +17,7 @@ pub struct OwnedTargetMachine { } impl OwnedTargetMachine { - pub fn new( + pub(crate) fn new( triple: &CStr, cpu: &CStr, features: &CStr, diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 4706744f353..9fa10e96068 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -40,7 +40,7 @@ use crate::errors::{ WithLlvmError, WriteBytecode, }; use crate::llvm::diagnostic::OptimizationDiagnosticKind::*; -use crate::llvm::{self, DiagnosticInfo, PassManager}; +use crate::llvm::{self, DiagnosticInfo}; use crate::type_::Type; use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util}; @@ -54,7 +54,7 @@ pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> Fatal fn write_output_file<'ll>( dcx: DiagCtxtHandle<'_>, target: &'ll llvm::TargetMachine, - pm: &llvm::PassManager<'ll>, + no_builtins: bool, m: &'ll llvm::Module, output: &Path, dwo_output: Option<&Path>, @@ -63,16 +63,19 @@ fn write_output_file<'ll>( verify_llvm_ir: bool, ) -> Result<(), FatalError> { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); - unsafe { - let output_c = path_to_c_string(output); - let dwo_output_c; - let dwo_output_ptr = if let Some(dwo_output) = dwo_output { - dwo_output_c = path_to_c_string(dwo_output); - dwo_output_c.as_ptr() - } else { - std::ptr::null() - }; - let result = llvm::LLVMRustWriteOutputFile( + let output_c = path_to_c_string(output); + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() + } else { + std::ptr::null() + }; + let result = unsafe { + let pm = llvm::LLVMCreatePassManager(); + llvm::LLVMAddAnalysisPasses(target, pm); + llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins); + llvm::LLVMRustWriteOutputFile( target, pm, m, @@ -80,22 +83,22 @@ fn write_output_file<'ll>( dwo_output_ptr, file_type, verify_llvm_ir, - ); + ) + }; - // Record artifact sizes for self-profiling - if result == llvm::LLVMRustResult::Success { - let artifact_kind = match file_type { - llvm::FileType::ObjectFile => "object_file", - llvm::FileType::AssemblyFile => "assembly_file", - }; - record_artifact_size(self_profiler_ref, artifact_kind, output); - if let Some(dwo_file) = dwo_output { - record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); - } + // Record artifact sizes for self-profiling + if result == llvm::LLVMRustResult::Success { + let artifact_kind = match file_type { + llvm::FileType::ObjectFile => "object_file", + llvm::FileType::AssemblyFile => "assembly_file", + }; + record_artifact_size(self_profiler_ref, artifact_kind, output); + if let Some(dwo_file) = dwo_output { + record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); } - - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } + + result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } pub(crate) fn create_informational_target_machine( @@ -138,7 +141,7 @@ fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm:: match cfg { No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), - Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), + More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), @@ -150,7 +153,7 @@ fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel match cfg { No => llvm::PassBuilderOptLevel::O0, Less => llvm::PassBuilderOptLevel::O1, - Default => llvm::PassBuilderOptLevel::O2, + More => llvm::PassBuilderOptLevel::O2, Aggressive => llvm::PassBuilderOptLevel::O3, Size => llvm::PassBuilderOptLevel::Os, SizeMin => llvm::PassBuilderOptLevel::Oz, @@ -325,13 +328,17 @@ pub(crate) fn save_temp_bitcode( if !cgcx.save_temps { return; } + let ext = format!("{name}.bc"); + let cgu = Some(&module.name[..]); + let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); + write_bitcode_to_file(module, &path) +} + +fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) { unsafe { - let ext = format!("{name}.bc"); - let cgu = Some(&module.name[..]); - let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); - let cstr = path_to_c_string(&path); + let path = path_to_c_string(&path); let llmod = module.module_llvm.llmod(); - llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); + llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr()); } } @@ -530,6 +537,16 @@ fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned()) } +// PreAD will run llvm opts but disable size increasing opts (vectorization, loop unrolling) +// DuringAD is the same as above, but also runs the enzyme opt and autodiff passes. +// PostAD will run all opts, including size increasing opts. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum AutodiffStage { + PreAD, + DuringAD, + PostAD, +} + pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, dcx: DiagCtxtHandle<'_>, @@ -537,7 +554,7 @@ pub(crate) unsafe fn llvm_optimize( config: &ModuleConfig, opt_level: config::OptLevel, opt_stage: llvm::OptStage, - skip_size_increasing_opts: bool, + autodiff_stage: AutodiffStage, ) -> Result<(), FatalError> { // Enzyme: // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized @@ -550,12 +567,16 @@ pub(crate) unsafe fn llvm_optimize( let unroll_loops; let vectorize_slp; let vectorize_loop; + let run_enzyme = cfg!(llvm_enzyme) && autodiff_stage == AutodiffStage::DuringAD; // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing - // optimizations until after differentiation. FIXME(ZuseZ4): Before shipping on nightly, + // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt). + // We therefore have two calls to llvm_optimize, if autodiff is used. + // + // FIXME(ZuseZ4): Before shipping on nightly, // we should make this more granular, or at least check that the user has at least one autodiff // call in their code, to justify altering the compilation pipeline. - if skip_size_increasing_opts && cfg!(llvm_enzyme) { + if cfg!(llvm_enzyme) && autodiff_stage != AutodiffStage::PostAD { unroll_loops = false; vectorize_slp = false; vectorize_loop = false; @@ -565,7 +586,7 @@ pub(crate) unsafe fn llvm_optimize( vectorize_slp = config.vectorize_slp; vectorize_loop = config.vectorize_loop; } - trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop); + trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme); let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); @@ -633,6 +654,7 @@ pub(crate) unsafe fn llvm_optimize( vectorize_loop, config.no_builtins, config.emit_lifetime_markers, + run_enzyme, sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), @@ -661,7 +683,6 @@ pub(crate) unsafe fn optimize( ) -> Result<(), FatalError> { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); - let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt); @@ -670,8 +691,7 @@ pub(crate) unsafe fn optimize( if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); - let out = path_to_c_string(&out); - unsafe { llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()) }; + write_bitcode_to_file(module, &out) } // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts @@ -684,18 +704,14 @@ pub(crate) unsafe fn optimize( _ => llvm::OptStage::PreLinkNoLTO, }; - // If we know that we will later run AD, then we disable vectorization and loop unrolling - let skip_size_increasing_opts = cfg!(llvm_enzyme); + // If we know that we will later run AD, then we disable vectorization and loop unrolling. + // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). + // FIXME(ZuseZ4): Make this more granular, only set PreAD if we actually have autodiff + // usages, not just if we build rustc with autodiff support. + let autodiff_stage = + if cfg!(llvm_enzyme) { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; return unsafe { - llvm_optimize( - cgcx, - dcx, - module, - config, - opt_level, - opt_stage, - skip_size_increasing_opts, - ) + llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage) }; } Ok(()) @@ -744,31 +760,6 @@ pub(crate) unsafe fn codegen( create_msvc_imps(cgcx, llcx, llmod); } - // A codegen-specific pass manager is used to generate object - // files for an LLVM module. - // - // Apparently each of these pass managers is a one-shot kind of - // thing, so we create a new one for each type of output. The - // pass manager passed to the closure should be ensured to not - // escape the closure itself, and the manager should only be - // used once. - unsafe fn with_codegen<'ll, F, R>( - tm: &'ll llvm::TargetMachine, - llmod: &'ll llvm::Module, - no_builtins: bool, - f: F, - ) -> R - where - F: FnOnce(&'ll mut PassManager<'ll>) -> R, - { - unsafe { - let cpm = llvm::LLVMCreatePassManager(); - llvm::LLVMAddAnalysisPasses(tm, cpm); - llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); - f(cpm) - } - } - // Note that if object files are just LLVM bitcode we write bitcode, // copy it to the .o file, and delete the bitcode if it wasn't // otherwise requested. @@ -887,21 +878,17 @@ pub(crate) unsafe fn codegen( } else { llmod }; - unsafe { - with_codegen(tm, llmod, config.no_builtins, |cpm| { - write_output_file( - dcx, - tm, - cpm, - llmod, - &path, - None, - llvm::FileType::AssemblyFile, - &cgcx.prof, - config.verify_llvm_ir, - ) - })?; - } + write_output_file( + dcx, + tm, + config.no_builtins, + llmod, + &path, + None, + llvm::FileType::AssemblyFile, + &cgcx.prof, + config.verify_llvm_ir, + )?; } match config.emit_obj { @@ -925,21 +912,17 @@ pub(crate) unsafe fn codegen( (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; - unsafe { - with_codegen(tm, llmod, config.no_builtins, |cpm| { - write_output_file( - dcx, - tm, - cpm, - llmod, - &obj_out, - dwo_out, - llvm::FileType::ObjectFile, - &cgcx.prof, - config.verify_llvm_ir, - ) - })?; - } + write_output_file( + dcx, + tm, + config.no_builtins, + llmod, + &obj_out, + dwo_out, + llvm::FileType::ObjectFile, + &cgcx.prof, + config.verify_llvm_ir, + )?; } EmitObj::Bitcode => { @@ -1066,24 +1049,18 @@ unsafe fn embed_bitcode( { // We don't need custom section flags, create LLVM globals. let llconst = common::bytes_in_context(llcx, bitcode); - let llglobal = llvm::LLVMAddGlobal( - llmod, - common::val_ty(llconst), - c"rustc.embedded.module".as_ptr(), - ); - llvm::LLVMSetInitializer(llglobal, llconst); + let llglobal = + llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module"); + llvm::set_initializer(llglobal, llconst); llvm::set_section(llglobal, bitcode_section_name(cgcx)); llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); - let llglobal = llvm::LLVMAddGlobal( - llmod, - common::val_ty(llconst), - c"rustc.embedded.cmdline".as_ptr(), - ); - llvm::LLVMSetInitializer(llglobal, llconst); + let llglobal = + llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); + llvm::set_initializer(llglobal, llconst); let section = if cgcx.target_is_like_osx { c"__LLVM,__cmdline" } else if cgcx.target_is_like_aix { @@ -1123,31 +1100,29 @@ fn create_msvc_imps( // underscores added in front). let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; - unsafe { - let ptr_ty = Type::ptr_llcx(llcx); - let globals = base::iter_globals(llmod) - .filter(|&val| { - llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage - && llvm::LLVMIsDeclaration(val) == 0 - }) - .filter_map(|val| { - // Exclude some symbols that we know are not Rust symbols. - let name = llvm::get_value_name(val); - if ignored(name) { None } else { Some((val, name)) } - }) - .map(move |(val, name)| { - let mut imp_name = prefix.as_bytes().to_vec(); - imp_name.extend(name); - let imp_name = CString::new(imp_name).unwrap(); - (imp_name, val) - }) - .collect::<Vec<_>>(); + let ptr_ty = Type::ptr_llcx(llcx); + let globals = base::iter_globals(llmod) + .filter(|&val| { + llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val) + }) + .filter_map(|val| { + // Exclude some symbols that we know are not Rust symbols. + let name = llvm::get_value_name(val); + if ignored(name) { None } else { Some((val, name)) } + }) + .map(move |(val, name)| { + let mut imp_name = prefix.as_bytes().to_vec(); + imp_name.extend(name); + let imp_name = CString::new(imp_name).unwrap(); + (imp_name, val) + }) + .collect::<Vec<_>>(); - for (imp_name, val) in globals { - let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr()); - llvm::LLVMSetInitializer(imp, val); - llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); - } + for (imp_name, val) in globals { + let imp = llvm::add_global(llmod, ptr_ty, &imp_name); + + llvm::set_initializer(imp, val); + llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); } // Use this function to exclude certain symbols from `__imp` generation. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index d05faf5577b..d35c7945bae 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -157,9 +157,7 @@ pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage, Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage, Linkage::WeakODR => llvm::Linkage::WeakODRLinkage, - Linkage::Appending => llvm::Linkage::AppendingLinkage, Linkage::Internal => llvm::Linkage::InternalLinkage, - Linkage::Private => llvm::Linkage::PrivateLinkage, Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage, Linkage::Common => llvm::Linkage::CommonLinkage, } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 2d007416263..57f09b1fa2f 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -421,6 +421,51 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unchecked_umul(x, y) => LLVMBuildNUWMul, } + fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(add).is_some() { + llvm::LLVMSetNUW(add, True); + llvm::LLVMSetNSW(add, True); + } + add + } + } + fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(sub).is_some() { + llvm::LLVMSetNUW(sub, True); + llvm::LLVMSetNSW(sub, True); + } + sub + } + } + fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(mul).is_some() { + llvm::LLVMSetNUW(mul, True); + llvm::LLVMSetNSW(mul, True); + } + mul + } + } + + fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED); + + // If a and b are both values, then `or` is a value, rather than + // an instruction, so we need to check before setting the flag. + // (See also `LLVMBuildNUWNeg` which also needs a check.) + if llvm::LLVMIsAInstruction(or).is_some() { + llvm::LLVMSetIsDisjoint(or, True); + } + or + } + } + set_math_builder_methods! { fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath), fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath), @@ -529,7 +574,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - alloca + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) } } @@ -538,7 +584,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - alloca + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) } } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 9e8e4e1c567..b2c1088e3fc 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -4,10 +4,9 @@ use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivit use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::back::write::ModuleConfig; use rustc_errors::FatalError; -use rustc_session::config::Lto; use tracing::{debug, trace}; -use crate::back::write::{llvm_err, llvm_optimize}; +use crate::back::write::llvm_err; use crate::builder::SBuilder; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; @@ -53,8 +52,6 @@ fn generate_enzyme_call<'ll>( let mut ad_name: String = match attrs.mode { DiffMode::Forward => "__enzyme_fwddiff", DiffMode::Reverse => "__enzyme_autodiff", - DiffMode::ForwardFirst => "__enzyme_fwddiff", - DiffMode::ReverseFirst => "__enzyme_autodiff", _ => panic!("logic bug in autodiff, unrecognized mode"), } .to_string(); @@ -153,7 +150,7 @@ fn generate_enzyme_call<'ll>( _ => {} } - trace!("matching autodiff arguments"); + debug!("matching autodiff arguments"); // We now handle the issue that Rust level arguments not always match the llvm-ir level // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on // llvm-ir level. The number of activities matches the number of Rust level arguments, so we @@ -164,10 +161,10 @@ fn generate_enzyme_call<'ll>( let mut activity_pos = 0; let outer_args: Vec<&llvm::Value> = get_params(outer_fn); while activity_pos < inputs.len() { - let activity = inputs[activity_pos as usize]; + let diff_activity = inputs[activity_pos as usize]; // Duplicated arguments received a shadow argument, into which enzyme will write the // gradient. - let (activity, duplicated): (&Metadata, bool) = match activity { + let (activity, duplicated): (&Metadata, bool) = match diff_activity { DiffActivity::None => panic!("not a valid input activity"), DiffActivity::Const => (enzyme_const, false), DiffActivity::Active => (enzyme_out, false), @@ -222,7 +219,15 @@ fn generate_enzyme_call<'ll>( // A duplicated pointer will have the following two outer_fn arguments: // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call: // (..., metadata! enzyme_dup, ptr, ptr, ...). - assert!(llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer); + if matches!( + diff_activity, + DiffActivity::Duplicated | DiffActivity::DuplicatedOnly + ) { + assert!( + llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer + ); + } + // In the case of Dual we don't have assumptions, e.g. f32 would be valid. args.push(next_outer_arg); outer_pos += 2; activity_pos += 1; @@ -277,7 +282,7 @@ pub(crate) fn differentiate<'ll>( module: &'ll ModuleCodegen<ModuleLlvm>, cgcx: &CodegenContext<LlvmCodegenBackend>, diff_items: Vec<AutoDiffItem>, - config: &ModuleConfig, + _config: &ModuleConfig, ) -> Result<(), FatalError> { for item in &diff_items { trace!("{}", item); @@ -291,20 +296,26 @@ pub(crate) fn differentiate<'ll>( let name = item.source.clone(); let fn_def: Option<&llvm::Value> = cx.get_function(&name); let Some(fn_def) = fn_def else { - return Err(llvm_err(diag_handler.handle(), LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find source function".to_owned(), - })); + return Err(llvm_err( + diag_handler.handle(), + LlvmError::PrepareAutoDiff { + src: item.source.clone(), + target: item.target.clone(), + error: "could not find source function".to_owned(), + }, + )); }; debug!(?item.target); let fn_target: Option<&llvm::Value> = cx.get_function(&item.target); let Some(fn_target) = fn_target else { - return Err(llvm_err(diag_handler.handle(), LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find target function".to_owned(), - })); + return Err(llvm_err( + diag_handler.handle(), + LlvmError::PrepareAutoDiff { + src: item.source.clone(), + target: item.target.clone(), + error: "could not find target function".to_owned(), + }, + )); }; generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); @@ -312,29 +323,6 @@ pub(crate) fn differentiate<'ll>( // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts - if let Some(opt_level) = config.opt_level { - let opt_stage = match cgcx.lto { - Lto::Fat => llvm::OptStage::PreLinkFatLTO, - Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, - _ => llvm::OptStage::PreLinkNoLTO, - }; - // This is our second opt call, so now we run all opts, - // to make sure we get the best performance. - let skip_size_increasing_opts = false; - trace!("running Module Optimization after differentiation"); - unsafe { - llvm_optimize( - cgcx, - diag_handler.handle(), - module, - config, - opt_level, - opt_stage, - skip_size_increasing_opts, - )? - }; - } trace!("done with differentiate()"); Ok(()) diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8c94a46ebf3..78b3a7f8541 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -219,8 +219,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { bug!("symbol `{}` is already defined", sym); }); + llvm::set_initializer(g, sc); unsafe { - llvm::LLVMSetInitializer(g, sc); llvm::LLVMSetGlobalConstant(g, True); llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index c6855dd42e5..4a5491ec7a1 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -191,7 +191,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( }) }); llvm::set_linkage(g2, llvm::Linkage::InternalLinkage); - unsafe { llvm::LLVMSetInitializer(g2, g1) }; + llvm::set_initializer(g2, g1); g2 } else if cx.tcx.sess.target.arch == "x86" && common::is_mingw_gnu_toolchain(&cx.tcx.sess.target) @@ -235,7 +235,7 @@ impl<'ll> CodegenCx<'ll, '_> { } _ => self.define_private_global(self.val_ty(cv)), }; - unsafe { llvm::LLVMSetInitializer(gv, cv) }; + llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); gv @@ -458,7 +458,7 @@ impl<'ll> CodegenCx<'ll, '_> { new_g }; set_global_alignment(self, g, alloc.align); - llvm::LLVMSetInitializer(g, v); + llvm::set_initializer(g, v); if self.should_assume_dso_local(g, true) { llvm::LLVMRustSetDSOLocal(g, true); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ebb4b421ad0..e7952bc95e7 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -622,12 +622,10 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let array = self.const_array(self.type_ptr(), values); - unsafe { - let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); - llvm::LLVMSetInitializer(g, array); - llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); - llvm::set_section(g, c"llvm.metadata"); - } + let g = llvm::add_global(self.llmod, self.val_ty(array), name); + llvm::set_initializer(g, array); + llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); + llvm::set_section(g, c"llvm.metadata"); } } impl<'ll> SimpleCx<'ll> { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index 460a4664615..c53ea6d4666 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -11,7 +11,8 @@ use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, }; use rustc_middle::mir::coverage::{ - CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, MappingKind, Op, + BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, + MappingKind, Op, }; use rustc_middle::ty::{Instance, TyCtxt}; use rustc_span::Span; @@ -53,7 +54,7 @@ pub(crate) fn prepare_covfun_record<'tcx>( let fn_cov_info = tcx.instance_mir(instance.def).function_coverage_info.as_deref()?; let ids_info = tcx.coverage_ids_info(instance.def)?; - let expressions = prepare_expressions(fn_cov_info, ids_info, is_used); + let expressions = prepare_expressions(ids_info); let mut covfun = CovfunRecord { mangled_function_name: tcx.symbol_name(instance).name, @@ -75,26 +76,14 @@ pub(crate) fn prepare_covfun_record<'tcx>( } /// Convert the function's coverage-counter expressions into a form suitable for FFI. -fn prepare_expressions( - fn_cov_info: &FunctionCoverageInfo, - ids_info: &CoverageIdsInfo, - is_used: bool, -) -> Vec<ffi::CounterExpression> { - // If any counters or expressions were removed by MIR opts, replace their - // terms with zero. - let counter_for_term = |term| { - if !is_used || ids_info.is_zero_term(term) { - ffi::Counter::ZERO - } else { - ffi::Counter::from_term(term) - } - }; +fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec<ffi::CounterExpression> { + let counter_for_term = ffi::Counter::from_term; // We know that LLVM will optimize out any unused expressions before // producing the final coverage map, so there's no need to do the same // thing on the Rust side unless we're confident we can do much better. // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.) - fn_cov_info + ids_info .expressions .iter() .map(move |&Expression { lhs, op, rhs }| ffi::CounterExpression { @@ -136,11 +125,16 @@ fn fill_region_tables<'tcx>( // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - let is_zero_term = |term| !covfun.is_used || ids_info.is_zero_term(term); for &Mapping { ref kind, span } in &fn_cov_info.mappings { - // If the mapping refers to counters/expressions that were removed by - // MIR opts, replace those occurrences with zero. - let kind = kind.map_terms(|term| if is_zero_term(term) { CovTerm::Zero } else { term }); + // If this function is unused, replace all counters with zero. + let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { + let term = if covfun.is_used { + ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term") + } else { + CovTerm::Zero + }; + ffi::Counter::from_term(term) + }; // Convert the `Span` into coordinates that we can pass to LLVM, or // discard the span if conversion fails. In rare, cases _all_ of a @@ -154,23 +148,22 @@ fn fill_region_tables<'tcx>( continue; } - match kind { - MappingKind::Code(term) => { - code_regions - .push(ffi::CodeRegion { cov_span, counter: ffi::Counter::from_term(term) }); + match *kind { + MappingKind::Code { bcb } => { + code_regions.push(ffi::CodeRegion { cov_span, counter: counter_for_bcb(bcb) }); } - MappingKind::Branch { true_term, false_term } => { + MappingKind::Branch { true_bcb, false_bcb } => { branch_regions.push(ffi::BranchRegion { cov_span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), + true_counter: counter_for_bcb(true_bcb), + false_counter: counter_for_bcb(false_bcb), }); } - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params } => { mcdc_branch_regions.push(ffi::MCDCBranchRegion { cov_span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), + true_counter: counter_for_bcb(true_bcb), + false_counter: counter_for_bcb(false_bcb), mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), }); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 021108cd51c..ea7f581a3cb 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -160,21 +160,12 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( "marker statement {kind:?} should have been removed by CleanupPostBorrowck" ), - CoverageKind::CounterIncrement { id } => { - // The number of counters passed to `llvm.instrprof.increment` might - // be smaller than the number originally inserted by the instrumentor, - // if some high-numbered counters were removed by MIR optimizations. - // If so, LLVM's profiler runtime will use fewer physical counters. - let num_counters = ids_info.num_counters_after_mir_opts(); - assert!( - num_counters as usize <= function_coverage_info.num_counters, - "num_counters disagreement: query says {num_counters} but function info only has {}", - function_coverage_info.num_counters - ); - + CoverageKind::VirtualCounter { bcb } + if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) => + { let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); - let num_counters = bx.const_u32(num_counters); + let num_counters = bx.const_u32(ids_info.num_counters); let index = bx.const_u32(id.as_u32()); debug!( "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", @@ -182,10 +173,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { ); bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::ExpressionUsed { id: _ } => { - // Expression-used statements are markers that are handled by - // `coverage_ids_info`, so there's nothing to codegen here. - } + // If a BCB doesn't have an associated physical counter, there's nothing to codegen. + CoverageKind::VirtualCounter { .. } => {} CoverageKind::CondBitmapUpdate { index, decision_depth } => { let cond_bitmap = coverage_cx .try_get_mcdc_condition_bitmap(&instance, decision_depth) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 11eb9651af6..f52991b3697 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -127,7 +127,7 @@ fn make_mir_scope<'ll, 'tcx>( }) } None => unsafe { - llvm::LLVMRustDIBuilderCreateLexicalBlock( + llvm::LLVMDIBuilderCreateLexicalBlock( DIB(cx), parent_scope.dbg_scope, file_metadata, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 2c9f1cda13a..54c5d445f66 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -73,7 +73,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); llvm::set_section(section_var, c".debug_gdb_scripts"); - llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); + llvm::set_initializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 3a0c7f007bd..59c3fe635d0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -319,19 +319,16 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); let (size, align) = match fn_ty.kind() { - ty::FnDef(..) => (0, 1), - ty::FnPtr(..) => ( - cx.tcx.data_layout.pointer_size.bits(), - cx.tcx.data_layout.pointer_align.abi.bits() as u32, - ), + ty::FnDef(..) => (Size::ZERO, Align::ONE), + ty::FnPtr(..) => (cx.tcx.data_layout.pointer_size, cx.tcx.data_layout.pointer_align.abi), _ => unreachable!(), }; let di_node = unsafe { llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), fn_di_node, - size, - align, + size.bits(), + align.bits() as u32, 0, // Ignore DWARF address space. name.as_c_char_ptr(), name.len(), @@ -931,7 +928,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( - debug_context.builder, + debug_context.builder.as_ref(), name_in_debuginfo.as_c_char_ptr(), name_in_debuginfo.len(), work_dir.as_c_char_ptr(), @@ -944,7 +941,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( ); let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit( - debug_context.builder, + debug_context.builder.as_ref(), dwarf_const::DW_LANG_Rust, compile_unit_file, producer.as_c_char_ptr(), @@ -1641,7 +1638,14 @@ pub(crate) fn extend_scope_to_file<'ll>( file: &SourceFile, ) -> &'ll DILexicalBlock { let file_metadata = file_metadata(cx, file); - unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } + unsafe { + llvm::LLVMDIBuilderCreateLexicalBlockFile( + DIB(cx), + scope_metadata, + file_metadata, + /* Discriminator (default) */ 0u32, + ) + } } fn tuple_field_name(field_index: usize) -> Cow<'static, str> { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 11824398f24..187d97c54c8 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -437,6 +437,12 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( .source_info .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + let discr = discr_value.opt_single_val().map(|value| { + let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); + let size = cx.size_of(tag_base_type); + cx.const_uint_big(cx.type_ix(size.bits()), value) + }); + unsafe { llvm::LLVMRustDIBuilderCreateVariantMemberType( DIB(cx), @@ -448,7 +454,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.opt_single_val().map(|value| cx.const_u128(value)), + discr, DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index b1ce52667bd..17f2d5f4e73 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -2,6 +2,7 @@ use std::cell::{OnceCell, RefCell}; use std::ops::Range; +use std::sync::Arc; use std::{iter, ptr}; use libc::c_uint; @@ -10,7 +11,6 @@ use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::sync::Lrc; use rustc_data_structures::unord::UnordMap; use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::IndexVec; @@ -34,7 +34,7 @@ use crate::builder::Builder; use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::{ - DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, + DIArray, DIBuilderBox, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, DIVariable, }; use crate::value::Value; @@ -61,7 +61,7 @@ const DW_TAG_arg_variable: c_uint = 0x101; /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, - builder: &'ll mut DIBuilder<'ll>, + builder: DIBuilderBox<'ll>, created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, @@ -69,18 +69,10 @@ pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { recursion_marker_type: OnceCell<&'ll DIType>, } -impl Drop for CodegenUnitDebugContext<'_, '_> { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _)); - } - } -} - impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { pub(crate) fn new(llmod: &'ll llvm::Module) -> Self { debug!("CodegenUnitDebugContext::new"); - let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; + let builder = DIBuilderBox::new(llmod); // DIBuilder inherits context from the module, so we'd better use the same one CodegenUnitDebugContext { llmod, @@ -93,7 +85,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { } pub(crate) fn finalize(&self, sess: &Session) { - unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder) }; + unsafe { llvm::LLVMDIBuilderFinalize(self.builder.as_ref()) }; match sess.target.debuginfo_kind { DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym => { @@ -105,7 +97,11 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { // Android has the same issue (#22398) llvm::add_module_flag_u32( self.llmod, - llvm::ModuleFlagMergeBehavior::Warning, + // In the case where multiple CGUs with different dwarf version + // values are being merged together, such as with cross-crate + // LTO, then we want to use the highest version of dwarf + // we can. This matches Clang's behavior as well. + llvm::ModuleFlagMergeBehavior::Max, "Dwarf Version", sess.dwarf_version(), ); @@ -248,7 +244,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { // `lookup_char_pos` return the right information instead. struct DebugLoc { /// Information about the original source file. - file: Lrc<SourceFile>, + file: Arc<SourceFile>, /// The (1-based) line number. line: u32, /// The (1-based) column number. @@ -552,14 +548,17 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - let scope = namespace::item_namespace(cx, DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }); + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); (scope, false) } } @@ -582,7 +581,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { (line, col) }; - unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) } + unsafe { llvm::LLVMDIBuilderCreateDebugLocation(self.llcx, line, col, scope, inlined_at) } } fn create_vtable_debuginfo( @@ -641,7 +640,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { true, DIFlags::FlagZero, argument_index, - align.bytes() as u32, + align.bits() as u32, ) } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index 33d9bc23890..b4d639368b0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -5,7 +5,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::ty::{self, Instance}; use super::utils::{DIB, debug_context}; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::llvm; use crate::llvm::debuginfo::DIScope; @@ -33,12 +33,12 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l }; let scope = unsafe { - llvm::LLVMRustDIBuilderCreateNameSpace( + llvm::LLVMDIBuilderCreateNameSpace( DIB(cx), parent_scope, - namespace_name_string.as_c_char_ptr(), + namespace_name_string.as_ptr(), namespace_name_string.len(), - false, // ExportSymbols (only relevant for C++ anonymous namespaces) + llvm::False, // ExportSymbols (only relevant for C++ anonymous namespaces) ) }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 6e841293477..cc1d504b430 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -41,7 +41,7 @@ pub(crate) fn debug_context<'a, 'll, 'tcx>( #[inline] #[allow(non_snake_case)] pub(crate) fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { - cx.dbg_cx.as_ref().unwrap().builder + cx.dbg_cx.as_ref().unwrap().builder.as_ref() } pub(crate) fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index bdc83267cca..cebceef1b93 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -235,7 +235,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// name. pub(crate) fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { self.get_declared_value(name).and_then(|val| { - let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; + let declaration = llvm::is_declaration(val); if !declaration { Some(val) } else { None } }) } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index f4c9491f758..97f49256165 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -131,8 +131,6 @@ pub enum LlvmError<'a> { LoadBitcode { name: CString }, #[diag(codegen_llvm_write_thinlto_key)] WriteThinLtoKey { err: std::io::Error }, - #[diag(codegen_llvm_multiple_source_dicompileunit)] - MultipleSourceDiCompileUnit, #[diag(codegen_llvm_prepare_thin_lto_module)] PrepareThinLtoModule, #[diag(codegen_llvm_parse_bitcode)] @@ -155,9 +153,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for WithLlvmError<'_> { PrepareThinLtoContext => fluent::codegen_llvm_prepare_thin_lto_context_with_llvm_err, LoadBitcode { .. } => fluent::codegen_llvm_load_bitcode_with_llvm_err, WriteThinLtoKey { .. } => fluent::codegen_llvm_write_thinlto_key_with_llvm_err, - MultipleSourceDiCompileUnit => { - fluent::codegen_llvm_multiple_source_dicompileunit_with_llvm_err - } PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, PrepareAutoDiff { .. } => fluent::codegen_llvm_prepare_autodiff_with_llvm_err, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 43d6ccfcb4a..8b976885904 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -333,12 +333,15 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::prefetch_write_instruction => (1, 0), _ => bug!(), }; - self.call_intrinsic("llvm.prefetch", &[ - args[0].immediate(), - self.const_i32(rw), - args[1].immediate(), - self.const_i32(cache_type), - ]) + self.call_intrinsic( + "llvm.prefetch", + &[ + args[0].immediate(), + self.const_i32(rw), + args[1].immediate(), + self.const_i32(cache_type), + ], + ) } sym::carrying_mul_add => { let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx); @@ -396,10 +399,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { match name { sym::ctlz | sym::cttz => { let y = self.const_bool(false); - let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[ - args[0].immediate(), - y, - ]); + let ret = self.call_intrinsic( + &format!("llvm.{name}.i{width}"), + &[args[0].immediate(), y], + ); self.intcast(ret, llret_ty, false) } @@ -416,24 +419,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.intcast(ret, llret_ty, false) } sym::ctpop => { - let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[ - args[0].immediate() - ]); + let ret = self.call_intrinsic( + &format!("llvm.ctpop.i{width}"), + &[args[0].immediate()], + ); self.intcast(ret, llret_ty, false) } sym::bswap => { if width == 8 { args[0].immediate() // byte swap a u8/i8 is just a no-op } else { - self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[ - args[0].immediate() - ]) + self.call_intrinsic( + &format!("llvm.bswap.i{width}"), + &[args[0].immediate()], + ) } } - sym::bitreverse => self - .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[ - args[0].immediate() - ]), + sym::bitreverse => self.call_intrinsic( + &format!("llvm.bitreverse.i{width}"), + &[args[0].immediate()], + ), sym::rotate_left | sym::rotate_right => { let is_left = name == sym::rotate_left; let val = args[0].immediate(); @@ -500,11 +505,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::compare_bytes => { // Here we assume that the `memcmp` provided by the target is a NOP for size 0. - let cmp = self.call_intrinsic("memcmp", &[ - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - ]); + let cmp = self.call_intrinsic( + "memcmp", + &[args[0].immediate(), args[1].immediate(), args[2].immediate()], + ); // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`. self.sext(cmp, self.type_ix(32)) } @@ -820,7 +824,7 @@ fn codegen_msvc_try<'ll>( if bx.cx.tcx.sess.target.supports_comdat() { llvm::SetUniqueComdat(bx.llmod, tydesc); } - unsafe { llvm::LLVMSetInitializer(tydesc, type_info) }; + llvm::set_initializer(tydesc, type_info); // The flag value of 8 indicates that we are catching the exception by // reference instead of by value. We can't use catch by value because @@ -1305,14 +1309,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if let Some(cmp_op) = comparison { let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); require!( bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty } @@ -1333,21 +1340,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let n = idx.len() as u64; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); - require!(out_len == n, InvalidMonomorphization::ReturnLength { - span, - name, - in_len: n, - ret_ty, - out_len - }); - require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement { - span, - name, - in_elem, - in_ty, - ret_ty, - out_ty - }); + require!( + out_len == n, + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } + ); + require!( + in_elem == out_ty, + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } + ); let total_len = in_len * 2; @@ -1392,21 +1392,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); - require!(out_len == n, InvalidMonomorphization::ReturnLength { - span, - name, - in_len: n, - ret_ty, - out_len - }); - require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement { - span, - name, - in_elem, - in_ty, - ret_ty, - out_ty - }); + require!( + out_len == n, + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } + ); + require!( + in_elem == out_ty, + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } + ); let total_len = u128::from(in_len) * 2; @@ -1431,13 +1424,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_insert { - require!(in_elem == arg_tys[2], InvalidMonomorphization::InsertedType { - span, - name, - in_elem, - in_ty, - out_ty: arg_tys[2] - }); + require!( + in_elem == arg_tys[2], + InvalidMonomorphization::InsertedType { + span, + name, + in_elem, + in_ty, + out_ty: arg_tys[2] + } + ); let idx = bx .const_to_opt_u128(args[1].immediate(), false) .expect("typeck should have ensure that this is a const"); @@ -1456,13 +1452,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( )); } if name == sym::simd_extract { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); let idx = bx .const_to_opt_u128(args[1].immediate(), false) .expect("typeck should have ensure that this is a const"); @@ -1481,18 +1474,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let m_elem_ty = in_elem; let m_len = in_len; let (v_len, _) = require_simd!(arg_tys[1], SimdArgument); - require!(m_len == v_len, InvalidMonomorphization::MismatchedLengths { - span, - name, - m_len, - v_len - }); - let in_elem_bitwidth = - require_int_ty!(m_elem_ty.kind(), InvalidMonomorphization::MaskType { - span, - name, - ty: m_elem_ty - }); + require!( + m_len == v_len, + InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } + ); + let in_elem_bitwidth = require_int_ty!( + m_elem_ty.kind(), + InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty } + ); let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len); return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } @@ -1510,13 +1499,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let expected_bytes = in_len.div_ceil(8); // Integer vector <i{in_bitwidth} x in_len>: - let in_elem_bitwidth = - require_int_or_uint_ty!(in_elem.kind(), InvalidMonomorphization::VectorArgument { - span, - name, - in_ty, - in_elem - }); + let in_elem_bitwidth = require_int_or_uint_ty!( + in_elem.kind(), + InvalidMonomorphization::VectorArgument { span, name, in_ty, in_elem } + ); let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len); // Bitcast <i1 x N> to iN: @@ -1698,30 +1684,34 @@ fn generic_simd_intrinsic<'ll, 'tcx>( require_simd!(ret_ty, SimdReturn); // Of the same length: - require!(in_len == out_len, InvalidMonomorphization::SecondArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[1], - out_len - }); - require!(in_len == out_len2, InvalidMonomorphization::ThirdArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[2], - out_len: out_len2 - }); + require!( + in_len == out_len, + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len + } + ); + require!( + in_len == out_len2, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: out_len2 + } + ); // The return type must match the first argument type - require!(ret_ty == in_ty, InvalidMonomorphization::ExpectedReturnType { - span, - name, - in_ty, - ret_ty - }); + require!( + ret_ty == in_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } + ); require!( matches!( @@ -1739,13 +1729,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ); - let mask_elem_bitwidth = - require_int_ty!(element_ty2.kind(), InvalidMonomorphization::ThirdArgElementType { + let mask_elem_bitwidth = require_int_ty!( + element_ty2.kind(), + InvalidMonomorphization::ThirdArgElementType { span, name, expected_element: element_ty2, third_arg: arg_tys[2] - }); + } + ); // Alignment of T, must be a constant integer value: let alignment_ty = bx.type_i32(); @@ -1805,22 +1797,23 @@ fn generic_simd_intrinsic<'ll, 'tcx>( require_simd!(ret_ty, SimdReturn); // Of the same length: - require!(values_len == mask_len, InvalidMonomorphization::ThirdArgumentLength { - span, - name, - in_len: mask_len, - in_ty: mask_ty, - arg_ty: values_ty, - out_len: values_len - }); + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); // The return type must match the last argument type - require!(ret_ty == values_ty, InvalidMonomorphization::ExpectedReturnType { - span, - name, - in_ty: values_ty, - ret_ty - }); + require!( + ret_ty == values_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty } + ); require!( matches!( @@ -1838,13 +1831,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ); - let m_elem_bitwidth = - require_int_ty!(mask_elem.kind(), InvalidMonomorphization::ThirdArgElementType { + let m_elem_bitwidth = require_int_ty!( + mask_elem.kind(), + InvalidMonomorphization::ThirdArgElementType { span, name, expected_element: values_elem, third_arg: mask_ty, - }); + } + ); let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len); let mask_ty = bx.type_vector(bx.type_i1(), mask_len); @@ -1896,14 +1891,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let (values_len, values_elem) = require_simd!(values_ty, SimdThird); // Of the same length: - require!(values_len == mask_len, InvalidMonomorphization::ThirdArgumentLength { - span, - name, - in_len: mask_len, - in_ty: mask_ty, - arg_ty: values_ty, - out_len: values_len - }); + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); // The second argument must be a mutable pointer type matching the element type require!( @@ -1923,13 +1921,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ); - let m_elem_bitwidth = - require_int_ty!(mask_elem.kind(), InvalidMonomorphization::ThirdArgElementType { + let m_elem_bitwidth = require_int_ty!( + mask_elem.kind(), + InvalidMonomorphization::ThirdArgElementType { span, name, expected_element: values_elem, third_arg: mask_ty, - }); + } + ); let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len); let mask_ty = bx.type_vector(bx.type_i1(), mask_len); @@ -1976,22 +1976,28 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird); // Of the same length: - require!(in_len == element_len1, InvalidMonomorphization::SecondArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[1], - out_len: element_len1 - }); - require!(in_len == element_len2, InvalidMonomorphization::ThirdArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[2], - out_len: element_len2 - }); + require!( + in_len == element_len1, + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len: element_len1 + } + ); + require!( + in_len == element_len2, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: element_len2 + } + ); require!( matches!( @@ -2011,13 +2017,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); // The element type of the third argument must be a signed integer type of any width: - let mask_elem_bitwidth = - require_int_ty!(element_ty2.kind(), InvalidMonomorphization::ThirdArgElementType { + let mask_elem_bitwidth = require_int_ty!( + element_ty2.kind(), + InvalidMonomorphization::ThirdArgElementType { span, name, expected_element: element_ty2, third_arg: arg_tys[2] - }); + } + ); // Alignment of T, must be a constant integer value: let alignment_ty = bx.type_i32(); @@ -2058,13 +2066,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident, $identity:expr) => { if name == sym::$name { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); return match in_elem.kind() { ty::Int(_) | ty::Uint(_) => { let r = bx.$integer_reduce(args[0].immediate()); @@ -2133,13 +2138,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( macro_rules! minmax_red { ($name:ident: $int_red:ident, $float_red:ident) => { if name == sym::$name { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); return match in_elem.kind() { ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), @@ -2164,13 +2166,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ($name:ident : $red:ident, $boolean:expr) => { if name == sym::$name { let input = if !$boolean { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); args[0].immediate() } else { let bitwidth = match in_elem.kind() { @@ -2218,25 +2217,27 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_cast_ptr { let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); match in_elem.kind() { ty::RawPtr(p_ty, _) => { let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(bx.typing_env(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { - span, - name, - ty: in_elem - }); + require!( + metadata.is_unit(), + InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem } + ); } _ => { return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) @@ -2247,11 +2248,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(bx.typing_env(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { - span, - name, - ty: out_elem - }); + require!( + metadata.is_unit(), + InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem } + ); } _ => { return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) @@ -2263,14 +2263,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_expose_provenance { let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); match in_elem.kind() { ty::RawPtr(_, _) => {} @@ -2288,14 +2291,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_with_exposed_provenance { let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); match in_elem.kind() { ty::Uint(ty::UintTy::Usize) => {} @@ -2313,14 +2319,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_cast || name == sym::simd_as { let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); // casting cares about nominal type, not just structural type if in_elem == out_elem { return Ok(args[0].immediate()); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 4a84fd29e44..e9e1b644f18 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -13,6 +13,7 @@ #![feature(extern_types)] #![feature(file_buffered)] #![feature(hash_raw_entry)] +#![feature(if_let_guard)] #![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] #![feature(let_chains)] @@ -29,7 +30,7 @@ use std::mem::ManuallyDrop; use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; use errors::{AutoDiffWithoutLTO, ParseTargetMachineConfig}; -pub use llvm_util::target_features_cfg; +pub(crate) use llvm_util::target_features_cfg; use rustc_ast::expand::allocator::AllocatorKind; use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; @@ -71,14 +72,9 @@ mod debuginfo; mod declare; mod errors; mod intrinsic; - -// The following is a workaround that replaces `pub mod llvm;` and that fixes issue 53912. -#[path = "llvm/mod.rs"] -mod llvm_; -pub mod llvm { - pub use super::llvm_::*; -} - +// FIXME(Zalathar): Fix all the unreachable-pub warnings that would occur if +// this isn't pub, then make it not pub. +pub mod llvm; mod llvm_util; mod mono_item; mod type_; @@ -249,9 +245,6 @@ impl WriteBackendMethods for LlvmCodegenBackend { } } -unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis -unsafe impl Sync for LlvmCodegenBackend {} - impl LlvmCodegenBackend { pub fn new() -> Box<dyn CodegenBackend> { Box::new(LlvmCodegenBackend(())) diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 4dabde55e98..63b2b15c514 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -26,7 +26,7 @@ impl ArchiveRO { /// /// If this archive is used with a mutable method, then an error will be /// raised. - pub fn open(dst: &Path) -> Result<ArchiveRO, String> { + pub(crate) fn open(dst: &Path) -> Result<ArchiveRO, String> { unsafe { let s = path_to_c_string(dst); let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { @@ -36,7 +36,7 @@ impl ArchiveRO { } } - pub fn iter(&self) -> Iter<'_> { + pub(crate) fn iter(&self) -> Iter<'_> { unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } } } @@ -71,7 +71,7 @@ impl<'a> Drop for Iter<'a> { } impl<'a> Child<'a> { - pub fn name(&self) -> Option<&'a str> { + pub(crate) fn name(&self) -> Option<&'a str> { unsafe { let mut name_len = 0; let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 92b0ce8ffe1..39bac13a968 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -1,4 +1,5 @@ #![allow(non_camel_case_types)] +#![expect(dead_code)] use libc::{c_char, c_uint}; @@ -8,23 +9,23 @@ use crate::llvm::Bool; #[link(name = "llvm-wrapper", kind = "static")] extern "C" { // Enzyme - pub fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool; - pub fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value); - pub fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>; - pub fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>; - pub fn LLVMRustEraseInstFromParent(V: &Value); - pub fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value; - pub fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; + pub(crate) fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool; + pub(crate) fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value); + pub(crate) fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>; + pub(crate) fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>; + pub(crate) fn LLVMRustEraseInstFromParent(V: &Value); + pub(crate) fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value; + pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; } extern "C" { // Enzyme - pub fn LLVMDumpModule(M: &Module); - pub fn LLVMDumpValue(V: &Value); - pub fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; - pub fn LLVMGetReturnType(T: &Type) -> &Type; - pub fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); - pub fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>; + pub(crate) fn LLVMDumpModule(M: &Module); + pub(crate) fn LLVMDumpValue(V: &Value); + pub(crate) fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; + pub(crate) fn LLVMGetReturnType(T: &Type) -> &Type; + pub(crate) fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); + pub(crate) fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>; } #[repr(C)] diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index cc7c5231aca..3b0187b9d37 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,3 +1,15 @@ +//! Bindings to the LLVM-C API (`LLVM*`), and to our own `extern "C"` wrapper +//! functions around the unstable LLVM C++ API (`LLVMRust*`). +//! +//! ## Passing pointer/length strings as `*const c_uchar` +//! +//! Normally it's a good idea for Rust-side bindings to match the corresponding +//! C-side function declarations as closely as possible. But when passing `&str` +//! or `&[u8]` data as a pointer/length pair, it's more convenient to declare +//! the Rust-side pointer as `*const c_uchar` instead of `*const c_char`. +//! Both pointer types have the same ABI, and using `*const c_uchar` avoids +//! the need for an extra cast from `*const u8` on the Rust side. + #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] @@ -5,17 +17,18 @@ use std::fmt::Debug; use std::marker::PhantomData; use std::ptr; -use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; +use bitflags::bitflags; +use libc::{c_char, c_int, c_uchar, c_uint, c_ulonglong, c_void, size_t}; use rustc_macros::TryFromU32; use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, - DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, - DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, - DebugEmissionKind, DebugNameTableKind, + DIFile, DIFlags, DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, + DISubrange, DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, }; +use crate::llvm; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. @@ -56,19 +69,6 @@ pub enum LLVMRustResult { Failure, } -/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h. -/// -/// We include only architectures supported on Windows. -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum LLVMMachineType { - AMD64 = 0x8664, - I386 = 0x14c, - ARM64 = 0xaa64, - ARM64EC = 0xa641, - ARM = 0x01c0, -} - /// Must match the layout of `LLVMRustModuleFlagMergeBehavior`. /// /// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are @@ -160,7 +160,7 @@ pub enum Visibility { } impl Visibility { - pub fn from_generic(visibility: SymbolVisibility) -> Self { + pub(crate) fn from_generic(visibility: SymbolVisibility) -> Self { match visibility { SymbolVisibility::Hidden => Visibility::Hidden, SymbolVisibility::Protected => Visibility::Protected, @@ -255,7 +255,7 @@ pub enum IntPredicate { } impl IntPredicate { - pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { + pub(crate) fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { use rustc_codegen_ssa::common::IntPredicate as Common; match intpre { Common::IntEQ => Self::IntEQ, @@ -295,7 +295,7 @@ pub enum RealPredicate { } impl RealPredicate { - pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { + pub(crate) fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { use rustc_codegen_ssa::common::RealPredicate as Common; match realp { Common::RealPredicateFalse => Self::RealPredicateFalse, @@ -344,7 +344,7 @@ pub enum TypeKind { } impl TypeKind { - pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { + pub(crate) fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { use rustc_codegen_ssa::common::TypeKind as Common; match self { Self::Void => Common::Void, @@ -388,7 +388,7 @@ pub enum AtomicRmwBinOp { } impl AtomicRmwBinOp { - pub fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { + pub(crate) fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { use rustc_codegen_ssa::common::AtomicRmwBinOp as Common; match op { Common::AtomicXchg => Self::AtomicXchg, @@ -422,7 +422,7 @@ pub enum AtomicOrdering { } impl AtomicOrdering { - pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { + pub(crate) fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { use rustc_codegen_ssa::common::AtomicOrdering as Common; match ao { Common::Unordered => Self::Unordered, @@ -632,16 +632,6 @@ pub enum ThreadLocalMode { LocalExec, } -/// LLVMRustTailCallKind -#[derive(Copy, Clone)] -#[repr(C)] -pub enum TailCallKind { - None, - Tail, - MustTail, - NoTail, -} - /// LLVMRustChecksumKind #[derive(Copy, Clone)] #[repr(C)] @@ -760,7 +750,6 @@ pub struct Builder<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct PassManager<'a>(InvariantOpaque<'a>); unsafe extern "C" { - pub type Pass; pub type TargetMachine; pub type Archive; } @@ -786,15 +775,52 @@ unsafe extern "C" { } pub type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); -pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint); pub mod debuginfo { + use std::ptr; + use bitflags::bitflags; use super::{InvariantOpaque, Metadata}; + use crate::llvm::{self, Module}; + /// Opaque target type for references to an LLVM debuginfo builder. + /// + /// `&'_ DIBuilder<'ll>` corresponds to `LLVMDIBuilderRef`, which is the + /// LLVM-C wrapper for `DIBuilder *`. + /// + /// Debuginfo builders are created and destroyed during codegen, so the + /// builder reference typically has a shorter lifetime than the LLVM + /// session (`'ll`) that it participates in. #[repr(C)] - pub struct DIBuilder<'a>(InvariantOpaque<'a>); + pub struct DIBuilder<'ll>(InvariantOpaque<'ll>); + + /// Owning pointer to a `DIBuilder<'ll>` that will dispose of the builder + /// when dropped. Use `.as_ref()` to get the underlying `&DIBuilder` + /// needed for debuginfo FFI calls. + pub(crate) struct DIBuilderBox<'ll> { + raw: ptr::NonNull<DIBuilder<'ll>>, + } + + impl<'ll> DIBuilderBox<'ll> { + pub(crate) fn new(llmod: &'ll Module) -> Self { + let raw = unsafe { llvm::LLVMCreateDIBuilder(llmod) }; + let raw = ptr::NonNull::new(raw).unwrap(); + Self { raw } + } + + pub(crate) fn as_ref(&self) -> &DIBuilder<'ll> { + // SAFETY: This is an owning pointer, so `&DIBuilder` is valid + // for as long as `&self` is. + unsafe { self.raw.as_ref() } + } + } + + impl<'ll> Drop for DIBuilderBox<'ll> { + fn drop(&mut self) { + unsafe { llvm::LLVMDisposeDIBuilder(self.raw) }; + } + } pub type DIDescriptor = Metadata; pub type DILocation = Metadata; @@ -802,7 +828,6 @@ pub mod debuginfo { pub type DIFile = DIScope; pub type DILexicalBlock = DIScope; pub type DISubprogram = DIScope; - pub type DINameSpace = DIScope; pub type DIType = DIDescriptor; pub type DIBasicType = DIType; pub type DIDerivedType = DIType; @@ -883,7 +908,7 @@ pub mod debuginfo { } impl DebugEmissionKind { - pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + pub(crate) fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { // We should be setting LLVM's emission kind to `LineTablesOnly` if // we are compiling with "limited" debuginfo. However, some of the // existing tools relied on slightly more debuginfo being generated than @@ -914,7 +939,6 @@ pub mod debuginfo { } } -use bitflags::bitflags; // These values **must** match with LLVMRustAllocKindFlags bitflags! { #[repr(transparent)] @@ -943,49 +967,55 @@ pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c unsafe extern "C" { // Create and destroy contexts. - pub fn LLVMContextDispose(C: &'static mut Context); - pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint; + pub(crate) fn LLVMContextDispose(C: &'static mut Context); + pub(crate) fn LLVMGetMDKindIDInContext( + C: &Context, + Name: *const c_char, + SLen: c_uint, + ) -> c_uint; // Create modules. - pub fn LLVMModuleCreateWithNameInContext(ModuleID: *const c_char, C: &Context) -> &Module; - pub fn LLVMGetModuleContext(M: &Module) -> &Context; - pub fn LLVMCloneModule(M: &Module) -> &Module; + pub(crate) fn LLVMModuleCreateWithNameInContext( + ModuleID: *const c_char, + C: &Context, + ) -> &Module; + pub(crate) fn LLVMCloneModule(M: &Module) -> &Module; /// Data layout. See Module::getDataLayout. - pub fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; - pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); + pub(crate) fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; + pub(crate) fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); + pub(crate) fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); // Operations on integer types - pub fn LLVMInt1TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt8TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt16TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt32TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt64TypeInContext(C: &Context) -> &Type; - pub fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; + pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt16TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt32TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt64TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; - pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; + pub(crate) fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; // Operations on real types - pub fn LLVMHalfTypeInContext(C: &Context) -> &Type; - pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; - pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; - pub fn LLVMFP128TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMHalfTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; // Operations on function types - pub fn LLVMFunctionType<'a>( + pub(crate) fn LLVMFunctionType<'a>( ReturnType: &'a Type, ParamTypes: *const &'a Type, ParamCount: c_uint, IsVarArg: Bool, ) -> &'a Type; - pub fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; - pub fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; + pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); // Operations on struct types - pub fn LLVMStructTypeInContext<'a>( + pub(crate) fn LLVMStructTypeInContext<'a>( C: &'a Context, ElementTypes: *const &'a Type, ElementCount: c_uint, @@ -993,111 +1023,123 @@ unsafe extern "C" { ) -> &'a Type; // Operations on array, pointer, and vector types (sequence types) - pub fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; - pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; + pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; + pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; - pub fn LLVMGetElementType(Ty: &Type) -> &Type; - pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; + pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type; + pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; // Operations on other types - pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMTokenTypeInContext(C: &Context) -> &Type; - pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMTokenTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values - pub fn LLVMIsUndef(Val: &Value) -> Bool; - pub fn LLVMTypeOf(Val: &Value) -> &Type; - pub fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; - pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); - pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); - pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); - pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + pub(crate) fn LLVMIsUndef(Val: &Value) -> Bool; + pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type; + pub(crate) fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; + pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); + pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); + pub(crate) fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); + pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub(crate) fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type - pub fn LLVMConstNull(Ty: &Type) -> &Value; - pub fn LLVMGetUndef(Ty: &Type) -> &Value; - pub fn LLVMGetPoison(Ty: &Type) -> &Value; + pub(crate) fn LLVMConstNull(Ty: &Type) -> &Value; + pub(crate) fn LLVMGetUndef(Ty: &Type) -> &Value; + pub(crate) fn LLVMGetPoison(Ty: &Type) -> &Value; // Operations on metadata - pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata; - pub fn LLVMMDNodeInContext2<'a>( + pub(crate) fn LLVMMDStringInContext2( + C: &Context, + Str: *const c_char, + SLen: size_t, + ) -> &Metadata; + pub(crate) fn LLVMMDNodeInContext2<'a>( C: &'a Context, Vals: *const &'a Metadata, Count: size_t, ) -> &'a Metadata; - pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); + pub(crate) fn LLVMAddNamedMetadataOperand<'a>( + M: &'a Module, + Name: *const c_char, + Val: &'a Value, + ); // Operations on scalar constants - pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; - pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; - pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; + pub(crate) fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; + pub(crate) fn LLVMConstIntOfArbitraryPrecision( + IntTy: &Type, + Wn: c_uint, + Ws: *const u64, + ) -> &Value; + pub(crate) fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; // Operations on composite constants - pub fn LLVMConstArray2<'a>( + pub(crate) fn LLVMConstArray2<'a>( ElementTy: &'a Type, ConstantVals: *const &'a Value, Length: u64, ) -> &'a Value; - pub fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; - pub fn LLVMConstStringInContext2( + pub(crate) fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; + pub(crate) fn LLVMConstStringInContext2( C: &Context, Str: *const c_char, Length: size_t, DontNullTerminate: Bool, ) -> &Value; - pub fn LLVMConstStructInContext<'a>( + pub(crate) fn LLVMConstStructInContext<'a>( C: &'a Context, ConstantVals: *const &'a Value, Count: c_uint, Packed: Bool, ) -> &'a Value; - pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; + pub(crate) fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions - pub fn LLVMConstInBoundsGEP2<'a>( + pub(crate) fn LLVMConstInBoundsGEP2<'a>( ty: &'a Type, ConstantVal: &'a Value, ConstantIndices: *const &'a Value, NumIndices: c_uint, ) -> &'a Value; - pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; - pub fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; - pub fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; + pub(crate) fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; + pub(crate) fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMIsDeclaration(Global: &Value) -> Bool; - pub fn LLVMGetLinkage(Global: &Value) -> RawEnum<Linkage>; - pub fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); - pub fn LLVMSetSection(Global: &Value, Section: *const c_char); - pub fn LLVMGetVisibility(Global: &Value) -> RawEnum<Visibility>; - pub fn LLVMSetVisibility(Global: &Value, Viz: Visibility); - pub fn LLVMGetAlignment(Global: &Value) -> c_uint; - pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); - pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); - pub fn LLVMGlobalGetValueType(Global: &Value) -> &Type; + pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub(crate) fn LLVMGetLinkage(Global: &Value) -> RawEnum<Linkage>; + pub(crate) fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); + pub(crate) fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub(crate) fn LLVMGetVisibility(Global: &Value) -> RawEnum<Visibility>; + pub(crate) fn LLVMSetVisibility(Global: &Value, Viz: Visibility); + pub(crate) fn LLVMGetAlignment(Global: &Value) -> c_uint; + pub(crate) fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); + pub(crate) fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); + pub(crate) fn LLVMGlobalGetValueType(Global: &Value) -> &Type; // Operations on global variables - pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; - pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; - pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMDeleteGlobal(GlobalVar: &Value); - pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); - pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; - pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); - pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; - pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); - pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub(crate) fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub(crate) fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; + pub(crate) fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; + pub(crate) fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMDeleteGlobal(GlobalVar: &Value); + pub(crate) fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); + pub(crate) fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; + pub(crate) fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); + pub(crate) fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; + pub(crate) fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); + pub(crate) fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); // Operations on attributes - pub fn LLVMCreateStringAttribute( + pub(crate) fn LLVMCreateStringAttribute( C: &Context, Name: *const c_char, NameLen: c_uint, @@ -1106,34 +1148,34 @@ unsafe extern "C" { ) -> &Attribute; // Operations on functions - pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); // Operations on parameters - pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; - pub fn LLVMCountParams(Fn: &Value) -> c_uint; - pub fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; + pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMCountParams(Fn: &Value) -> c_uint; + pub(crate) fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; // Operations on basic blocks - pub fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; - pub fn LLVMAppendBasicBlockInContext<'a>( + pub(crate) fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; + pub(crate) fn LLVMAppendBasicBlockInContext<'a>( C: &'a Context, Fn: &'a Value, Name: *const c_char, ) -> &'a BasicBlock; // Operations on instructions - pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; - pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; - pub fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; + pub(crate) fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; + pub(crate) fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; // Operations on call sites - pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); + pub(crate) fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); // Operations on load/store instructions (only) - pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); + pub(crate) fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); // Operations on phi nodes - pub fn LLVMAddIncoming<'a>( + pub(crate) fn LLVMAddIncoming<'a>( PhiNode: &'a Value, IncomingValues: *const &'a Value, IncomingBlocks: *const &'a BasicBlock, @@ -1141,263 +1183,278 @@ unsafe extern "C" { ); // Instruction builders - pub fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; - pub fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); - pub fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; - pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); + pub(crate) fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; + pub(crate) fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); + pub(crate) fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; + pub(crate) fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); - pub fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>; + pub(crate) fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); + pub(crate) fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>; // Terminators - pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; - pub fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; - pub fn LLVMBuildCondBr<'a>( + pub(crate) fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; + pub(crate) fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; + pub(crate) fn LLVMBuildCondBr<'a>( B: &Builder<'a>, If: &'a Value, Then: &'a BasicBlock, Else: &'a BasicBlock, ) -> &'a Value; - pub fn LLVMBuildSwitch<'a>( + pub(crate) fn LLVMBuildSwitch<'a>( B: &Builder<'a>, V: &'a Value, Else: &'a BasicBlock, NumCases: c_uint, ) -> &'a Value; - pub fn LLVMBuildLandingPad<'a>( + pub(crate) fn LLVMBuildLandingPad<'a>( B: &Builder<'a>, Ty: &'a Type, PersFn: Option<&'a Value>, NumClauses: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; - pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; + pub(crate) fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMBuildCleanupPad<'a>( + pub(crate) fn LLVMBuildCleanupPad<'a>( B: &Builder<'a>, ParentPad: Option<&'a Value>, Args: *const &'a Value, NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMBuildCleanupRet<'a>( + pub(crate) fn LLVMBuildCleanupRet<'a>( B: &Builder<'a>, CleanupPad: &'a Value, BB: Option<&'a BasicBlock>, ) -> Option<&'a Value>; - pub fn LLVMBuildCatchPad<'a>( + pub(crate) fn LLVMBuildCatchPad<'a>( B: &Builder<'a>, ParentPad: &'a Value, Args: *const &'a Value, NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMBuildCatchRet<'a>( + pub(crate) fn LLVMBuildCatchRet<'a>( B: &Builder<'a>, CatchPad: &'a Value, BB: &'a BasicBlock, ) -> Option<&'a Value>; - pub fn LLVMBuildCatchSwitch<'a>( + pub(crate) fn LLVMBuildCatchSwitch<'a>( Builder: &Builder<'a>, ParentPad: Option<&'a Value>, UnwindBB: Option<&'a BasicBlock>, NumHandlers: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); - pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); + pub(crate) fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); + pub(crate) fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); // Add a case to the switch instruction - pub fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); + pub(crate) fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); // Add a clause to the landing pad instruction - pub fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); + pub(crate) fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); // Set the cleanup on a landing pad instruction - pub fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); + pub(crate) fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); // Arithmetic - pub fn LLVMBuildAdd<'a>( + pub(crate) fn LLVMBuildAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFAdd<'a>( + pub(crate) fn LLVMBuildFAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSub<'a>( + pub(crate) fn LLVMBuildSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFSub<'a>( + pub(crate) fn LLVMBuildFSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildMul<'a>( + pub(crate) fn LLVMBuildMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFMul<'a>( + pub(crate) fn LLVMBuildFMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildUDiv<'a>( + pub(crate) fn LLVMBuildUDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExactUDiv<'a>( + pub(crate) fn LLVMBuildExactUDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSDiv<'a>( + pub(crate) fn LLVMBuildSDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExactSDiv<'a>( + pub(crate) fn LLVMBuildExactSDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFDiv<'a>( + pub(crate) fn LLVMBuildFDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildURem<'a>( + pub(crate) fn LLVMBuildURem<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSRem<'a>( + pub(crate) fn LLVMBuildSRem<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFRem<'a>( + pub(crate) fn LLVMBuildFRem<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildShl<'a>( + pub(crate) fn LLVMBuildShl<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildLShr<'a>( + pub(crate) fn LLVMBuildLShr<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildAShr<'a>( + pub(crate) fn LLVMBuildAShr<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNSWAdd<'a>( + pub(crate) fn LLVMBuildNSWAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNUWAdd<'a>( + pub(crate) fn LLVMBuildNUWAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNSWSub<'a>( + pub(crate) fn LLVMBuildNSWSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNUWSub<'a>( + pub(crate) fn LLVMBuildNUWSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNSWMul<'a>( + pub(crate) fn LLVMBuildNSWMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNUWMul<'a>( + pub(crate) fn LLVMBuildNUWMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildAnd<'a>( + pub(crate) fn LLVMBuildAnd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildOr<'a>( + pub(crate) fn LLVMBuildOr<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildXor<'a>( + pub(crate) fn LLVMBuildXor<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub(crate) fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMBuildFNeg<'a>( + B: &Builder<'a>, + V: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) + -> &'a Value; + + // Extra flags on arithmetic + pub(crate) fn LLVMSetIsDisjoint(Instr: &Value, IsDisjoint: Bool); + pub(crate) fn LLVMSetNUW(ArithInst: &Value, HasNUW: Bool); + pub(crate) fn LLVMSetNSW(ArithInst: &Value, HasNSW: Bool); // Memory - pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildArrayAlloca<'a>( + pub(crate) fn LLVMBuildAlloca<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildArrayAlloca<'a>( B: &Builder<'a>, Ty: &'a Type, Val: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildLoad2<'a>( + pub(crate) fn LLVMBuildLoad2<'a>( B: &Builder<'a>, Ty: &'a Type, PointerVal: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; - pub fn LLVMBuildGEP2<'a>( + pub(crate) fn LLVMBuildGEP2<'a>( B: &Builder<'a>, Ty: &'a Type, Pointer: &'a Value, @@ -1405,7 +1462,7 @@ unsafe extern "C" { NumIndices: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildInBoundsGEP2<'a>( + pub(crate) fn LLVMBuildInBoundsGEP2<'a>( B: &Builder<'a>, Ty: &'a Type, Pointer: &'a Value, @@ -1415,85 +1472,85 @@ unsafe extern "C" { ) -> &'a Value; // Casts - pub fn LLVMBuildTrunc<'a>( + pub(crate) fn LLVMBuildTrunc<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildZExt<'a>( + pub(crate) fn LLVMBuildZExt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSExt<'a>( + pub(crate) fn LLVMBuildSExt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPToUI<'a>( + pub(crate) fn LLVMBuildFPToUI<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPToSI<'a>( + pub(crate) fn LLVMBuildFPToSI<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildUIToFP<'a>( + pub(crate) fn LLVMBuildUIToFP<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSIToFP<'a>( + pub(crate) fn LLVMBuildSIToFP<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPTrunc<'a>( + pub(crate) fn LLVMBuildFPTrunc<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPExt<'a>( + pub(crate) fn LLVMBuildFPExt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildPtrToInt<'a>( + pub(crate) fn LLVMBuildPtrToInt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildIntToPtr<'a>( + pub(crate) fn LLVMBuildIntToPtr<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildBitCast<'a>( + pub(crate) fn LLVMBuildBitCast<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildPointerCast<'a>( + pub(crate) fn LLVMBuildPointerCast<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildIntCast2<'a>( + pub(crate) fn LLVMBuildIntCast2<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, @@ -1502,14 +1559,14 @@ unsafe extern "C" { ) -> &'a Value; // Comparisons - pub fn LLVMBuildICmp<'a>( + pub(crate) fn LLVMBuildICmp<'a>( B: &Builder<'a>, Op: c_uint, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFCmp<'a>( + pub(crate) fn LLVMBuildFCmp<'a>( B: &Builder<'a>, Op: c_uint, LHS: &'a Value, @@ -1518,47 +1575,48 @@ unsafe extern "C" { ) -> &'a Value; // Miscellaneous instructions - pub fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildSelect<'a>( + pub(crate) fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMBuildSelect<'a>( B: &Builder<'a>, If: &'a Value, Then: &'a Value, Else: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildVAArg<'a>( + pub(crate) fn LLVMBuildVAArg<'a>( B: &Builder<'a>, list: &'a Value, Ty: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExtractElement<'a>( + pub(crate) fn LLVMBuildExtractElement<'a>( B: &Builder<'a>, VecVal: &'a Value, Index: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildInsertElement<'a>( + pub(crate) fn LLVMBuildInsertElement<'a>( B: &Builder<'a>, VecVal: &'a Value, EltVal: &'a Value, Index: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildShuffleVector<'a>( + pub(crate) fn LLVMBuildShuffleVector<'a>( B: &Builder<'a>, V1: &'a Value, V2: &'a Value, Mask: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExtractValue<'a>( + pub(crate) fn LLVMBuildExtractValue<'a>( B: &Builder<'a>, AggVal: &'a Value, Index: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildInsertValue<'a>( + pub(crate) fn LLVMBuildInsertValue<'a>( B: &Builder<'a>, AggVal: &'a Value, EltVal: &'a Value, @@ -1567,7 +1625,7 @@ unsafe extern "C" { ) -> &'a Value; // Atomic Operations - pub fn LLVMBuildAtomicCmpXchg<'a>( + pub(crate) fn LLVMBuildAtomicCmpXchg<'a>( B: &Builder<'a>, LHS: &'a Value, CMP: &'a Value, @@ -1577,9 +1635,9 @@ unsafe extern "C" { SingleThreaded: Bool, ) -> &'a Value; - pub fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool); + pub(crate) fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool); - pub fn LLVMBuildAtomicRMW<'a>( + pub(crate) fn LLVMBuildAtomicRMW<'a>( B: &Builder<'a>, Op: AtomicRmwBinOp, LHS: &'a Value, @@ -1588,7 +1646,7 @@ unsafe extern "C" { SingleThreaded: Bool, ) -> &'a Value; - pub fn LLVMBuildFence<'a>( + pub(crate) fn LLVMBuildFence<'a>( B: &Builder<'a>, Order: AtomicOrdering, SingleThreaded: Bool, @@ -1596,36 +1654,36 @@ unsafe extern "C" { ) -> &'a Value; /// Writes a module to the specified path. Returns 0 on success. - pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; + pub(crate) fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; /// Creates a legacy pass manager -- only used for final codegen. - pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; + pub(crate) fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; - pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); + pub(crate) fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - pub fn LLVMGetHostCPUFeatures() -> *mut c_char; + pub(crate) fn LLVMGetHostCPUFeatures() -> *mut c_char; - pub fn LLVMDisposeMessage(message: *mut c_char); + pub(crate) fn LLVMDisposeMessage(message: *mut c_char); - pub fn LLVMIsMultithreaded() -> Bool; + pub(crate) fn LLVMIsMultithreaded() -> Bool; - pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; + pub(crate) fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; - pub fn LLVMStructSetBody<'a>( + pub(crate) fn LLVMStructSetBody<'a>( StructTy: &'a Type, ElementTypes: *const &'a Type, ElementCount: c_uint, Packed: Bool, ); - pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + pub(crate) fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; - pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); + pub(crate) fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); - pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + pub(crate) fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - pub fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; - pub fn LLVMSetComdat(V: &Value, C: &Comdat); + pub(crate) fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; + pub(crate) fn LLVMSetComdat(V: &Value, C: &Comdat); pub(crate) fn LLVMCreateOperandBundle( Tag: *const c_char, @@ -1672,24 +1730,72 @@ unsafe extern "C" { ) -> &'a Value; } +// FFI bindings for `DIBuilder` functions in the LLVM-C API. +// Try to keep these in the same order as in `llvm/include/llvm-c/DebugInfo.h`. +// +// FIXME(#134001): Audit all `Option` parameters, especially in lists, to check +// that they really are nullable on the C/C++ side. LLVM doesn't appear to +// actually document which ones are nullable. +unsafe extern "C" { + pub(crate) fn LLVMCreateDIBuilder<'ll>(M: &'ll Module) -> *mut DIBuilder<'ll>; + pub(crate) fn LLVMDisposeDIBuilder<'ll>(Builder: ptr::NonNull<DIBuilder<'ll>>); + + pub(crate) fn LLVMDIBuilderFinalize<'ll>(Builder: &DIBuilder<'ll>); + + pub(crate) fn LLVMDIBuilderCreateNameSpace<'ll>( + Builder: &DIBuilder<'ll>, + ParentScope: Option<&'ll Metadata>, + Name: *const c_uchar, + NameLen: size_t, + ExportSymbols: llvm::Bool, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateLexicalBlock<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + File: &'ll Metadata, + Line: c_uint, + Column: c_uint, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateLexicalBlockFile<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + File: &'ll Metadata, + Discriminator: c_uint, // (optional "DWARF path discriminator"; default is 0) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateDebugLocation<'ll>( + Ctx: &'ll Context, + Line: c_uint, + Column: c_uint, + Scope: &'ll Metadata, + InlinedAt: Option<&'ll Metadata>, + ) -> &'ll Metadata; +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { - pub fn LLVMRustInstallErrorHandlers(); - pub fn LLVMRustDisableSystemDialogsOnCrash(); + pub(crate) fn LLVMRustInstallErrorHandlers(); + pub(crate) fn LLVMRustDisableSystemDialogsOnCrash(); // Create and destroy contexts. - pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; + pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; /// See llvm::LLVMTypeKind::getTypeID. - pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; + pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; // Operations on all values - pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); - pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; + pub(crate) fn LLVMRustGlobalAddMetadata<'a>( + Val: &'a Value, + KindID: c_uint, + Metadata: &'a Metadata, + ); + pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; // Operations on scalar constants - pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; - pub fn LLVMRustConstInt128Get( + pub(crate) fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; + pub(crate) fn LLVMRustConstInt128Get( ConstantVal: &ConstantInt, SExt: bool, high: &mut u64, @@ -1697,36 +1803,38 @@ unsafe extern "C" { ) -> bool; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); + pub(crate) fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); // Operations on global variables - pub fn LLVMRustGetOrInsertGlobal<'a>( + pub(crate) fn LLVMRustGetOrInsertGlobal<'a>( M: &'a Module, Name: *const c_char, NameLen: size_t, T: &'a Type, ) -> &'a Value; - pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; - pub fn LLVMRustGetNamedValue( + pub(crate) fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; + pub(crate) fn LLVMRustGetNamedValue( M: &Module, Name: *const c_char, NameLen: size_t, ) -> Option<&Value>; - pub fn LLVMRustSetTailCallKind(CallInst: &Value, TKC: TailCallKind); // Operations on attributes - pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; - pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; - pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; - pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; - pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute; - pub fn LLVMRustCreateRangeAttribute( + pub(crate) fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; + pub(crate) fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub(crate) fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub(crate) fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateMemoryEffectsAttr( + C: &Context, + effects: MemoryEffects, + ) -> &Attribute; + pub(crate) fn LLVMRustCreateRangeAttribute( C: &Context, num_bits: c_uint, lower_words: *const u64, @@ -1734,13 +1842,13 @@ unsafe extern "C" { ) -> &Attribute; // Operations on functions - pub fn LLVMRustGetOrInsertFunction<'a>( + pub(crate) fn LLVMRustGetOrInsertFunction<'a>( M: &'a Module, Name: *const c_char, NameLen: size_t, FunctionTy: &'a Type, ) -> &'a Value; - pub fn LLVMRustAddFunctionAttributes<'a>( + pub(crate) fn LLVMRustAddFunctionAttributes<'a>( Fn: &'a Value, index: c_uint, Attrs: *const &'a Attribute, @@ -1748,19 +1856,19 @@ unsafe extern "C" { ); // Operations on call sites - pub fn LLVMRustAddCallSiteAttributes<'a>( + pub(crate) fn LLVMRustAddCallSiteAttributes<'a>( Instr: &'a Value, index: c_uint, Attrs: *const &'a Attribute, AttrsLen: size_t, ); - pub fn LLVMRustSetFastMath(Instr: &Value); - pub fn LLVMRustSetAlgebraicMath(Instr: &Value); - pub fn LLVMRustSetAllowReassoc(Instr: &Value); + pub(crate) fn LLVMRustSetFastMath(Instr: &Value); + pub(crate) fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub(crate) fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustBuildMemCpy<'a>( + pub(crate) fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, @@ -1769,7 +1877,7 @@ unsafe extern "C" { Size: &'a Value, IsVolatile: bool, ) -> &'a Value; - pub fn LLVMRustBuildMemMove<'a>( + pub(crate) fn LLVMRustBuildMemMove<'a>( B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, @@ -1778,7 +1886,7 @@ unsafe extern "C" { Size: &'a Value, IsVolatile: bool, ) -> &'a Value; - pub fn LLVMRustBuildMemSet<'a>( + pub(crate) fn LLVMRustBuildMemSet<'a>( B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, @@ -1787,47 +1895,55 @@ unsafe extern "C" { IsVolatile: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFAdd<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFAdd<'a>( B: &Builder<'a>, Acc: &'a Value, Src: &'a Value, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMul<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFMul<'a>( B: &Builder<'a>, Acc: &'a Value, Src: &'a Value, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMin<'a>( + pub(crate) fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMin<'a>( B: &Builder<'a>, Src: &'a Value, IsSigned: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMax<'a>( + pub(crate) fn LLVMRustBuildVectorReduceMax<'a>( B: &Builder<'a>, Src: &'a Value, IsSigned: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMin<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFMin<'a>( B: &Builder<'a>, Src: &'a Value, IsNaN: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMax<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFMax<'a>( B: &Builder<'a>, Src: &'a Value, IsNaN: bool, ) -> &'a Value; - pub fn LLVMRustBuildMinNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; - pub fn LLVMRustBuildMaxNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildMinNum<'a>( + B: &Builder<'a>, + LHS: &'a Value, + LHS: &'a Value, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildMaxNum<'a>( + B: &Builder<'a>, + LHS: &'a Value, + LHS: &'a Value, + ) -> &'a Value; // Atomic Operations - pub fn LLVMRustBuildAtomicLoad<'a>( + pub(crate) fn LLVMRustBuildAtomicLoad<'a>( B: &Builder<'a>, ElementType: &'a Type, PointerVal: &'a Value, @@ -1835,21 +1951,21 @@ unsafe extern "C" { Order: AtomicOrdering, ) -> &'a Value; - pub fn LLVMRustBuildAtomicStore<'a>( + pub(crate) fn LLVMRustBuildAtomicStore<'a>( B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value, Order: AtomicOrdering, ) -> &'a Value; - pub fn LLVMRustTimeTraceProfilerInitialize(); + pub(crate) fn LLVMRustTimeTraceProfilerInitialize(); - pub fn LLVMRustTimeTraceProfilerFinishThread(); + pub(crate) fn LLVMRustTimeTraceProfilerFinishThread(); - pub fn LLVMRustTimeTraceProfilerFinish(FileName: *const c_char); + pub(crate) fn LLVMRustTimeTraceProfilerFinish(FileName: *const c_char); /// Returns a string describing the last error caused by an LLVMRust* call. - pub fn LLVMRustGetLastError() -> *const c_char; + pub(crate) fn LLVMRustGetLastError() -> *const c_char; /// Prints the timing information collected by `-Ztime-llvm-passes`. pub(crate) fn LLVMRustPrintPassTimings(OutStr: &RustString); @@ -1858,7 +1974,7 @@ unsafe extern "C" { pub(crate) fn LLVMRustPrintStatistics(OutStr: &RustString); /// Prepares inline assembly. - pub fn LLVMRustInlineAsm( + pub(crate) fn LLVMRustInlineAsm( Ty: &Type, AsmString: *const c_char, AsmStringLen: size_t, @@ -1869,7 +1985,7 @@ unsafe extern "C" { Dialect: AsmDialect, CanThrow: Bool, ) -> &Value; - pub fn LLVMRustInlineAsmVerify( + pub(crate) fn LLVMRustInlineAsmVerify( Ty: &Type, Constraints: *const c_char, ConstraintsLen: size_t, @@ -1913,16 +2029,16 @@ unsafe extern "C" { pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; - pub fn LLVMRustDebugMetadataVersion() -> u32; - pub fn LLVMRustVersionMajor() -> u32; - pub fn LLVMRustVersionMinor() -> u32; - pub fn LLVMRustVersionPatch() -> u32; + pub(crate) fn LLVMRustDebugMetadataVersion() -> u32; + pub(crate) fn LLVMRustVersionMajor() -> u32; + pub(crate) fn LLVMRustVersionMinor() -> u32; + pub(crate) fn LLVMRustVersionPatch() -> u32; /// Add LLVM module flags. /// /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What /// "compatible" means depends on the merge behaviors involved. - pub fn LLVMRustAddModuleFlagU32( + pub(crate) fn LLVMRustAddModuleFlagU32( M: &Module, MergeBehavior: ModuleFlagMergeBehavior, Name: *const c_char, @@ -1930,7 +2046,7 @@ unsafe extern "C" { Value: u32, ); - pub fn LLVMRustAddModuleFlagString( + pub(crate) fn LLVMRustAddModuleFlagString( M: &Module, MergeBehavior: ModuleFlagMergeBehavior, Name: *const c_char, @@ -1939,13 +2055,7 @@ unsafe extern "C" { ValueLen: size_t, ); - pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; - - pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); - - pub fn LLVMRustDIBuilderFinalize(Builder: &DIBuilder<'_>); - - pub fn LLVMRustDIBuilderCreateCompileUnit<'a>( + pub(crate) fn LLVMRustDIBuilderCreateCompileUnit<'a>( Builder: &DIBuilder<'a>, Lang: c_uint, File: &'a DIFile, @@ -1962,7 +2072,7 @@ unsafe extern "C" { DebugNameTableKind: DebugNameTableKind, ) -> &'a DIDescriptor; - pub fn LLVMRustDIBuilderCreateFile<'a>( + pub(crate) fn LLVMRustDIBuilderCreateFile<'a>( Builder: &DIBuilder<'a>, Filename: *const c_char, FilenameLen: size_t, @@ -1975,12 +2085,12 @@ unsafe extern "C" { SourceLen: size_t, ) -> &'a DIFile; - pub fn LLVMRustDIBuilderCreateSubroutineType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>( Builder: &DIBuilder<'a>, ParameterTypes: &'a DIArray, ) -> &'a DICompositeType; - pub fn LLVMRustDIBuilderCreateFunction<'a>( + pub(crate) fn LLVMRustDIBuilderCreateFunction<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -1998,7 +2108,7 @@ unsafe extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; - pub fn LLVMRustDIBuilderCreateMethod<'a>( + pub(crate) fn LLVMRustDIBuilderCreateMethod<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2013,7 +2123,7 @@ unsafe extern "C" { TParam: &'a DIArray, ) -> &'a DISubprogram; - pub fn LLVMRustDIBuilderCreateBasicType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, NameLen: size_t, @@ -2021,7 +2131,7 @@ unsafe extern "C" { Encoding: c_uint, ) -> &'a DIBasicType; - pub fn LLVMRustDIBuilderCreateTypedef<'a>( + pub(crate) fn LLVMRustDIBuilderCreateTypedef<'a>( Builder: &DIBuilder<'a>, Type: &'a DIBasicType, Name: *const c_char, @@ -2031,7 +2141,7 @@ unsafe extern "C" { Scope: Option<&'a DIScope>, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreatePointerType<'a>( + pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>( Builder: &DIBuilder<'a>, PointeeTy: &'a DIType, SizeInBits: u64, @@ -2041,7 +2151,7 @@ unsafe extern "C" { NameLen: size_t, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateStructType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateStructType<'a>( Builder: &DIBuilder<'a>, Scope: Option<&'a DIDescriptor>, Name: *const c_char, @@ -2059,7 +2169,7 @@ unsafe extern "C" { UniqueIdLen: size_t, ) -> &'a DICompositeType; - pub fn LLVMRustDIBuilderCreateMemberType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2073,7 +2183,7 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateVariantMemberType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, Name: *const c_char, @@ -2088,7 +2198,7 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIType; - pub fn LLVMRustDIBuilderCreateStaticMemberType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateStaticMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2101,27 +2211,13 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateQualifiedType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateQualifiedType<'a>( Builder: &DIBuilder<'a>, Tag: c_uint, Type: &'a DIType, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - File: &'a DIFile, - Line: c_uint, - Col: c_uint, - ) -> &'a DILexicalBlock; - - pub fn LLVMRustDIBuilderCreateLexicalBlockFile<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - File: &'a DIFile, - ) -> &'a DILexicalBlock; - - pub fn LLVMRustDIBuilderCreateStaticVariable<'a>( + pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( Builder: &DIBuilder<'a>, Context: Option<&'a DIScope>, Name: *const c_char, @@ -2137,7 +2233,7 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub fn LLVMRustDIBuilderCreateVariable<'a>( + pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( Builder: &DIBuilder<'a>, Tag: c_uint, Scope: &'a DIDescriptor, @@ -2152,7 +2248,7 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIVariable; - pub fn LLVMRustDIBuilderCreateArrayType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateArrayType<'a>( Builder: &DIBuilder<'a>, Size: u64, AlignInBits: u32, @@ -2160,19 +2256,19 @@ unsafe extern "C" { Subscripts: &'a DIArray, ) -> &'a DIType; - pub fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( + pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( Builder: &DIBuilder<'a>, Lo: i64, Count: i64, ) -> &'a DISubrange; - pub fn LLVMRustDIBuilderGetOrCreateArray<'a>( + pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( Builder: &DIBuilder<'a>, Ptr: *const Option<&'a DIDescriptor>, Count: c_uint, ) -> &'a DIArray; - pub fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( + pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( Builder: &DIBuilder<'a>, Val: &'a Value, VarInfo: &'a DIVariable, @@ -2182,7 +2278,7 @@ unsafe extern "C" { InsertAtEnd: &'a BasicBlock, ); - pub fn LLVMRustDIBuilderCreateEnumerator<'a>( + pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, NameLen: size_t, @@ -2191,7 +2287,7 @@ unsafe extern "C" { IsUnsigned: bool, ) -> &'a DIEnumerator; - pub fn LLVMRustDIBuilderCreateEnumerationType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateEnumerationType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, Name: *const c_char, @@ -2205,7 +2301,7 @@ unsafe extern "C" { IsScoped: bool, ) -> &'a DIType; - pub fn LLVMRustDIBuilderCreateUnionType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateUnionType<'a>( Builder: &DIBuilder<'a>, Scope: Option<&'a DIScope>, Name: *const c_char, @@ -2221,7 +2317,7 @@ unsafe extern "C" { UniqueIdLen: size_t, ) -> &'a DIType; - pub fn LLVMRustDIBuilderCreateVariantPart<'a>( + pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, Name: *const c_char, @@ -2237,7 +2333,7 @@ unsafe extern "C" { UniqueIdLen: size_t, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( + pub(crate) fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( Builder: &DIBuilder<'a>, Scope: Option<&'a DIScope>, Name: *const c_char, @@ -2245,51 +2341,37 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DITemplateTypeParameter; - pub fn LLVMRustDIBuilderCreateNameSpace<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - ExportSymbols: bool, - ) -> &'a DINameSpace; - - pub fn LLVMRustDICompositeTypeReplaceArrays<'a>( + pub(crate) fn LLVMRustDICompositeTypeReplaceArrays<'a>( Builder: &DIBuilder<'a>, CompositeType: &'a DIType, Elements: Option<&'a DIArray>, Params: Option<&'a DIArray>, ); - pub fn LLVMRustDIBuilderCreateDebugLocation<'a>( - Line: c_uint, - Column: c_uint, - Scope: &'a DIScope, - InlinedAt: Option<&'a DILocation>, - ) -> &'a DILocation; - pub fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( + pub(crate) fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( Location: &'a DILocation, BD: c_uint, ) -> Option<&'a DILocation>; - pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); - pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); + pub(crate) fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); + pub(crate) fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); - pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; + pub(crate) fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); - pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; - pub fn LLVMRustGetTargetFeature( + pub(crate) fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; + pub(crate) fn LLVMRustGetTargetFeature( T: &TargetMachine, Index: size_t, Feature: &mut *const c_char, Desc: &mut *const c_char, ); - pub fn LLVMRustGetHostCPUName(LenOut: &mut size_t) -> *const u8; + pub(crate) fn LLVMRustGetHostCPUName(LenOut: &mut size_t) -> *const u8; // This function makes copies of pointed to data, so the data's lifetime may end after this // function returns. - pub fn LLVMRustCreateTargetMachine( + pub(crate) fn LLVMRustCreateTargetMachine( Triple: *const c_char, CPU: *const c_char, Features: *const c_char, @@ -2315,22 +2397,22 @@ unsafe extern "C" { ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; - pub fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); - pub fn LLVMRustAddLibraryInfo<'a>( + pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); + pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, DisableSimplifyLibCalls: bool, ); - pub fn LLVMRustWriteOutputFile<'a>( + pub(crate) fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, - PM: &PassManager<'a>, + PM: *mut PassManager<'a>, M: &'a Module, Output: *const c_char, DwoOutput: *const c_char, FileType: FileType, VerifyIR: bool, ) -> LLVMRustResult; - pub fn LLVMRustOptimize<'a>( + pub(crate) fn LLVMRustOptimize<'a>( M: &'a Module, TM: &'a TargetMachine, OptLevel: PassBuilderOptLevel, @@ -2346,6 +2428,7 @@ unsafe extern "C" { LoopVectorize: bool, DisableSimplifyLibCalls: bool, EmitLifetimeMarkers: bool, + RunEnzyme: bool, SanitizerOptions: Option<&SanitizerOptions>, PGOGenPath: *const c_char, PGOUsePath: *const c_char, @@ -2361,29 +2444,32 @@ unsafe extern "C" { LLVMPlugins: *const c_char, LLVMPluginsLen: size_t, ) -> LLVMRustResult; - pub fn LLVMRustPrintModule( + pub(crate) fn LLVMRustPrintModule( M: &Module, Output: *const c_char, Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t, ) -> LLVMRustResult; - pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); - pub fn LLVMRustPrintPasses(); - pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); - pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - - pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub fn LLVMRustArchiveIteratorNext<'a>( + pub(crate) fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); + pub(crate) fn LLVMRustPrintPasses(); + pub(crate) fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); + pub(crate) fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); + + pub(crate) fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; + pub(crate) fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; + pub(crate) fn LLVMRustArchiveIteratorNext<'a>( AIR: &ArchiveIterator<'a>, ) -> Option<&'a mut ArchiveChild<'a>>; - pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); + pub(crate) fn LLVMRustArchiveChildName( + ACR: &ArchiveChild<'_>, + size: &mut size_t, + ) -> *const c_char; + pub(crate) fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); + pub(crate) fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); + pub(crate) fn LLVMRustDestroyArchive(AR: &'static mut Archive); - pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); + pub(crate) fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); - pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( + pub(crate) fn LLVMRustUnpackOptimizationDiagnostic<'a>( DI: &'a DiagnosticInfo, pass_name_out: &RustString, function_out: &mut Option<&'a Value>, @@ -2393,22 +2479,22 @@ unsafe extern "C" { message_out: &RustString, ); - pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( + pub(crate) fn LLVMRustUnpackInlineAsmDiagnostic<'a>( DI: &'a DiagnosticInfo, level_out: &mut DiagnosticLevel, cookie_out: &mut u64, message_out: &mut Option<&'a Twine>, ); - pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); - pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; + pub(crate) fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); + pub(crate) fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; - pub fn LLVMRustGetSMDiagnostic<'a>( + pub(crate) fn LLVMRustGetSMDiagnostic<'a>( DI: &'a DiagnosticInfo, cookie_out: &mut u64, ) -> &'a SMDiagnostic; - pub fn LLVMRustUnpackSMDiagnostic( + pub(crate) fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, message_out: &RustString, buffer_out: &RustString, @@ -2418,7 +2504,7 @@ unsafe extern "C" { num_ranges: &mut usize, ) -> bool; - pub fn LLVMRustWriteArchive( + pub(crate) fn LLVMRustWriteArchive( Dst: *const c_char, NumMembers: size_t, Members: *const &RustArchiveMember<'_>, @@ -2426,63 +2512,63 @@ unsafe extern "C" { Kind: ArchiveKind, isEC: bool, ) -> LLVMRustResult; - pub fn LLVMRustArchiveMemberNew<'a>( + pub(crate) fn LLVMRustArchiveMemberNew<'a>( Filename: *const c_char, Name: *const c_char, Child: Option<&ArchiveChild<'a>>, ) -> &'a mut RustArchiveMember<'a>; - pub fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); + pub(crate) fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); + pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); + pub(crate) fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); - pub fn LLVMRustSetModulePICLevel(M: &Module); - pub fn LLVMRustSetModulePIELevel(M: &Module); - pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); - pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; - pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; - pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; - pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); - pub fn LLVMRustModuleCost(M: &Module) -> u64; - pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustSetModulePICLevel(M: &Module); + pub(crate) fn LLVMRustSetModulePIELevel(M: &Module); + pub(crate) fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); + pub(crate) fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; + pub(crate) fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; + pub(crate) fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; + pub(crate) fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); + pub(crate) fn LLVMRustModuleCost(M: &Module) -> u64; + pub(crate) fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); - pub fn LLVMRustThinLTOBufferCreate( + pub(crate) fn LLVMRustThinLTOBufferCreate( M: &Module, is_thin: bool, emit_summary: bool, ) -> &'static mut ThinLTOBuffer; - pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); - pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; - pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; - pub fn LLVMRustThinLTOBufferThinLinkDataPtr(M: &ThinLTOBuffer) -> *const c_char; - pub fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; - pub fn LLVMRustCreateThinLTOData( + pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); + pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; + pub(crate) fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; + pub(crate) fn LLVMRustThinLTOBufferThinLinkDataPtr(M: &ThinLTOBuffer) -> *const c_char; + pub(crate) fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; + pub(crate) fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, NumModules: size_t, PreservedSymbols: *const *const c_char, PreservedSymbolsLen: size_t, ) -> Option<&'static mut ThinLTOData>; - pub fn LLVMRustPrepareThinLTORename( + pub(crate) fn LLVMRustPrepareThinLTORename( Data: &ThinLTOData, Module: &Module, Target: &TargetMachine, ); - pub fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; - pub fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; - pub fn LLVMRustPrepareThinLTOImport( + pub(crate) fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; + pub(crate) fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; + pub(crate) fn LLVMRustPrepareThinLTOImport( Data: &ThinLTOData, Module: &Module, Target: &TargetMachine, ) -> bool; - pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); - pub fn LLVMRustParseBitcodeForLTO( + pub(crate) fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); + pub(crate) fn LLVMRustParseBitcodeForLTO( Context: &Context, Data: *const u8, len: usize, Identifier: *const c_char, ) -> Option<&Module>; - pub fn LLVMRustGetSliceFromObjectDataByName( + pub(crate) fn LLVMRustGetSliceFromObjectDataByName( data: *const u8, len: usize, name: *const u8, @@ -2490,25 +2576,27 @@ unsafe extern "C" { out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; - pub fn LLVMRustLinkerAdd( + pub(crate) fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; + pub(crate) fn LLVMRustLinkerAdd( linker: &Linker<'_>, bytecode: *const c_char, bytecode_len: usize, ) -> bool; - pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); - pub fn LLVMRustComputeLTOCacheKey( + pub(crate) fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); + pub(crate) fn LLVMRustComputeLTOCacheKey( key_out: &RustString, mod_id: *const c_char, data: &ThinLTOData, ); - pub fn LLVMRustContextGetDiagnosticHandler(Context: &Context) -> Option<&DiagnosticHandler>; - pub fn LLVMRustContextSetDiagnosticHandler( + pub(crate) fn LLVMRustContextGetDiagnosticHandler( + Context: &Context, + ) -> Option<&DiagnosticHandler>; + pub(crate) fn LLVMRustContextSetDiagnosticHandler( context: &Context, diagnostic_handler: Option<&DiagnosticHandler>, ); - pub fn LLVMRustContextConfigureDiagnosticHandler( + pub(crate) fn LLVMRustContextConfigureDiagnosticHandler( context: &Context, diagnostic_handler_callback: DiagnosticHandlerTy, diagnostic_handler_context: *mut c_void, @@ -2519,17 +2607,15 @@ unsafe extern "C" { pgo_available: bool, ); - pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); - - pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; + pub(crate) fn LLVMRustGetMangledName(V: &Value, out: &RustString); - pub fn LLVMRustIsBitcode(ptr: *const u8, len: usize) -> bool; + pub(crate) fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; - pub fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; + pub(crate) fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; - pub fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; + pub(crate) fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; - pub fn LLVMRustGetSymbols( + pub(crate) fn LLVMRustGetSymbols( buf_ptr: *const u8, buf_len: usize, state: *mut c_void, @@ -2537,10 +2623,10 @@ unsafe extern "C" { error_callback: GetSymbolsErrorCallback, ) -> *mut c_void; - pub fn LLVMRustIs64BitSymbolicFile(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIs64BitSymbolicFile(buf_ptr: *const u8, buf_len: usize) -> bool; - pub fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; - pub fn LLVMRustSetNoSanitizeAddress(Global: &Value); - pub fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); + pub(crate) fn LLVMRustSetNoSanitizeAddress(Global: &Value); + pub(crate) fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 2592a7df95c..efc9cf2ef69 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -10,13 +10,9 @@ use libc::c_uint; use rustc_abi::{Align, Size, WrappingRange}; use rustc_llvm::RustString; -pub use self::AtomicRmwBinOp::*; pub use self::CallConv::*; pub use self::CodeGenOptSize::*; -pub use self::IntPredicate::*; -pub use self::Linkage::*; pub use self::MetadataType::*; -pub use self::RealPredicate::*; pub use self::ffi::*; use crate::common::AsCCharPtr; @@ -28,7 +24,7 @@ mod ffi; pub use self::enzyme_ffi::*; impl LLVMRustResult { - pub fn into_result(self) -> Result<(), ()> { + pub(crate) fn into_result(self) -> Result<(), ()> { match self { LLVMRustResult::Success => Ok(()), LLVMRustResult::Failure => Err(()), @@ -36,13 +32,17 @@ impl LLVMRustResult { } } -pub fn AddFunctionAttributes<'ll>(llfn: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute]) { +pub(crate) fn AddFunctionAttributes<'ll>( + llfn: &'ll Value, + idx: AttributePlace, + attrs: &[&'ll Attribute], +) { unsafe { LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len()); } } -pub fn AddCallSiteAttributes<'ll>( +pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute], @@ -52,7 +52,11 @@ pub fn AddCallSiteAttributes<'ll>( } } -pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) -> &'ll Attribute { +pub(crate) fn CreateAttrStringValue<'ll>( + llcx: &'ll Context, + attr: &str, + value: &str, +) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, @@ -64,7 +68,7 @@ pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) - } } -pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { +pub(crate) fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, @@ -76,39 +80,39 @@ pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { } } -pub fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { +pub(crate) fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) } } -pub fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { +pub(crate) fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) } } -pub fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { +pub(crate) fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) } } -pub fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { +pub(crate) fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { unsafe { LLVMRustCreateByValAttr(llcx, ty) } } -pub fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { +pub(crate) fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } } -pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { +pub(crate) fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } } -pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { +pub(crate) fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } } -pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { +pub(crate) fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } } -pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { +pub(crate) fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { let lower = range.start; let upper = range.end.wrapping_add(1); let lower_words = [lower as u64, (lower >> 64) as u64]; @@ -131,7 +135,7 @@ pub enum AttributePlace { } impl AttributePlace { - pub fn as_uint(self) -> c_uint { + pub(crate) fn as_uint(self) -> c_uint { match self { AttributePlace::ReturnValue => 0, AttributePlace::Argument(i) => 1 + i, @@ -163,12 +167,12 @@ impl FromStr for ArchiveKind { } } -pub fn SetInstructionCallConv(instr: &Value, cc: CallConv) { +pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); } } -pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { +pub(crate) fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { unsafe { LLVMSetFunctionCallConv(fn_, cc as c_uint); } @@ -180,20 +184,20 @@ pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { // value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the // function. // For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 -pub fn SetUniqueComdat(llmod: &Module, val: &Value) { +pub(crate) fn SetUniqueComdat(llmod: &Module, val: &Value) { let name_buf = get_value_name(val).to_vec(); let name = CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap(); set_comdat(llmod, val, &name); } -pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { +pub(crate) fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { unsafe { LLVMSetUnnamedAddress(global, unnamed); } } -pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { +pub(crate) fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { unsafe { LLVMSetThreadLocalMode(global, mode); } @@ -201,61 +205,65 @@ pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { impl AttributeKind { /// Create an LLVM Attribute with no associated value. - pub fn create_attr(self, llcx: &Context) -> &Attribute { + pub(crate) fn create_attr(self, llcx: &Context) -> &Attribute { unsafe { LLVMRustCreateAttrNoValue(llcx, self) } } } impl MemoryEffects { /// Create an LLVM Attribute with these memory effects. - pub fn create_attr(self, llcx: &Context) -> &Attribute { + pub(crate) fn create_attr(self, llcx: &Context) -> &Attribute { unsafe { LLVMRustCreateMemoryEffectsAttr(llcx, self) } } } -pub fn set_section(llglobal: &Value, section_name: &CStr) { +pub(crate) fn set_section(llglobal: &Value, section_name: &CStr) { unsafe { LLVMSetSection(llglobal, section_name.as_ptr()); } } -pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { +pub(crate) fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } } -pub fn set_initializer(llglobal: &Value, constant_val: &Value) { +pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) { unsafe { LLVMSetInitializer(llglobal, constant_val); } } -pub fn set_global_constant(llglobal: &Value, is_constant: bool) { +pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) { unsafe { LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); } } -pub fn get_linkage(llglobal: &Value) -> Linkage { +pub(crate) fn get_linkage(llglobal: &Value) -> Linkage { unsafe { LLVMGetLinkage(llglobal) }.to_rust() } -pub fn set_linkage(llglobal: &Value, linkage: Linkage) { +pub(crate) fn set_linkage(llglobal: &Value, linkage: Linkage) { unsafe { LLVMSetLinkage(llglobal, linkage); } } -pub fn get_visibility(llglobal: &Value) -> Visibility { +pub(crate) fn is_declaration(llglobal: &Value) -> bool { + unsafe { LLVMIsDeclaration(llglobal) == ffi::True } +} + +pub(crate) fn get_visibility(llglobal: &Value) -> Visibility { unsafe { LLVMGetVisibility(llglobal) }.to_rust() } -pub fn set_visibility(llglobal: &Value, visibility: Visibility) { +pub(crate) fn set_visibility(llglobal: &Value, visibility: Visibility) { unsafe { LLVMSetVisibility(llglobal, visibility); } } -pub fn set_alignment(llglobal: &Value, align: Align) { +pub(crate) fn set_alignment(llglobal: &Value, align: Align) { unsafe { ffi::LLVMSetAlignment(llglobal, align.bytes() as c_uint); } @@ -265,7 +273,7 @@ pub fn set_alignment(llglobal: &Value, align: Align) { /// /// Inserts the comdat into `llmod` if it does not exist. /// It is an error to call this if the target does not support comdat. -pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { +pub(crate) fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { unsafe { let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr()); LLVMSetComdat(llglobal, comdat); @@ -273,7 +281,7 @@ pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { } /// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. -pub fn get_param(llfn: &Value, index: c_uint) -> &Value { +pub(crate) fn get_param(llfn: &Value, index: c_uint) -> &Value { unsafe { assert!( index < LLVMCountParams(llfn), @@ -286,7 +294,7 @@ pub fn get_param(llfn: &Value, index: c_uint) -> &Value { } /// Safe wrapper for `LLVMGetValueName2` into a byte slice -pub fn get_value_name(value: &Value) -> &[u8] { +pub(crate) fn get_value_name(value: &Value) -> &[u8] { unsafe { let mut len = 0; let data = LLVMGetValueName2(value, &mut len); @@ -295,28 +303,28 @@ pub fn get_value_name(value: &Value) -> &[u8] { } /// Safe wrapper for `LLVMSetValueName2` from a byte slice -pub fn set_value_name(value: &Value, name: &[u8]) { +pub(crate) fn set_value_name(value: &Value, name: &[u8]) { unsafe { let data = name.as_c_char_ptr(); LLVMSetValueName2(value, data, name.len()); } } -pub fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> { +pub(crate) fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> { String::from_utf8(RustString::build_byte_buffer(f)) } -pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> { +pub(crate) fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> { RustString::build_byte_buffer(f) } -pub fn twine_to_string(tr: &Twine) -> String { +pub(crate) fn twine_to_string(tr: &Twine) -> String { unsafe { build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM") } } -pub fn last_error() -> Option<String> { +pub(crate) fn last_error() -> Option<String> { unsafe { let cstr = LLVMRustGetLastError(); if cstr.is_null() { diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 53611c746a7..b4b5d6a5b19 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -271,6 +271,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")), // Filter out features that are not supported by the current LLVM version ("aarch64", "fpmr") if get_version().0 != 18 => None, + ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single // feature called `fast-unaligned-access`. In LLVM 19, it was split back out. ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { @@ -303,7 +304,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea /// Must express features in the way Rust understands them. /// /// We do not have to worry about RUSTC_SPECIFIC_FEATURES here, those are handled outside codegen. -pub fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { +pub(crate) fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { let mut features: FxHashSet<Symbol> = Default::default(); // Add base features for the target. diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 33789c6261f..70edee21bd6 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -71,10 +71,7 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { // compiler-rt, then we want to implicitly compile everything with hidden // visibility as we're going to link this object all over the place but // don't want the symbols to get exported. - if linkage != Linkage::Internal - && linkage != Linkage::Private - && self.tcx.is_compiler_builtins(LOCAL_CRATE) - { + if linkage != Linkage::Internal && self.tcx.is_compiler_builtins(LOCAL_CRATE) { llvm::set_visibility(lldecl, llvm::Visibility::Hidden); } else { llvm::set_visibility(lldecl, base::visibility_to_llvm(visibility)); diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index c56ad886120..d61ce417562 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -1,14 +1,14 @@ use std::{fmt, ptr}; use libc::{c_char, c_uint}; -use rustc_abi::{AddressSpace, Align, Integer, Size}; +use rustc_abi::{AddressSpace, Align, Integer, Reg, Size}; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_middle::bug; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty}; -use rustc_target::callconv::{CastTarget, FnAbi, Reg}; +use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::{CodegenCx, SimpleCx}; @@ -237,11 +237,11 @@ impl<'ll, 'tcx> BaseTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { impl Type { /// Creates an integer type with the given number of bits, e.g., i24 - pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { + pub(crate) fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } } - pub fn ptr_llcx(llcx: &llvm::Context) -> &Type { + pub(crate) fn ptr_llcx(llcx: &llvm::Context) -> &Type { unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::DATA.0) } } } |