diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
25 files changed, 393 insertions, 236 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index cd55a61cbaf..dca9c1f04d3 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -511,7 +511,12 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value) { - // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite. + if self.ret.layout.abi.is_uninhabited() { + llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite); + } + if !self.can_unwind { + llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite); + } let mut i = 0; let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { @@ -521,7 +526,7 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { }; match self.ret.mode { PassMode::Direct(ref attrs) => { - attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, &bx.cx, callsite); + attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); @@ -536,16 +541,13 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } _ => {} } - if let abi::Abi::Scalar(ref scalar) = self.ret.layout.abi { + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected // by the LLVM verifier. if let Int(..) = scalar.value { - if !scalar.is_bool() { - let range = scalar.valid_range_exclusive(bx); - if range.start != range.end { - bx.range_metadata(callsite, range); - } + if !scalar.is_bool() && !scalar.is_always_valid(bx) { + bx.range_metadata(callsite, scalar.valid_range); } } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index e0d312727a5..341a8882416 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -105,7 +105,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { let r = r.unwrap(); // Again, based on how many outputs we have - let outputs = ia.outputs.iter().zip(&outputs).filter(|&(ref o, _)| !o.is_indirect); + let outputs = ia.outputs.iter().zip(&outputs).filter(|&(o, _)| !o.is_indirect); for (i, (_, &place)) in outputs.enumerate() { let v = if num_outputs == 1 { r } else { self.extract_value(r, i as u64) }; OperandValue::Immediate(v).store(self, place); @@ -331,7 +331,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { let output_type = match &output_types[..] { [] => self.type_void(), [ty] => ty, - tys => self.type_struct(&tys, false), + tys => self.type_struct(tys, false), }; let dialect = match asm_arch { InlineAsmArch::X86 | InlineAsmArch::X86_64 @@ -792,7 +792,7 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll /// Helper function to get the LLVM type for a Scalar. Pointers are returned as /// the equivalent integer type. -fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: &Scalar) -> &'ll Type { +fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: Scalar) -> &'ll Type { match scalar.value { Primitive::Int(Integer::I8, _) => cx.type_i8(), Primitive::Int(Integer::I16, _) => cx.type_i16(), @@ -812,7 +812,7 @@ fn llvm_fixup_input( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value { - match (reg, &layout.abi) { + match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.value { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); @@ -835,7 +835,7 @@ fn llvm_fixup_input( Abi::Vector { element, count }, ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); - let vec_ty = bx.cx.type_vector(elem_ty, *count); + let vec_ty = bx.cx.type_vector(elem_ty, count); let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } @@ -890,7 +890,7 @@ fn llvm_fixup_output( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value { - match (reg, &layout.abi) { + match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.value { bx.extract_element(value, bx.const_i32(0)) @@ -910,8 +910,8 @@ fn llvm_fixup_output( Abi::Vector { element, count }, ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); - let vec_ty = bx.cx.type_vector(elem_ty, *count * 2); - let indices: Vec<_> = (0..*count).map(|x| bx.const_i32(x as i32)).collect(); + let vec_ty = bx.cx.type_vector(elem_ty, count * 2); + let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) @@ -965,7 +965,7 @@ fn llvm_fixup_output_type( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Type { - match (reg, &layout.abi) { + match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.value { cx.type_vector(cx.type_i8(), 8) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 56b93f83466..51c70f0868f 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -305,9 +305,12 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: let mut function_features = codegen_fn_attrs .target_features .iter() - .map(|f| { + .flat_map(|f| { let feature = &f.as_str(); - format!("+{}", llvm_util::to_llvm_feature(cx.tcx.sess, feature)) + llvm_util::to_llvm_feature(cx.tcx.sess, feature) + .into_iter() + .map(|f| format!("+{}", f)) + .collect::<Vec<String>>() }) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index f612785e5a4..e2b33509b40 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -109,7 +109,7 @@ fn prepare_lto( .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); } - let archive = ArchiveRO::open(&path).expect("wanted an rlib"); + let archive = ArchiveRO::open(path).expect("wanted an rlib"); let obj_files = archive .iter() .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) @@ -316,14 +316,28 @@ fn fat_lto( .generic_activity_with_arg("LLVM_fat_lto_link_module", format!("{:?}", name)); info!("linking {:?}", name); let data = bc_decoded.data(); - linker.add(&data).map_err(|()| { + linker.add(data).map_err(|()| { let msg = format!("failed to load bc of {:?}", name); - write::llvm_err(&diag_handler, &msg) + write::llvm_err(diag_handler, &msg) })?; serialized_bitcode.push(bc_decoded); } drop(linker); - save_temp_bitcode(&cgcx, &module, "lto.input"); + save_temp_bitcode(cgcx, &module, "lto.input"); + + // Fat LTO also suffers from the invalid DWARF issue similar to Thin LTO. + // Here we rewrite all `DICompileUnit` pointers if there is only one `DICompileUnit`. + // This only works around the problem when codegen-units = 1. + // Refer to the comments in the `optimize_thin_module` function for more details. + let mut cu1 = ptr::null_mut(); + let mut cu2 = ptr::null_mut(); + unsafe { llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2) }; + if !cu2.is_null() { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_fat_lto_patch_debuginfo", &*module.name); + unsafe { llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1) }; + save_temp_bitcode(cgcx, &module, "fat-lto-after-patch"); + } // Internalize everything below threshold to help strip out more modules and such. unsafe { @@ -333,14 +347,14 @@ fn fat_lto( ptr as *const *const libc::c_char, symbols_below_threshold.len() as libc::size_t, ); - save_temp_bitcode(&cgcx, &module, "lto.after-restriction"); + save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } if cgcx.no_landing_pads { unsafe { llvm::LLVMRustMarkAllFunctionsNounwind(llmod); } - save_temp_bitcode(&cgcx, &module, "lto.after-nounwind"); + save_temp_bitcode(cgcx, &module, "lto.after-nounwind"); } } @@ -484,7 +498,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len() as u32, ) - .ok_or_else(|| write::llvm_err(&diag_handler, "failed to prepare thin LTO context"))?; + .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?; let data = ThinData(data); @@ -558,7 +572,7 @@ fn thin_lto( if let Some(path) = key_map_path { if let Err(err) = curr_key_map.save_to_file(&path) { let msg = format!("Error while writing ThinLTO key data: {}", err); - return Err(write::llvm_err(&diag_handler, &msg)); + return Err(write::llvm_err(diag_handler, &msg)); } } @@ -730,8 +744,7 @@ pub unsafe fn optimize_thin_module( // crates but for locally codegened modules we may be able to reuse // that LLVM Context and Module. let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = - parse_module(llcx, &module_name, thin_module.data(), &diag_handler)? as *const _; + let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; let module = ModuleCodegen { module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, name: thin_module.name().to_string(), @@ -740,7 +753,7 @@ pub unsafe fn optimize_thin_module( { let target = &*module.module_llvm.tm; let llmod = module.module_llvm.llmod(); - save_temp_bitcode(&cgcx, &module, "thin-lto-input"); + save_temp_bitcode(cgcx, &module, "thin-lto-input"); // Before we do much else find the "main" `DICompileUnit` that we'll be // using below. If we find more than one though then rustc has changed @@ -748,7 +761,7 @@ pub unsafe fn optimize_thin_module( // an error. let mut cu1 = ptr::null_mut(); let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); + llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); if !cu2.is_null() { let msg = "multiple source DICompileUnits found"; return Err(write::llvm_err(&diag_handler, msg)); @@ -761,7 +774,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_remove_landing_pads", thin_module.name()); llvm::LLVMRustMarkAllFunctionsNounwind(llmod); - save_temp_bitcode(&cgcx, &module, "thin-lto-after-nounwind"); + save_temp_bitcode(cgcx, &module, "thin-lto-after-nounwind"); } // Up next comes the per-module local analyses that we do for Thin LTO. @@ -847,7 +860,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); + llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1); save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); } @@ -933,7 +946,7 @@ pub fn parse_module<'a>( llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( || { let msg = "failed to parse bitcode for LTO module"; - write::llvm_err(&diag_handler, msg) + write::llvm_err(diag_handler, msg) }, ) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 791604a1827..ab48c56a626 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -41,7 +41,7 @@ use std::sync::Arc; pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { match llvm::last_error() { Some(err) => handler.fatal(&format!("{}: {}", msg, err)), - None => handler.fatal(&msg), + None => handler.fatal(msg), } } @@ -96,7 +96,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll None }; let config = TargetMachineFactoryConfig { split_dwarf_file }; - target_machine_factory(&tcx.sess, tcx.backend_optimization_level(()))(config) + target_machine_factory(tcx.sess, tcx.backend_optimization_level(()))(config) .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) } @@ -129,7 +129,8 @@ fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel { match relocation_model { RelocModel::Static => llvm::RelocModel::Static, - RelocModel::Pic => llvm::RelocModel::PIC, + // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute. + RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC, RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic, RelocModel::Ropi => llvm::RelocModel::ROPI, RelocModel::Rwpi => llvm::RelocModel::RWPI, @@ -370,8 +371,9 @@ fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> { } pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool { - // The new pass manager is disabled by default. - config.new_llvm_pass_manager.unwrap_or(false) + // The new pass manager is enabled by default for LLVM >= 13. + // This matches Clang, which also enables it since Clang 13. + config.new_llvm_pass_manager.unwrap_or_else(|| llvm_util::get_version() >= (13, 0, 0)) } pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( @@ -404,13 +406,15 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( None }; - let llvm_selfprofiler = if cgcx.prof.llvm_recording_enabled() { - let mut llvm_profiler = LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()); - &mut llvm_profiler as *mut _ as *mut c_void + let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { + Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) } else { - std::ptr::null_mut() + None }; + let llvm_selfprofiler = + llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); + let extra_passes = config.passes.join(","); // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. @@ -554,7 +558,7 @@ pub(crate) unsafe fn optimize( let prepare_for_thin_lto = cgcx.lto == Lto::Thin || cgcx.lto == Lto::ThinLocal || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); - with_llvm_pmb(llmod, &config, opt_level, prepare_for_thin_lto, &mut |b| { + with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { llvm::LLVMRustAddLastExtensionPasses( b, extra_passes.as_ptr(), @@ -656,9 +660,9 @@ pub(crate) fn link( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", format!("{:?}", module.name)); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); - linker.add(&buffer.data()).map_err(|()| { + linker.add(buffer.data()).map_err(|()| { let msg = format!("failed to serialize module {:?}", module.name); - llvm_err(&diag_handler, &msg) + llvm_err(diag_handler, &msg) })?; } drop(linker); diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index a6bdbd11899..3026c2fa030 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -25,9 +25,9 @@ use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::middle::exported_symbols; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::TyCtxt; @@ -64,7 +64,7 @@ pub fn write_compressed_metadata<'tcx>( let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod()); let mut compressed = rustc_metadata::METADATA_HEADER.to_vec(); - FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); + FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap(); let llmeta = common::bytes_in_context(metadata_llcx, &compressed); let llconst = common::struct_in_context(metadata_llcx, &[llmeta], false); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index da24fe08f0d..9f7b8616d78 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -15,15 +15,17 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; -use rustc_middle::ty::layout::{LayoutError, LayoutOfHelpers, TyAndLayout}; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, +}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; -use rustc_target::abi::{self, Align, Size}; +use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, Target}; use std::borrow::Cow; use std::ffi::CStr; use std::iter; -use std::ops::{Deref, Range}; +use std::ops::Deref; use std::ptr; use tracing::debug; @@ -84,7 +86,7 @@ impl ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { impl HasTargetSpec for Builder<'_, '_, 'tcx> { #[inline] fn target_spec(&self) -> &Target { - &self.cx.target_spec() + self.cx.target_spec() } } @@ -97,6 +99,20 @@ impl LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { } } +impl FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + self.cx.handle_fn_abi_err(err, span, fn_abi_request) + } +} + impl Deref for Builder<'_, 'll, 'tcx> { type Target = CodegenCx<'ll, 'tcx>; @@ -382,7 +398,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { val } } - fn to_immediate_scalar(&mut self, val: Self::Value, scalar: &abi::Scalar) -> Self::Value { + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { if scalar.is_bool() { return self.trunc(val, self.cx().type_i1()); } @@ -460,16 +476,15 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn scalar_load_metadata<'a, 'll, 'tcx>( bx: &mut Builder<'a, 'll, 'tcx>, load: &'ll Value, - scalar: &abi::Scalar, + scalar: abi::Scalar, ) { match scalar.value { abi::Int(..) => { - let range = scalar.valid_range_exclusive(bx); - if range.start != range.end { - bx.range_metadata(load, range); + if !scalar.is_always_valid(bx) { + bx.range_metadata(load, scalar.valid_range); } } - abi::Pointer if !scalar.valid_range.contains_zero() => { + abi::Pointer if !scalar.valid_range.contains(0) => { bx.nonnull_metadata(load); } _ => {} @@ -489,17 +504,17 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } let llval = const_llval.unwrap_or_else(|| { let load = self.load(place.layout.llvm_type(self), place.llval, place.align); - if let abi::Abi::Scalar(ref scalar) = place.layout.abi { + if let abi::Abi::Scalar(scalar) = place.layout.abi { scalar_load_metadata(self, load, scalar); } load }); OperandValue::Immediate(self.to_immediate(llval, place.layout)) - } else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi { + } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { let b_offset = a.value.size(self).align_to(b.value.align(self).abi); let pair_ty = place.layout.llvm_type(self); - let mut load = |i, scalar: &abi::Scalar, align| { + let mut load = |i, scalar: abi::Scalar, align| { let llptr = self.struct_gep(pair_ty, place.llval, i as u64); let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); let load = self.load(llty, llptr, align); @@ -555,7 +570,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { next_bx } - fn range_metadata(&mut self, load: &'ll Value, range: Range<u128>) { + fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { if self.sess().target.arch == "amdgpu" { // amdgpu/LLVM does something weird and thinks an i64 value is // split into a v2i32, halving the bitwidth LLVM expects, @@ -568,7 +583,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let llty = self.cx.val_ty(load); let v = [ self.cx.const_uint_big(llty, range.start), - self.cx.const_uint_big(llty, range.end), + self.cx.const_uint_big(llty, range.end.wrapping_add(1)), ]; llvm::LLVMSetMetadata( diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index bb16c90cd12..5d68d2b77d4 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -4,7 +4,7 @@ //! and methods are represented as just a fn ptr and not a full //! closure. -use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::context::CodegenCx; use crate::llvm; @@ -12,7 +12,7 @@ use crate::value::Value; use rustc_codegen_ssa::traits::*; use tracing::debug; -use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeFoldable}; /// Codegens a reference to a fn/method item, monomorphizing and @@ -42,9 +42,9 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value sym ); - let fn_abi = FnAbi::of_instance(cx, instance, &[]); + let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); - let llfn = if let Some(llfn) = cx.get_declared_value(&sym) { + let llfn = if let Some(llfn) = cx.get_declared_value(sym) { // Create a fn pointer with the new signature. let llptrty = fn_abi.ptr_to_llvm_type(cx); @@ -79,7 +79,7 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value llfn } } else { - let llfn = cx.declare_fn(&sym, &fn_abi); + let llfn = cx.declare_fn(sym, fn_abi); debug!("get_fn: not casting pointer!"); attributes::from_fn_attrs(cx, llfn, instance); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index cee582aec95..73a8d464431 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -228,7 +228,7 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } - fn scalar_to_backend(&self, cv: Scalar, layout: &abi::Scalar, llty: &'ll Type) -> &'ll Value { + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.value.size(self).bits() }; match cv { Scalar::Int(ScalarInt::ZST) => { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index e673b06f155..1afa6f02836 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -111,7 +111,7 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), &cx.tcx, ), - &Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } }, + Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } }, cx.type_i8p_ext(address_space), )); next_offset = offset + pointer_size; @@ -178,7 +178,7 @@ fn check_and_apply_linkage( }; unsafe { // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(&sym, llty2); + let g1 = cx.declare_global(sym, llty2); llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); // Declare an internal global `extern_with_linkage_foo` which @@ -188,7 +188,7 @@ fn check_and_apply_linkage( // `extern_with_linkage_foo` will instead be initialized to // zero. let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(&sym); + real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { cx.sess().span_fatal( cx.tcx.def_span(span_def_id), @@ -202,7 +202,7 @@ fn check_and_apply_linkage( } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global - cx.declare_global(&sym, llty) + cx.declare_global(sym, llty) } } @@ -234,7 +234,7 @@ impl CodegenCx<'ll, 'tcx> { _ => self.define_private_global(self.val_ty(cv)), }; llvm::LLVMSetInitializer(gv, cv); - set_global_alignment(&self, gv, align); + set_global_alignment(self, gv, align); llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); gv } @@ -279,7 +279,7 @@ impl CodegenCx<'ll, 'tcx> { g } else { - check_and_apply_linkage(&self, &fn_attrs, ty, sym, def_id) + check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) }; // Thread-local statics in some other crate need to *always* be linked @@ -369,7 +369,7 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { unsafe { let attrs = self.tcx.codegen_fn_attrs(def_id); - let (v, alloc) = match codegen_static_initializer(&self, def_id) { + let (v, alloc) = match codegen_static_initializer(self, def_id) { Ok(v) => v, // Error has already been reported Err(_) => return, @@ -417,7 +417,7 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; - set_global_alignment(&self, g, self.align_of(ty)); + set_global_alignment(self, g, self.align_of(ty)); llvm::LLVMSetInitializer(g, v); if self.should_assume_dso_local(g, true) { @@ -430,7 +430,7 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { llvm::LLVMSetGlobalConstant(g, llvm::True); } - debuginfo::create_global_var_metadata(&self, def_id, g); + debuginfo::create_global_var_metadata(self, def_id, g); if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { llvm::set_thread_local_mode(g, self.tls_model); @@ -518,7 +518,7 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { ); } } else { - base::set_link_section(g, &attrs); + base::set_link_section(g, attrs); } if attrs.flags.contains(CodegenFnAttrFlags::USED) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 2d397dc5835..257a0ac89d8 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -15,14 +15,19 @@ use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; use rustc_middle::mir::mono::CodegenUnit; -use rustc_middle::ty::layout::{HasParamEnv, LayoutError, LayoutOfHelpers, TyAndLayout}; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, + TyAndLayout, +}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::{CFGuard, CrateType, DebugInfo}; use rustc_session::Session; use rustc_span::source_map::Span; use rustc_span::symbol::Symbol; -use rustc_target::abi::{HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx}; +use rustc_target::abi::{ + call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, +}; use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; @@ -190,11 +195,14 @@ pub unsafe fn create_module( let llvm_target = SmallCStr::new(&sess.target.llvm_target); llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr()); - if sess.relocation_model() == RelocModel::Pic { + let reloc_model = sess.relocation_model(); + if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) { llvm::LLVMRustSetModulePICLevel(llmod); // PIE is potentially more effective than PIC, but can only be used in executables. // If all our outputs are executables, then we can relax PIC to PIE. - if sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) { + if reloc_model == RelocModel::Pie + || sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) + { llvm::LLVMRustSetModulePIELevel(llmod); } } @@ -355,7 +363,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let section = cstr!("llvm.metadata"); - let array = self.const_array(&self.type_ptr_to(self.type_i8()), values); + let array = self.const_array(self.type_ptr_to(self.type_i8()), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); @@ -439,7 +447,7 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn sess(&self) -> &Session { - &self.tcx.sess + self.tcx.sess } fn check_overflow(&self) -> bool { @@ -835,6 +843,12 @@ impl ty::layout::HasTyCtxt<'tcx> for CodegenCx<'ll, 'tcx> { } } +impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { + fn param_env(&self) -> ty::ParamEnv<'tcx> { + ty::ParamEnv::reveal_all() + } +} + impl LayoutOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> { type LayoutOfResult = TyAndLayout<'tcx>; @@ -848,8 +862,39 @@ impl LayoutOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> { } } -impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { - fn param_env(&self) -> ty::ParamEnv<'tcx> { - ty::ParamEnv::reveal_all() +impl FnAbiOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { + self.sess().span_fatal(span, &err.to_string()) + } else { + match fn_abi_request { + FnAbiRequest::OfFnPtr { sig, extra_args } => { + span_bug!( + span, + "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", + sig, + extra_args, + err + ); + } + FnAbiRequest::OfInstance { instance, extra_args } => { + span_bug!( + span, + "`fn_abi_of_instance({}, {:?})` failed: {}", + instance, + extra_args, + err + ); + } + } + } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index d2a2e739ff3..6830864ba04 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -73,7 +73,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); }); debug_assert!( - coverage_mapping_buffer.len() > 0, + !coverage_mapping_buffer.is_empty(), "Every `FunctionCoverage` should have at least one counter" ); @@ -311,8 +311,7 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { // for each region in it's MIR. // Convert the `HashSet` of `codegenned_def_ids` to a sortable vector, and sort them. - let mut sorted_codegenned_def_ids: Vec<DefId> = - codegenned_def_ids.iter().map(|def_id| *def_id).collect(); + let mut sorted_codegenned_def_ids: Vec<DefId> = codegenned_def_ids.iter().copied().collect(); sorted_codegenned_def_ids.sort_unstable(); let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default(); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 019bf4a09a7..ef11e2972ea 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,6 +1,6 @@ use crate::llvm; -use crate::abi::{Abi, FnAbi}; +use crate::abi::Abi; use crate::builder::Builder; use crate::common::CodegenCx; @@ -20,7 +20,7 @@ use rustc_middle::mir::coverage::{ CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, }; use rustc_middle::ty; -use rustc_middle::ty::layout::FnAbiExt; +use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::subst::InternalSubsts; use rustc_middle::ty::Instance; @@ -199,9 +199,8 @@ fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx ); let llfn = cx.declare_fn( - &tcx.symbol_name(instance).name, - &FnAbi::of_fn_ptr( - cx, + tcx.symbol_name(instance).name, + cx.fn_abi_of_fn_ptr( ty::Binder::dummy(tcx.mk_fn_sig( iter::once(tcx.mk_unit()), tcx.mk_unit(), @@ -209,7 +208,7 @@ fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx hir::Unsafety::Unsafe, Abi::Rust, )), - &[], + ty::List::empty(), ), ); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index c2725b83f50..58f8573a2ac 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -3,12 +3,11 @@ use super::utils::DIB; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; use rustc_codegen_ssa::traits::*; -use crate::abi::FnAbi; use crate::common::CodegenCx; use crate::llvm; use crate::llvm::debuginfo::{DILocation, DIScope}; use rustc_middle::mir::{Body, SourceScope}; -use rustc_middle::ty::layout::FnAbiExt; +use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; @@ -42,7 +41,7 @@ pub fn compute_mir_scopes( // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { let scope = SourceScope::new(idx); - make_mir_scope(cx, instance, &mir, fn_dbg_scope, &has_variables, debug_context, scope); + make_mir_scope(cx, instance, mir, fn_dbg_scope, &has_variables, debug_context, scope); } } @@ -94,8 +93,8 @@ fn make_mir_scope( ty::ParamEnv::reveal_all(), callee, ); - let callee_fn_abi = FnAbi::of_instance(cx, callee, &[]); - cx.dbg_scope_fn(callee, &callee_fn_abi, None) + let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); + cx.dbg_scope_fn(callee, callee_fn_abi, None) } None => unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlock( diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index c33d35cc285..ae1f83d944f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -59,10 +59,8 @@ pub fn get_or_insert_gdb_debug_scripts_section_global(cx: &CodegenCx<'ll, '_>) - } pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { - let omit_gdb_pretty_printer_section = cx - .tcx - .sess - .contains_name(&cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + let omit_gdb_pretty_printer_section = + cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); !omit_gdb_pretty_printer_section && cx.sess().opts.debuginfo != DebugInfo::None diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 9a6391443dd..f6ec5e6395f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -34,7 +34,7 @@ use rustc_middle::ty::Instance; use rustc_middle::ty::{self, AdtKind, GeneratorSubsts, ParamEnv, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::{self, DebugInfo}; -use rustc_span::symbol::{Interner, Symbol}; +use rustc_span::symbol::Symbol; use rustc_span::FileNameDisplayPreference; use rustc_span::{self, SourceFile, SourceFileHash, Span}; use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, TagEncoding}; @@ -89,8 +89,54 @@ pub const UNKNOWN_COLUMN_NUMBER: c_uint = 0; pub const NO_SCOPE_METADATA: Option<&DIScope> = None; -#[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)] -pub struct UniqueTypeId(Symbol); +mod unique_type_id { + use super::*; + use rustc_arena::DroplessArena; + + #[derive(Copy, Hash, Eq, PartialEq, Clone)] + pub(super) struct UniqueTypeId(u32); + + // The `&'static str`s in this type actually point into the arena. + // + // The `FxHashMap`+`Vec` pair could be replaced by `FxIndexSet`, but #75278 + // found that to regress performance up to 2% in some cases. This might be + // revisited after further improvements to `indexmap`. + #[derive(Default)] + pub(super) struct TypeIdInterner { + arena: DroplessArena, + names: FxHashMap<&'static str, UniqueTypeId>, + strings: Vec<&'static str>, + } + + impl TypeIdInterner { + #[inline] + pub(super) fn intern(&mut self, string: &str) -> UniqueTypeId { + if let Some(&name) = self.names.get(string) { + return name; + } + + let name = UniqueTypeId(self.strings.len() as u32); + + // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be + // UTF-8. + let string: &str = + unsafe { std::str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) }; + // It is safe to extend the arena allocation to `'static` because we only access + // these while the arena is still alive. + let string: &'static str = unsafe { &*(string as *const str) }; + self.strings.push(string); + self.names.insert(string, name); + name + } + + // Get the symbol as a string. `Symbol::as_str()` should be used in + // preference to this function. + pub(super) fn get(&self, symbol: UniqueTypeId) -> &str { + self.strings[symbol.0 as usize] + } + } +} +use unique_type_id::*; /// The `TypeMap` is where the `CrateDebugContext` holds the type metadata nodes /// created so far. The metadata nodes are indexed by `UniqueTypeId`, and, for @@ -99,7 +145,7 @@ pub struct UniqueTypeId(Symbol); #[derive(Default)] pub struct TypeMap<'ll, 'tcx> { /// The `UniqueTypeId`s created so far. - unique_id_interner: Interner, + unique_id_interner: TypeIdInterner, /// A map from `UniqueTypeId` to debuginfo metadata for that type. This is a 1:1 mapping. unique_id_to_metadata: FxHashMap<UniqueTypeId, &'ll DIType>, /// A map from types to debuginfo metadata. This is an N:1 mapping. @@ -166,8 +212,7 @@ impl TypeMap<'ll, 'tcx> { /// Gets the string representation of a `UniqueTypeId`. This method will fail if /// the ID is unknown. fn get_unique_type_id_as_string(&self, unique_type_id: UniqueTypeId) -> &str { - let UniqueTypeId(interner_key) = unique_type_id; - self.unique_id_interner.get(interner_key) + self.unique_id_interner.get(unique_type_id) } /// Gets the `UniqueTypeId` for the given type. If the `UniqueTypeId` for the given @@ -197,9 +242,9 @@ impl TypeMap<'ll, 'tcx> { let unique_type_id = hasher.finish::<Fingerprint>().to_hex(); let key = self.unique_id_interner.intern(&unique_type_id); - self.type_to_unique_id.insert(type_, UniqueTypeId(key)); + self.type_to_unique_id.insert(type_, key); - UniqueTypeId(key) + key } /// Gets the `UniqueTypeId` for an enum variant. Enum variants are not really @@ -215,7 +260,7 @@ impl TypeMap<'ll, 'tcx> { let enum_variant_type_id = format!("{}::{}", self.get_unique_type_id_as_string(enum_type_id), variant_name); let interner_key = self.unique_id_interner.intern(&enum_variant_type_id); - UniqueTypeId(interner_key) + interner_key } /// Gets the unique type ID string for an enum variant part. @@ -432,7 +477,7 @@ fn subroutine_type_metadata( let signature_metadata: Vec<_> = iter::once( // return type match signature.output().kind() { - ty::Tuple(ref tys) if tys.is_empty() => None, + ty::Tuple(tys) if tys.is_empty() => None, _ => Some(type_metadata(cx, signature.output(), span)), }, ) @@ -602,7 +647,7 @@ pub fn type_metadata(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>, usage_site_span: Sp ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { MetadataCreationResult::new(basic_type_metadata(cx, t), false) } - ty::Tuple(ref elements) if elements.is_empty() => { + ty::Tuple(elements) if elements.is_empty() => { MetadataCreationResult::new(basic_type_metadata(cx, t), false) } ty::Array(typ, _) | ty::Slice(typ) => { @@ -701,7 +746,7 @@ pub fn type_metadata(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>, usage_site_span: Sp .finalize(cx) } }, - ty::Tuple(ref elements) => { + ty::Tuple(elements) => { let tys: Vec<_> = elements.iter().map(|k| k.expect_ty()).collect(); prepare_tuple_metadata(cx, t, &tys, unique_type_id, usage_site_span, NO_SCOPE_METADATA) .finalize(cx) @@ -887,7 +932,7 @@ fn basic_type_metadata(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { let (name, encoding) = match t.kind() { ty::Never => ("!", DW_ATE_unsigned), - ty::Tuple(ref elements) if elements.is_empty() => ("()", DW_ATE_unsigned), + ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned), ty::Bool => ("bool", DW_ATE_boolean), ty::Char => ("char", DW_ATE_unsigned_char), ty::Int(int_ty) if msvc_like_names => (int_ty.msvc_basic_name(), DW_ATE_signed), @@ -1078,7 +1123,7 @@ pub fn compile_unit_metadata( let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), - path_to_mdstring(debug_context.llcontext, &gcda_path), + path_to_mdstring(debug_context.llcontext, gcda_path), cu_desc_metadata, ]; let gcov_metadata = llvm::LLVMMDNodeInContext( @@ -1656,7 +1701,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { Variants::Multiple { tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, - ref tag, + tag, ref variants, tag_field, } => { @@ -1918,17 +1963,13 @@ impl<'tcx> VariantInfo<'_, 'tcx> { } fn source_info(&self, cx: &CodegenCx<'ll, 'tcx>) -> Option<SourceInfo<'ll>> { - match self { - VariantInfo::Generator { def_id, variant_index, .. } => { - let span = cx.tcx.generator_layout(*def_id).unwrap().variant_source_info - [*variant_index] - .span; - if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - return Some(SourceInfo { file: file_metadata(cx, &loc.file), line: loc.line }); - } + if let VariantInfo::Generator { def_id, variant_index, .. } = self { + let span = + cx.tcx.generator_layout(*def_id).unwrap().variant_source_info[*variant_index].span; + if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + return Some(SourceInfo { file: file_metadata(cx, &loc.file), line: loc.line }); } - _ => {} } None } @@ -1949,11 +1990,11 @@ fn describe_enum_variant( let unique_type_id = debug_context(cx) .type_map .borrow_mut() - .get_unique_type_id_of_enum_variant(cx, layout.ty, &variant_name); + .get_unique_type_id_of_enum_variant(cx, layout.ty, variant_name); create_struct_stub( cx, layout.ty, - &variant_name, + variant_name, unique_type_id, Some(containing_scope), DIFlags::FlagZero, @@ -2082,10 +2123,8 @@ fn prepare_enum_metadata( let layout = cx.layout_of(enum_type); - if let ( - &Abi::Scalar(_), - &Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. }, - ) = (&layout.abi, &layout.variants) + if let (Abi::Scalar(_), Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. }) = + (layout.abi, &layout.variants) { return FinalMetadata(discriminant_type_metadata(tag.value)); } @@ -2093,8 +2132,8 @@ fn prepare_enum_metadata( if use_enum_fallback(cx) { let discriminant_type_metadata = match layout.variants { Variants::Single { .. } => None, - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, ref tag, .. } - | Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => { + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } + | Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { Some(discriminant_type_metadata(tag.value)) } }; @@ -2146,9 +2185,7 @@ fn prepare_enum_metadata( // A single-variant enum has no discriminant. Variants::Single { .. } => None, - Variants::Multiple { - tag_encoding: TagEncoding::Niche { .. }, ref tag, tag_field, .. - } => { + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, tag_field, .. } => { // Find the integer type of the correct size. let size = tag.value.size(cx); let align = tag.value.align(cx); @@ -2179,7 +2216,7 @@ fn prepare_enum_metadata( } } - Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, tag_field, .. } => { + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, tag_field, .. } => { let discr_type = tag.value.to_ty(cx.tcx); let (size, align) = cx.size_and_align_of(discr_type); @@ -2344,7 +2381,7 @@ fn set_members_of_composite_type( { let mut composite_types_completed = debug_context(cx).composite_types_completed.borrow_mut(); - if !composite_types_completed.insert(&composite_type_metadata) { + if !composite_types_completed.insert(composite_type_metadata) { bug!( "debuginfo::set_members_of_composite_type() - \ Already completed forward declaration re-encountered." diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index fbaf8c8bdf6..894320a7982 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -328,7 +328,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // name if necessary. let generics = self.tcx().generics_of(enclosing_fn_def_id); let substs = instance.substs.truncate_to(self.tcx(), generics); - let template_parameters = get_template_parameters(self, &generics, substs, &mut name); + let template_parameters = get_template_parameters(self, generics, substs, &mut name); let linkage_name = &mangled_name_of_instance(self, instance).name; // Omit the linkage_name if it is the same as subprogram name. @@ -559,7 +559,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { scope_metadata: &'ll DIScope, file: &rustc_span::SourceFile, ) -> &'ll DILexicalBlock { - metadata::extend_scope_to_file(&self, scope_metadata, file) + metadata::extend_scope_to_file(self, scope_metadata, file) } fn debuginfo_finalize(&self) { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 1aa52d975e9..22dc8d101c8 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -15,7 +15,7 @@ use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; -use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt, LayoutOf}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; @@ -71,7 +71,7 @@ fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<(&'ll T sym::roundf64 => "llvm.round.f64", _ => return None, }; - Some(cx.get_intrinsic(&llvm_name)) + Some(cx.get_intrinsic(llvm_name)) } impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { @@ -96,7 +96,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let arg_tys = sig.inputs(); let ret_ty = sig.output(); let name = tcx.item_name(def_id); - let name_str = &*name.as_str(); let llret_ty = self.layout_of(ret_ty).llvm_type(self); let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); @@ -133,7 +132,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } sym::va_arg => { match fn_abi.ret.layout.abi { - abi::Abi::Scalar(ref scalar) => { + abi::Abi::Scalar(scalar) => { match scalar.value { Primitive::Int(..) => { if self.cx().size_of(ret_ty).bytes() < 4 { @@ -230,9 +229,14 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { &[args[0].immediate(), y], ) } - sym::ctlz_nonzero | sym::cttz_nonzero => { + sym::ctlz_nonzero => { let y = self.const_bool(true); - let llvm_name = &format!("llvm.{}.i{}", &name_str[..4], width); + let llvm_name = &format!("llvm.ctlz.i{}", width); + self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + } + sym::cttz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.cttz.i{}", width); self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) } sym::ctpop => self.call_intrinsic( @@ -353,7 +357,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { return; } - _ if name_str.starts_with("simd_") => { + _ if name.as_str().starts_with("simd_") => { match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { Ok(llval) => llval, Err(()) => return, @@ -737,9 +741,9 @@ fn gen_fn<'ll, 'tcx>( rust_fn_sig: ty::PolyFnSig<'tcx>, codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), ) -> (&'ll Type, &'ll Value) { - let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]); + let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, &fn_abi); + let llfn = cx.declare_fn(name, fn_abi); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. @@ -843,7 +847,6 @@ fn generic_simd_intrinsic( let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); let arg_tys = sig.inputs(); - let name_str = &*name.as_str(); if name == sym::simd_select_bitmask { let in_ty = arg_tys[0]; @@ -917,13 +920,30 @@ fn generic_simd_intrinsic( )); } - if let Some(stripped) = name_str.strip_prefix("simd_shuffle") { - let n: u64 = stripped.parse().unwrap_or_else(|_| { - span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") - }); + if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { + // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer. + // If there is no suffix, use the index array length. + let n: u64 = if stripped.is_empty() { + // Make sure this is actually an array, since typeck only checks the length-suffixed + // version of this intrinsic. + match args[2].layout.ty.kind() { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { + len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { + span_bug!(span, "could not evaluate shuffle index array length") + }) + } + _ => return_error!( + "simd_shuffle index must be an array of `u32`, got `{}`", + args[2].layout.ty + ), + } + } else { + stripped.parse().unwrap_or_else(|_| { + span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") + }) + }; require_simd!(ret_ty, "return"); - let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); require!( out_len == n, @@ -1139,7 +1159,7 @@ fn generic_simd_intrinsic( _ => return_error!("unrecognized intrinsic `{}`", name), }; let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); - let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty); + let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); let c = bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); Ok(c) @@ -1773,7 +1793,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); let v = bx.call(fn_ty, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 1da14344b1d..8f4d79e7147 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -27,8 +27,8 @@ use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{ErrorReported, FatalError, Handler}; +use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; use rustc_session::Session; @@ -211,9 +211,16 @@ impl CodegenBackend for LlvmCodegenBackend { match req { PrintRequest::RelocationModels => { println!("Available relocation models:"); - for name in - &["static", "pic", "dynamic-no-pic", "ropi", "rwpi", "ropi-rwpi", "default"] - { + for name in &[ + "static", + "pic", + "pie", + "dynamic-no-pic", + "ropi", + "rwpi", + "ropi-rwpi", + "default", + ] { println!(" {}", name); } println!(); @@ -332,7 +339,7 @@ impl ModuleLlvm { unsafe { let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?; - let tm_factory_config = TargetMachineFactoryConfig::new(&cgcx, name.to_str().unwrap()); + let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap()); let tm = match (cgcx.tm_factory)(tm_factory_config) { Ok(m) => m, Err(e) => { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 3f2ed02d90d..436d906827b 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -789,7 +789,7 @@ pub mod coverageinfo { start_line, start_col, end_line, - end_col: ((1 as u32) << 31) | end_col, + end_col: (1_u32 << 31) | end_col, kind: RegionKind::GapRegion, } } @@ -2377,12 +2377,8 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustThinLTOGetDICompileUnit( - M: &Module, - CU1: &mut *mut c_void, - CU2: &mut *mut c_void, - ); - pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + pub fn LLVMRustLTOGetDICompileUnit(M: &Module, CU1: &mut *mut c_void, CU2: &mut *mut c_void); + pub fn LLVMRustLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); pub fn LLVMRustLinkerNew(M: &'a Module) -> &'a mut Linker<'a>; pub fn LLVMRustLinkerAdd( diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 3b64ec1a991..c5deb11edd0 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -166,25 +166,32 @@ pub fn time_trace_profiler_finish(file_name: &str) { // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str { +pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> Vec<&'a str> { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { - ("x86", "pclmulqdq") => "pclmul", - ("x86", "rdrand") => "rdrnd", - ("x86", "bmi1") => "bmi", - ("x86", "cmpxchg16b") => "cx16", - ("x86", "avx512vaes") => "vaes", - ("x86", "avx512gfni") => "gfni", - ("x86", "avx512vpclmulqdq") => "vpclmulqdq", - ("aarch64", "fp") => "fp-armv8", - ("aarch64", "fp16") => "fullfp16", - ("aarch64", "fhm") => "fp16fml", - ("aarch64", "rcpc2") => "rcpc-immo", - ("aarch64", "dpb") => "ccpp", - ("aarch64", "dpb2") => "ccdp", - ("aarch64", "frintts") => "fptoint", - ("aarch64", "fcma") => "complxnum", - (_, s) => s, + ("x86", "sse4.2") => { + if get_version() >= (14, 0, 0) { + vec!["sse4.2", "crc32"] + } else { + vec!["sse4.2"] + } + } + ("x86", "pclmulqdq") => vec!["pclmul"], + ("x86", "rdrand") => vec!["rdrnd"], + ("x86", "bmi1") => vec!["bmi"], + ("x86", "cmpxchg16b") => vec!["cx16"], + ("x86", "avx512vaes") => vec!["vaes"], + ("x86", "avx512gfni") => vec!["gfni"], + ("x86", "avx512vpclmulqdq") => vec!["vpclmulqdq"], + ("aarch64", "fp") => vec!["fp-armv8"], + ("aarch64", "fp16") => vec!["fullfp16"], + ("aarch64", "fhm") => vec!["fp16fml"], + ("aarch64", "rcpc2") => vec!["rcpc-immo"], + ("aarch64", "dpb") => vec!["ccpp"], + ("aarch64", "dpb2") => vec!["ccdp"], + ("aarch64", "frintts") => vec!["fptoint"], + ("aarch64", "fcma") => vec!["complxnum"], + (_, s) => vec![s], } } @@ -198,9 +205,13 @@ pub fn target_features(sess: &Session) -> Vec<Symbol> { }, ) .filter(|feature| { - let llvm_feature = to_llvm_feature(sess, feature); - let cstr = CString::new(llvm_feature).unwrap(); - unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } + for llvm_feature in to_llvm_feature(sess, feature) { + let cstr = CString::new(llvm_feature).unwrap(); + if unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return true; + } + } + false }) .map(|feature| Symbol::intern(feature)) .collect() @@ -253,12 +264,19 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { let mut rustc_target_features = supported_target_features(sess) .iter() .filter_map(|(feature, _gate)| { - let llvm_feature = to_llvm_feature(sess, *feature); - // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. - target_features.binary_search_by_key(&llvm_feature, |(f, _d)| *f).ok().map(|index| { - let (_f, desc) = target_features.remove(index); - (*feature, desc) - }) + for llvm_feature in to_llvm_feature(sess, *feature) { + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| (*f)).ok().map( + |index| { + let (_f, desc) = target_features.remove(index); + (*feature, desc) + }, + ) { + Some(v) => return Some(v), + None => {} + } + } + None }) .collect::<Vec<_>>(); rustc_target_features.extend_from_slice(&[( @@ -280,7 +298,7 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { for (feature, desc) in &target_features { println!(" {1:0$} - {2}.", max_feature_len, feature, desc); } - if target_features.len() == 0 { + if target_features.is_empty() { println!(" Target features listing is not supported by this LLVM version."); } println!("\nUse +feature to enable a feature, or -feature to disable it."); @@ -373,30 +391,30 @@ pub fn llvm_global_features(sess: &Session) -> Vec<String> { let filter = |s: &str| { if s.is_empty() { - return None; + return vec![]; } let feature = if s.starts_with('+') || s.starts_with('-') { &s[1..] } else { - return Some(s.to_string()); + return vec![s.to_string()]; }; // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return None; + return vec![]; } // ... otherwise though we run through `to_llvm_feature` feature when // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. - Some(format!("{}{}", &s[..1], to_llvm_feature(sess, feature))) + to_llvm_feature(sess, feature).iter().map(|f| format!("{}{}", &s[..1], f)).collect() }; // Features implied by an implicit or explicit `--target`. - features.extend(sess.target.features.split(',').filter_map(&filter)); + features.extend(sess.target.features.split(',').flat_map(&filter)); // -Ctarget-features - features.extend(sess.opts.cg.target_feature.split(',').filter_map(&filter)); + features.extend(sess.opts.cg.target_feature.split(',').flat_map(&filter)); features } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 8ba3e870fbb..88498cf47d8 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,4 +1,3 @@ -use crate::abi::FnAbi; use crate::attributes; use crate::base; use crate::context::CodegenCx; @@ -8,7 +7,7 @@ use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; -use rustc_middle::ty::layout::{FnAbiExt, LayoutOf}; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeFoldable}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; @@ -53,11 +52,11 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { ) { assert!(!instance.substs.needs_infer()); - let fn_abi = FnAbi::of_instance(self, instance, &[]); - let lldecl = self.declare_fn(symbol_name, &fn_abi); + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let lldecl = self.declare_fn(symbol_name, fn_abi); unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); - base::set_link_section(lldecl, &attrs); + base::set_link_section(lldecl, attrs); if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { llvm::SetUniqueComdat(self.llmod, lldecl); } @@ -144,6 +143,8 @@ impl CodegenCx<'ll, 'tcx> { return true; } - return false; + // With pie relocation model calls of functions defined in the translation + // unit can use copy relocations. + self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration } } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index c7f4287e28e..2ae0a08f192 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -248,7 +248,7 @@ impl Type { } fn ptr_to(&self, address_space: AddressSpace) -> &Type { - unsafe { llvm::LLVMPointerType(&self, address_space.0) } + unsafe { llvm::LLVMPointerType(self, address_space.0) } } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index d615d230ea0..f8c919ec2aa 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,11 +1,10 @@ -use crate::abi::FnAbi; use crate::common::*; use crate::context::TypeLowering; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; -use rustc_middle::ty::layout::{FnAbiExt, LayoutOf, TyAndLayout}; -use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; +use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty, TypeFoldable}; use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; @@ -23,7 +22,7 @@ fn uncached_llvm_type<'a, 'tcx>( ) -> &'a Type { match layout.abi { Abi::Scalar(_) => bug!("handled elsewhere"), - Abi::Vector { ref element, count } => { + Abi::Vector { element, count } => { let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO); return cx.type_vector(element, count); } @@ -44,7 +43,8 @@ fn uncached_llvm_type<'a, 'tcx>( // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str => { - let mut name = with_no_trimmed_paths(|| layout.ty.to_string()); + let mut name = + with_no_visible_paths(|| with_no_trimmed_paths(|| layout.ty.to_string())); if let (&ty::Adt(def, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { @@ -177,7 +177,7 @@ pub trait LayoutLlvmExt<'tcx> { fn scalar_llvm_type_at<'a>( &self, cx: &CodegenCx<'a, 'tcx>, - scalar: &Scalar, + scalar: Scalar, offset: Size, ) -> &'a Type; fn scalar_pair_element_llvm_type<'a>( @@ -218,7 +218,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(ref scalar) = self.abi { + if let Abi::Scalar(scalar) = self.abi { // Use a different cache for scalars because pointers to DSTs // can be either fat or thin (data pointers of fat pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { @@ -231,7 +231,9 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { ty::Adt(def, _) if def.is_box() => { cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx)) } - ty::FnPtr(sig) => cx.fn_ptr_backend_type(&FnAbi::of_fn_ptr(cx, sig, &[])), + ty::FnPtr(sig) => { + cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty())) + } _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO), }; cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); @@ -243,7 +245,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { Variants::Single { index } => Some(index), _ => None, }; - if let Some(ref llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { + if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { return llty.lltype; } @@ -268,10 +270,9 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { }; debug!("--> mapped {:#?} to llty={:?}", self, llty); - cx.type_lowering.borrow_mut().insert( - (self.ty, variant_index), - TypeLowering { lltype: llty, field_remapping: field_remapping }, - ); + cx.type_lowering + .borrow_mut() + .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); if let Some((llty, layout)) = defer { let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); @@ -286,7 +287,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(ref scalar) = self.abi { + if let Abi::Scalar(scalar) = self.abi { if scalar.is_bool() { return cx.type_i1(); } @@ -297,7 +298,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn scalar_llvm_type_at<'a>( &self, cx: &CodegenCx<'a, 'tcx>, - scalar: &Scalar, + scalar: Scalar, offset: Size, ) -> &'a Type { match scalar.value { @@ -337,7 +338,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } let (a, b) = match self.abi { - Abi::ScalarPair(ref a, ref b) => (a, b), + Abi::ScalarPair(a, b) => (a, b), _ => bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self), }; let scalar = [a, b][index]; diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index caafae6c267..591f659f11b 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -125,7 +125,7 @@ fn emit_aapcs_va_arg( // if the offset >= 0 then the value will be on the stack let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align); let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero); - bx.cond_br(use_stack, &on_stack.llbb(), &maybe_reg.llbb()); + bx.cond_br(use_stack, on_stack.llbb(), maybe_reg.llbb()); // The value at this point might be in a register, but there is a chance that // it could be on the stack so we have to update the offset and then check @@ -142,7 +142,7 @@ fn emit_aapcs_va_arg( // Check to see if we have overflowed the registers as a result of this. // If we have then we need to use the stack for this value let use_stack = maybe_reg.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); - maybe_reg.cond_br(use_stack, &on_stack.llbb(), &in_reg.llbb()); + maybe_reg.cond_br(use_stack, on_stack.llbb(), in_reg.llbb()); let top_type = bx.type_i8p(); let top = in_reg.struct_gep(va_list_ty, va_list_addr, reg_top_index); @@ -158,17 +158,17 @@ fn emit_aapcs_va_arg( let reg_type = layout.llvm_type(bx); let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); let reg_value = in_reg.load(reg_type, reg_addr, layout.align.abi); - in_reg.br(&end.llbb()); + in_reg.br(end.llbb()); // On Stack block let stack_value = emit_ptr_va_arg(&mut on_stack, list, target_ty, false, Align::from_bytes(8).unwrap(), true); - on_stack.br(&end.llbb()); + on_stack.br(end.llbb()); let val = end.phi( layout.immediate_llvm_type(bx), &[reg_value, stack_value], - &[&in_reg.llbb(), &on_stack.llbb()], + &[in_reg.llbb(), on_stack.llbb()], ); *bx = end; |