diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 81 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 12 |
8 files changed, 98 insertions, 56 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index cd55a61cbaf..1a0a3a0c340 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -511,7 +511,12 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value) { - // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite. + if self.ret.layout.abi.is_uninhabited() { + llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite); + } + if !self.can_unwind { + llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite); + } let mut i = 0; let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { @@ -536,16 +541,13 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } _ => {} } - if let abi::Abi::Scalar(ref scalar) = self.ret.layout.abi { + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected // by the LLVM verifier. if let Int(..) = scalar.value { - if !scalar.is_bool() { - let range = scalar.valid_range_exclusive(bx); - if range.start != range.end { - bx.range_metadata(callsite, range); - } + if !scalar.is_bool() && !scalar.is_always_valid(bx) { + bx.range_metadata(callsite, scalar.valid_range); } } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index e0d312727a5..9690ad8b246 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -792,7 +792,7 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll /// Helper function to get the LLVM type for a Scalar. Pointers are returned as /// the equivalent integer type. -fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: &Scalar) -> &'ll Type { +fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: Scalar) -> &'ll Type { match scalar.value { Primitive::Int(Integer::I8, _) => cx.type_i8(), Primitive::Int(Integer::I16, _) => cx.type_i16(), @@ -812,7 +812,7 @@ fn llvm_fixup_input( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value { - match (reg, &layout.abi) { + match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.value { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); @@ -835,7 +835,7 @@ fn llvm_fixup_input( Abi::Vector { element, count }, ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); - let vec_ty = bx.cx.type_vector(elem_ty, *count); + let vec_ty = bx.cx.type_vector(elem_ty, count); let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } @@ -890,7 +890,7 @@ fn llvm_fixup_output( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value { - match (reg, &layout.abi) { + match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.value { bx.extract_element(value, bx.const_i32(0)) @@ -910,8 +910,8 @@ fn llvm_fixup_output( Abi::Vector { element, count }, ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); - let vec_ty = bx.cx.type_vector(elem_ty, *count * 2); - let indices: Vec<_> = (0..*count).map(|x| bx.const_i32(x as i32)).collect(); + let vec_ty = bx.cx.type_vector(elem_ty, count * 2); + let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) @@ -965,7 +965,7 @@ fn llvm_fixup_output_type( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Type { - match (reg, &layout.abi) { + match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.value { cx.type_vector(cx.type_i8(), 8) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index da24fe08f0d..799f9a57e93 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -18,12 +18,12 @@ use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{LayoutError, LayoutOfHelpers, TyAndLayout}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; -use rustc_target::abi::{self, Align, Size}; +use rustc_target::abi::{self, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, Target}; use std::borrow::Cow; use std::ffi::CStr; use std::iter; -use std::ops::{Deref, Range}; +use std::ops::Deref; use std::ptr; use tracing::debug; @@ -382,7 +382,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { val } } - fn to_immediate_scalar(&mut self, val: Self::Value, scalar: &abi::Scalar) -> Self::Value { + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { if scalar.is_bool() { return self.trunc(val, self.cx().type_i1()); } @@ -460,16 +460,15 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn scalar_load_metadata<'a, 'll, 'tcx>( bx: &mut Builder<'a, 'll, 'tcx>, load: &'ll Value, - scalar: &abi::Scalar, + scalar: abi::Scalar, ) { match scalar.value { abi::Int(..) => { - let range = scalar.valid_range_exclusive(bx); - if range.start != range.end { - bx.range_metadata(load, range); + if !scalar.is_always_valid(bx) { + bx.range_metadata(load, scalar.valid_range); } } - abi::Pointer if !scalar.valid_range.contains_zero() => { + abi::Pointer if !scalar.valid_range.contains(0) => { bx.nonnull_metadata(load); } _ => {} @@ -489,17 +488,17 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } let llval = const_llval.unwrap_or_else(|| { let load = self.load(place.layout.llvm_type(self), place.llval, place.align); - if let abi::Abi::Scalar(ref scalar) = place.layout.abi { + if let abi::Abi::Scalar(scalar) = place.layout.abi { scalar_load_metadata(self, load, scalar); } load }); OperandValue::Immediate(self.to_immediate(llval, place.layout)) - } else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi { + } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { let b_offset = a.value.size(self).align_to(b.value.align(self).abi); let pair_ty = place.layout.llvm_type(self); - let mut load = |i, scalar: &abi::Scalar, align| { + let mut load = |i, scalar: abi::Scalar, align| { let llptr = self.struct_gep(pair_ty, place.llval, i as u64); let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); let load = self.load(llty, llptr, align); @@ -555,7 +554,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { next_bx } - fn range_metadata(&mut self, load: &'ll Value, range: Range<u128>) { + fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { if self.sess().target.arch == "amdgpu" { // amdgpu/LLVM does something weird and thinks an i64 value is // split into a v2i32, halving the bitwidth LLVM expects, @@ -568,7 +567,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let llty = self.cx.val_ty(load); let v = [ self.cx.const_uint_big(llty, range.start), - self.cx.const_uint_big(llty, range.end), + self.cx.const_uint_big(llty, range.end.wrapping_add(1)), ]; llvm::LLVMSetMetadata( diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index cee582aec95..73a8d464431 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -228,7 +228,7 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } - fn scalar_to_backend(&self, cv: Scalar, layout: &abi::Scalar, llty: &'ll Type) -> &'ll Value { + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.value.size(self).bits() }; match cv { Scalar::Int(ScalarInt::ZST) => { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index e673b06f155..ef3a90fdeca 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -111,7 +111,7 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), &cx.tcx, ), - &Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } }, + Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } }, cx.type_i8p_ext(address_space), )); next_offset = offset + pointer_size; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 9a6391443dd..9272435a330 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -34,7 +34,7 @@ use rustc_middle::ty::Instance; use rustc_middle::ty::{self, AdtKind, GeneratorSubsts, ParamEnv, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::{self, DebugInfo}; -use rustc_span::symbol::{Interner, Symbol}; +use rustc_span::symbol::Symbol; use rustc_span::FileNameDisplayPreference; use rustc_span::{self, SourceFile, SourceFileHash, Span}; use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, TagEncoding}; @@ -89,8 +89,54 @@ pub const UNKNOWN_COLUMN_NUMBER: c_uint = 0; pub const NO_SCOPE_METADATA: Option<&DIScope> = None; -#[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)] -pub struct UniqueTypeId(Symbol); +mod unique_type_id { + use super::*; + use rustc_arena::DroplessArena; + + #[derive(Copy, Hash, Eq, PartialEq, Clone)] + pub(super) struct UniqueTypeId(u32); + + // The `&'static str`s in this type actually point into the arena. + // + // The `FxHashMap`+`Vec` pair could be replaced by `FxIndexSet`, but #75278 + // found that to regress performance up to 2% in some cases. This might be + // revisited after further improvements to `indexmap`. + #[derive(Default)] + pub(super) struct TypeIdInterner { + arena: DroplessArena, + names: FxHashMap<&'static str, UniqueTypeId>, + strings: Vec<&'static str>, + } + + impl TypeIdInterner { + #[inline] + pub(super) fn intern(&mut self, string: &str) -> UniqueTypeId { + if let Some(&name) = self.names.get(string) { + return name; + } + + let name = UniqueTypeId(self.strings.len() as u32); + + // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be + // UTF-8. + let string: &str = + unsafe { std::str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) }; + // It is safe to extend the arena allocation to `'static` because we only access + // these while the arena is still alive. + let string: &'static str = unsafe { &*(string as *const str) }; + self.strings.push(string); + self.names.insert(string, name); + name + } + + // Get the symbol as a string. `Symbol::as_str()` should be used in + // preference to this function. + pub(super) fn get(&self, symbol: UniqueTypeId) -> &str { + self.strings[symbol.0 as usize] + } + } +} +use unique_type_id::*; /// The `TypeMap` is where the `CrateDebugContext` holds the type metadata nodes /// created so far. The metadata nodes are indexed by `UniqueTypeId`, and, for @@ -99,7 +145,7 @@ pub struct UniqueTypeId(Symbol); #[derive(Default)] pub struct TypeMap<'ll, 'tcx> { /// The `UniqueTypeId`s created so far. - unique_id_interner: Interner, + unique_id_interner: TypeIdInterner, /// A map from `UniqueTypeId` to debuginfo metadata for that type. This is a 1:1 mapping. unique_id_to_metadata: FxHashMap<UniqueTypeId, &'ll DIType>, /// A map from types to debuginfo metadata. This is an N:1 mapping. @@ -166,8 +212,7 @@ impl TypeMap<'ll, 'tcx> { /// Gets the string representation of a `UniqueTypeId`. This method will fail if /// the ID is unknown. fn get_unique_type_id_as_string(&self, unique_type_id: UniqueTypeId) -> &str { - let UniqueTypeId(interner_key) = unique_type_id; - self.unique_id_interner.get(interner_key) + self.unique_id_interner.get(unique_type_id) } /// Gets the `UniqueTypeId` for the given type. If the `UniqueTypeId` for the given @@ -197,9 +242,9 @@ impl TypeMap<'ll, 'tcx> { let unique_type_id = hasher.finish::<Fingerprint>().to_hex(); let key = self.unique_id_interner.intern(&unique_type_id); - self.type_to_unique_id.insert(type_, UniqueTypeId(key)); + self.type_to_unique_id.insert(type_, key); - UniqueTypeId(key) + key } /// Gets the `UniqueTypeId` for an enum variant. Enum variants are not really @@ -215,7 +260,7 @@ impl TypeMap<'ll, 'tcx> { let enum_variant_type_id = format!("{}::{}", self.get_unique_type_id_as_string(enum_type_id), variant_name); let interner_key = self.unique_id_interner.intern(&enum_variant_type_id); - UniqueTypeId(interner_key) + interner_key } /// Gets the unique type ID string for an enum variant part. @@ -1656,7 +1701,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { Variants::Multiple { tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, - ref tag, + tag, ref variants, tag_field, } => { @@ -2082,10 +2127,8 @@ fn prepare_enum_metadata( let layout = cx.layout_of(enum_type); - if let ( - &Abi::Scalar(_), - &Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. }, - ) = (&layout.abi, &layout.variants) + if let (Abi::Scalar(_), Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. }) = + (layout.abi, &layout.variants) { return FinalMetadata(discriminant_type_metadata(tag.value)); } @@ -2093,8 +2136,8 @@ fn prepare_enum_metadata( if use_enum_fallback(cx) { let discriminant_type_metadata = match layout.variants { Variants::Single { .. } => None, - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, ref tag, .. } - | Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => { + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } + | Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { Some(discriminant_type_metadata(tag.value)) } }; @@ -2146,9 +2189,7 @@ fn prepare_enum_metadata( // A single-variant enum has no discriminant. Variants::Single { .. } => None, - Variants::Multiple { - tag_encoding: TagEncoding::Niche { .. }, ref tag, tag_field, .. - } => { + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, tag_field, .. } => { // Find the integer type of the correct size. let size = tag.value.size(cx); let align = tag.value.align(cx); @@ -2179,7 +2220,7 @@ fn prepare_enum_metadata( } } - Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, tag_field, .. } => { + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, tag_field, .. } => { let discr_type = tag.value.to_ty(cx.tcx); let (size, align) = cx.size_and_align_of(discr_type); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 1aa52d975e9..1060f911a9e 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -133,7 +133,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } sym::va_arg => { match fn_abi.ret.layout.abi { - abi::Abi::Scalar(ref scalar) => { + abi::Abi::Scalar(scalar) => { match scalar.value { Primitive::Int(..) => { if self.cx().size_of(ret_ty).bytes() < 4 { diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index d615d230ea0..3e39bf3e995 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -23,7 +23,7 @@ fn uncached_llvm_type<'a, 'tcx>( ) -> &'a Type { match layout.abi { Abi::Scalar(_) => bug!("handled elsewhere"), - Abi::Vector { ref element, count } => { + Abi::Vector { element, count } => { let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO); return cx.type_vector(element, count); } @@ -177,7 +177,7 @@ pub trait LayoutLlvmExt<'tcx> { fn scalar_llvm_type_at<'a>( &self, cx: &CodegenCx<'a, 'tcx>, - scalar: &Scalar, + scalar: Scalar, offset: Size, ) -> &'a Type; fn scalar_pair_element_llvm_type<'a>( @@ -218,7 +218,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(ref scalar) = self.abi { + if let Abi::Scalar(scalar) = self.abi { // Use a different cache for scalars because pointers to DSTs // can be either fat or thin (data pointers of fat pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { @@ -286,7 +286,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(ref scalar) = self.abi { + if let Abi::Scalar(scalar) = self.abi { if scalar.is_bool() { return cx.type_i1(); } @@ -297,7 +297,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn scalar_llvm_type_at<'a>( &self, cx: &CodegenCx<'a, 'tcx>, - scalar: &Scalar, + scalar: Scalar, offset: Size, ) -> &'a Type { match scalar.value { @@ -337,7 +337,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } let (a, b) = match self.abi { - Abi::ScalarPair(ref a, ref b) => (a, b), + Abi::ScalarPair(a, b) => (a, b), _ => bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self), }; let scalar = [a, b][index]; |