diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 79 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/declare.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 |
10 files changed, 115 insertions, 31 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 28f423efc21..8c75125e009 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -2,19 +2,18 @@ use std::borrow::Borrow; use std::cmp; use libc::c_uint; -use rustc_abi as abi; -pub(crate) use rustc_abi::ExternAbi; -use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind, Size}; +use rustc_abi::{BackendRepr, HasDataLayout, Primitive, Reg, RegKind, Size}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::LayoutOf; -pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA}; use rustc_middle::{bug, ty}; use rustc_session::config; -pub(crate) use rustc_target::callconv::*; +use rustc_target::callconv::{ + ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, Conv, FnAbi, PassMode, +}; use rustc_target::spec::SanitizerSet; use smallvec::SmallVec; @@ -458,7 +457,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { match &self.ret.mode { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); - if let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr { + if let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr { apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); } } @@ -499,7 +498,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(attrs) => { let i = apply(attrs); - if let abi::BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { apply_range_attr(llvm::AttributePlace::Argument(i), scalar); } } @@ -514,9 +513,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Pair(a, b) => { let i = apply(a); let ii = apply(b); - if let abi::BackendRepr::ScalarPair(scalar_a, scalar_b) = - arg.layout.backend_repr - { + if let BackendRepr::ScalarPair(scalar_a, scalar_b) = arg.layout.backend_repr { apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a); apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b); } @@ -576,7 +573,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } if bx.cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() < (19, 0, 0) - && let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr + && let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr && matches!(scalar.primitive(), Primitive::Int(..)) // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index acae0b444c0..e10a4d63407 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -4,7 +4,7 @@ use std::{iter, ptr}; pub(crate) mod autodiff; -use libc::{c_char, c_uint}; +use libc::{c_char, c_uint, size_t}; use rustc_abi as abi; use rustc_abi::{Align, Size, WrappingRange}; use rustc_codegen_ssa::MemFlags; @@ -32,7 +32,7 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, SimpleCx}; -use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; +use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, Metadata, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -333,6 +333,50 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn switch_with_weights( + &mut self, + v: Self::Value, + else_llbb: Self::BasicBlock, + else_is_cold: bool, + cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>, + ) { + if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No { + self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest))); + return; + } + + let id_str = "branch_weights"; + let id = unsafe { + llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len()) + }; + + // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used. + // This function handles switch instructions with more than 2 targets and it needs to + // emit branch weights metadata instead of using the intrinsic. + // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic. + let cold_weight = unsafe { llvm::LLVMValueAsMetadata(self.cx.const_u32(1)) }; + let hot_weight = unsafe { llvm::LLVMValueAsMetadata(self.cx.const_u32(2000)) }; + let weight = + |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } }; + + let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2); + md.push(id); + md.push(weight(else_is_cold)); + + let switch = + unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) }; + for (on_val, dest, is_cold) in cases { + let on_val = self.const_uint_big(self.val_ty(v), on_val); + unsafe { llvm::LLVMAddCase(switch, on_val, dest) } + md.push(weight(is_cold)); + } + + unsafe { + let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t); + self.cx.set_metadata(switch, llvm::MD_prof, md_node); + } + } + fn invoke( &mut self, llty: &'ll Type, @@ -421,6 +465,37 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unchecked_umul(x, y) => LLVMBuildNUWMul, } + fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(add).is_some() { + llvm::LLVMSetNUW(add, True); + llvm::LLVMSetNSW(add, True); + } + add + } + } + fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(sub).is_some() { + llvm::LLVMSetNUW(sub, True); + llvm::LLVMSetNSW(sub, True); + } + sub + } + } + fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(mul).is_some() { + llvm::LLVMSetNUW(mul, True); + llvm::LLVMSetNSW(mul, True); + } + mul + } + } + fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { unsafe { let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 78b3a7f8541..f17d98fa242 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -7,7 +7,8 @@ use rustc_abi::{AddressSpace, HasDataLayout}; use rustc_ast::Mutability; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_hashes::Hash128; use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 7fe527a4c07..e7952bc95e7 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -194,6 +194,12 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("-i128:128", ""); } } + if llvm_version < (21, 0, 0) { + if sess.target.arch == "nvptx64" { + // LLVM 21 updated the default layout on nvptx: https://github.com/llvm/llvm-project/pull/124961 + target_data_layout = target_data_layout.replace("e-p6:32:32-i64", "e-i64"); + } + } // Ensure the data-layout values hardcoded remain the defaults. { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 54c5d445f66..4ffe551df09 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -87,7 +87,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = - attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + attr::contains_name(cx.tcx.hir_krate_attrs(), sym::omit_gdb_pretty_printer_section); // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 59c3fe635d0..98d59f5a8ae 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -11,7 +11,9 @@ use rustc_codegen_ssa::traits::*; use rustc_hir::def::{CtorKind, DefKind}; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; -use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{ + HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA, +}; use rustc_middle::ty::{ self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, }; @@ -34,12 +36,12 @@ use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::dwarf_const; use crate::debuginfo::metadata::type_map::build_type_with_children; use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind}; +use crate::llvm; use crate::llvm::debuginfo::{ DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, DebugNameTableKind, }; use crate::value::Value; -use crate::{abi, llvm}; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -211,16 +213,16 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( }; let layout = cx.layout_of(layout_type); - let addr_field = layout.field(cx, abi::WIDE_PTR_ADDR); - let extra_field = layout.field(cx, abi::WIDE_PTR_EXTRA); + let addr_field = layout.field(cx, WIDE_PTR_ADDR); + let extra_field = layout.field(cx, WIDE_PTR_EXTRA); let (addr_field_name, extra_field_name) = match wide_pointer_kind { WidePtrKind::Dyn => ("pointer", "vtable"), WidePtrKind::Slice => ("data_ptr", "length"), }; - assert_eq!(abi::WIDE_PTR_ADDR, 0); - assert_eq!(abi::WIDE_PTR_EXTRA, 1); + assert_eq!(WIDE_PTR_ADDR, 0); + assert_eq!(WIDE_PTR_EXTRA, 1); // The data pointer type is a regular, thin pointer, regardless of whether this // is a slice or a trait object. @@ -242,7 +244,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( owner, addr_field_name, (addr_field.size, addr_field.align.abi), - layout.fields.offset(abi::WIDE_PTR_ADDR), + layout.fields.offset(WIDE_PTR_ADDR), DIFlags::FlagZero, data_ptr_type_di_node, None, @@ -252,7 +254,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( owner, extra_field_name, (extra_field.size, extra_field.align.abi), - layout.fields.offset(abi::WIDE_PTR_EXTRA), + layout.fields.offset(WIDE_PTR_EXTRA), DIFlags::FlagZero, type_di_node(cx, extra_field.ty), None, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 17f2d5f4e73..10819a53b1d 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -22,6 +22,7 @@ use rustc_session::config::{self, DebugInfo}; use rustc_span::{ BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, Symbol, }; +use rustc_target::callconv::FnAbi; use rustc_target::spec::DebuginfoKind; use smallvec::SmallVec; use tracing::debug; @@ -29,7 +30,6 @@ use tracing::debug; use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, type_di_node}; use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; -use crate::abi::FnAbi; use crate::builder::Builder; use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index cebceef1b93..e79662ebc64 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -16,10 +16,11 @@ use rustc_codegen_ssa::traits::TypeMembershipCodegenMethods; use rustc_data_structures::fx::FxIndexSet; use rustc_middle::ty::{Instance, Ty}; use rustc_sanitizers::{cfi, kcfi}; +use rustc_target::callconv::FnAbi; use smallvec::SmallVec; use tracing::debug; -use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::abi::FnAbiLlvmExt; use crate::common::AsCCharPtr; use crate::context::{CodegenCx, SimpleCx}; use crate::llvm::AttributePlace::Function; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 8b976885904..7e1a9d361e6 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,7 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; -use rustc_abi::{self as abi, Align, Float, HasDataLayout, Primitive, Size}; +use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; @@ -14,10 +14,11 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; +use rustc_target::callconv::{FnAbi, PassMode}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use tracing::debug; -use crate::abi::{ExternAbi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; +use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Metadata}; @@ -257,7 +258,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::va_arg => { match fn_abi.ret.layout.backend_repr { - abi::BackendRepr::Scalar(scalar) => { + BackendRepr::Scalar(scalar) => { match scalar.primitive() { Primitive::Int(..) => { if self.cx().size_of(ret_ty).bytes() < 4 { @@ -470,7 +471,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::raw_eq => { - use abi::BackendRepr::*; + use BackendRepr::*; let tp_ty = fn_args.type_at(0); let layout = self.layout_of(tp_ty).layout; let use_integer_compare = match layout.backend_repr() { @@ -582,8 +583,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } let llret_ty = if ret_ty.is_simd() - && let abi::BackendRepr::Memory { .. } = - self.layout_of(ret_ty).layout.backend_repr + && let BackendRepr::Memory { .. } = self.layout_of(ret_ty).layout.backend_repr { let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx()); let elem_ll_ty = match elem_ty.kind() { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 618fdb09d4d..3b0187b9d37 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1430,6 +1430,8 @@ unsafe extern "C" { // Extra flags on arithmetic pub(crate) fn LLVMSetIsDisjoint(Instr: &Value, IsDisjoint: Bool); + pub(crate) fn LLVMSetNUW(ArithInst: &Value, HasNUW: Bool); + pub(crate) fn LLVMSetNSW(ArithInst: &Value, HasNSW: Bool); // Memory pub(crate) fn LLVMBuildAlloca<'a>( |