diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
22 files changed, 372 insertions, 386 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 4b07c8aef91..009e7e2487b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -146,7 +146,7 @@ impl LlvmType for CastTarget { "total size {:?} cannot be divided into units of zero size", self.rest.total ); - if self.rest.total.bytes() % self.rest.unit.size.bytes() != 0 { + if !self.rest.total.bytes().is_multiple_of(self.rest.unit.size.bytes()) { assert_eq!(self.rest.unit.kind, RegKind::Integer, "only int regs can be split"); } self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes()) diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 9dca63cfc8d..2b5090ed6db 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -11,7 +11,7 @@ use rustc_symbol_mangling::mangle_internal_symbol; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; -use crate::llvm::{self, False, True, Type}; +use crate::llvm::{self, False, True, Type, Value}; use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( @@ -73,13 +73,14 @@ pub(crate) unsafe fn codegen( ); unsafe { - // __rust_alloc_error_handler_should_panic - let name = mangle_internal_symbol(tcx, OomStrategy::SYMBOL); - let ll_g = cx.declare_global(&name, i8); - llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); - let val = tcx.sess.opts.unstable_opts.oom.should_panic(); - let llval = llvm::LLVMConstInt(i8, val as u64, False); - llvm::set_initializer(ll_g, llval); + // __rust_alloc_error_handler_should_panic_v2 + create_const_value_function( + tcx, + &cx, + &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), + &i8, + &llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, False), + ); // __rust_no_alloc_shim_is_unstable_v2 create_wrapper_function( @@ -100,6 +101,34 @@ pub(crate) unsafe fn codegen( } } +fn create_const_value_function( + tcx: TyCtxt<'_>, + cx: &SimpleCx<'_>, + name: &str, + output: &Type, + value: &Value, +) { + let ty = cx.type_func(&[], output); + let llfn = declare_simple_fn( + &cx, + name, + llvm::CallConv::CCallConv, + llvm::UnnamedAddr::Global, + llvm::Visibility::from_generic(tcx.sess.default_visibility()), + ty, + ); + + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Function, + &[llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx)], + ); + + let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; + let mut bx = SBuilder::build(&cx, llbb); + bx.ret(value); +} + fn create_wrapper_function( tcx: TyCtxt<'_>, cx: &SimpleCx<'_>, diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 9ddadcf16aa..a643a91141e 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -384,15 +384,19 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); - // Default to Intel syntax on x86 - let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) - && !options.contains(InlineAsmOptions::ATT_SYNTAX); - // Build the template string let mut template_str = String::new(); - if intel_syntax { - template_str.push_str(".intel_syntax\n"); + + // On X86 platforms there are two assembly syntaxes. Rust uses intel by default, + // but AT&T can be specified explicitly. + if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) { + if options.contains(InlineAsmOptions::ATT_SYNTAX) { + template_str.push_str(".att_syntax\n") + } else { + template_str.push_str(".intel_syntax\n") + } } + for piece in template { match *piece { InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), @@ -431,7 +435,11 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { } } } - if intel_syntax { + + // Just to play it safe, if intel was used, reset the assembly syntax to att. + if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) + && !options.contains(InlineAsmOptions::ATT_SYNTAX) + { template_str.push_str("\n.att_syntax\n"); } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index adb53e0b66c..1ea5a062254 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -370,22 +370,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( }; to_add.extend(inline_attr(cx, inline)); - // The `uwtable` attribute according to LLVM is: - // - // This attribute indicates that the ABI being targeted requires that an - // unwind table entry be produced for this function even if we can show - // that no exceptions passes by it. This is normally the case for the - // ELF x86-64 abi, but it can be disabled for some compilation units. - // - // Typically when we're compiling with `-C panic=abort` (which implies this - // `no_landing_pads` check) we don't need `uwtable` because we can't - // generate any exceptions! On Windows, however, exceptions include other - // events such as illegal instructions, segfaults, etc. This means that on - // Windows we end up still needing the `uwtable` attribute even if the `-C - // panic=abort` flag is passed. - // - // You can also find more info on why Windows always requires uwtables here: - // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 if cx.sess().must_emit_unwind_tables() { to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 0198b9f0cf0..655e1c95373 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -677,7 +677,7 @@ pub(crate) fn run_pass_manager( if attributes::has_string_attr(function, enzyme_marker) { // Sanity check: Ensure 'noinline' is present before replacing it. assert!( - !attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), + attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'" ); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 1f7a785bbe7..68279008c03 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -881,9 +881,7 @@ pub(crate) fn codegen( .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); let thin_bc = module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode"); - unsafe { - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); - } + embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); } } @@ -947,7 +945,7 @@ pub(crate) fn codegen( // binaries. So we must clone the module to produce the asm output // if we are also producing object code. let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj { - unsafe { llvm::LLVMCloneModule(llmod) } + llvm::LLVMCloneModule(llmod) } else { llmod }; @@ -1075,7 +1073,7 @@ pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> } /// Embed the bitcode of an LLVM module for LTO in the LLVM module itself. -unsafe fn embed_bitcode( +fn embed_bitcode( cgcx: &CodegenContext<LlvmCodegenBackend>, llcx: &llvm::Context, llmod: &llvm::Module, @@ -1117,43 +1115,40 @@ unsafe fn embed_bitcode( // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). - unsafe { - if cgcx.target_is_like_darwin - || cgcx.target_is_like_aix - || cgcx.target_arch == "wasm32" - || cgcx.target_arch == "wasm64" - { - // We don't need custom section flags, create LLVM globals. - let llconst = common::bytes_in_context(llcx, bitcode); - let llglobal = - llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module"); - llvm::set_initializer(llglobal, llconst); - - llvm::set_section(llglobal, bitcode_section_name(cgcx)); - llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::LLVMSetGlobalConstant(llglobal, llvm::True); - - let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); - let llglobal = - llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); - llvm::set_initializer(llglobal, llconst); - let section = if cgcx.target_is_like_darwin { - c"__LLVM,__cmdline" - } else if cgcx.target_is_like_aix { - c".info" - } else { - c".llvmcmd" - }; - llvm::set_section(llglobal, section); - llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + + if cgcx.target_is_like_darwin + || cgcx.target_is_like_aix + || cgcx.target_arch == "wasm32" + || cgcx.target_arch == "wasm64" + { + // We don't need custom section flags, create LLVM globals. + let llconst = common::bytes_in_context(llcx, bitcode); + let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module"); + llvm::set_initializer(llglobal, llconst); + + llvm::set_section(llglobal, bitcode_section_name(cgcx)); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::LLVMSetGlobalConstant(llglobal, llvm::True); + + let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); + llvm::set_initializer(llglobal, llconst); + let section = if cgcx.target_is_like_darwin { + c"__LLVM,__cmdline" + } else if cgcx.target_is_like_aix { + c".info" } else { - // We need custom section flags, so emit module-level inline assembly. - let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; - let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::append_module_inline_asm(llmod, &asm); - let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::append_module_inline_asm(llmod, &asm); - } + c".llvmcmd" + }; + llvm::set_section(llglobal, section); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + } else { + // We need custom section flags, so emit module-level inline assembly. + let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; + let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); + llvm::append_module_inline_asm(llmod, &asm); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + llvm::append_module_inline_asm(llmod, &asm); } } @@ -1184,7 +1179,7 @@ fn create_msvc_imps( .filter_map(|val| { // Exclude some symbols that we know are not Rust symbols. let name = llvm::get_value_name(val); - if ignored(name) { None } else { Some((val, name)) } + if ignored(&name) { None } else { Some((val, name)) } }) .map(move |(val, name)| { let mut imp_name = prefix.as_bytes().to_vec(); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index d0aa7320b4b..514923ad6f3 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -302,10 +302,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { return; } - let id_str = "branch_weights"; - let id = unsafe { - llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len()) - }; + let id = self.cx.create_metadata(b"branch_weights"); // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used. // This function handles switch instructions with more than 2 targets and it needs to @@ -538,16 +535,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value { - unsafe { - let alloca = - llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED); - llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - // Cast to default addrspace if necessary - llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) - } - } - fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); @@ -647,17 +634,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } else if place.layout.is_llvm_immediate() { let mut const_llval = None; let llty = place.layout.llvm_type(self); - unsafe { - if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { - if llvm::LLVMIsGlobalConstant(global) == llvm::True { - if let Some(init) = llvm::LLVMGetInitializer(global) { - if self.val_ty(init) == llty { - const_llval = Some(init); - } + if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { + if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); } } } } + let llval = const_llval.unwrap_or_else(|| { let load = self.load(llty, place.val.llval, place.val.align); if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr { @@ -1731,7 +1717,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } else { cfi::typeid_for_fnabi(self.tcx, fn_abi, options) }; - let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + let typeid_metadata = self.cx.create_metadata(typeid.as_bytes()); let dbg_loc = self.get_dbg_loc(); // Test whether the function pointer is associated with the type identifier using the diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index c27e161ba99..829b3c513c2 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -75,12 +75,12 @@ fn match_args_from_caller_to_enzyme<'ll>( outer_pos = 1; } - let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap(); - let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap(); - let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap(); - let enzyme_dupv = cx.create_metadata("enzyme_dupv".to_string()).unwrap(); - let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap(); - let enzyme_dupnoneedv = cx.create_metadata("enzyme_dupnoneedv".to_string()).unwrap(); + let enzyme_const = cx.create_metadata(b"enzyme_const"); + let enzyme_out = cx.create_metadata(b"enzyme_out"); + let enzyme_dup = cx.create_metadata(b"enzyme_dup"); + let enzyme_dupv = cx.create_metadata(b"enzyme_dupv"); + let enzyme_dupnoneed = cx.create_metadata(b"enzyme_dupnoneed"); + let enzyme_dupnoneedv = cx.create_metadata(b"enzyme_dupnoneedv"); while activity_pos < inputs.len() { let diff_activity = inputs[activity_pos as usize]; @@ -305,7 +305,7 @@ fn generate_enzyme_call<'ll>( // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::str::from_utf8(name).unwrap(); + let outer_fn_name = std::str::from_utf8(&name).unwrap(); ad_name.push_str(outer_fn_name); // Let us assume the user wrote the following function square: @@ -377,12 +377,12 @@ fn generate_enzyme_call<'ll>( let mut args = Vec::with_capacity(num_args as usize + 1); args.push(fn_to_diff); - let enzyme_primal_ret = cx.create_metadata("enzyme_primal_return".to_string()).unwrap(); + let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { args.push(cx.get_metadata_value(enzyme_primal_ret)); } if attrs.width > 1 { - let enzyme_width = cx.create_metadata("enzyme_width".to_string()).unwrap(); + let enzyme_width = cx.create_metadata(b"enzyme_width"); args.push(cx.get_metadata_value(enzyme_width)); args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 7cfab25bc50..f9ab96b5789 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -3,9 +3,8 @@ use std::borrow::Borrow; use libc::{c_char, c_uint}; -use rustc_abi as abi; -use rustc_abi::HasDataLayout; use rustc_abi::Primitive::Pointer; +use rustc_abi::{self as abi, HasDataLayout as _}; use rustc_ast::Mutability; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; @@ -175,7 +174,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { } fn const_usize(&self, i: u64) -> &'ll Value { - let bit_size = self.data_layout().pointer_size.bits(); + let bit_size = self.data_layout().pointer_size().bits(); if bit_size < 64 { // make sure it doesn't overflow assert!(i < (1 << bit_size)); @@ -216,10 +215,10 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { bug!("symbol `{}` is already defined", sym); }); llvm::set_initializer(g, sc); - unsafe { - llvm::LLVMSetGlobalConstant(g, True); - llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); - } + + llvm::set_global_constant(g, true); + llvm::set_unnamed_address(g, llvm::UnnamedAddr::Global); + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); // Cast to default address space if globals are in a different addrspace let g = self.const_pointercast(g, self.type_ptr()); @@ -284,7 +283,8 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.const_bitcast(llval, llty) }; } else { - let init = const_alloc_to_llvm(self, alloc, /*static*/ false); + let init = + const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); let alloc = alloc.inner(); let value = match alloc.mutability { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), @@ -316,15 +316,19 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { }), ))) .unwrap_memory(); - let init = const_alloc_to_llvm(self, alloc, /*static*/ false); - let value = self.static_addr_of_impl(init, alloc.inner().align, None); - value + let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + self.static_addr_of_impl(init, alloc.inner().align, None) } GlobalAlloc::Static(def_id) => { assert!(self.tcx.is_static(def_id)); assert!(!self.tcx.is_thread_local_static(def_id)); self.get_static(def_id) } + GlobalAlloc::TypeId { .. } => { + // Drop the provenance, the offset contains the bytes of the hash + let llval = self.const_usize(offset.bytes()); + return unsafe { llvm::LLVMConstIntToPtr(llval, llty) }; + } }; let base_addr_space = global_alloc.address_space(self); let llval = unsafe { @@ -346,7 +350,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { } fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value { - const_alloc_to_llvm(self, alloc, /*static*/ false) + const_alloc_to_llvm(self, alloc.inner(), /*static*/ false) } fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index a4492d76c3c..0b96b63bc85 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -17,20 +17,18 @@ use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use tracing::{debug, instrument, trace}; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::errors::SymbolAlreadyDefined; -use crate::llvm::{self, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use crate::{base, debuginfo}; +use crate::{base, debuginfo, llvm}; pub(crate) fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, - alloc: ConstAllocation<'_>, + alloc: &Allocation, is_static: bool, ) -> &'ll Value { - let alloc = alloc.inner(); // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be // producing empty LLVM allocations as they're just adding noise to binaries and forcing less @@ -43,7 +41,8 @@ pub(crate) fn const_alloc_to_llvm<'ll>( } let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); let dl = cx.data_layout(); - let pointer_size = dl.pointer_size.bytes() as usize; + let pointer_size = dl.pointer_size(); + let pointer_size_bytes = pointer_size.bytes() as usize; // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. @@ -100,7 +99,9 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since it is within the bounds of the allocation, it doesn't // affect interpreter execution (we inspect the result after interpreter execution), // and we properly interpret the provenance as a relocation pointer offset. - alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), + alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..(offset + pointer_size_bytes), + ), ) .expect("const_alloc_to_llvm: could not read relocation pointer") as u64; @@ -111,11 +112,11 @@ pub(crate) fn const_alloc_to_llvm<'ll>( InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), Scalar::Initialized { value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(dl.pointer_size), + valid_range: WrappingRange::full(pointer_size), }, cx.type_ptr_ext(address_space), )); - next_offset = offset + pointer_size; + next_offset = offset + pointer_size_bytes; } if alloc.len() >= next_offset { let range = next_offset..alloc.len(); @@ -138,7 +139,7 @@ fn codegen_static_initializer<'ll, 'tcx>( def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { let alloc = cx.tcx.eval_static_initializer(def_id)?; - Ok((const_alloc_to_llvm(cx, alloc, /*static*/ true), alloc)) + Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true), alloc)) } fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { @@ -245,7 +246,7 @@ impl<'ll> CodegenCx<'ll, '_> { }; llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); - llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global); gv } @@ -270,9 +271,8 @@ impl<'ll> CodegenCx<'ll, '_> { return gv; } let gv = self.static_addr_of_mut(cv, align, kind); - unsafe { - llvm::LLVMSetGlobalConstant(gv, True); - } + llvm::set_global_constant(gv, true); + self.const_globals.borrow_mut().insert(cv, gv); gv } @@ -396,149 +396,140 @@ impl<'ll> CodegenCx<'ll, '_> { } fn codegen_static_item(&mut self, def_id: DefId) { - unsafe { - assert!( - llvm::LLVMGetInitializer( - self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap() - ) - .is_none() - ); - let attrs = self.tcx.codegen_fn_attrs(def_id); + assert!( + llvm::LLVMGetInitializer( + self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap() + ) + .is_none() + ); + let attrs = self.tcx.codegen_fn_attrs(def_id); - let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { - // Error has already been reported - return; - }; - let alloc = alloc.inner(); + let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { + // Error has already been reported + return; + }; + let alloc = alloc.inner(); - let val_llty = self.val_ty(v); + let val_llty = self.val_ty(v); - let g = self.get_static_inner(def_id, val_llty); - let llty = self.get_type_of_global(g); + let g = self.get_static_inner(def_id, val_llty); + let llty = self.get_type_of_global(g); - let g = if val_llty == llty { - g - } else { - // codegen_static_initializer creates the global value just from the - // `Allocation` data by generating one big struct value that is just - // all the bytes and pointers after each other. This will almost never - // match the type that the static was declared with. Unfortunately - // we can't just LLVMConstBitCast our way out of it because that has very - // specific rules on what can be cast. So instead of adding a new way to - // generate static initializers that match the static's type, we picked - // the easier option and retroactively change the type of the static item itself. - let name = llvm::get_value_name(g).to_vec(); - llvm::set_value_name(g, b""); - - let linkage = llvm::get_linkage(g); - let visibility = llvm::get_visibility(g); - - let new_g = llvm::LLVMRustGetOrInsertGlobal( - self.llmod, - name.as_c_char_ptr(), - name.len(), - val_llty, - ); - - llvm::set_linkage(new_g, linkage); - llvm::set_visibility(new_g, visibility); - - // The old global has had its name removed but is returned by - // get_static since it is in the instance cache. Provide an - // alternative lookup that points to the new global so that - // global_asm! can compute the correct mangled symbol name - // for the global. - self.renamed_statics.borrow_mut().insert(def_id, new_g); - - // To avoid breaking any invariants, we leave around the old - // global for the moment; we'll replace all references to it - // with the new global later. (See base::codegen_backend.) - self.statics_to_rauw.borrow_mut().push((g, new_g)); - new_g - }; - set_global_alignment(self, g, alloc.align); - llvm::set_initializer(g, v); - - self.assume_dso_local(g, true); - - // Forward the allocation's mutability (picked by the const interner) to LLVM. - if alloc.mutability.is_not() { - llvm::LLVMSetGlobalConstant(g, llvm::True); - } + let g = if val_llty == llty { + g + } else { + // codegen_static_initializer creates the global value just from the + // `Allocation` data by generating one big struct value that is just + // all the bytes and pointers after each other. This will almost never + // match the type that the static was declared with. Unfortunately + // we can't just LLVMConstBitCast our way out of it because that has very + // specific rules on what can be cast. So instead of adding a new way to + // generate static initializers that match the static's type, we picked + // the easier option and retroactively change the type of the static item itself. + let name = String::from_utf8(llvm::get_value_name(g)) + .expect("we declare our statics with a utf8-valid name"); + llvm::set_value_name(g, b""); + + let linkage = llvm::get_linkage(g); + let visibility = llvm::get_visibility(g); + + let new_g = self.declare_global(&name, val_llty); + + llvm::set_linkage(new_g, linkage); + llvm::set_visibility(new_g, visibility); + + // The old global has had its name removed but is returned by + // get_static since it is in the instance cache. Provide an + // alternative lookup that points to the new global so that + // global_asm! can compute the correct mangled symbol name + // for the global. + self.renamed_statics.borrow_mut().insert(def_id, new_g); + + // To avoid breaking any invariants, we leave around the old + // global for the moment; we'll replace all references to it + // with the new global later. (See base::codegen_backend.) + self.statics_to_rauw.borrow_mut().push((g, new_g)); + new_g + }; + set_global_alignment(self, g, alloc.align); + llvm::set_initializer(g, v); - debuginfo::build_global_var_di_node(self, def_id, g); + self.assume_dso_local(g, true); - if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { - llvm::set_thread_local_mode(g, self.tls_model); - } + // Forward the allocation's mutability (picked by the const interner) to LLVM. + if alloc.mutability.is_not() { + llvm::set_global_constant(g, true); + } - // Wasm statics with custom link sections get special treatment as they - // go into custom sections of the wasm executable. The exception to this - // is the `.init_array` section which are treated specially by the wasm linker. - if self.tcx.sess.target.is_like_wasm - && attrs - .link_section - .map(|link_section| !link_section.as_str().starts_with(".init_array")) - .unwrap_or(true) - { - if let Some(section) = attrs.link_section { - let section = llvm::LLVMMDStringInContext2( - self.llcx, - section.as_str().as_c_char_ptr(), - section.as_str().len(), - ); - assert!(alloc.provenance().ptrs().is_empty()); - - // The `inspect` method is okay here because we checked for provenance, and - // because we are doing this access to inspect the final interpreter state (not - // as part of the interpreter execution). - let bytes = - alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); - let alloc = - llvm::LLVMMDStringInContext2(self.llcx, bytes.as_c_char_ptr(), bytes.len()); - let data = [section, alloc]; - let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); - let val = self.get_metadata_value(meta); + debuginfo::build_global_var_di_node(self, def_id, g); + + if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + llvm::set_thread_local_mode(g, self.tls_model); + } + + // Wasm statics with custom link sections get special treatment as they + // go into custom sections of the wasm executable. The exception to this + // is the `.init_array` section which are treated specially by the wasm linker. + if self.tcx.sess.target.is_like_wasm + && attrs + .link_section + .map(|link_section| !link_section.as_str().starts_with(".init_array")) + .unwrap_or(true) + { + if let Some(section) = attrs.link_section { + let section = self.create_metadata(section.as_str().as_bytes()); + assert!(alloc.provenance().ptrs().is_empty()); + + // The `inspect` method is okay here because we checked for provenance, and + // because we are doing this access to inspect the final interpreter state (not + // as part of the interpreter execution). + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); + let alloc = self.create_metadata(bytes); + let data = [section, alloc]; + let meta = + unsafe { llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()) }; + let val = self.get_metadata_value(meta); + unsafe { llvm::LLVMAddNamedMetadataOperand( self.llmod, c"wasm.custom_sections".as_ptr(), val, - ); - } - } else { - base::set_link_section(g, attrs); + ) + }; } + } else { + base::set_link_section(g, attrs); + } - base::set_variable_sanitizer_attrs(g, attrs); - - if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER) { - // `USED` and `USED_LINKER` can't be used together. - assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); - - // The semantics of #[used] in Rust only require the symbol to make it into the - // object file. It is explicitly allowed for the linker to strip the symbol if it - // is dead, which means we are allowed to use `llvm.compiler.used` instead of - // `llvm.used` here. - // - // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique - // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs - // in the handling of `.init_array` (the static constructor list) in versions of - // the gold linker (prior to the one released with binutils 2.36). - // - // That said, we only ever emit these when `#[used(compiler)]` is explicitly - // requested. This is to avoid similar breakage on other targets, in particular - // MachO targets have *their* static constructor lists broken if `llvm.compiler.used` - // is emitted rather than `llvm.used`. However, that check happens when assigning - // the `CodegenFnAttrFlags` in the `codegen_fn_attrs` query, so we don't need to - // take care of it here. - self.add_compiler_used_global(g); - } - if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { - // `USED` and `USED_LINKER` can't be used together. - assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)); + base::set_variable_sanitizer_attrs(g, attrs); + + if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); + + // The semantics of #[used] in Rust only require the symbol to make it into the + // object file. It is explicitly allowed for the linker to strip the symbol if it + // is dead, which means we are allowed to use `llvm.compiler.used` instead of + // `llvm.used` here. + // + // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique + // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs + // in the handling of `.init_array` (the static constructor list) in versions of + // the gold linker (prior to the one released with binutils 2.36). + // + // That said, we only ever emit these when `#[used(compiler)]` is explicitly + // requested. This is to avoid similar breakage on other targets, in particular + // MachO targets have *their* static constructor lists broken if `llvm.compiler.used` + // is emitted rather than `llvm.used`. However, that check happens when assigning + // the `CodegenFnAttrFlags` in the `codegen_fn_attrs` query, so we don't need to + // take care of it here. + self.add_compiler_used_global(g); + } + if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)); - self.add_used_global(g); - } + self.add_used_global(g); } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 0324dff6ff2..6a23becaa96 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -34,7 +34,6 @@ use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; -use crate::common::AsCCharPtr; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::Metadata; use crate::type_::Type; @@ -169,6 +168,8 @@ pub(crate) unsafe fn create_module<'ll>( let mod_name = SmallCStr::new(mod_name); let llmod = unsafe { llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx) }; + let cx = SimpleCx::new(llmod, llcx, tcx.data_layout.pointer_size()); + let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); @@ -473,18 +474,14 @@ pub(crate) unsafe fn create_module<'ll>( #[allow(clippy::option_env_unwrap)] let rustc_producer = format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION")); - let name_metadata = unsafe { - llvm::LLVMMDStringInContext2( - llcx, - rustc_producer.as_c_char_ptr(), - rustc_producer.as_bytes().len(), - ) - }; + + let name_metadata = cx.create_metadata(rustc_producer.as_bytes()); + unsafe { llvm::LLVMAddNamedMetadataOperand( llmod, c"llvm.ident".as_ptr(), - &llvm::LLVMMetadataAsValue(llcx, llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)), + &cx.get_metadata_value(llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)), ); } @@ -605,7 +602,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { GenericCx( FullCx { tcx, - scx: SimpleCx::new(llmod, llcx, tcx.data_layout.pointer_size), + scx: SimpleCx::new(llmod, llcx, tcx.data_layout.pointer_size()), use_dll_storage_attrs, tls_model, codegen_unit, @@ -698,10 +695,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { } } - pub(crate) fn create_metadata(&self, name: String) -> Option<&'ll Metadata> { - Some(unsafe { + pub(crate) fn create_metadata(&self, name: &[u8]) -> &'ll Metadata { + unsafe { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) - }) + } } pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 8f0948b8183..61555ac2f6f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -1,13 +1,12 @@ // .debug_gdb_scripts binary section. -use rustc_ast::attr; +use rustc_attr_data_structures::{AttributeKind, find_attr}; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; -use rustc_span::sym; use crate::builder::Builder; use crate::common::CodegenCx; @@ -75,7 +74,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( llvm::set_section(section_var, c".debug_gdb_scripts"); llvm::set_initializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); - llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); + llvm::set_unnamed_address(section_var, llvm::UnnamedAddr::Global); llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than // the string it contains. Otherwise we get a warning from GDB. @@ -87,7 +86,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = - attr::contains_name(cx.tcx.hir_krate_attrs(), sym::omit_gdb_pretty_printer_section); + find_attr!(cx.tcx.hir_krate_attrs(), AttributeKind::OmitGdbPrettyPrinterSection); // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 7f3e486ca31..0e9dbfba658 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::{iter, ptr}; -use libc::{c_char, c_longlong, c_uint}; +use libc::{c_longlong, c_uint}; use rustc_abi::{Align, Size}; use rustc_codegen_ssa::debuginfo::type_names::{VTableNameKind, cpp_like_debuginfo}; use rustc_codegen_ssa::traits::*; @@ -159,13 +159,15 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( return_if_di_node_created_in_meantime!(cx, unique_type_id); let data_layout = &cx.tcx.data_layout; + let pointer_size = data_layout.pointer_size(); + let pointer_align = data_layout.pointer_align(); let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); match wide_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. assert_eq!( - (data_layout.pointer_size, data_layout.pointer_align.abi), + (pointer_size, pointer_align.abi), cx.size_and_align_of(ptr_type), "ptr_type={ptr_type}, pointee_type={pointee_type}", ); @@ -174,8 +176,8 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), pointee_type_di_node, - data_layout.pointer_size.bits(), - data_layout.pointer_align.abi.bits() as u32, + pointer_size.bits(), + pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. ptr_type_debuginfo_name.as_c_char_ptr(), ptr_type_debuginfo_name.len(), @@ -319,7 +321,9 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); let (size, align) = match fn_ty.kind() { ty::FnDef(..) => (Size::ZERO, Align::ONE), - ty::FnPtr(..) => (cx.tcx.data_layout.pointer_size, cx.tcx.data_layout.pointer_align.abi), + ty::FnPtr(..) => { + (cx.tcx.data_layout.pointer_size(), cx.tcx.data_layout.pointer_align().abi) + } _ => unreachable!(), }; let di_node = unsafe { @@ -504,7 +508,7 @@ fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll D create_basic_type( cx, "<recur_type>", - cx.tcx.data_layout.pointer_size, + cx.tcx.data_layout.pointer_size(), dwarf_const::DW_ATE_unsigned, ) }) @@ -1578,13 +1582,9 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( }; let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); unsafe { - let typeid = llvm::LLVMMDStringInContext2( - cx.llcx, - trait_ref_typeid.as_ptr() as *const c_char, - trait_ref_typeid.as_bytes().len(), - ); let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; llvm::LLVMRustGlobalAddMetadata( vtable, @@ -1626,7 +1626,7 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>( // When full debuginfo is enabled, we want to try and prevent vtables from being // merged. Otherwise debuggers will have a hard time mapping from dyn pointer // to concrete type. - llvm::SetUnnamedAddress(vtable, llvm::UnnamedAddr::No); + llvm::set_unnamed_address(vtable, llvm::UnnamedAddr::No); let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 2419ec1f888..eb75716d768 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -49,7 +49,7 @@ pub(crate) fn declare_simple_fn<'ll>( }; llvm::SetFunctionCallConv(llfn, callconv); - llvm::SetUnnamedAddress(llfn, unnamed); + llvm::set_unnamed_address(llfn, unnamed); llvm::set_visibility(llfn, visibility); llfn @@ -176,7 +176,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { { let typeid = cfi::typeid_for_instance(self.tcx, instance, options); if typeids.insert(typeid.clone()) { - self.add_type_metadata(llfn, typeid); + self.add_type_metadata(llfn, typeid.as_bytes()); } } } else { @@ -189,7 +189,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { .map(cfi::TypeIdOptions::from_iter) { let typeid = cfi::typeid_for_fnabi(self.tcx, fn_abi, options); - self.add_type_metadata(llfn, typeid); + self.add_type_metadata(llfn, typeid.as_bytes()); } } } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9930eae3fe7..fcc0d378f06 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -103,23 +103,25 @@ fn call_simple_intrinsic<'ll, 'tcx>( sym::minnumf64 => ("llvm.minnum", &[bx.type_f64()]), sym::minnumf128 => ("llvm.minnum", &[bx.type_f128()]), - sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]), - sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]), - sym::minimumf64 => ("llvm.minimum", &[bx.type_f64()]), - // There are issues on x86_64 and aarch64 with the f128 variant, - // let's instead use the instrinsic fallback body. - // sym::minimumf128 => ("llvm.minimum", &[cx.type_f128()]), + // FIXME: LLVM currently mis-compile those intrinsics, re-enable them + // when llvm/llvm-project#{139380,139381,140445} are fixed. + //sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]), + //sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]), + //sym::minimumf64 => ("llvm.minimum", &[bx.type_f64()]), + //sym::minimumf128 => ("llvm.minimum", &[cx.type_f128()]), + // sym::maxnumf16 => ("llvm.maxnum", &[bx.type_f16()]), sym::maxnumf32 => ("llvm.maxnum", &[bx.type_f32()]), sym::maxnumf64 => ("llvm.maxnum", &[bx.type_f64()]), sym::maxnumf128 => ("llvm.maxnum", &[bx.type_f128()]), - sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]), - sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]), - sym::maximumf64 => ("llvm.maximum", &[bx.type_f64()]), - // There are issues on x86_64 and aarch64 with the f128 variant, - // let's instead use the instrinsic fallback body. - // sym::maximumf128 => ("llvm.maximum", &[cx.type_f128()]), + // FIXME: LLVM currently mis-compile those intrinsics, re-enable them + // when llvm/llvm-project#{139380,139381,140445} are fixed. + //sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]), + //sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]), + //sym::maximumf64 => ("llvm.maximum", &[bx.type_f64()]), + //sym::maximumf128 => ("llvm.maximum", &[cx.type_f128()]), + // sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]), sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]), sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]), @@ -456,7 +458,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), // so we re-use that same threshold here. - layout.size() <= self.data_layout().pointer_size * 2 + layout.size() <= self.data_layout().pointer_size() * 2 } }; @@ -756,8 +758,8 @@ fn codegen_msvc_try<'ll, 'tcx>( // } // // More information can be found in libstd's seh.rs implementation. - let ptr_size = bx.tcx().data_layout.pointer_size; - let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let ptr_size = bx.tcx().data_layout.pointer_size(); + let ptr_align = bx.tcx().data_layout.pointer_align().abi; let slot = bx.alloca(ptr_size, ptr_align); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); @@ -1029,8 +1031,8 @@ fn codegen_emcc_try<'ll, 'tcx>( // We need to pass two values to catch_func (ptr and is_rust_panic), so // create an alloca and pass a pointer to that. - let ptr_size = bx.tcx().data_layout.pointer_size; - let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let ptr_size = bx.tcx().data_layout.pointer_size(); + let ptr_align = bx.tcx().data_layout.pointer_align().abi; let i8_align = bx.tcx().data_layout.i8_align.abi; // Required in order for there to be no padding between the fields. assert!(i8_align <= ptr_align); @@ -1156,9 +1158,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( macro_rules! require_int_or_uint_ty { ($ty: expr, $diag: expr) => { match $ty { - ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), + ty::Int(i) => { + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits()) + } ty::Uint(i) => { - i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()) + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits()) } _ => { return_error!($diag); @@ -2012,10 +2016,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } else { let bitwidth = match in_elem.kind() { ty::Int(i) => { - i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()) + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits()) } ty::Uint(i) => { - i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()) + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits()) } _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { span, diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 5d551c3af87..6db4e122ad6 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -113,7 +113,7 @@ impl ExtraBackendMethods for LlvmCodegenBackend { ) -> ModuleLlvm { let module_llvm = ModuleLlvm::new_metadata(tcx, module_name); let cx = - SimpleCx::new(module_llvm.llmod(), &module_llvm.llcx, tcx.data_layout.pointer_size); + SimpleCx::new(module_llvm.llmod(), &module_llvm.llcx, tcx.data_layout.pointer_size()); unsafe { allocator::codegen(tcx, cx, module_name, kind, alloc_error_handler_kind); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index b94716b89d6..c696b8d8ff2 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -1,4 +1,3 @@ -#![allow(non_camel_case_types)] #![expect(dead_code)] use libc::{c_char, c_uint}; @@ -40,7 +39,7 @@ unsafe extern "C" { pub(crate) fn LLVMDumpValue(V: &Value); pub(crate) fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; pub(crate) fn LLVMGetReturnType(T: &Type) -> &Type; - pub(crate) fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); + pub(crate) fn LLVMGetParams(Fnc: &Value, params: *mut &Value); pub(crate) fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 91ada856d59..0b1e632cbc4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1009,7 +1009,7 @@ unsafe extern "C" { ModuleID: *const c_char, C: &Context, ) -> &Module; - pub(crate) fn LLVMCloneModule(M: &Module) -> &Module; + pub(crate) safe fn LLVMCloneModule(M: &Module) -> &Module; /// Data layout. See Module::getDataLayout. pub(crate) fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; @@ -1168,18 +1168,18 @@ unsafe extern "C" { pub(crate) fn LLVMGlobalGetValueType(Global: &Value) -> &Type; // Operations on global variables - pub(crate) fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; + pub(crate) safe fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; pub(crate) fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; pub(crate) fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; pub(crate) fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; pub(crate) fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; pub(crate) fn LLVMDeleteGlobal(GlobalVar: &Value); - pub(crate) fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; + pub(crate) safe fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; pub(crate) fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); - pub(crate) fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; + pub(crate) safe fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; pub(crate) fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); - pub(crate) fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; - pub(crate) fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); + pub(crate) safe fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; + pub(crate) safe fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); // Operations on attributes @@ -1492,12 +1492,6 @@ unsafe extern "C" { Ty: &'a Type, Name: *const c_char, ) -> &'a Value; - pub(crate) fn LLVMBuildArrayAlloca<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Val: &'a Value, - Name: *const c_char, - ) -> &'a Value; pub(crate) fn LLVMBuildLoad2<'a>( B: &Builder<'a>, Ty: &'a Type, @@ -1724,7 +1718,7 @@ unsafe extern "C" { pub(crate) safe fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; - pub(crate) fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); + pub(crate) safe fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); pub(crate) fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; @@ -1980,12 +1974,12 @@ unsafe extern "C" { pub(crate) fn LLVMRustBuildMinNum<'a>( B: &Builder<'a>, LHS: &'a Value, - LHS: &'a Value, + RHS: &'a Value, ) -> &'a Value; pub(crate) fn LLVMRustBuildMaxNum<'a>( B: &Builder<'a>, LHS: &'a Value, - LHS: &'a Value, + RHS: &'a Value, ) -> &'a Value; // Atomic Operations diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 661174a80df..154ba4fd690 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -211,16 +211,14 @@ pub(crate) fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { // function. // For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 pub(crate) fn SetUniqueComdat(llmod: &Module, val: &Value) { - let name_buf = get_value_name(val).to_vec(); + let name_buf = get_value_name(val); let name = CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap(); set_comdat(llmod, val, &name); } -pub(crate) fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { - unsafe { - LLVMSetUnnamedAddress(global, unnamed); - } +pub(crate) fn set_unnamed_address(global: &Value, unnamed: UnnamedAddr) { + LLVMSetUnnamedAddress(global, unnamed); } pub(crate) fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { @@ -260,9 +258,7 @@ pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) { } pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) { - unsafe { - LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); - } + LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); } pub(crate) fn get_linkage(llglobal: &Value) -> Linkage { @@ -319,12 +315,14 @@ pub(crate) fn get_param(llfn: &Value, index: c_uint) -> &Value { } } -/// Safe wrapper for `LLVMGetValueName2` into a byte slice -pub(crate) fn get_value_name(value: &Value) -> &[u8] { +/// Safe wrapper for `LLVMGetValueName2` +/// Needs to allocate the value, because `set_value_name` will invalidate +/// the pointer. +pub(crate) fn get_value_name(value: &Value) -> Vec<u8> { unsafe { let mut len = 0; let data = LLVMGetValueName2(value, &mut len); - std::slice::from_raw_parts(data.cast(), len) + std::slice::from_raw_parts(data.cast(), len).to_vec() } } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 3f38e1e191b..8f70270f203 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -131,8 +131,8 @@ impl CodegenCx<'_, '_> { } // Thread-local variables generally don't support copy relocations. - let is_thread_local_var = unsafe { llvm::LLVMIsAGlobalVariable(llval) } - .is_some_and(|v| unsafe { llvm::LLVMIsThreadLocal(v) } == llvm::True); + let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) + .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 453eca2bbe1..89365503138 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -2,7 +2,7 @@ use std::borrow::Borrow; use std::hash::{Hash, Hasher}; use std::{fmt, ptr}; -use libc::{c_char, c_uint}; +use libc::c_uint; use rustc_abi::{AddressSpace, Align, Integer, Reg, Size}; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; @@ -208,7 +208,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_ptr(&self) -> &'ll Type { - self.type_ptr_ext(AddressSpace::DATA) + self.type_ptr_ext(AddressSpace::ZERO) } fn type_ptr_ext(&self, address_space: AddressSpace) -> &'ll Type { @@ -258,7 +258,7 @@ impl Type { } pub(crate) fn ptr_llcx(llcx: &llvm::Context) -> &Type { - unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::DATA.0) } + unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::ZERO.0) } } } @@ -298,8 +298,8 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn add_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + fn add_type_metadata(&self, function: &'ll Value, typeid: &[u8]) { + let typeid_metadata = self.create_metadata(typeid); unsafe { let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; llvm::LLVMRustGlobalAddMetadata( @@ -310,8 +310,8 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - fn set_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + fn set_type_metadata(&self, function: &'ll Value, typeid: &[u8]) { + let typeid_metadata = self.create_metadata(typeid); unsafe { let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; llvm::LLVMGlobalSetMetadata( @@ -322,10 +322,8 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - fn typeid_metadata(&self, typeid: String) -> Option<&'ll Metadata> { - Some(unsafe { - llvm::LLVMMDStringInContext2(self.llcx, typeid.as_ptr() as *const c_char, typeid.len()) - }) + fn typeid_metadata(&self, typeid: &[u8]) -> Option<&'ll Metadata> { + Some(self.create_metadata(typeid)) } fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 236568590be..ce079f3cb0a 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -45,7 +45,8 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>( let va_list_ty = bx.type_ptr(); let va_list_addr = list.immediate(); - let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi; + let ptr = bx.load(va_list_ty, va_list_addr, ptr_align_abi); let (addr, addr_align) = if allow_higher_align && align > slot_size { (round_pointer_up_to_alignment(bx, ptr, align, bx.type_ptr()), align) @@ -56,7 +57,7 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>( let aligned_size = size.align_to(slot_size).bytes() as i32; let full_direct_size = bx.cx().const_i32(aligned_size); let next = bx.inbounds_ptradd(addr, full_direct_size); - bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + bx.store(next, va_list_addr, ptr_align_abi); if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big @@ -108,8 +109,8 @@ fn emit_ptr_va_arg<'ll, 'tcx>( let (llty, size, align) = if indirect { ( bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx), - bx.cx.data_layout().pointer_size, - bx.cx.data_layout().pointer_align, + bx.cx.data_layout().pointer_size(), + bx.cx.data_layout().pointer_align(), ) } else { (layout.llvm_type(bx.cx), layout.size, layout.align) @@ -172,10 +173,10 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); let (reg_off, reg_top, slot_size) = if gr_type { - let nreg = (layout.size.bytes() + 7) / 8; + let nreg = layout.size.bytes().div_ceil(8); (gr_offs, gr_top, nreg * 8) } else { - let nreg = (layout.size.bytes() + 15) / 16; + let nreg = layout.size.bytes().div_ceil(16); (vr_offs, vr_top, nreg * 16) }; @@ -204,7 +205,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); let top_type = bx.type_ptr(); - let top = bx.load(top_type, reg_top, dl.pointer_align.abi); + let top = bx.load(top_type, reg_top, dl.pointer_align().abi); // reg_value = *(@top + reg_off_v); let mut reg_addr = bx.ptradd(top, reg_off_v); @@ -297,6 +298,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( let max_regs = 8u8; let use_regs = bx.icmp(IntPredicate::IntULT, num_regs, bx.const_u8(max_regs)); + let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi; let in_reg = bx.append_sibling_block("va_arg.in_reg"); let in_mem = bx.append_sibling_block("va_arg.in_mem"); @@ -308,7 +310,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); let reg_safe_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(1 + 1 + 2 + 4)); - let mut reg_addr = bx.load(bx.type_ptr(), reg_safe_area_ptr, dl.pointer_align.abi); + let mut reg_addr = bx.load(bx.type_ptr(), reg_safe_area_ptr, ptr_align_abi); // Floating-point registers start after the general-purpose registers. if !is_int && !is_soft_float_abi { @@ -342,11 +344,11 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( let size = if !is_indirect { layout.layout.size.align_to(overflow_area_align) } else { - dl.pointer_size + dl.pointer_size() }; let overflow_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(1 + 1 + 2)); - let mut overflow_area = bx.load(bx.type_ptr(), overflow_area_ptr, dl.pointer_align.abi); + let mut overflow_area = bx.load(bx.type_ptr(), overflow_area_ptr, ptr_align_abi); // Round up address of argument to alignment if layout.layout.align.abi > overflow_area_align { @@ -362,7 +364,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( // Increase the overflow area. overflow_area = bx.inbounds_ptradd(overflow_area, bx.const_usize(size.bytes())); - bx.store(overflow_area, overflow_area_ptr, dl.pointer_align.abi); + bx.store(overflow_area, overflow_area_ptr, ptr_align_abi); bx.br(end); @@ -373,11 +375,8 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( bx.switch_to_block(end); let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]); let val_type = layout.llvm_type(bx); - let val_addr = if is_indirect { - bx.load(bx.cx.type_ptr(), val_addr, dl.pointer_align.abi) - } else { - val_addr - }; + let val_addr = + if is_indirect { bx.load(bx.cx.type_ptr(), val_addr, ptr_align_abi) } else { val_addr }; bx.load(val_type, val_addr, layout.align.abi) } @@ -414,6 +413,7 @@ fn emit_s390x_va_arg<'ll, 'tcx>( let in_reg = bx.append_sibling_block("va_arg.in_reg"); let in_mem = bx.append_sibling_block("va_arg.in_mem"); let end = bx.append_sibling_block("va_arg.end"); + let ptr_align_abi = dl.pointer_align().abi; // FIXME: vector ABI not yet supported. let target_ty_size = bx.cx.size_of(target_ty).bytes(); @@ -435,7 +435,7 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); // Work out the address of the value in the register save area. - let reg_ptr_v = bx.load(bx.type_ptr(), reg_save_area, dl.pointer_align.abi); + let reg_ptr_v = bx.load(bx.type_ptr(), reg_save_area, ptr_align_abi); let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8)); let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding)); let reg_addr = bx.ptradd(reg_ptr_v, reg_off); @@ -449,15 +449,14 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.switch_to_block(in_mem); // Work out the address of the value in the argument overflow area. - let arg_ptr_v = - bx.load(bx.type_ptr(), overflow_arg_area, bx.tcx().data_layout.pointer_align.abi); + let arg_ptr_v = bx.load(bx.type_ptr(), overflow_arg_area, ptr_align_abi); let arg_off = bx.const_u64(padding); let mem_addr = bx.ptradd(arg_ptr_v, arg_off); // Update the argument overflow area pointer. let arg_size = bx.cx().const_u64(padded_size); let new_arg_ptr_v = bx.inbounds_ptradd(arg_ptr_v, arg_size); - bx.store(new_arg_ptr_v, overflow_arg_area, dl.pointer_align.abi); + bx.store(new_arg_ptr_v, overflow_arg_area, ptr_align_abi); bx.br(end); // Return the appropriate result. @@ -465,7 +464,7 @@ fn emit_s390x_va_arg<'ll, 'tcx>( let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]); let val_type = layout.llvm_type(bx); let val_addr = - if indirect { bx.load(bx.cx.type_ptr(), val_addr, dl.pointer_align.abi) } else { val_addr }; + if indirect { bx.load(bx.cx.type_ptr(), val_addr, ptr_align_abi) } else { val_addr }; bx.load(val_type, val_addr, layout.align.abi) } @@ -607,7 +606,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>( // loads than necessary. Can we clean this up? let reg_save_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * unsigned_int_offset + ptr_offset)); - let reg_save_area_v = bx.load(bx.type_ptr(), reg_save_area_ptr, dl.pointer_align.abi); + let reg_save_area_v = bx.load(bx.type_ptr(), reg_save_area_ptr, dl.pointer_align().abi); let reg_addr = match layout.layout.backend_repr() { BackendRepr::Scalar(scalar) => match scalar.primitive() { @@ -749,10 +748,11 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>( layout: TyAndLayout<'tcx, Ty<'tcx>>, ) -> &'ll Value { let dl = bx.cx.data_layout(); + let ptr_align_abi = dl.data_layout().pointer_align().abi; let overflow_arg_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(8)); - let overflow_arg_area_v = bx.load(bx.type_ptr(), overflow_arg_area_ptr, dl.pointer_align.abi); + let overflow_arg_area_v = bx.load(bx.type_ptr(), overflow_arg_area_ptr, ptr_align_abi); // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16 // byte boundary if alignment needed by type exceeds 8 byte boundary. // It isn't stated explicitly in the standard, but in practice we use @@ -771,7 +771,7 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>( let size_in_bytes = layout.layout.size().bytes(); let offset = bx.const_i32(size_in_bytes.next_multiple_of(8) as i32); let overflow_arg_area = bx.inbounds_ptradd(overflow_arg_area_v, offset); - bx.store(overflow_arg_area, overflow_arg_area_ptr, dl.pointer_align.abi); + bx.store(overflow_arg_area, overflow_arg_area_ptr, ptr_align_abi); mem_addr } @@ -803,6 +803,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( let from_stack = bx.append_sibling_block("va_arg.from_stack"); let from_regsave = bx.append_sibling_block("va_arg.from_regsave"); let end = bx.append_sibling_block("va_arg.end"); + let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi; // (*va).va_ndx let va_reg_offset = 4; @@ -825,12 +826,11 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( bx.switch_to_block(from_regsave); // update va_ndx - bx.store(offset_next, offset_ptr, bx.tcx().data_layout.pointer_align.abi); + bx.store(offset_next, offset_ptr, ptr_align_abi); // (*va).va_reg let regsave_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_reg_offset)); - let regsave_area = - bx.load(bx.type_ptr(), regsave_area_ptr, bx.tcx().data_layout.pointer_align.abi); + let regsave_area = bx.load(bx.type_ptr(), regsave_area_ptr, ptr_align_abi); let regsave_value_ptr = bx.inbounds_ptradd(regsave_area, offset); bx.br(end); @@ -849,11 +849,11 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( // va_ndx = offset_next_corrected; let offset_next_corrected = bx.add(offset_next, bx.const_i32(slot_size)); // update va_ndx - bx.store(offset_next_corrected, offset_ptr, bx.tcx().data_layout.pointer_align.abi); + bx.store(offset_next_corrected, offset_ptr, ptr_align_abi); // let stack_value_ptr = unsafe { (*va).va_stk.byte_add(offset_corrected) }; let stack_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(0)); - let stack_area = bx.load(bx.type_ptr(), stack_area_ptr, bx.tcx().data_layout.pointer_align.abi); + let stack_area = bx.load(bx.type_ptr(), stack_area_ptr, ptr_align_abi); let stack_value_ptr = bx.inbounds_ptradd(stack_area, offset_corrected); bx.br(end); @@ -861,7 +861,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( // On big-endian, for values smaller than the slot size we'd have to align the read to the end // of the slot rather than the start. While the ISA and GCC support big-endian, all the Xtensa - // targets supported by rustc are litte-endian so don't worry about it. + // targets supported by rustc are little-endian so don't worry about it. // if from_regsave { // unsafe { *regsave_value_ptr } |