diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
| -rw-r--r-- | compiler/rustc_codegen_llvm/Cargo.toml | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 51 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 41 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 29 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 26 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/va_arg.rs | 24 |
15 files changed, 162 insertions, 78 deletions
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index b04310f3d54..2d11628250c 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -8,15 +8,15 @@ test = false [dependencies] # tidy-alphabetical-start -bitflags.workspace = true +bitflags = "2.4.1" # To avoid duplicate dependencies, this should match the version of gimli used # by `rustc_codegen_ssa` via its `thorin-dwp` dependency. gimli = "0.31" -itertools.workspace = true -libc.workspace = true -measureme.workspace = true +itertools = "0.12" +libc = "0.2" +measureme = "12.0.1" object = { version = "0.37.0", default-features = false, features = ["std", "read"] } -rustc-demangle.workspace = true +rustc-demangle = "0.1.21" rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } @@ -38,9 +38,9 @@ rustc_span = { path = "../rustc_span" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_target = { path = "../rustc_target" } serde = { version = "1", features = ["derive"] } -serde_json.workspace = true +serde_json = "1" smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } -tracing.workspace = true +tracing = "0.1" # tidy-alphabetical-end [features] diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 5affb26483a..573c51a9539 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -29,8 +29,18 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } /// Get LLVM attribute for the provided inline heuristic. -#[inline] -fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { +pub(crate) fn inline_attr<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: ty::Instance<'tcx>, +) -> Option<&'ll Attribute> { + // `optnone` requires `noinline` + let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { + (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, + (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, + (inline, _) => inline, + }; + if !cx.tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); @@ -286,6 +296,19 @@ pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribu .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } +/// Get the `target-features` LLVM attribute. +pub(crate) fn target_features_attr<'ll>( + cx: &CodegenCx<'ll, '_>, + function_features: Vec<String>, +) -> Option<&'ll Attribute> { + let global_features = cx.tcx.global_backend_features(()).iter().map(String::as_str); + let function_features = function_features.iter().map(String::as_str); + let target_features = + global_features.chain(function_features).intersperse(",").collect::<String>(); + (!target_features.is_empty()) + .then(|| llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)) +} + /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { @@ -346,14 +369,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - // `optnone` requires `noinline` - let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { - (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, - (inline, _) => inline, - }; - to_add.extend(inline_attr(cx, inline)); - if cx.sess().must_emit_unwind_tables() { to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); } @@ -488,6 +503,14 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); + // Apply function attributes as per usual if there are no user defined + // target features otherwise this will get applied at the callsite. + if function_features.is_empty() { + if let Some(inline_attr) = inline_attr(cx, instance) { + to_add.push(inline_attr); + } + } + let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones @@ -513,13 +536,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( } } - let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); - let function_features = function_features.iter().map(|s| s.as_str()); - let target_features: String = - global_features.chain(function_features).intersperse(",").collect(); - if !target_features.is_empty() { - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); - } + to_add.extend(target_features_attr(cx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 7a340ae83f3..f9dc48e3aba 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -26,6 +26,7 @@ static LLVM_OBJECT_READER: ObjectReader = ObjectReader { get_symbols: get_llvm_object_symbols, is_64_bit_object_file: llvm_is_64_bit_object_file, is_ec_object_file: llvm_is_ec_object_file, + is_any_arm64_coff: llvm_is_any_arm64_coff, get_xcoff_member_alignment: DEFAULT_OBJECT_READER.get_xcoff_member_alignment, }; @@ -95,3 +96,7 @@ fn llvm_is_64_bit_object_file(buf: &[u8]) -> bool { fn llvm_is_ec_object_file(buf: &[u8]) -> bool { unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } + +fn llvm_is_any_arm64_coff(buf: &[u8]) -> bool { + unsafe { llvm::LLVMRustIsAnyArm64Coff(buf.as_ptr(), buf.len()) } +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index fc38c4f3e51..f571716d9dd 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -43,9 +43,7 @@ fn prepare_lto( .map(|symbol| CString::new(symbol.to_owned()).unwrap()) .collect::<Vec<CString>>(); - if cgcx.regular_module_config.instrument_coverage - || cgcx.regular_module_config.pgo_gen.enabled() - { + if cgcx.module_config.instrument_coverage || cgcx.module_config.pgo_gen.enabled() { // These are weak symbols that point to the profile version and the // profile name, which need to be treated as exported so LTO doesn't nix // them. @@ -55,15 +53,15 @@ fn prepare_lto( symbols_below_threshold.extend(PROFILER_WEAK_SYMBOLS.iter().map(|&sym| sym.to_owned())); } - if cgcx.regular_module_config.sanitizer.contains(SanitizerSet::MEMORY) { + if cgcx.module_config.sanitizer.contains(SanitizerSet::MEMORY) { let mut msan_weak_symbols = Vec::new(); // Similar to profiling, preserve weak msan symbol during LTO. - if cgcx.regular_module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { + if cgcx.module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { msan_weak_symbols.push(c"__msan_keep_going"); } - if cgcx.regular_module_config.sanitizer_memory_track_origins != 0 { + if cgcx.module_config.sanitizer_memory_track_origins != 0 { msan_weak_symbols.push(c"__msan_track_origins"); } @@ -187,12 +185,9 @@ pub(crate) fn run_thin( thin_lto(cgcx, dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold) } -pub(crate) fn prepare_thin( - module: ModuleCodegen<ModuleLlvm>, - emit_summary: bool, -) -> (String, ThinBuffer) { +pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { let name = module.name; - let buffer = ThinBuffer::new(module.module_llvm.llmod(), true, emit_summary); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -583,7 +578,7 @@ pub(crate) fn run_pass_manager( thin: bool, ) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. @@ -689,9 +684,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub(crate) fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { + pub(crate) fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); ThinBuffer(buffer) } } @@ -700,21 +695,21 @@ impl ThinBuffer { let mut ptr = NonNull::new(ptr).unwrap(); ThinBuffer(unsafe { ptr.as_mut() }) } -} -impl ThinBufferMethods for ThinBuffer { - fn data(&self) -> &[u8] { + pub(crate) fn thin_link_data(&self) -> &[u8] { unsafe { - let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferLen(self.0); + let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); slice::from_raw_parts(ptr, len) } } +} - fn thin_link_data(&self) -> &[u8] { +impl ThinBufferMethods for ThinBuffer { + fn data(&self) -> &[u8] { unsafe { - let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); + let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferLen(self.0); slice::from_raw_parts(ptr, len) } } @@ -745,7 +740,7 @@ pub(crate) fn optimize_thin_module( let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx); let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. - if cgcx.config(ModuleKind::Regular).embed_bitcode() { + if cgcx.module_config.embed_bitcode() { module.thin_lto_buffer = Some(thin_module.data().to_vec()); } { diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 7ea2ae6673b..423f0da4878 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -837,7 +837,7 @@ pub(crate) fn codegen( "LLVM_module_codegen_make_bitcode", &*module.name, ); - ThinBuffer::new(llmod, config.emit_thin_lto, false) + ThinBuffer::new(llmod, config.emit_thin_lto) }; let data = thin.data(); let _timer = cgcx diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 37379586d58..7d0691366e6 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1392,7 +1392,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, - fn_attrs: Option<&CodegenFnAttrs>, + fn_call_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1409,10 +1409,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } // Emit CFI pointer type membership test - self.cfi_type_test(fn_attrs, fn_abi, instance, llfn); + self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn); // Emit KCFI operand bundle - let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); + let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn); if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1429,6 +1429,29 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { c"".as_ptr(), ) }; + + if let Some(instance) = instance { + // Attributes on the function definition being called + let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id()); + if let Some(fn_call_attrs) = fn_call_attrs + && !fn_call_attrs.target_features.is_empty() + // If there is an inline attribute and a target feature that matches + // we will add the attribute to the callsite otherwise we'll omit + // this and not add the attribute to prevent soundness issues. + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance) + && self.cx.tcx.is_target_feature_call_safe( + &fn_call_attrs.target_features, + &fn_defn_attrs.target_features, + ) + { + attributes::apply_to_callsite( + call, + llvm::AttributePlace::Function, + &[inlining_rule], + ); + } + } + if let Some(fn_abi) = fn_abi { fn_abi.apply_attrs_callsite(self, call); } diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 1280ab1442a..0737a18384b 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -193,7 +193,7 @@ fn gen_define_handling<'ll>( // reference) types. let num_ptr_types = types .iter() - .map(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)) + .filter(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)) .count(); // We do not know their size anymore at this level, so hardcode a placeholder. diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 9ec7b0f80ae..dc9bb743560 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -452,6 +452,8 @@ impl<'ll> CodegenCx<'ll, '_> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; + + // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, g, alloc.align); llvm::set_initializer(g, v); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4fd6110ac4a..a69fa54a54a 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -217,6 +217,10 @@ pub(crate) unsafe fn create_module<'ll>( // LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010 target_data_layout = target_data_layout.replace("n8:16", "n8") } + if sess.target.arch == "nvptx64" { + // LLVM 22 updated the NVPTX layout to indicate 256-bit vector load/store: https://github.com/llvm/llvm-project/pull/155198 + target_data_layout = target_data_layout.replace("-i256:256", ""); + } } // Ensure the data-layout values hardcoded remain the defaults. @@ -849,7 +853,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { let entry_name = self.sess().target.entry_name.as_ref(); if self.get_declared_value(entry_name).is_none() { - Some(self.declare_entry_fn( + let llfn = self.declare_entry_fn( entry_name, llvm::CallConv::from_conv( self.sess().target.entry_abi, @@ -857,7 +861,13 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { ), llvm::UnnamedAddr::Global, fn_type, - )) + ); + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Function, + attributes::target_features_attr(self, vec![]).as_slice(), + ); + Some(llfn) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index caa3369f413..dc3a84b6a15 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -103,16 +103,17 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, array_type: Ty<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let ty::Array(element_type, len) = array_type.kind() else { bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) }; - let element_type_di_node = type_di_node(cx, *element_type); + let element_type_di_node = spanned_type_di_node(cx, *element_type, span); return_if_di_node_created_in_meantime!(cx, unique_type_id); - let (size, align) = cx.size_and_align_of(array_type); + let (size, align) = cx.spanned_size_and_align_of(array_type, span); let upper_bound = len .try_to_target_usize(cx.tcx) @@ -447,7 +448,7 @@ pub(crate) fn spanned_type_di_node<'ll, 'tcx>( build_basic_type_di_node(cx, t) } ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), - ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t, span), ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), @@ -1435,7 +1436,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); - let trait_ref = tcx.erase_regions(trait_ref); + let trait_ref = tcx.erase_and_anonymize_regions(trait_ref); tcx.vtable_entries(trait_ref) } else { @@ -1562,7 +1563,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( // Unwrap potential addrspacecast let vtable = find_vtable_behind_cast(vtable); let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); - let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_ref_self = cx.tcx.erase_and_anonymize_regions(trait_ref_self); let trait_def_id = trait_ref_self.def_id; let trait_vis = cx.tcx.visibility(trait_def_id); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 49d3dedbeab..85f71f331a4 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -383,7 +383,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::rotate_left | sym::rotate_right | sym::saturating_add - | sym::saturating_sub => { + | sym::saturating_sub + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { let ty = args[0].layout.ty; if !ty.is_integral() { tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { @@ -424,18 +426,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::bitreverse => { self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()]) } - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same + sym::rotate_left + | sym::rotate_right + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { + let is_left = name == sym::rotate_left || name == sym::unchecked_funnel_shl; + let lhs = args[0].immediate(); + let (rhs, raw_shift) = + if name == sym::rotate_left || name == sym::rotate_right { + // rotate = funnel shift with first two args the same + (lhs, args[1].immediate()) + } else { + (args[1].immediate(), args[2].immediate()) + }; let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' }); // llvm expects shift to be the same type as the values, but rust // always uses `u32`. - let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false); - self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift]) + self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 628cb34fd9e..6fb23d09843 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -211,11 +211,8 @@ impl WriteBackendMethods for LlvmCodegenBackend { ) -> CompiledModule { back::write::codegen(cgcx, module, config) } - fn prepare_thin( - module: ModuleCodegen<Self::Module>, - emit_summary: bool, - ) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module, emit_summary) + fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module) } fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer) { (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index ba590851dbd..0679f55ab7f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2602,7 +2602,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustThinLTOBufferCreate( M: &Module, is_thin: bool, - emit_summary: bool, ) -> &'static mut ThinLTOBuffer; pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; @@ -2686,6 +2685,8 @@ unsafe extern "C" { pub(crate) fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIsAnyArm64Coff(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustSetNoSanitizeAddress(Global: &Value); pub(crate) fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 4e7096da502..84998b5499b 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -7,6 +7,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; +use rustc_span::{DUMMY_SP, Span}; use tracing::debug; use crate::common::*; @@ -149,7 +150,11 @@ impl<'a, 'tcx> CodegenCx<'a, 'tcx> { } pub(crate) fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { - let layout = self.layout_of(ty); + self.spanned_size_and_align_of(ty, DUMMY_SP) + } + + pub(crate) fn spanned_size_and_align_of(&self, ty: Ty<'tcx>, span: Span) -> (Size, Align) { + let layout = self.spanned_layout_of(ty, span); (layout.size, layout.align.abi) } } @@ -226,7 +231,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // Make sure lifetimes are erased, to avoid generating distinct LLVM // types for Rust types that only differ in the choice of lifetimes. - let normal_ty = cx.tcx.erase_regions(self.ty); + let normal_ty = cx.tcx.erase_and_anonymize_regions(self.ty); let mut defer = None; let llty = if self.ty != normal_ty { diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ce079f3cb0a..ab08125217f 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -28,9 +28,12 @@ fn round_pointer_up_to_alignment<'ll>( align: Align, ptr_ty: &'ll Type, ) -> &'ll Value { - let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); - ptr_as_int = round_up_to_alignment(bx, ptr_as_int, align); - bx.inttoptr(ptr_as_int, ptr_ty) + let ptr = bx.inbounds_ptradd(addr, bx.const_i32(align.bytes() as i32 - 1)); + bx.call_intrinsic( + "llvm.ptrmask", + &[ptr_ty, bx.type_i32()], + &[ptr, bx.const_int(bx.isize_ty, -(align.bytes() as isize) as i64)], + ) } fn emit_direct_ptr_va_arg<'ll, 'tcx>( @@ -905,6 +908,21 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( ) } "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + "arm" => { + // Types wider than 16 bytes are not currently supported. Clang has special logic for + // such types, but `VaArgSafe` is not implemented for any type that is this large. + assert!(bx.cx.size_of(target_ty).bytes() <= 16); + + emit_ptr_va_arg( + bx, + addr, + target_ty, + PassMode::Direct, + SlotSize::Bytes4, + AllowHigherAlign::Yes, + ForceRightAdjust::No, + ) + } "s390x" => emit_s390x_va_arg(bx, addr, target_ty), "powerpc" => emit_powerpc_va_arg(bx, addr, target_ty), "powerpc64" | "powerpc64le" => emit_ptr_va_arg( |