Merge commit 'de5d6523738fd44a0521b6abf3e73ae1df210741' into sync_cg_clif-2024-04-23

11 files changed, 68 insertions, 28 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 6363a0d59a4..6f346af25c6 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -7,11 +7,13 @@ mod returning;
 use std::borrow::Cow;
 
 use cranelift_codegen::ir::SigRef;
+use cranelift_codegen::isa::CallConv;
 use cranelift_module::ModuleError;
 use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_monomorphize::is_call_from_compiler_builtins_to_upstream_monomorphization;
 use rustc_session::Session;
 use rustc_span::source_map::Spanned;
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index f428c4c7c0d..e3d050df4cd 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -2,12 +2,14 @@
 
 use cranelift_codegen::ir::UserFuncName;
 use cranelift_codegen::CodegenError;
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
 use cranelift_module::ModuleError;
 use rustc_ast::InlineAsmOptions;
 use rustc_index::IndexVec;
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_monomorphize::is_call_from_compiler_builtins_to_upstream_monomorphization;
 
 use crate::constant::ConstantCx;
@@ -823,7 +825,13 @@ fn codegen_stmt<'tcx>(
                     };
                     let data = codegen_operand(fx, data);
                     let meta = codegen_operand(fx, meta);
-                    let ptr_val = CValue::pointer_from_data_and_meta(data, meta, layout);
+                    assert!(data.layout().ty.is_unsafe_ptr());
+                    assert!(layout.ty.is_unsafe_ptr());
+                    let ptr_val = if meta.layout().is_zst() {
+                        data.cast_pointer_to(layout)
+                    } else {
+                        CValue::by_val_pair(data.load_scalar(fx), meta.load_scalar(fx), layout)
+                    };
                     lval.write_cvalue(fx, ptr_val);
                 }
                 Rvalue::Aggregate(ref kind, ref operands) => {
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index cf0b065414d..2a24d6150bd 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -1,8 +1,10 @@
 use cranelift_codegen::isa::TargetFrontendConfig;
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
 use rustc_index::IndexVec;
 use rustc_middle::ty::layout::{
-    FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
+    self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
 };
+use rustc_middle::ty::TypeFoldable;
 use rustc_span::source_map::Spanned;
 use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{Integer, Primitive};
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index cb05c17ec2a..cdf499a22f8 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -137,18 +137,23 @@ pub(crate) fn codegen_const_value<'tcx>(
                 let alloc_id = prov.alloc_id();
                 let base_addr = match fx.tcx.global_alloc(alloc_id) {
                     GlobalAlloc::Memory(alloc) => {
-                        let data_id = data_id_for_alloc_id(
-                            &mut fx.constants_cx,
-                            fx.module,
-                            alloc_id,
-                            alloc.inner().mutability,
-                        );
-                        let local_data_id =
-                            fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                        if fx.clif_comments.enabled() {
-                            fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+                        if alloc.inner().len() == 0 {
+                            assert_eq!(offset, Size::ZERO);
+                            fx.bcx.ins().iconst(fx.pointer_type, alloc.inner().align.bytes() as i64)
+                        } else {
+                            let data_id = data_id_for_alloc_id(
+                                &mut fx.constants_cx,
+                                fx.module,
+                                alloc_id,
+                                alloc.inner().mutability,
+                            );
+                            let local_data_id =
+                                fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+                            }
+                            fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
-                        fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                     }
                     GlobalAlloc::Function(instance) => {
                         let func_id = crate::abi::import_function(fx.tcx, fx.module, instance);
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 5d943b5d996..f0b78e5d7c6 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -19,7 +19,7 @@ use rustc_codegen_ssa::debuginfo::type_names;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefIdMap;
 use rustc_session::Session;
-use rustc_span::{SourceFileHash, StableSourceFileId};
+use rustc_span::{FileNameDisplayPreference, SourceFileHash, StableSourceFileId};
 use rustc_target::abi::call::FnAbi;
 
 pub(crate) use self::emit::{DebugReloc, DebugRelocName};
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
index 27eabd8a0a6..65f4c67b21f 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
@@ -2,7 +2,7 @@ use cranelift_module::{DataId, FuncId};
 use cranelift_object::ObjectProduct;
 use gimli::SectionId;
 use object::write::{Relocation, StandardSegment};
-use object::{RelocationEncoding, SectionKind};
+use object::{RelocationEncoding, RelocationFlags, SectionKind};
 use rustc_data_structures::fx::FxHashMap;
 
 use crate::debuginfo::{DebugReloc, DebugRelocName};
@@ -72,9 +72,11 @@ impl WriteDebugInfo for ObjectProduct {
                 Relocation {
                     offset: u64::from(reloc.offset),
                     symbol,
-                    kind: reloc.kind,
-                    encoding: RelocationEncoding::Generic,
-                    size: reloc.size * 8,
+                    flags: RelocationFlags::Generic {
+                        kind: reloc.kind,
+                        encoding: RelocationEncoding::Generic,
+                        size: reloc.size * 8,
+                    },
                     addend: i64::try_from(symbol_offset).unwrap() + reloc.addend,
                 },
             )
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 6dbc3f19127..929fa92596d 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -6,6 +6,7 @@ use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 use std::sync::{mpsc, Mutex, OnceLock};
 
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
 use cranelift_jit::{JITBuilder, JITModule};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 171ee88a11c..28b92f730da 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -2,6 +2,7 @@
 
 use std::fmt::Write;
 
+use cranelift_codegen::isa::CallConv;
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_span::sym;
 use rustc_target::asm::*;
@@ -785,9 +786,9 @@ fn call_inline_asm<'tcx>(
     for (offset, place) in outputs {
         let ty = if place.layout().ty.is_simd() {
             let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
-            fx.clif_type(lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
+            asm_clif_type(fx, lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
         } else {
-            fx.clif_type(place.layout().ty).unwrap()
+            asm_clif_type(fx, place.layout().ty).unwrap()
         };
         let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
             fx,
@@ -797,3 +798,24 @@ fn call_inline_asm<'tcx>(
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
+
+fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<types::Type> {
+    match ty.kind() {
+        // Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151
+        ty::Adt(adt, args) if Some(adt.did()) == fx.tcx.lang_items().maybe_uninit() => {
+            let fields = &adt.non_enum_variant().fields;
+            let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args);
+            let ty::Adt(ty, args) = ty.kind() else {
+                unreachable!("expected first field of `MaybeUninit` to be an ADT")
+            };
+            assert!(
+                ty.is_manually_drop(),
+                "expected first field of `MaybeUninit` to be `ManuallyDrop`"
+            );
+            let fields = &ty.non_enum_variant().fields;
+            let ty = fields[FieldIdx::ZERO].ty(fx.tcx, args);
+            fx.clif_type(ty)
+        }
+        _ => fx.clif_type(ty),
+    }
+}
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index d0ab64a5584..c9f84b69997 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -87,21 +87,17 @@ mod prelude {
         AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, StackSlot,
         StackSlotData, StackSlotKind, TrapCode, Type, Value,
     };
-    pub(crate) use cranelift_codegen::isa::{self, CallConv};
     pub(crate) use cranelift_codegen::Context;
-    pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
     pub(crate) use cranelift_module::{self, DataDescription, FuncId, Linkage, Module};
     pub(crate) use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
     pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
     pub(crate) use rustc_index::Idx;
-    pub(crate) use rustc_middle::bug;
     pub(crate) use rustc_middle::mir::{self, *};
-    pub(crate) use rustc_middle::ty::layout::{self, LayoutOf, TyAndLayout};
+    pub(crate) use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
     pub(crate) use rustc_middle::ty::{
-        self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
-        TypeFoldable, TypeVisitableExt, UintTy,
+        self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut, UintTy,
     };
-    pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
+    pub(crate) use rustc_span::Span;
     pub(crate) use rustc_target::abi::{Abi, FieldIdx, Scalar, Size, VariantIdx, FIRST_VARIANT};
 
     pub(crate) use crate::abi::*;
@@ -261,7 +257,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
     }
 }
 
-fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::TargetIsa + 'static> {
+fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIsa + 'static> {
     use target_lexicon::BinaryFormat;
 
     let target_triple = crate::target_triple(sess);
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index 1abfded8b11..1f20ec42ddb 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -1,3 +1,4 @@
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
 use rustc_hir::LangItem;
 use rustc_middle::ty::AssocKind;
 use rustc_middle::ty::GenericArg;
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 8d52fd9d7a8..38fedb6036c 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -2,6 +2,7 @@
 
 use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
+use cranelift_frontend::Variable;
 use rustc_middle::ty::FnSig;
 
 use crate::prelude::*;