diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
18 files changed, 463 insertions, 175 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 0497c2570e6..99059e788a0 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -465,7 +465,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); - (CallTarget::Indirect(sig, method), Some(ptr)) + (CallTarget::Indirect(sig, method), Some(ptr.get_addr(fx))) } // Normal call @@ -560,7 +560,19 @@ pub(crate) fn codegen_drop<'tcx>( // we don't actually need to drop anything } else { match ty.kind() { - ty::Dynamic(..) => { + ty::Dynamic(_, _, ty::Dyn) => { + // IN THIS ARM, WE HAVE: + // ty = *mut (dyn Trait) + // which is: exists<T> ( *mut T, Vtable<T: Trait> ) + // args[0] args[1] + // + // args = ( Data, Vtable ) + // | + // v + // /-------\ + // | ... | + // \-------/ + // let (ptr, vtable) = drop_place.to_ptr_maybe_unsized(); let ptr = ptr.get_addr(fx); let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap()); @@ -578,6 +590,43 @@ pub(crate) fn codegen_drop<'tcx>( let sig = fx.bcx.import_signature(sig); fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); } + ty::Dynamic(_, _, ty::DynStar) => { + // IN THIS ARM, WE HAVE: + // ty = *mut (dyn* Trait) + // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) + // + // args = [ * ] + // | + // v + // ( Data, Vtable ) + // | + // v + // /-------\ + // | ... | + // \-------/ + // + // + // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING + // + // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) + // vtable = (*args[0]).1 // loads the vtable out + // (data, vtable) // an equivalent Rust `*mut dyn Trait` + // + // SO THEN WE CAN USE THE ABOVE CODE. + let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx); + let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); + + let virtual_drop = Instance { + def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0), + substs: drop_instance.substs, + }; + let fn_abi = + RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty()); + + let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); + let sig = fx.bcx.import_signature(sig); + fx.bcx.ins().call_indirect(sig, drop_fn, &[data]); + } _ => { assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _))); diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index 6d321c7b298..bad8a87b9be 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -78,7 +78,7 @@ fn codegen_inner( let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone()); + ctx.func.signature = sig.clone(); { let mut func_ctx = FunctionBuilderContext::new(); let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); @@ -116,7 +116,7 @@ fn codegen_inner( let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig); + ctx.func.signature = sig; { let mut func_ctx = FunctionBuilderContext::new(); let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs index b4c79096170..31d3d0e0615 100644 --- a/compiler/rustc_codegen_cranelift/src/archive.rs +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -159,6 +159,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { let err = err.to_string(); if err == "Unknown file magic" { // Not an object file; skip it. + } else if object::read::archive::ArchiveFile::parse(&*data).is_ok() { + // Nested archive file; skip it. } else { sess.fatal(&format!( "error parsing `{}` during archive creation: {}", diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 4303d63fe21..a41b561598f 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -6,6 +6,8 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; +use cranelift_codegen::ir::UserFuncName; + use crate::constant::ConstantCx; use crate::debuginfo::FunctionDebugContext; use crate::prelude::*; @@ -64,7 +66,7 @@ pub(crate) fn codegen_fn<'tcx>( let mut func_ctx = FunctionBuilderContext::new(); let mut func = cached_func; func.clear(); - func.name = ExternalName::user(0, func_id.as_u32()); + func.name = UserFuncName::user(0, func_id.as_u32()); func.signature = sig; func.collect_debug_info(); @@ -706,9 +708,9 @@ fn codegen_stmt<'tcx>( let operand = codegen_operand(fx, operand); operand.unsize_value(fx, lval); } - Rvalue::Cast(CastKind::DynStar, _, _) => { - // FIXME(dyn-star) - unimplemented!() + Rvalue::Cast(CastKind::DynStar, ref operand, _) => { + let operand = codegen_operand(fx, operand); + operand.coerce_dyn_star(fx, lval); } Rvalue::Discriminant(place) => { let place = codegen_place(fx, place); @@ -922,7 +924,7 @@ pub(crate) fn codegen_operand<'tcx>( let cplace = codegen_place(fx, *place); cplace.to_cvalue(fx) } - Operand::Constant(const_) => crate::constant::codegen_constant(fx, const_), + Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_), } } diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs index dfde9792046..f855e20e0a1 100644 --- a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs +++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs @@ -10,6 +10,7 @@ pub(super) struct ConcurrencyLimiter { helper_thread: Option<HelperThread>, state: Arc<Mutex<state::ConcurrencyLimiterState>>, available_token_condvar: Arc<Condvar>, + finished: bool, } impl ConcurrencyLimiter { @@ -32,6 +33,7 @@ impl ConcurrencyLimiter { helper_thread: Some(helper_thread), state, available_token_condvar: Arc::new(Condvar::new()), + finished: false, } } @@ -56,16 +58,23 @@ impl ConcurrencyLimiter { let mut state = self.state.lock().unwrap(); state.job_already_done(); } -} -impl Drop for ConcurrencyLimiter { - fn drop(&mut self) { - // + pub(crate) fn finished(mut self) { self.helper_thread.take(); // Assert that all jobs have finished let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap(); state.assert_done(); + + self.finished = true; + } +} + +impl Drop for ConcurrencyLimiter { + fn drop(&mut self) { + if !self.finished && !std::thread::panicking() { + panic!("Forgot to call finished() on ConcurrencyLimiter"); + } } } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index c5f44bb8479..d4bc3543b2d 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -7,7 +7,6 @@ use rustc_middle::mir::interpret::{ }; use rustc_span::DUMMY_SP; -use cranelift_codegen::ir::GlobalValueData; use cranelift_module::*; use crate::prelude::*; @@ -81,53 +80,46 @@ pub(crate) fn codegen_tls_ref<'tcx>( CValue::by_val(tls_ptr, layout) } -fn codegen_static_ref<'tcx>( - fx: &mut FunctionCx<'_, '_, 'tcx>, - def_id: DefId, - layout: TyAndLayout<'tcx>, -) -> CPlace<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); - let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - if fx.clif_comments.enabled() { - fx.add_comment(local_data_id, format!("{:?}", def_id)); - } - let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); - assert!(!layout.is_unsized(), "unsized statics aren't supported"); - assert!( - matches!( - fx.bcx.func.global_values[local_data_id], - GlobalValueData::Symbol { tls: false, .. } - ), - "tls static referenced without Rvalue::ThreadLocalRef" - ); - CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) -} - -pub(crate) fn codegen_constant<'tcx>( +pub(crate) fn eval_mir_constant<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, constant: &Constant<'tcx>, -) -> CValue<'tcx> { - let (const_val, ty) = match fx.monomorphize(constant.literal) { - ConstantKind::Ty(const_) => unreachable!("{:?}", const_), - ConstantKind::Unevaluated(mir::UnevaluatedConst { def, substs, promoted }, ty) +) -> (ConstValue<'tcx>, Ty<'tcx>) { + let constant_kind = fx.monomorphize(constant.literal); + let uv = match constant_kind { + ConstantKind::Ty(const_) => match const_.kind() { + ty::ConstKind::Unevaluated(uv) => uv.expand(), + ty::ConstKind::Value(val) => { + return (fx.tcx.valtree_to_const_val((const_.ty(), val)), const_.ty()); + } + err => span_bug!( + constant.span, + "encountered bad ConstKind after monomorphizing: {:?}", + err + ), + }, + ConstantKind::Unevaluated(mir::UnevaluatedConst { def, .. }, _) if fx.tcx.is_static(def.did) => { - assert!(substs.is_empty()); - assert!(promoted.is_none()); - - return codegen_static_ref(fx, def.did, fx.layout_of(ty)).to_cvalue(fx); - } - ConstantKind::Unevaluated(unevaluated, ty) => { - match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) { - Ok(const_val) => (const_val, ty), - Err(_) => { - span_bug!(constant.span, "erroneous constant not captured by required_consts"); - } - } + span_bug!(constant.span, "MIR constant refers to static"); } - ConstantKind::Val(val, ty) => (val, ty), + ConstantKind::Unevaluated(uv, _) => uv, + ConstantKind::Val(val, _) => return (val, constant_kind.ty()), }; + ( + fx.tcx.const_eval_resolve(ty::ParamEnv::reveal_all(), uv, None).unwrap_or_else(|_err| { + span_bug!(constant.span, "erroneous constant not captured by required_consts"); + }), + constant_kind.ty(), + ) +} + +pub(crate) fn codegen_constant_operand<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + constant: &Constant<'tcx>, +) -> CValue<'tcx> { + let (const_val, ty) = eval_mir_constant(fx, constant); + codegen_const_value(fx, const_val, ty) } @@ -244,7 +236,7 @@ pub(crate) fn codegen_const_value<'tcx>( } } -pub(crate) fn pointer_for_allocation<'tcx>( +fn pointer_for_allocation<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, alloc: ConstAllocation<'tcx>, ) -> crate::pointer::Pointer { @@ -467,14 +459,13 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( operand: &Operand<'tcx>, ) -> Option<ConstValue<'tcx>> { match operand { - Operand::Constant(const_) => match const_.literal { - ConstantKind::Ty(const_) => fx - .monomorphize(const_) - .eval_for_mir(fx.tcx, ParamEnv::reveal_all()) - .try_to_value(fx.tcx), + Operand::Constant(const_) => match fx.monomorphize(const_.literal) { + ConstantKind::Ty(const_) => Some( + const_.eval_for_mir(fx.tcx, ParamEnv::reveal_all()).try_to_value(fx.tcx).unwrap(), + ), ConstantKind::Val(val, _) => Some(val), ConstantKind::Unevaluated(uv, _) => { - fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).ok() + Some(fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).unwrap()) } }, // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 8eabe1cbcb1..f873561c171 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -106,7 +106,7 @@ impl OngoingCodegen { } } - drop(self.concurrency_limiter); + self.concurrency_limiter.finished(); ( CodegenResults { diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 0e77e4004c0..6a430b5215e 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -67,13 +67,12 @@ fn create_jit_module( hotswap: bool, ) -> (JITModule, CodegenCx) { let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); - let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info); let isa = crate::build_isa(tcx.sess, backend_config); let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); - jit_builder.symbols(imported_symbols); + jit_builder.symbol_lookup_fn(dep_symbol_lookup_fn(tcx.sess, crate_info)); jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8); let mut jit_module = JITModule::new(jit_builder); @@ -286,10 +285,10 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> }) } -fn load_imported_symbols_for_jit( +fn dep_symbol_lookup_fn( sess: &Session, crate_info: CrateInfo, -) -> Vec<(String, *const u8)> { +) -> Box<dyn Fn(&str) -> Option<*const u8>> { use rustc_middle::middle::dependency_format::Linkage; let mut dylib_paths = Vec::new(); @@ -316,39 +315,23 @@ fn load_imported_symbols_for_jit( } } - let mut imported_symbols = Vec::new(); - for path in dylib_paths { - use object::{Object, ObjectSymbol}; - let lib = libloading::Library::new(&path).unwrap(); - let obj = std::fs::read(path).unwrap(); - let obj = object::File::parse(&*obj).unwrap(); - imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { - let name = symbol.name().unwrap().to_string(); - if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { - return None; - } - if name.starts_with("rust_metadata_") { - // The metadata is part of a section that is not loaded by the dynamic linker in - // case of cg_llvm. - return None; - } - let dlsym_name = if cfg!(target_os = "macos") { - // On macOS `dlsym` expects the name without leading `_`. - assert!(name.starts_with('_'), "{:?}", name); - &name[1..] - } else { - &name - }; - let symbol: libloading::Symbol<'_, *const u8> = - unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap(); - Some((name, *symbol)) - })); - std::mem::forget(lib) - } + let imported_dylibs = Box::leak( + dylib_paths + .into_iter() + .map(|path| unsafe { libloading::Library::new(&path).unwrap() }) + .collect::<Box<[_]>>(), + ); sess.abort_if_errors(); - imported_symbols + Box::new(move |sym_name| { + for dylib in &*imported_dylibs { + if let Ok(sym) = unsafe { dylib.get::<*const u8>(sym_name.as_bytes()) } { + return Some(*sym); + } + } + None + }) } fn codegen_shim<'tcx>( diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 8b3d475cb18..3fcc84d3929 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -27,7 +27,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( } // Used by stdarch - if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) + if template[0] == InlineAsmTemplatePiece::String("mov ".to_string()) && matches!( template[1], InlineAsmTemplatePiece::Placeholder { @@ -36,24 +36,26 @@ pub(crate) fn codegen_inline_asm<'tcx>( span: _ } ) - && template[2] == InlineAsmTemplatePiece::String("\n".to_string()) - && template[3] == InlineAsmTemplatePiece::String("cpuid".to_string()) - && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) - && template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string()) + && template[2] == InlineAsmTemplatePiece::String(", rbx".to_string()) + && template[3] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[4] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[5] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[6] == InlineAsmTemplatePiece::String("xchg ".to_string()) && matches!( - template[6], + template[7], InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } ) + && template[8] == InlineAsmTemplatePiece::String(", rbx".to_string()) { assert_eq!(operands.len(), 4); let (leaf, eax_place) = match operands[1] { InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), - late: true, + late: _, ref in_value, out_place: Some(out_place), } => ( @@ -68,7 +70,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( X86InlineAsmRegClass::reg, )), - late: true, + late: _, place: Some(place), } => crate::base::codegen_place(fx, place), _ => unreachable!(), @@ -76,7 +78,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( let (sub_leaf, ecx_place) = match operands[2] { InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), - late: true, + late: _, ref in_value, out_place: Some(out_place), } => ( @@ -88,7 +90,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( let edx_place = match operands[3] { InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), - late: true, + late: _, place: Some(place), } => crate::base::codegen_place(fx, place), _ => unreachable!(), diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index a799dca938e..783d426c30b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -14,6 +14,10 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( target: Option<BasicBlock>, ) { match intrinsic { + "llvm.x86.sse2.pause" | "llvm.aarch64.isb" => { + // Spin loop hint + } + // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd" => { intrinsic_args!(fx, args => (a); intrinsic); @@ -25,8 +29,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let mut res = fx.bcx.ins().iconst(types::I32, 0); for lane in (0..lane_count).rev() { - let a_lane = - a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx); + let a_lane = a.value_lane(fx, lane).load_scalar(fx); // cast float to int let a_lane = match lane_ty { diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 2e4ca594f91..0302b843aa2 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -84,6 +84,30 @@ fn simd_for_each_lane<'tcx>( } } +fn simd_pair_for_each_lane_typed<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + x: CValue<'tcx>, + y: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, CValue<'tcx>, CValue<'tcx>) -> CValue<'tcx>, +) { + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, _ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let x_lane = x.value_lane(fx, lane_idx); + let y_lane = y.value_lane(fx, lane_idx); + + let res_lane = f(fx, x_lane, y_lane); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + fn simd_pair_for_each_lane<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, x: CValue<'tcx>, @@ -504,37 +528,7 @@ fn codegen_regular_intrinsic_call<'tcx>( _ => unreachable!(), }; - let signed = type_sign(lhs.layout().ty); - - let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs); - - let (val, has_overflow) = checked_res.load_scalar_pair(fx); - let clif_ty = fx.clif_type(lhs.layout().ty).unwrap(); - - let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); - - let val = match (intrinsic, signed) { - (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val), - (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), - (sym::saturating_add, true) => { - let rhs = rhs.load_scalar(fx); - let rhs_ge_zero = - fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); - let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); - fx.bcx.ins().select(has_overflow, sat_val, val) - } - (sym::saturating_sub, true) => { - let rhs = rhs.load_scalar(fx); - let rhs_ge_zero = - fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); - let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); - fx.bcx.ins().select(has_overflow, sat_val, val) - } - _ => unreachable!(), - }; - - let res = CValue::by_val(val, lhs.layout()); - + let res = crate::num::codegen_saturating_int_binop(fx, bin_op, lhs, rhs); ret.write_cvalue(fx, res); } sym::rotate_left => { @@ -819,8 +813,8 @@ fn codegen_regular_intrinsic_call<'tcx>( sym::ptr_guaranteed_cmp => { intrinsic_args!(fx, args => (a, b); intrinsic); - let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b); - ret.write_cvalue(fx, val); + let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b).load_scalar(fx); + ret.write_cvalue(fx, CValue::by_val(val, fx.layout_of(fx.tcx.types.u8))); } sym::caller_location => { @@ -1206,7 +1200,7 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: fx.target_config.default_call_conv, - params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))], + params: vec![AbiParam::new(pointer_ty(fx.tcx))], returns: vec![], }); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 1f358b1bbb9..51fce8c854b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -2,6 +2,7 @@ use rustc_middle::ty::subst::SubstsRef; use rustc_span::Symbol; +use rustc_target::abi::Endian; use super::*; use crate::prelude::*; @@ -26,7 +27,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( span: Span, ) { match intrinsic { - sym::simd_cast => { + sym::simd_as | sym::simd_cast => { intrinsic_args!(fx, args => (a); intrinsic); if !a.layout().ty.is_simd() { @@ -162,6 +163,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } } } else { + // FIXME remove this case intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap() }; @@ -650,8 +652,128 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } } - // simd_saturating_* - // simd_bitmask + sym::simd_select_bitmask => { + intrinsic_args!(fx, args => (m, a, b); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + assert_eq!(a.layout(), b.layout()); + + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + + let m = m.load_scalar(fx); + + for lane in 0..lane_count { + let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64); + let m_lane = fx.bcx.ins().band_imm(m_lane, 1); + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + let b_lane = b.value_lane(fx, lane).load_scalar(fx); + + let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); + let res_lane = + CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); + + ret.place_lane(fx, lane).write_cvalue(fx, res_lane); + } + } + + sym::simd_bitmask => { + intrinsic_args!(fx, args => (a); intrinsic); + + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_clif_ty = fx.clif_type(lane_ty).unwrap(); + + // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. + let expected_int_bits = lane_count.max(8); + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); + + match lane_ty.kind() { + ty::Int(_) | ty::Uint(_) => {} + _ => { + fx.tcx.sess.span_fatal( + span, + &format!( + "invalid monomorphization of `simd_bitmask` intrinsic: \ + vector argument `{}`'s element type `{}`, expected integer element \ + type", + a.layout().ty, + lane_ty + ), + ); + } + } + + let res_type = + Type::int_with_byte_size(u16::try_from(expected_bytes).unwrap()).unwrap(); + let mut res = fx.bcx.ins().iconst(res_type, 0); + + let lanes = match fx.tcx.sess.target.endian { + Endian::Big => Box::new(0..lane_count) as Box<dyn Iterator<Item = u64>>, + Endian::Little => Box::new((0..lane_count).rev()) as Box<dyn Iterator<Item = u64>>, + }; + for lane in lanes { + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + + // extract sign bit of an int + let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1)); + + // shift sign bit into result + let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false); + res = fx.bcx.ins().ishl_imm(res, 1); + res = fx.bcx.ins().bor(res, a_lane_sign); + } + + match ret.layout().ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {} + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => {} + _ => { + fx.tcx.sess.span_fatal( + span, + &format!( + "invalid monomorphization of `simd_bitmask` intrinsic: \ + cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret.layout().ty, + expected_int_bits, + expected_bytes + ), + ); + } + } + + let res = CValue::by_val(res, ret.layout()); + ret.write_cvalue(fx, res); + } + + sym::simd_saturating_add | sym::simd_saturating_sub => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + let bin_op = match intrinsic { + sym::simd_saturating_add => BinOp::Add, + sym::simd_saturating_sub => BinOp::Sub, + _ => unreachable!(), + }; + + // FIXME use vector instructions when possible + simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| { + crate::num::codegen_saturating_int_binop(fx, bin_op, x_lane, y_lane) + }); + } + + // simd_arith_offset // simd_scatter // simd_gather _ => { diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 913414e7618..629d79d5012 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -96,8 +96,8 @@ mod prelude { pub(crate) use cranelift_codegen::ir::function::Function; pub(crate) use cranelift_codegen::ir::types; pub(crate) use cranelift_codegen::ir::{ - AbiParam, Block, ExternalName, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, - StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value, + AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, StackSlot, + StackSlotData, StackSlotKind, TrapCode, Type, Value, }; pub(crate) use cranelift_codegen::isa::{self, CallConv}; pub(crate) use cranelift_codegen::Context; @@ -251,7 +251,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); - flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; flags_builder.set("enable_verifier", enable_verifier).unwrap(); flags_builder.set("regalloc_checker", enable_verifier).unwrap(); @@ -279,6 +278,15 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar } } + if target_triple.architecture == target_lexicon::Architecture::X86_64 { + // Windows depends on stack probes to grow the committed part of the stack + flags_builder.enable("enable_probestack").unwrap(); + flags_builder.set("probestack_strategy", "inline").unwrap(); + } else { + // __cranelift_probestack is not provided and inline stack probes are only supported on x86_64 + flags_builder.set("enable_probestack", "false").unwrap(); + } + let flags = settings::Flags::new(flags_builder); let isa_builder = match sess.opts.cg.target_cpu.as_deref() { diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs index 3c024a84d90..cae6312a607 100644 --- a/compiler/rustc_codegen_cranelift/src/main_shim.rs +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -75,7 +75,7 @@ pub(crate) fn maybe_create_entry_wrapper( let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig); + ctx.func.signature = cmain_sig; { let mut func_ctx = FunctionBuilderContext::new(); let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index 4ce8adb182e..ecbab408ded 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -150,18 +150,12 @@ pub(crate) fn codegen_int_binop<'tcx>( BinOp::BitXor => b.bxor(lhs, rhs), BinOp::BitAnd => b.band(lhs, rhs), BinOp::BitOr => b.bor(lhs, rhs), - BinOp::Shl => { - let lhs_ty = fx.bcx.func.dfg.value_type(lhs); - let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); - fx.bcx.ins().ishl(lhs, actual_shift) - } + BinOp::Shl => b.ishl(lhs, rhs), BinOp::Shr => { - let lhs_ty = fx.bcx.func.dfg.value_type(lhs); - let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); if signed { - fx.bcx.ins().sshr(lhs, actual_shift) + b.sshr(lhs, rhs) } else { - fx.bcx.ins().ushr(lhs, actual_shift) + b.ushr(lhs, rhs) } } // Compare binops handles by `codegen_binop`. @@ -279,22 +273,15 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( } } BinOp::Shl => { - let lhs_ty = fx.bcx.func.dfg.value_type(lhs); - let masked_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); - let val = fx.bcx.ins().ishl(lhs, masked_shift); + let val = fx.bcx.ins().ishl(lhs, rhs); let ty = fx.bcx.func.dfg.value_type(val); let max_shift = i64::from(ty.bits()) - 1; let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift); (val, has_overflow) } BinOp::Shr => { - let lhs_ty = fx.bcx.func.dfg.value_type(lhs); - let masked_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); - let val = if !signed { - fx.bcx.ins().ushr(lhs, masked_shift) - } else { - fx.bcx.ins().sshr(lhs, masked_shift) - }; + let val = + if !signed { fx.bcx.ins().ushr(lhs, rhs) } else { fx.bcx.ins().sshr(lhs, rhs) }; let ty = fx.bcx.func.dfg.value_type(val); let max_shift = i64::from(ty.bits()) - 1; let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift); @@ -309,6 +296,42 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( CValue::by_val_pair(res, has_overflow, out_layout) } +pub(crate) fn codegen_saturating_int_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + lhs: CValue<'tcx>, + rhs: CValue<'tcx>, +) -> CValue<'tcx> { + assert_eq!(lhs.layout().ty, rhs.layout().ty); + + let signed = type_sign(lhs.layout().ty); + let clif_ty = fx.clif_type(lhs.layout().ty).unwrap(); + let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); + + let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs); + let (val, has_overflow) = checked_res.load_scalar_pair(fx); + + let val = match (bin_op, signed) { + (BinOp::Add, false) => fx.bcx.ins().select(has_overflow, max, val), + (BinOp::Sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (BinOp::Add, true) => { + let rhs = rhs.load_scalar(fx); + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); + fx.bcx.ins().select(has_overflow, sat_val, val) + } + (BinOp::Sub, true) => { + let rhs = rhs.load_scalar(fx); + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); + fx.bcx.ins().select(has_overflow, sat_val, val) + } + _ => unreachable!(), + }; + + CValue::by_val(val, lhs.layout()) +} + pub(crate) fn codegen_float_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs index dd9d891ddbd..9c88f7dbcda 100644 --- a/compiler/rustc_codegen_cranelift/src/unsize.rs +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -25,7 +25,12 @@ pub(crate) fn unsized_info<'tcx>( .bcx .ins() .iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64), - (&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => { + ( + &ty::Dynamic(ref data_a, _, src_dyn_kind), + &ty::Dynamic(ref data_b, _, target_dyn_kind), + ) => { + assert_eq!(src_dyn_kind, target_dyn_kind); + let old_info = old_info.expect("unsized_info: missing old info for trait upcasting coercion"); if data_a.principal_def_id() == data_b.principal_def_id() { @@ -101,6 +106,21 @@ fn unsize_ptr<'tcx>( } } +/// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type. +pub(crate) fn cast_to_dyn_star<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + src: Value, + src_ty_and_layout: TyAndLayout<'tcx>, + dst_ty: Ty<'tcx>, + old_info: Option<Value>, +) -> (Value, Value) { + assert!( + matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)), + "destination type must be a dyn*" + ); + (src, unsized_info(fx, src_ty_and_layout.ty, dst_ty, old_info)) +} + /// Coerce `src`, which is a reference to a value of type `src_ty`, /// to a value of type `dst_ty` and store the result in `dst` pub(crate) fn coerce_unsized_into<'tcx>( @@ -147,6 +167,24 @@ pub(crate) fn coerce_unsized_into<'tcx>( } } +pub(crate) fn coerce_dyn_star<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + src: CValue<'tcx>, + dst: CPlace<'tcx>, +) { + let (data, extra) = if let ty::Dynamic(_, _, ty::DynStar) = src.layout().ty.kind() { + let (data, vtable) = src.load_scalar_pair(fx); + (data, Some(vtable)) + } else { + let data = src.load_scalar(fx); + (data, None) + }; + + let (data, vtable) = cast_to_dyn_star(fx, data, src.layout(), dst.layout().ty, extra); + + dst.write_cvalue(fx, CValue::by_val_pair(data, vtable, dst.layout())); +} + // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs pub(crate) fn size_and_align_of_dst<'tcx>( diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index 3fa3e3657cb..c3dfbd37279 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -107,6 +107,50 @@ impl<'tcx> CValue<'tcx> { } } + // FIXME remove + // Forces the data value of a dyn* value to the stack and returns a pointer to it as well as the + // vtable pointer. + pub(crate) fn dyn_star_force_data_on_stack( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + ) -> (Value, Value) { + assert!(self.1.ty.is_dyn_star()); + + match self.0 { + CValueInner::ByRef(ptr, None) => { + let (a_scalar, b_scalar) = match self.1.abi { + Abi::ScalarPair(a, b) => (a, b), + _ => unreachable!("dyn_star_force_data_on_stack({:?})", self), + }; + let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); + let mut flags = MemFlags::new(); + flags.set_notrap(); + let vtable = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags); + (ptr.get_addr(fx), vtable) + } + CValueInner::ByValPair(data, vtable) => { + let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (u32::try_from(fx.target_config.pointer_type().bytes()).unwrap() + 15) + / 16 + * 16, + }); + let data_ptr = Pointer::stack_slot(stack_slot); + let mut flags = MemFlags::new(); + flags.set_notrap(); + data_ptr.store(fx, data, flags); + + (data_ptr.get_addr(fx), vtable) + } + CValueInner::ByRef(_, Some(_)) | CValueInner::ByVal(_) => { + unreachable!("dyn_star_force_data_on_stack({:?})", self) + } + } + } + pub(crate) fn try_to_ptr(self) -> Option<(Pointer, Option<Value>)> { match self.0 { CValueInner::ByRef(ptr, meta) => Some((ptr, meta)), @@ -236,6 +280,10 @@ impl<'tcx> CValue<'tcx> { crate::unsize::coerce_unsized_into(fx, self, dest); } + pub(crate) fn coerce_dyn_star(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) { + crate::unsize::coerce_dyn_star(fx, self, dest); + } + /// If `ty` is signed, `const_val` must already be sign extended. pub(crate) fn const_val( fx: &mut FunctionCx<'_, '_, 'tcx>, diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs index 36b3725ef42..f04fb82de8c 100644 --- a/compiler/rustc_codegen_cranelift/src/vtable.rs +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -45,12 +45,26 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, arg: CValue<'tcx>, idx: usize, -) -> (Value, Value) { - let (ptr, vtable) = if let Abi::ScalarPair(_, _) = arg.layout().abi { - arg.load_scalar_pair(fx) - } else { - let (ptr, vtable) = arg.try_to_ptr().unwrap(); - (ptr.get_addr(fx), vtable.unwrap()) +) -> (Pointer, Value) { + let (ptr, vtable) = 'block: { + if let ty::Ref(_, ty, _) = arg.layout().ty.kind() { + if ty.is_dyn_star() { + let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap().ty); + let dyn_star = CPlace::for_ptr(Pointer::new(arg.load_scalar(fx)), inner_layout); + let ptr = dyn_star.place_field(fx, mir::Field::new(0)).to_ptr(); + let vtable = + dyn_star.place_field(fx, mir::Field::new(1)).to_cvalue(fx).load_scalar(fx); + break 'block (ptr, vtable); + } + } + + if let Abi::ScalarPair(_, _) = arg.layout().abi { + let (ptr, vtable) = arg.load_scalar_pair(fx); + (Pointer::new(ptr), vtable) + } else { + let (ptr, vtable) = arg.try_to_ptr().unwrap(); + (ptr, vtable.unwrap()) + } }; let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes(); |