diff options
| -rw-r--r-- | src/abi/mod.rs | 19 | ||||
| -rw-r--r-- | src/abi/returning.rs | 51 | ||||
| -rw-r--r-- | src/base.rs | 10 | ||||
| -rw-r--r-- | src/constant.rs | 4 | ||||
| -rw-r--r-- | src/driver/aot.rs | 33 | ||||
| -rw-r--r-- | src/intrinsics/llvm.rs | 7 | ||||
| -rw-r--r-- | src/intrinsics/mod.rs | 70 |
7 files changed, 99 insertions, 95 deletions
diff --git a/src/abi/mod.rs b/src/abi/mod.rs index b163a426191..ffa5d747b11 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -23,6 +23,7 @@ fn clif_sig_from_fn_abi<'tcx>( ) -> Signature { let call_conv = match fn_abi.conv { Conv::Rust | Conv::C => default_call_conv, + Conv::RustCold => CallConv::Cold, Conv::X86_64SysV => CallConv::SystemV, Conv::X86_64Win64 => CallConv::WindowsFastcall, Conv::ArmAapcs @@ -312,13 +313,14 @@ pub(crate) fn codegen_terminator_call<'tcx>( source_info: mir::SourceInfo, func: &Operand<'tcx>, args: &[Operand<'tcx>], - mir_dest: Option<(Place<'tcx>, BasicBlock)>, + destination: Place<'tcx>, + target: Option<BasicBlock>, ) { let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx)); let fn_sig = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx)); - let destination = mir_dest.map(|(place, bb)| (codegen_place(fx, place), bb)); + let ret_place = codegen_place(fx, destination); // Handle special calls like instrinsics and empty drop glue. let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() { @@ -333,7 +335,8 @@ pub(crate) fn codegen_terminator_call<'tcx>( &fx.tcx.symbol_name(instance).name, substs, args, - destination, + ret_place, + target, ); return; } @@ -344,14 +347,15 @@ pub(crate) fn codegen_terminator_call<'tcx>( fx, instance, args, - destination, + ret_place, + target, source_info, ); return; } InstanceDef::DropGlue(_, None) => { // empty drop glue - a nop. - let (_, dest) = destination.expect("Non terminating drop_in_place_real???"); + let dest = target.expect("Non terminating drop_in_place_real???"); let ret_block = fx.get_block(dest); fx.bcx.ins().jump(ret_block, &[]); return; @@ -377,7 +381,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( .unwrap_or(false); if is_cold { fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); - if let Some((_place, destination_block)) = destination { + if let Some(destination_block) = target { fx.bcx.set_cold_block(fx.get_block(destination_block)); } } @@ -459,7 +463,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( } }; - let ret_place = destination.map(|(place, _)| place); self::returning::codegen_with_call_return_arg(fx, &fn_abi.ret, ret_place, |fx, return_ptr| { let call_args = return_ptr .into_iter() @@ -511,7 +514,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( call_inst }); - if let Some((_, dest)) = destination { + if let Some(dest) = target { let ret_block = fx.get_block(dest); fx.bcx.ins().jump(ret_block, &[]); } else { diff --git a/src/abi/returning.rs b/src/abi/returning.rs index c1bdba43e6c..ff3bb2dfd00 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -56,23 +56,22 @@ pub(super) fn codegen_return_param<'tcx>( pub(super) fn codegen_with_call_return_arg<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - ret_place: Option<CPlace<'tcx>>, + ret_place: CPlace<'tcx>, f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> Inst, ) { let (ret_temp_place, return_ptr) = match ret_arg_abi.mode { PassMode::Ignore => (None, None), - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => match ret_place { - Some(ret_place) if matches!(ret_place.inner(), CPlaceInner::Addr(_, None)) => { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + if matches!(ret_place.inner(), CPlaceInner::Addr(_, None)) { // This is an optimization to prevent unnecessary copies of the return value when // the return place is already a memory place as opposed to a register. // This match arm can be safely removed. (None, Some(ret_place.to_ptr().get_addr(fx))) - } - _ => { + } else { let place = CPlace::new_stack_slot(fx, ret_arg_abi.layout); (Some(place), Some(place.to_ptr().get_addr(fx))) } - }, + } PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } @@ -84,39 +83,25 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( match ret_arg_abi.mode { PassMode::Ignore => {} PassMode::Direct(_) => { - if let Some(ret_place) = ret_place { - let ret_val = fx.bcx.inst_results(call_inst)[0]; - ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout)); - } + let ret_val = fx.bcx.inst_results(call_inst)[0]; + ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout)); } PassMode::Pair(_, _) => { - if let Some(ret_place) = ret_place { - let ret_val_a = fx.bcx.inst_results(call_inst)[0]; - let ret_val_b = fx.bcx.inst_results(call_inst)[1]; - ret_place.write_cvalue( - fx, - CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout), - ); - } + let ret_val_a = fx.bcx.inst_results(call_inst)[0]; + let ret_val_b = fx.bcx.inst_results(call_inst)[1]; + ret_place + .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout)); } PassMode::Cast(cast) => { - if let Some(ret_place) = ret_place { - let results = fx - .bcx - .inst_results(call_inst) - .iter() - .copied() - .collect::<SmallVec<[Value; 2]>>(); - let result = - super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); - ret_place.write_cvalue(fx, result); - } + let results = + fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>(); + let result = + super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); + ret_place.write_cvalue(fx, result); } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { - if let (Some(ret_place), Some(ret_temp_place)) = (ret_place, ret_temp_place) { - // Both ret_place and ret_temp_place must be Some. If ret_place is None, this is - // a non-returning call. If ret_temp_place is None, it is not necessary to copy the - // return value. + if let Some(ret_temp_place) = ret_temp_place { + // If ret_temp_place is None, it is not necessary to copy the return value. let ret_temp_value = ret_temp_place.to_cvalue(fx); ret_place.write_cvalue(fx, ret_temp_value); } diff --git a/src/base.rs b/src/base.rs index 65e5812a8a5..07136e1b76a 100644 --- a/src/base.rs +++ b/src/base.rs @@ -393,6 +393,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { func, args, destination, + target, fn_span, cleanup: _, from_hir_call: _, @@ -404,6 +405,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { func, args, *destination, + *target, ) }); } @@ -605,7 +607,13 @@ fn codegen_stmt<'tcx>( let operand = codegen_operand(fx, operand); lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); } - Rvalue::Cast(CastKind::Misc, ref operand, to_ty) => { + Rvalue::Cast( + CastKind::Misc + | CastKind::PointerExposeAddress + | CastKind::PointerFromExposedAddress, + ref operand, + to_ty, + ) => { let operand = codegen_operand(fx, operand); let from_ty = operand.layout().ty; let to_ty = fx.monomorphize(to_ty); diff --git a/src/constant.rs b/src/constant.rs index 349a726d7b0..c7b05239eb7 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -542,8 +542,8 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( | TerminatorKind::FalseEdge { .. } | TerminatorKind::FalseUnwind { .. } => unreachable!(), TerminatorKind::InlineAsm { .. } => return None, - TerminatorKind::Call { destination: Some((call_place, _)), .. } - if call_place == place => + TerminatorKind::Call { destination, target: Some(_), .. } + if destination == place => { return None; } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 5e1e1c81d26..05457ce15e9 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -66,11 +66,7 @@ fn emit_module( let work_product = if backend_config.disable_incr_cache { None } else { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - tcx.sess, - &name, - &Some(tmp_file.clone()), - ) + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(tcx.sess, &name, &tmp_file) }; ModuleCodegenResult( @@ -84,21 +80,16 @@ fn reuse_workproduct_for_cgu( cgu: &CodegenUnit<'_>, work_products: &mut FxHashMap<WorkProductId, WorkProduct>, ) -> CompiledModule { - let mut object = None; - let work_product = cgu.work_product(tcx); - if let Some(saved_file) = &work_product.saved_file { - let obj_out = - tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); - object = Some(obj_out.clone()); - let source_file = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, &saved_file); - if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { - tcx.sess.err(&format!( - "unable to copy {} to {}: {}", - source_file.display(), - obj_out.display(), - err - )); - } + let work_product = cgu.previous_work_product(tcx); + let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); + let source_file = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, &work_product.saved_file); + if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { + tcx.sess.err(&format!( + "unable to copy {} to {}: {}", + source_file.display(), + obj_out.display(), + err + )); } work_products.insert(cgu.work_product_id(), work_product); @@ -106,7 +97,7 @@ fn reuse_workproduct_for_cgu( CompiledModule { name: cgu.name().to_string(), kind: ModuleKind::Regular, - object, + object: Some(obj_out), dwarf_object: None, bytecode: None, } diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 0e4f7ee907a..77ac46540a9 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -10,10 +10,9 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( intrinsic: &str, _substs: SubstsRef<'tcx>, args: &[mir::Operand<'tcx>], - destination: Option<(CPlace<'tcx>, BasicBlock)>, + ret: CPlace<'tcx>, + target: Option<BasicBlock>, ) { - let ret = destination.unwrap().0; - intrinsic_match! { fx, intrinsic, args, _ => { @@ -126,7 +125,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }; } - let dest = destination.expect("all llvm intrinsics used by stdlib should return").1; + let dest = target.expect("all llvm intrinsics used by stdlib should return"); let ret_block = fx.get_block(dest); fx.bcx.ins().jump(ret_block, &[]); } diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 29b3f36b2be..6937e658ed5 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -58,6 +58,7 @@ pub(crate) use llvm::codegen_llvm_intrinsic_call; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::subst::SubstsRef; use rustc_span::symbol::{kw, sym, Symbol}; +use rustc_target::abi::InitKind; use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; @@ -217,35 +218,42 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, instance: Instance<'tcx>, args: &[mir::Operand<'tcx>], - destination: Option<(CPlace<'tcx>, BasicBlock)>, + destination: CPlace<'tcx>, + target: Option<BasicBlock>, source_info: mir::SourceInfo, ) { let intrinsic = fx.tcx.item_name(instance.def_id()); let substs = instance.substs; - let ret = match destination { - Some((place, _)) => place, - None => { - // Insert non returning intrinsics here - match intrinsic { - sym::abort => { - fx.bcx.ins().trap(TrapCode::User(0)); - } - sym::transmute => { - crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info); - } - _ => unimplemented!("unsupported instrinsic {}", intrinsic), + let target = if let Some(target) = target { + target + } else { + // Insert non returning intrinsics here + match intrinsic { + sym::abort => { + fx.bcx.ins().trap(TrapCode::User(0)); + } + sym::transmute => { + crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info); } - return; + _ => unimplemented!("unsupported instrinsic {}", intrinsic), } + return; }; if intrinsic.as_str().starts_with("simd_") { - self::simd::codegen_simd_intrinsic_call(fx, intrinsic, substs, args, ret, source_info.span); - let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); + self::simd::codegen_simd_intrinsic_call( + fx, + intrinsic, + substs, + args, + destination, + source_info.span, + ); + let ret_block = fx.get_block(target); fx.bcx.ins().jump(ret_block, &[]); - } else if codegen_float_intrinsic_call(fx, intrinsic, args, ret) { - let ret_block = fx.get_block(destination.expect("Float intrinsics don't diverge").1); + } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) { + let ret_block = fx.get_block(target); fx.bcx.ins().jump(ret_block, &[]); } else { codegen_regular_intrinsic_call( @@ -254,9 +262,9 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( intrinsic, substs, args, - ret, - source_info, destination, + Some(target), + source_info, ); } } @@ -339,8 +347,8 @@ fn codegen_regular_intrinsic_call<'tcx>( substs: SubstsRef<'tcx>, args: &[mir::Operand<'tcx>], ret: CPlace<'tcx>, + destination: Option<BasicBlock>, source_info: mir::SourceInfo, - destination: Option<(CPlace<'tcx>, BasicBlock)>, ) { let usize_layout = fx.layout_of(fx.tcx.types.usize); @@ -664,7 +672,12 @@ fn codegen_regular_intrinsic_call<'tcx>( return; } - if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true) { + if intrinsic == sym::assert_zero_valid + && !layout.might_permit_raw_init( + fx, + InitKind::Zero, + fx.tcx.sess.opts.debugging_opts.strict_init_checks) { + with_no_trimmed_paths!({ crate::base::codegen_panic( fx, @@ -675,7 +688,12 @@ fn codegen_regular_intrinsic_call<'tcx>( return; } - if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false) { + if intrinsic == sym::assert_uninit_valid + && !layout.might_permit_raw_init( + fx, + InitKind::Uninit, + fx.tcx.sess.opts.debugging_opts.strict_init_checks) { + with_no_trimmed_paths!({ crate::base::codegen_panic( fx, @@ -761,7 +779,7 @@ fn codegen_regular_intrinsic_call<'tcx>( if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { // special case for compiler-builtins to avoid having to patch it crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); - let ret_block = fx.get_block(destination.unwrap().1); + let ret_block = fx.get_block(destination.unwrap()); fx.bcx.ins().jump(ret_block, &[]); return; } else { @@ -789,7 +807,7 @@ fn codegen_regular_intrinsic_call<'tcx>( if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { // special case for compiler-builtins to avoid having to patch it crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); - let ret_block = fx.get_block(destination.unwrap().1); + let ret_block = fx.get_block(destination.unwrap()); fx.bcx.ins().jump(ret_block, &[]); return; } else { @@ -1130,6 +1148,6 @@ fn codegen_regular_intrinsic_call<'tcx>( }; } - let ret_block = fx.get_block(destination.unwrap().1); + let ret_block = fx.get_block(destination.unwrap()); fx.bcx.ins().jump(ret_block, &[]); } |