diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src/base.rs')
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/base.rs | 271 |
1 files changed, 82 insertions, 189 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 4842628a99d..0a7734d6a04 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -8,7 +8,7 @@ use rustc_target::abi::call::FnAbi; use crate::prelude::*; pub(crate) fn codegen_fn<'tcx>( - cx: &mut crate::CodegenCx<'tcx, impl Module>, + cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>, linkage: Linkage, ) { @@ -38,9 +38,8 @@ pub(crate) fn codegen_fn<'tcx>( // Predefine blocks let start_block = bcx.create_block(); - let block_map: IndexVec<BasicBlock, Block> = (0..mir.basic_blocks().len()) - .map(|_| bcx.create_block()) - .collect(); + let block_map: IndexVec<BasicBlock, Block> = + (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx let pointer_type = cx.module.target_config().pointer_type(); @@ -68,22 +67,23 @@ pub(crate) fn codegen_fn<'tcx>( inline_asm_index: 0, }; - let arg_uninhabited = fx.mir.args_iter().any(|arg| { - fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)) - .abi - .is_uninhabited() - }); + let arg_uninhabited = fx + .mir + .args_iter() + .any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); - if arg_uninhabited { - fx.bcx - .append_block_params_for_function_params(fx.block_map[START_BLOCK]); + if !crate::constant::check_constants(&mut fx) { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted"); + } else if arg_uninhabited { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument"); } else { tcx.sess.time("codegen clif ir", || { - tcx.sess.time("codegen prelude", || { - crate::abi::codegen_fn_prelude(&mut fx, start_block) - }); + tcx.sess + .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block)); codegen_fn_content(&mut fx); }); } @@ -131,11 +131,7 @@ pub(crate) fn codegen_fn<'tcx>( let module = &mut cx.module; tcx.sess.time("define function", || { module - .define_function( - func_id, - context, - &mut cranelift_codegen::binemit::NullTrapSink {}, - ) + .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {}) .unwrap() }); @@ -149,14 +145,12 @@ pub(crate) fn codegen_fn<'tcx>( &clif_comments, ); - if let Some(mach_compile_result) = &context.mach_compile_result { - if let Some(disasm) = &mach_compile_result.disasm { - crate::pretty_clif::write_ir_file( - tcx, - &format!("{}.vcode", tcx.symbol_name(instance).name), - |file| file.write_all(disasm.as_bytes()), - ) - } + if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { + crate::pretty_clif::write_ir_file( + tcx, + &format!("{}.vcode", tcx.symbol_name(instance).name), + |file| file.write_all(disasm.as_bytes()), + ) } // Define debuginfo for function @@ -199,16 +193,13 @@ pub(crate) fn verify_func( Some(Box::new(writer)), err, ); - tcx.sess - .fatal(&format!("cranelift verify error:\n{}", pretty_error)); + tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error)); } } }); } -fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { - crate::constant::check_constants(fx); - +fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() { let block = fx.get_block(bb); fx.bcx.switch_to_block(block); @@ -231,11 +222,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { #[cfg(debug_assertions)] { let mut terminator_head = "\n".to_string(); - bb_data - .terminator() - .kind - .fmt_head(&mut terminator_head) - .unwrap(); + bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); let inst = fx.bcx.func.layout.last_inst(block).unwrap(); fx.add_comment(inst, terminator_head); } @@ -267,13 +254,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { TerminatorKind::Return => { crate::abi::codegen_return(fx); } - TerminatorKind::Assert { - cond, - expected, - msg, - target, - cleanup: _, - } => { + TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => { if !fx.tcx.sess.overflow_checks() { if let mir::AssertKind::OverflowNeg(_) = *msg { let target = fx.get_block(*target); @@ -319,11 +300,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { } } - TerminatorKind::SwitchInt { - discr, - switch_ty, - targets, - } => { + TerminatorKind::SwitchInt { discr, switch_ty, targets } => { let discr = codegen_operand(fx, discr).load_scalar(fx); let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() @@ -433,11 +410,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { | TerminatorKind::GeneratorDrop => { bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); } - TerminatorKind::Drop { - place, - target, - unwind: _, - } => { + TerminatorKind::Drop { place, target, unwind: _ } => { let drop_place = codegen_place(fx, *place); crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place); @@ -452,7 +425,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { } fn codegen_stmt<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, + fx: &mut FunctionCx<'_, '_, 'tcx>, #[allow(unused_variables)] cur_block: Block, stmt: &Statement<'tcx>, ) { @@ -470,10 +443,7 @@ fn codegen_stmt<'tcx>( } match &stmt.kind { - StatementKind::SetDiscriminant { - place, - variant_index, - } => { + StatementKind::SetDiscriminant { place, variant_index } => { let place = codegen_place(fx, **place); crate::discriminant::codegen_set_discriminant(fx, place, *variant_index); } @@ -594,19 +564,11 @@ fn codegen_stmt<'tcx>( let from_ty = operand.layout().ty; let to_ty = fx.monomorphize(to_ty); - fn is_fat_ptr<'tcx>( - fx: &FunctionCx<'_, 'tcx, impl Module>, - ty: Ty<'tcx>, - ) -> bool { + fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { ty.builtin_deref(true) - .map( - |ty::TypeAndMut { - ty: pointee_ty, - mutbl: _, - }| { - has_ptr_meta(fx.tcx, pointee_ty) - }, - ) + .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| { + has_ptr_meta(fx.tcx, pointee_ty) + }) .unwrap_or(false) } @@ -626,50 +588,22 @@ fn codegen_stmt<'tcx>( ty::Uint(_) | ty::Int(_) => {} _ => unreachable!("cast adt {} -> {}", from_ty, to_ty), } + let to_clif_ty = fx.clif_type(to_ty).unwrap(); - use rustc_target::abi::{Int, TagEncoding, Variants}; - - match operand.layout().variants { - Variants::Single { index } => { - let discr = operand - .layout() - .ty - .discriminant_for_variant(fx.tcx, index) - .unwrap(); - let discr = if discr.ty.is_signed() { - fx.layout_of(discr.ty).size.sign_extend(discr.val) - } else { - discr.val - }; - let discr = discr.into(); - - let discr = CValue::const_val(fx, fx.layout_of(to_ty), discr); - lval.write_cvalue(fx, discr); - } - Variants::Multiple { - ref tag, - tag_field, - tag_encoding: TagEncoding::Direct, - variants: _, - } => { - let cast_to = fx.clif_type(dest_layout.ty).unwrap(); - - // Read the tag/niche-encoded discriminant from memory. - let encoded_discr = - operand.value_field(fx, mir::Field::new(tag_field)); - let encoded_discr = encoded_discr.load_scalar(fx); - - // Decode the discriminant (specifically if it's niche-encoded). - let signed = match tag.value { - Int(_, signed) => signed, - _ => false, - }; - let val = clif_intcast(fx, encoded_discr, cast_to, signed); - let val = CValue::by_val(val, dest_layout); - lval.write_cvalue(fx, val); - } - Variants::Multiple { .. } => unreachable!(), - } + let discriminant = crate::discriminant::codegen_get_discriminant( + fx, + operand, + fx.layout_of(operand.layout().ty.discriminant_ty(fx.tcx)), + ) + .load_scalar(fx); + + let res = crate::cast::clif_intcast( + fx, + discriminant, + to_clif_ty, + to_ty.is_signed(), + ); + lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); } else { let to_clif_ty = fx.clif_type(to_ty).unwrap(); let from = operand.load_scalar(fx); @@ -730,8 +664,7 @@ fn codegen_stmt<'tcx>( // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); let val = operand.load_scalar(fx); - fx.bcx - .call_memset(fx.cx.module.target_config(), addr, val, times); + fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); } else { let loop_block = fx.bcx.create_block(); let loop_block2 = fx.bcx.create_block(); @@ -766,25 +699,19 @@ fn codegen_stmt<'tcx>( let content_ty = fx.monomorphize(content_ty); let layout = fx.layout_of(content_ty); let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64); - let llalign = fx - .bcx - .ins() - .iconst(usize_type, layout.align.abi.bytes() as i64); + let llalign = fx.bcx.ins().iconst(usize_type, layout.align.abi.bytes() as i64); let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty)); // Allocate space: - let def_id = match fx - .tcx - .lang_items() - .require(rustc_hir::LangItem::ExchangeMalloc) - { - Ok(id) => id, - Err(s) => { - fx.tcx - .sess - .fatal(&format!("allocation of `{}` {}", box_layout.ty, s)); - } - }; + let def_id = + match fx.tcx.lang_items().require(rustc_hir::LangItem::ExchangeMalloc) { + Ok(id) => id, + Err(s) => { + fx.tcx + .sess + .fatal(&format!("allocation of `{}` {}", box_layout.ty, s)); + } + }; let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); let func_ref = fx.get_function_ref(instance); let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]); @@ -792,10 +719,11 @@ fn codegen_stmt<'tcx>( lval.write_cvalue(fx, CValue::by_val(ptr, box_layout)); } Rvalue::NullaryOp(NullOp::SizeOf, ty) => { - assert!(lval - .layout() - .ty - .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all())); + assert!( + lval.layout() + .ty + .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all()) + ); let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes(); let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into()); @@ -823,11 +751,7 @@ fn codegen_stmt<'tcx>( StatementKind::LlvmInlineAsm(asm) => { use rustc_span::symbol::Symbol; - let LlvmInlineAsm { - asm, - outputs, - inputs, - } = &**asm; + let LlvmInlineAsm { asm, outputs, inputs } = &**asm; let rustc_hir::LlvmInlineAsmInner { asm: asm_code, // Name outputs: output_names, // Vec<LlvmInlineAsmOutput> @@ -843,15 +767,9 @@ fn codegen_stmt<'tcx>( // Black box } "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { - assert_eq!( - input_names, - &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")] - ); + assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]); assert_eq!(output_names.len(), 4); - for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]) - .iter() - .enumerate() - { + for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() { assert_eq!(&output_names[i].constraint.as_str(), c); assert!(!output_names[i].is_rw); assert!(!output_names[i].is_indirect); @@ -897,12 +815,7 @@ fn codegen_stmt<'tcx>( crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); } // ___chkstk, ___chkstk_ms and __alloca are only used on Windows - _ if fx - .tcx - .symbol_name(fx.instance) - .name - .starts_with("___chkstk") => - { + _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => { crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); } _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { @@ -922,27 +835,21 @@ fn codegen_stmt<'tcx>( } } -fn codegen_array_len<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, - place: CPlace<'tcx>, -) -> Value { +fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value { match *place.layout().ty.kind() { ty::Array(_elem_ty, len) => { - let len = fx - .monomorphize(len) - .eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64; + let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64; fx.bcx.ins().iconst(fx.pointer_type, len) } - ty::Slice(_elem_ty) => place - .to_ptr_maybe_unsized() - .1 - .expect("Length metadata for slice place"), + ty::Slice(_elem_ty) => { + place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place") + } _ => bug!("Rvalue::Len({:?})", place), } } pub(crate) fn codegen_place<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, + fx: &mut FunctionCx<'_, '_, 'tcx>, place: Place<'tcx>, ) -> CPlace<'tcx> { let mut cplace = fx.get_local_place(place.local); @@ -959,11 +866,7 @@ pub(crate) fn codegen_place<'tcx>( let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx); cplace = cplace.place_index(fx, index); } - PlaceElem::ConstantIndex { - offset, - min_length: _, - from_end, - } => { + PlaceElem::ConstantIndex { offset, min_length: _, from_end } => { let offset: u64 = offset; let index = if !from_end { fx.bcx.ins().iconst(fx.pointer_type, offset as i64) @@ -1014,7 +917,7 @@ pub(crate) fn codegen_place<'tcx>( } pub(crate) fn codegen_operand<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, + fx: &mut FunctionCx<'_, '_, 'tcx>, operand: &Operand<'tcx>, ) -> CValue<'tcx> { match operand { @@ -1026,34 +929,24 @@ pub(crate) fn codegen_operand<'tcx>( } } -pub(crate) fn codegen_panic<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, - msg_str: &str, - span: Span, -) { +pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) { let location = fx.get_caller_location(span).load_scalar(fx); let msg_ptr = fx.anonymous_str("assert", msg_str); - let msg_len = fx - .bcx - .ins() - .iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); + let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); let args = [msg_ptr, msg_len, location]; codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span); } pub(crate) fn codegen_panic_inner<'tcx>( - fx: &mut FunctionCx<'_, 'tcx, impl Module>, + fx: &mut FunctionCx<'_, '_, 'tcx>, lang_item: rustc_hir::LangItem, args: &[Value], span: Span, ) { - let def_id = fx - .tcx - .lang_items() - .require(lang_item) - .unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s)); + let def_id = + fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s)); let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); let symbol_name = fx.tcx.symbol_name(instance).name; |