diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 551 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/va_arg.rs | 84 |
10 files changed, 400 insertions, 293 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index a6fd2a7de6b..546540dfd76 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -34,13 +34,6 @@ pub trait ArgAttributesExt { ); } -fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // LLVM prior to version 12 had known miscompiles in the presence of - // noalias attributes (see #54878), but we don't support earlier - // versions at all anymore. We now enable mutable noalias by default. - cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) -} - const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; @@ -88,9 +81,6 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' attrs.push(llattr.create_attr(cx.llcx)); } } - if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { - attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); - } } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects // memory sanitizer's behavior. diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index f3bdacf6085..95baa95b021 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -102,10 +102,10 @@ pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { let mut fp = cx.sess().target.frame_pointer; + let opts = &cx.sess().opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. - if cx.sess().instrument_mcount() || matches!(cx.sess().opts.cg.force_frame_pointers, Some(true)) - { + if opts.unstable_opts.instrument_mcount || matches!(opts.cg.force_frame_pointers, Some(true)) { fp = FramePointer::Always; } let attr_value = match fp { @@ -119,7 +119,7 @@ pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attr /// Tell LLVM what instrument function to insert. #[inline] fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().instrument_mcount() { + if cx.sess().opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. @@ -137,6 +137,14 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribu } } +fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + if !cx.sess().opts.unstable_opts.no_jump_tables { + return None; + } + + Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) +} + fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from @@ -293,6 +301,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // FIXME: none of these three functions interact with source level attributes. to_add.extend(frame_pointer_type_attr(cx)); to_add.extend(instrument_function_attr(cx)); + to_add.extend(nojumptables_attr(cx)); to_add.extend(probestack_attr(cx)); to_add.extend(stackprotector_attr(cx)); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 97d0de47b3a..e23c88b62c1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -203,7 +203,7 @@ pub fn target_machine_factory( sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; - let asm_comments = sess.asm_comments(); + let asm_comments = sess.opts.unstable_opts.asm_comments; let relax_elf_relocations = sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f3bff5d5716..d9ccba07a34 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -280,29 +280,35 @@ pub unsafe fn create_module<'ll>( } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + let behavior = if llvm_version >= (15, 0, 0) { + llvm::LLVMModFlagBehavior::Min + } else { + llvm::LLVMModFlagBehavior::Error + }; + if sess.target.arch == "aarch64" { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "branch-target-enforcement\0".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "sign-return-address\0".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "sign-return-address-all\0".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "sign-return-address-with-bkey\0".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 86580d05d41..393bf30e9f8 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -30,7 +30,7 @@ use std::ffi::CString; /// implementing this Rust version, and though the format documentation is very explicit and /// detailed, some undocumented details in Clang's implementation (that may or may not be important) /// were also replicated for Rust's Coverage Map. -pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { +pub fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure the installed version of LLVM supports at least Coverage Map @@ -284,7 +284,7 @@ fn save_function_record( /// "code coverage dead code cgu" during the partitioning process. This prevents us from generating /// code regions for the same function more than once which can lead to linker errors regarding /// duplicate symbols. -fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { +fn add_unused_functions(cx: &CodegenCx<'_, '_>) { assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); let tcx = cx.tcx; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index a9e3dcf4cb3..48e3a812e4f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -111,7 +111,7 @@ macro_rules! return_if_di_node_created_in_meantime { /// Extract size and alignment from a TyAndLayout. #[inline] -fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { +fn size_and_align_of(ty_and_layout: TyAndLayout<'_>) -> (Size, Align) { (ty_and_layout.size, ty_and_layout.align.abi) } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 907517bf6ce..680d810f78e 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -8,8 +8,8 @@ use crate::va_arg::emit_va_arg; use crate::value::Value; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; -use rustc_codegen_ssa::common::span_invalid_monomorphization_error; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; +use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; @@ -284,15 +284,11 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }, None => { - span_invalid_monomorphization_error( - tcx.sess, + tcx.sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", - name, ty - ), - ); + name, + ty, + }); return; } } @@ -567,7 +563,7 @@ fn codegen_msvc_try<'ll>( // module. // // When modifying, make sure that the type_name string exactly matches - // the one used in src/libpanic_unwind/seh.rs. + // the one used in library/panic_unwind/src/seh.rs. let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); let type_name = bx.const_bytes(b"rust_panic\0"); let type_info = @@ -838,40 +834,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( llret_ty: &'ll Type, span: Span, ) -> Result<&'ll Value, ()> { - // macros for error handling: - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } - macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } + ($diag: expr) => {{ + bx.sess().emit_err($diag); + return Err(()); + }}; } macro_rules! require { - ($cond: expr, $($fmt: tt)*) => { + ($cond: expr, $diag: expr) => { if !$cond { - return_error!($($fmt)*); + return_error!($diag); } }; } macro_rules! require_simd { - ($ty: expr, $position: expr) => { - require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) + ($ty: expr, $diag: expr) => { + require!($ty.is_simd(), $diag) }; } @@ -881,7 +861,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - require_simd!(arg_tys[1], "argument"); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let expected_int_bits = (len.max(8) - 1).next_power_of_two(); @@ -902,12 +886,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); bx.load(int_ty, ptr, Align::ONE) } - _ => return_error!( - "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + _ => return_error!(InvalidMonomorphization::InvalidBitmask { + span, + name, mask_ty, expected_int_bits, expected_bytes - ), + }), }; let i1 = bx.type_i1(); @@ -919,7 +904,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // every intrinsic below takes a SIMD vector as its first argument - require_simd!(arg_tys[0], "input"); + require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] }); let in_ty = arg_tys[0]; let comparison = match name { @@ -934,23 +919,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); if let Some(cmp_op) = comparison { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); require!( bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, - "expected return type with integer elements, found `{}` with non-integer `{}`", - ret_ty, - out_ty + InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty } ); return Ok(compare_simd_types( @@ -975,10 +961,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span_bug!(span, "could not evaluate shuffle index array length") }) } - _ => return_error!( - "simd_shuffle index must be an array of `u32`, got `{}`", - args[2].layout.ty - ), + _ => return_error!(InvalidMonomorphization::SimdShuffle { + span, + name, + ty: args[2].layout.ty + }), } } else { stripped.parse().unwrap_or_else(|_| { @@ -986,23 +973,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }) }; - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); require!( out_len == n, - "expected return type of length {}, found `{}` with length {}", - n, - ret_ty, - out_len + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } ); require!( in_elem == out_ty, - "expected return element type `{}` (element of input `{}`), \ - found `{}` with element type `{}`", - in_elem, - in_ty, - ret_ty, - out_ty + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } ); let total_len = u128::from(in_len) * 2; @@ -1015,15 +994,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - emit_error!("shuffle index #{} is not a constant", arg_idx); + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexNotConstant { + span, + name, + arg_idx, + }); None } Some(idx) if idx >= total_len => { - emit_error!( - "shuffle index #{} is out of bounds (limit {})", + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + span, + name, arg_idx, - total_len - ); + total_len, + }); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1044,10 +1028,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_insert { require!( in_elem == arg_tys[2], - "expected inserted type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - arg_tys[2] + InvalidMonomorphization::InsertedType { + span, + name, + in_elem, + in_ty, + out_ty: arg_tys[2] + } ); return Ok(bx.insert_element( args[0].immediate(), @@ -1058,10 +1045,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_extract { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); } @@ -1069,17 +1053,18 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_select { let m_elem_ty = in_elem; let m_len = in_len; - require_simd!(arg_tys[1], "argument"); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); require!( m_len == v_len, - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len + InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } ); match m_elem_ty.kind() { ty::Int(_) => {} - _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), + _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }), } // truncate the mask to a vector of i1s let i1 = bx.type_i1(); @@ -1111,11 +1096,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( args[0].immediate(), i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), ), - _ => return_error!( - "vector argument `{}`'s element type `{}`, expected integer element type", + _ => return_error!(InvalidMonomorphization::VectorArgument { + span, + name, in_ty, in_elem - ), + }), }; // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. @@ -1150,12 +1136,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); return Ok(bx.load(array_ty, ptr, Align::ONE)); } - _ => return_error!( - "cannot return `{}`, expected `u{}` or `[u8; {}]`", + _ => return_error!(InvalidMonomorphization::CannotReturn { + span, + name, ret_ty, expected_int_bits, expected_bytes - ), + }), } } @@ -1168,25 +1155,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } + ($diag: expr) => {{ + bx.sess().emit_err($diag); + return Err(()); + }}; } let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { @@ -1194,16 +1167,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match f.bit_width() { 32 => ("f32", elem_ty), 64 => ("f64", elem_ty), - _ => { - return_error!( - "unsupported element type `{}` of floating-point vector `{}`", - f.name_str(), - in_ty - ); - } + _ => return_error!(InvalidMonomorphization::FloatingPointVector { + span, + name, + f_ty: *f, + in_ty, + }), } } else { - return_error!("`{}` is not a floating-point type", in_ty); + return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty }); }; let vec_ty = bx.type_vector(elem_ty, in_len); @@ -1225,7 +1197,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), - _ => return_error!("unrecognized intrinsic `{}`", name), + _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }), }; let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); @@ -1319,37 +1291,48 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); - require_simd!(ret_ty, "return"); + require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); // Of the same length: let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - out_len + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len + } ); require!( in_len == out_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - out_len2 + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: out_len2 + } ); // The return type must match the first argument type - require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); + require!( + ret_ty == in_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } + ); // This counts how many pointers fn ptr_count(t: Ty<'_>) -> usize { @@ -1376,15 +1359,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*_ {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Not, + } ); unreachable!(); } @@ -1400,10 +1383,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of third argument `{}` \ - to be a signed integer type", - element_ty2, - arg_tys[2] + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } ); } } @@ -1452,32 +1437,40 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); + require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); // Of the same length: let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == element_len1, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - element_len1 + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len: element_len1 + } ); require!( in_len == element_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - element_len2 + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: element_len2 + } ); // This counts how many pointers @@ -1508,15 +1501,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*mut {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Mut, + } ); unreachable!(); } @@ -1531,10 +1524,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of third argument `{}` \ - be a signed integer type", - element_ty2, - arg_tys[2] + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } ); } } @@ -1581,10 +1576,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::$name { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { ty::Int(_) | ty::Uint(_) => { @@ -1607,25 +1599,28 @@ fn generic_simd_intrinsic<'ll, 'tcx>( 32 => bx.const_real(bx.type_f32(), $identity), 64 => bx.const_real(bx.type_f64(), $identity), v => return_error!( - r#" -unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, - sym::$name, - in_ty, - in_elem, - v, - ret_ty + InvalidMonomorphization::UnsupportedSymbolOfSize { + span, + name, + symbol: sym::$name, + in_ty, + in_elem, + size: v, + ret_ty + } ), } }; Ok(bx.$float_reduce(acc, args[0].immediate())) } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1653,22 +1648,20 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, if name == sym::$name { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1686,22 +1679,20 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let input = if !$boolean { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); args[0].immediate() } else { match in_elem.kind() { ty::Int(_) | ty::Uint(_) => {} - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), } // boolean reductions operate on vectors of i1s: @@ -1714,13 +1705,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let r = bx.$red(input); Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1733,16 +1725,18 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bitwise_red!(simd_reduce_any: vector_reduce_or, true); if name == sym::simd_cast_ptr { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); match in_elem.kind() { @@ -1751,9 +1745,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); assert!(!check_sized); // we are in codegen, so we shouldn't see these types - require!(metadata.is_unit(), "cannot cast fat pointer `{}`", in_elem) + require!( + metadata.is_unit(), + InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } + ); + } + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) } - _ => return_error!("expected pointer, got `{}`", in_elem), } match out_elem.kind() { ty::RawPtr(p) => { @@ -1761,9 +1760,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); assert!(!check_sized); // we are in codegen, so we shouldn't see these types - require!(metadata.is_unit(), "cannot cast to fat pointer `{}`", out_elem) + require!( + metadata.is_unit(), + InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } + ); + } + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) } - _ => return_error!("expected pointer, got `{}`", out_elem), } if in_elem == out_elem { @@ -1774,66 +1778,76 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, } if name == sym::simd_expose_addr { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); match in_elem.kind() { ty::RawPtr(_) => {} - _ => return_error!("expected pointer, got `{}`", in_elem), + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) + } } match out_elem.kind() { ty::Uint(ty::UintTy::Usize) => {} - _ => return_error!("expected `usize`, got `{}`", out_elem), + _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }), } return Ok(bx.ptrtoint(args[0].immediate(), llret_ty)); } if name == sym::simd_from_exposed_addr { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); match in_elem.kind() { ty::Uint(ty::UintTy::Usize) => {} - _ => return_error!("expected `usize`, got `{}`", in_elem), + _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }), } match out_elem.kind() { ty::RawPtr(_) => {} - _ => return_error!("expected pointer, got `{}`", out_elem), + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) + } } return Ok(bx.inttoptr(args[0].immediate(), llret_ty)); } if name == sym::simd_cast || name == sym::simd_as { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); // casting cares about nominal type, not just structural type if in_elem == out_elem { @@ -1912,11 +1926,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, } require!( false, - "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", - in_ty, - in_elem, - ret_ty, - out_elem + InvalidMonomorphization::UnsupportedCast { + span, + name, + in_ty, + in_elem, + ret_ty, + out_elem + } ); } macro_rules! arith_binary { @@ -1928,10 +1945,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* _ => {}, } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) + require!( + false, + InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } + ); })* } } @@ -1959,10 +1976,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* _ => {}, } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) + require!( + false, + InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } + ); })* } } @@ -2000,12 +2017,12 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), _ => { - return_error!( - "expected element type `{}` of vector type `{}` \ - to be a signed or unsigned integer type", - arg_tys[0].simd_size_and_type(bx.tcx()).1, - arg_tys[0] - ); + return_error!(InvalidMonomorphization::ExpectedVectorElementType { + span, + name, + expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1, + vector_type: arg_tys[0] + }); } }; let llvm_intrinsic = &format!( diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e61dbe8b8fc..8b4861962b2 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -79,6 +79,7 @@ pub enum LLVMModFlagBehavior { Append = 5, AppendUnique = 6, Max = 7, + Min = 8, } // Consts for the LLVM CallConv type, pre-cast to usize. @@ -2389,11 +2390,11 @@ extern "C" { pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustBuildOperandBundleDef<'a>( + pub fn LLVMRustBuildOperandBundleDef( Name: *const c_char, - Inputs: *const &'a Value, + Inputs: *const &'_ Value, NumInputs: c_uint, - ) -> &'a mut OperandBundleDef<'a>; + ) -> &mut OperandBundleDef<'_>; pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 2fa602520dc..79b243f73d1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -81,10 +81,10 @@ unsafe fn configure_llvm(sess: &Session) { }; // Set the llvm "program name" to make usage and invalid argument messages more clear. add("rustc -Cllvm-args=\"...\" with", true); - if sess.time_llvm_passes() { + if sess.opts.unstable_opts.time_llvm_passes { add("-time-passes", false); } - if sess.print_llvm_passes() { + if sess.opts.unstable_opts.print_llvm_passes { add("-debug-pass=Structure", false); } if sess.target.generate_arange_section diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ceb3d5a84ab..b19398e68c2 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -175,6 +175,89 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( val } +fn emit_s390x_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of the s390x ELF ABI calling convention for va_args see + // https://github.com/IBM/s390x-abi (chapter 1.2.4) + let va_list_addr = list.immediate(); + let va_list_layout = list.deref(bx.cx).layout; + let va_list_ty = va_list_layout.llvm_type(bx); + let layout = bx.cx.layout_of(target_ty); + + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let in_mem = bx.append_sibling_block("va_arg.in_mem"); + let end = bx.append_sibling_block("va_arg.end"); + + // FIXME: vector ABI not yet supported. + let target_ty_size = bx.cx.size_of(target_ty).bytes(); + let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); + let unpadded_size = if indirect { 8 } else { target_ty_size }; + let padded_size = 8; + let padding = padded_size - unpadded_size; + + let gpr_type = indirect || !layout.is_single_fp_element(bx.cx); + let (max_regs, reg_count_field, reg_save_index, reg_padding) = + if gpr_type { (5, 0, 2, padding) } else { (4, 1, 16, 0) }; + + // Check whether the value was passed in a register or in memory. + let reg_count = bx.struct_gep( + va_list_ty, + va_list_addr, + va_list_layout.llvm_field_index(bx.cx, reg_count_field), + ); + let reg_count_v = bx.load(bx.type_i64(), reg_count, Align::from_bytes(8).unwrap()); + let use_regs = bx.icmp(IntPredicate::IntULT, reg_count_v, bx.const_u64(max_regs)); + bx.cond_br(use_regs, in_reg, in_mem); + + // Emit code to load the value if it was passed in a register. + bx.switch_to_block(in_reg); + + // Work out the address of the value in the register save area. + let reg_ptr = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let reg_ptr_v = bx.load(bx.type_i8p(), reg_ptr, bx.tcx().data_layout.pointer_align.abi); + let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8)); + let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding)); + let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]); + + // Update the register count. + let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1)); + bx.store(new_reg_count_v, reg_count, Align::from_bytes(8).unwrap()); + bx.br(end); + + // Emit code to load the value if it was passed in memory. + bx.switch_to_block(in_mem); + + // Work out the address of the value in the argument overflow area. + let arg_ptr = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 2)); + let arg_ptr_v = bx.load(bx.type_i8p(), arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let arg_off = bx.const_u64(padding); + let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]); + + // Update the argument overflow area pointer. + let arg_size = bx.cx().const_u64(padded_size); + let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]); + bx.store(new_arg_ptr_v, arg_ptr, bx.tcx().data_layout.pointer_align.abi); + bx.br(end); + + // Return the appropriate result. + bx.switch_to_block(end); + let val_addr = bx.phi(bx.type_i8p(), &[reg_addr, mem_addr], &[in_reg, in_mem]); + let val_type = layout.llvm_type(bx); + let val_addr = if indirect { + let ptr_type = bx.cx.type_ptr_to(val_type); + let ptr_addr = bx.bitcast(val_addr, bx.cx.type_ptr_to(ptr_type)); + bx.load(ptr_type, ptr_addr, bx.tcx().data_layout.pointer_align.abi) + } else { + bx.bitcast(val_addr, bx.cx.type_ptr_to(val_type)) + }; + bx.load(val_type, val_addr, layout.align.abi) +} + pub(super) fn emit_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, addr: OperandRef<'tcx, &'ll Value>, @@ -200,6 +283,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) } "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + "s390x" => emit_s390x_va_arg(bx, addr, target_ty), // Windows x86_64 "x86_64" if target.is_like_windows => { let target_ty_size = bx.cx.size_of(target_ty).bytes(); |