diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/allocator.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 43 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 44 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 3 |
8 files changed, 78 insertions, 51 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index e02b457fd0b..c6a7dc95d77 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -383,13 +383,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { - let mut func_attrs = SmallVec::<[_; 2]>::new(); + let mut func_attrs = SmallVec::<[_; 3]>::new(); if self.ret.layout.abi.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); } if !self.can_unwind { func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); } + if let Conv::RiscvInterrupt { kind } = self.conv { + func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", kind.as_str())); + } attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); let mut i = 0; @@ -565,7 +568,9 @@ impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { impl From<Conv> for llvm::CallConv { fn from(conv: Conv) -> Self { match conv { - Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::C | Conv::Rust | Conv::CCmseNonSecureCall | Conv::RiscvInterrupt { .. } => { + llvm::CCallConv + } Conv::RustCold => llvm::ColdCallConv, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index ca123334fca..8bb93025c45 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -34,8 +34,8 @@ pub(crate) unsafe fn codegen( if kind == AllocatorKind::Default { for method in ALLOCATOR_METHODS { let mut args = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match *ty { + for input in method.inputs.iter() { + match input.ty { AllocatorTy::Layout => { args.push(usize); // size args.push(usize); // align diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 46e6daed21f..47cc5bd52e2 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -320,6 +320,7 @@ impl<'a> DiagnosticHandlers<'a> { }) .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok())); + let pgo_available = cgcx.opts.cg.profile_use.is_some(); let data = Box::into_raw(Box::new((cgcx, handler))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); @@ -333,6 +334,7 @@ impl<'a> DiagnosticHandlers<'a> { // The `as_ref()` is important here, otherwise the `CString` will be dropped // too soon! remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()), + pgo_available, ); DiagnosticHandlers { data, llcx, old_handler } } @@ -470,6 +472,8 @@ pub(crate) unsafe fn llvm_optimize( Some(llvm::SanitizerOptions { sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), + sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI), + sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI), sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, @@ -505,6 +509,7 @@ pub(crate) unsafe fn llvm_optimize( &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), opt_stage, + cgcx.opts.cg.linker_plugin_lto.enabled(), config.no_prepopulate_passes, config.verify_llvm_ir, using_thin_buffers, diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 5408481df48..ac6d8f84142 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1512,9 +1512,9 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, ) { - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - if is_indirect_call && fn_abi.is_some() && self.tcx.sess.is_sanitizer_cfi_enabled() { - if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::CFI) { + let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; + if self.tcx.sess.is_sanitizer_cfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { + if let Some(fn_attrs) = fn_attrs && fn_attrs.no_sanitize.contains(SanitizerSet::CFI) { return; } @@ -1526,7 +1526,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { options.insert(TypeIdOptions::NORMALIZE_INTEGERS); } - let typeid = typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, options); let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); // Test whether the function pointer is associated with the type identifier. @@ -1550,25 +1550,26 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, ) -> Option<llvm::OperandBundleDef<'ll>> { - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = if is_indirect_call && self.tcx.sess.is_sanitizer_kcfi_enabled() { - if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::KCFI) { - return None; - } + let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; + let kcfi_bundle = + if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { + if let Some(fn_attrs) = fn_attrs && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI) { + return None; + } - let mut options = TypeIdOptions::empty(); - if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { - options.insert(TypeIdOptions::GENERALIZE_POINTERS); - } - if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { - options.insert(TypeIdOptions::NORMALIZE_INTEGERS); - } + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; kcfi_bundle } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index cb093996d1d..b4f7e20e05d 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -145,6 +145,17 @@ pub unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("-n32:64-", "-n64-"); } } + if llvm_version < (17, 0, 0) { + if sess.target.arch.starts_with("powerpc") { + // LLVM 17 specifies function pointer alignment for ppc: + // https://reviews.llvm.org/D147016 + target_data_layout = target_data_layout + .replace("-Fn32", "") + .replace("-Fi32", "") + .replace("-Fn64", "") + .replace("-Fi64", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. if sess.target.is_builtin { @@ -891,7 +902,8 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.prefetch", fn(ptr, t_i32, t_i32, t_i32) -> void); // This isn't an "LLVM intrinsic", but LLVM's optimization passes - // recognize it like one and we assume it exists in `core::slice::cmp` + // recognize it like one (including turning it into `bcmp` sometimes) + // and we use it to implement intrinsics like `raw_eq` and `compare_bytes` match self.sess().target.arch.as_ref() { "avr" | "msp430" => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i16), _ => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i32), @@ -985,7 +997,7 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) = err { + if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { span_bug!(span, "failed to get layout for `{ty}`: {err:?}") diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 86c39ab5e94..a9b06030e70 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -329,6 +329,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } + sym::compare_bytes => { + // Here we assume that the `memcmp` provided by the target is a NOP for size 0. + let cmp = self.call_intrinsic( + "memcmp", + &[args[0].immediate(), args[1].immediate(), args[2].immediate()], + ); + // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`. + self.sext(cmp, self.type_ix(32)) + } + sym::black_box => { args[0].val.store(self, result); let result_val_span = [result.llval]; @@ -1020,28 +1030,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>( )); } - if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { - // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer. - // If there is no suffix, use the index array length. - let n: u64 = if stripped.is_empty() { - // Make sure this is actually an array, since typeck only checks the length-suffixed - // version of this intrinsic. - match args[2].layout.ty.kind() { - ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( - || span_bug!(span, "could not evaluate shuffle index array length"), - ) - } - _ => return_error!(InvalidMonomorphization::SimdShuffle { - span, - name, - ty: args[2].layout.ty - }), + if name == sym::simd_shuffle { + // Make sure this is actually an array, since typeck only checks the length-suffixed + // version of this intrinsic. + let n: u64 = match args[2].layout.ty.kind() { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { + len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( + || span_bug!(span, "could not evaluate shuffle index array length"), + ) } - } else { - stripped.parse().unwrap_or_else(|_| { - span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") - }) + _ => return_error!(InvalidMonomorphization::SimdShuffle { + span, + name, + ty: args[2].layout.ty + }), }; require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 69b1fd9c2d2..84157d1e25c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -475,6 +475,8 @@ pub enum OptStage { pub struct SanitizerOptions { pub sanitize_address: bool, pub sanitize_address_recover: bool, + pub sanitize_cfi: bool, + pub sanitize_kcfi: bool, pub sanitize_memory: bool, pub sanitize_memory_recover: bool, pub sanitize_memory_track_origins: c_int, @@ -894,6 +896,7 @@ extern "C" { pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>; + pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; // Operations on constants of any type pub fn LLVMConstNull(Ty: &Type) -> &Value; @@ -2144,6 +2147,7 @@ extern "C" { TM: &'a TargetMachine, OptLevel: PassBuilderOptLevel, OptStage: OptStage, + IsLinkerPluginLTO: bool, NoPrepopulatePasses: bool, VerifyIR: bool, UseThinLTOBuffers: bool, @@ -2338,6 +2342,7 @@ extern "C" { remark_passes: *const *const c_char, remark_passes_len: usize, remark_file: *const c_char, + pgo_available: bool, ); #[allow(improper_ctypes)] diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 32e71c6a641..2be7bce115d 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -61,9 +61,6 @@ fn uncached_llvm_type<'a, 'tcx>( } Some(name) } - // Use identified structure types for ADT. Due to pointee types in LLVM IR their definition - // might be recursive. Other cases are non-recursive and we can use literal structure types. - ty::Adt(..) => Some(String::new()), _ => None, }; |