diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
| -rw-r--r-- | compiler/rustc_codegen_llvm/Cargo.toml | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 80 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 124 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/declare.rs | 24 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 143 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/va_arg.rs | 12 |
12 files changed, 151 insertions, 274 deletions
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index f9373640dce..ebbb852f21c 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -29,6 +29,6 @@ rustc_llvm = { path = "../rustc_llvm" } rustc_session = { path = "../rustc_session" } rustc_serialize = { path = "../rustc_serialize" } rustc_target = { path = "../rustc_target" } -smallvec = { version = "1.0", features = ["union", "may_dangle"] } +smallvec = { version = "1.6.1", features = ["union", "may_dangle"] } rustc_ast = { path = "../rustc_ast" } rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 915dd3d9eda..a69241e456f 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -389,7 +389,7 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_cconv(&self) -> llvm::CallConv { match self.conv { - Conv::C | Conv::Rust => llvm::CCallConv, + Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, @@ -546,6 +546,18 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { if cconv != llvm::CCallConv { llvm::SetInstructionCallConv(callsite, cconv); } + + if self.conv == Conv::CCmseNonSecureCall { + // This will probably get ignored on all targets but those supporting the TrustZone-M + // extension (thumbv8m targets). + unsafe { + llvm::AddCallSiteAttrString( + callsite, + llvm::AttributePlace::Function, + rustc_data_structures::const_cstr!("cmse_nonsecure_call"), + ); + } + } } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 8801211d51b..4aa25aae747 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -487,6 +487,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { // LLVM doesn't recognize x30 "{lr}".to_string() + } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { + // LLVM doesn't recognize r14 + "{lr}".to_string() } else { format!("{{{}}}", reg.name()) } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index a78d692aaa7..26111729ba5 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -53,6 +53,9 @@ pub fn sanitize(cx: &CodegenCx<'ll, '_>, no_sanitize: SanitizerSet, llfn: &'ll V if enabled.contains(SanitizerSet::THREAD) { llvm::Attribute::SanitizeThread.apply_llfn(Function, llfn); } + if enabled.contains(SanitizerSet::HWADDRESS) { + llvm::Attribute::SanitizeHWAddress.apply_llfn(Function, llfn); + } } /// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 326ae354ccf..5f8a11ab94e 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -11,6 +11,7 @@ use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; use crate::ModuleLlvm; +use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -440,6 +441,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), + sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), + sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), }) } else { None @@ -652,6 +655,10 @@ unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static if config.sanitizer.contains(SanitizerSet::THREAD) { passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); } + if config.sanitizer.contains(SanitizerSet::HWADDRESS) { + let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); + passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); + } } pub(crate) fn link( @@ -873,9 +880,7 @@ pub(crate) unsafe fn codegen( if !config.emit_bc { debug!("removing_bitcode {:?}", bc_out); - if let Err(e) = fs::remove_file(&bc_out) { - diag_handler.err(&format!("failed to remove bitcode: {}", e)); - } + ensure_removed(diag_handler, &bc_out); } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 14dd245625d..16e1a8a1242 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -8,9 +8,7 @@ use crate::value::Value; use libc::c_uint; use rustc_codegen_ssa::traits::*; use rustc_data_structures::const_cstr; -use rustc_hir as hir; use rustc_hir::def_id::DefId; -use rustc_hir::Node; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ read_target_uint, Allocation, ErrorHandled, GlobalAlloc, Pointer, @@ -18,7 +16,6 @@ use rustc_middle::mir::interpret::{ use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, span_bug}; -use rustc_span::symbol::sym; use rustc_target::abi::{AddressSpace, Align, HasDataLayout, LayoutOf, Primitive, Scalar, Size}; use tracing::debug; @@ -209,70 +206,42 @@ impl CodegenCx<'ll, 'tcx> { let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); let sym = self.tcx.symbol_name(instance).name; + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); - debug!("get_static: sym={} instance={:?}", sym, instance); + debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); - let g = if let Some(local_def_id) = def_id.as_local() { - let id = self.tcx.hir().local_def_id_to_hir_id(local_def_id); + let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { let llty = self.layout_of(ty).llvm_type(self); - // FIXME: refactor this to work without accessing the HIR - let (g, attrs) = match self.tcx.hir().get(id) { - Node::Item(&hir::Item { attrs, kind: hir::ItemKind::Static(..), .. }) => { - if let Some(g) = self.get_declared_value(sym) { - if self.val_ty(g) != self.type_ptr_to(llty) { - span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); - } - } - - let g = self.declare_global(sym, llty); - - if !self.tcx.is_reachable_non_generic(local_def_id) { - unsafe { - llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); - } - } - - (g, attrs) + if let Some(g) = self.get_declared_value(sym) { + if self.val_ty(g) != self.type_ptr_to(llty) { + span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); } + } - Node::ForeignItem(&hir::ForeignItem { - ref attrs, - kind: hir::ForeignItemKind::Static(..), - .. - }) => { - let fn_attrs = self.tcx.codegen_fn_attrs(local_def_id); - (check_and_apply_linkage(&self, &fn_attrs, ty, sym, def_id), &**attrs) - } - - item => bug!("get_static: expected static, found {:?}", item), - }; - - debug!("get_static: sym={} attrs={:?}", sym, attrs); + let g = self.declare_global(sym, llty); - for attr in attrs { - if self.tcx.sess.check_name(attr, sym::thread_local) { - llvm::set_thread_local_mode(g, self.tls_model); + if !self.tcx.is_reachable_non_generic(def_id) { + unsafe { + llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); } } g } else { - // FIXME(nagisa): perhaps the map of externs could be offloaded to llvm somehow? - debug!("get_static: sym={} item_attr={:?}", sym, self.tcx.item_attrs(def_id)); + check_and_apply_linkage(&self, &fn_attrs, ty, sym, def_id) + }; - let attrs = self.tcx.codegen_fn_attrs(def_id); - let g = check_and_apply_linkage(&self, &attrs, ty, sym, def_id); - - // Thread-local statics in some other crate need to *always* be linked - // against in a thread-local fashion, so we need to be sure to apply the - // thread-local attribute locally if it was present remotely. If we - // don't do this then linker errors can be generated where the linker - // complains that one object files has a thread local version of the - // symbol and another one doesn't. - if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { - llvm::set_thread_local_mode(g, self.tls_model); - } + // Thread-local statics in some other crate need to *always* be linked + // against in a thread-local fashion, so we need to be sure to apply the + // thread-local attribute locally if it was present remotely. If we + // don't do this then linker errors can be generated where the linker + // complains that one object files has a thread local version of the + // symbol and another one doesn't. + if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + llvm::set_thread_local_mode(g, self.tls_model); + } + if !def_id.is_local() { let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && // ThinLTO can't handle this workaround in all cases, so we don't // emit the attrs. Instead we make them unnecessary by disallowing @@ -304,8 +273,7 @@ impl CodegenCx<'ll, 'tcx> { } } } - g - }; + } if self.use_dll_storage_attrs && self.tcx.is_dllimport_foreign_item(def_id) { // For foreign (native) libs we know the exact storage type to use. diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 6acd26bd415..ee099f93258 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -380,7 +380,7 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { "rust_eh_personality" }; let fty = self.type_variadic_func(&[], self.type_i32()); - self.declare_cfn(name, fty) + self.declare_cfn(name, llvm::UnnamedAddr::Global, fty) } }; attributes::apply_target_cpu_attr(self, llfn); @@ -429,7 +429,7 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { if self.get_declared_value("main").is_none() { - Some(self.declare_cfn("main", fn_type)) + Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} @@ -459,8 +459,7 @@ impl CodegenCx<'b, 'tcx> { } else { self.type_variadic_func(&[], ret) }; - let f = self.declare_cfn(name, fn_ty); - llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); + let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty); self.intrinsics.borrow_mut().insert(name, f); f } @@ -498,25 +497,6 @@ impl CodegenCx<'b, 'tcx> { let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); - macro_rules! vector_types { - ($id_out:ident: $elem_ty:ident, $len:expr) => { - let $id_out = self.type_vector($elem_ty, $len); - }; - ($($id_out:ident: $elem_ty:ident, $len:expr;)*) => { - $(vector_types!($id_out: $elem_ty, $len);)* - } - } - vector_types! { - t_v2f32: t_f32, 2; - t_v4f32: t_f32, 4; - t_v8f32: t_f32, 8; - t_v16f32: t_f32, 16; - - t_v2f64: t_f64, 2; - t_v4f64: t_f64, 4; - t_v8f64: t_f64, 8; - } - ifn!("llvm.wasm.trunc.saturate.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.saturate.unsigned.i32.f64", fn(t_f64) -> t_i32); ifn!("llvm.wasm.trunc.saturate.unsigned.i64.f32", fn(t_f32) -> t_i64); @@ -540,124 +520,40 @@ impl CodegenCx<'b, 'tcx> { ifn!("llvm.sideeffect", fn() -> void); ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); - ifn!("llvm.powi.v2f32", fn(t_v2f32, t_i32) -> t_v2f32); - ifn!("llvm.powi.v4f32", fn(t_v4f32, t_i32) -> t_v4f32); - ifn!("llvm.powi.v8f32", fn(t_v8f32, t_i32) -> t_v8f32); - ifn!("llvm.powi.v16f32", fn(t_v16f32, t_i32) -> t_v16f32); ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); - ifn!("llvm.powi.v2f64", fn(t_v2f64, t_i32) -> t_v2f64); - ifn!("llvm.powi.v4f64", fn(t_v4f64, t_i32) -> t_v4f64); - ifn!("llvm.powi.v8f64", fn(t_v8f64, t_i32) -> t_v8f64); ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.pow.v2f32", fn(t_v2f32, t_v2f32) -> t_v2f32); - ifn!("llvm.pow.v4f32", fn(t_v4f32, t_v4f32) -> t_v4f32); - ifn!("llvm.pow.v8f32", fn(t_v8f32, t_v8f32) -> t_v8f32); - ifn!("llvm.pow.v16f32", fn(t_v16f32, t_v16f32) -> t_v16f32); ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.pow.v2f64", fn(t_v2f64, t_v2f64) -> t_v2f64); - ifn!("llvm.pow.v4f64", fn(t_v4f64, t_v4f64) -> t_v4f64); - ifn!("llvm.pow.v8f64", fn(t_v8f64, t_v8f64) -> t_v8f64); ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); - ifn!("llvm.sqrt.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.sqrt.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.sqrt.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.sqrt.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); - ifn!("llvm.sqrt.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.sqrt.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.sqrt.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); - ifn!("llvm.sin.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.sin.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.sin.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.sin.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); - ifn!("llvm.sin.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.sin.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.sin.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); - ifn!("llvm.cos.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.cos.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.cos.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.cos.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); - ifn!("llvm.cos.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.cos.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.cos.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); - ifn!("llvm.exp.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.exp.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.exp.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.exp.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); - ifn!("llvm.exp.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.exp.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.exp.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); - ifn!("llvm.exp2.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.exp2.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.exp2.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.exp2.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); - ifn!("llvm.exp2.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.exp2.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.exp2.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.log.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.log.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.log.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.log.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.log.f64", fn(t_f64) -> t_f64); - ifn!("llvm.log.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.log.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.log.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log10.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.log10.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.log10.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.log10.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); - ifn!("llvm.log10.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.log10.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.log10.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log2.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.log2.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.log2.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.log2.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); - ifn!("llvm.log2.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.log2.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.log2.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); - ifn!("llvm.fma.v2f32", fn(t_v2f32, t_v2f32, t_v2f32) -> t_v2f32); - ifn!("llvm.fma.v4f32", fn(t_v4f32, t_v4f32, t_v4f32) -> t_v4f32); - ifn!("llvm.fma.v8f32", fn(t_v8f32, t_v8f32, t_v8f32) -> t_v8f32); - ifn!("llvm.fma.v16f32", fn(t_v16f32, t_v16f32, t_v16f32) -> t_v16f32); ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); - ifn!("llvm.fma.v2f64", fn(t_v2f64, t_v2f64, t_v2f64) -> t_v2f64); - ifn!("llvm.fma.v4f64", fn(t_v4f64, t_v4f64, t_v4f64) -> t_v4f64); - ifn!("llvm.fma.v8f64", fn(t_v8f64, t_v8f64, t_v8f64) -> t_v8f64); ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); - ifn!("llvm.fabs.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.fabs.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.fabs.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.fabs.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); - ifn!("llvm.fabs.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.fabs.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.fabs.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); @@ -665,24 +561,10 @@ impl CodegenCx<'b, 'tcx> { ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); - ifn!("llvm.floor.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.floor.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.floor.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.floor.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); - ifn!("llvm.floor.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.floor.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.floor.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); - ifn!("llvm.ceil.v2f32", fn(t_v2f32) -> t_v2f32); - ifn!("llvm.ceil.v4f32", fn(t_v4f32) -> t_v4f32); - ifn!("llvm.ceil.v8f32", fn(t_v8f32) -> t_v8f32); - ifn!("llvm.ceil.v16f32", fn(t_v16f32) -> t_v16f32); ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); - ifn!("llvm.ceil.v2f64", fn(t_v2f64) -> t_v2f64); - ifn!("llvm.ceil.v4f64", fn(t_v4f64) -> t_v4f64); - ifn!("llvm.ceil.v8f64", fn(t_v8f64) -> t_v8f64); ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 0591e0a5c12..8977fa085b9 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -30,6 +30,7 @@ fn declare_raw_fn( cx: &CodegenCx<'ll, '_>, name: &str, callconv: llvm::CallConv, + unnamed: llvm::UnnamedAddr, ty: &'ll Type, ) -> &'ll Value { debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); @@ -38,9 +39,7 @@ fn declare_raw_fn( }; llvm::SetFunctionCallConv(llfn, callconv); - // Function addresses in Rust are never significant, allowing functions to - // be merged. - llvm::SetUnnamedAddress(llfn, llvm::UnnamedAddr::Global); + llvm::SetUnnamedAddress(llfn, unnamed); if cx.tcx.sess.opts.cg.no_redzone.unwrap_or(cx.tcx.sess.target.disable_redzone) { llvm::Attribute::NoRedZone.apply_llfn(Function, llfn); @@ -68,8 +67,13 @@ impl CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_cfn(&self, name: &str, fn_type: &'ll Type) -> &'ll Value { - declare_raw_fn(self, name, llvm::CCallConv, fn_type) + pub fn declare_cfn( + &self, + name: &str, + unnamed: llvm::UnnamedAddr, + fn_type: &'ll Type, + ) -> &'ll Value { + declare_raw_fn(self, name, llvm::CCallConv, unnamed, fn_type) } /// Declare a Rust function. @@ -79,7 +83,15 @@ impl CodegenCx<'ll, 'tcx> { pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); - let llfn = declare_raw_fn(self, name, fn_abi.llvm_cconv(), fn_abi.llvm_type(self)); + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + name, + fn_abi.llvm_cconv(), + llvm::UnnamedAddr::Global, + fn_abi.llvm_type(self), + ); fn_abi.apply_attrs_llfn(self, llfn); llfn } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index bf0d499e6c4..668daa52ed2 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1009,7 +1009,7 @@ fn generic_simd_intrinsic( } fn simd_simple_float_intrinsic( - name: &str, + name: Symbol, in_elem: &::rustc_middle::ty::TyS<'_>, in_ty: &::rustc_middle::ty::TyS<'_>, in_len: u64, @@ -1036,93 +1036,69 @@ fn generic_simd_intrinsic( } } } - let ety = match in_elem.kind() { - ty::Float(f) if f.bit_width() == 32 => { - if in_len < 2 || in_len > 16 { - return_error!( - "unsupported floating-point vector `{}` with length `{}` \ - out-of-range [2, 16]", - in_ty, - in_len - ); - } - "f32" - } - ty::Float(f) if f.bit_width() == 64 => { - if in_len < 2 || in_len > 8 { + + let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { + let elem_ty = bx.cx.type_float_from_ty(*f); + match f.bit_width() { + 32 => ("f32", elem_ty), + 64 => ("f64", elem_ty), + _ => { return_error!( - "unsupported floating-point vector `{}` with length `{}` \ - out-of-range [2, 8]", - in_ty, - in_len + "unsupported element type `{}` of floating-point vector `{}`", + f.name_str(), + in_ty ); } - "f64" - } - ty::Float(f) => { - return_error!( - "unsupported element type `{}` of floating-point vector `{}`", - f.name_str(), - in_ty - ); - } - _ => { - return_error!("`{}` is not a floating-point type", in_ty); } + } else { + return_error!("`{}` is not a floating-point type", in_ty); }; - let llvm_name = &format!("llvm.{0}.v{1}{2}", name, in_len, ety); - let intrinsic = bx.get_intrinsic(&llvm_name); - let c = - bx.call(intrinsic, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); + let vec_ty = bx.type_vector(elem_ty, in_len); + + let (intr_name, fn_ty) = match name { + sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), + sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), + sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), + _ => return_error!("unrecognized intrinsic `{}`", name), + }; + + let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); + let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty); + let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) }; Ok(c) } - match name { - sym::simd_fsqrt => { - return simd_simple_float_intrinsic("sqrt", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fsin => { - return simd_simple_float_intrinsic("sin", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fcos => { - return simd_simple_float_intrinsic("cos", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fabs => { - return simd_simple_float_intrinsic("fabs", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_floor => { - return simd_simple_float_intrinsic("floor", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_ceil => { - return simd_simple_float_intrinsic("ceil", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fexp => { - return simd_simple_float_intrinsic("exp", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fexp2 => { - return simd_simple_float_intrinsic("exp2", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_flog10 => { - return simd_simple_float_intrinsic("log10", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_flog2 => { - return simd_simple_float_intrinsic("log2", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_flog => { - return simd_simple_float_intrinsic("log", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fpowi => { - return simd_simple_float_intrinsic("powi", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fpow => { - return simd_simple_float_intrinsic("pow", in_elem, in_ty, in_len, bx, span, args); - } - sym::simd_fma => { - return simd_simple_float_intrinsic("fma", in_elem, in_ty, in_len, bx, span, args); - } - _ => { /* fallthrough */ } + if std::matches!( + name, + sym::simd_fsqrt + | sym::simd_fsin + | sym::simd_fcos + | sym::simd_fabs + | sym::simd_floor + | sym::simd_ceil + | sym::simd_fexp + | sym::simd_fexp2 + | sym::simd_flog10 + | sym::simd_flog2 + | sym::simd_flog + | sym::simd_fpowi + | sym::simd_fpow + | sym::simd_fma + ) { + return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); } // FIXME: use: @@ -1278,12 +1254,12 @@ fn generic_simd_intrinsic( format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); let f = bx.declare_cfn( &llvm_intrinsic, + llvm::UnnamedAddr::No, bx.type_func( &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty, ), ); - llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); return Ok(v); } @@ -1408,9 +1384,9 @@ fn generic_simd_intrinsic( format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); let f = bx.declare_cfn( &llvm_intrinsic, + llvm::UnnamedAddr::No, bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t), ); - llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); return Ok(v); } @@ -1714,8 +1690,11 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ); let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); - let f = bx.declare_cfn(&llvm_intrinsic, bx.type_func(&[vec_ty, vec_ty], vec_ty)); - llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); + let f = bx.declare_cfn( + &llvm_intrinsic, + llvm::UnnamedAddr::No, + bx.type_func(&[vec_ty, vec_ty], vec_ty), + ); let v = bx.call(f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d9f42efebab..8c1740d8f25 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -131,6 +131,7 @@ pub enum Attribute { ReturnsTwice = 25, ReadNone = 26, InaccessibleMemOnly = 27, + SanitizeHWAddress = 28, } /// LLVMIntPredicate @@ -439,6 +440,8 @@ pub struct SanitizerOptions { pub sanitize_memory_recover: bool, pub sanitize_memory_track_origins: c_int, pub sanitize_thread: bool, + pub sanitize_hwaddress: bool, + pub sanitize_hwaddress_recover: bool, } /// LLVMRelocMode @@ -1100,6 +1103,7 @@ extern "C" { // Operations on call sites pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); pub fn LLVMRustAddCallSiteAttribute(Instr: &Value, index: c_uint, attr: Attribute); + pub fn LLVMRustAddCallSiteAttrString(Instr: &Value, index: c_uint, Name: *const c_char); pub fn LLVMRustAddAlignmentCallSiteAttr(Instr: &Value, index: c_uint, bytes: u32); pub fn LLVMRustAddDereferenceableCallSiteAttr(Instr: &Value, index: c_uint, bytes: u64); pub fn LLVMRustAddDereferenceableOrNullCallSiteAttr(Instr: &Value, index: c_uint, bytes: u64); @@ -2127,6 +2131,7 @@ extern "C" { Recover: bool, ) -> &'static mut Pass; pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; + pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); pub fn LLVMRustAddLastExtensionPasses( PMB: &PassManagerBuilder, diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index fc40065a966..bb9c6d47373 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -43,6 +43,10 @@ pub fn AddFunctionAttrString(llfn: &'a Value, idx: AttributePlace, attr: &CStr) } } +pub fn AddCallSiteAttrString(callsite: &Value, idx: AttributePlace, attr: &CStr) { + unsafe { LLVMRustAddCallSiteAttrString(callsite, idx.as_uint(), attr.as_ptr()) } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 07fde27b5a3..39d08fbee3b 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -105,7 +105,6 @@ fn emit_aapcs_va_arg( let mut end = bx.build_sibling_block("va_arg.end"); let zero = bx.const_i32(0); let offset_align = Align::from_bytes(4).unwrap(); - assert_eq!(bx.tcx().sess.target.endian, Endian::Little); let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); let (reg_off, reg_top_index, slot_size) = if gr_type { @@ -144,9 +143,14 @@ fn emit_aapcs_va_arg( let top = in_reg.load(top, bx.tcx().data_layout.pointer_align.abi); // reg_value = *(@top + reg_off_v); - let top = in_reg.gep(top, &[reg_off_v]); - let top = in_reg.bitcast(top, bx.cx.type_ptr_to(layout.llvm_type(bx))); - let reg_value = in_reg.load(top, layout.align.abi); + let mut reg_addr = in_reg.gep(top, &[reg_off_v]); + if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { + // On big-endian systems the value is right-aligned in its slot. + let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); + reg_addr = in_reg.gep(reg_addr, &[offset]); + } + let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(layout.llvm_type(bx))); + let reg_value = in_reg.load(reg_addr, layout.align.abi); in_reg.br(&end.llbb()); // On Stack block |