diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 118 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 43 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 27 |
10 files changed, 137 insertions, 107 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index e481b99afcc..9e3893d5314 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -14,7 +14,7 @@ use smallvec::SmallVec; use tracing::debug; use crate::builder::Builder; -use crate::common::{AsCCharPtr, Funclet}; +use crate::common::Funclet; use crate::context::CodegenCx; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -435,13 +435,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { template_str.push_str("\n.att_syntax\n"); } - unsafe { - llvm::LLVMAppendModuleInlineAsm( - self.llmod, - template_str.as_c_char_ptr(), - template_str.len(), - ); - } + llvm::append_module_inline_asm(self.llmod, template_str.as_bytes()); } fn mangled_name(&self, instance: Instance<'tcx>) -> String { @@ -482,67 +476,67 @@ pub(crate) fn inline_asm_call<'ll>( debug!("Asm Output Type: {:?}", output); let fty = bx.cx.type_func(&argtys, output); + // Ask LLVM to verify that the constraints are well-formed. - let constraints_ok = - unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_c_char_ptr(), cons.len()) }; + let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) }; debug!("constraint verification result: {:?}", constraints_ok); - if constraints_ok { - let v = unsafe { - llvm::LLVMRustInlineAsm( - fty, - asm.as_c_char_ptr(), - asm.len(), - cons.as_c_char_ptr(), - cons.len(), - volatile, - alignstack, - dia, - can_throw, - ) - }; + if !constraints_ok { + // LLVM has detected an issue with our constraints, so bail out. + return None; + } - let call = if !labels.is_empty() { - assert!(catch_funclet.is_none()); - bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None) - } else if let Some((catch, funclet)) = catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None) - } else { - bx.call(fty, None, None, v, inputs, None, None) - }; + let v = unsafe { + llvm::LLVMGetInlineAsm( + fty, + asm.as_ptr(), + asm.len(), + cons.as_ptr(), + cons.len(), + volatile, + alignstack, + dia, + can_throw, + ) + }; - // Store mark in a metadata node so we can map LLVM errors - // back to source locations. See #17552. - let key = "srcloc"; - let kind = bx.get_md_kind_id(key); + let call = if !labels.is_empty() { + assert!(catch_funclet.is_none()); + bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None) + } else if let Some((catch, funclet)) = catch_funclet { + bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None) + } else { + bx.call(fty, None, None, v, inputs, None, None) + }; - // `srcloc` contains one 64-bit integer for each line of assembly code, - // where the lower 32 bits hold the lo byte position and the upper 32 bits - // hold the hi byte position. - let mut srcloc = vec![]; - if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { - // LLVM inserts an extra line to add the ".intel_syntax", so add - // a dummy srcloc entry for it. - // - // Don't do this if we only have 1 line span since that may be - // due to the asm template string coming from a macro. LLVM will - // default to the first srcloc for lines that don't have an - // associated srcloc. - srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0))); - } - srcloc.extend(line_spans.iter().map(|span| { - llvm::LLVMValueAsMetadata( - bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)), - ) - })); - let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) }; - let md = bx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, kind, md); + // Store mark in a metadata node so we can map LLVM errors + // back to source locations. See #17552. + let key = "srcloc"; + let kind = bx.get_md_kind_id(key); - Some(call) - } else { - // LLVM has detected an issue with our constraints, bail out - None + // `srcloc` contains one 64-bit integer for each line of assembly code, + // where the lower 32 bits hold the lo byte position and the upper 32 bits + // hold the hi byte position. + let mut srcloc = vec![]; + if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { + // LLVM inserts an extra line to add the ".intel_syntax", so add + // a dummy srcloc entry for it. + // + // Don't do this if we only have 1 line span since that may be + // due to the asm template string coming from a macro. LLVM will + // default to the first srcloc for lines that don't have an + // associated srcloc. + srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0))); } + srcloc.extend(line_spans.iter().map(|span| { + llvm::LLVMValueAsMetadata( + bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)), + ) + })); + let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) }; + let md = bx.get_metadata_value(md); + llvm::LLVMSetMetadata(call, kind, md); + + Some(call) } /// If the register is an xmm/ymm/zmm register then return its index. diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 4ac77c8f7f1..20721c74608 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1148,9 +1148,9 @@ unsafe fn embed_bitcode( // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); + llvm::append_module_inline_asm(llmod, &asm); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); + llvm::append_module_inline_asm(llmod, &asm); } } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 04c8118b616..5238755c8eb 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -361,7 +361,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) { + if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1416,7 +1416,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) { + if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1749,7 +1749,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) { + if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1836,7 +1836,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, instance: Option<Instance<'tcx>>, llfn: &'ll Value, - ) -> Option<llvm::OperandBundleOwned<'ll>> { + ) -> Option<llvm::OperandBundleBox<'ll>> { let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi @@ -1862,7 +1862,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi::typeid_for_fnabi(self.tcx, fn_abi, options) }; - Some(llvm::OperandBundleOwned::new("kcfi", &[self.const_u32(kcfi_typeid)])) + Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)])) } else { None }; diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index a6f277e4455..3cfa96393e9 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -67,12 +67,12 @@ use crate::value::Value; /// the `OperandBundleDef` value created for MSVC landing pads. pub(crate) struct Funclet<'ll> { cleanuppad: &'ll Value, - operand: llvm::OperandBundleOwned<'ll>, + operand: llvm::OperandBundleBox<'ll>, } impl<'ll> Funclet<'ll> { pub(crate) fn new(cleanuppad: &'ll Value) -> Self { - Funclet { cleanuppad, operand: llvm::OperandBundleOwned::new("funclet", &[cleanuppad]) } + Funclet { cleanuppad, operand: llvm::OperandBundleBox::new("funclet", &[cleanuppad]) } } pub(crate) fn cleanuppad(&self) -> &'ll Value { @@ -80,7 +80,7 @@ impl<'ll> Funclet<'ll> { } pub(crate) fn bundle(&self) -> &llvm::OperandBundle<'ll> { - self.operand.raw() + self.operand.as_ref() } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index cbac55c7153..bf81eb648f8 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -364,12 +364,7 @@ impl<'ll> CodegenCx<'ll, '_> { if !def_id.is_local() { let needs_dll_storage_attr = self.use_dll_storage_attrs - // If the symbol is a foreign item, then don't automatically apply DLLImport, as - // we'll rely on the #[link] attribute instead. BUT, if this is an internal symbol - // then it may be generated by the compiler in some crate, so we do need to apply - // DLLImport when linking with the MSVC linker. - && (!self.tcx.is_foreign_item(def_id) - || (self.sess().target.is_like_msvc && fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL))) + && !self.tcx.is_foreign_item(def_id) // Local definitions can never be imported, so we must not apply // the DLLImport annotation. && !dso_local diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ed50515b707..b0d8e11d1fb 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1009,11 +1009,27 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.minimum.f16", fn(t_f16, t_f16) -> t_f16); + ifn!("llvm.minimum.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.minimum.f64", fn(t_f64, t_f64) -> t_f64); + // There are issues on x86_64 and aarch64 with the f128 variant. + // - https://github.com/llvm/llvm-project/issues/139380 + // - https://github.com/llvm/llvm-project/issues/139381 + // ifn!("llvm.minimum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.maximum.f16", fn(t_f16, t_f16) -> t_f16); + ifn!("llvm.maximum.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.maximum.f64", fn(t_f64, t_f64) -> t_f64); + // There are issues on x86_64 and aarch64 with the f128 variant. + // - https://github.com/llvm/llvm-project/issues/139380 + // - https://github.com/llvm/llvm-project/issues/139381 + // ifn!("llvm.maximum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.floor.f16", fn(t_f16) -> t_f16); ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index bfaad8f2f1e..5ca57375292 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -103,11 +103,23 @@ fn get_simple_intrinsic<'ll>( sym::minnumf64 => "llvm.minnum.f64", sym::minnumf128 => "llvm.minnum.f128", + sym::minimumf16 => "llvm.minimum.f16", + sym::minimumf32 => "llvm.minimum.f32", + sym::minimumf64 => "llvm.minimum.f64", + // There are issues on x86_64 and aarch64 with the f128 variant, + // let's instead use the instrinsic fallback body. + // sym::minimumf128 => "llvm.minimum.f128", sym::maxnumf16 => "llvm.maxnum.f16", sym::maxnumf32 => "llvm.maxnum.f32", sym::maxnumf64 => "llvm.maxnum.f64", sym::maxnumf128 => "llvm.maxnum.f128", + sym::maximumf16 => "llvm.maximum.f16", + sym::maximumf32 => "llvm.maximum.f32", + sym::maximumf64 => "llvm.maximum.f64", + // There are issues on x86_64 and aarch64 with the f128 variant, + // let's instead use the instrinsic fallback body. + // sym::maximumf128 => "llvm.maximum.f128", sym::copysignf16 => "llvm.copysign.f16", sym::copysignf32 => "llvm.copysign.f32", sym::copysignf64 => "llvm.copysign.f64", diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index e8010ec9fc4..5736314b96a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -6,7 +6,6 @@ // tidy-alphabetical-start #![allow(internal_features)] -#![cfg_attr(bootstrap, feature(let_chains))] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] #![feature(assert_matches)] diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index ffb490dcdc2..67a66e6ec79 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,7 +1,7 @@ //! Bindings to the LLVM-C API (`LLVM*`), and to our own `extern "C"` wrapper //! functions around the unstable LLVM C++ API (`LLVMRust*`). //! -//! ## Passing pointer/length strings as `*const c_uchar` +//! ## Passing pointer/length strings as `*const c_uchar` (PTR_LEN_STR) //! //! Normally it's a good idea for Rust-side bindings to match the corresponding //! C-side function declarations as closely as possible. But when passing `&str` @@ -415,6 +415,7 @@ impl AtomicRmwBinOp { pub(crate) enum AtomicOrdering { #[allow(dead_code)] NotAtomic = 0, + #[allow(dead_code)] Unordered = 1, Monotonic = 2, // Consume = 3, // Not specified yet. @@ -428,7 +429,6 @@ impl AtomicOrdering { pub(crate) fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { use rustc_codegen_ssa::common::AtomicOrdering as Common; match ao { - Common::Unordered => Self::Unordered, Common::Relaxed => Self::Monotonic, Common::Acquire => Self::Acquire, Common::Release => Self::Release, @@ -471,7 +471,7 @@ pub(crate) enum MetadataType { MD_kcfi_type = 36, } -/// LLVMRustAsmDialect +/// Must match the layout of `LLVMInlineAsmDialect`. #[derive(Copy, Clone, PartialEq)] #[repr(C)] pub(crate) enum AsmDialect { @@ -1014,8 +1014,25 @@ unsafe extern "C" { pub(crate) fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; pub(crate) fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); - /// See Module::setModuleInlineAsm. - pub(crate) fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); + /// Append inline assembly to a module. See `Module::appendModuleInlineAsm`. + pub(crate) fn LLVMAppendModuleInlineAsm( + M: &Module, + Asm: *const c_uchar, // See "PTR_LEN_STR". + Len: size_t, + ); + + /// Create the specified uniqued inline asm string. See `InlineAsm::get()`. + pub(crate) fn LLVMGetInlineAsm<'ll>( + Ty: &'ll Type, + AsmString: *const c_uchar, // See "PTR_LEN_STR". + AsmStringSize: size_t, + Constraints: *const c_uchar, // See "PTR_LEN_STR". + ConstraintsSize: size_t, + HasSideEffects: llvm::Bool, + IsAlignStack: llvm::Bool, + Dialect: AsmDialect, + CanThrow: llvm::Bool, + ) -> &'ll Value; // Operations on integer types pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; @@ -1766,7 +1783,7 @@ unsafe extern "C" { pub(crate) fn LLVMDIBuilderCreateNameSpace<'ll>( Builder: &DIBuilder<'ll>, ParentScope: Option<&'ll Metadata>, - Name: *const c_uchar, + Name: *const c_uchar, // See "PTR_LEN_STR". NameLen: size_t, ExportSymbols: llvm::Bool, ) -> &'ll Metadata; @@ -1994,21 +2011,9 @@ unsafe extern "C" { /// Prints the statistics collected by `-Zprint-codegen-stats`. pub(crate) fn LLVMRustPrintStatistics(OutStr: &RustString); - /// Prepares inline assembly. - pub(crate) fn LLVMRustInlineAsm( - Ty: &Type, - AsmString: *const c_char, - AsmStringLen: size_t, - Constraints: *const c_char, - ConstraintsLen: size_t, - SideEffects: Bool, - AlignStack: Bool, - Dialect: AsmDialect, - CanThrow: Bool, - ) -> &Value; pub(crate) fn LLVMRustInlineAsmVerify( Ty: &Type, - Constraints: *const c_char, + Constraints: *const c_uchar, // See "PTR_LEN_STR". ConstraintsLen: size_t, ) -> bool; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index d14aab06073..ed23f911930 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -363,12 +363,13 @@ pub(crate) fn last_error() -> Option<String> { } } -/// Owns an [`OperandBundle`], and will dispose of it when dropped. -pub(crate) struct OperandBundleOwned<'a> { +/// Owning pointer to an [`OperandBundle`] that will dispose of the bundle +/// when dropped. +pub(crate) struct OperandBundleBox<'a> { raw: ptr::NonNull<OperandBundle<'a>>, } -impl<'a> OperandBundleOwned<'a> { +impl<'a> OperandBundleBox<'a> { pub(crate) fn new(name: &str, vals: &[&'a Value]) -> Self { let raw = unsafe { LLVMCreateOperandBundle( @@ -378,21 +379,21 @@ impl<'a> OperandBundleOwned<'a> { vals.len() as c_uint, ) }; - OperandBundleOwned { raw: ptr::NonNull::new(raw).unwrap() } + Self { raw: ptr::NonNull::new(raw).unwrap() } } - /// Returns inner `OperandBundle` type. + /// Dereferences to the underlying `&OperandBundle`. /// - /// This could be a `Deref` implementation, but `OperandBundle` contains an extern type and - /// `Deref::Target: ?Sized`. - pub(crate) fn raw(&self) -> &OperandBundle<'a> { + /// This can't be a `Deref` implementation because `OperandBundle` transitively + /// contains an extern type, which is incompatible with `Deref::Target: ?Sized`. + pub(crate) fn as_ref(&self) -> &OperandBundle<'a> { // SAFETY: The returned reference is opaque and can only used for FFI. // It is valid for as long as `&self` is. unsafe { self.raw.as_ref() } } } -impl Drop for OperandBundleOwned<'_> { +impl Drop for OperandBundleBox<'_> { fn drop(&mut self) { unsafe { LLVMDisposeOperandBundle(self.raw); @@ -440,3 +441,11 @@ pub(crate) fn set_dso_local<'ll>(v: &'ll Value) { LLVMRustSetDSOLocal(v, true); } } + +/// Safe wrapper for `LLVMAppendModuleInlineAsm`, which delegates to +/// `Module::appendModuleInlineAsm`. +pub(crate) fn append_module_inline_asm<'ll>(llmod: &'ll Module, asm: &[u8]) { + unsafe { + LLVMAppendModuleInlineAsm(llmod, asm.as_ptr(), asm.len()); + } +} |