diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src/builder.rs')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 145 |
1 files changed, 38 insertions, 107 deletions
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 40783825cae..6ffe90997f5 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -26,13 +26,13 @@ use smallvec::SmallVec; use tracing::{debug, instrument}; use crate::abi::FnAbiLlvmExt; +use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use crate::{attributes, llvm_util}; // All Builders must have an llfn associated with them #[must_use] @@ -93,8 +93,6 @@ impl HasTargetSpec for Builder<'_, '_, '_> { } impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { - type LayoutOfResult = TyAndLayout<'tcx>; - #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { self.cx.handle_layout_err(err, span, ty) @@ -102,8 +100,6 @@ impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { } impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { - type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; - #[inline] fn handle_fn_abi_err( &self, @@ -124,11 +120,7 @@ impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { } } -impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> { - type CodegenCx = CodegenCx<'ll, 'tcx>; -} - -macro_rules! builder_methods_for_value_instructions { +macro_rules! math_builder_methods { ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { unsafe { @@ -138,7 +130,21 @@ macro_rules! builder_methods_for_value_instructions { } } +macro_rules! set_math_builder_methods { + ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => { + $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { + unsafe { + let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED); + llvm::$llvm_set_math(instr); + instr + } + })+ + } +} + impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { + type CodegenCx = CodegenCx<'ll, 'tcx>; + fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { let bx = Builder::with_cx(cx); unsafe { @@ -273,7 +279,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - builder_methods_for_value_instructions! { + math_builder_methods! { add(a, b) => LLVMBuildAdd, fadd(a, b) => LLVMBuildFAdd, sub(a, b) => LLVMBuildSub, @@ -305,84 +311,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unchecked_umul(x, y) => LLVMBuildNUWMul, } - fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetAlgebraicMath(instr); - instr - } - } - - fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetAlgebraicMath(instr); - instr - } - } - - fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetAlgebraicMath(instr); - instr - } - } - - fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetAlgebraicMath(instr); - instr - } - } - - fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetAlgebraicMath(instr); - instr - } + set_math_builder_methods! { + fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath), + fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath), + fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath), + fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath), + frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath), + fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath), + fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath), + fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath), + fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath), + frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath), } fn checked_binop( @@ -465,6 +404,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { val } } + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { if scalar.is_bool() { return self.trunc(val, self.cx().type_i1()); @@ -733,11 +673,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // for performance. LLVM doesn't seem to care about this, and will happily treat // `!nontemporal` stores as-if they were normal stores (for reordering optimizations // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like - // regular stores but require special fences. - // So we keep a list of architectures where `!nontemporal` is known to be truly just - // a hint, and use regular stores everywhere else. - // (In the future, we could alternatively ensure that an sfence gets emitted after a sequence of movnt - // before any kind of synchronizing operation. But it's not clear how to do that with LLVM.) + // regular stores but require special fences. So we keep a list of architectures + // where `!nontemporal` is known to be truly just a hint, and use regular stores + // everywhere else. (In the future, we could alternatively ensure that an sfence + // gets emitted after a sequence of movnt before any kind of synchronizing + // operation. But it's not clear how to do that with LLVM.) // For more context, see <https://github.com/rust-lang/rust/issues/114582> and // <https://github.com/llvm/llvm-project/issues/64521>. const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] = @@ -1166,6 +1106,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { (val, success) } } + fn atomic_rmw( &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, @@ -1317,15 +1258,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { - if llvm_util::get_version() < (17, 0, 2) { - // Work around https://github.com/llvm/llvm-project/issues/66984. - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); - } else { - // Cleanup is always the cold path. - let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); - } + // Cleanup is always the cold path. + let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); } } @@ -1767,8 +1702,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) { debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); - assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], @@ -1802,7 +1735,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp ); - assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; @@ -1844,7 +1776,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", fn_name, hash, cond_loc, mcdc_temp, bool_value ); - assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( &[ |