diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 64 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 115 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 2 |
7 files changed, 118 insertions, 74 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 67bc86e4c90..b6bbc81732e 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -313,7 +313,7 @@ fn get_llvm_object_symbols( llvm::LLVMRustGetSymbols( buf.as_ptr(), buf.len(), - &mut *state as *mut &mut _ as *mut c_void, + std::ptr::addr_of_mut!(*state) as *mut c_void, callback, error_callback, ) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7ed27b33dce..8cab2a3f27c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + fn checked_binop( &mut self, oop: OverflowOp, @@ -1327,17 +1367,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } @@ -1366,22 +1406,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 87e3774068b..24b3aa4223a 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -103,8 +103,7 @@ impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_ fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { let diag: DiagnosticBuilder<'_, G> = self.0.into_diagnostic(dcx, level); let (message, _) = diag.messages.first().expect("`LlvmError` with no message"); - let message = dcx.eagerly_translate_to_string(message.clone(), diag.args()); - + let message = dcx.eagerly_translate_to_string(message.clone(), diag.args.iter()); DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) .with_arg("error", message) } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4415c51acf6..58c0765d814 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1079,7 +1079,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .map(|(arg_idx, val)| { let idx = val.unwrap_leaf().try_to_i32().unwrap(); if idx >= i32::try_from(total_len).unwrap() { - bx.sess().dcx().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: arg_idx as u64, @@ -1138,24 +1138,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - bx.sess().dcx().emit_err( - InvalidMonomorphization::ShuffleIndexNotConstant { - span, - name, - arg_idx, - }, - ); - None + bug!("typeck should have already ensured that these are const") } Some(idx) if idx >= total_len => { - bx.sess().dcx().emit_err( - InvalidMonomorphization::ShuffleIndexOutOfBounds { - span, - name, - arg_idx, - total_len, - }, - ); + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx, + total_len, + }); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1184,10 +1175,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>( out_ty: arg_tys[2] } ); + let idx = bx + .const_to_opt_u128(args[1].immediate(), false) + .expect("typeck should have ensure that this is a const"); + if idx >= in_len.into() { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: 1, + total_len: in_len.into(), + }); + return Ok(bx.const_null(llret_ty)); + } return Ok(bx.insert_element( args[0].immediate(), args[2].immediate(), - args[1].immediate(), + bx.const_i32(idx as i32), )); } if name == sym::simd_extract { @@ -1195,7 +1198,19 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ret_ty == in_elem, InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); - return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); + let idx = bx + .const_to_opt_u128(args[1].immediate(), false) + .expect("typeck should have ensure that this is a const"); + if idx >= in_len.into() { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: 1, + total_len: in_len.into(), + }); + return Ok(bx.const_null(llret_ty)); + } + return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32))); } if name == sym::simd_select { @@ -1880,14 +1895,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, + vector_reduce_fadd_reassoc, false, add, 0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, + vector_reduce_fmul_reassoc, false, mul, 1.0 @@ -1920,9 +1935,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); - minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); - minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); - macro_rules! bitwise_red { ($name:ident : $red:ident, $boolean:expr) => { if name == sym::$name { @@ -2094,9 +2106,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(args[0].immediate()); } + #[derive(Copy, Clone)] + enum Sign { + Unsigned, + Signed, + } + use Sign::*; + enum Style { Float, - Int(/* is signed? */ bool), + Int(Sign), Unsupported, } @@ -2104,11 +2123,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // vectors of pointer-sized integers should've been // disallowed before here, so this unwrap is safe. ty::Int(i) => ( - Style::Int(true), + Style::Int(Signed), i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Uint(u) => ( - Style::Int(false), + Style::Int(Unsigned), u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Float(f) => (Style::Float, f.bit_width()), @@ -2116,11 +2135,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; let (out_style, out_width) = match out_elem.kind() { ty::Int(i) => ( - Style::Int(true), + Style::Int(Signed), i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Uint(u) => ( - Style::Int(false), + Style::Int(Unsigned), u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Float(f) => (Style::Float, f.bit_width()), @@ -2128,31 +2147,31 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; match (in_style, out_style) { - (Style::Int(in_is_signed), Style::Int(_)) => { + (Style::Int(sign), Style::Int(_)) => { return Ok(match in_width.cmp(&out_width) { Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), Ordering::Equal => args[0].immediate(), - Ordering::Less => { - if in_is_signed { - bx.sext(args[0].immediate(), llret_ty) - } else { - bx.zext(args[0].immediate(), llret_ty) - } - } + Ordering::Less => match sign { + Sign::Signed => bx.sext(args[0].immediate(), llret_ty), + Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty), + }, }); } - (Style::Int(in_is_signed), Style::Float) => { - return Ok(if in_is_signed { - bx.sitofp(args[0].immediate(), llret_ty) - } else { - bx.uitofp(args[0].immediate(), llret_ty) - }); + (Style::Int(Sign::Signed), Style::Float) => { + return Ok(bx.sitofp(args[0].immediate(), llret_ty)); + } + (Style::Int(Sign::Unsigned), Style::Float) => { + return Ok(bx.uitofp(args[0].immediate(), llret_ty)); } - (Style::Float, Style::Int(out_is_signed)) => { - return Ok(match (out_is_signed, name == sym::simd_as) { - (false, false) => bx.fptoui(args[0].immediate(), llret_ty), - (true, false) => bx.fptosi(args[0].immediate(), llret_ty), - (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), + (Style::Float, Style::Int(sign)) => { + return Ok(match (sign, name == sym::simd_as) { + (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty), + (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int( + matches!(sign, Sign::Signed), + args[0].immediate(), + llret_ty, + ), }); } (Style::Float, Style::Float) => { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 35210b0b2e8..c84461e53eb 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -169,7 +169,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_pass_timings(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintPassTimings(&mut size as *mut usize); + let cstr = llvm::LLVMRustPrintPassTimings(std::ptr::addr_of_mut!(size)); if cstr.is_null() { println!("failed to get pass timings"); } else { @@ -182,7 +182,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_statistics(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintStatistics(&mut size as *mut usize); + let cstr = llvm::LLVMRustPrintStatistics(std::ptr::addr_of_mut!(size)); if cstr.is_null() { println!("failed to get pass stats"); } else { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d0044086c61..dbf35e5f499 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1618,6 +1618,8 @@ extern "C" { ) -> &'a Value; pub fn LLVMRustSetFastMath(Instr: &Value); + pub fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 54e8ed85e32..1b2beac56a2 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -435,7 +435,7 @@ pub(crate) fn print(req: &PrintRequest, mut out: &mut dyn PrintBackendInfo, sess &tm, cpu_cstring.as_ptr(), callback, - &mut out as *mut &mut dyn PrintBackendInfo as *mut c_void, + std::ptr::addr_of_mut!(out) as *mut c_void, ); } } |