diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 48 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 110 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 49 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 15 |
11 files changed, 182 insertions, 74 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index dca9c1f04d3..bedd3523d89 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,7 +1,6 @@ use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, AttributePlace}; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -53,15 +52,10 @@ pub trait ArgAttributesExt { } fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // LLVM prior to version 12 has known miscompiles in the presence of - // noalias attributes (see #54878). Only enable mutable noalias by - // default for versions we believe to be safe. - cx.tcx - .sess - .opts - .debugging_opts - .mutable_noalias - .unwrap_or_else(|| llvm_util::get_version() >= (12, 0, 0)) + // LLVM prior to version 12 had known miscompiles in the presence of + // noalias attributes (see #54878), but we don't support earlier + // versions at all anymore. We now enable mutable noalias by default. + cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) } impl ArgAttributesExt for ArgAttributes { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index f128f769580..02096f4abfa 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -320,7 +320,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} InlineAsmArch::S390x => {} InlineAsmArch::SpirV => {} - InlineAsmArch::Wasm32 => {} + InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 82c3c2006eb..3d05fc15b38 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -17,6 +17,7 @@ use rustc_codegen_ssa::back::write::{ }; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; @@ -53,6 +54,7 @@ pub fn write_output_file( output: &Path, dwo_output: Option<&Path>, file_type: llvm::FileType, + self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { unsafe { let output_c = path_to_c_string(output); @@ -76,6 +78,19 @@ pub fn write_output_file( file_type, ) }; + + // Record artifact sizes for self-profiling + if result == llvm::LLVMRustResult::Success { + let artifact_kind = match file_type { + llvm::FileType::ObjectFile => "object_file", + llvm::FileType::AssemblyFile => "assembly_file", + }; + record_artifact_size(self_profiler_ref, artifact_kind, output); + if let Some(dwo_file) = dwo_output { + record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); + } + } + result.into_result().map_err(|()| { let msg = format!("could not write output to {}", output.display()); llvm_err(handler, &msg) @@ -286,7 +301,7 @@ fn report_inline_asm( cookie = 0; } let level = match level { - llvm::DiagnosticLevel::Error => Level::Error, + llvm::DiagnosticLevel::Error => Level::Error { lint: false }, llvm::DiagnosticLevel::Warning => Level::Warning, llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; @@ -752,6 +767,14 @@ pub(crate) unsafe fn codegen( let thin = ThinBuffer::new(llmod); let data = thin.data(); + if let Some(bitcode_filename) = bc_out.file_name() { + cgcx.prof.artifact_size( + "llvm_bitcode", + bitcode_filename.to_string_lossy(), + data.len() as u64, + ); + } + if config.emit_bc || config.emit_obj == EmitObj::Bitcode { let _timer = cgcx.prof.generic_activity_with_arg( "LLVM_module_codegen_emit_bitcode", @@ -812,6 +835,11 @@ pub(crate) unsafe fn codegen( } let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback); + + if result == llvm::LLVMRustResult::Success { + record_artifact_size(&cgcx.prof, "llvm_ir", &out); + } + result.into_result().map_err(|()| { let msg = format!("failed to write LLVM IR to {}", out.display()); llvm_err(diag_handler, &msg) @@ -842,6 +870,7 @@ pub(crate) unsafe fn codegen( &path, None, llvm::FileType::AssemblyFile, + &cgcx.prof, ) })?; } @@ -875,6 +904,7 @@ pub(crate) unsafe fn codegen( &obj_out, dwo_out, llvm::FileType::ObjectFile, + &cgcx.prof, ) })?; } @@ -1131,3 +1161,19 @@ fn create_msvc_imps( symbol_name.starts_with(b"__llvm_profile_") } } + +fn record_artifact_size( + self_profiler_ref: &SelfProfilerRef, + artifact_kind: &'static str, + path: &Path, +) { + // Don't stat the file if we are not going to record its size. + if !self_profiler_ref.enabled() { + return; + } + + if let Some(artifact_name) = path.file_name() { + let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); + } +} diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0707faf610c..ff88302bf7a 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -731,7 +731,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() { + if !self.fptoint_sat_broken_in_llvm() { let src_ty = self.cx.val_ty(val); let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); @@ -743,7 +743,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() { + if !self.fptoint_sat_broken_in_llvm() { let src_ty = self.cx.val_ty(val); let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); @@ -769,7 +769,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // we like. To ensure that LLVM picks the right instruction we choose // the raw wasm intrinsic functions which avoid LLVM inserting all the // other control flow automatically. - if self.sess().target.arch == "wasm32" { + if self.sess().target.is_like_wasm { let src_ty = self.cx.val_ty(val); if self.cx.type_kind(src_ty) != TypeKind::Vector { let float_width = self.cx.float_width(src_ty); @@ -791,7 +791,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { // see `fptoui` above for why wasm is different here - if self.sess().target.arch == "wasm32" { + if self.sess().target.is_like_wasm { let src_ty = self.cx.val_ty(val); if self.cx.type_kind(src_ty) != TypeKind::Vector { let float_width = self.cx.float_width(src_ty); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 1afa6f02836..b154ced42f0 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -490,7 +490,7 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { // Wasm statics with custom link sections get special treatment as they // go into custom sections of the wasm executable. - if self.tcx.sess.opts.target_triple.triple().starts_with("wasm32") { + if self.tcx.sess.target.is_like_wasm { if let Some(section) = attrs.link_section { let section = llvm::LLVMMDStringInContext( self.llcx, diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index cda766039c1..613a8df891c 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -134,9 +134,6 @@ pub unsafe fn create_module( let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); let mut target_data_layout = sess.target.data_layout.clone(); - if llvm_util::get_version() < (12, 0, 0) && sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-v256:256:256-v512:512:512", ""); - } if llvm_util::get_version() < (13, 0, 0) { if sess.target.arch == "powerpc64" { target_data_layout = target_data_layout.replace("-S128", ""); @@ -600,7 +597,6 @@ impl CodegenCx<'b, 'tcx> { ifn!("llvm.trap", fn() -> void); ifn!("llvm.debugtrap", fn() -> void); ifn!("llvm.frameaddress", fn(t_i32) -> i8p); - ifn!("llvm.sideeffect", fn() -> void); ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 1f1bd73c7d0..2a6bf7d9b1a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -322,7 +322,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { type_names::push_item_name(self.tcx(), def_id, false, &mut name); // Find the enclosing function, in case this is a closure. - let enclosing_fn_def_id = self.tcx().closure_base_def_id(def_id); + let enclosing_fn_def_id = self.tcx().typeck_root_def_id(def_id); // Get_template_parameters() will append a `<...>` clause to the function // name if necessary. diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index e63fb22829a..a7e34b08059 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -19,7 +19,7 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; -use rustc_target::abi::{self, HasDataLayout, Primitive}; +use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; @@ -392,15 +392,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) } - fn sideeffect(&mut self) { - // This kind of check would make a ton of sense in the caller, but currently the only - // caller of this function is in `rustc_codegen_ssa`, which is agnostic to whether LLVM - // codegen backend being used, and so is unable to check the LLVM version. - if unsafe { llvm::LLVMRustVersionMajor() } < 12 { - self.call_intrinsic("llvm.sideeffect", &[]); - } - } - fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time // optimization pass replaces calls to this intrinsic with code to test type membership. @@ -857,28 +848,39 @@ fn generic_simd_intrinsic( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - let in_ty = arg_tys[0]; - let m_len = match in_ty.kind() { - // Note that this `.unwrap()` crashes for isize/usize, that's sort - // of intentional as there's not currently a use case for that. - ty::Int(i) => i.bit_width().unwrap(), - ty::Uint(i) => i.bit_width().unwrap(), - _ => return_error!("`{}` is not an integral type", in_ty), - }; require_simd!(arg_tys[1], "argument"); - let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - require!( - // Allow masks for vectors with fewer than 8 elements to be - // represented with a u8 or i8. - m_len == v_len || (m_len == 8 && v_len < 8), - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len - ); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + + let expected_int_bits = (len.max(8) - 1).next_power_of_two(); + let expected_bytes = len / 8 + ((len % 8 > 0) as u64); + + let mask_ty = arg_tys[0]; + let mask = match mask_ty.kind() { + ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + let place = PlaceRef::alloca(bx, args[0].layout); + args[0].val.store(bx, place); + let int_ty = bx.type_ix(expected_bytes * 8); + let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); + bx.load(int_ty, ptr, Align::ONE) + } + _ => return_error!( + "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + mask_ty, + expected_int_bits, + expected_bytes + ), + }; + let i1 = bx.type_i1(); - let im = bx.type_ix(v_len); - let i1xn = bx.type_vector(i1, v_len); - let m_im = bx.trunc(args[0].immediate(), im); + let im = bx.type_ix(len); + let i1xn = bx.type_vector(i1, len); + let m_im = bx.trunc(mask, im); let m_i1s = bx.bitcast(m_im, i1xn); return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } @@ -1056,16 +1058,16 @@ fn generic_simd_intrinsic( if name == sym::simd_bitmask { // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a - // vector mask and returns an unsigned integer containing the most - // significant bit (MSB) of each lane. - - // If the vector has less than 8 lanes, a u8 is returned with zeroed - // trailing bits. + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. let expected_int_bits = in_len.max(8); - match ret_ty.kind() { - ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (), - _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits), - } + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); // Integer vector <i{in_bitwidth} x in_len>: let (i_xn, in_elem_bitwidth) = match in_elem.kind() { @@ -1095,8 +1097,34 @@ fn generic_simd_intrinsic( let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); // Bitcast <i1 x N> to iN: let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); - // Zero-extend iN to the bitmask type: - return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + + match ret_ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { + // Zero-extend iN to the bitmask type: + return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + } + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + // Zero-extend iN to the array lengh: + let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); + + // Convert the integer to a byte array + let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); + bx.store(ze, ptr, Align::ONE); + let array_ty = bx.type_array(bx.type_i8(), expected_bytes); + let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); + return Ok(bx.load(array_ty, ptr, Align::ONE)); + } + _ => return_error!( + "cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret_ty, + expected_int_bits, + expected_bytes + ), + } } fn simd_simple_float_intrinsic( diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 8f4d79e7147..64fedb7bc1a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -76,6 +76,27 @@ mod value; #[derive(Clone)] pub struct LlvmCodegenBackend(()); +struct TimeTraceProfiler { + enabled: bool, +} + +impl TimeTraceProfiler { + fn new(enabled: bool) -> Self { + if enabled { + unsafe { llvm::LLVMTimeTraceProfilerInitialize() } + } + TimeTraceProfiler { enabled } + } +} + +impl Drop for TimeTraceProfiler { + fn drop(&mut self) { + if self.enabled { + unsafe { llvm::LLVMTimeTraceProfilerFinishThread() } + } + } +} + impl ExtraBackendMethods for LlvmCodegenBackend { fn new_metadata(&self, tcx: TyCtxt<'_>, mod_name: &str) -> ModuleLlvm { ModuleLlvm::new_metadata(tcx, mod_name) @@ -119,6 +140,34 @@ impl ExtraBackendMethods for LlvmCodegenBackend { fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { llvm_util::tune_cpu(sess) } + + fn spawn_thread<F, T>(time_trace: bool, f: F) -> std::thread::JoinHandle<T> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } + + fn spawn_named_thread<F, T>( + time_trace: bool, + name: String, + f: F, + ) -> std::io::Result<std::thread::JoinHandle<T>> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::Builder::new().name(name).spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } } impl WriteBackendMethods for LlvmCodegenBackend { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 21d2388fc30..749eec459ac 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1737,6 +1737,8 @@ extern "C" { pub fn LLVMTimeTraceProfilerInitialize(); + pub fn LLVMTimeTraceProfilerFinishThread(); + pub fn LLVMTimeTraceProfilerFinish(FileName: *const c_char); pub fn LLVMAddAnalysisPasses(T: &'a TargetMachine, PM: &PassManager<'a>); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index baa257069aa..3393c9baa28 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -75,7 +75,9 @@ unsafe fn configure_llvm(sess: &Session) { if sess.print_llvm_passes() { add("-debug-pass=Structure", false); } - if !sess.opts.debugging_opts.no_generate_arange_section { + if sess.target.generate_arange_section + && !sess.opts.debugging_opts.no_generate_arange_section + { add("-generate-arange-section", false); } @@ -113,11 +115,6 @@ unsafe fn configure_llvm(sess: &Session) { } if sess.opts.debugging_opts.llvm_time_trace { - // time-trace is not thread safe and running it in parallel will cause seg faults. - if !sess.opts.debugging_opts.no_parallel_llvm { - bug!("`-Z llvm-time-trace` requires `-Z no-parallel-llvm") - } - llvm::LLVMTimeTraceProfilerInitialize(); } @@ -180,6 +177,7 @@ pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> Vec<&'a str> { ("aarch64", "dpb2") => vec!["ccdp"], ("aarch64", "frintts") => vec!["fptoint"], ("aarch64", "fcma") => vec!["complxnum"], + ("aarch64", "pmuv3") => vec!["perfmon"], (_, s) => vec![s], } } @@ -405,11 +403,6 @@ pub fn llvm_global_features(sess: &Session) -> Vec<String> { // -Ctarget-features features.extend(sess.opts.cg.target_feature.split(',').flat_map(&filter)); - // FIXME: Move outline-atomics to target definition when earliest supported LLVM is 12. - if get_version() >= (12, 0, 0) && sess.target.llvm_target.contains("aarch64-unknown-linux") { - features.push("+outline-atomics".to_string()); - } - features } |