From e5b76892cc5e6fbfb495bf6a7339962a287849a7 Mon Sep 17 00:00:00 2001 From: quininer Date: Mon, 13 Nov 2023 20:48:23 +0800 Subject: Add emulated TLS support Currently LLVM uses emutls by default for some targets (such as android, openbsd), but rust does not use it, because `has_thread_local` is false. This commit has some changes to allow users to enable emutls: 1. add `-Zhas-thread-local` flag to specify that std uses `#[thread_local]` instead of pthread key. 2. when using emutls, decorate symbol names to find thread local symbol correctly. 3. change `-Zforce-emulated-tls` to `-Ztls-model=emulated` to explicitly specify whether to generate emutls. --- compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 4 ++-- compiler/rustc_codegen_llvm/src/back/write.rs | 6 +++--- compiler/rustc_codegen_llvm/src/context.rs | 1 + compiler/rustc_codegen_llvm/src/lib.rs | 4 +++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 36484c3c3fc..28a88dd2efe 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -39,7 +39,7 @@ impl OwnedTargetMachine { split_dwarf_file: &CStr, output_obj_file: &CStr, debug_info_compression: &CStr, - force_emulated_tls: bool, + use_emulated_tls: bool, args_cstr_buff: &[u8], ) -> Result> { assert!(args_cstr_buff.len() > 0); @@ -71,7 +71,7 @@ impl OwnedTargetMachine { split_dwarf_file.as_ptr(), output_obj_file.as_ptr(), debug_info_compression.as_ptr(), - force_emulated_tls, + use_emulated_tls, args_cstr_buff.as_ptr() as *const c_char, args_cstr_buff.len(), ) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 1a567c0fce8..bdabb9129a7 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -33,7 +33,7 @@ use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, Switc use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; -use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; @@ -223,7 +223,7 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); - let force_emulated_tls = sess.target.force_emulated_tls; + let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); // copy the exe path, followed by path all into one buffer // null terminating them so we can use them as null terminated strings @@ -297,7 +297,7 @@ pub fn target_machine_factory( &split_dwarf_file, &output_obj_file, &debuginfo_compression, - force_emulated_tls, + use_emulated_tls, &args_cstr_buff, ) }) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 92a8c00510b..ed0d5e68cbf 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -120,6 +120,7 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic, TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec, TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec, + TlsModel::Emulated => llvm::ThreadLocalMode::GeneralDynamic, } } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 915cf31de08..33b19ab362a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -306,7 +306,9 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:"); - for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { + for name in + &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] + { writeln!(out, " {name}"); } writeln!(out); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6c3ccc9cf0d..432cfe203c8 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2159,7 +2159,7 @@ extern "C" { SplitDwarfFile: *const c_char, OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, - ForceEmulatedTls: bool, + UseEmulatedTls: bool, ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; -- cgit 1.4.1-3-g733a5 From 44b47aa976d6a6bdd7eb1f99a8ee5270afbe993e Mon Sep 17 00:00:00 2001 From: Zalathar Date: Thu, 23 Nov 2023 11:50:39 +1100 Subject: coverage: Add `CoverageKind::SpanMarker` for including extra spans in MIR There are cases where coverage instrumentation wants to show a span for some syntax element, but there is no MIR node that naturally carries that span, so the instrumentor can't see it. MIR building can now use this new kind of coverage statement to deliberately include those spans in MIR, attached to a dummy statement that has no other effect. --- compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 3 +++ compiler/rustc_middle/src/mir/coverage.rs | 8 ++++++++ compiler/rustc_mir_build/src/build/cfg.rs | 13 +++++++++++++ .../rustc_mir_transform/src/coverage/spans/from_mir.rs | 15 ++++++++++++--- 4 files changed, 36 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 7d69756181a..8386f067baf 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -100,6 +100,9 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let Coverage { kind } = coverage; match *kind { + // Span markers are only meaningful during MIR instrumentation, + // and have no effect during codegen. + CoverageKind::SpanMarker => {} CoverageKind::CounterIncrement { id } => { func_coverage.mark_counter_id_seen(id); // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index f15ee0082ce..ec5edceb269 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -76,6 +76,13 @@ impl Debug for CovTerm { #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] pub enum CoverageKind { + /// Marks a span that might otherwise not be represented in MIR, so that + /// coverage instrumentation can associate it with its enclosing block/BCB. + /// + /// Only used by the `InstrumentCoverage` pass, and has no effect during + /// codegen. + SpanMarker, + /// Marks the point in MIR control flow represented by a coverage counter. /// /// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR. @@ -99,6 +106,7 @@ impl Debug for CoverageKind { fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { use CoverageKind::*; match self { + SpanMarker => write!(fmt, "SpanMarker"), CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()), ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()), } diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs index fddcf9de7c7..2bd0e289731 100644 --- a/compiler/rustc_mir_build/src/build/cfg.rs +++ b/compiler/rustc_mir_build/src/build/cfg.rs @@ -101,6 +101,19 @@ impl<'tcx> CFG<'tcx> { self.push(block, stmt); } + /// Adds a dummy statement whose only role is to associate a span with its + /// enclosing block for the purposes of coverage instrumentation. + /// + /// This results in more accurate coverage reports for certain kinds of + /// syntax (e.g. `continue` or `if !`) that would otherwise not appear in MIR. + pub(crate) fn push_coverage_span_marker(&mut self, block: BasicBlock, source_info: SourceInfo) { + let kind = StatementKind::Coverage(Box::new(Coverage { + kind: coverage::CoverageKind::SpanMarker, + })); + let stmt = Statement { source_info, kind }; + self.push(block, stmt); + } + pub(crate) fn terminate( &mut self, block: BasicBlock, diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index e1531f2c239..6f7d8d9dd75 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -92,13 +92,13 @@ fn is_closure(statement: &Statement<'_>) -> bool { /// If the MIR `Statement` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. fn filtered_statement_span(statement: &Statement<'_>) -> Option { + use mir::coverage::CoverageKind; + match statement.kind { // These statements have spans that are often outside the scope of the executed source code // for their parent `BasicBlock`. StatementKind::StorageLive(_) | StatementKind::StorageDead(_) - // Coverage should not be encountered, but don't inject coverage coverage - | StatementKind::Coverage(_) // Ignore `ConstEvalCounter`s | StatementKind::ConstEvalCounter // Ignore `Nop`s @@ -122,9 +122,13 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option { // If and when the Issue is resolved, remove this special case match pattern: StatementKind::FakeRead(box (FakeReadCause::ForGuardBinding, _)) => None, - // Retain spans from all other statements + // Retain spans from most other statements. StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding` | StatementKind::Intrinsic(..) + | StatementKind::Coverage(box mir::Coverage { + // The purpose of `SpanMarker` is to be matched and accepted here. + kind: CoverageKind::SpanMarker + }) | StatementKind::Assign(_) | StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(..) @@ -133,6 +137,11 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option { | StatementKind::AscribeUserType(_, _) => { Some(statement.source_info.span) } + + StatementKind::Coverage(box mir::Coverage { + // These coverage statements should not exist prior to coverage instrumentation. + kind: CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. } + }) => bug!("Unexpected coverage statement found during coverage instrumentation: {statement:?}"), } } -- cgit 1.4.1-3-g733a5 From b378059e6b2573c5356423fa31d184a89a3b6029 Mon Sep 17 00:00:00 2001 From: Krasimir Georgiev Date: Mon, 4 Dec 2023 15:52:26 +0000 Subject: update target feature following LLVM API change LLVM commit https://github.com/llvm/llvm-project/commit/e81796671890b59c110f8e41adc7ca26f8484d20 renamed the `unaligned-scalar-mem` target feature to `fast-unaligned-access`. --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 4 ++++ compiler/rustc_codegen_ssa/src/target_features.rs | 2 +- tests/ui/abi/riscv-discoverability-guidance.rs | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index eb69efb0d59..93cb7327a01 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -263,6 +263,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { "sve2-bitperm", TargetFeatureFoldStrength::EnableOnly("neon"), ), + // The unaligned-scalar-mem feature was renamed to fast-unaligned-access. + ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { + LLVMFeature::new("unaligned-scalar-mem") + } (_, s) => LLVMFeature::new(s), } } diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index d802816bb75..c3b8859c779 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -291,9 +291,9 @@ const RISCV_ALLOWED_FEATURES: &[(&str, Stability)] = &[ ("d", Unstable(sym::riscv_target_feature)), ("e", Unstable(sym::riscv_target_feature)), ("f", Unstable(sym::riscv_target_feature)), + ("fast-unaligned-access", Unstable(sym::riscv_target_feature)), ("m", Stable), ("relax", Unstable(sym::riscv_target_feature)), - ("unaligned-scalar-mem", Unstable(sym::riscv_target_feature)), ("v", Unstable(sym::riscv_target_feature)), ("zba", Stable), ("zbb", Stable), diff --git a/tests/ui/abi/riscv-discoverability-guidance.rs b/tests/ui/abi/riscv-discoverability-guidance.rs index f57fcd6044f..361ed8f3d91 100644 --- a/tests/ui/abi/riscv-discoverability-guidance.rs +++ b/tests/ui/abi/riscv-discoverability-guidance.rs @@ -2,9 +2,9 @@ // revisions: riscv32 riscv64 // // [riscv32] needs-llvm-components: riscv -// [riscv32] compile-flags: --target=riscv32i-unknown-none-elf -C target-feature=-unaligned-scalar-mem --crate-type=rlib +// [riscv32] compile-flags: --target=riscv32i-unknown-none-elf -C target-feature=-fast-unaligned-access --crate-type=rlib // [riscv64] needs-llvm-components: riscv -// [riscv64] compile-flags: --target=riscv64gc-unknown-none-elf -C target-feature=-unaligned-scalar-mem --crate-type=rlib +// [riscv64] compile-flags: --target=riscv64gc-unknown-none-elf -C target-feature=-fast-unaligned-access --crate-type=rlib #![no_core] #![feature( no_core, -- cgit 1.4.1-3-g733a5 From 97ae5095f52100b98170ab476e516d2be5b2c297 Mon Sep 17 00:00:00 2001 From: Jakub Okoński Date: Wed, 15 Nov 2023 23:08:02 +0100 Subject: Add simd_masked_{load,store} platform-intrinsics This maps to the LLVM intrinsics: llvm.masked.load and llvm.masked.store --- .../rustc_codegen_cranelift/src/intrinsics/simd.rs | 52 +++++- compiler/rustc_codegen_llvm/src/intrinsic.rs | 192 +++++++++++++++++++++ compiler/rustc_hir_analysis/src/check/intrinsic.rs | 2 + compiler/rustc_span/src/symbol.rs | 2 + .../simd-intrinsic-generic-masked-load.rs | 34 ++++ .../simd-intrinsic-generic-masked-store.rs | 32 ++++ tests/ui/simd/masked-load-store-build-fail.rs | 74 ++++++++ tests/ui/simd/masked-load-store-build-fail.stderr | 83 +++++++++ tests/ui/simd/masked-load-store-check-fail.rs | 32 ++++ tests/ui/simd/masked-load-store-check-fail.stderr | 59 +++++++ tests/ui/simd/masked-load-store.rs | 33 ++++ 11 files changed, 594 insertions(+), 1 deletion(-) create mode 100644 tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs create mode 100644 tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs create mode 100644 tests/ui/simd/masked-load-store-build-fail.rs create mode 100644 tests/ui/simd/masked-load-store-build-fail.stderr create mode 100644 tests/ui/simd/masked-load-store-check-fail.rs create mode 100644 tests/ui/simd/masked-load-store-check-fail.stderr create mode 100644 tests/ui/simd/masked-load-store.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 0bd211fd614..5997e6026b4 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -1,5 +1,6 @@ //! Codegen `extern "platform-intrinsic"` intrinsics. +use cranelift_codegen::ir::immediates::Offset32; use rustc_middle::ty::GenericArgsRef; use rustc_span::Symbol; use rustc_target::abi::Endian; @@ -1008,8 +1009,57 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } } + sym::simd_masked_load => { + intrinsic_args!(fx, args => (mask, ptr, val); intrinsic); + + let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(val_lane_count, mask_lane_count); + assert_eq!(val_lane_count, ret_lane_count); + + let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap(); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + let ptr_val = ptr.load_scalar(fx); + + for lane_idx in 0..ret_lane_count { + let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); + + let if_enabled = fx.bcx.create_block(); + let if_disabled = fx.bcx.create_block(); + let next = fx.bcx.create_block(); + let res_lane = fx.bcx.append_block_param(next, lane_clif_ty); + + fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]); + fx.bcx.seal_block(if_enabled); + fx.bcx.seal_block(if_disabled); + + fx.bcx.switch_to_block(if_enabled); + let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32; + let res = fx.bcx.ins().load( + lane_clif_ty, + MemFlags::trusted(), + ptr_val, + Offset32::new(offset), + ); + fx.bcx.ins().jump(next, &[res]); + + fx.bcx.switch_to_block(if_disabled); + fx.bcx.ins().jump(next, &[val_lane]); + + fx.bcx.seal_block(next); + fx.bcx.switch_to_block(next); + + fx.bcx.ins().nop(); + + ret.place_lane(fx, lane_idx) + .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout)); + } + } + sym::simd_scatter => { - intrinsic_args!(fx, args => (val, ptr, mask); intrinsic); + intrinsic_args!(fx, args => (mask, ptr, val); intrinsic); let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index cc7e78b9c62..f16014e1361 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1492,6 +1492,198 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(v); } + if name == sym::simd_masked_load { + // simd_masked_load(mask: , pointer: *_ T, values: ) -> + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + // Loads contiguous elements from memory behind `pointer`, but only for + // those lanes whose `mask` bit is enabled. + // The memory addresses corresponding to the “off” lanes are not accessed. + + // The element type of the "mask" argument must be a signed integer type of any width + let mask_ty = in_ty; + let (mask_len, mask_elem) = (in_len, in_elem); + + // The second argument must be a pointer matching the element type + let pointer_ty = arg_tys[1]; + + // The last argument is a passthrough vector providing values for disabled lanes + let values_ty = arg_tys[2]; + let (values_len, values_elem) = require_simd!(values_ty, SimdThird); + + require_simd!(ret_ty, SimdReturn); + + // Of the same length: + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); + + // The return type must match the last argument type + require!( + ret_ty == values_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty } + ); + + require!( + matches!( + pointer_ty.kind(), + ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: values_elem, + second_arg: pointer_ty, + in_elem: values_elem, + in_ty: values_ty, + mutability: ExpectedPointerMutability::Not, + } + ); + + require!( + matches!(mask_elem.kind(), ty::Int(_)), + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: values_elem, + third_arg: mask_ty, + } + ); + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(values_ty).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, mask_len); + (bx.trunc(args[0].immediate(), i1xn), i1xn) + }; + + let llvm_pointer = bx.type_ptr(); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); + let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); + + let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0"); + let fn_ty = bx + .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[1].immediate(), alignment, mask, args[2].immediate()], + None, + ); + return Ok(v); + } + + if name == sym::simd_masked_store { + // simd_masked_store(mask: , pointer: *mut T, values: ) -> () + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + // Stores contiguous elements to memory behind `pointer`, but only for + // those lanes whose `mask` bit is enabled. + // The memory addresses corresponding to the “off” lanes are not accessed. + + // The element type of the "mask" argument must be a signed integer type of any width + let mask_ty = in_ty; + let (mask_len, mask_elem) = (in_len, in_elem); + + // The second argument must be a pointer matching the element type + let pointer_ty = arg_tys[1]; + + // The last argument specifies the values to store to memory + let values_ty = arg_tys[2]; + let (values_len, values_elem) = require_simd!(values_ty, SimdThird); + + // Of the same length: + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); + + // The second argument must be a mutable pointer type matching the element type + require!( + matches!( + pointer_ty.kind(), + ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: values_elem, + second_arg: pointer_ty, + in_elem: values_elem, + in_ty: values_ty, + mutability: ExpectedPointerMutability::Mut, + } + ); + + require!( + matches!(mask_elem.kind(), ty::Int(_)), + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: values_elem, + third_arg: mask_ty, + } + ); + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[0].immediate(), i1xn), i1xn) + }; + + let ret_t = bx.type_void(); + + let llvm_pointer = bx.type_ptr(); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); + let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); + + let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0"); + let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[2].immediate(), args[1].immediate(), alignment, mask], + None, + ); + return Ok(v); + } + if name == sym::simd_scatter { // simd_scatter(values: , pointers: , // mask: ) -> () diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 7ea21b24fc8..33337190562 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -521,6 +521,8 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) sym::simd_fpowi => (1, 0, vec![param(0), tcx.types.i32], param(0)), sym::simd_fma => (1, 0, vec![param(0), param(0), param(0)], param(0)), sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)), + sym::simd_masked_load => (3, 0, vec![param(0), param(1), param(2)], param(2)), + sym::simd_masked_store => (3, 0, vec![param(0), param(1), param(2)], Ty::new_unit(tcx)), sym::simd_scatter => (3, 0, vec![param(0), param(1), param(2)], Ty::new_unit(tcx)), sym::simd_insert => (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0)), sym::simd_extract => (2, 0, vec![param(0), tcx.types.u32], param(1)), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 7b9b7b85293..485265e6889 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1516,6 +1516,8 @@ symbols! { simd_insert, simd_le, simd_lt, + simd_masked_load, + simd_masked_store, simd_mul, simd_ne, simd_neg, diff --git a/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs new file mode 100644 index 00000000000..7b1fb320894 --- /dev/null +++ b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-load.rs @@ -0,0 +1,34 @@ +// compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] + +#![feature(repr_simd, platform_intrinsics)] +#![allow(non_camel_case_types)] + +#[repr(simd)] +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct Vec2(pub T, pub T); + +#[repr(simd)] +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct Vec4(pub T, pub T, pub T, pub T); + +extern "platform-intrinsic" { + fn simd_masked_load(mask: M, pointer: P, values: T) -> T; +} + +// CHECK-LABEL: @load_f32x2 +#[no_mangle] +pub unsafe fn load_f32x2(mask: Vec2, pointer: *const f32, + values: Vec2) -> Vec2 { + // CHECK: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 {{.*}}, <2 x i1> {{.*}}, <2 x float> {{.*}}) + simd_masked_load(mask, pointer, values) +} + +// CHECK-LABEL: @load_pf32x4 +#[no_mangle] +pub unsafe fn load_pf32x4(mask: Vec4, pointer: *const *const f32, + values: Vec4<*const f32>) -> Vec4<*const f32> { + // CHECK: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> {{.*}}, <4 x ptr> {{.*}}) + simd_masked_load(mask, pointer, values) +} diff --git a/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs new file mode 100644 index 00000000000..d8a37020f23 --- /dev/null +++ b/tests/codegen/simd-intrinsic/simd-intrinsic-generic-masked-store.rs @@ -0,0 +1,32 @@ +// compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] + +#![feature(repr_simd, platform_intrinsics)] +#![allow(non_camel_case_types)] + +#[repr(simd)] +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct Vec2(pub T, pub T); + +#[repr(simd)] +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct Vec4(pub T, pub T, pub T, pub T); + +extern "platform-intrinsic" { + fn simd_masked_store(mask: M, pointer: P, values: T) -> (); +} + +// CHECK-LABEL: @store_f32x2 +#[no_mangle] +pub unsafe fn store_f32x2(mask: Vec2, pointer: *mut f32, values: Vec2) { + // CHECK: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 {{.*}}, <2 x i1> {{.*}}) + simd_masked_store(mask, pointer, values) +} + +// CHECK-LABEL: @store_pf32x4 +#[no_mangle] +pub unsafe fn store_pf32x4(mask: Vec4, pointer: *mut *const f32, values: Vec4<*const f32>) { + // CHECK: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> {{.*}}) + simd_masked_store(mask, pointer, values) +} diff --git a/tests/ui/simd/masked-load-store-build-fail.rs b/tests/ui/simd/masked-load-store-build-fail.rs new file mode 100644 index 00000000000..9b79b3bd6ea --- /dev/null +++ b/tests/ui/simd/masked-load-store-build-fail.rs @@ -0,0 +1,74 @@ +// build-fail +#![feature(repr_simd, platform_intrinsics)] + +extern "platform-intrinsic" { + fn simd_masked_load(mask: M, pointer: P, values: T) -> T; + fn simd_masked_store(mask: M, pointer: P, values: T) -> (); +} + +#[derive(Copy, Clone)] +#[repr(simd)] +struct Simd([T; N]); + +fn main() { + unsafe { + let mut arr = [4u8, 5, 6, 7]; + let default = Simd::([9; 4]); + + simd_masked_load( + Simd::([-1, 0, -1, -1, 0, 0, 0, 0]), + arr.as_ptr(), + default + ); + //~^^^^^ ERROR expected third argument with length 8 (same as input type `Simd`), found `Simd` with length 4 + + simd_masked_load( + Simd::([-1, 0, -1, -1]), + arr.as_ptr() as *const i8, + default + ); + //~^^^^^ ERROR expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd`, found `u8` != `*_ u8` + + simd_masked_load( + Simd::([-1, 0, -1, -1]), + arr.as_ptr(), + Simd::([9; 4]) + ); + //~^^^^^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd`, found `u32` != `*_ u32` + + simd_masked_load( + Simd::([1, 0, 1, 1]), + arr.as_ptr(), + default + ); + //~^^^^^ ERROR expected element type `u8` of third argument `Simd` to be a signed integer type + + simd_masked_store( + Simd([-1i8; 4]), + arr.as_ptr(), + Simd([5u32; 4]) + ); + //~^^^^^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd`, found `u32` != `*mut u32` + + simd_masked_store( + Simd([-1i8; 4]), + arr.as_ptr(), + Simd([5u8; 4]) + ); + //~^^^^^ ERROR expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd`, found `u8` != `*mut u8` + + simd_masked_store( + Simd([-1i8; 4]), + arr.as_mut_ptr(), + Simd([5u8; 2]) + ); + //~^^^^^ ERROR expected third argument with length 4 (same as input type `Simd`), found `Simd` with length 2 + + simd_masked_store( + Simd([1u32; 4]), + arr.as_mut_ptr(), + Simd([5u8; 4]) + ); + //~^^^^^ ERROR expected element type `u8` of third argument `Simd` to be a signed integer type + } +} diff --git a/tests/ui/simd/masked-load-store-build-fail.stderr b/tests/ui/simd/masked-load-store-build-fail.stderr new file mode 100644 index 00000000000..59af83fe0e8 --- /dev/null +++ b/tests/ui/simd/masked-load-store-build-fail.stderr @@ -0,0 +1,83 @@ +error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected third argument with length 8 (same as input type `Simd`), found `Simd` with length 4 + --> $DIR/masked-load-store-build-fail.rs:18:9 + | +LL | / simd_masked_load( +LL | | Simd::([-1, 0, -1, -1, 0, 0, 0, 0]), +LL | | arr.as_ptr(), +LL | | default +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u8` of second argument `*const i8` to be a pointer to the element type `u8` of the first argument `Simd`, found `u8` != `*_ u8` + --> $DIR/masked-load-store-build-fail.rs:25:9 + | +LL | / simd_masked_load( +LL | | Simd::([-1, 0, -1, -1]), +LL | | arr.as_ptr() as *const i8, +LL | | default +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd`, found `u32` != `*_ u32` + --> $DIR/masked-load-store-build-fail.rs:32:9 + | +LL | / simd_masked_load( +LL | | Simd::([-1, 0, -1, -1]), +LL | | arr.as_ptr(), +LL | | Simd::([9; 4]) +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_load` intrinsic: expected element type `u8` of third argument `Simd` to be a signed integer type + --> $DIR/masked-load-store-build-fail.rs:39:9 + | +LL | / simd_masked_load( +LL | | Simd::([1, 0, 1, 1]), +LL | | arr.as_ptr(), +LL | | default +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd`, found `u32` != `*mut u32` + --> $DIR/masked-load-store-build-fail.rs:46:9 + | +LL | / simd_masked_store( +LL | | Simd([-1i8; 4]), +LL | | arr.as_ptr(), +LL | | Simd([5u32; 4]) +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd`, found `u8` != `*mut u8` + --> $DIR/masked-load-store-build-fail.rs:53:9 + | +LL | / simd_masked_store( +LL | | Simd([-1i8; 4]), +LL | | arr.as_ptr(), +LL | | Simd([5u8; 4]) +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected third argument with length 4 (same as input type `Simd`), found `Simd` with length 2 + --> $DIR/masked-load-store-build-fail.rs:60:9 + | +LL | / simd_masked_store( +LL | | Simd([-1i8; 4]), +LL | | arr.as_mut_ptr(), +LL | | Simd([5u8; 2]) +LL | | ); + | |_________^ + +error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of third argument `Simd` to be a signed integer type + --> $DIR/masked-load-store-build-fail.rs:67:9 + | +LL | / simd_masked_store( +LL | | Simd([1u32; 4]), +LL | | arr.as_mut_ptr(), +LL | | Simd([5u8; 4]) +LL | | ); + | |_________^ + +error: aborting due to 8 previous errors + +For more information about this error, try `rustc --explain E0511`. diff --git a/tests/ui/simd/masked-load-store-check-fail.rs b/tests/ui/simd/masked-load-store-check-fail.rs new file mode 100644 index 00000000000..d4b35e211c8 --- /dev/null +++ b/tests/ui/simd/masked-load-store-check-fail.rs @@ -0,0 +1,32 @@ +// check-fail +#![feature(repr_simd, platform_intrinsics)] + +extern "platform-intrinsic" { + fn simd_masked_load(mask: M, pointer: P, values: T) -> T; + fn simd_masked_store(mask: M, pointer: P, values: T) -> (); +} + +#[derive(Copy, Clone)] +#[repr(simd)] +struct Simd([T; N]); + +fn main() { + unsafe { + let mut arr = [4u8, 5, 6, 7]; + let default = Simd::([9; 4]); + + let _x: Simd = simd_masked_load( + Simd::([-1, 0, -1, -1]), + arr.as_ptr(), + Simd::([9; 4]) + ); + //~^^ ERROR mismatched types + + let _x: Simd = simd_masked_load( + Simd::([1, 0, 1, 1]), + arr.as_ptr(), + default + ); + //~^^ ERROR mismatched types + } +} diff --git a/tests/ui/simd/masked-load-store-check-fail.stderr b/tests/ui/simd/masked-load-store-check-fail.stderr new file mode 100644 index 00000000000..5d205d607c9 --- /dev/null +++ b/tests/ui/simd/masked-load-store-check-fail.stderr @@ -0,0 +1,59 @@ +error[E0308]: mismatched types + --> $DIR/masked-load-store-check-fail.rs:21:13 + | +LL | let _x: Simd = simd_masked_load( + | ---------------- arguments to this function are incorrect +... +LL | Simd::([9; 4]) + | ^^^^^^^^^^^^^^^^^^^^^ expected `2`, found `4` + | + = note: expected struct `Simd<_, 2>` + found struct `Simd<_, 4>` +help: the return type of this call is `Simd` due to the type of the argument passed + --> $DIR/masked-load-store-check-fail.rs:18:31 + | +LL | let _x: Simd = simd_masked_load( + | _______________________________^ +LL | | Simd::([-1, 0, -1, -1]), +LL | | arr.as_ptr(), +LL | | Simd::([9; 4]) + | | --------------------- this argument influences the return type of `simd_masked_load` +LL | | ); + | |_________^ +note: function defined here + --> $DIR/masked-load-store-check-fail.rs:5:8 + | +LL | fn simd_masked_load(mask: M, pointer: P, values: T) -> T; + | ^^^^^^^^^^^^^^^^ + +error[E0308]: mismatched types + --> $DIR/masked-load-store-check-fail.rs:28:13 + | +LL | let _x: Simd = simd_masked_load( + | ---------------- arguments to this function are incorrect +... +LL | default + | ^^^^^^^ expected `Simd`, found `Simd` + | + = note: expected struct `Simd` + found struct `Simd` +help: the return type of this call is `Simd` due to the type of the argument passed + --> $DIR/masked-load-store-check-fail.rs:25:32 + | +LL | let _x: Simd = simd_masked_load( + | ________________________________^ +LL | | Simd::([1, 0, 1, 1]), +LL | | arr.as_ptr(), +LL | | default + | | ------- this argument influences the return type of `simd_masked_load` +LL | | ); + | |_________^ +note: function defined here + --> $DIR/masked-load-store-check-fail.rs:5:8 + | +LL | fn simd_masked_load(mask: M, pointer: P, values: T) -> T; + | ^^^^^^^^^^^^^^^^ + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0308`. diff --git a/tests/ui/simd/masked-load-store.rs b/tests/ui/simd/masked-load-store.rs new file mode 100644 index 00000000000..74ee652ec6e --- /dev/null +++ b/tests/ui/simd/masked-load-store.rs @@ -0,0 +1,33 @@ +// run-pass +#![feature(repr_simd, platform_intrinsics)] + +extern "platform-intrinsic" { + fn simd_masked_load(mask: M, pointer: P, values: T) -> T; + fn simd_masked_store(mask: M, pointer: P, values: T) -> (); +} + +#[derive(Copy, Clone)] +#[repr(simd)] +struct Simd([T; N]); + +fn main() { + unsafe { + let a = Simd::([0, 1, 2, 3]); + let b_src = [4u8, 5, 6, 7]; + let b_default = Simd::([9; 4]); + let b: Simd:: = simd_masked_load( + Simd::([-1, 0, -1, -1]), + b_src.as_ptr(), + b_default + ); + + assert_eq!(&b.0, &[4, 9, 6, 7]); + + let mut output = [u8::MAX; 5]; + + simd_masked_store(Simd::([-1, -1, -1, 0]), output.as_mut_ptr(), a); + assert_eq!(&output, &[0, 1, 2, u8::MAX, u8::MAX]); + simd_masked_store(Simd::([0, -1, -1, 0]), output[1..].as_mut_ptr(), b); + assert_eq!(&output, &[0, 1, 9, 6, u8::MAX]); + } +} -- cgit 1.4.1-3-g733a5