From 3f076451202409bca6262d7b5bf9a4fee3d18fb9 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Thu, 30 Jun 2022 08:16:05 +0000 Subject: Lower the assume intrinsic to a MIR statement --- compiler/rustc_mir_transform/src/simplify.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'compiler/rustc_mir_transform/src/simplify.rs') diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index bed48db959a..b51d6d6f4fe 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -500,6 +500,7 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match statement.kind { StatementKind::CopyNonOverlapping(..) + | StatementKind::Assume(..) | StatementKind::Retag(..) | StatementKind::Coverage(..) | StatementKind::FakeRead(..) -- cgit 1.4.1-3-g733a5 From b7413511dc85ec01ef4b91785f86614589ac6103 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Tue, 12 Jul 2022 10:05:00 +0000 Subject: Generalize the Assume intrinsic statement to a general Intrinsic statement --- compiler/rustc_borrowck/src/dataflow.rs | 3 +- compiler/rustc_borrowck/src/invalidation.rs | 11 ++--- compiler/rustc_borrowck/src/lib.rs | 14 +++--- compiler/rustc_borrowck/src/type_check/mod.rs | 14 +++--- compiler/rustc_codegen_cranelift/src/base.rs | 41 +++++++++++------- compiler/rustc_codegen_cranelift/src/constant.rs | 9 ++-- compiler/rustc_codegen_ssa/src/mir/statement.rs | 19 ++++---- compiler/rustc_const_eval/src/interpret/step.rs | 23 +++++----- .../src/transform/check_consts/check.rs | 3 +- .../rustc_const_eval/src/transform/validate.rs | 17 ++++---- compiler/rustc_middle/src/mir/mod.rs | 9 +--- compiler/rustc_middle/src/mir/spanview.rs | 3 +- compiler/rustc_middle/src/mir/syntax.rs | 50 +++++++++++++++++----- compiler/rustc_middle/src/mir/visit.rs | 20 ++++----- compiler/rustc_mir_dataflow/src/impls/liveness.rs | 3 +- .../src/impls/storage_liveness.rs | 3 +- .../rustc_mir_dataflow/src/move_paths/builder.rs | 3 +- compiler/rustc_mir_transform/src/check_unsafety.rs | 4 +- compiler/rustc_mir_transform/src/coverage/spans.rs | 3 +- .../src/dead_store_elimination.rs | 3 +- compiler/rustc_mir_transform/src/dest_prop.rs | 3 +- compiler/rustc_mir_transform/src/generator.rs | 3 +- .../rustc_mir_transform/src/lower_intrinsics.rs | 24 +++++++---- .../src/remove_noop_landing_pads.rs | 3 +- .../src/separate_const_switch.rs | 6 +-- compiler/rustc_mir_transform/src/simplify.rs | 3 +- ...sics.f_copy_nonoverlapping.LowerIntrinsics.diff | 2 +- .../clippy_utils/src/qualify_min_const_fn.rs | 10 +++-- 28 files changed, 166 insertions(+), 143 deletions(-) (limited to 'compiler/rustc_mir_transform/src/simplify.rs') diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs index ec5fa76ff8b..9f7a4d49989 100644 --- a/compiler/rustc_borrowck/src/dataflow.rs +++ b/compiler/rustc_borrowck/src/dataflow.rs @@ -391,8 +391,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> { | mir::StatementKind::Retag { .. } | mir::StatementKind::AscribeUserType(..) | mir::StatementKind::Coverage(..) - | mir::StatementKind::CopyNonOverlapping(..) - | mir::StatementKind::Assume(..) + | mir::StatementKind::Intrinsic(..) | mir::StatementKind::Nop => {} } } diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/invalidation.rs index 03ad973cf49..3157f861d93 100644 --- a/compiler/rustc_borrowck/src/invalidation.rs +++ b/compiler/rustc_borrowck/src/invalidation.rs @@ -1,6 +1,6 @@ use rustc_data_structures::graph::dominators::Dominators; use rustc_middle::mir::visit::Visitor; -use rustc_middle::mir::{BasicBlock, Body, Location, Place, Rvalue}; +use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue}; use rustc_middle::mir::{BorrowKind, Mutability, Operand}; use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind}; use rustc_middle::mir::{Statement, StatementKind}; @@ -63,11 +63,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> { StatementKind::FakeRead(box (_, _)) => { // Only relevant for initialized/liveness/safety checks. } - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { + StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => { + self.consume_operand(location, op); + } + StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { ref src, ref dst, ref count, - }) => { + })) => { self.consume_operand(location, src); self.consume_operand(location, dst); self.consume_operand(location, count); @@ -76,8 +79,6 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> { StatementKind::AscribeUserType(..) // Doesn't have any language semantics | StatementKind::Coverage(..) - // Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck - | StatementKind::Assume(..) // Does not actually affect borrowck | StatementKind::StorageLive(..) => {} StatementKind::StorageDead(local) => { diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 663c880d0ae..3b2bcd7e6d3 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -26,8 +26,8 @@ use rustc_index::bit_set::ChunkedBitSet; use rustc_index::vec::IndexVec; use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt}; use rustc_middle::mir::{ - traversal, Body, ClearCrossCrate, Local, Location, Mutability, Operand, Place, PlaceElem, - PlaceRef, VarDebugInfoContents, + traversal, Body, ClearCrossCrate, Local, Location, Mutability, NonDivergingIntrinsic, Operand, + Place, PlaceElem, PlaceRef, VarDebugInfoContents, }; use rustc_middle::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind}; use rustc_middle::mir::{Field, ProjectionElem, Promoted, Rvalue, Statement, StatementKind}; @@ -591,10 +591,10 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx flow_state, ); } - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { - .. - }) => { - span_bug!( + StatementKind::Intrinsic(box ref kind) => match kind { + // Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck + NonDivergingIntrinsic::Assume(..) => {}, + NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!( span, "Unexpected CopyNonOverlapping, should only appear after lower_intrinsics", ) @@ -603,8 +603,6 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx StatementKind::AscribeUserType(..) // Doesn't have any language semantics | StatementKind::Coverage(..) - // Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck - | StatementKind::Assume(..) // Does not actually affect borrowck | StatementKind::StorageLive(..) => {} StatementKind::StorageDead(local) => { diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index e7713bda82c..2ba9b2be901 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1302,14 +1302,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { ); } } - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { - .. - }) => span_bug!( - stmt.source_info.span, - "Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics", - ), + StatementKind::Intrinsic(box ref kind) => match kind { + NonDivergingIntrinsic::Assume(..) => {} + NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!( + stmt.source_info.span, + "Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics", + ), + }, StatementKind::FakeRead(..) - | StatementKind::Assume(..) | StatementKind::StorageLive(..) | StatementKind::StorageDead(..) | StatementKind::Retag { .. } diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 1f49ead93a3..2aa11ac2eea 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -791,25 +791,34 @@ fn codegen_stmt<'tcx>( | StatementKind::Nop | StatementKind::FakeRead(..) | StatementKind::Retag { .. } - // We ignore `assume` intrinsics, they are only useful for optimizations - | StatementKind::Assume(..) | StatementKind::AscribeUserType(..) => {} StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), - StatementKind::CopyNonOverlapping(inner) => { - let dst = codegen_operand(fx, &inner.dst); - let pointee = dst - .layout() - .pointee_info_at(fx, rustc_target::abi::Size::ZERO) - .expect("Expected pointer"); - let dst = dst.load_scalar(fx); - let src = codegen_operand(fx, &inner.src).load_scalar(fx); - let count = codegen_operand(fx, &inner.count).load_scalar(fx); - let elem_size: u64 = pointee.size.bytes(); - let bytes = - if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); - } + StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic { + // We ignore `assume` intrinsics, they are only useful for optimizations + NonDivergingIntrinsic::Assume(_) => {} + NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { + src, + dst, + count, + }) => { + let dst = codegen_operand(fx, dst); + let pointee = dst + .layout() + .pointee_info_at(fx, rustc_target::abi::Size::ZERO) + .expect("Expected pointer"); + let dst = dst.load_scalar(fx); + let src = codegen_operand(fx, src).load_scalar(fx); + let count = codegen_operand(fx, count).load_scalar(fx); + let elem_size: u64 = pointee.size.bytes(); + let bytes = if elem_size != 1 { + fx.bcx.ins().imul_imm(count, elem_size as i64) + } else { + count + }; + fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); + } + }, } } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index f75de9096f3..0305341da78 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -536,11 +536,12 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( { return None; } - StatementKind::CopyNonOverlapping(_) => { - return None; - } // conservative handling + StatementKind::Intrinsic(ref intrinsic) => match **intrinsic { + NonDivergingIntrinsic::CopyNonOverlapping(..) => return None, + NonDivergingIntrinsic::Assume(..) => {} + }, + // conservative handling StatementKind::Assign(_) - | StatementKind::Assume(_) | StatementKind::FakeRead(_) | StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(_) diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 9869e61280d..1db0fb3a6f1 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -1,4 +1,5 @@ use rustc_middle::mir; +use rustc_middle::mir::NonDivergingIntrinsic; use super::FunctionCx; use super::LocalRef; @@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); bx } - mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping { - ref src, - ref dst, - ref count, - }) => { + mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => { + let op_val = self.codegen_operand(&mut bx, op); + bx.assume(op_val.immediate()); + bx + } + mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping( + mir::CopyNonOverlapping { ref count, ref src, ref dst }, + )) => { let dst_val = self.codegen_operand(&mut bx, dst); let src_val = self.codegen_operand(&mut bx, src); let count = self.codegen_operand(&mut bx, count).immediate(); @@ -93,11 +97,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty()); bx } - mir::StatementKind::Assume(box ref op) => { - let op_val = self.codegen_operand(&mut bx, op); - bx.assume(op_val.immediate()); - bx - } mir::StatementKind::FakeRead(..) | mir::StatementKind::Retag { .. } | mir::StatementKind::AscribeUserType(..) diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 3ac1d1b3198..3cf38258b75 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -2,8 +2,8 @@ //! //! The main entry point is the `step` method. -use rustc_middle::mir; use rustc_middle::mir::interpret::{InterpResult, Scalar}; +use rustc_middle::mir::{self, NonDivergingIntrinsic}; use rustc_middle::ty::layout::LayoutOf; use super::{InterpCx, Machine}; @@ -114,22 +114,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { M::retag(self, *kind, &dest)?; } - // Call CopyNonOverlapping - CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => { - let src = self.eval_operand(src, None)?; - let dst = self.eval_operand(dst, None)?; - let count = self.eval_operand(count, None)?; - self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?; - } - - // Call Assume - Assume(box op) => { + Intrinsic(box NonDivergingIntrinsic::Assume(op)) => { let op = self.eval_operand(op, None)?; let cond = self.read_scalar(&op)?.to_bool()?; if !cond { throw_ub_format!("`assume` called with `false`"); } } + Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { + ref count, + ref src, + ref dst, + })) => { + let src = self.eval_operand(src, None)?; + let dst = self.eval_operand(dst, None)?; + let count = self.eval_operand(count, None)?; + self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?; + } // Statements we do not track. AscribeUserType(..) => {} diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs index 347c31ceeeb..3fa40dc3059 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs @@ -678,8 +678,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { | StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index dea445f6066..783ab350faa 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -7,9 +7,10 @@ use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::NonUseContext::VarDebugInfo; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::{ - traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location, - MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, RuntimePhase, Rvalue, - SourceScope, Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK, + traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping, + Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef, + ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator, + TerminatorKind, UnOp, START_BLOCK, }; use rustc_middle::ty::fold::BottomUpFolder; use rustc_middle::ty::subst::Subst; @@ -636,7 +637,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ); } } - StatementKind::Assume(box ref op) => { + StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => { let ty = op.ty(&self.body.local_decls, self.tcx); if !ty.is_bool() { self.fail( @@ -645,11 +646,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ); } } - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { - ref src, - ref dst, - ref count, - }) => { + StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping( + CopyNonOverlapping { src, dst, count }, + )) => { let src_ty = src.ty(&self.body.local_decls, self.tcx); let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) { src_deref.ty diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 48eb292c490..091cc84af3c 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1370,14 +1370,7 @@ impl Debug for Statement<'_> { write!(fmt, "Coverage::{:?} for {:?}", kind, rgn) } Coverage(box ref coverage) => write!(fmt, "Coverage::{:?}", coverage.kind), - CopyNonOverlapping(box crate::mir::CopyNonOverlapping { - ref src, - ref dst, - ref count, - }) => { - write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count) - } - Assume(box ref cond) => write!(fmt, "assume({:?})", cond), + Intrinsic(box ref intrinsic) => write!(fmt, "{intrinsic}"), Nop => write!(fmt, "nop"), } } diff --git a/compiler/rustc_middle/src/mir/spanview.rs b/compiler/rustc_middle/src/mir/spanview.rs index cb5adee6910..4e06d91012c 100644 --- a/compiler/rustc_middle/src/mir/spanview.rs +++ b/compiler/rustc_middle/src/mir/spanview.rs @@ -249,8 +249,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str { Retag(..) => "Retag", AscribeUserType(..) => "AscribeUserType", Coverage(..) => "Coverage", - CopyNonOverlapping(..) => "CopyNonOverlapping", - Assume(..) => "Assume", + Intrinsic(..) => "Intrinsic", Nop => "Nop", } } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 01ac2456a13..62306159e88 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -327,6 +327,34 @@ pub enum StatementKind<'tcx> { /// executed. Coverage(Box), + /// Denotes a call to an intrinsic that does not require an unwind path. This avoids + /// adding a new block and a terminator for simple intrinsics. + Intrinsic(Box>), + + /// No-op. Useful for deleting instructions without affecting statement indices. + Nop, +} + +#[derive( + Clone, + TyEncodable, + TyDecodable, + Debug, + PartialEq, + Hash, + HashStable, + TypeFoldable, + TypeVisitable +)] +pub enum NonDivergingIntrinsic<'tcx> { + /// Denotes a call to the intrinsic function `assume`. + /// + /// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its + /// computation to infer information about other variables. So if the boolean came from a + /// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks. + /// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`. + Assume(Operand<'tcx>), + /// Denotes a call to the intrinsic function `copy_nonoverlapping`. /// /// First, all three operands are evaluated. `src` and `dest` must each be a reference, pointer, @@ -340,18 +368,18 @@ pub enum StatementKind<'tcx> { /// /// **Needs clarification**: Is this typed or not, ie is there a typed load and store involved? /// I vaguely remember Ralf saying somewhere that he thought it should not be. - CopyNonOverlapping(Box>), - - /// Denotes a call to the intrinsic function `assume`. - /// - /// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its - /// computation to infer information about other variables. So if the boolean came from a - /// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks. - /// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`. - Assume(Box>), + CopyNonOverlapping(CopyNonOverlapping<'tcx>), +} - /// No-op. Useful for deleting instructions without affecting statement indices. - Nop, +impl std::fmt::Display for NonDivergingIntrinsic<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Assume(op) => write!(f, "assume({op:?})"), + Self::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => { + write!(f, "copy_nonoverlapping(dst = {dst:?}, src = {src:?}, count = {count:?})") + } + } + } } /// Describes what kind of retag is to be performed. diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index a0273e78b8f..708ea4398c8 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -425,17 +425,15 @@ macro_rules! make_mir_visitor { location ) } - StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{ - src, - dst, - count, - }) => { - self.visit_operand(src, location); - self.visit_operand(dst, location); - self.visit_operand(count, location) - } - StatementKind::Assume(box ref $($mutability)? val) => { - self.visit_operand(val, location) + StatementKind::Intrinsic(box ref $($mutability)? intrinsic) => { + match intrinsic { + NonDivergingIntrinsic::Assume(op) => self.visit_operand(op, location), + NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => { + self.visit_operand(src, location); + self.visit_operand(dst, location); + self.visit_operand(count, location); + } + } } StatementKind::Nop => {} } diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs index f3ea84e6350..3e08a8799ef 100644 --- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs @@ -270,8 +270,7 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> { | StatementKind::Retag(..) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => None, }; if let Some(destination) = destination { diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs index bab224a7e04..18760b6c6fa 100644 --- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs @@ -142,8 +142,7 @@ impl<'mir, 'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, 'tc | StatementKind::FakeRead(..) | StatementKind::Nop | StatementKind::Retag(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::StorageLive(..) => {} } } diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index d7ea3339a69..f46fd118bde 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -330,8 +330,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> { StatementKind::Retag { .. } | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 1a360a8d40b..beff19a3ab2 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -106,9 +106,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { } // Move to above list once mir construction uses it. - StatementKind::Assume(..) => unreachable!(), - - StatementKind::CopyNonOverlapping(..) => unreachable!(), + StatementKind::Intrinsic(..) => unreachable!(), } self.super_statement(statement, location); } diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index f5e91371b58..9f842c929dc 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -825,8 +825,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option // Retain spans from all other statements StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding` - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Assign(_) | StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(..) diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index e95c5d888b2..3f3870cc7ba 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -52,8 +52,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Coverage(_) - | StatementKind::CopyNonOverlapping(_) - | StatementKind::Assume(_) + | StatementKind::Intrinsic(_) | StatementKind::Nop => (), StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index a2e12e40be9..9bc47613e4c 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -537,8 +537,7 @@ impl<'a> Conflicts<'a> { | StatementKind::FakeRead(..) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 2bfcd82de5e..705cf776fb2 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -1452,8 +1452,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { | StatementKind::Retag(..) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 652c344ee14..9892580e63d 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -46,12 +46,14 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { let mut args = args.drain(..); block.statements.push(Statement { source_info: terminator.source_info, - kind: StatementKind::CopyNonOverlapping(Box::new( - rustc_middle::mir::CopyNonOverlapping { - src: args.next().unwrap(), - dst: args.next().unwrap(), - count: args.next().unwrap(), - }, + kind: StatementKind::Intrinsic(Box::new( + NonDivergingIntrinsic::CopyNonOverlapping( + rustc_middle::mir::CopyNonOverlapping { + src: args.next().unwrap(), + dst: args.next().unwrap(), + count: args.next().unwrap(), + }, + ), )), }); assert_eq!( @@ -67,9 +69,15 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { let mut args = args.drain(..); block.statements.push(Statement { source_info: terminator.source_info, - kind: StatementKind::Assume(Box::new(args.next().unwrap())), + kind: StatementKind::Intrinsic(Box::new( + NonDivergingIntrinsic::Assume(args.next().unwrap()), + )), }); - assert_eq!(args.next(), None, "Extra argument for assume intrinsic"); + assert_eq!( + args.next(), + None, + "Extra argument for copy_non_overlapping intrinsic" + ); drop(args); terminator.kind = TerminatorKind::Goto { target }; } diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index 7d1ab06c3bd..f1bbf2ea7e8 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -51,8 +51,7 @@ impl RemoveNoopLandingPads { StatementKind::Assign { .. } | StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(..) - | StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + | StatementKind::Intrinsic(..) | StatementKind::Retag { .. } => { return false; } diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 34df693cf68..2f116aaa958 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -249,8 +249,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData< | StatementKind::AscribeUserType(_, _) | StatementKind::Coverage(_) | StatementKind::StorageDead(_) - | StatementKind::CopyNonOverlapping(_) - | StatementKind::Assume(_) + | StatementKind::Intrinsic(_) | StatementKind::Nop => {} } } @@ -318,8 +317,7 @@ fn find_determining_place<'tcx>( | StatementKind::Retag(_, _) | StatementKind::AscribeUserType(_, _) | StatementKind::Coverage(_) - | StatementKind::CopyNonOverlapping(_) - | StatementKind::Assume(_) + | StatementKind::Intrinsic(_) | StatementKind::Nop => {} // If the discriminant is set, it is always set diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index b51d6d6f4fe..57d372fda56 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -499,8 +499,7 @@ impl UsedLocals { impl<'tcx> Visitor<'tcx> for UsedLocals { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match statement.kind { - StatementKind::CopyNonOverlapping(..) - | StatementKind::Assume(..) + StatementKind::Intrinsic(..) | StatementKind::Retag(..) | StatementKind::Coverage(..) | StatementKind::FakeRead(..) diff --git a/src/test/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.diff b/src/test/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.diff index 9681e3f696b..74e22ebb7ff 100644 --- a/src/test/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.diff +++ b/src/test/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.diff @@ -51,7 +51,7 @@ - // mir::Constant - // + span: $DIR/lower_intrinsics.rs:90:9: 90:28 - // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(*const i32, *mut i32, usize) {copy_nonoverlapping::}, val: Value() } -+ copy_nonoverlapping(src=move _4, dst=move _8, count=const 0_usize); // scope 3 at $DIR/lower_intrinsics.rs:+4:9: +4:95 ++ copy_nonoverlapping(dst = move _8, src = move _4, count = const 0_usize); // scope 3 at $DIR/lower_intrinsics.rs:+4:9: +4:95 + goto -> bb1; // scope 3 at $DIR/lower_intrinsics.rs:+4:9: +4:95 } diff --git a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs index 1c12da7f741..b22a9c81746 100644 --- a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs +++ b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs @@ -7,7 +7,7 @@ use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_middle::mir::{ Body, CastKind, NullOp, Operand, Place, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, - TerminatorKind, + TerminatorKind, NonDivergingIntrinsic }; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{self, adjustment::PointerCast, Ty, TyCtxt}; @@ -211,15 +211,19 @@ fn check_statement<'tcx>( StatementKind::SetDiscriminant { place, .. } | StatementKind::Deinit(place) => { check_place(tcx, **place, span, body) }, - StatementKind::Assume(box op) => { + + StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => { check_operand(tcx, op, span, body) }, - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { dst, src, count }) => { + StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping( + rustc_middle::mir::CopyNonOverlapping { dst, src, count }, + )) => { check_operand(tcx, dst, span, body)?; check_operand(tcx, src, span, body)?; check_operand(tcx, count, span, body) }, + // These are all NOPs StatementKind::StorageLive(_) | StatementKind::StorageDead(_) -- cgit 1.4.1-3-g733a5