diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
31 files changed, 1071 insertions, 616 deletions
diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs index bacff287859..26839ebf61e 100644 --- a/compiler/rustc_mir_transform/src/add_call_guards.rs +++ b/compiler/rustc_mir_transform/src/add_call_guards.rs @@ -44,11 +44,10 @@ impl<'tcx> crate::MirPass<'tcx> for AddCallGuards { let cur_len = body.basic_blocks.len(); let mut new_block = |source_info: SourceInfo, is_cleanup: bool, target: BasicBlock| { - let block = BasicBlockData { - statements: vec![], + let block = BasicBlockData::new( + Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), is_cleanup, - terminator: Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), - }; + ); let idx = cur_len + new_blocks.len(); new_blocks.push(block); BasicBlock::new(idx) diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs index a414d120e68..7ae2ebaf4ff 100644 --- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs +++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs @@ -93,11 +93,11 @@ fn add_move_for_packed_drop<'tcx>( let ty = place.ty(body, tcx).ty; let temp = patch.new_temp(ty, source_info.span); - let storage_dead_block = patch.new_block(BasicBlockData { - statements: vec![Statement { source_info, kind: StatementKind::StorageDead(temp) }], - terminator: Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), + let storage_dead_block = patch.new_block(BasicBlockData::new_stmts( + vec![Statement::new(source_info, StatementKind::StorageDead(temp))], + Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), is_cleanup, - }); + )); patch.add_statement(loc, StatementKind::StorageLive(temp)); patch.add_assign(loc, Place::from(temp), Rvalue::Use(Operand::Move(*place))); diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index e5a28d1b66c..3c29d4624b7 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -81,9 +81,11 @@ impl<'tcx> crate::MirPass<'tcx> for AddRetag { // Emit their retags. basic_blocks[START_BLOCK].statements.splice( 0..0, - places.map(|(place, source_info)| Statement { - source_info, - kind: StatementKind::Retag(RetagKind::FnEntry, Box::new(place)), + places.map(|(place, source_info)| { + Statement::new( + source_info, + StatementKind::Retag(RetagKind::FnEntry, Box::new(place)), + ) }), ); } @@ -113,10 +115,10 @@ impl<'tcx> crate::MirPass<'tcx> for AddRetag { for (source_info, dest_place, dest_block) in returns { basic_blocks[dest_block].statements.insert( 0, - Statement { + Statement::new( source_info, - kind: StatementKind::Retag(RetagKind::Default, Box::new(dest_place)), - }, + StatementKind::Retag(RetagKind::Default, Box::new(dest_place)), + ), ); } @@ -174,10 +176,7 @@ impl<'tcx> crate::MirPass<'tcx> for AddRetag { let source_info = block_data.statements[i].source_info; block_data.statements.insert( i + 1, - Statement { - source_info, - kind: StatementKind::Retag(retag_kind, Box::new(place)), - }, + Statement::new(source_info, StatementKind::Retag(retag_kind, Box::new(place))), ); } } diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index 8f88613b79f..989787504b7 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -51,22 +51,18 @@ fn insert_alignment_check<'tcx>( let const_raw_ptr = Ty::new_imm_ptr(tcx, tcx.types.unit); let rvalue = Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(pointer), const_raw_ptr); let thin_ptr = local_decls.push(LocalDecl::with_source_info(const_raw_ptr, source_info)).into(); - stmts - .push(Statement { source_info, kind: StatementKind::Assign(Box::new((thin_ptr, rvalue))) }); + stmts.push(Statement::new(source_info, StatementKind::Assign(Box::new((thin_ptr, rvalue))))); // Transmute the pointer to a usize (equivalent to `ptr.addr()`). let rvalue = Rvalue::Cast(CastKind::Transmute, Operand::Copy(thin_ptr), tcx.types.usize); let addr = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into(); - stmts.push(Statement { source_info, kind: StatementKind::Assign(Box::new((addr, rvalue))) }); + stmts.push(Statement::new(source_info, StatementKind::Assign(Box::new((addr, rvalue))))); // Get the alignment of the pointee let alignment = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into(); let rvalue = Rvalue::NullaryOp(NullOp::AlignOf, pointee_ty); - stmts.push(Statement { - source_info, - kind: StatementKind::Assign(Box::new((alignment, rvalue))), - }); + stmts.push(Statement::new(source_info, StatementKind::Assign(Box::new((alignment, rvalue))))); // Subtract 1 from the alignment to get the alignment mask let alignment_mask = @@ -76,13 +72,13 @@ fn insert_alignment_check<'tcx>( user_ty: None, const_: Const::Val(ConstValue::Scalar(Scalar::from_target_usize(1, &tcx)), tcx.types.usize), })); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( alignment_mask, Rvalue::BinaryOp(BinOp::Sub, Box::new((Operand::Copy(alignment), one))), ))), - }); + )); // If this target does not have reliable alignment, further limit the mask by anding it with // the mask for the highest reliable alignment. @@ -99,31 +95,31 @@ fn insert_alignment_check<'tcx>( tcx.types.usize, ), })); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( alignment_mask, Rvalue::BinaryOp( BinOp::BitAnd, Box::new((Operand::Copy(alignment_mask), max_mask)), ), ))), - }); + )); } // BitAnd the alignment mask with the pointer let alignment_bits = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into(); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( alignment_bits, Rvalue::BinaryOp( BinOp::BitAnd, Box::new((Operand::Copy(addr), Operand::Copy(alignment_mask))), ), ))), - }); + )); // Check if the alignment bits are all zero let is_ok = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); @@ -132,13 +128,13 @@ fn insert_alignment_check<'tcx>( user_ty: None, const_: Const::Val(ConstValue::Scalar(Scalar::from_target_usize(0, &tcx)), tcx.types.usize), })); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( is_ok, Rvalue::BinaryOp(BinOp::Eq, Box::new((Operand::Copy(alignment_bits), zero.clone()))), ))), - }); + )); // Emit a check that asserts on the alignment and otherwise triggers a // AssertKind::MisalignedPointerDereference. diff --git a/compiler/rustc_mir_transform/src/check_enums.rs b/compiler/rustc_mir_transform/src/check_enums.rs new file mode 100644 index 00000000000..240da87ab27 --- /dev/null +++ b/compiler/rustc_mir_transform/src/check_enums.rs @@ -0,0 +1,500 @@ +use rustc_abi::{Scalar, Size, TagEncoding, Variants, WrappingRange}; +use rustc_hir::LangItem; +use rustc_index::IndexVec; +use rustc_middle::bug; +use rustc_middle::mir::visit::Visitor; +use rustc_middle::mir::*; +use rustc_middle::ty::layout::PrimitiveExt; +use rustc_middle::ty::{self, Ty, TyCtxt, TypingEnv}; +use rustc_session::Session; +use tracing::debug; + +/// This pass inserts checks for a valid enum discriminant where they are most +/// likely to find UB, because checking everywhere like Miri would generate too +/// much MIR. +pub(super) struct CheckEnums; + +impl<'tcx> crate::MirPass<'tcx> for CheckEnums { + fn is_enabled(&self, sess: &Session) -> bool { + sess.ub_checks() + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + // This pass emits new panics. If for whatever reason we do not have a panic + // implementation, running this pass may cause otherwise-valid code to not compile. + if tcx.lang_items().get(LangItem::PanicImpl).is_none() { + return; + } + + let typing_env = body.typing_env(tcx); + let basic_blocks = body.basic_blocks.as_mut(); + let local_decls = &mut body.local_decls; + + // This operation inserts new blocks. Each insertion changes the Location for all + // statements/blocks after. Iterating or visiting the MIR in order would require updating + // our current location after every insertion. By iterating backwards, we dodge this issue: + // The only Locations that an insertion changes have already been handled. + for block in basic_blocks.indices().rev() { + for statement_index in (0..basic_blocks[block].statements.len()).rev() { + let location = Location { block, statement_index }; + let statement = &basic_blocks[block].statements[statement_index]; + let source_info = statement.source_info; + + let mut finder = EnumFinder::new(tcx, local_decls, typing_env); + finder.visit_statement(statement, location); + + for check in finder.into_found_enums() { + debug!("Inserting enum check"); + let new_block = split_block(basic_blocks, location); + + match check { + EnumCheckType::Direct { source_op, discr, op_size, valid_discrs } => { + insert_direct_enum_check( + tcx, + local_decls, + basic_blocks, + block, + source_op, + discr, + op_size, + valid_discrs, + source_info, + new_block, + ) + } + EnumCheckType::Uninhabited => insert_uninhabited_enum_check( + tcx, + local_decls, + &mut basic_blocks[block], + source_info, + new_block, + ), + EnumCheckType::WithNiche { + source_op, + discr, + op_size, + offset, + valid_range, + } => insert_niche_check( + tcx, + local_decls, + &mut basic_blocks[block], + source_op, + valid_range, + discr, + op_size, + offset, + source_info, + new_block, + ), + } + } + } + } + } + + fn is_required(&self) -> bool { + true + } +} + +/// Represent the different kind of enum checks we can insert. +enum EnumCheckType<'tcx> { + /// We know we try to create an uninhabited enum from an inhabited variant. + Uninhabited, + /// We know the enum does no niche optimizations and can thus easily compute + /// the valid discriminants. + Direct { + source_op: Operand<'tcx>, + discr: TyAndSize<'tcx>, + op_size: Size, + valid_discrs: Vec<u128>, + }, + /// We try to construct an enum that has a niche. + WithNiche { + source_op: Operand<'tcx>, + discr: TyAndSize<'tcx>, + op_size: Size, + offset: Size, + valid_range: WrappingRange, + }, +} + +struct TyAndSize<'tcx> { + pub ty: Ty<'tcx>, + pub size: Size, +} + +/// A [Visitor] that finds the construction of enums and evaluates which checks +/// we should apply. +struct EnumFinder<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a mut LocalDecls<'tcx>, + typing_env: TypingEnv<'tcx>, + enums: Vec<EnumCheckType<'tcx>>, +} + +impl<'a, 'tcx> EnumFinder<'a, 'tcx> { + fn new( + tcx: TyCtxt<'tcx>, + local_decls: &'a mut LocalDecls<'tcx>, + typing_env: TypingEnv<'tcx>, + ) -> Self { + EnumFinder { tcx, local_decls, typing_env, enums: Vec::new() } + } + + /// Returns the found enum creations and which checks should be inserted. + fn into_found_enums(self) -> Vec<EnumCheckType<'tcx>> { + self.enums + } +} + +impl<'a, 'tcx> Visitor<'tcx> for EnumFinder<'a, 'tcx> { + fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { + if let Rvalue::Cast(CastKind::Transmute, op, ty) = rvalue { + let ty::Adt(adt_def, _) = ty.kind() else { + return; + }; + if !adt_def.is_enum() { + return; + } + + let Ok(enum_layout) = self.tcx.layout_of(self.typing_env.as_query_input(*ty)) else { + return; + }; + let Ok(op_layout) = self + .tcx + .layout_of(self.typing_env.as_query_input(op.ty(self.local_decls, self.tcx))) + else { + return; + }; + + match enum_layout.variants { + Variants::Empty if op_layout.is_uninhabited() => return, + // An empty enum that tries to be constructed from an inhabited value, this + // is never correct. + Variants::Empty => { + // The enum layout is uninhabited but we construct it from sth inhabited. + // This is always UB. + self.enums.push(EnumCheckType::Uninhabited); + } + // Construction of Single value enums is always fine. + Variants::Single { .. } => {} + // Construction of an enum with multiple variants but no niche optimizations. + Variants::Multiple { + tag_encoding: TagEncoding::Direct, + tag: Scalar::Initialized { value, .. }, + .. + } => { + let valid_discrs = + adt_def.discriminants(self.tcx).map(|(_, discr)| discr.val).collect(); + + let discr = + TyAndSize { ty: value.to_int_ty(self.tcx), size: value.size(&self.tcx) }; + self.enums.push(EnumCheckType::Direct { + source_op: op.to_copy(), + discr, + op_size: op_layout.size, + valid_discrs, + }); + } + // Construction of an enum with multiple variants and niche optimizations. + Variants::Multiple { + tag_encoding: TagEncoding::Niche { .. }, + tag: Scalar::Initialized { value, valid_range, .. }, + tag_field, + .. + } => { + let discr = + TyAndSize { ty: value.to_int_ty(self.tcx), size: value.size(&self.tcx) }; + self.enums.push(EnumCheckType::WithNiche { + source_op: op.to_copy(), + discr, + op_size: op_layout.size, + offset: enum_layout.fields.offset(tag_field.as_usize()), + valid_range, + }); + } + _ => return, + } + + self.super_rvalue(rvalue, location); + } + } +} + +fn split_block( + basic_blocks: &mut IndexVec<BasicBlock, BasicBlockData<'_>>, + location: Location, +) -> BasicBlock { + let block_data = &mut basic_blocks[location.block]; + + // Drain every statement after this one and move the current terminator to a new basic block. + let new_block = BasicBlockData::new_stmts( + block_data.statements.split_off(location.statement_index), + block_data.terminator.take(), + block_data.is_cleanup, + ); + + basic_blocks.push(new_block) +} + +/// Inserts the cast of an operand (any type) to a u128 value that holds the discriminant value. +fn insert_discr_cast_to_u128<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>, + block_data: &mut BasicBlockData<'tcx>, + source_op: Operand<'tcx>, + discr: TyAndSize<'tcx>, + op_size: Size, + offset: Option<Size>, + source_info: SourceInfo, +) -> Place<'tcx> { + let get_ty_for_size = |tcx: TyCtxt<'tcx>, size: Size| -> Ty<'tcx> { + match size.bytes() { + 1 => tcx.types.u8, + 2 => tcx.types.u16, + 4 => tcx.types.u32, + 8 => tcx.types.u64, + 16 => tcx.types.u128, + invalid => bug!("Found discriminant with invalid size, has {} bytes", invalid), + } + }; + + let (cast_kind, discr_ty_bits) = if discr.size.bytes() < op_size.bytes() { + // The discriminant is less wide than the operand, cast the operand into + // [MaybeUninit; N] and then index into it. + let mu = Ty::new_maybe_uninit(tcx, tcx.types.u8); + let array_len = op_size.bytes(); + let mu_array_ty = Ty::new_array(tcx, mu, array_len); + let mu_array = + local_decls.push(LocalDecl::with_source_info(mu_array_ty, source_info)).into(); + let rvalue = Rvalue::Cast(CastKind::Transmute, source_op, mu_array_ty); + block_data + .statements + .push(Statement::new(source_info, StatementKind::Assign(Box::new((mu_array, rvalue))))); + + // Index into the array of MaybeUninit to get something that is actually + // as wide as the discriminant. + let offset = offset.unwrap_or(Size::ZERO); + let smaller_mu_array = mu_array.project_deeper( + &[ProjectionElem::Subslice { + from: offset.bytes(), + to: offset.bytes() + discr.size.bytes(), + from_end: false, + }], + tcx, + ); + + (CastKind::Transmute, Operand::Copy(smaller_mu_array)) + } else { + let operand_int_ty = get_ty_for_size(tcx, op_size); + + let op_as_int = + local_decls.push(LocalDecl::with_source_info(operand_int_ty, source_info)).into(); + let rvalue = Rvalue::Cast(CastKind::Transmute, source_op, operand_int_ty); + block_data.statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new((op_as_int, rvalue))), + )); + + (CastKind::IntToInt, Operand::Copy(op_as_int)) + }; + + // Cast the resulting value to the actual discriminant integer type. + let rvalue = Rvalue::Cast(cast_kind, discr_ty_bits, discr.ty); + let discr_in_discr_ty = + local_decls.push(LocalDecl::with_source_info(discr.ty, source_info)).into(); + block_data.statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new((discr_in_discr_ty, rvalue))), + )); + + // Cast the discriminant to a u128 (base for comparisions of enum discriminants). + let const_u128 = Ty::new_uint(tcx, ty::UintTy::U128); + let rvalue = Rvalue::Cast(CastKind::IntToInt, Operand::Copy(discr_in_discr_ty), const_u128); + let discr = local_decls.push(LocalDecl::with_source_info(const_u128, source_info)).into(); + block_data + .statements + .push(Statement::new(source_info, StatementKind::Assign(Box::new((discr, rvalue))))); + + discr +} + +fn insert_direct_enum_check<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>, + basic_blocks: &mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, + current_block: BasicBlock, + source_op: Operand<'tcx>, + discr: TyAndSize<'tcx>, + op_size: Size, + discriminants: Vec<u128>, + source_info: SourceInfo, + new_block: BasicBlock, +) { + // Insert a new target block that is branched to in case of an invalid discriminant. + let invalid_discr_block_data = BasicBlockData::new(None, false); + let invalid_discr_block = basic_blocks.push(invalid_discr_block_data); + let block_data = &mut basic_blocks[current_block]; + let discr = insert_discr_cast_to_u128( + tcx, + local_decls, + block_data, + source_op, + discr, + op_size, + None, + source_info, + ); + + // Branch based on the discriminant value. + block_data.terminator = Some(Terminator { + source_info, + kind: TerminatorKind::SwitchInt { + discr: Operand::Copy(discr), + targets: SwitchTargets::new( + discriminants.into_iter().map(|discr| (discr, new_block)), + invalid_discr_block, + ), + }, + }); + + // Abort in case of an invalid enum discriminant. + basic_blocks[invalid_discr_block].terminator = Some(Terminator { + source_info, + kind: TerminatorKind::Assert { + cond: Operand::Constant(Box::new(ConstOperand { + span: source_info.span, + user_ty: None, + const_: Const::Val(ConstValue::from_bool(false), tcx.types.bool), + })), + expected: true, + target: new_block, + msg: Box::new(AssertKind::InvalidEnumConstruction(Operand::Copy(discr))), + // This calls panic_invalid_enum_construction, which is #[rustc_nounwind]. + // We never want to insert an unwind into unsafe code, because unwinding could + // make a failing UB check turn into much worse UB when we start unwinding. + unwind: UnwindAction::Unreachable, + }, + }); +} + +fn insert_uninhabited_enum_check<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>, + block_data: &mut BasicBlockData<'tcx>, + source_info: SourceInfo, + new_block: BasicBlock, +) { + let is_ok: Place<'_> = + local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); + block_data.statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + is_ok, + Rvalue::Use(Operand::Constant(Box::new(ConstOperand { + span: source_info.span, + user_ty: None, + const_: Const::Val(ConstValue::from_bool(false), tcx.types.bool), + }))), + ))), + )); + + block_data.terminator = Some(Terminator { + source_info, + kind: TerminatorKind::Assert { + cond: Operand::Copy(is_ok), + expected: true, + target: new_block, + msg: Box::new(AssertKind::InvalidEnumConstruction(Operand::Constant(Box::new( + ConstOperand { + span: source_info.span, + user_ty: None, + const_: Const::Val(ConstValue::from_u128(0), tcx.types.u128), + }, + )))), + // This calls panic_invalid_enum_construction, which is #[rustc_nounwind]. + // We never want to insert an unwind into unsafe code, because unwinding could + // make a failing UB check turn into much worse UB when we start unwinding. + unwind: UnwindAction::Unreachable, + }, + }); +} + +fn insert_niche_check<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>, + block_data: &mut BasicBlockData<'tcx>, + source_op: Operand<'tcx>, + valid_range: WrappingRange, + discr: TyAndSize<'tcx>, + op_size: Size, + offset: Size, + source_info: SourceInfo, + new_block: BasicBlock, +) { + let discr = insert_discr_cast_to_u128( + tcx, + local_decls, + block_data, + source_op, + discr, + op_size, + Some(offset), + source_info, + ); + + // Compare the discriminant agains the valid_range. + let start_const = Operand::Constant(Box::new(ConstOperand { + span: source_info.span, + user_ty: None, + const_: Const::Val(ConstValue::from_u128(valid_range.start), tcx.types.u128), + })); + let end_start_diff_const = Operand::Constant(Box::new(ConstOperand { + span: source_info.span, + user_ty: None, + const_: Const::Val( + ConstValue::from_u128(u128::wrapping_sub(valid_range.end, valid_range.start)), + tcx.types.u128, + ), + })); + + let discr_diff: Place<'_> = + local_decls.push(LocalDecl::with_source_info(tcx.types.u128, source_info)).into(); + block_data.statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + discr_diff, + Rvalue::BinaryOp(BinOp::Sub, Box::new((Operand::Copy(discr), start_const))), + ))), + )); + + let is_ok: Place<'_> = + local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); + block_data.statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + is_ok, + Rvalue::BinaryOp( + // This is a `WrappingRange`, so make sure to get the wrapping right. + BinOp::Le, + Box::new((Operand::Copy(discr_diff), end_start_diff_const)), + ), + ))), + )); + + block_data.terminator = Some(Terminator { + source_info, + kind: TerminatorKind::Assert { + cond: Operand::Copy(is_ok), + expected: true, + target: new_block, + msg: Box::new(AssertKind::InvalidEnumConstruction(Operand::Copy(discr))), + // This calls panic_invalid_enum_construction, which is #[rustc_nounwind]. + // We never want to insert an unwind into unsafe code, because unwinding could + // make a failing UB check turn into much worse UB when we start unwinding. + unwind: UnwindAction::Unreachable, + }, + }); +} diff --git a/compiler/rustc_mir_transform/src/check_null.rs b/compiler/rustc_mir_transform/src/check_null.rs index ad74e335bd9..e365d36580a 100644 --- a/compiler/rustc_mir_transform/src/check_null.rs +++ b/compiler/rustc_mir_transform/src/check_null.rs @@ -41,13 +41,12 @@ fn insert_null_check<'tcx>( let const_raw_ptr = Ty::new_imm_ptr(tcx, tcx.types.unit); let rvalue = Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(pointer), const_raw_ptr); let thin_ptr = local_decls.push(LocalDecl::with_source_info(const_raw_ptr, source_info)).into(); - stmts - .push(Statement { source_info, kind: StatementKind::Assign(Box::new((thin_ptr, rvalue))) }); + stmts.push(Statement::new(source_info, StatementKind::Assign(Box::new((thin_ptr, rvalue))))); // Transmute the pointer to a usize (equivalent to `ptr.addr()`). let rvalue = Rvalue::Cast(CastKind::Transmute, Operand::Copy(thin_ptr), tcx.types.usize); let addr = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into(); - stmts.push(Statement { source_info, kind: StatementKind::Assign(Box::new((addr, rvalue))) }); + stmts.push(Statement::new(source_info, StatementKind::Assign(Box::new((addr, rvalue))))); let zero = Operand::Constant(Box::new(ConstOperand { span: source_info.span, @@ -71,24 +70,24 @@ fn insert_null_check<'tcx>( let rvalue = Rvalue::NullaryOp(NullOp::SizeOf, pointee_ty); let sizeof_pointee = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into(); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new((sizeof_pointee, rvalue))), - }); + StatementKind::Assign(Box::new((sizeof_pointee, rvalue))), + )); // Check that the pointee is not a ZST. let is_pointee_not_zst = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( is_pointee_not_zst, Rvalue::BinaryOp( BinOp::Ne, Box::new((Operand::Copy(sizeof_pointee), zero.clone())), ), ))), - }); + )); // Pointer needs to be checked only if pointee is not a ZST. Operand::Copy(is_pointee_not_zst) @@ -97,38 +96,38 @@ fn insert_null_check<'tcx>( // Check whether the pointer is null. let is_null = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( is_null, Rvalue::BinaryOp(BinOp::Eq, Box::new((Operand::Copy(addr), zero))), ))), - }); + )); // We want to throw an exception if the pointer is null and the pointee is not unconditionally // allowed (which for all non-borrow place uses, is when the pointee is ZST). let should_throw_exception = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( should_throw_exception, Rvalue::BinaryOp( BinOp::BitAnd, Box::new((Operand::Copy(is_null), pointee_should_be_checked)), ), ))), - }); + )); // The final condition whether this pointer usage is ok or not. let is_ok = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); - stmts.push(Statement { + stmts.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( is_ok, Rvalue::UnaryOp(UnOp::Not, Operand::Copy(should_throw_exception)), ))), - }); + )); // Emit a PointerCheck that asserts on the condition and otherwise triggers // a AssertKind::NullPointerDereference. diff --git a/compiler/rustc_mir_transform/src/check_pointers.rs b/compiler/rustc_mir_transform/src/check_pointers.rs index bf94f1aad24..4f913c1fca0 100644 --- a/compiler/rustc_mir_transform/src/check_pointers.rs +++ b/compiler/rustc_mir_transform/src/check_pointers.rs @@ -235,11 +235,11 @@ fn split_block( let block_data = &mut basic_blocks[location.block]; // Drain every statement after this one and move the current terminator to a new basic block. - let new_block = BasicBlockData { - statements: block_data.statements.split_off(location.statement_index), - terminator: block_data.terminator.take(), - is_cleanup: block_data.is_cleanup, - }; + let new_block = BasicBlockData::new_stmts( + block_data.statements.split_off(location.statement_index), + block_data.terminator.take(), + block_data.is_cleanup, + ); basic_blocks.push(new_block) } diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index fe78a104fa0..cddeefca681 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -33,24 +33,26 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { debug!(borrowed_locals = ?ssa.borrowed_locals()); debug!(copy_classes = ?ssa.copy_classes()); - let fully_moved = fully_moved_locals(&ssa, body); - debug!(?fully_moved); - - let mut storage_to_remove = DenseBitSet::new_empty(fully_moved.domain_size()); + let mut any_replacement = false; + let mut storage_to_remove = DenseBitSet::new_empty(body.local_decls.len()); for (local, &head) in ssa.copy_classes().iter_enumerated() { if local != head { + any_replacement = true; storage_to_remove.insert(head); } } - let any_replacement = ssa.copy_classes().iter_enumerated().any(|(l, &h)| l != h); + if !any_replacement { + return; + } + + let fully_moved = fully_moved_locals(&ssa, body); + debug!(?fully_moved); Replacer { tcx, copy_classes: ssa.copy_classes(), fully_moved, storage_to_remove } .visit_body_preserves_cfg(body); - if any_replacement { - crate::simplify::remove_unused_definitions(body); - } + crate::simplify::remove_unused_definitions(body); } fn is_required(&self) -> bool { diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 06c6b46a9c2..761d5461a99 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -252,16 +252,16 @@ impl<'tcx> TransformVisitor<'tcx> { } }; - let statements = vec![Statement { - kind: StatementKind::Assign(Box::new((Place::return_place(), none_value))), + let statements = vec![Statement::new( source_info, - }]; + StatementKind::Assign(Box::new((Place::return_place(), none_value))), + )]; - body.basic_blocks_mut().push(BasicBlockData { + body.basic_blocks_mut().push(BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }); + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + )); block } @@ -342,10 +342,10 @@ impl<'tcx> TransformVisitor<'tcx> { } }; - statements.push(Statement { - kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))), + statements.push(Statement::new( source_info, - }); + StatementKind::Assign(Box::new((Place::return_place(), rvalue))), + )); } // Create a Place referencing a coroutine struct field @@ -361,13 +361,13 @@ impl<'tcx> TransformVisitor<'tcx> { // Create a statement which changes the discriminant fn set_discr(&self, state_disc: VariantIdx, source_info: SourceInfo) -> Statement<'tcx> { let self_place = Place::from(SELF_ARG); - Statement { + Statement::new( source_info, - kind: StatementKind::SetDiscriminant { + StatementKind::SetDiscriminant { place: Box::new(self_place), variant_index: state_disc, }, - } + ) } // Create a statement which reads the discriminant into a temporary @@ -377,10 +377,10 @@ impl<'tcx> TransformVisitor<'tcx> { let temp = Place::from(local_decls_len); let self_place = Place::from(SELF_ARG); - let assign = Statement { - source_info: SourceInfo::outermost(body.span), - kind: StatementKind::Assign(Box::new((temp, Rvalue::Discriminant(self_place)))), - }; + let assign = Statement::new( + SourceInfo::outermost(body.span), + StatementKind::Assign(Box::new((temp, Rvalue::Discriminant(self_place)))), + ); (assign, temp) } } @@ -450,7 +450,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { && !self.always_live_locals.contains(l); if needs_storage_dead { data.statements - .push(Statement { source_info, kind: StatementKind::StorageDead(l) }); + .push(Statement::new(source_info, StatementKind::StorageDead(l))); } } @@ -596,10 +596,8 @@ fn eliminate_get_context_call<'tcx>(bb_data: &mut BasicBlockData<'tcx>) -> Local let local = arg.node.place().unwrap().local; let arg = Rvalue::Use(arg.node); - let assign = Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new((destination, arg))), - }; + let assign = + Statement::new(terminator.source_info, StatementKind::Assign(Box::new((destination, arg)))); bb_data.statements.push(assign); bb_data.terminator = Some(Terminator { source_info: terminator.source_info, @@ -1075,11 +1073,11 @@ fn insert_switch<'tcx>( let source_info = SourceInfo::outermost(body.span); body.basic_blocks_mut().raw.insert( 0, - BasicBlockData { - statements: vec![assign], - terminator: Some(Terminator { source_info, kind: switch }), - is_cleanup: false, - }, + BasicBlockData::new_stmts( + vec![assign], + Some(Terminator { source_info, kind: switch }), + false, + ), ); for b in body.basic_blocks_mut().iter_mut() { @@ -1089,11 +1087,7 @@ fn insert_switch<'tcx>( fn insert_term_block<'tcx>(body: &mut Body<'tcx>, kind: TerminatorKind<'tcx>) -> BasicBlock { let source_info = SourceInfo::outermost(body.span); - body.basic_blocks_mut().push(BasicBlockData { - statements: Vec::new(), - terminator: Some(Terminator { source_info, kind }), - is_cleanup: false, - }) + body.basic_blocks_mut().push(BasicBlockData::new(Some(Terminator { source_info, kind }), false)) } fn return_poll_ready_assign<'tcx>(tcx: TyCtxt<'tcx>, source_info: SourceInfo) -> Statement<'tcx> { @@ -1109,19 +1103,16 @@ fn return_poll_ready_assign<'tcx>(tcx: TyCtxt<'tcx>, source_info: SourceInfo) -> Box::new(AggregateKind::Adt(poll_def_id, VariantIdx::from_usize(0), args, None, None)), IndexVec::from_raw(vec![val]), ); - Statement { - kind: StatementKind::Assign(Box::new((Place::return_place(), ready_val))), - source_info, - } + Statement::new(source_info, StatementKind::Assign(Box::new((Place::return_place(), ready_val)))) } fn insert_poll_ready_block<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> BasicBlock { let source_info = SourceInfo::outermost(body.span); - body.basic_blocks_mut().push(BasicBlockData { - statements: [return_poll_ready_assign(tcx, source_info)].to_vec(), - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }) + body.basic_blocks_mut().push(BasicBlockData::new_stmts( + [return_poll_ready_assign(tcx, source_info)].to_vec(), + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + )) } fn insert_panic_block<'tcx>( @@ -1205,13 +1196,11 @@ fn generate_poison_block_and_redirect_unwinds_there<'tcx>( body: &mut Body<'tcx>, ) { let source_info = SourceInfo::outermost(body.span); - let poison_block = body.basic_blocks_mut().push(BasicBlockData { - statements: vec![ - transform.set_discr(VariantIdx::new(CoroutineArgs::POISONED), source_info), - ], - terminator: Some(Terminator { source_info, kind: TerminatorKind::UnwindResume }), - is_cleanup: true, - }); + let poison_block = body.basic_blocks_mut().push(BasicBlockData::new_stmts( + vec![transform.set_discr(VariantIdx::new(CoroutineArgs::POISONED), source_info)], + Some(Terminator { source_info, kind: TerminatorKind::UnwindResume }), + true, + )); for (idx, block) in body.basic_blocks_mut().iter_enumerated_mut() { let source_info = block.terminator().source_info; @@ -1345,32 +1334,28 @@ fn create_cases<'tcx>( && !transform.remap.contains(l) && !transform.always_live_locals.contains(l); if needs_storage_live { - statements - .push(Statement { source_info, kind: StatementKind::StorageLive(l) }); + statements.push(Statement::new(source_info, StatementKind::StorageLive(l))); } } if operation == Operation::Resume { // Move the resume argument to the destination place of the `Yield` terminator let resume_arg = CTX_ARG; - statements.push(Statement { + statements.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( point.resume_arg, Rvalue::Use(Operand::Move(resume_arg.into())), ))), - }); + )); } // Then jump to the real target - let block = body.basic_blocks_mut().push(BasicBlockData { + let block = body.basic_blocks_mut().push(BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { - source_info, - kind: TerminatorKind::Goto { target }, - }), - is_cleanup: false, - }); + Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), + false, + )); (point.state, block) }) @@ -1540,13 +1525,13 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { let stmts = &mut body.basic_blocks_mut()[START_BLOCK].statements; stmts.insert( 0, - Statement { + Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( old_resume_local.into(), Rvalue::Use(Operand::Move(resume_local.into())), ))), - }, + ), ); let always_live_locals = always_storage_live_locals(body); diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs index dc68629ec0d..406575c4f43 100644 --- a/compiler/rustc_mir_transform/src/coroutine/drop.rs +++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs @@ -87,12 +87,11 @@ fn build_pin_fut<'tcx>( const_: Const::zero_sized(pin_fut_new_unchecked_fn), })); - let storage_live = - Statement { source_info, kind: StatementKind::StorageLive(fut_pin_place.local) }; + let storage_live = Statement::new(source_info, StatementKind::StorageLive(fut_pin_place.local)); - let fut_ref_assign = Statement { + let fut_ref_assign = Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( fut_ref_place, Rvalue::Ref( tcx.lifetimes.re_erased, @@ -100,12 +99,12 @@ fn build_pin_fut<'tcx>( fut_place, ), ))), - }; + ); // call Pin<FutTy>::new_unchecked(&mut fut) - let pin_fut_bb = body.basic_blocks_mut().push(BasicBlockData { - statements: [storage_live, fut_ref_assign].to_vec(), - terminator: Some(Terminator { + let pin_fut_bb = body.basic_blocks_mut().push(BasicBlockData::new_stmts( + [storage_live, fut_ref_assign].to_vec(), + Some(Terminator { source_info, kind: TerminatorKind::Call { func: pin_fut_new_unchecked_fn, @@ -117,8 +116,8 @@ fn build_pin_fut<'tcx>( fn_span: span, }, }), - is_cleanup: false, - }); + false, + )); (pin_fut_bb, fut_pin_place) } @@ -156,19 +155,15 @@ fn build_poll_switch<'tcx>( let source_info = SourceInfo::outermost(body.span); let poll_discr_place = Place::from(body.local_decls.push(LocalDecl::new(poll_discr_ty, source_info.span))); - let discr_assign = Statement { + let discr_assign = Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( - poll_discr_place, - Rvalue::Discriminant(*poll_unit_place), - ))), - }; - let storage_dead = - Statement { source_info, kind: StatementKind::StorageDead(fut_pin_place.local) }; + StatementKind::Assign(Box::new((poll_discr_place, Rvalue::Discriminant(*poll_unit_place)))), + ); + let storage_dead = Statement::new(source_info, StatementKind::StorageDead(fut_pin_place.local)); let unreachable_block = insert_term_block(body, TerminatorKind::Unreachable); - body.basic_blocks_mut().push(BasicBlockData { - statements: [storage_dead, discr_assign].to_vec(), - terminator: Some(Terminator { + body.basic_blocks_mut().push(BasicBlockData::new_stmts( + [storage_dead, discr_assign].to_vec(), + Some(Terminator { source_info, kind: TerminatorKind::SwitchInt { discr: Operand::Move(poll_discr_place), @@ -179,8 +174,8 @@ fn build_poll_switch<'tcx>( ), }, }), - is_cleanup: false, - }) + false, + )) } // Gather blocks, reachable through 'drop' targets of Yield and Drop terminators (chained) @@ -330,10 +325,10 @@ pub(super) fn expand_async_drops<'tcx>( let context_ref_place = Place::from(body.local_decls.push(LocalDecl::new(context_mut_ref, source_info.span))); let arg = Rvalue::Use(Operand::Move(Place::from(CTX_ARG))); - body[bb].statements.push(Statement { + body[bb].statements.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new((context_ref_place, arg))), - }); + StatementKind::Assign(Box::new((context_ref_place, arg))), + )); let yield_block = insert_term_block(body, TerminatorKind::Unreachable); // `kind` replaced later to yield let (pin_bb, fut_pin_place) = build_pin_fut(tcx, body, fut_place.clone(), UnwindAction::Continue); @@ -551,11 +546,8 @@ pub(super) fn insert_clean_drop<'tcx>( }; // Create a block to destroy an unresumed coroutines. This can only destroy upvars. - body.basic_blocks_mut().push(BasicBlockData { - statements: Vec::new(), - terminator: Some(Terminator { source_info, kind: term }), - is_cleanup: false, - }) + body.basic_blocks_mut() + .push(BasicBlockData::new(Some(Terminator { source_info, kind: term }), false)) } pub(super) fn create_coroutine_drop_shim<'tcx>( @@ -734,11 +726,7 @@ pub(super) fn create_coroutine_drop_shim_proxy_async<'tcx>( body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(poll_enum, source_info); // call coroutine_drop() - let call_bb = body.basic_blocks_mut().push(BasicBlockData { - statements: Vec::new(), - terminator: None, - is_cleanup: false, - }); + let call_bb = body.basic_blocks_mut().push(BasicBlockData::new(None, false)); // return Poll::Ready() let ret_bb = insert_poll_ready_block(tcx, &mut body); diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 702c62eddc7..f253d1662ca 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -259,7 +259,7 @@ fn inject_statement(mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb debug!(" injecting statement {counter_kind:?} for {bb:?}"); let data = &mut mir_body[bb]; let source_info = data.terminator().source_info; - let statement = Statement { source_info, kind: StatementKind::Coverage(counter_kind) }; + let statement = Statement::new(source_info, StatementKind::Coverage(counter_kind)); data.statements.insert(0, statement); } diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 3c0053c610d..b0fc5e90f07 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -68,14 +68,13 @@ impl<'tcx> MockBlocks<'tcx> { BytePos(1) }; let next_hi = next_lo + BytePos(1); - self.blocks.push(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + self.blocks.push(BasicBlockData::new( + Some(Terminator { source_info: SourceInfo::outermost(Span::with_root_ctxt(next_lo, next_hi)), kind, }), - is_cleanup: false, - }) + false, + )) } fn link(&mut self, from_block: BasicBlock, to_block: BasicBlock) { diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs index d0b313e149a..fb17cca30f4 100644 --- a/compiler/rustc_mir_transform/src/ctfe_limit.rs +++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs @@ -55,8 +55,8 @@ fn has_back_edge( } fn insert_counter(basic_block_data: &mut BasicBlockData<'_>) { - basic_block_data.statements.push(Statement { - source_info: basic_block_data.terminator().source_info, - kind: StatementKind::ConstEvalCounter, - }); + basic_block_data.statements.push(Statement::new( + basic_block_data.terminator().source_info, + StatementKind::ConstEvalCounter, + )); } diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 0cf8142a560..fe53de31f75 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -16,7 +16,6 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; -use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::fmt::DebugWithContext; use rustc_mir_dataflow::lattice::{FlatSet, HasBottom}; diff --git a/compiler/rustc_mir_transform/src/elaborate_drop.rs b/compiler/rustc_mir_transform/src/elaborate_drop.rs index c9bc52c6c7e..de96b1f255a 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drop.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drop.rs @@ -222,15 +222,14 @@ where let span = self.source_info.span; let pin_obj_bb = bb.unwrap_or_else(|| { - self.elaborator.patch().new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + self.elaborator.patch().new_block(BasicBlockData::new( + Some(Terminator { // Temporary terminator, will be replaced by patch source_info: self.source_info, kind: TerminatorKind::Return, }), - is_cleanup: false, - }) + false, + )) }); let (fut_ty, drop_fn_def_id, trait_args) = if call_destructor_only { @@ -366,10 +365,8 @@ where call_statements.push(self.assign(obj_ptr_place, addr)); obj_ptr_place }; - call_statements.push(Statement { - source_info: self.source_info, - kind: StatementKind::StorageLive(fut.local), - }); + call_statements + .push(Statement::new(self.source_info, StatementKind::StorageLive(fut.local))); let call_drop_bb = self.new_block_with_statements( unwind, @@ -732,17 +729,17 @@ where let do_drop_bb = self.drop_subpath(interior, interior_path, succ, unwind, dropline); - let setup_bbd = BasicBlockData { - statements: vec![self.assign( + let setup_bbd = BasicBlockData::new_stmts( + vec![self.assign( Place::from(ptr_local), Rvalue::Cast(CastKind::Transmute, Operand::Copy(nonnull_place), ptr_ty), )], - terminator: Some(Terminator { + Some(Terminator { kind: TerminatorKind::Goto { target: do_drop_bb }, source_info: self.source_info, }), - is_cleanup: unwind.is_cleanup(), - }; + unwind.is_cleanup(), + ); self.elaborator.patch().new_block(setup_bbd) } @@ -753,14 +750,13 @@ where args: GenericArgsRef<'tcx>, ) -> BasicBlock { if adt.variants().is_empty() { - return self.elaborator.patch().new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + return self.elaborator.patch().new_block(BasicBlockData::new( + Some(Terminator { source_info: self.source_info, kind: TerminatorKind::Unreachable, }), - is_cleanup: self.unwind.is_cleanup(), - }); + self.unwind.is_cleanup(), + )); } let skip_contents = adt.is_union() || adt.is_manually_drop(); @@ -927,9 +923,9 @@ where let discr_ty = adt.repr().discr_type().to_ty(self.tcx()); let discr = Place::from(self.new_temp(discr_ty)); let discr_rv = Rvalue::Discriminant(self.place); - let switch_block = BasicBlockData { - statements: vec![self.assign(discr, discr_rv)], - terminator: Some(Terminator { + let switch_block = BasicBlockData::new_stmts( + vec![self.assign(discr, discr_rv)], + Some(Terminator { source_info: self.source_info, kind: TerminatorKind::SwitchInt { discr: Operand::Move(discr), @@ -939,8 +935,8 @@ where ), }, }), - is_cleanup: unwind.is_cleanup(), - }; + unwind.is_cleanup(), + ); let switch_block = self.elaborator.patch().new_block(switch_block); self.drop_flag_test_block(switch_block, succ, unwind) } @@ -956,8 +952,8 @@ where let ref_place = self.new_temp(ref_ty); let unit_temp = Place::from(self.new_temp(tcx.types.unit)); - let result = BasicBlockData { - statements: vec![self.assign( + let result = BasicBlockData::new_stmts( + vec![self.assign( Place::from(ref_place), Rvalue::Ref( tcx.lifetimes.re_erased, @@ -965,7 +961,7 @@ where self.place, ), )], - terminator: Some(Terminator { + Some(Terminator { kind: TerminatorKind::Call { func: Operand::function_handle( tcx, @@ -983,8 +979,8 @@ where }, source_info: self.source_info, }), - is_cleanup: unwind.is_cleanup(), - }; + unwind.is_cleanup(), + ); let destructor_block = self.elaborator.patch().new_block(result); @@ -1047,8 +1043,8 @@ where let can_go = Place::from(self.new_temp(tcx.types.bool)); let one = self.constant_usize(1); - let drop_block = BasicBlockData { - statements: vec![ + let drop_block = BasicBlockData::new_stmts( + vec![ self.assign( ptr, Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_index(self.place, cur)), @@ -1058,26 +1054,26 @@ where Rvalue::BinaryOp(BinOp::Add, Box::new((move_(cur.into()), one))), ), ], - is_cleanup: unwind.is_cleanup(), - terminator: Some(Terminator { + Some(Terminator { source_info: self.source_info, // this gets overwritten by drop elaboration. kind: TerminatorKind::Unreachable, }), - }; + unwind.is_cleanup(), + ); let drop_block = self.elaborator.patch().new_block(drop_block); - let loop_block = BasicBlockData { - statements: vec![self.assign( + let loop_block = BasicBlockData::new_stmts( + vec![self.assign( can_go, Rvalue::BinaryOp(BinOp::Eq, Box::new((copy(Place::from(cur)), copy(len.into())))), )], - is_cleanup: unwind.is_cleanup(), - terminator: Some(Terminator { + Some(Terminator { source_info: self.source_info, kind: TerminatorKind::if_(move_(can_go), succ, drop_block), }), - }; + unwind.is_cleanup(), + ); let loop_block = self.elaborator.patch().new_block(loop_block); let place = tcx.mk_place_deref(ptr); @@ -1187,8 +1183,8 @@ where let slice_ptr_ty = Ty::new_mut_ptr(tcx, slice_ty); let slice_ptr = self.new_temp(slice_ptr_ty); - let mut delegate_block = BasicBlockData { - statements: vec![ + let mut delegate_block = BasicBlockData::new_stmts( + vec![ self.assign(Place::from(array_ptr), Rvalue::RawPtr(RawPtrKind::Mut, self.place)), self.assign( Place::from(slice_ptr), @@ -1202,9 +1198,9 @@ where ), ), ], - is_cleanup: self.unwind.is_cleanup(), - terminator: None, - }; + None, + self.unwind.is_cleanup(), + ); let array_place = mem::replace( &mut self.place, @@ -1246,8 +1242,8 @@ where }; let zero = self.constant_usize(0); - let block = BasicBlockData { - statements: vec![ + let block = BasicBlockData::new_stmts( + vec![ self.assign( len.into(), Rvalue::UnaryOp( @@ -1257,12 +1253,12 @@ where ), self.assign(cur.into(), Rvalue::Use(zero)), ], - is_cleanup: unwind.is_cleanup(), - terminator: Some(Terminator { + Some(Terminator { source_info: self.source_info, kind: TerminatorKind::Goto { target: loop_block }, }), - }; + unwind.is_cleanup(), + ); let drop_block = self.elaborator.patch().new_block(block); // FIXME(#34708): handle partially-dropped array/slice elements. @@ -1308,14 +1304,13 @@ where self.source_info.span, "open drop for unsafe binder shouldn't be encountered", ); - self.elaborator.patch().new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + self.elaborator.patch().new_block(BasicBlockData::new( + Some(Terminator { source_info: self.source_info, kind: TerminatorKind::Unreachable, }), - is_cleanup: self.unwind.is_cleanup(), - }) + self.unwind.is_cleanup(), + )) } _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty), @@ -1434,11 +1429,10 @@ where } fn new_block(&mut self, unwind: Unwind, k: TerminatorKind<'tcx>) -> BasicBlock { - self.elaborator.patch().new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { source_info: self.source_info, kind: k }), - is_cleanup: unwind.is_cleanup(), - }) + self.elaborator.patch().new_block(BasicBlockData::new( + Some(Terminator { source_info: self.source_info, kind: k }), + unwind.is_cleanup(), + )) } fn new_block_with_statements( @@ -1447,11 +1441,11 @@ where statements: Vec<Statement<'tcx>>, k: TerminatorKind<'tcx>, ) -> BasicBlock { - self.elaborator.patch().new_block(BasicBlockData { + self.elaborator.patch().new_block(BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { source_info: self.source_info, kind: k }), - is_cleanup: unwind.is_cleanup(), - }) + Some(Terminator { source_info: self.source_info, kind: k }), + unwind.is_cleanup(), + )) } fn new_temp(&mut self, ty: Ty<'tcx>) -> Local { @@ -1467,9 +1461,6 @@ where } fn assign(&self, lhs: Place<'tcx>, rhs: Rvalue<'tcx>) -> Statement<'tcx> { - Statement { - source_info: self.source_info, - kind: StatementKind::Assign(Box::new((lhs, rhs))), - } + Statement::new(self.source_info, StatementKind::Assign(Box::new((lhs, rhs)))) } } diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index bda71ceaa55..335354c23b6 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -102,7 +102,7 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::GlobalAlloc; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; +use rustc_middle::ty::layout::HasTypingEnv; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::DUMMY_SP; use rustc_span::def_id::DefId; @@ -1625,7 +1625,7 @@ fn op_to_prop_const<'tcx>( // If this constant is already represented as an `Allocation`, // try putting it into global memory to return it. if let Either::Left(mplace) = op.as_mplace_or_imm() { - let (size, _align) = ecx.size_and_align_of_mplace(&mplace).discard_err()??; + let (size, _align) = ecx.size_and_align_of_val(&mplace).discard_err()??; // Do not try interning a value that contains provenance. // Due to https://github.com/rust-lang/rust/issues/79738, doing so could lead to bugs. @@ -1636,7 +1636,7 @@ fn op_to_prop_const<'tcx>( } let pointer = mplace.ptr().into_pointer_or_addr().ok()?; - let (prov, offset) = pointer.into_parts(); + let (prov, offset) = pointer.prov_and_relative_offset(); let alloc_id = prov.alloc_id(); intern_const_alloc_for_constprop(ecx, alloc_id).discard_err()?; diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index f48dba9663a..7852bb7ae2f 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -770,14 +770,15 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>( return Ok(()); } - if callee_def_id.is_local() + if let Some(callee_def_id) = callee_def_id.as_local() && !inliner .tcx() .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce) { // If we know for sure that the function we're calling will itself try to // call us, then we avoid inlining that function. - if inliner.tcx().mir_callgraph_reachable((callee, caller_def_id.expect_local())) { + if inliner.tcx().mir_callgraph_cyclic(caller_def_id.expect_local()).contains(&callee_def_id) + { debug!("query cycle avoidance"); return Err("caller might be reachable from callee"); } @@ -899,10 +900,10 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( ); let dest_ty = dest.ty(caller_body, tcx); let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block)); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((temp, dest))), - }); + caller_body[callsite.block].statements.push(Statement::new( + callsite.source_info, + StatementKind::Assign(Box::new((temp, dest))), + )); tcx.mk_place_deref(temp) } else { destination @@ -946,10 +947,9 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( for local in callee_body.vars_and_temps_iter() { if integrator.always_live_locals.contains(local) { let new_local = integrator.map_local(local); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageLive(new_local), - }); + caller_body[callsite.block] + .statements + .push(Statement::new(callsite.source_info, StatementKind::StorageLive(new_local))); } } if let Some(block) = return_block { @@ -957,22 +957,22 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( // the slice once. let mut n = 0; if remap_destination { - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new(( + caller_body[block].statements.push(Statement::new( + callsite.source_info, + StatementKind::Assign(Box::new(( dest, Rvalue::Use(Operand::Move(destination_local.into())), ))), - }); + )); n += 1; } for local in callee_body.vars_and_temps_iter().rev() { if integrator.always_live_locals.contains(local) { let new_local = integrator.map_local(local); - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageDead(new_local), - }); + caller_body[block].statements.push(Statement::new( + callsite.source_info, + StatementKind::StorageDead(new_local), + )); n += 1; } } @@ -1125,10 +1125,10 @@ fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>( trace!("creating temp for argument {:?}", arg); let arg_ty = arg.ty(caller_body, inliner.tcx()); let local = new_call_temp(caller_body, callsite, arg_ty, return_block); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), - }); + caller_body[callsite.block].statements.push(Statement::new( + callsite.source_info, + StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), + )); local } @@ -1141,19 +1141,14 @@ fn new_call_temp<'tcx>( ) -> Local { let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageLive(local), - }); + caller_body[callsite.block] + .statements + .push(Statement::new(callsite.source_info, StatementKind::StorageLive(local))); if let Some(block) = return_block { - caller_body[block].statements.insert( - 0, - Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageDead(local), - }, - ); + caller_body[block] + .statements + .insert(0, Statement::new(callsite.source_info, StatementKind::StorageDead(local))); } local diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index a944960ce4a..08f3ce5fd67 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -1,5 +1,6 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_data_structures::stack::ensure_sufficient_stack; +use rustc_data_structures::unord::UnordSet; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::mir::TerminatorKind; use rustc_middle::ty::{self, GenericArgsRef, InstanceKind, TyCtxt, TypeVisitableExt}; @@ -7,137 +8,143 @@ use rustc_session::Limit; use rustc_span::sym; use tracing::{instrument, trace}; -// FIXME: check whether it is cheaper to precompute the entire call graph instead of invoking -// this query ridiculously often. -#[instrument(level = "debug", skip(tcx, root, target))] -pub(crate) fn mir_callgraph_reachable<'tcx>( +#[instrument(level = "debug", skip(tcx), ret)] +fn should_recurse<'tcx>(tcx: TyCtxt<'tcx>, callee: ty::Instance<'tcx>) -> bool { + match callee.def { + // If there is no MIR available (either because it was not in metadata or + // because it has no MIR because it's an extern function), then the inliner + // won't cause cycles on this. + InstanceKind::Item(_) => { + if !tcx.is_mir_available(callee.def_id()) { + return false; + } + } + + // These have no own callable MIR. + InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => return false, + + // These have MIR and if that MIR is inlined, instantiated and then inlining is run + // again, a function item can end up getting inlined. Thus we'll be able to cause + // a cycle that way + InstanceKind::VTableShim(_) + | InstanceKind::ReifyShim(..) + | InstanceKind::FnPtrShim(..) + | InstanceKind::ClosureOnceShim { .. } + | InstanceKind::ConstructCoroutineInClosureShim { .. } + | InstanceKind::ThreadLocalShim { .. } + | InstanceKind::CloneShim(..) => {} + + // This shim does not call any other functions, thus there can be no recursion. + InstanceKind::FnPtrAddrShim(..) => return false, + + // FIXME: A not fully instantiated drop shim can cause ICEs if one attempts to + // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this + // needs some more analysis. + InstanceKind::DropGlue(..) + | InstanceKind::FutureDropPollShim(..) + | InstanceKind::AsyncDropGlue(..) + | InstanceKind::AsyncDropGlueCtorShim(..) => { + if callee.has_param() { + return false; + } + } + } + + crate::pm::should_run_pass(tcx, &crate::inline::Inline, crate::pm::Optimizations::Allowed) + || crate::inline::ForceInline::should_run_pass_for_callee(tcx, callee.def.def_id()) +} + +#[instrument( + level = "debug", + skip(tcx, typing_env, seen, involved, recursion_limiter, recursion_limit), + ret +)] +fn process<'tcx>( tcx: TyCtxt<'tcx>, - (root, target): (ty::Instance<'tcx>, LocalDefId), + typing_env: ty::TypingEnv<'tcx>, + caller: ty::Instance<'tcx>, + target: LocalDefId, + seen: &mut FxHashSet<ty::Instance<'tcx>>, + involved: &mut FxHashSet<LocalDefId>, + recursion_limiter: &mut FxHashMap<DefId, usize>, + recursion_limit: Limit, ) -> bool { - trace!(%root, target = %tcx.def_path_str(target)); - assert_ne!( - root.def_id().expect_local(), - target, - "you should not call `mir_callgraph_reachable` on immediate self recursion" - ); - assert!( - matches!(root.def, InstanceKind::Item(_)), - "you should not call `mir_callgraph_reachable` on shims" - ); - assert!( - !tcx.is_constructor(root.def_id()), - "you should not call `mir_callgraph_reachable` on enum/struct constructor functions" - ); - #[instrument( - level = "debug", - skip(tcx, typing_env, target, stack, seen, recursion_limiter, caller, recursion_limit) - )] - fn process<'tcx>( - tcx: TyCtxt<'tcx>, - typing_env: ty::TypingEnv<'tcx>, - caller: ty::Instance<'tcx>, - target: LocalDefId, - stack: &mut Vec<ty::Instance<'tcx>>, - seen: &mut FxHashSet<ty::Instance<'tcx>>, - recursion_limiter: &mut FxHashMap<DefId, usize>, - recursion_limit: Limit, - ) -> bool { - trace!(%caller); - for &(callee, args) in tcx.mir_inliner_callees(caller.def) { - let Ok(args) = caller.try_instantiate_mir_and_normalize_erasing_regions( - tcx, - typing_env, - ty::EarlyBinder::bind(args), - ) else { - trace!(?caller, ?typing_env, ?args, "cannot normalize, skipping"); - continue; - }; - let Ok(Some(callee)) = ty::Instance::try_resolve(tcx, typing_env, callee, args) else { - trace!(?callee, "cannot resolve, skipping"); - continue; - }; + trace!(%caller); + let mut cycle_found = false; - // Found a path. - if callee.def_id() == target.to_def_id() { - return true; - } + for &(callee, args) in tcx.mir_inliner_callees(caller.def) { + let Ok(args) = caller.try_instantiate_mir_and_normalize_erasing_regions( + tcx, + typing_env, + ty::EarlyBinder::bind(args), + ) else { + trace!(?caller, ?typing_env, ?args, "cannot normalize, skipping"); + continue; + }; + let Ok(Some(callee)) = ty::Instance::try_resolve(tcx, typing_env, callee, args) else { + trace!(?callee, "cannot resolve, skipping"); + continue; + }; - if tcx.is_constructor(callee.def_id()) { - trace!("constructors always have MIR"); - // Constructor functions cannot cause a query cycle. - continue; - } + // Found a path. + if callee.def_id() == target.to_def_id() { + cycle_found = true; + } - match callee.def { - InstanceKind::Item(_) => { - // If there is no MIR available (either because it was not in metadata or - // because it has no MIR because it's an extern function), then the inliner - // won't cause cycles on this. - if !tcx.is_mir_available(callee.def_id()) { - trace!(?callee, "no mir available, skipping"); - continue; - } - } - // These have no own callable MIR. - InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => continue, - // These have MIR and if that MIR is inlined, instantiated and then inlining is run - // again, a function item can end up getting inlined. Thus we'll be able to cause - // a cycle that way - InstanceKind::VTableShim(_) - | InstanceKind::ReifyShim(..) - | InstanceKind::FnPtrShim(..) - | InstanceKind::ClosureOnceShim { .. } - | InstanceKind::ConstructCoroutineInClosureShim { .. } - | InstanceKind::ThreadLocalShim { .. } - | InstanceKind::CloneShim(..) => {} - - // This shim does not call any other functions, thus there can be no recursion. - InstanceKind::FnPtrAddrShim(..) => { - continue; - } - InstanceKind::DropGlue(..) - | InstanceKind::FutureDropPollShim(..) - | InstanceKind::AsyncDropGlue(..) - | InstanceKind::AsyncDropGlueCtorShim(..) => { - // FIXME: A not fully instantiated drop shim can cause ICEs if one attempts to - // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this - // needs some more analysis. - if callee.has_param() { - continue; - } - } - } + if tcx.is_constructor(callee.def_id()) { + trace!("constructors always have MIR"); + // Constructor functions cannot cause a query cycle. + continue; + } + + if !should_recurse(tcx, callee) { + continue; + } - if seen.insert(callee) { - let recursion = recursion_limiter.entry(callee.def_id()).or_default(); - trace!(?callee, recursion = *recursion); - if recursion_limit.value_within_limit(*recursion) { - *recursion += 1; - stack.push(callee); - let found_recursion = ensure_sufficient_stack(|| { - process( - tcx, - typing_env, - callee, - target, - stack, - seen, - recursion_limiter, - recursion_limit, - ) - }); - if found_recursion { - return true; - } - stack.pop(); - } else { - // Pessimistically assume that there could be recursion. - return true; + if seen.insert(callee) { + let recursion = recursion_limiter.entry(callee.def_id()).or_default(); + trace!(?callee, recursion = *recursion); + let found_recursion = if recursion_limit.value_within_limit(*recursion) { + *recursion += 1; + ensure_sufficient_stack(|| { + process( + tcx, + typing_env, + callee, + target, + seen, + involved, + recursion_limiter, + recursion_limit, + ) + }) + } else { + // Pessimistically assume that there could be recursion. + true + }; + if found_recursion { + if let Some(callee) = callee.def_id().as_local() { + // Calling `optimized_mir` of a non-local definition cannot cycle. + involved.insert(callee); } + cycle_found = true; } } - false } + + cycle_found +} + +#[instrument(level = "debug", skip(tcx), ret)] +pub(crate) fn mir_callgraph_cyclic<'tcx>( + tcx: TyCtxt<'tcx>, + root: LocalDefId, +) -> UnordSet<LocalDefId> { + assert!( + !tcx.is_constructor(root.to_def_id()), + "you should not call `mir_callgraph_reachable` on enum/struct constructor functions" + ); + // FIXME(-Znext-solver=no): Remove this hack when trait solver overflow can return an error. // In code like that pointed out in #128887, the type complexity we ask the solver to deal with // grows as we recurse into the call graph. If we use the same recursion limit here and in the @@ -146,16 +153,32 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( // the default recursion limits are quite generous for us. If we need to recurse 64 times // into the call graph, we're probably not going to find any useful MIR inlining. let recursion_limit = tcx.recursion_limit() / 2; + let mut involved = FxHashSet::default(); + let typing_env = ty::TypingEnv::post_analysis(tcx, root); + let Ok(Some(root_instance)) = ty::Instance::try_resolve( + tcx, + typing_env, + root.to_def_id(), + ty::GenericArgs::identity_for_item(tcx, root.to_def_id()), + ) else { + trace!("cannot resolve, skipping"); + return involved.into(); + }; + if !should_recurse(tcx, root_instance) { + trace!("cannot walk, skipping"); + return involved.into(); + } process( tcx, - ty::TypingEnv::post_analysis(tcx, target), + typing_env, + root_instance, root, - target, - &mut Vec::new(), &mut FxHashSet::default(), + &mut involved, &mut FxHashMap::default(), recursion_limit, - ) + ); + involved.into() } pub(crate) fn mir_inliner_callees<'tcx>( diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 5f0c55ddc09..dbcaed20953 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -240,15 +240,15 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { let Some(arg_place) = arg.node.place() else { return }; - statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::Use(Operand::Copy( arg_place.project_deeper(&[ProjectionElem::Deref], self.tcx), )), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target: *destination_block }; } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index b45bff2af44..f9e642e28eb 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -45,7 +45,6 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; -use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, ScalarInt, TyCtxt}; use rustc_mir_dataflow::lattice::HasBottom; use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem}; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 572ad585c8c..08f25276cec 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -117,6 +117,7 @@ declare_passes! { mod check_inline : CheckForceInline; mod check_call_recursion : CheckCallRecursion, CheckDropRecursion; mod check_alignment : CheckAlignment; + mod check_enums : CheckEnums; mod check_const_item_mutation : CheckConstItemMutation; mod check_null : CheckNull; mod check_packed_ref : CheckPackedRef; @@ -215,7 +216,7 @@ pub fn provide(providers: &mut Providers) { optimized_mir, is_mir_available, is_ctfe_mir_available: is_mir_available, - mir_callgraph_reachable: inline::cycle::mir_callgraph_reachable, + mir_callgraph_cyclic: inline::cycle::mir_callgraph_cyclic, mir_inliner_callees: inline::cycle::mir_inliner_callees, promoted_mir, deduced_param_attrs: deduce_param_attrs::deduced_param_attrs, @@ -258,13 +259,13 @@ fn remap_mir_for_const_eval_select<'tcx>( // (const generic stuff) so we just create a temporary and deconstruct // that. let local = body.local_decls.push(LocalDecl::new(ty, fn_span)); - bb.statements.push(Statement { - source_info: SourceInfo::outermost(fn_span), - kind: StatementKind::Assign(Box::new(( + bb.statements.push(Statement::new( + SourceInfo::outermost(fn_span), + StatementKind::Assign(Box::new(( local.into(), Rvalue::Use(tupled_args.node.clone()), ))), - }); + )); (Operand::Move, local.into()) } Operand::Move(place) => (Operand::Move, place), @@ -666,6 +667,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' // Add some UB checks before any UB gets optimized away. &check_alignment::CheckAlignment, &check_null::CheckNull, + &check_enums::CheckEnums, // Before inlining: trim down MIR with passes to reduce inlining work. // Has to be done before inlining, otherwise actual call will be almost always inlined. diff --git a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs index 75f351f05c3..1bd770a8526 100644 --- a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs +++ b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs @@ -516,8 +516,12 @@ struct LocalLabel<'a> { /// A custom `Subdiagnostic` implementation so that the notes are delivered in a specific order impl Subdiagnostic for LocalLabel<'_> { fn add_to_diag<G: rustc_errors::EmissionGuarantee>(self, diag: &mut rustc_errors::Diag<'_, G>) { + // Becuase parent uses this field , we need to remove it delay before adding it. + diag.remove_arg("name"); diag.arg("name", self.name); + diag.remove_arg("is_generated_name"); diag.arg("is_generated_name", self.is_generated_name); + diag.remove_arg("is_dropped_first_edition_2024"); diag.arg("is_dropped_first_edition_2024", self.is_dropped_first_edition_2024); let msg = diag.eagerly_translate(crate::fluent_generated::mir_transform_tail_expr_local); diag.span_label(self.span, msg); diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index fa29ab985b7..8dadce0d448 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -25,31 +25,31 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { } sym::ub_checks => { let target = target.unwrap(); - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::NullaryOp(NullOp::UbChecks, tcx.types.bool), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::contract_checks => { let target = target.unwrap(); - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::NullaryOp(NullOp::ContractChecks, tcx.types.bool), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::forget => { let target = target.unwrap(); - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::Use(Operand::Constant(Box::new(ConstOperand { span: terminator.source_info.span, @@ -57,7 +57,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { const_: Const::zero_sized(tcx.types.unit), }))), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::copy_nonoverlapping => { @@ -65,9 +65,9 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { let Ok([src, dst, count]) = take_array(args) else { bug!("Wrong arguments for copy_non_overlapping intrinsic"); }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Intrinsic(Box::new( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Intrinsic(Box::new( NonDivergingIntrinsic::CopyNonOverlapping( rustc_middle::mir::CopyNonOverlapping { src: src.node, @@ -76,7 +76,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { }, ), )), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::assume => { @@ -84,12 +84,12 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { let Ok([arg]) = take_array(args) else { bug!("Wrong arguments for assume intrinsic"); }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Intrinsic(Box::new( - NonDivergingIntrinsic::Assume(arg.node), - )), - }); + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume( + arg.node, + ))), + )); terminator.kind = TerminatorKind::Goto { target }; } sym::wrapping_add @@ -121,13 +121,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { sym::unchecked_shr => BinOp::ShrUnchecked, _ => bug!("unexpected intrinsic"), }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => { @@ -141,13 +141,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { sym::mul_with_overflow => BinOp::MulWithOverflow, _ => bug!("unexpected intrinsic"), }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::size_of | sym::align_of => { @@ -158,13 +158,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { sym::align_of => NullOp::AlignOf, _ => bug!("unexpected intrinsic"), }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::NullaryOp(null_op, tp_ty), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::read_via_copy => { @@ -183,13 +183,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { }; // Add new statement at the end of the block that does the read, and patch // up the terminator. - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::Use(Operand::Copy(derefed_place)), ))), - }); + )); terminator.kind = match *target { None => { // No target means this read something uninhabited, @@ -217,13 +217,10 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { "Only passing a local is supported" ); }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( - derefed_place, - Rvalue::Use(val.node), - ))), - }); + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new((derefed_place, Rvalue::Use(val.node)))), + )); terminator.kind = TerminatorKind::Goto { target }; } sym::discriminant_value => { @@ -236,13 +233,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { }; let arg = arg.node.place().unwrap(); let arg = tcx.mk_place_deref(arg); - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::Discriminant(arg), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::offset => { @@ -253,13 +250,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { "Wrong number of arguments for offset intrinsic", ); }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::slice_get_unchecked => { @@ -302,10 +299,10 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { _ => bug!("Unknown return type {ret_ty:?}"), }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new((*destination, rvalue))), - }); + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new((*destination, rvalue))), + )); terminator.kind = TerminatorKind::Goto { target }; } sym::transmute | sym::transmute_unchecked => { @@ -320,13 +317,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { // Always emit the cast, even if we transmute to an uninhabited type, // because that lets CTFE and codegen generate better error messages // when such a transmute actually ends up reachable. - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty), ))), - }); + )); if let Some(target) = *target { terminator.kind = TerminatorKind::Goto { target }; } else { @@ -351,13 +348,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { ); }; let fields = [data.node, meta.node]; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::Aggregate(Box::new(kind), fields.into()), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } sym::ptr_metadata => { @@ -368,13 +365,13 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { ); }; let target = target.unwrap(); - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( *destination, Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node), ))), - }); + )); terminator.kind = TerminatorKind::Goto { target }; } _ => {} diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index aca80e36e33..79a9017de3e 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -56,8 +56,7 @@ fn lower_slice_len_call<'tcx>(block: &mut BasicBlockData<'tcx>, slice_len_fn_ite // make new RValue for Len let r_value = Rvalue::UnaryOp(UnOp::PtrMetadata, arg.node.clone()); let len_statement_kind = StatementKind::Assign(Box::new((*destination, r_value))); - let add_statement = - Statement { kind: len_statement_kind, source_info: terminator.source_info }; + let add_statement = Statement::new(terminator.source_info, len_statement_kind); // modify terminator into simple Goto let new_terminator_kind = TerminatorKind::Goto { target: *bb }; diff --git a/compiler/rustc_mir_transform/src/mentioned_items.rs b/compiler/rustc_mir_transform/src/mentioned_items.rs index 9fd8d81d64a..f011d394f61 100644 --- a/compiler/rustc_mir_transform/src/mentioned_items.rs +++ b/compiler/rustc_mir_transform/src/mentioned_items.rs @@ -74,8 +74,7 @@ impl<'tcx> Visitor<'tcx> for MentionedItemsVisitor<'_, 'tcx> { match *rvalue { // We need to detect unsizing casts that required vtables. mir::Rvalue::Cast( - mir::CastKind::PointerCoercion(PointerCoercion::Unsize, _) - | mir::CastKind::PointerCoercion(PointerCoercion::DynStar, _), + mir::CastKind::PointerCoercion(PointerCoercion::Unsize, _), ref operand, target_ty, ) => { diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index a872eae15f1..c781d1a5324 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -78,14 +78,13 @@ impl<'tcx> MirPatch<'tcx> { return bb; } - let bb = self.new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + let bb = self.new_block(BasicBlockData::new( + Some(Terminator { source_info: SourceInfo::outermost(self.body_span), kind: TerminatorKind::UnwindResume, }), - is_cleanup: true, - }); + true, + )); self.resume_block = Some(bb); bb } @@ -95,14 +94,13 @@ impl<'tcx> MirPatch<'tcx> { return bb; } - let bb = self.new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + let bb = self.new_block(BasicBlockData::new( + Some(Terminator { source_info: SourceInfo::outermost(self.body_span), kind: TerminatorKind::Unreachable, }), - is_cleanup: true, - }); + true, + )); self.unreachable_cleanup_block = Some(bb); bb } @@ -112,14 +110,13 @@ impl<'tcx> MirPatch<'tcx> { return bb; } - let bb = self.new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + let bb = self.new_block(BasicBlockData::new( + Some(Terminator { source_info: SourceInfo::outermost(self.body_span), kind: TerminatorKind::Unreachable, }), - is_cleanup: false, - }); + false, + )); self.unreachable_no_cleanup_block = Some(bb); bb } @@ -131,14 +128,13 @@ impl<'tcx> MirPatch<'tcx> { return cached_bb; } - let bb = self.new_block(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + let bb = self.new_block(BasicBlockData::new( + Some(Terminator { source_info: SourceInfo::outermost(self.body_span), kind: TerminatorKind::UnwindTerminate(reason), }), - is_cleanup: true, - }); + true, + )); self.terminate_block = Some((bb, reason)); bb } @@ -280,7 +276,7 @@ impl<'tcx> MirPatch<'tcx> { let source_info = Self::source_info_for_index(&body[loc.block], loc); body[loc.block] .statements - .insert(loc.statement_index, Statement { source_info, kind: stmt }); + .insert(loc.statement_index, Statement::new(source_info, stmt)); delta += 1; } } diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 47d43830970..4e8f30e077b 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -731,23 +731,22 @@ struct Promoter<'a, 'tcx> { impl<'a, 'tcx> Promoter<'a, 'tcx> { fn new_block(&mut self) -> BasicBlock { let span = self.promoted.span; - self.promoted.basic_blocks_mut().push(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + self.promoted.basic_blocks_mut().push(BasicBlockData::new( + Some(Terminator { source_info: SourceInfo::outermost(span), kind: TerminatorKind::Return, }), - is_cleanup: false, - }) + false, + )) } fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) { let last = self.promoted.basic_blocks.last_index().unwrap(); let data = &mut self.promoted[last]; - data.statements.push(Statement { - source_info: SourceInfo::outermost(span), - kind: StatementKind::Assign(Box::new((Place::from(dest), rvalue))), - }); + data.statements.push(Statement::new( + SourceInfo::outermost(span), + StatementKind::Assign(Box::new((Place::from(dest), rvalue))), + )); } fn is_temp_kind(&self, local: Local) -> bool { @@ -914,13 +913,13 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { assert_eq!(self.temps.push(TempState::Unpromotable), promoted_ref); let promoted_operand = promoted_operand(ref_ty, span); - let promoted_ref_statement = Statement { - source_info: statement.source_info, - kind: StatementKind::Assign(Box::new(( + let promoted_ref_statement = Statement::new( + statement.source_info, + StatementKind::Assign(Box::new(( Place::from(promoted_ref), Rvalue::Use(Operand::Constant(Box::new(promoted_operand))), ))), - }; + ); self.extra_statements.push((loc, promoted_ref_statement)); ( diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 6d45bbc6e16..4083038cbb6 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -323,9 +323,7 @@ fn dropee_emit_retag<'tcx>( StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)), ]; for s in new_statements { - body.basic_blocks_mut()[START_BLOCK] - .statements - .push(Statement { source_info, kind: s }); + body.basic_blocks_mut()[START_BLOCK].statements.push(Statement::new(source_info, s)); } } dropee_ptr @@ -350,11 +348,7 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) let return_block = BasicBlock::new(1); let mut blocks = IndexVec::with_capacity(2); let block = |blocks: &mut IndexVec<_, _>, kind| { - blocks.push(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { source_info, kind }), - is_cleanup: false, - }) + blocks.push(BasicBlockData::new(Some(Terminator { source_info, kind }), false)) }; block(&mut blocks, TerminatorKind::Goto { target: return_block }); block(&mut blocks, TerminatorKind::Return); @@ -515,17 +509,17 @@ fn build_thread_local_shim<'tcx>( let span = tcx.def_span(def_id); let source_info = SourceInfo::outermost(span); - let blocks = IndexVec::from_raw(vec![BasicBlockData { - statements: vec![Statement { + let blocks = IndexVec::from_raw(vec![BasicBlockData::new_stmts( + vec![Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( Place::return_place(), Rvalue::ThreadLocalRef(def_id), ))), - }], - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }]); + )], + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + )]); new_body( MirSource::from_instance(instance), @@ -609,11 +603,11 @@ impl<'tcx> CloneShimBuilder<'tcx> { is_cleanup: bool, ) -> BasicBlock { let source_info = self.source_info(); - self.blocks.push(BasicBlockData { + self.blocks.push(BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { source_info, kind }), + Some(Terminator { source_info, kind }), is_cleanup, - }) + )) } /// Gives the index of an upcoming BasicBlock, with an offset. @@ -625,7 +619,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { } fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> { - Statement { source_info: self.source_info(), kind } + Statement::new(self.source_info(), kind) } fn copy_shim(&mut self) { @@ -901,13 +895,13 @@ fn build_call_shim<'tcx>( .immutable(), ); let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default }; - statements.push(Statement { + statements.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( Place::from(ref_rcvr), Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()), ))), - }); + )); Operand::Move(Place::from(ref_rcvr)) } }); @@ -956,11 +950,11 @@ fn build_call_shim<'tcx>( let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 }; let mut blocks = IndexVec::with_capacity(n_blocks); let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| { - blocks.push(BasicBlockData { + blocks.push(BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { source_info, kind }), + Some(Terminator { source_info, kind }), is_cleanup, - }) + )) }; // BB #0 @@ -1071,8 +1065,9 @@ pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { let kind = AggregateKind::Adt(adt_def.did(), variant_index, args, None, None); let variant = adt_def.variant(variant_index); - let statement = Statement { - kind: StatementKind::Assign(Box::new(( + let statement = Statement::new( + source_info, + StatementKind::Assign(Box::new(( Place::return_place(), Rvalue::Aggregate( Box::new(kind), @@ -1081,14 +1076,13 @@ pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { .collect(), ), ))), - source_info, - }; + ); - let start_block = BasicBlockData { - statements: vec![statement], - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }; + let start_block = BasicBlockData::new_stmts( + vec![statement], + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + ); let source = MirSource::item(ctor_id); let mut body = new_body( @@ -1130,16 +1124,16 @@ fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'t Operand::Move(Place::from(Local::new(1))), Ty::new_imm_ptr(tcx, tcx.types.unit), ); - let stmt = Statement { + let stmt = Statement::new( source_info, - kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))), - }; + StatementKind::Assign(Box::new((Place::return_place(), rvalue))), + ); let statements = vec![stmt]; - let start_block = BasicBlockData { + let start_block = BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }; + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + ); let source = MirSource::from_instance(ty::InstanceKind::FnPtrAddrShim(def_id, self_ty)); new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span) } @@ -1230,16 +1224,16 @@ fn build_construct_coroutine_by_move_shim<'tcx>( Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)), IndexVec::from_raw(fields), ); - let stmt = Statement { + let stmt = Statement::new( source_info, - kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))), - }; + StatementKind::Assign(Box::new((Place::return_place(), rvalue))), + ); let statements = vec![stmt]; - let start_block = BasicBlockData { + let start_block = BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }; + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + ); let source = MirSource::from_instance(ty::InstanceKind::ConstructCoroutineInClosureShim { coroutine_closure_def_id, diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index fd7b7362cd9..18d09473c19 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -88,11 +88,7 @@ pub(super) fn build_async_drop_shim<'tcx>( let return_block = BasicBlock::new(1); let mut blocks = IndexVec::with_capacity(2); let block = |blocks: &mut IndexVec<_, _>, kind| { - blocks.push(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { source_info, kind }), - is_cleanup: false, - }) + blocks.push(BasicBlockData::new(Some(Terminator { source_info, kind }), false)) }; block( &mut blocks, @@ -133,7 +129,7 @@ pub(super) fn build_async_drop_shim<'tcx>( dropee_ptr, Rvalue::Use(Operand::Move(coroutine_layout_dropee)), ))); - body.basic_blocks_mut()[START_BLOCK].statements.push(Statement { source_info, kind: st_kind }); + body.basic_blocks_mut()[START_BLOCK].statements.push(Statement::new(source_info, st_kind)); dropee_ptr = dropee_emit_retag(tcx, &mut body, dropee_ptr, span); let dropline = body.basic_blocks.last_index(); @@ -240,13 +236,13 @@ fn build_adrop_for_coroutine_shim<'tcx>( .project_deeper(&[PlaceElem::Field(FieldIdx::ZERO, proxy_ref)], tcx); body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, - Statement { + Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( Place::from(proxy_ref_local), Rvalue::CopyForDeref(proxy_ref_place), ))), - }, + ), ); idx += 1; let mut cor_ptr_local = proxy_ref_local; @@ -261,13 +257,13 @@ fn build_adrop_for_coroutine_shim<'tcx>( // _cor_ptr = _proxy.0.0 (... .0) body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, - Statement { + Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( Place::from(cor_ptr_local), Rvalue::CopyForDeref(impl_ptr_place), ))), - }, + ), ); idx += 1; } @@ -281,10 +277,10 @@ fn build_adrop_for_coroutine_shim<'tcx>( ); body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, - Statement { + Statement::new( source_info, - kind: StatementKind::Assign(Box::new((Place::from(cor_ref_local), reborrow))), - }, + StatementKind::Assign(Box::new((Place::from(cor_ref_local), reborrow))), + ), ); } body @@ -334,13 +330,13 @@ fn build_adrop_for_adrop_shim<'tcx>( let mut statements = Vec::new(); - statements.push(Statement { + statements.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( Place::from(proxy_ref_local), Rvalue::CopyForDeref(proxy_ref_place), ))), - }); + )); let mut cor_ptr_local = proxy_ref_local; proxy_ty.find_async_drop_impl_coroutine(tcx, |ty| { @@ -350,13 +346,13 @@ fn build_adrop_for_adrop_shim<'tcx>( .project_deeper(&[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], tcx); cor_ptr_local = locals.push(LocalDecl::new(ty_ptr, span)); // _cor_ptr = _proxy.0.0 (... .0) - statements.push(Statement { + statements.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new(( + StatementKind::Assign(Box::new(( Place::from(cor_ptr_local), Rvalue::CopyForDeref(impl_ptr_place), ))), - }); + )); } }); @@ -367,10 +363,10 @@ fn build_adrop_for_adrop_shim<'tcx>( tcx.mk_place_deref(Place::from(cor_ptr_local)), ); let cor_ref_place = Place::from(locals.push(LocalDecl::new(cor_ref, span))); - statements.push(Statement { + statements.push(Statement::new( source_info, - kind: StatementKind::Assign(Box::new((cor_ref_place, reborrow))), - }); + StatementKind::Assign(Box::new((cor_ref_place, reborrow))), + )); // cor_pin_ty = `Pin<&mut cor_ref>` let cor_pin_ty = Ty::new_adt(tcx, pin_adt_ref, tcx.mk_args(&[cor_ref.into()])); @@ -378,9 +374,9 @@ fn build_adrop_for_adrop_shim<'tcx>( let pin_fn = tcx.require_lang_item(LangItem::PinNewUnchecked, span); // call Pin<FutTy>::new_unchecked(&mut impl_cor) - blocks.push(BasicBlockData { + blocks.push(BasicBlockData::new_stmts( statements, - terminator: Some(Terminator { + Some(Terminator { source_info, kind: TerminatorKind::Call { func: Operand::function_handle(tcx, pin_fn, [cor_ref.into()], span), @@ -392,15 +388,14 @@ fn build_adrop_for_adrop_shim<'tcx>( fn_span: span, }, }), - is_cleanup: false, - }); + false, + )); // When dropping async drop coroutine, we continue its execution: // we call impl::poll (impl_layout, ctx) let poll_fn = tcx.require_lang_item(LangItem::FuturePoll, span); let resume_ctx = Place::from(Local::new(2)); - blocks.push(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { + blocks.push(BasicBlockData::new( + Some(Terminator { source_info, kind: TerminatorKind::Call { func: Operand::function_handle(tcx, poll_fn, [impl_ty.into()], span), @@ -416,13 +411,12 @@ fn build_adrop_for_adrop_shim<'tcx>( fn_span: span, }, }), - is_cleanup: false, - }); - blocks.push(BasicBlockData { - statements: vec![], - terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), - is_cleanup: false, - }); + false, + )); + blocks.push(BasicBlockData::new( + Some(Terminator { source_info, kind: TerminatorKind::Return }), + false, + )); let source = MirSource::from_instance(instance); let mut body = new_body(source, blocks, locals, sig.inputs().len(), span); diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index bd008230731..c60eb566521 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -115,10 +115,10 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral { for bb_idx in new_targets.all_targets() { storage_deads_to_insert.push(( *bb_idx, - Statement { - source_info: terminator.source_info, - kind: StatementKind::StorageDead(opt.to_switch_on.local), - }, + Statement::new( + terminator.source_info, + StatementKind::StorageDead(opt.to_switch_on.local), + ), )); } } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 7dcdd7999f2..cbb9bbfd12f 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1260,9 +1260,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.fail(location, format!("Unsize coercion, but `{op_ty}` isn't coercible to `{target_type}`")); } } - CastKind::PointerCoercion(PointerCoercion::DynStar, _) => { - // FIXME(dyn-star): make sure nothing needs to be done here. - } CastKind::IntToInt | CastKind::IntToFloat => { let input_valid = op_ty.is_integral() || op_ty.is_char() || op_ty.is_bool(); let target_valid = target_type.is_numeric() || target_type.is_char(); |