diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
18 files changed, 240 insertions, 883 deletions
diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs deleted file mode 100644 index a0c3de3af58..00000000000 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ /dev/null @@ -1,615 +0,0 @@ -use rustc_data_structures::unord::{ExtendUnord, UnordItems, UnordSet}; -use rustc_hir as hir; -use rustc_hir::def::DefKind; -use rustc_hir::def_id::{DefId, LocalDefId}; -use rustc_hir::hir_id::HirId; -use rustc_hir::intravisit; -use rustc_hir::{BlockCheckMode, ExprKind, Node}; -use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor}; -use rustc_middle::mir::*; -use rustc_middle::query::Providers; -use rustc_middle::ty::{self, TyCtxt}; -use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE}; -use rustc_session::lint::Level; - -use std::ops::Bound; - -use crate::errors; - -pub struct UnsafetyChecker<'a, 'tcx> { - body: &'a Body<'tcx>, - body_did: LocalDefId, - violations: Vec<UnsafetyViolation>, - source_info: SourceInfo, - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - - /// Used `unsafe` blocks in this function. This is used for the "unused_unsafe" lint. - used_unsafe_blocks: UnordSet<HirId>, -} - -impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { - fn new( - body: &'a Body<'tcx>, - body_did: LocalDefId, - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - ) -> Self { - Self { - body, - body_did, - violations: vec![], - source_info: SourceInfo::outermost(body.span), - tcx, - param_env, - used_unsafe_blocks: Default::default(), - } - } -} - -impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { - self.source_info = terminator.source_info; - match terminator.kind { - TerminatorKind::Goto { .. } - | TerminatorKind::SwitchInt { .. } - | TerminatorKind::Drop { .. } - | TerminatorKind::Yield { .. } - | TerminatorKind::Assert { .. } - | TerminatorKind::CoroutineDrop - | TerminatorKind::UnwindResume - | TerminatorKind::UnwindTerminate(_) - | TerminatorKind::Return - | TerminatorKind::Unreachable - | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } => { - // safe (at least as emitted during MIR construction) - } - - TerminatorKind::Call { ref func, .. } => { - let func_ty = func.ty(self.body, self.tcx); - let func_id = - if let ty::FnDef(func_id, _) = func_ty.kind() { Some(func_id) } else { None }; - let sig = func_ty.fn_sig(self.tcx); - if let hir::Unsafety::Unsafe = sig.unsafety() { - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::CallToUnsafeFunction, - ) - } - - if let Some(func_id) = func_id { - self.check_target_features(*func_id); - } - } - - TerminatorKind::InlineAsm { .. } => self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::UseOfInlineAssembly, - ), - } - self.super_terminator(terminator, location); - } - - fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { - self.source_info = statement.source_info; - match statement.kind { - StatementKind::Assign(..) - | StatementKind::FakeRead(..) - | StatementKind::SetDiscriminant { .. } - | StatementKind::Deinit(..) - | StatementKind::StorageLive(..) - | StatementKind::StorageDead(..) - | StatementKind::Retag { .. } - | StatementKind::PlaceMention(..) - | StatementKind::Coverage(..) - | StatementKind::Intrinsic(..) - | StatementKind::ConstEvalCounter - | StatementKind::Nop => { - // safe (at least as emitted during MIR construction) - } - // `AscribeUserType` just exists to help MIR borrowck. - // It has no semantics, and everything is already reported by `PlaceMention`. - StatementKind::AscribeUserType(..) => return, - } - self.super_statement(statement, location); - } - - fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { - match rvalue { - Rvalue::Aggregate(box ref aggregate, _) => match aggregate { - &AggregateKind::Array(..) | &AggregateKind::Tuple => {} - &AggregateKind::Adt(adt_did, ..) => { - match self.tcx.layout_scalar_valid_range(adt_did) { - (Bound::Unbounded, Bound::Unbounded) => {} - _ => self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::InitializingTypeWith, - ), - } - } - &AggregateKind::Closure(def_id, _) - | &AggregateKind::CoroutineClosure(def_id, _) - | &AggregateKind::Coroutine(def_id, _) => { - let def_id = def_id.expect_local(); - let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } = - self.tcx.mir_unsafety_check_result(def_id); - self.register_violations(violations, used_unsafe_blocks.items().copied()); - } - }, - _ => {} - } - self.super_rvalue(rvalue, location); - } - - fn visit_operand(&mut self, op: &Operand<'tcx>, location: Location) { - if let Operand::Constant(constant) = op { - let maybe_uneval = match constant.const_ { - Const::Val(..) | Const::Ty(_) => None, - Const::Unevaluated(uv, _) => Some(uv), - }; - - if let Some(uv) = maybe_uneval { - if uv.promoted.is_none() { - let def_id = uv.def; - if self.tcx.def_kind(def_id) == DefKind::InlineConst { - let local_def_id = def_id.expect_local(); - let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } = - self.tcx.mir_unsafety_check_result(local_def_id); - self.register_violations(violations, used_unsafe_blocks.items().copied()); - } - } - } - } - self.super_operand(op, location); - } - - fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) { - // On types with `scalar_valid_range`, prevent - // * `&mut x.field` - // * `x.field = y;` - // * `&x.field` if `field`'s type has interior mutability - // because either of these would allow modifying the layout constrained field and - // insert values that violate the layout constraints. - if context.is_mutating_use() || context.is_borrow() { - self.check_mut_borrowing_layout_constrained_field(*place, context.is_mutating_use()); - } - - // Some checks below need the extra meta info of the local declaration. - let decl = &self.body.local_decls[place.local]; - - // Check the base local: it might be an unsafe-to-access static. We only check derefs of the - // temporary holding the static pointer to avoid duplicate errors - // <https://github.com/rust-lang/rust/pull/78068#issuecomment-731753506>. - if place.projection.first() == Some(&ProjectionElem::Deref) { - // If the projection root is an artificial local that we introduced when - // desugaring `static`, give a more specific error message - // (avoid the general "raw pointer" clause below, that would only be confusing). - if let LocalInfo::StaticRef { def_id, .. } = *decl.local_info() { - if self.tcx.is_mutable_static(def_id) { - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::UseOfMutableStatic, - ); - return; - } else if self.tcx.is_foreign_item(def_id) { - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::UseOfExternStatic, - ); - return; - } - } - } - - // Check for raw pointer `Deref`. - for (base, proj) in place.iter_projections() { - if proj == ProjectionElem::Deref { - let base_ty = base.ty(self.body, self.tcx).ty; - if base_ty.is_unsafe_ptr() { - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::DerefOfRawPointer, - ) - } - } - } - - // Check for union fields. For this we traverse right-to-left, as the last `Deref` changes - // whether we *read* the union field or potentially *write* to it (if this place is being assigned to). - let mut saw_deref = false; - for (base, proj) in place.iter_projections().rev() { - if proj == ProjectionElem::Deref { - saw_deref = true; - continue; - } - - let base_ty = base.ty(self.body, self.tcx).ty; - if base_ty.is_union() { - // If we did not hit a `Deref` yet and the overall place use is an assignment, the - // rules are different. - let assign_to_field = !saw_deref - && matches!( - context, - PlaceContext::MutatingUse( - MutatingUseContext::Store - | MutatingUseContext::Drop - | MutatingUseContext::AsmOutput - ) - ); - // If this is just an assignment, determine if the assigned type needs dropping. - if assign_to_field { - // We have to check the actual type of the assignment, as that determines if the - // old value is being dropped. - let assigned_ty = place.ty(&self.body.local_decls, self.tcx).ty; - if assigned_ty.needs_drop(self.tcx, self.param_env) { - // This would be unsafe, but should be outright impossible since we reject - // such unions. - assert!( - self.tcx.dcx().has_errors().is_some(), - "union fields that need dropping should be impossible: {assigned_ty}" - ); - } - } else { - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::AccessToUnionField, - ) - } - } - } - } -} - -impl<'tcx> UnsafetyChecker<'_, 'tcx> { - fn require_unsafe(&mut self, kind: UnsafetyViolationKind, details: UnsafetyViolationDetails) { - // Violations can turn out to be `UnsafeFn` during analysis, but they should not start out as such. - assert_ne!(kind, UnsafetyViolationKind::UnsafeFn); - - let source_info = self.source_info; - let lint_root = self.body.source_scopes[self.source_info.scope] - .local_data - .as_ref() - .assert_crate_local() - .lint_root; - self.register_violations( - [&UnsafetyViolation { source_info, lint_root, kind, details }], - UnordItems::empty(), - ); - } - - fn register_violations<'a>( - &mut self, - violations: impl IntoIterator<Item = &'a UnsafetyViolation>, - new_used_unsafe_blocks: UnordItems<HirId, impl Iterator<Item = HirId>>, - ) { - let safety = self.body.source_scopes[self.source_info.scope] - .local_data - .as_ref() - .assert_crate_local() - .safety; - match safety { - // `unsafe` blocks are required in safe code - Safety::Safe => violations.into_iter().for_each(|violation| { - match violation.kind { - UnsafetyViolationKind::General => {} - UnsafetyViolationKind::UnsafeFn => { - bug!("`UnsafetyViolationKind::UnsafeFn` in an `Safe` context") - } - } - if !self.violations.contains(violation) { - self.violations.push(violation.clone()) - } - }), - // With the RFC 2585, no longer allow `unsafe` operations in `unsafe fn`s - Safety::FnUnsafe => violations.into_iter().for_each(|violation| { - let mut violation = violation.clone(); - violation.kind = UnsafetyViolationKind::UnsafeFn; - if !self.violations.contains(&violation) { - self.violations.push(violation) - } - }), - Safety::BuiltinUnsafe => {} - Safety::ExplicitUnsafe(hir_id) => violations.into_iter().for_each(|_violation| { - self.used_unsafe_blocks.insert(hir_id); - }), - }; - - self.used_unsafe_blocks.extend_unord(new_used_unsafe_blocks); - } - fn check_mut_borrowing_layout_constrained_field( - &mut self, - place: Place<'tcx>, - is_mut_use: bool, - ) { - for (place_base, elem) in place.iter_projections().rev() { - match elem { - // Modifications behind a dereference don't affect the value of - // the pointer. - ProjectionElem::Deref => return, - ProjectionElem::Field(..) => { - let ty = place_base.ty(&self.body.local_decls, self.tcx).ty; - if let ty::Adt(def, _) = ty.kind() { - if self.tcx.layout_scalar_valid_range(def.did()) - != (Bound::Unbounded, Bound::Unbounded) - { - let details = if is_mut_use { - UnsafetyViolationDetails::MutationOfLayoutConstrainedField - - // Check `is_freeze` as late as possible to avoid cycle errors - // with opaque types. - } else if !place - .ty(self.body, self.tcx) - .ty - .is_freeze(self.tcx, self.param_env) - { - UnsafetyViolationDetails::BorrowOfLayoutConstrainedField - } else { - continue; - }; - self.require_unsafe(UnsafetyViolationKind::General, details); - } - } - } - _ => {} - } - } - } - - /// Checks whether calling `func_did` needs an `unsafe` context or not, i.e. whether - /// the called function has target features the calling function hasn't. - fn check_target_features(&mut self, func_did: DefId) { - // Unsafety isn't required on wasm targets. For more information see - // the corresponding check in typeck/src/collect.rs - if self.tcx.sess.target.options.is_like_wasm { - return; - } - - let callee_features = &self.tcx.codegen_fn_attrs(func_did).target_features; - // The body might be a constant, so it doesn't have codegen attributes. - let self_features = &self.tcx.body_codegen_attrs(self.body_did.to_def_id()).target_features; - - // Is `callee_features` a subset of `calling_features`? - if !callee_features.iter().all(|feature| self_features.contains(feature)) { - let missing: Vec<_> = callee_features - .iter() - .copied() - .filter(|feature| !self_features.contains(feature)) - .collect(); - let build_enabled = self - .tcx - .sess - .target_features - .iter() - .copied() - .filter(|feature| missing.contains(feature)) - .collect(); - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::CallToFunctionWith { missing, build_enabled }, - ) - } - } -} - -pub(crate) fn provide(providers: &mut Providers) { - *providers = Providers { mir_unsafety_check_result, ..*providers }; -} - -/// Context information for [`UnusedUnsafeVisitor`] traversal, -/// saves (innermost) relevant context -#[derive(Copy, Clone, Debug)] -enum Context { - Safe, - /// in an `unsafe fn` - UnsafeFn, - /// in a *used* `unsafe` block - /// (i.e. a block without unused-unsafe warning) - UnsafeBlock(HirId), -} - -struct UnusedUnsafeVisitor<'a, 'tcx> { - tcx: TyCtxt<'tcx>, - used_unsafe_blocks: &'a UnordSet<HirId>, - context: Context, - unused_unsafes: &'a mut Vec<(HirId, UnusedUnsafe)>, -} - -impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> { - fn visit_block(&mut self, block: &'tcx hir::Block<'tcx>) { - if let hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::UserProvided) = block.rules { - let used = match self.tcx.lint_level_at_node(UNUSED_UNSAFE, block.hir_id) { - (Level::Allow, _) => true, - _ => self.used_unsafe_blocks.contains(&block.hir_id), - }; - let unused_unsafe = match (self.context, used) { - (_, false) => UnusedUnsafe::Unused, - (Context::Safe, true) | (Context::UnsafeFn, true) => { - let previous_context = self.context; - self.context = Context::UnsafeBlock(block.hir_id); - intravisit::walk_block(self, block); - self.context = previous_context; - return; - } - (Context::UnsafeBlock(hir_id), true) => UnusedUnsafe::InUnsafeBlock(hir_id), - }; - self.unused_unsafes.push((block.hir_id, unused_unsafe)); - } - intravisit::walk_block(self, block); - } - - fn visit_inline_const(&mut self, c: &'tcx hir::ConstBlock) { - self.visit_body(self.tcx.hir().body(c.body)) - } - - fn visit_fn( - &mut self, - fk: intravisit::FnKind<'tcx>, - _fd: &'tcx hir::FnDecl<'tcx>, - b: hir::BodyId, - _s: rustc_span::Span, - _id: LocalDefId, - ) { - if matches!(fk, intravisit::FnKind::Closure) { - self.visit_body(self.tcx.hir().body(b)) - } - } -} - -fn check_unused_unsafe( - tcx: TyCtxt<'_>, - def_id: LocalDefId, - used_unsafe_blocks: &UnordSet<HirId>, -) -> Vec<(HirId, UnusedUnsafe)> { - let body_id = tcx.hir().maybe_body_owned_by(def_id); - - let Some(body_id) = body_id else { - debug!("check_unused_unsafe({:?}) - no body found", def_id); - return vec![]; - }; - - let body = tcx.hir().body(body_id); - let hir_id = tcx.local_def_id_to_hir_id(def_id); - let context = match tcx.hir().fn_sig_by_hir_id(hir_id) { - Some(sig) if sig.header.unsafety == hir::Unsafety::Unsafe => Context::UnsafeFn, - _ => Context::Safe, - }; - - debug!( - "check_unused_unsafe({:?}, context={:?}, body={:?}, used_unsafe_blocks={:?})", - def_id, body, context, used_unsafe_blocks - ); - - let mut unused_unsafes = vec![]; - - let mut visitor = UnusedUnsafeVisitor { - tcx, - used_unsafe_blocks, - context, - unused_unsafes: &mut unused_unsafes, - }; - intravisit::Visitor::visit_body(&mut visitor, body); - - unused_unsafes -} - -fn mir_unsafety_check_result(tcx: TyCtxt<'_>, def: LocalDefId) -> &UnsafetyCheckResult { - debug!("unsafety_violations({:?})", def); - - // N.B., this borrow is valid because all the consumers of - // `mir_built` force this. - let body = &tcx.mir_built(def).borrow(); - - if body.is_custom_mir() || body.tainted_by_errors.is_some() { - return tcx.arena.alloc(UnsafetyCheckResult { - violations: Vec::new(), - used_unsafe_blocks: Default::default(), - unused_unsafes: Some(Vec::new()), - }); - } - - let param_env = tcx.param_env(def); - - let mut checker = UnsafetyChecker::new(body, def, tcx, param_env); - checker.visit_body(body); - - let unused_unsafes = (!tcx.is_typeck_child(def.to_def_id())) - .then(|| check_unused_unsafe(tcx, def, &checker.used_unsafe_blocks)); - - tcx.arena.alloc(UnsafetyCheckResult { - violations: checker.violations, - used_unsafe_blocks: checker.used_unsafe_blocks, - unused_unsafes, - }) -} - -fn report_unused_unsafe(tcx: TyCtxt<'_>, kind: UnusedUnsafe, id: HirId) { - let span = tcx.sess.source_map().guess_head_span(tcx.hir().span(id)); - let nested_parent = if let UnusedUnsafe::InUnsafeBlock(id) = kind { - Some(tcx.sess.source_map().guess_head_span(tcx.hir().span(id))) - } else { - None - }; - tcx.emit_node_span_lint(UNUSED_UNSAFE, id, span, errors::UnusedUnsafe { span, nested_parent }); -} - -pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) { - debug!("check_unsafety({:?})", def_id); - - // closures and inline consts are handled by their parent fn. - if tcx.is_typeck_child(def_id.to_def_id()) { - return; - } - - let UnsafetyCheckResult { violations, unused_unsafes, .. } = - tcx.mir_unsafety_check_result(def_id); - // Only suggest wrapping the entire function body in an unsafe block once - let mut suggest_unsafe_block = true; - - for &UnsafetyViolation { source_info, lint_root, kind, ref details } in violations.iter() { - let details = - errors::RequiresUnsafeDetail { violation: details.clone(), span: source_info.span }; - - match kind { - UnsafetyViolationKind::General => { - let op_in_unsafe_fn_allowed = unsafe_op_in_unsafe_fn_allowed(tcx, lint_root); - let note_non_inherited = tcx.hir().parent_iter(lint_root).find(|(id, node)| { - if let Node::Expr(block) = node - && let ExprKind::Block(block, _) = block.kind - && let BlockCheckMode::UnsafeBlock(_) = block.rules - { - true - } else if let Some(sig) = tcx.hir().fn_sig_by_hir_id(*id) - && sig.header.is_unsafe() - { - true - } else { - false - } - }); - let enclosing = if let Some((id, _)) = note_non_inherited { - Some(tcx.sess.source_map().guess_head_span(tcx.hir().span(id))) - } else { - None - }; - tcx.dcx().emit_err(errors::RequiresUnsafe { - span: source_info.span, - enclosing, - details, - op_in_unsafe_fn_allowed, - }); - } - UnsafetyViolationKind::UnsafeFn => { - tcx.emit_node_span_lint( - UNSAFE_OP_IN_UNSAFE_FN, - lint_root, - source_info.span, - errors::UnsafeOpInUnsafeFn { - details, - suggest_unsafe_block: suggest_unsafe_block.then(|| { - let hir_id = tcx.local_def_id_to_hir_id(def_id); - let fn_sig = tcx - .hir() - .fn_sig_by_hir_id(hir_id) - .expect("this violation only occurs in fn"); - let body = tcx.hir().body_owned_by(def_id); - let body_span = tcx.hir().body(body).value.span; - let start = tcx.sess.source_map().start_point(body_span).shrink_to_hi(); - let end = tcx.sess.source_map().end_point(body_span).shrink_to_lo(); - (start, end, fn_sig.span) - }), - }, - ); - suggest_unsafe_block = false; - } - } - } - - for &(block_id, kind) in unused_unsafes.as_ref().unwrap() { - report_unused_unsafe(tcx, kind, block_id); - } -} - -fn unsafe_op_in_unsafe_fn_allowed(tcx: TyCtxt<'_>, id: HirId) -> bool { - tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, id).0 == Level::Allow -} diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index f0a13f66555..e2a911f0dc7 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -168,7 +168,7 @@ impl<'tcx> MutVisitor<'tcx> for PinArgVisitor<'tcx> { Place { local: SELF_ARG, projection: self.tcx().mk_place_elems(&[ProjectionElem::Field( - FieldIdx::new(0), + FieldIdx::ZERO, self.ref_coroutine_ty, )]), }, @@ -267,7 +267,7 @@ impl<'tcx> TransformVisitor<'tcx> { Rvalue::Aggregate( Box::new(AggregateKind::Adt( option_def_id, - VariantIdx::from_usize(0), + VariantIdx::ZERO, self.tcx.mk_args(&[self.old_yield_ty.into()]), None, None, @@ -329,7 +329,7 @@ impl<'tcx> TransformVisitor<'tcx> { Rvalue::Aggregate( Box::new(AggregateKind::Adt( poll_def_id, - VariantIdx::from_usize(0), + VariantIdx::ZERO, args, None, None, @@ -358,7 +358,7 @@ impl<'tcx> TransformVisitor<'tcx> { Rvalue::Aggregate( Box::new(AggregateKind::Adt( option_def_id, - VariantIdx::from_usize(0), + VariantIdx::ZERO, args, None, None, @@ -420,7 +420,7 @@ impl<'tcx> TransformVisitor<'tcx> { Rvalue::Aggregate( Box::new(AggregateKind::Adt( coroutine_state_def_id, - VariantIdx::from_usize(0), + VariantIdx::ZERO, args, None, None, @@ -1226,7 +1226,7 @@ fn create_coroutine_drop_shim<'tcx>( tcx: TyCtxt<'tcx>, transform: &TransformVisitor<'tcx>, coroutine_ty: Ty<'tcx>, - body: &mut Body<'tcx>, + body: &Body<'tcx>, drop_clean: BasicBlock, ) -> Body<'tcx> { let mut body = body.clone(); diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index e0bbd582d88..de43f9faff9 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -1,9 +1,68 @@ -//! A MIR pass which duplicates a coroutine's body and removes any derefs which -//! would be present for upvars that are taken by-ref. The result of which will -//! be a coroutine body that takes all of its upvars by-move, and which we stash -//! into the `CoroutineInfo` for all coroutines returned by coroutine-closures. +//! This pass constructs a second coroutine body sufficient for return from +//! `FnOnce`/`AsyncFnOnce` implementations for coroutine-closures (e.g. async closures). +//! +//! Consider an async closure like: +//! ```rust +//! #![feature(async_closure)] +//! +//! let x = vec![1, 2, 3]; +//! +//! let closure = async move || { +//! println!("{x:#?}"); +//! }; +//! ``` +//! +//! This desugars to something like: +//! ```rust,ignore (invalid-borrowck) +//! let x = vec![1, 2, 3]; +//! +//! let closure = move || { +//! async { +//! println!("{x:#?}"); +//! } +//! }; +//! ``` +//! +//! Important to note here is that while the outer closure *moves* `x: Vec<i32>` +//! into its upvars, the inner `async` coroutine simply captures a ref of `x`. +//! This is the "magic" of async closures -- the futures that they return are +//! allowed to borrow from their parent closure's upvars. +//! +//! However, what happens when we call `closure` with `AsyncFnOnce` (or `FnOnce`, +//! since all async closures implement that too)? Well, recall the signature: +//! ``` +//! use std::future::Future; +//! pub trait AsyncFnOnce<Args> +//! { +//! type CallOnceFuture: Future<Output = Self::Output>; +//! type Output; +//! fn async_call_once( +//! self, +//! args: Args +//! ) -> Self::CallOnceFuture; +//! } +//! ``` +//! +//! This signature *consumes* the async closure (`self`) and returns a `CallOnceFuture`. +//! How do we deal with the fact that the coroutine is supposed to take a reference +//! to the captured `x` from the parent closure, when that parent closure has been +//! destroyed? +//! +//! This is the second piece of magic of async closures. We can simply create a +//! *second* `async` coroutine body where that `x` that was previously captured +//! by reference is now captured by value. This means that we consume the outer +//! closure and return a new coroutine that will hold onto all of these captures, +//! and drop them when it is finished (i.e. after it has been `.await`ed). +//! +//! We do this with the analysis below, which detects the captures that come from +//! borrowing from the outer closure, and we simply peel off a `deref` projection +//! from them. This second body is stored alongside the first body, and optimized +//! with it in lockstep. When we need to resolve a body for `FnOnce` or `AsyncFnOnce`, +//! we use this "by move" body instead. -use rustc_data_structures::fx::FxIndexSet; +use itertools::Itertools; + +use rustc_data_structures::unord::UnordSet; use rustc_hir as hir; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, dump_mir, MirPass}; @@ -14,6 +73,8 @@ pub struct ByMoveBody; impl<'tcx> MirPass<'tcx> for ByMoveBody { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { + // We only need to generate by-move coroutine bodies for coroutines that come + // from coroutine-closures. let Some(coroutine_def_id) = body.source.def_id().as_local() else { return; }; @@ -22,44 +83,78 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { else { return; }; + + // Also, let's skip processing any bodies with errors, since there's no guarantee + // the MIR body will be constructed well. let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; if coroutine_ty.references_error() { return; } - let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") }; - let coroutine_kind = args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(); - if coroutine_kind == ty::ClosureKind::FnOnce { + // We don't need to generate a by-move coroutine if the coroutine body was + // produced by the `CoroutineKindShim`, since it's already by-move. + if matches!(body.source.instance, ty::InstanceDef::CoroutineKindShim { .. }) { return; } - let mut by_ref_fields = FxIndexSet::default(); - let by_move_upvars = Ty::new_tup_from_iter( - tcx, - tcx.closure_captures(coroutine_def_id).iter().enumerate().map(|(idx, capture)| { - if capture.is_by_ref() { - by_ref_fields.insert(FieldIdx::from_usize(idx)); - } - capture.place.ty() - }), - ); - let by_move_coroutine_ty = Ty::new_coroutine( - tcx, - coroutine_def_id.to_def_id(), - ty::CoroutineArgs::new( + let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") }; + let args = args.as_coroutine(); + + let coroutine_kind = args.kind_ty().to_opt_closure_kind().unwrap(); + + let parent_def_id = tcx.local_parent(coroutine_def_id); + let ty::CoroutineClosure(_, parent_args) = + *tcx.type_of(parent_def_id).instantiate_identity().kind() + else { + bug!(); + }; + let parent_closure_args = parent_args.as_coroutine_closure(); + let num_args = parent_closure_args + .coroutine_closure_sig() + .skip_binder() + .tupled_inputs_ty + .tuple_fields() + .len(); + + let mut by_ref_fields = UnordSet::default(); + for (idx, (coroutine_capture, parent_capture)) in tcx + .closure_captures(coroutine_def_id) + .iter() + // By construction we capture all the args first. + .skip(num_args) + .zip_eq(tcx.closure_captures(parent_def_id)) + .enumerate() + { + // This upvar is captured by-move from the parent closure, but by-ref + // from the inner async block. That means that it's being borrowed from + // the outer closure body -- we need to change the coroutine to take the + // upvar by value. + if coroutine_capture.is_by_ref() && !parent_capture.is_by_ref() { + assert_ne!( + coroutine_kind, + ty::ClosureKind::FnOnce, + "`FnOnce` coroutine-closures return coroutines that capture from \ + their body; it will always result in a borrowck error!" + ); + by_ref_fields.insert(FieldIdx::from_usize(num_args + idx)); + } + + // Make sure we're actually talking about the same capture. + // FIXME(async_closures): We could look at the `hir::Upvar` instead? + assert_eq!(coroutine_capture.place.ty(), parent_capture.place.ty()); + } + + let by_move_coroutine_ty = tcx + .instantiate_bound_regions_with_erased(parent_closure_args.coroutine_closure_sig()) + .to_coroutine_given_kind_and_upvars( tcx, - ty::CoroutineArgsParts { - parent_args: args.as_coroutine().parent_args(), - kind_ty: Ty::from_closure_kind(tcx, ty::ClosureKind::FnOnce), - resume_ty: args.as_coroutine().resume_ty(), - yield_ty: args.as_coroutine().yield_ty(), - return_ty: args.as_coroutine().return_ty(), - witness: args.as_coroutine().witness(), - tupled_upvars_ty: by_move_upvars, - }, - ) - .args, - ); + parent_closure_args.parent_args(), + coroutine_def_id.to_def_id(), + ty::ClosureKind::FnOnce, + tcx.lifetimes.re_erased, + parent_closure_args.tupled_upvars_ty(), + parent_closure_args.coroutine_captures_by_ref_ty(), + ); let mut by_move_body = body.clone(); MakeByMoveBody { tcx, by_ref_fields, by_move_coroutine_ty }.visit_body(&mut by_move_body); @@ -73,7 +168,7 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - by_ref_fields: FxIndexSet<FieldIdx>, + by_ref_fields: UnordSet<FieldIdx>, by_move_coroutine_ty: Ty<'tcx>, } @@ -89,11 +184,11 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { location: mir::Location, ) { if place.local == ty::CAPTURE_STRUCT_LOCAL - && !place.projection.is_empty() - && let mir::ProjectionElem::Field(idx, ty) = place.projection[0] + && let Some((&mir::ProjectionElem::Field(idx, ty), projection)) = + place.projection.split_first() && self.by_ref_fields.contains(&idx) { - let (begin, end) = place.projection[1..].split_first().unwrap(); + let (begin, end) = projection.split_first().unwrap(); // FIXME(async_closures): I'm actually a bit surprised to see that we always // initially deref the by-ref upvars. If this is not actually true, then we // will at least get an ICE that explains why this isn't true :^) diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index ae3b1a3d1af..d382d2c03c2 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -123,8 +123,11 @@ fn create_mappings<'tcx>( let body_span = hir_info.body_span; let source_file = source_map.lookup_source_file(body_span.lo()); - use rustc_session::RemapFileNameExt; - let file_name = Symbol::intern(&source_file.name.for_codegen(tcx.sess).to_string_lossy()); + + use rustc_session::{config::RemapPathScopeComponents, RemapFileNameExt}; + let file_name = Symbol::intern( + &source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy(), + ); let term_for_bcb = |bcb| { coverage_counters diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index b5dd9dcc7b4..65715253647 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -59,7 +59,7 @@ fn coverage_ids_info<'tcx>( _ => None, }) .max() - .unwrap_or(CounterId::START); + .unwrap_or(CounterId::ZERO); CoverageIdsInfo { max_counter_id } } diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs index 96943435bab..318674f24e7 100644 --- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs +++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs @@ -3,7 +3,6 @@ //! Box is not actually a pointer so it is incorrect to dereference it directly. use rustc_hir::def_id::DefId; -use rustc_index::Idx; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::*; @@ -32,9 +31,9 @@ pub fn build_projection<'tcx>( ptr_ty: Ty<'tcx>, ) -> [PlaceElem<'tcx>; 3] { [ - PlaceElem::Field(FieldIdx::new(0), unique_ty), - PlaceElem::Field(FieldIdx::new(0), nonnull_ty), - PlaceElem::Field(FieldIdx::new(0), ptr_ty), + PlaceElem::Field(FieldIdx::ZERO, unique_ty), + PlaceElem::Field(FieldIdx::ZERO, nonnull_ty), + PlaceElem::Field(FieldIdx::ZERO, ptr_ty), ] } @@ -91,15 +90,14 @@ pub struct ElaborateBoxDerefs; impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { if let Some(def_id) = tcx.lang_items().owned_box() { - let unique_did = - tcx.adt_def(def_id).non_enum_variant().fields[FieldIdx::from_u32(0)].did; + let unique_did = tcx.adt_def(def_id).non_enum_variant().fields[FieldIdx::ZERO].did; let Some(nonnull_def) = tcx.type_of(unique_did).instantiate_identity().ty_adt_def() else { span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique") }; - let nonnull_did = nonnull_def.non_enum_variant().fields[FieldIdx::from_u32(0)].did; + let nonnull_did = nonnull_def.non_enum_variant().fields[FieldIdx::ZERO].did; let patch = MirPatch::new(body); diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 9297bc51fad..0634e321ea3 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -1,11 +1,6 @@ -use std::borrow::Cow; - -use rustc_errors::{ - codes::*, Applicability, Diag, DiagArgValue, DiagCtxt, DiagMessage, Diagnostic, - EmissionGuarantee, Level, LintDiagnostic, -}; +use rustc_errors::{codes::*, Diag, DiagMessage, LintDiagnostic}; use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; -use rustc_middle::mir::{AssertKind, UnsafetyViolationDetails}; +use rustc_middle::mir::AssertKind; use rustc_middle::ty::TyCtxt; use rustc_session::lint::{self, Lint}; use rustc_span::def_id::DefId; @@ -42,168 +37,6 @@ pub(crate) struct UnalignedPackedRef { pub span: Span, } -#[derive(LintDiagnostic)] -#[diag(mir_transform_unused_unsafe)] -pub(crate) struct UnusedUnsafe { - #[label(mir_transform_unused_unsafe)] - pub span: Span, - #[label] - pub nested_parent: Option<Span>, -} - -pub(crate) struct RequiresUnsafe { - pub span: Span, - pub details: RequiresUnsafeDetail, - pub enclosing: Option<Span>, - pub op_in_unsafe_fn_allowed: bool, -} - -// The primary message for this diagnostic should be '{$label} is unsafe and...', -// so we need to eagerly translate the label here, which isn't supported by the derive API -// We could also exhaustively list out the primary messages for all unsafe violations, -// but this would result in a lot of duplication. -impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for RequiresUnsafe { - #[track_caller] - fn into_diag(self, dcx: &'a DiagCtxt, level: Level) -> Diag<'a, G> { - let mut diag = Diag::new(dcx, level, fluent::mir_transform_requires_unsafe); - diag.code(E0133); - diag.span(self.span); - diag.span_label(self.span, self.details.label()); - let desc = dcx.eagerly_translate_to_string(self.details.label(), [].into_iter()); - diag.arg("details", desc); - diag.arg("op_in_unsafe_fn_allowed", self.op_in_unsafe_fn_allowed); - self.details.add_subdiagnostics(&mut diag); - if let Some(sp) = self.enclosing { - diag.span_label(sp, fluent::mir_transform_not_inherited); - } - diag - } -} - -#[derive(Clone)] -pub(crate) struct RequiresUnsafeDetail { - pub span: Span, - pub violation: UnsafetyViolationDetails, -} - -impl RequiresUnsafeDetail { - // FIXME: make this translatable - #[allow(rustc::diagnostic_outside_of_impl)] - #[allow(rustc::untranslatable_diagnostic)] - fn add_subdiagnostics<G: EmissionGuarantee>(&self, diag: &mut Diag<'_, G>) { - use UnsafetyViolationDetails::*; - match self.violation { - CallToUnsafeFunction => { - diag.note(fluent::mir_transform_call_to_unsafe_note); - } - UseOfInlineAssembly => { - diag.note(fluent::mir_transform_use_of_asm_note); - } - InitializingTypeWith => { - diag.note(fluent::mir_transform_initializing_valid_range_note); - } - CastOfPointerToInt => { - diag.note(fluent::mir_transform_const_ptr2int_note); - } - UseOfMutableStatic => { - diag.note(fluent::mir_transform_use_of_static_mut_note); - } - UseOfExternStatic => { - diag.note(fluent::mir_transform_use_of_extern_static_note); - } - DerefOfRawPointer => { - diag.note(fluent::mir_transform_deref_ptr_note); - } - AccessToUnionField => { - diag.note(fluent::mir_transform_union_access_note); - } - MutationOfLayoutConstrainedField => { - diag.note(fluent::mir_transform_mutation_layout_constrained_note); - } - BorrowOfLayoutConstrainedField => { - diag.note(fluent::mir_transform_mutation_layout_constrained_borrow_note); - } - CallToFunctionWith { ref missing, ref build_enabled } => { - diag.help(fluent::mir_transform_target_feature_call_help); - diag.arg( - "missing_target_features", - DiagArgValue::StrListSepByAnd( - missing.iter().map(|feature| Cow::from(feature.to_string())).collect(), - ), - ); - diag.arg("missing_target_features_count", missing.len()); - if !build_enabled.is_empty() { - diag.note(fluent::mir_transform_target_feature_call_note); - diag.arg( - "build_target_features", - DiagArgValue::StrListSepByAnd( - build_enabled - .iter() - .map(|feature| Cow::from(feature.to_string())) - .collect(), - ), - ); - diag.arg("build_target_features_count", build_enabled.len()); - } - } - } - } - - fn label(&self) -> DiagMessage { - use UnsafetyViolationDetails::*; - match self.violation { - CallToUnsafeFunction => fluent::mir_transform_call_to_unsafe_label, - UseOfInlineAssembly => fluent::mir_transform_use_of_asm_label, - InitializingTypeWith => fluent::mir_transform_initializing_valid_range_label, - CastOfPointerToInt => fluent::mir_transform_const_ptr2int_label, - UseOfMutableStatic => fluent::mir_transform_use_of_static_mut_label, - UseOfExternStatic => fluent::mir_transform_use_of_extern_static_label, - DerefOfRawPointer => fluent::mir_transform_deref_ptr_label, - AccessToUnionField => fluent::mir_transform_union_access_label, - MutationOfLayoutConstrainedField => { - fluent::mir_transform_mutation_layout_constrained_label - } - BorrowOfLayoutConstrainedField => { - fluent::mir_transform_mutation_layout_constrained_borrow_label - } - CallToFunctionWith { .. } => fluent::mir_transform_target_feature_call_label, - } - } -} - -pub(crate) struct UnsafeOpInUnsafeFn { - pub details: RequiresUnsafeDetail, - - /// These spans point to: - /// 1. the start of the function body - /// 2. the end of the function body - /// 3. the function signature - pub suggest_unsafe_block: Option<(Span, Span, Span)>, -} - -impl<'a> LintDiagnostic<'a, ()> for UnsafeOpInUnsafeFn { - #[track_caller] - fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) { - let desc = diag.dcx.eagerly_translate_to_string(self.details.label(), [].into_iter()); - diag.arg("details", desc); - diag.span_label(self.details.span, self.details.label()); - self.details.add_subdiagnostics(diag); - - if let Some((start, end, fn_sig)) = self.suggest_unsafe_block { - diag.span_note(fn_sig, fluent::mir_transform_note); - diag.tool_only_multipart_suggestion( - fluent::mir_transform_suggestion, - vec![(start, " unsafe {".into()), (end, "}".into())], - Applicability::MaybeIncorrect, - ); - } - } - - fn msg(&self) -> DiagMessage { - fluent::mir_transform_unsafe_op_in_unsafe_fn - } -} - pub(crate) struct AssertLint<P> { pub span: Span, pub assert_kind: AssertKind<P>, diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 59d6d89cf1f..d4f736d2a50 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -355,7 +355,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } fn insert_tuple(&mut self, values: Vec<VnIndex>) -> VnIndex { - self.insert(Value::Aggregate(AggregateTy::Tuple, VariantIdx::from_u32(0), values)) + self.insert(Value::Aggregate(AggregateTy::Tuple, VariantIdx::ZERO, values)) } #[instrument(level = "trace", skip(self), ret)] diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 5f74841151c..60513a674af 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -324,7 +324,7 @@ impl<'tcx> Inliner<'tcx> { // do not need to catch this here, we can wait until the inliner decides to continue // inlining a second time. InstanceDef::VTableShim(_) - | InstanceDef::ReifyShim(_) + | InstanceDef::ReifyShim(..) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } | InstanceDef::ConstructCoroutineInClosureShim { .. } @@ -1077,7 +1077,7 @@ fn try_instance_mir<'tcx>( let fields = def.all_fields(); for field in fields { let field_ty = field.ty(tcx, args); - if field_ty.has_param() && field_ty.has_projections() { + if field_ty.has_param() && field_ty.has_aliases() { return Err("cannot build drop shim for polymorphic type"); } } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index f2b6dcac586..99c7b616f1b 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -5,6 +5,7 @@ use rustc_middle::mir::TerminatorKind; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, GenericArgsRef, InstanceDef, TyCtxt}; use rustc_session::Limit; +use rustc_span::sym; // FIXME: check whether it is cheaper to precompute the entire call graph instead of invoking // this query ridiculously often. @@ -84,7 +85,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( // again, a function item can end up getting inlined. Thus we'll be able to cause // a cycle that way InstanceDef::VTableShim(_) - | InstanceDef::ReifyShim(_) + | InstanceDef::ReifyShim(..) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } | InstanceDef::ConstructCoroutineInClosureShim { .. } @@ -164,11 +165,20 @@ pub(crate) fn mir_inliner_callees<'tcx>( let mut calls = FxIndexSet::default(); for bb_data in body.basic_blocks.iter() { let terminator = bb_data.terminator(); - if let TerminatorKind::Call { func, .. } = &terminator.kind { + if let TerminatorKind::Call { func, args: call_args, .. } = &terminator.kind { let ty = func.ty(&body.local_decls, tcx); - let call = match ty.kind() { - ty::FnDef(def_id, args) => (*def_id, *args), - _ => continue, + let ty::FnDef(def_id, generic_args) = ty.kind() else { + continue; + }; + let call = if tcx.is_intrinsic(*def_id, sym::const_eval_select) { + let func = &call_args[2].node; + let ty = func.ty(&body.local_decls, tcx); + let ty::FnDef(def_id, generic_args) = ty.kind() else { + continue; + }; + (*def_id, *generic_args) + } else { + (*def_id, *generic_args) }; calls.insert(call); } diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index a20958e74df..2218154ea5e 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -13,7 +13,7 @@ use rustc_const_eval::interpret::{ use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_hir::HirId; -use rustc_index::{bit_set::BitSet, Idx, IndexVec}; +use rustc_index::{bit_set::BitSet, IndexVec}; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; @@ -124,10 +124,8 @@ impl<'tcx> Value<'tcx> { fields.ensure_contains_elem(*idx, || Value::Uninit) } (PlaceElem::Field(..), val @ Value::Uninit) => { - *val = Value::Aggregate { - variant: VariantIdx::new(0), - fields: Default::default(), - }; + *val = + Value::Aggregate { variant: VariantIdx::ZERO, fields: Default::default() }; val.project_mut(&[*proj])? } _ => return None, @@ -572,7 +570,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { self.use_ecx(|this| this.ecx.overflowing_binary_op(bin_op, &left, &right))?; let overflowed = ImmTy::from_bool(overflowed, self.tcx); Value::Aggregate { - variant: VariantIdx::new(0), + variant: VariantIdx::ZERO, fields: [Value::from(val), overflowed.into()].into_iter().collect(), } } @@ -607,7 +605,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { | AggregateKind::Tuple | AggregateKind::Closure(_, _) | AggregateKind::Coroutine(_, _) - | AggregateKind::CoroutineClosure(_, _) => VariantIdx::new(0), + | AggregateKind::CoroutineClosure(_, _) => VariantIdx::ZERO, }, } } diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 15988c0ea6b..e477c068229 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -53,7 +53,6 @@ mod add_moves_for_packed_drops; mod add_retag; mod check_const_item_mutation; mod check_packed_ref; -pub mod check_unsafety; mod remove_place_mention; // This pass is public to allow external drivers to perform MIR cleanup mod add_subtyping_projections; @@ -110,7 +109,7 @@ pub mod simplify; mod simplify_branches; mod simplify_comparison_integral; mod sroa; -mod uninhabited_enum_branching; +mod unreachable_enum_branching; mod unreachable_prop; use rustc_const_eval::transform::check_consts::{self, ConstCx}; @@ -120,7 +119,6 @@ use rustc_mir_dataflow::rustc_peek; rustc_fluent_macro::fluent_messages! { "../messages.ftl" } pub fn provide(providers: &mut Providers) { - check_unsafety::provide(providers); coverage::query::provide(providers); ffi_unwind_calls::provide(providers); shim::provide(providers); @@ -280,11 +278,6 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs { } fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> { - // MIR unsafety check uses the raw mir, so make sure it is run. - if !tcx.sess.opts.unstable_opts.thir_unsafeck { - tcx.ensure_with_value().mir_unsafety_check_result(def); - } - let mut body = tcx.build_mir(def); pass_manager::dump_mir_for_phase_change(tcx, &body); @@ -580,9 +573,10 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &remove_zsts::RemoveZsts, &remove_unneeded_drops::RemoveUnneededDrops, // Type instantiation may create uninhabited enums. - &uninhabited_enum_branching::UninhabitedEnumBranching, + // Also eliminates some unreachable branches based on variants of enums. + &unreachable_enum_branching::UnreachableEnumBranching, &unreachable_prop::UnreachablePropagation, - &o1(simplify::SimplifyCfg::AfterUninhabitedEnumBranching), + &o1(simplify::SimplifyCfg::AfterUnreachableEnumBranching), // Inlining may have introduced a lot of redundant code and a large move pattern. // Now, we need to shrink the generated MIR. diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 7d4c1b9c21a..7e8920604c1 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -90,6 +90,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul + | sym::three_way_compare | sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul @@ -109,6 +110,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { sym::wrapping_add => BinOp::Add, sym::wrapping_sub => BinOp::Sub, sym::wrapping_mul => BinOp::Mul, + sym::three_way_compare => BinOp::Cmp, sym::unchecked_add => BinOp::AddUnchecked, sym::unchecked_sub => BinOp::SubUnchecked, sym::unchecked_mul => BinOp::MulUnchecked, diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index c3a92911bbf..232c290e0fb 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -84,7 +84,7 @@ impl<'tcx> MirPass<'tcx> for RenameReturnPlace { /// /// If the MIR fulfills both these conditions, this function returns the `Local` that is assigned /// to the return place along all possible paths through the control-flow graph. -fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { +fn local_eligible_for_nrvo(body: &mir::Body<'_>) -> Option<Local> { if IsReturnPlaceRead::run(body) { return None; } @@ -118,10 +118,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { copied_to_return_place } -fn find_local_assigned_to_return_place( - start: BasicBlock, - body: &mut mir::Body<'_>, -) -> Option<Local> { +fn find_local_assigned_to_return_place(start: BasicBlock, body: &mir::Body<'_>) -> Option<Local> { let mut block = start; let mut seen = BitSet::new_empty(body.basic_blocks.len()); diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 2951897ebd6..a9d4b860b7a 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -434,7 +434,7 @@ impl<'tcx> Validator<'_, 'tcx> { Rvalue::ThreadLocalRef(_) => return Err(Unpromotable), // ptr-to-int casts are not possible in consts and thus not promotable - Rvalue::Cast(CastKind::PointerExposeAddress, _, _) => return Err(Unpromotable), + Rvalue::Cast(CastKind::PointerExposeProvenance, _, _) => return Err(Unpromotable), // all other casts including int-to-ptr casts are fine, they just use the integer value // at pointer type. @@ -525,6 +525,7 @@ impl<'tcx> Validator<'_, 'tcx> { | BinOp::Lt | BinOp::Ge | BinOp::Gt + | BinOp::Cmp | BinOp::Offset | BinOp::Add | BinOp::AddUnchecked diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index b60ee7649b2..fa6906bdd55 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -55,7 +55,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' // a virtual call, or a direct call to a function for which // indirect calls must be codegen'd differently than direct ones // (such as `#[track_caller]`). - ty::InstanceDef::ReifyShim(def_id) => { + ty::InstanceDef::ReifyShim(def_id, _) => { build_call_shim(tcx, instance, None, CallKind::Direct(def_id)) } ty::InstanceDef::ClosureOnceShim { call_once: _, track_caller: _ } => { @@ -985,7 +985,7 @@ fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'t let locals = local_decls_for_sig(&sig, span); let source_info = SourceInfo::outermost(span); - // FIXME: use `expose_addr` once we figure out whether function pointers have meaningful provenance. + // FIXME: use `expose_provenance` once we figure out whether function pointers have meaningful provenance. let rvalue = Rvalue::Cast( CastKind::FnPtrToPtr, Operand::Move(Place::from(Local::new(1))), diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 574330cc355..5bbe3bb747f 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -44,7 +44,7 @@ pub enum SimplifyCfg { PreOptimizations, Final, MakeShim, - AfterUninhabitedEnumBranching, + AfterUnreachableEnumBranching, } impl SimplifyCfg { @@ -57,8 +57,8 @@ impl SimplifyCfg { SimplifyCfg::PreOptimizations => "SimplifyCfg-pre-optimizations", SimplifyCfg::Final => "SimplifyCfg-final", SimplifyCfg::MakeShim => "SimplifyCfg-make_shim", - SimplifyCfg::AfterUninhabitedEnumBranching => { - "SimplifyCfg-after-uninhabited-enum-branching" + SimplifyCfg::AfterUnreachableEnumBranching => { + "SimplifyCfg-after-unreachable-enum-branching" } } } @@ -415,7 +415,7 @@ fn make_local_map<V>( used_locals: &UsedLocals, ) -> IndexVec<Local, Option<Local>> { let mut map: IndexVec<Local, Option<Local>> = IndexVec::from_elem(None, local_decls); - let mut used = Local::new(0); + let mut used = Local::ZERO; for alive_index in local_decls.indices() { // `is_used` treats the `RETURN_PLACE` and arguments as used. diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs index 57fe46ad75a..66b6235eb93 100644 --- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs @@ -1,4 +1,4 @@ -//! A pass that eliminates branches on uninhabited enum variants. +//! A pass that eliminates branches on uninhabited or unreachable enum variants. use crate::MirPass; use rustc_data_structures::fx::FxHashSet; @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_target::abi::{Abi, Variants}; -pub struct UninhabitedEnumBranching; +pub struct UnreachableEnumBranching; fn get_discriminant_local(terminator: &TerminatorKind<'_>) -> Option<Local> { if let TerminatorKind::SwitchInt { discr: Operand::Move(p), .. } = terminator { @@ -71,13 +71,13 @@ fn variant_discriminants<'tcx>( } } -impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { +impl<'tcx> MirPass<'tcx> for UnreachableEnumBranching { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - trace!("UninhabitedEnumBranching starting for {:?}", body.source); + trace!("UnreachableEnumBranching starting for {:?}", body.source); let mut unreachable_targets = Vec::new(); let mut patch = MirPatch::new(body); @@ -96,8 +96,10 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { ); let mut allowed_variants = if let Ok(layout) = layout { + // Find allowed variants based on uninhabited. variant_discriminants(&layout, discriminant_ty, tcx) } else if let Some(variant_range) = discriminant_ty.variant_range(tcx) { + // If there are some generics, we can still get the allowed variants. variant_range .map(|variant| { discriminant_ty.discriminant_for_variant(tcx, variant).unwrap().val @@ -121,9 +123,26 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { } let otherwise_is_empty_unreachable = body.basic_blocks[targets.otherwise()].is_empty_unreachable(); - // After resolving https://github.com/llvm/llvm-project/issues/78578, - // we can remove the limit on the number of successors. fn check_successors(basic_blocks: &BasicBlocks<'_>, bb: BasicBlock) -> bool { + // After resolving https://github.com/llvm/llvm-project/issues/78578, + // We can remove this check. + // The main issue here is that `early-tailduplication` causes compile time overhead + // and potential performance problems. + // Simply put, when encounter a switch (indirect branch) statement, + // `early-tailduplication` tries to duplicate the switch branch statement with BB + // into (each) predecessors. This makes CFG very complex. + // We can understand it as it transforms the following code + // ```rust + // match a { ... many cases }; + // match b { ... many cases }; + // ``` + // into + // ```rust + // match a { ... many match b { goto BB cases } } + // ... BB cases + // ``` + // Abandon this transformation when it is possible (the best effort) + // to encounter the problem. let mut successors = basic_blocks[bb].terminator().successors(); let Some(first_successor) = successors.next() else { return true }; if successors.next().is_some() { @@ -136,11 +155,32 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { }; true } + // If and only if there is a variant that does not have a branch set, + // change the current of otherwise as the variant branch and set otherwise to unreachable. + // It transforms following code + // ```rust + // match c { + // Ordering::Less => 1, + // Ordering::Equal => 2, + // _ => 3, + // } + // ``` + // to + // ```rust + // match c { + // Ordering::Less => 1, + // Ordering::Equal => 2, + // Ordering::Greater => 3, + // } + // ``` let otherwise_is_last_variant = !otherwise_is_empty_unreachable && allowed_variants.len() == 1 - && check_successors(&body.basic_blocks, targets.otherwise()); + // Despite the LLVM issue, we hope that small enum can still be transformed. + // This is valuable for both `a <= b` and `if let Some/Ok(v)`. + && (targets.all_targets().len() <= 3 + || check_successors(&body.basic_blocks, targets.otherwise())); let replace_otherwise_to_unreachable = otherwise_is_last_variant - || !otherwise_is_empty_unreachable && allowed_variants.is_empty(); + || (!otherwise_is_empty_unreachable && allowed_variants.is_empty()); if unreachable_targets.is_empty() && !replace_otherwise_to_unreachable { continue; @@ -150,6 +190,7 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { let mut targets = targets.clone(); if replace_otherwise_to_unreachable { if otherwise_is_last_variant { + // We have checked that `allowed_variants` has only one element. #[allow(rustc::potential_query_instability)] let last_variant = *allowed_variants.iter().next().unwrap(); targets.add_target(last_variant, targets.otherwise()); |