diff options
Diffstat (limited to 'compiler')
| -rw-r--r-- | compiler/rustc_metadata/src/rmeta/decoder.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_metadata/src/rmeta/encoder.rs | 120 | ||||
| -rw-r--r-- | compiler/rustc_metadata/src/rmeta/mod.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/query.rs | 13 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/query/mod.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/mod.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/const_eval/machine.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/interpret/eval_context.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/interpret/machine.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/monomorphize/polymorphize.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/transform/const_prop.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/transform/coverage/query.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/transform/mod.rs | 124 | ||||
| -rw-r--r-- | compiler/rustc_mir/src/util/pretty.rs | 27 | ||||
| -rw-r--r-- | compiler/rustc_trait_selection/src/traits/const_evaluatable.rs | 2 |
16 files changed, 333 insertions, 91 deletions
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 6e381fd2965..eace1747577 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -1160,6 +1160,10 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } } + fn is_ctfe_mir_available(&self, id: DefIndex) -> bool { + self.root.tables.mir_for_ctfe.get(self, id).is_some() + } + fn is_item_mir_available(&self, id: DefIndex) -> bool { self.root.tables.mir.get(self, id).is_some() } @@ -1183,6 +1187,17 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .decode((self, tcx)) } + fn get_mir_for_ctfe(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> Body<'tcx> { + self.root + .tables + .mir_for_ctfe + .get(self, id) + .unwrap_or_else(|| { + bug!("get_mir_for_ctfe: missing MIR for `{:?}`", self.local_def_id(id)) + }) + .decode((self, tcx)) + } + fn get_mir_abstract_const( &self, tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index b7f22885217..96db0157422 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -115,6 +115,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, }) } optimized_mir => { tcx.arena.alloc(cdata.get_optimized_mir(tcx, def_id.index)) } + mir_for_ctfe => { tcx.arena.alloc(cdata.get_mir_for_ctfe(tcx, def_id.index)) } promoted_mir => { tcx.arena.alloc(cdata.get_promoted_mir(tcx, def_id.index)) } mir_abstract_const => { cdata.get_mir_abstract_const(tcx, def_id.index) } unused_generic_params => { cdata.get_unused_generic_params(def_id.index) } @@ -145,6 +146,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, impl_parent => { cdata.get_parent_impl(def_id.index) } trait_of_item => { cdata.get_trait_of_item(def_id.index) } is_mir_available => { cdata.is_item_mir_available(def_id.index) } + is_ctfe_mir_available => { cdata.is_ctfe_mir_available(def_id.index) } dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) } is_panic_runtime => { cdata.root.panic_runtime } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 8abae6924b5..ccaee8608b6 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -758,8 +758,6 @@ impl EncodeContext<'a, 'tcx> { self.encode_generics(def_id); self.encode_explicit_predicates(def_id); self.encode_inferred_outlives(def_id); - self.encode_optimized_mir(def_id.expect_local()); - self.encode_promoted_mir(def_id.expect_local()); } fn encode_enum_variant_ctor(&mut self, def: &ty::AdtDef, index: VariantIdx) { @@ -789,6 +787,7 @@ impl EncodeContext<'a, 'tcx> { self.encode_generics(def_id); self.encode_explicit_predicates(def_id); self.encode_inferred_outlives(def_id); + self.encode_mir_for_ctfe(def_id.expect_local()); self.encode_optimized_mir(def_id.expect_local()); self.encode_promoted_mir(def_id.expect_local()); } @@ -897,6 +896,7 @@ impl EncodeContext<'a, 'tcx> { self.encode_explicit_predicates(def_id); self.encode_inferred_outlives(def_id); self.encode_optimized_mir(def_id.expect_local()); + self.encode_mir_for_ctfe(def_id.expect_local()); self.encode_promoted_mir(def_id.expect_local()); } @@ -1015,8 +1015,21 @@ impl EncodeContext<'a, 'tcx> { self.encode_inferred_outlives(def_id); // This should be kept in sync with `PrefetchVisitor.visit_trait_item`. - self.encode_optimized_mir(def_id.expect_local()); - self.encode_promoted_mir(def_id.expect_local()); + match trait_item.kind { + ty::AssocKind::Type => {} + ty::AssocKind::Const => { + if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id.expect_local()) { + self.encode_mir_for_ctfe(def_id.expect_local()); + self.encode_promoted_mir(def_id.expect_local()); + } + } + ty::AssocKind::Fn => { + if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id.expect_local()) { + self.encode_optimized_mir(def_id.expect_local()); + self.encode_promoted_mir(def_id.expect_local()); + } + } + } } fn metadata_output_only(&self) -> bool { @@ -1089,8 +1102,8 @@ impl EncodeContext<'a, 'tcx> { // The following part should be kept in sync with `PrefetchVisitor.visit_impl_item`. - let mir = match ast_item.kind { - hir::ImplItemKind::Const(..) => true, + let (mir, mir_const) = match ast_item.kind { + hir::ImplItemKind::Const(..) => (false, true), hir::ImplItemKind::Fn(ref sig, _) => { let generics = self.tcx.generics_of(def_id); let needs_inline = (generics.requires_monomorphization(self.tcx) @@ -1098,14 +1111,19 @@ impl EncodeContext<'a, 'tcx> { && !self.metadata_output_only(); let is_const_fn = sig.header.constness == hir::Constness::Const; let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir; - needs_inline || is_const_fn || always_encode_mir + (needs_inline || always_encode_mir, is_const_fn) } - hir::ImplItemKind::TyAlias(..) => false, + hir::ImplItemKind::TyAlias(..) => (false, false), }; if mir { self.encode_optimized_mir(def_id.expect_local()); + } + if mir || mir_const { self.encode_promoted_mir(def_id.expect_local()); } + if mir_const { + self.encode_mir_for_ctfe(def_id.expect_local()); + } } fn encode_fn_param_names_for_body(&mut self, body_id: hir::BodyId) -> Lazy<[Ident]> { @@ -1116,28 +1134,34 @@ impl EncodeContext<'a, 'tcx> { self.lazy(param_names.iter()) } - fn encode_optimized_mir(&mut self, def_id: LocalDefId) { - debug!("EntryBuilder::encode_mir({:?})", def_id); - if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id) { - record!(self.tables.mir[def_id.to_def_id()] <- self.tcx.optimized_mir(def_id)); + fn encode_mir_for_ctfe(&mut self, def_id: LocalDefId) { + debug!("EntryBuilder::encode_mir_for_ctfe({:?})", def_id); + record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- self.tcx.mir_for_ctfe(def_id)); - let unused = self.tcx.unused_generic_params(def_id); - if !unused.is_empty() { - record!(self.tables.unused_generic_params[def_id.to_def_id()] <- unused); - } + let unused = self.tcx.unused_generic_params(def_id); + if !unused.is_empty() { + record!(self.tables.unused_generic_params[def_id.to_def_id()] <- unused); + } - let abstract_const = self.tcx.mir_abstract_const(def_id); - if let Ok(Some(abstract_const)) = abstract_const { - record!(self.tables.mir_abstract_consts[def_id.to_def_id()] <- abstract_const); - } + let abstract_const = self.tcx.mir_abstract_const(def_id); + if let Ok(Some(abstract_const)) = abstract_const { + record!(self.tables.mir_abstract_consts[def_id.to_def_id()] <- abstract_const); + } + } + + fn encode_optimized_mir(&mut self, def_id: LocalDefId) { + debug!("EntryBuilder::encode_optimized_mir({:?})", def_id); + record!(self.tables.mir[def_id.to_def_id()] <- self.tcx.optimized_mir(def_id)); + + let unused = self.tcx.unused_generic_params(def_id); + if !unused.is_empty() { + record!(self.tables.unused_generic_params[def_id.to_def_id()] <- unused); } } fn encode_promoted_mir(&mut self, def_id: LocalDefId) { debug!("EncodeContext::encode_promoted_mir({:?})", def_id); - if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id) { - record!(self.tables.promoted_mir[def_id.to_def_id()] <- self.tcx.promoted_mir(def_id)); - } + record!(self.tables.promoted_mir[def_id.to_def_id()] <- self.tcx.promoted_mir(def_id)); } // Encodes the inherent implementations of a structure, enumeration, or trait. @@ -1406,22 +1430,31 @@ impl EncodeContext<'a, 'tcx> { // The following part should be kept in sync with `PrefetchVisitor.visit_item`. - let mir = match item.kind { - hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => true, + let (mir, const_mir) = match item.kind { + hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => (false, true), hir::ItemKind::Fn(ref sig, ..) => { let generics = tcx.generics_of(def_id); let needs_inline = (generics.requires_monomorphization(tcx) || tcx.codegen_fn_attrs(def_id).requests_inline()) && !self.metadata_output_only(); + + let is_const_fn = sig.header.constness == hir::Constness::Const; let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir; - needs_inline || sig.header.constness == hir::Constness::Const || always_encode_mir + let mir = needs_inline || always_encode_mir; + // We don't need the optimized MIR for const fns. + (mir, is_const_fn) } - _ => false, + _ => (false, false), }; if mir { self.encode_optimized_mir(def_id.expect_local()); + } + if mir || const_mir { self.encode_promoted_mir(def_id.expect_local()); } + if const_mir { + self.encode_mir_for_ctfe(def_id.expect_local()); + } } /// Serialize the text of exported macros @@ -1486,7 +1519,7 @@ impl EncodeContext<'a, 'tcx> { self.encode_generics(def_id.to_def_id()); self.encode_explicit_predicates(def_id.to_def_id()); self.encode_inferred_outlives(def_id.to_def_id()); - self.encode_optimized_mir(def_id); + self.encode_mir_for_ctfe(def_id); self.encode_promoted_mir(def_id); } @@ -1951,6 +1984,12 @@ struct PrefetchVisitor<'tcx> { } impl<'tcx> PrefetchVisitor<'tcx> { + fn prefetch_ctfe_mir(&self, def_id: LocalDefId) { + if self.mir_keys.contains(&def_id) { + self.tcx.ensure().mir_for_ctfe(def_id); + self.tcx.ensure().promoted_mir(def_id); + } + } fn prefetch_mir(&self, def_id: LocalDefId) { if self.mir_keys.contains(&def_id) { self.tcx.ensure().optimized_mir(def_id); @@ -1965,16 +2004,19 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> { let tcx = self.tcx; match item.kind { hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => { - self.prefetch_mir(tcx.hir().local_def_id(item.hir_id)) + self.prefetch_ctfe_mir(tcx.hir().local_def_id(item.hir_id)) } hir::ItemKind::Fn(ref sig, ..) => { let def_id = tcx.hir().local_def_id(item.hir_id); let generics = tcx.generics_of(def_id.to_def_id()); let needs_inline = generics.requires_monomorphization(tcx) || tcx.codegen_fn_attrs(def_id.to_def_id()).requests_inline(); - if needs_inline || sig.header.constness == hir::Constness::Const { + if needs_inline { self.prefetch_mir(def_id) } + if sig.header.constness == hir::Constness::Const { + self.prefetch_ctfe_mir(def_id); + } } _ => (), } @@ -1982,7 +2024,16 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> { fn visit_trait_item(&self, trait_item: &'v hir::TraitItem<'v>) { // This should be kept in sync with `encode_info_for_trait_item`. - self.prefetch_mir(self.tcx.hir().local_def_id(trait_item.hir_id)); + let def_id = self.tcx.hir().local_def_id(trait_item.hir_id); + match trait_item.kind { + hir::TraitItemKind::Type(..) => {} + hir::TraitItemKind::Const(..) => { + self.prefetch_ctfe_mir(def_id); + } + hir::TraitItemKind::Fn(..) => { + self.prefetch_mir(def_id); + } + } } fn visit_impl_item(&self, impl_item: &'v hir::ImplItem<'v>) { @@ -1990,7 +2041,7 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> { let tcx = self.tcx; match impl_item.kind { hir::ImplItemKind::Const(..) => { - self.prefetch_mir(tcx.hir().local_def_id(impl_item.hir_id)) + self.prefetch_ctfe_mir(tcx.hir().local_def_id(impl_item.hir_id)) } hir::ImplItemKind::Fn(ref sig, _) => { let def_id = tcx.hir().local_def_id(impl_item.hir_id); @@ -1998,9 +2049,12 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> { let needs_inline = generics.requires_monomorphization(tcx) || tcx.codegen_fn_attrs(def_id.to_def_id()).requests_inline(); let is_const_fn = sig.header.constness == hir::Constness::Const; - if needs_inline || is_const_fn { + if needs_inline { self.prefetch_mir(def_id) } + if is_const_fn { + self.prefetch_ctfe_mir(def_id); + } } hir::ImplItemKind::TyAlias(..) => (), } diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 53606178909..59a8bc7fac1 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -302,6 +302,7 @@ define_tables! { // As an optimization, a missing entry indicates an empty `&[]`. explicit_item_bounds: Table<DefIndex, Lazy!([(ty::Predicate<'tcx>, Span)])>, mir: Table<DefIndex, Lazy!(mir::Body<'tcx>)>, + mir_for_ctfe: Table<DefIndex, Lazy!(mir::Body<'tcx>)>, promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>, mir_abstract_consts: Table<DefIndex, Lazy!(&'tcx [mir::abstract_const::Node<'tcx>])>, unused_generic_params: Table<DefIndex, Lazy<FiniteBitSet<u32>>>, diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 89a93096f1c..a7b847fc5e0 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -439,18 +439,27 @@ impl<'tcx> TyCtxt<'tcx> { } #[inline] - pub fn optimized_mir_opt_const_arg( + pub fn optimized_mir_or_const_arg_mir( self, def: ty::WithOptConstParam<DefId>, ) -> &'tcx Body<'tcx> { if let Some((did, param_did)) = def.as_const_arg() { - self.optimized_mir_of_const_arg((did, param_did)) + self.mir_for_ctfe_of_const_arg((did, param_did)) } else { self.optimized_mir(def.did) } } #[inline] + pub fn mir_for_ctfe_opt_const_arg(self, def: ty::WithOptConstParam<DefId>) -> &'tcx Body<'tcx> { + if let Some((did, param_did)) = def.as_const_arg() { + self.mir_for_ctfe_of_const_arg((did, param_did)) + } else { + self.mir_for_ctfe(def.did) + } + } + + #[inline] pub fn mir_abstract_const_opt_const_arg( self, def: ty::WithOptConstParam<DefId>, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 1e836d0a842..fd8c1ac23d1 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -312,6 +312,20 @@ rustc_queries! { desc { |tcx| "elaborating drops for `{}`", tcx.def_path_str(key.did.to_def_id()) } } + query mir_for_ctfe( + key: DefId + ) -> &'tcx mir::Body<'tcx> { + desc { |tcx| "caching mir of `{}` for CTFE", tcx.def_path_str(key) } + cache_on_disk_if { key.is_local() } + } + + query mir_for_ctfe_of_const_arg(key: (LocalDefId, DefId)) -> &'tcx mir::Body<'tcx> { + desc { + |tcx| "MIR for CTFE of the const argument `{}`", + tcx.def_path_str(key.0.to_def_id()) + } + } + query mir_promoted(key: ty::WithOptConstParam<LocalDefId>) -> ( &'tcx Steal<mir::Body<'tcx>>, @@ -331,12 +345,6 @@ rustc_queries! { desc { |tcx| "optimizing MIR for `{}`", tcx.def_path_str(key) } cache_on_disk_if { key.is_local() } } - query optimized_mir_of_const_arg(key: (LocalDefId, DefId)) -> &'tcx mir::Body<'tcx> { - desc { - |tcx| "optimizing MIR for the const argument `{}`", - tcx.def_path_str(key.0.to_def_id()) - } - } /// Returns coverage summary info for a function, after executing the `InstrumentCoverage` /// MIR pass (assuming the -Zinstrument-coverage option is enabled). @@ -927,6 +935,9 @@ rustc_queries! { } Codegen { + query is_ctfe_mir_available(key: DefId) -> bool { + desc { |tcx| "checking if item has ctfe mir available: `{}`", tcx.def_path_str(key) } + } query is_mir_available(key: DefId) -> bool { desc { |tcx| "checking if item has mir available: `{}`", tcx.def_path_str(key) } } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 94186d490c3..cebe0594c02 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -3010,7 +3010,16 @@ impl<'tcx> TyCtxt<'tcx> { /// Returns the possibly-auto-generated MIR of a `(DefId, Subst)` pair. pub fn instance_mir(self, instance: ty::InstanceDef<'tcx>) -> &'tcx Body<'tcx> { match instance { - ty::InstanceDef::Item(def) => self.optimized_mir_opt_const_arg(def), + ty::InstanceDef::Item(def) => match self.def_kind(def.did) { + DefKind::Const + | DefKind::Static + | DefKind::AssocConst + | DefKind::Ctor(..) + | DefKind::AnonConst => self.mir_for_ctfe_opt_const_arg(def), + // If the caller wants `mir_for_ctfe` of a function they should not be using + // `instance_mir`, so we'll assume const fn also wants the optimized version. + _ => self.optimized_mir_or_const_arg_mir(def), + }, ty::InstanceDef::VtableShim(..) | ty::InstanceDef::ReifyShim(..) | ty::InstanceDef::Intrinsic(..) diff --git a/compiler/rustc_mir/src/const_eval/machine.rs b/compiler/rustc_mir/src/const_eval/machine.rs index 02a9ec4df16..49126cfec6b 100644 --- a/compiler/rustc_mir/src/const_eval/machine.rs +++ b/compiler/rustc_mir/src/const_eval/machine.rs @@ -201,6 +201,22 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, type MemoryExtra = MemoryExtra; + fn load_mir( + ecx: &InterpCx<'mir, 'tcx, Self>, + instance: ty::InstanceDef<'tcx>, + ) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> { + match instance { + ty::InstanceDef::Item(def) => { + if ecx.tcx.is_ctfe_mir_available(def.did) { + Ok(ecx.tcx.mir_for_ctfe_opt_const_arg(def)) + } else { + throw_unsup!(NoMirFor(def.did)) + } + } + _ => Ok(ecx.tcx.instance_mir(instance)), + } + } + fn find_mir_or_eval_fn( ecx: &mut InterpCx<'mir, 'tcx, Self>, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_mir/src/interpret/eval_context.rs b/compiler/rustc_mir/src/interpret/eval_context.rs index 3d955576f0f..6d7781671d8 100644 --- a/compiler/rustc_mir/src/interpret/eval_context.rs +++ b/compiler/rustc_mir/src/interpret/eval_context.rs @@ -477,16 +477,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { if let Some(promoted) = promoted { return Ok(&self.tcx.promoted_mir_opt_const_arg(def)[promoted]); } - match instance { - ty::InstanceDef::Item(def) => { - if self.tcx.is_mir_available(def.did) { - Ok(self.tcx.optimized_mir_opt_const_arg(def)) - } else { - throw_unsup!(NoMirFor(def.did)) - } - } - _ => Ok(self.tcx.instance_mir(instance)), - } + M::load_mir(self, instance) } /// Call this on things you got out of the MIR (so it is as generic as the current diff --git a/compiler/rustc_mir/src/interpret/machine.rs b/compiler/rustc_mir/src/interpret/machine.rs index a1a825b3268..53ac62d4351 100644 --- a/compiler/rustc_mir/src/interpret/machine.rs +++ b/compiler/rustc_mir/src/interpret/machine.rs @@ -132,6 +132,16 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Whether to enforce the validity invariant fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; + /// Entry point for obtaining the MIR of anything that should get evaluated. + /// So not just functions and shims, but also const/static initializers, anonymous + /// constants, ... + fn load_mir( + ecx: &InterpCx<'mir, 'tcx, Self>, + instance: ty::InstanceDef<'tcx>, + ) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> { + Ok(ecx.tcx.instance_mir(instance)) + } + /// Entry point to all function calls. /// /// Returns either the mir to use for the call, or `None` if execution should diff --git a/compiler/rustc_mir/src/monomorphize/polymorphize.rs b/compiler/rustc_mir/src/monomorphize/polymorphize.rs index 0ce1c5a0489..4ad71ab4913 100644 --- a/compiler/rustc_mir/src/monomorphize/polymorphize.rs +++ b/compiler/rustc_mir/src/monomorphize/polymorphize.rs @@ -5,7 +5,7 @@ //! generic parameters are unused (and eventually, in what ways generic parameters are used - only //! for their size, offset of a field, etc.). -use rustc_hir::{def::DefKind, def_id::DefId}; +use rustc_hir::{def::DefKind, def_id::DefId, ConstContext}; use rustc_index::bit_set::FiniteBitSet; use rustc_middle::mir::{ visit::{TyContext, Visitor}, @@ -54,9 +54,17 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> { } // Exit early when there is no MIR available. - if !tcx.is_mir_available(def_id) { - debug!("unused_generic_params: (no mir available) def_id={:?}", def_id); - return FiniteBitSet::new_empty(); + let context = tcx.hir().body_const_context(def_id.expect_local()); + match context { + Some(ConstContext::ConstFn) | None if !tcx.is_mir_available(def_id) => { + debug!("unused_generic_params: (no mir available) def_id={:?}", def_id); + return FiniteBitSet::new_empty(); + } + Some(_) if !tcx.is_ctfe_mir_available(def_id) => { + debug!("unused_generic_params: (no ctfe mir available) def_id={:?}", def_id); + return FiniteBitSet::new_empty(); + } + _ => {} } // Create a bitset with N rightmost ones for each parameter. @@ -69,7 +77,12 @@ fn unused_generic_params(tcx: TyCtxt<'_>, def_id: DefId) -> FiniteBitSet<u32> { debug!("unused_generic_params: (after default) unused_parameters={:?}", unused_parameters); // Visit MIR and accumululate used generic parameters. - let body = tcx.optimized_mir(def_id); + let body = match context { + // Const functions are actually called and should thus be considered for polymorphization + // via their runtime MIR + Some(ConstContext::ConstFn) | None => tcx.optimized_mir(def_id), + Some(_) => tcx.mir_for_ctfe(def_id), + }; let mut vis = MarkUsedGenericParams { tcx, def_id, unused_parameters: &mut unused_parameters }; vis.visit_body(body); debug!("unused_generic_params: (after visitor) unused_parameters={:?}", unused_parameters); diff --git a/compiler/rustc_mir/src/transform/const_prop.rs b/compiler/rustc_mir/src/transform/const_prop.rs index 2d6d0adf3bc..a311e262dd4 100644 --- a/compiler/rustc_mir/src/transform/const_prop.rs +++ b/compiler/rustc_mir/src/transform/const_prop.rs @@ -185,6 +185,13 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> type MemoryExtra = (); + fn load_mir( + _ecx: &InterpCx<'mir, 'tcx, Self>, + _instance: ty::InstanceDef<'tcx>, + ) -> InterpResult<'tcx, &'tcx Body<'tcx>> { + throw_machine_stop_str!("calling functions isn't supported in ConstProp") + } + fn find_mir_or_eval_fn( _ecx: &mut InterpCx<'mir, 'tcx, Self>, _instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_mir/src/transform/coverage/query.rs b/compiler/rustc_mir/src/transform/coverage/query.rs index aa34ae70ef1..4b455a6a1ba 100644 --- a/compiler/rustc_mir/src/transform/coverage/query.rs +++ b/compiler/rustc_mir/src/transform/coverage/query.rs @@ -4,7 +4,7 @@ use rustc_middle::mir::coverage::*; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::{self, Coverage, CoverageInfo, Location}; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{self, TyCtxt}; use rustc_span::def_id::DefId; /// The `query` provider for `CoverageInfo`, requested by `codegen_coverage()` (to inject each @@ -112,7 +112,7 @@ impl Visitor<'_> for CoverageVisitor { } fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> CoverageInfo { - let mir_body = tcx.optimized_mir(def_id); + let mir_body = mir_body(tcx, def_id); let mut coverage_visitor = CoverageVisitor { // num_counters always has at least the `ZERO` counter. @@ -129,8 +129,7 @@ fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> CoverageInfo } fn covered_file_name<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<Symbol> { - let mir_body = tcx.optimized_mir(def_id); - for bb_data in mir_body.basic_blocks().iter() { + for bb_data in mir_body(tcx, def_id).basic_blocks().iter() { for statement in bb_data.statements.iter() { if let StatementKind::Coverage(box ref coverage) = statement.kind { if let Some(code_region) = coverage.code_region.as_ref() { @@ -142,9 +141,17 @@ fn covered_file_name<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<Symbol> { None } +/// This function ensures we obtain the correct MIR for the given item irrespective of +/// whether that means const mir or runtime mir. For `const fn` this opts for runtime +/// mir. +fn mir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx mir::Body<'tcx> { + let id = ty::WithOptConstParam::unknown(def_id); + let def = ty::InstanceDef::Item(id); + tcx.instance_mir(def) +} + fn covered_code_regions<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Vec<&'tcx CodeRegion> { - let mir_body: &'tcx mir::Body<'tcx> = tcx.optimized_mir(def_id); - mir_body + mir_body(tcx, def_id) .basic_blocks() .iter() .map(|data| { diff --git a/compiler/rustc_mir/src/transform/mod.rs b/compiler/rustc_mir/src/transform/mod.rs index 7f3b421cf76..11f7e6922cc 100644 --- a/compiler/rustc_mir/src/transform/mod.rs +++ b/compiler/rustc_mir/src/transform/mod.rs @@ -71,9 +71,11 @@ pub(crate) fn provide(providers: &mut Providers) { }, mir_promoted, mir_drops_elaborated_and_const_checked, + mir_for_ctfe, + mir_for_ctfe_of_const_arg, optimized_mir, - optimized_mir_of_const_arg, is_mir_available, + is_ctfe_mir_available: |tcx, did| is_mir_available(tcx, did), promoted_mir: |tcx, def_id| { let def_id = def_id.expect_local(); if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { @@ -278,6 +280,7 @@ fn mir_const<'tcx>( tcx.alloc_steal_mir(body) } +/// Compute the main MIR body and the list of MIR bodies of the promoteds. fn mir_promoted( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, @@ -319,6 +322,87 @@ fn mir_promoted( (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted)) } +/// Compute the MIR that is used during CTFE (and thus has no optimizations run on it) +fn mir_for_ctfe<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx Body<'tcx> { + let did = def_id.expect_local(); + if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) { + tcx.mir_for_ctfe_of_const_arg(def) + } else { + tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(did))) + } +} + +/// Same as `mir_for_ctfe`, but used to get the MIR of a const generic parameter. +/// The docs on `WithOptConstParam` explain this a bit more, but the TLDR is that +/// we'd get cycle errors with `mir_for_ctfe`, because typeck would need to typeck +/// the const parameter while type checking the main body, which in turn would try +/// to type check the main body again. +fn mir_for_ctfe_of_const_arg<'tcx>( + tcx: TyCtxt<'tcx>, + (did, param_did): (LocalDefId, DefId), +) -> &'tcx Body<'tcx> { + tcx.arena.alloc(inner_mir_for_ctfe( + tcx, + ty::WithOptConstParam { did, const_param_did: Some(param_did) }, + )) +} + +fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> { + // FIXME: don't duplicate this between the optimized_mir/mir_for_ctfe queries + if tcx.is_constructor(def.did.to_def_id()) { + // There's no reason to run all of the MIR passes on constructors when + // we can just output the MIR we want directly. This also saves const + // qualification and borrow checking the trouble of special casing + // constructors. + return shim::build_adt_ctor(tcx, def.did.to_def_id()); + } + + let context = tcx + .hir() + .body_const_context(def.did) + .expect("mir_for_ctfe should not be used for runtime functions"); + + let mut body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); + + match context { + // Do not const prop functions, either they get executed at runtime or exported to metadata, + // so we run const prop on them, or they don't, in which case we const evaluate some control + // flow paths of the function and any errors in those paths will get emitted as const eval + // errors. + hir::ConstContext::ConstFn => {} + // Static items always get evaluated, so we can just let const eval see if any erroneous + // control flow paths get executed. + hir::ConstContext::Static(_) => {} + // Associated constants get const prop run so we detect common failure situations in the + // crate that defined the constant. + // Technically we want to not run on regular const items, but oli-obk doesn't know how to + // conveniently detect that at this point without looking at the HIR. + hir::ConstContext::Const => { + #[rustfmt::skip] + let optimizations: &[&dyn MirPass<'_>] = &[ + &const_prop::ConstProp, + ]; + + #[rustfmt::skip] + run_passes( + tcx, + &mut body, + MirPhase::Optimization, + &[ + optimizations, + ], + ); + } + } + + debug_assert!(!body.has_free_regions(), "Free regions in MIR for CTFE"); + + body +} + +/// Obtain just the main MIR (no promoteds) and run some cleanups on it. This also runs +/// mir borrowck *before* doing so in order to ensure that borrowck can be run and doesn't +/// end up missing the source MIR due to stealing happening. fn mir_drops_elaborated_and_const_checked<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, @@ -456,35 +540,32 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { ); } +/// Optimize the MIR and prepare it for codegen. fn optimized_mir<'tcx>(tcx: TyCtxt<'tcx>, did: DefId) -> &'tcx Body<'tcx> { let did = did.expect_local(); - if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) { - tcx.optimized_mir_of_const_arg(def) - } else { - tcx.arena.alloc(inner_optimized_mir(tcx, ty::WithOptConstParam::unknown(did))) - } + assert_eq!(ty::WithOptConstParam::try_lookup(did, tcx), None); + tcx.arena.alloc(inner_optimized_mir(tcx, did)) } -fn optimized_mir_of_const_arg<'tcx>( - tcx: TyCtxt<'tcx>, - (did, param_did): (LocalDefId, DefId), -) -> &'tcx Body<'tcx> { - tcx.arena.alloc(inner_optimized_mir( - tcx, - ty::WithOptConstParam { did, const_param_did: Some(param_did) }, - )) -} - -fn inner_optimized_mir(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> { - if tcx.is_constructor(def.did.to_def_id()) { +fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { + if tcx.is_constructor(did.to_def_id()) { // There's no reason to run all of the MIR passes on constructors when // we can just output the MIR we want directly. This also saves const // qualification and borrow checking the trouble of special casing // constructors. - return shim::build_adt_ctor(tcx, def.did.to_def_id()); + return shim::build_adt_ctor(tcx, did.to_def_id()); } - let mut body = tcx.mir_drops_elaborated_and_const_checked(def).steal(); + match tcx.hir().body_const_context(did) { + // Run the `mir_for_ctfe` query, which depends on `mir_drops_elaborated_and_const_checked` + // which we are going to steal below. Thus we need to run `mir_for_ctfe` first, so it + // computes and caches its result. + Some(hir::ConstContext::ConstFn) => tcx.ensure().mir_for_ctfe(did), + None => {} + Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), + } + let mut body = + tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); run_optimization_passes(tcx, &mut body); debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); @@ -492,6 +573,8 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) body } +/// Fetch all the promoteds of an item and prepare their MIR bodies to be ready for +/// constant evaluation once all substitutions become known. fn promoted_mir<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, @@ -510,7 +593,6 @@ fn promoted_mir<'tcx>( for body in &mut promoted { run_post_borrowck_cleanup_passes(tcx, body); - run_optimization_passes(tcx, body); } debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); diff --git a/compiler/rustc_mir/src/util/pretty.rs b/compiler/rustc_mir/src/util/pretty.rs index 89ce29bd101..7fc1c3a73af 100644 --- a/compiler/rustc_mir/src/util/pretty.rs +++ b/compiler/rustc_mir/src/util/pretty.rs @@ -273,8 +273,6 @@ pub fn write_mir_pretty<'tcx>( let mut first = true; for def_id in dump_mir_def_ids(tcx, single) { - let body = &tcx.optimized_mir(def_id); - if first { first = false; } else { @@ -282,11 +280,28 @@ pub fn write_mir_pretty<'tcx>( writeln!(w)?; } - write_mir_fn(tcx, body, &mut |_, _| Ok(()), w)?; - - for body in tcx.promoted_mir(def_id) { - writeln!(w)?; + let render_body = |w: &mut dyn Write, body| -> io::Result<()> { write_mir_fn(tcx, body, &mut |_, _| Ok(()), w)?; + + for body in tcx.promoted_mir(def_id) { + writeln!(w)?; + write_mir_fn(tcx, body, &mut |_, _| Ok(()), w)?; + } + Ok(()) + }; + match tcx.hir().body_const_context(def_id.expect_local()) { + None => render_body(w, tcx.optimized_mir(def_id))?, + // For `const fn` we want to render the optimized MIR. If you want the mir used in + // ctfe, you can dump the MIR after the `Deaggregator` optimization pass. + Some(rustc_hir::ConstContext::ConstFn) => { + render_body(w, tcx.optimized_mir(def_id))?; + writeln!(w)?; + writeln!(w, "// MIR FOR CTFE")?; + // Do not use `render_body`, as that would render the promoteds again, but these + // are shared between mir_for_ctfe and optimized_mir + write_mir_fn(tcx, tcx.mir_for_ctfe(def_id), &mut |_, _| Ok(()), w)?; + } + Some(_) => render_body(w, tcx.mir_for_ctfe(def_id))?, } } Ok(()) diff --git a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs index fdb2361ba03..370ad577170 100644 --- a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs +++ b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs @@ -152,7 +152,7 @@ pub fn is_const_evaluatable<'cx, 'tcx>( if concrete.is_ok() && substs.has_param_types_or_consts() { match infcx.tcx.def_kind(def.did) { DefKind::AnonConst => { - let mir_body = infcx.tcx.optimized_mir_opt_const_arg(def); + let mir_body = infcx.tcx.mir_for_ctfe_opt_const_arg(def); if mir_body.is_polymorphic { future_compat_lint(); |