diff options
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/intern.rs | 47 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/memory.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/mir/interpret/allocation.rs | 17 |
3 files changed, 61 insertions, 8 deletions
diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index c1d42c9ae7c..9dfdafcb38e 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -168,8 +168,51 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory mplace: &MPlaceTy<'tcx>, fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>, ) -> InterpResult<'tcx> { - // ZSTs cannot contain pointers, so we can skip them. - if mplace.layout.is_zst() { + // We want to walk the aggregate to look for references to intern. While doing that we + // also need to take special care of interior mutability. + // + // As an optimization, however, if the allocation does not contain any references: we don't + // need to do the walk. It can be costly for big arrays for example (e.g. issue #93215). + let is_walk_needed = |mplace: &MPlaceTy<'tcx>| -> InterpResult<'tcx, bool> { + // ZSTs cannot contain pointers, we can avoid the interning walk. + if mplace.layout.is_zst() { + return Ok(false); + } + + // Now, check whether this allocation could contain references. + // + // Note, this check may sometimes not be cheap, so we only do it when the walk we'd like + // to avoid could be expensive: on the potentially larger types, arrays and slices, + // rather than on all aggregates unconditionally. + if matches!(mplace.layout.ty.kind(), ty::Array(..) | ty::Slice(..)) { + let Some((size, align)) = self.ecx.size_and_align_of_mplace(&mplace)? else { + // We do the walk if we can't determine the size of the mplace: we may be + // dealing with extern types here in the future. + return Ok(true); + }; + + // If there are no relocations in this allocation, it does not contain references + // that point to another allocation, and we can avoid the interning walk. + if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? { + if !alloc.has_relocations() { + return Ok(false); + } + } else { + // We're encountering a ZST here, and can avoid the walk as well. + return Ok(false); + } + } + + // In the general case, we do the walk. + Ok(true) + }; + + // If this allocation contains no references to intern, we avoid the potentially costly + // walk. + // + // We can do this before the checks for interior mutability below, because only references + // are relevant in that situation, and we're checking if there are any here. + if !is_walk_needed(mplace)? { return Ok(()); } diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index d5e68dbd5b7..c2a5b71b8f9 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -942,6 +942,11 @@ impl<'tcx, 'a, Tag: Provenance, Extra> AllocRef<'a, 'tcx, Tag, Extra> { .check_bytes(&self.tcx, self.range.subrange(range), allow_uninit, allow_ptr) .map_err(|e| e.to_interp_error(self.alloc_id))?) } + + /// Returns whether the allocation has relocations for the entire range of the `AllocRef`. + pub(crate) fn has_relocations(&self) -> bool { + self.alloc.has_relocations(&self.tcx, self.range) + } } impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs index 10c4ea63a68..1e2b53040d2 100644 --- a/compiler/rustc_middle/src/mir/interpret/allocation.rs +++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs @@ -537,21 +537,26 @@ impl<Tag: Provenance, Extra> Allocation<Tag, Extra> { /// Relocations. impl<Tag: Copy, Extra> Allocation<Tag, Extra> { /// Returns all relocations overlapping with the given pointer-offset pair. - pub fn get_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Tag)] { + fn get_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Tag)] { // We have to go back `pointer_size - 1` bytes, as that one would still overlap with // the beginning of this range. let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1); self.relocations.range(Size::from_bytes(start)..range.end()) } + /// Returns whether this allocation has relocations overlapping with the given range. + /// + /// Note: this function exists to allow `get_relocations` to be private, in order to somewhat + /// limit access to relocations outside of the `Allocation` abstraction. + /// + pub fn has_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool { + !self.get_relocations(cx, range).is_empty() + } + /// Checks that there are no relocations overlapping with the given range. #[inline(always)] fn check_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult { - if self.get_relocations(cx, range).is_empty() { - Ok(()) - } else { - Err(AllocError::ReadPointerAsBytes) - } + if self.has_relocations(cx, range) { Err(AllocError::ReadPointerAsBytes) } else { Ok(()) } } /// Removes all relocations inside the given range. |