diff options
Diffstat (limited to 'library/alloc/src/raw_vec/mod.rs')
| -rw-r--r-- | library/alloc/src/raw_vec/mod.rs | 166 |
1 files changed, 118 insertions, 48 deletions
diff --git a/library/alloc/src/raw_vec/mod.rs b/library/alloc/src/raw_vec/mod.rs index b7c153b825d..bc9692f5b6c 100644 --- a/library/alloc/src/raw_vec/mod.rs +++ b/library/alloc/src/raw_vec/mod.rs @@ -24,7 +24,6 @@ mod tests; // only one location which panics rather than a bunch throughout the module. #[cfg(not(no_global_oom_handling))] #[cfg_attr(not(panic = "immediate-abort"), inline(never))] -#[track_caller] fn capacity_overflow() -> ! { panic!("capacity overflow"); } @@ -123,7 +122,6 @@ impl<T> RawVec<T, Global> { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] pub(crate) fn with_capacity(capacity: usize) -> Self { Self { inner: RawVecInner::with_capacity(capacity, T::LAYOUT), _marker: PhantomData } } @@ -132,7 +130,6 @@ impl<T> RawVec<T, Global> { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] pub(crate) fn with_capacity_zeroed(capacity: usize) -> Self { Self { inner: RawVecInner::with_capacity_zeroed_in(capacity, Global, T::LAYOUT), @@ -145,7 +142,6 @@ impl RawVecInner<Global> { #[cfg(not(any(no_global_oom_handling, test)))] #[must_use] #[inline] - #[track_caller] fn with_capacity(capacity: usize, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Uninitialized, Global, elem_layout) { Ok(res) => res, @@ -177,6 +173,8 @@ impl<T, A: Allocator> RawVec<T, A> { /// the returned `RawVec`. #[inline] pub(crate) const fn new_in(alloc: A) -> Self { + // Check assumption made in `current_memory` + const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; Self { inner: RawVecInner::new_in(alloc, Alignment::of::<T>()), _marker: PhantomData } } @@ -184,7 +182,6 @@ impl<T, A: Allocator> RawVec<T, A> { /// allocator for the returned `RawVec`. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn with_capacity_in(capacity: usize, alloc: A) -> Self { Self { inner: RawVecInner::with_capacity_in(capacity, alloc, T::LAYOUT), @@ -206,7 +203,6 @@ impl<T, A: Allocator> RawVec<T, A> { /// of allocator for the returned `RawVec`. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { Self { inner: RawVecInner::with_capacity_zeroed_in(capacity, alloc, T::LAYOUT), @@ -326,18 +322,18 @@ impl<T, A: Allocator> RawVec<T, A> { /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] pub(crate) fn reserve(&mut self, len: usize, additional: usize) { - self.inner.reserve(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.reserve(len, additional, T::LAYOUT) } } /// A specialized version of `self.reserve(len, 1)` which requires the /// caller to ensure `len == self.capacity()`. #[cfg(not(no_global_oom_handling))] #[inline(never)] - #[track_caller] pub(crate) fn grow_one(&mut self) { - self.inner.grow_one(T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.grow_one(T::LAYOUT) } } /// The same as `reserve`, but returns on errors instead of panicking or aborting. @@ -346,7 +342,8 @@ impl<T, A: Allocator> RawVec<T, A> { len: usize, additional: usize, ) -> Result<(), TryReserveError> { - self.inner.try_reserve(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.try_reserve(len, additional, T::LAYOUT) } } /// Ensures that the buffer contains at least enough space to hold `len + @@ -367,9 +364,9 @@ impl<T, A: Allocator> RawVec<T, A> { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] - #[track_caller] pub(crate) fn reserve_exact(&mut self, len: usize, additional: usize) { - self.inner.reserve_exact(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT) } } /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. @@ -378,7 +375,8 @@ impl<T, A: Allocator> RawVec<T, A> { len: usize, additional: usize, ) -> Result<(), TryReserveError> { - self.inner.try_reserve_exact(len, additional, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -392,10 +390,10 @@ impl<T, A: Allocator> RawVec<T, A> { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] - #[track_caller] #[inline] pub(crate) fn shrink_to_fit(&mut self, cap: usize) { - self.inner.shrink_to_fit(cap, T::LAYOUT) + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout + unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT) } } } @@ -417,7 +415,6 @@ impl<A: Allocator> RawVecInner<A> { #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] fn with_capacity_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { Ok(this) => { @@ -442,7 +439,6 @@ impl<A: Allocator> RawVecInner<A> { #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] fn with_capacity_zeroed_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { match Self::try_allocate_in(capacity, AllocInit::Zeroed, alloc, elem_layout) { Ok(res) => res, @@ -518,8 +514,12 @@ impl<A: Allocator> RawVecInner<A> { &self.alloc } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[inline] - fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull<u8>, Layout)> { + unsafe fn current_memory(&self, elem_layout: Layout) -> Option<(NonNull<u8>, Layout)> { if elem_layout.size() == 0 || self.cap.as_inner() == 0 { None } else { @@ -535,48 +535,65 @@ impl<A: Allocator> RawVecInner<A> { } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { // Callers expect this function to be very cheap when there is already sufficient capacity. // Therefore, we move all the resizing and error-handling logic from grow_amortized and // handle_reserve behind a call, while making sure that this function is likely to be // inlined as just a comparison and a call if the comparison fails. #[cold] - fn do_reserve_and_handle<A: Allocator>( + unsafe fn do_reserve_and_handle<A: Allocator>( slf: &mut RawVecInner<A>, len: usize, additional: usize, elem_layout: Layout, ) { - if let Err(err) = slf.grow_amortized(len, additional, elem_layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { slf.grow_amortized(len, additional, elem_layout) } { handle_error(err); } } if self.needs_to_grow(len, additional, elem_layout) { - do_reserve_and_handle(self, len, additional, elem_layout); + unsafe { + do_reserve_and_handle(self, len, additional, elem_layout); + } } } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn grow_one(&mut self, elem_layout: Layout) { - if let Err(err) = self.grow_amortized(self.cap.as_inner(), 1, elem_layout) { + unsafe fn grow_one(&mut self, elem_layout: Layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { self.grow_amortized(self.cap.as_inner(), 1, elem_layout) } { handle_error(err); } } - fn try_reserve( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + unsafe fn try_reserve( &mut self, len: usize, additional: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { - self.grow_amortized(len, additional, elem_layout)?; + // SAFETY: Precondition passed to caller + unsafe { + self.grow_amortized(len, additional, elem_layout)?; + } } unsafe { // Inform the optimizer that the reservation has succeeded or wasn't needed @@ -585,22 +602,33 @@ impl<A: Allocator> RawVecInner<A> { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] - #[track_caller] - fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { - if let Err(err) = self.try_reserve_exact(len, additional, elem_layout) { + unsafe fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { + // SAFETY: Precondition passed to caller + if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout) } { handle_error(err); } } - fn try_reserve_exact( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + unsafe fn try_reserve_exact( &mut self, len: usize, additional: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { - self.grow_exact(len, additional, elem_layout)?; + // SAFETY: Precondition passed to caller + unsafe { + self.grow_exact(len, additional, elem_layout)?; + } } unsafe { // Inform the optimizer that the reservation has succeeded or wasn't needed @@ -609,11 +637,15 @@ impl<A: Allocator> RawVecInner<A> { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - #[track_caller] - fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { - if let Err(err) = self.shrink(cap, elem_layout) { + unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { + if let Err(err) = unsafe { self.shrink(cap, elem_layout) } { handle_error(err); } } @@ -632,7 +664,13 @@ impl<A: Allocator> RawVecInner<A> { self.cap = unsafe { Cap::new_unchecked(cap) }; } - fn grow_amortized( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - The sum of `len` and `additional` must be greater than or equal to + /// `self.capacity(elem_layout.size())` + unsafe fn grow_amortized( &mut self, len: usize, additional: usize, @@ -657,14 +695,25 @@ impl<A: Allocator> RawVecInner<A> { let new_layout = layout_array(cap, elem_layout)?; - let ptr = finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)?; - // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + let ptr = + unsafe { finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)? }; + // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items unsafe { self.set_ptr_and_cap(ptr, cap) }; Ok(()) } - fn grow_exact( + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - The sum of `len` and `additional` must be greater than or equal to + /// `self.capacity(elem_layout.size())` + unsafe fn grow_exact( &mut self, len: usize, additional: usize, @@ -679,7 +728,12 @@ impl<A: Allocator> RawVecInner<A> { let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; let new_layout = layout_array(cap, elem_layout)?; - let ptr = finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)?; + // SAFETY: + // - For the `current_memory` call: Precondition passed to caller + // - For the `finish_grow` call: Precondition passed to caller + // + `current_memory` does the right thing + let ptr = + unsafe { finish_grow(new_layout, self.current_memory(elem_layout), &mut self.alloc)? }; // SAFETY: layout_array would have resulted in a capacity overflow if we tried to allocate more than `isize::MAX` items unsafe { self.set_ptr_and_cap(ptr, cap); @@ -687,9 +741,14 @@ impl<A: Allocator> RawVecInner<A> { Ok(()) } + /// # Safety + /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to + /// initially construct `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { + unsafe fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(elem_layout.size()), "Tried to shrink to a larger capacity"); // SAFETY: Just checked this isn't trying to grow unsafe { self.shrink_unchecked(cap, elem_layout) } @@ -711,8 +770,12 @@ impl<A: Allocator> RawVecInner<A> { cap: usize, elem_layout: Layout, ) -> Result<(), TryReserveError> { - let (ptr, layout) = - if let Some(mem) = self.current_memory(elem_layout) { mem } else { return Ok(()) }; + // SAFETY: Precondition passed to caller + let (ptr, layout) = if let Some(mem) = unsafe { self.current_memory(elem_layout) } { + mem + } else { + return Ok(()); + }; // If shrinking to 0, deallocate the buffer. We don't reach this point // for the T::IS_ZST case since current_memory() will have returned @@ -748,7 +811,8 @@ impl<A: Allocator> RawVecInner<A> { /// Ideally this function would take `self` by move, but it cannot because it exists to be /// called from a `Drop` impl. unsafe fn deallocate(&mut self, elem_layout: Layout) { - if let Some((ptr, layout)) = self.current_memory(elem_layout) { + // SAFETY: Precondition passed to caller + if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { unsafe { self.alloc.deallocate(ptr, layout); } @@ -756,10 +820,17 @@ impl<A: Allocator> RawVecInner<A> { } } +/// # Safety +/// If `current_memory` matches `Some((ptr, old_layout))`: +/// - `ptr` must denote a block of memory *currently allocated* via `alloc` +/// - `old_layout` must *fit* that block of memory +/// - `new_layout` must have the same alignment as `old_layout` +/// - `new_layout.size()` must be greater than or equal to `old_layout.size()` +/// If `current_memory` is `None`, this function is safe. // not marked inline(never) since we want optimizers to be able to observe the specifics of this // function, see tests/codegen-llvm/vec-reserve-extend.rs. #[cold] -fn finish_grow<A>( +unsafe fn finish_grow<A>( new_layout: Layout, current_memory: Option<(NonNull<u8>, Layout)>, alloc: &mut A, @@ -785,7 +856,6 @@ where #[cfg(not(no_global_oom_handling))] #[cold] #[optimize(size)] -#[track_caller] fn handle_error(e: TryReserveError) -> ! { match e.kind() { CapacityOverflow => capacity_overflow(), |