diff --git a/Cargo.toml b/Cargo.toml index 687c3ae..4c9c277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,10 @@ specialization = [] may_dangle = [] serde = ["dep:serde_core"] internals = [] +default = ["allocator-api2"] [dependencies] +allocator-api2 = { version = "0.4", optional = true, features = ["alloc"], default-features = false } arbitrary = { version = "1.4", optional = true, default-features = false } borsh = { version = "1.8", optional = true, features = ["derive", "unstable__schema"], default-features = false } bytes = { version = "1.12", optional = true, default-features = false } diff --git a/src/comparisons.rs b/src/comparisons.rs index 3fb7e6a..37f83f2 100644 --- a/src/comparisons.rs +++ b/src/comparisons.rs @@ -1,5 +1,8 @@ use { - crate::SmallVec, + crate::{ + Allocator, + SmallVec + }, alloc::{ borrow::Cow, collections::VecDeque, @@ -9,7 +12,7 @@ use { macro_rules! __impl_slice_eq1 { ([$($vars:tt)*] $lhs:ty, $rhs:ty $(where $ty:ty: $bound:ident)?) => { - impl PartialEq<$rhs> for $lhs + impl PartialEq<$rhs> for $lhs where T: PartialEq, $($ty: $bound)? @@ -20,25 +23,25 @@ macro_rules! __impl_slice_eq1 { }; } -__impl_slice_eq1! { [const N: usize, const M: usize] SmallVec, SmallVec } -__impl_slice_eq1! { [const N: usize, const M: usize] SmallVec, [U; N] } -__impl_slice_eq1! { [const N: usize, const M: usize] SmallVec, &[U; N] } -__impl_slice_eq1! { [const N: usize] SmallVec, [U] } -__impl_slice_eq1! { [const N: usize] SmallVec, &[U] } -__impl_slice_eq1! { [const N: usize] SmallVec, &mut [U] } -__impl_slice_eq1! { [const N: usize] [T], SmallVec } -__impl_slice_eq1! { [const N: usize] &[T], SmallVec } -__impl_slice_eq1! { [const N: usize] &mut [T], SmallVec } -__impl_slice_eq1! { [const N: usize] Vec, SmallVec } -__impl_slice_eq1! { [const N: usize] SmallVec, Vec } -__impl_slice_eq1! { [const N: usize] Cow<'_, [T]>, SmallVec where T: Clone } -__impl_slice_eq1! { [const N: usize] SmallVec, Cow<'_, [U]> where U: Clone } +__impl_slice_eq1! { [const N: usize, const M: usize] SmallVec, SmallVec } +__impl_slice_eq1! { [const N: usize, const M: usize] SmallVec, [U; N] } +__impl_slice_eq1! { [const N: usize, const M: usize] SmallVec, &[U; N] } +__impl_slice_eq1! { [const N: usize] SmallVec, [U] } +__impl_slice_eq1! { [const N: usize] SmallVec, &[U] } +__impl_slice_eq1! { [const N: usize] SmallVec, &mut [U] } +__impl_slice_eq1! { [const N: usize] [T], SmallVec } +__impl_slice_eq1! { [const N: usize] &[T], SmallVec } +__impl_slice_eq1! { [const N: usize] &mut [T], SmallVec } +__impl_slice_eq1! { [const N: usize] Vec, SmallVec } +__impl_slice_eq1! { [const N: usize] SmallVec, Vec } +__impl_slice_eq1! { [const N: usize] Cow<'_, [T]>, SmallVec where T: Clone } +__impl_slice_eq1! { [const N: usize] SmallVec, Cow<'_, [U]> where U: Clone } -impl PartialEq> for VecDeque +impl PartialEq> for VecDeque where T: PartialEq { #[inline] - fn eq(&self, other: &SmallVec) -> bool { + fn eq(&self, other: &SmallVec) -> bool { let other = other.as_slice(); if self.len() != other.len() { return false; @@ -49,22 +52,22 @@ where T: PartialEq } } -impl Eq for SmallVec where T: Eq {} +impl Eq for SmallVec where T: Eq {} -impl PartialOrd for SmallVec +impl PartialOrd for SmallVec where T: PartialOrd { #[inline] - fn partial_cmp(&self, other: &SmallVec) -> Option { + fn partial_cmp(&self, other: &SmallVec) -> Option { self.as_slice().partial_cmp(other.as_slice()) } } -impl Ord for SmallVec +impl Ord for SmallVec where T: Ord { #[inline] - fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { + fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { self.as_slice().cmp(other.as_slice()) } } diff --git a/src/conversions.rs b/src/conversions.rs index 4883bb2..3e5bded 100644 --- a/src/conversions.rs +++ b/src/conversions.rs @@ -1,5 +1,8 @@ use { - crate::SmallVec, + crate::{ + Allocator, + SmallVec + }, alloc::vec::Vec, core::{ mem::ManuallyDrop, @@ -73,11 +76,11 @@ impl From<[T; M]> for SmallVec { } } -impl TryFrom> for [T; M] { - type Error = SmallVec; +impl TryFrom> for [T; M] { + type Error = SmallVec; #[inline] - fn try_from(mut this: SmallVec) -> Result<[T; M], SmallVec> { + fn try_from(mut this: SmallVec) -> Result<[T; M], SmallVec> { if this.len() != M { Err(this) } else { diff --git a/src/lib.rs b/src/lib.rs index c54900a..98a6dd7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -88,6 +88,16 @@ mod serde; mod specialization; mod taggedlen; +#[cfg(not(feature = "allocator-api2"))] +use alloc::{ + Allocator, + Global +}; +#[cfg(feature = "allocator-api2")] +use allocator_api2::alloc::{ + Allocator, + Global +}; #[cfg(feature = "bytes")] use bytes::{ BufMut, @@ -115,7 +125,6 @@ use { Hasher }, iter::repeat_n, - marker::PhantomData, mem::{ ManuallyDrop, MaybeUninit, @@ -197,14 +206,13 @@ where R: core::ops::RangeBounds { } #[repr(C)] -pub struct SmallVec { +pub struct SmallVec { len: TaggedLen, - raw: RawSmallVec, - _marker: PhantomData + raw: RawSmallVec } -unsafe impl Send for SmallVec {} -unsafe impl Sync for SmallVec {} +unsafe impl Send for SmallVec {} +unsafe impl Sync for SmallVec {} impl Default for SmallVec { #[inline] @@ -219,7 +227,7 @@ impl Default for SmallVec { /// Returned from [`SmallVec::drain`][1]. /// /// [1]: struct.SmallVec.html#method.drain -pub struct Drain<'a, T: 'a, const N: usize> { +pub struct Drain<'a, T: 'a, const N: usize, A: Allocator> { // `vec` points to a valid object within its lifetime. // This is ensured by the fact that we're holding an iterator to its items. // @@ -230,10 +238,10 @@ pub struct Drain<'a, T: 'a, const N: usize> { tail_start: usize, tail_len: usize, iter: core::slice::Iter<'a, T>, - vec: core::ptr::NonNull> + vec: core::ptr::NonNull> } -impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { +impl<'a, T: 'a, const N: usize, A: Allocator> Iterator for Drain<'a, T, N, A> { type Item = T; #[inline] @@ -251,7 +259,7 @@ impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { } } -impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { +impl<'a, T: 'a, const N: usize, A: Allocator> DoubleEndedIterator for Drain<'a, T, N, A> { #[inline] fn next_back(&mut self) -> Option { // SAFETY: see above @@ -261,21 +269,21 @@ impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { } } -impl ExactSizeIterator for Drain<'_, T, N> { +impl ExactSizeIterator for Drain<'_, T, N, A> { #[inline] fn len(&self) -> usize { self.iter.len() } } -impl core::iter::FusedIterator for Drain<'_, T, N> {} +impl core::iter::FusedIterator for Drain<'_, T, N, A> {} -impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { +impl<'a, T: 'a, const N: usize, A: Allocator> Drop for Drain<'a, T, N, A> { fn drop(&mut self) { /// Moves back the un-`Drain`ed elements to restore the original `Vec`. - struct DropGuard<'r, 'a, T, const N: usize>(&'r mut Drain<'a, T, N>); + struct DropGuard<'r, 'a, T, const N: usize, A: Allocator>(&'r mut Drain<'a, T, N, A>); - impl<'r, 'a, T, const N: usize> Drop for DropGuard<'r, 'a, T, N> { + impl<'r, 'a, T, const N: usize, A: Allocator> Drop for DropGuard<'r, 'a, T, N, A> { fn drop(&mut self) { if self.0.tail_len > 0 { unsafe { @@ -300,7 +308,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { let mut vec = self.vec; - if SmallVec::::IS_ZST { + if SmallVec::::IS_ZST { // ZSTs have no identity, so we don't need to move them around, we // only need to drop the correct amount. this can be // achieved by manipulating the Vec length instead of @@ -348,7 +356,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { } } -impl Drain<'_, T, N> { +impl Drain<'_, T, N, A> { #[must_use] pub fn as_slice(&self) -> &[T] { self.iter.as_slice() @@ -371,7 +379,7 @@ impl Drain<'_, T, N> { // SAFETY: len < tail_start <= capacity unsafe { vec.as_mut_ptr().add(len).write(new_item); - vec.set_len(len + 1); + vec.len.add(1); } } true @@ -408,10 +416,10 @@ impl Drain<'_, T, N> { /// Returned from [`SmallVec::extract_if`][1]. /// /// [1]: struct.SmallVec.html#method.extract_if -pub struct ExtractIf<'a, T, const N: usize, F> +pub struct ExtractIf<'a, T, const N: usize, A: Allocator, F> where F: FnMut(&mut T) -> bool { - vec: &'a mut SmallVec, + vec: &'a mut SmallVec, /// The index of the item that will be inspected by the next call to `next`. idx: usize, /// Elements at and beyond this point will be retained. Must be equal or @@ -425,7 +433,7 @@ where F: FnMut(&mut T) -> bool pred: F } -impl core::fmt::Debug for ExtractIf<'_, T, N, F> +impl core::fmt::Debug for ExtractIf<'_, T, N, A, F> where F: FnMut(&mut T) -> bool, T: core::fmt::Debug @@ -437,7 +445,7 @@ where } } -impl Iterator for ExtractIf<'_, T, N, F> +impl Iterator for ExtractIf<'_, T, N, A, F> where F: FnMut(&mut T) -> bool { type Item = T; @@ -450,8 +458,9 @@ where F: FnMut(&mut T) -> bool let cur = self.vec.as_mut_ptr().add(i); let drained = (self.pred)(&mut *cur); // Update the index *after* the predicate is called. If the - // index is updated prior and the predicate panics, - // the element at this index would be leaked. + // index is updated prior and the predicate + // panics, the element at this index would be + // leaked. self.idx += 1; if drained { self.del += 1; @@ -470,7 +479,7 @@ where F: FnMut(&mut T) -> bool } } -impl Drop for ExtractIf<'_, T, N, F> +impl Drop for ExtractIf<'_, T, N, A, F> where F: FnMut(&mut T) -> bool { fn drop(&mut self) { @@ -495,7 +504,7 @@ where F: FnMut(&mut T) -> bool } pub struct Splice<'a, I: Iterator + 'a, const N: usize> { - drain: Drain<'a, I::Item, N>, + drain: Drain<'a, I::Item, N, Global>, replace_with: I } @@ -585,24 +594,23 @@ impl Drop for Splice<'_, I, N> { /// Returned from [`SmallVec::into_iter`][1]. /// /// [1]: struct.SmallVec.html#method.into_iter -pub struct IntoIter { +pub struct IntoIter { // # Safety // // `end` decides whether the data lives on the heap or not // // The members from begin..end are initialized - raw: RawSmallVec, + raw: RawSmallVec, begin: usize, - end: TaggedLen, - _marker: PhantomData + end: TaggedLen } // SAFETY: IntoIter has unique ownership of its contents. Sending (or sharing) // an `IntoIter` is equivalent to sending (or sharing) a `SmallVec`. -unsafe impl Send for IntoIter where T: Send {} -unsafe impl Sync for IntoIter where T: Sync {} +unsafe impl Send for IntoIter where T: Send {} +unsafe impl Sync for IntoIter where T: Sync {} -impl IntoIter { +impl IntoIter { #[inline] pub const fn as_slice(&self) -> &[T] { let (end, on_heap) = self.end.parts(); @@ -626,7 +634,7 @@ impl IntoIter { } } -impl Iterator for IntoIter { +impl Iterator for IntoIter { type Item = T; #[inline] @@ -652,7 +660,7 @@ impl Iterator for IntoIter { } } -impl DoubleEndedIterator for IntoIter { +impl DoubleEndedIterator for IntoIter { #[inline] fn next_back(&mut self) -> Option { let (end, on_heap) = self.end.parts(); @@ -669,34 +677,22 @@ impl DoubleEndedIterator for IntoIter { } } } -impl ExactSizeIterator for IntoIter {} -impl core::iter::FusedIterator for IntoIter {} +impl ExactSizeIterator for IntoIter {} +impl core::iter::FusedIterator for IntoIter {} impl SmallVec { #[inline] pub const fn new() -> SmallVec { - Self { - len: TaggedLen::new(0, false), - raw: RawSmallVec::new(), - _marker: PhantomData - } + Self::new_in(Global) } pub fn try_with_capacity(capacity: usize) -> Result { - let mut this = Self::new(); - if capacity > Self::inline_size() && !Self::IS_ZST { - // SAFETY: we checked all the preconditions - unsafe { this.raw.try_grow_raw(TaggedLen::new(0, false), capacity) }?; - - // SAFETY: the allocation succeeded, so self.raw.heap is now active - unsafe { this.set_on_heap() }; - } - Ok(this) + Self::try_with_capacity_in(capacity, Global) } #[inline] pub fn with_capacity(capacity: usize) -> Self { - infallible(Self::try_with_capacity(capacity)) + Self::with_capacity_in(capacity, Global) } #[inline] @@ -723,8 +719,7 @@ impl SmallVec { // SAFETY: all the members in 0..S are initialized Self { len: TaggedLen::new(S, false), - raw: RawSmallVec::new_inline(buf), - _marker: PhantomData + raw: RawSmallVec::new_inline(buf, Global) } } @@ -734,8 +729,7 @@ impl SmallVec { // SAFETY: all the members in 0..len are initialized let mut vec = Self { len: TaggedLen::new(len, false), - raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), - _marker: PhantomData + raw: RawSmallVec::new_inline(MaybeUninit::new(buf), Global) }; // Deallocate the remaining elements so no memory is leaked. unsafe { @@ -755,9 +749,9 @@ impl SmallVec { vec } - /// Constructs a new `SmallVec` on the stack from an A without copying + /// Constructs a new `SmallVec` on the stack from a buffer without copying /// elements. Also sets the length. The user is responsible for ensuring - /// that `len <= A::size()`. + /// that `len <= N`. /// /// # Examples /// @@ -781,14 +775,9 @@ impl SmallVec { debug_assert!(len <= N); Self { len: TaggedLen::new(len, false), - raw: RawSmallVec::new_inline(buf), - _marker: PhantomData + raw: RawSmallVec::new_inline(buf, Global) } } -} - -impl SmallVec { - const IS_ZST: bool = size_of::() == 0; #[inline] pub fn from_vec(vec: Vec) -> Self { @@ -811,8 +800,7 @@ impl SmallVec { unsafe { vec.set_len(0) }; Self { len: TaggedLen::new(len, false), - raw: RawSmallVec::new(), - _marker: PhantomData + raw: RawSmallVec::new(Global) } } else { let mut vec = ManuallyDrop::new(vec); @@ -824,12 +812,127 @@ impl SmallVec { Self { len: TaggedLen::new(len, true), - raw: RawSmallVec::new_heap(ptr, cap), - _marker: PhantomData + raw: RawSmallVec::new_heap(ptr, cap, Global) } } } + pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, N> + where + R: core::ops::RangeBounds, + I: IntoIterator + { + Splice { + drain: self.drain(range), + replace_with: replace_with.into_iter() + } + } + + /// Creates a `SmallVec` directly from the raw components of another + /// `SmallVec`. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren’t + /// checked: + /// + /// - `ptr` needs to have been previously allocated via `SmallVec` from its + /// spilled storage (at least, it’s highly likely to be incorrect if it + /// wasn’t). + /// - `ptr`’s `T` type needs to be the same size and alignment that it was + /// allocated with + /// - `length` needs to be less than or equal to `capacity`. + /// - `capacity` needs to be the capacity that the pointer was allocated + /// with. + /// + /// Violating these may cause problems like corrupting the allocator’s + /// internal data structures. + /// + /// Additionally, `capacity` must be greater than `N`; that is, the new + /// `SmallVec` must need to spill over into heap allocated storage. This + /// condition is asserted against. + /// + /// The ownership of `ptr` is effectively transferred to the `SmallVec` + /// which may then deallocate, reallocate or change the contents of memory + /// pointed to by the pointer at will. Ensure that nothing else uses the + /// pointer after calling this function. + /// + /// # Examples + /// + /// ``` + /// use smallvec::SmallVec; + /// + /// let mut v: SmallVec<_, 1> = SmallVec::from([1, 2, 3]); + /// + /// // Pull out the important parts of `v`. + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// let spilled = v.spilled(); + /// + /// unsafe { + /// // Forget all about `v`. The heap allocation that stored the + /// // three values won't be deallocated. + /// std::mem::forget(v); + /// + /// // Overwrite memory with [4, 5, 6]. + /// // + /// // This is only safe if `spilled` is true! Otherwise, we are + /// // writing into the old `SmallVec`'s inline storage on the + /// // stack. + /// assert!(spilled); + /// for i in 0..len { + /// std::ptr::write(p.add(i), 4 + i); + /// } + /// + /// // Put everything back together into a SmallVec with a different + /// // amount of inline storage, but which is still less than `cap`. + /// let rebuilt = SmallVec::<_, 2>::from_raw_parts(p, len, cap); + /// assert_eq!(&*rebuilt, &[4, 5, 6]); + /// } + /// ``` + #[inline] + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> SmallVec { + assert!(!Self::IS_ZST); + + // SAFETY: We require caller to provide same ptr as we alloc + // and we never alloc null pointer. + let ptr = unsafe { + debug_assert!(!ptr.is_null(), "Called `from_raw_parts` with null pointer."); + NonNull::new_unchecked(ptr) + }; + + SmallVec { + len: TaggedLen::new(length, true), + raw: RawSmallVec::new_heap(ptr, capacity, Global) + } + } +} + +impl SmallVec { + /// A function for creating [`SmallVec`] values out of slices + /// for types with the [`Copy`] trait. + pub fn from_slice_copy(slice: &[T]) -> Self + where T: Copy { + let src = slice.as_ptr(); + let len = slice.len(); + let mut result = Self::with_capacity(len); + + // SAFETY: By using `with_capacity`, the pointer will point to valid + // memory. + unsafe { + let dst = result.as_mut_ptr(); + copy_nonoverlapping(src, dst, len); + result.set_len(len); + } + + result + } +} + +impl SmallVec { + const IS_ZST: bool = size_of::() == 0; + /// Sets the tag to be on the heap /// /// # Safety @@ -868,7 +971,7 @@ impl SmallVec { #[inline] pub const fn inline_size() -> usize { - RawSmallVec::::INLINE_CAP + RawSmallVec::::INLINE_CAP } #[inline] @@ -893,51 +996,7 @@ impl SmallVec { self.len.on_heap() } - /// Splits the collection into two at the given index. - /// - /// Returns a newly allocated vector containing the elements in the range - /// `[at, len)`. After the call, the original vector will be left containing - /// the elements `[0, at)` with its previous capacity unchanged. - /// - /// - If you want to take ownership of the entire contents and capacity of - /// the vector, see [`core::mem::take`] or [`core::mem::replace`]. - /// - If you don't need the returned vector at all, see - /// [`SmallVec::truncate`]. - /// - If you want to take ownership of an arbitrary subslice, or you don't - /// necessarily want to store the removed items in a vector, see - /// [`SmallVec::drain`]. - /// - /// # Panics - /// - /// Panics if `at > len`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// let vec2 = vec.split_off(1); - /// assert_eq!(vec, [1]); - /// assert_eq!(vec2, [2, 3]); - /// ``` - #[inline] - pub fn split_off(&mut self, at: usize) -> Self { - let len = self.len(); - assert!(at <= len); - - let other_len = len - at; - let mut other = Self::with_capacity(other_len); - - // Unsafely `set_len` and copy items to `other`. - unsafe { - self.set_len(at); - other.set_len(other_len); - - core::ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other_len); - } - other - } - - pub fn drain(&mut self, range: R) -> Drain<'_, T, N> + pub fn drain(&mut self, range: R) -> Drain<'_, T, N, A> where R: core::ops::RangeBounds { let len = self.len(); let core::ops::Range { @@ -1048,7 +1107,7 @@ impl SmallVec { /// ); /// assert_eq!(ones.len(), 3); /// ``` - pub fn extract_if(&mut self, range: R, filter: F) -> ExtractIf<'_, T, N, F> + pub fn extract_if(&mut self, range: R, filter: F) -> ExtractIf<'_, T, N, A, F> where F: FnMut(&mut T) -> bool, R: core::ops::RangeBounds @@ -1074,17 +1133,6 @@ impl SmallVec { } } - pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, N> - where - R: core::ops::RangeBounds, - I: IntoIterator - { - Splice { - drain: self.drain(range), - replace_with: replace_with.into_iter() - } - } - #[inline] pub fn push(&mut self, value: T) { _ = self.push_mut(value); @@ -1111,6 +1159,7 @@ impl SmallVec { // This block is an exact copy of `self.set_len`. // We have to do this so that Miri doesn't report a "Stacked // Borrows" rule violation. See PR/406 + // SAFETY: we have wrote the value to the address already unsafe { self.len.add(1); @@ -1148,7 +1197,7 @@ impl SmallVec { } #[inline] - pub fn append(&mut self, other: &mut SmallVec) { + pub fn append(&mut self, other: &mut SmallVec) { // can't overflow since both are smaller than isize::MAX and 2 * // isize::MAX < usize::MAX let len = self.len(); @@ -1196,7 +1245,7 @@ impl SmallVec { if on_heap { unsafe { // SAFETY: heap member is active - let (ptr, old_cap) = self.raw.heap; + let (ptr, old_cap) = self.raw.inner.heap; // inline member is now active // SAFETY: len <= new_capacity <= Self::inline_size() @@ -1205,7 +1254,8 @@ impl SmallVec { drop(DropDealloc { ptr: ptr.cast(), size_bytes: old_cap * size_of::(), - align: align_of::() + align: align_of::(), + alloc: &self.raw.alloc }); self.set_inline(); } @@ -1277,12 +1327,11 @@ impl SmallVec { if len <= Self::inline_size() { // SAFETY: on_heap is true, so we're on the heap unsafe { - let (ptr, capacity) = self.raw.heap; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); + let (ptr, capacity) = self.raw.inner.heap; copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); self.set_inline(); - alloc::alloc::dealloc( - ptr.cast().as_ptr(), + self.raw.alloc.deallocate( + ptr.cast(), Layout::from_size_align_unchecked(capacity * size_of::(), align_of::()) ); } @@ -1290,7 +1339,7 @@ impl SmallVec { // SAFETY: len > Self::inline_size() >= 0 // so new capacity is non zero, it is equal to the length // T can't be a ZST because SmallVec is never spilled. - unsafe { infallible(self.raw.try_grow_raw(self.len, len)) }; + unsafe { self.raw.shrink_to_raw(len) }; } } @@ -1301,18 +1350,17 @@ impl SmallVec { return; } // SAFETY: the vector is on the heap - let capacity = unsafe { self.raw.heap.1 }; + let capacity = unsafe { self.raw.inner.heap.1 }; if capacity > min_capacity { let target = core::cmp::max(len, min_capacity); if target <= Self::inline_size() { // SAFETY: on_heap is true, so we're on the heap unsafe { - let (ptr, capacity) = self.raw.heap; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); + let (ptr, capacity) = self.raw.inner.heap; copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); self.set_inline(); - alloc::alloc::dealloc( - ptr.cast().as_ptr(), + self.raw.alloc.deallocate( + ptr.cast(), Layout::from_size_align_unchecked( capacity * size_of::(), align_of::() @@ -1323,7 +1371,7 @@ impl SmallVec { // SAFETY: len > Self::inline_size() >= 0 // so new capacity is non zero, it is equal to the length // T can't be a ZST because SmallVec is never spilled. - unsafe { infallible(self.raw.try_grow_raw(self.len, target)) }; + unsafe { self.raw.shrink_to_raw(target) }; } } } @@ -1517,7 +1565,7 @@ impl SmallVec { // - the first `len` entries are proper `T`-values // - the allocation is not larger than `isize::MAX` unsafe { - let (ptr, cap) = this.raw.heap; + let (ptr, cap) = this.raw.inner.heap; Vec::from_raw_parts(ptr.as_ptr(), len, cap) } } @@ -1576,14 +1624,14 @@ impl SmallVec { // panicked. It shifts unchecked elements to cover holes and // `set_len` to the correct length. In cases when predicate and // `drop` never panic, it will be optimized out. - struct PanicGuard<'a, T, const N: usize> { - v: &'a mut SmallVec, + struct PanicGuard<'a, T, const N: usize, A: Allocator> { + v: &'a mut SmallVec, read: usize, write: usize, original_len: usize } - impl Drop for PanicGuard<'_, T, N> { + impl Drop for PanicGuard<'_, T, N, A> { #[cold] fn drop(&mut self) { let remaining = self.original_len - self.read; @@ -1741,127 +1789,6 @@ impl SmallVec { } } - /// Creates a `SmallVec` directly from the raw components of another - /// `SmallVec`. - /// - /// # Safety - /// - /// This is highly unsafe, due to the number of invariants that aren’t - /// checked: - /// - /// - `ptr` needs to have been previously allocated via `SmallVec` from its - /// spilled storage (at least, it’s highly likely to be incorrect if it - /// wasn’t). - /// - `ptr`’s `A::Item` type needs to be the same size and alignment that it - /// was allocated with - /// - `length` needs to be less than or equal to `capacity`. - /// - `capacity` needs to be the capacity that the pointer was allocated - /// with. - /// - /// Violating these may cause problems like corrupting the allocator’s - /// internal data structures. - /// - /// Additionally, `capacity` must be greater than the amount of inline - /// storage `A` has; that is, the new `SmallVec` must need to spill over - /// into heap allocated storage. This condition is asserted against. - /// - /// The ownership of `ptr` is effectively transferred to the `SmallVec` - /// which may then deallocate, reallocate or change the contents of memory - /// pointed to by the pointer at will. Ensure that nothing else uses the - /// pointer after calling this function. - /// - /// # Examples - /// - /// ``` - /// use smallvec::SmallVec; - /// - /// let mut v: SmallVec<_, 1> = SmallVec::from([1, 2, 3]); - /// - /// // Pull out the important parts of `v`. - /// let p = v.as_mut_ptr(); - /// let len = v.len(); - /// let cap = v.capacity(); - /// let spilled = v.spilled(); - /// - /// unsafe { - /// // Forget all about `v`. The heap allocation that stored the - /// // three values won't be deallocated. - /// std::mem::forget(v); - /// - /// // Overwrite memory with [4, 5, 6]. - /// // - /// // This is only safe if `spilled` is true! Otherwise, we are - /// // writing into the old `SmallVec`'s inline storage on the - /// // stack. - /// assert!(spilled); - /// for i in 0..len { - /// std::ptr::write(p.add(i), 4 + i); - /// } - /// - /// // Put everything back together into a SmallVec with a different - /// // amount of inline storage, but which is still less than `cap`. - /// let rebuilt = SmallVec::<_, 2>::from_raw_parts(p, len, cap); - /// assert_eq!(&*rebuilt, &[4, 5, 6]); - /// } - /// ``` - #[inline] - pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> SmallVec { - assert!(!Self::IS_ZST); - - // SAFETY: We require caller to provide same ptr as we alloc - // and we never alloc null pointer. - let ptr = unsafe { - debug_assert!(!ptr.is_null(), "Called `from_raw_parts` with null pointer."); - NonNull::new_unchecked(ptr) - }; - - SmallVec { - len: TaggedLen::new(length, true), - raw: RawSmallVec::new_heap(ptr, capacity), - _marker: PhantomData - } - } -} - -impl SmallVec { - #[inline] - pub fn resize(&mut self, len: usize, value: T) { - let old_len = self.len(); - if len > old_len { - self.extend(core::iter::repeat_n(value, len - old_len)); - } else { - self.truncate(len); - } - } - - #[inline] - pub fn extend_from_slice(&mut self, other: &[T]) { - self.extend(other.iter()) - } - - pub fn extend_from_within(&mut self, src: R) - where R: core::ops::RangeBounds { - let src = slice_range(src, ..self.len()); - self.reserve(src.len()); - - // SAFETY: The call to `reserve` ensures that the capacity is large - // enough. The range is within bounds through the use of - // `core::slice::range`. - unsafe { - #[cfg(feature = "specialization")] - { - >::spec_extend_from_within( - self, src - ); - } - - #[cfg(not(feature = "specialization"))] - { - self.extend_from_within_fallback(src); - } - } - } - #[inline] pub fn extend_from_slice_copy(&mut self, other: &[T]) where T: Copy { @@ -1925,23 +1852,113 @@ impl SmallVec { } } - /// A function for creating [`SmallVec`] values out of slices - /// for types with the [`Copy`] trait. - pub fn from_slice_copy(slice: &[T]) -> Self - where T: Copy { - let src = slice.as_ptr(); - let len = slice.len(); - let mut result = Self::with_capacity(len); + pub const fn new_in(alloc: A) -> SmallVec { + Self { + len: TaggedLen::new(0, false), + raw: RawSmallVec::new(alloc) + } + } - // SAFETY: By using `with_capacity`, the pointer will point to valid - // memory. + pub fn try_with_capacity_in(capacity: usize, alloc: A) -> Result { + let mut this = Self::new_in(alloc); + if capacity > Self::inline_size() && !Self::IS_ZST { + // SAFETY: we checked all the preconditions + unsafe { this.raw.try_grow_raw(TaggedLen::new(0, false), capacity) }?; + + // SAFETY: the allocation succeeded, so self.raw.heap is now active + unsafe { this.set_on_heap() }; + } + Ok(this) + } + + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + infallible(Self::try_with_capacity_in(capacity, alloc)) + } +} + +impl SmallVec { + #[inline] + pub fn resize(&mut self, len: usize, value: T) { + let old_len = self.len(); + if len > old_len { + self.extend(core::iter::repeat_n(value, len - old_len)); + } else { + self.truncate(len); + } + } + + #[inline] + pub fn extend_from_slice(&mut self, other: &[T]) { + self.extend(other.iter()) + } + + pub fn extend_from_within(&mut self, src: R) + where R: core::ops::RangeBounds { + let src = slice_range(src, ..self.len()); + self.reserve(src.len()); + + // SAFETY: The call to `reserve` ensures that the capacity is large + // enough. The range is within bounds through the use of + // `core::slice::range`. unsafe { - let dst = result.as_mut_ptr(); - copy_nonoverlapping(src, dst, len); - result.set_len(len); + #[cfg(feature = "specialization")] + { + >::spec_extend_from_within( + self, src + ); + } + + #[cfg(not(feature = "specialization"))] + { + self.extend_from_within_fallback(src); + } } + } +} - result +impl SmallVec { + /// Splits the collection into two at the given index. + /// + /// Returns a newly allocated vector containing the elements in the range + /// `[at, len)`. After the call, the original vector will be left containing + /// the elements `[0, at)` with its previous capacity unchanged. + /// + /// - If you want to take ownership of the entire contents and capacity of + /// the vector, see [`core::mem::take`] or [`core::mem::replace`]. + /// - If you don't need the returned vector at all, see + /// [`SmallVec::truncate`]. + /// - If you want to take ownership of an arbitrary subslice, or you don't + /// necessarily want to store the removed items in a vector, see + /// [`SmallVec::drain`]. + /// + /// # Panics + /// + /// Panics if `at > len`. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// let vec2 = vec.split_off(1); + /// assert_eq!(vec, [1]); + /// assert_eq!(vec2, [2, 3]); + /// ``` + #[inline] + pub fn split_off(&mut self, at: usize) -> Self { + let len = self.len(); + assert!(at <= len); + + let other_len = len - at; + let mut other = Self::with_capacity_in(other_len, self.raw.alloc.clone()); + + // Unsafely `set_len` and copy items to `other`. + unsafe { + self.set_len(at); + other.set_len(other_len); + + core::ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other_len); + } + other } } @@ -1958,19 +1975,20 @@ impl Drop for DropGuard { } } -struct DropDealloc { +struct DropDealloc<'a, A: Allocator> { ptr: NonNull, size_bytes: usize, - align: usize + align: usize, + alloc: &'a A } -impl Drop for DropDealloc { +impl Drop for DropDealloc<'_, A> { #[inline] fn drop(&mut self) { unsafe { if self.size_bytes > 0 { - alloc::alloc::dealloc( - self.ptr.as_ptr(), + self.alloc.deallocate( + self.ptr, Layout::from_size_align_unchecked(self.size_bytes, self.align) ); } @@ -1979,7 +1997,7 @@ impl Drop for DropDealloc { } #[cfg(feature = "may_dangle")] -unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { +unsafe impl<#[may_dangle] T, const N: usize, A: Allocator> Drop for SmallVec { fn drop(&mut self) { let (len, on_heap) = self.len.parts(); let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; @@ -1987,11 +2005,12 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { // releasing memory we used to own unsafe { let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.inner.heap.1; Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), - align: align_of::() + align: align_of::(), + alloc: &self.raw.alloc }) } else { None @@ -2002,7 +2021,7 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { } #[cfg(not(feature = "may_dangle"))] -impl Drop for SmallVec { +impl Drop for SmallVec { fn drop(&mut self) { let (len, on_heap) = self.len.parts(); // SAFETY: the tag tells which member is active @@ -2010,11 +2029,12 @@ impl Drop for SmallVec { // SAFETY: see above unsafe { let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.inner.heap.1; Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), - align: align_of::() + align: align_of::(), + alloc: &self.raw.alloc }) } else { None @@ -2024,7 +2044,7 @@ impl Drop for SmallVec { } } -impl Drop for IntoIter { +impl Drop for IntoIter { fn drop(&mut self) { // SAFETY: see above unsafe { @@ -2032,11 +2052,12 @@ impl Drop for IntoIter { let begin = self.begin; let ptr = self.raw.as_mut_ptr(on_heap); let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.inner.heap.1; Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), - align: align_of::() + align: align_of::(), + alloc: &self.raw.alloc }) } else { None @@ -2112,13 +2133,60 @@ impl SmallVec { result } + fn from_iter_fallback(iter: I) -> Self + where I: Iterator { + let (size, _) = iter.size_hint(); + let mut v = Self::with_capacity(size); + v.extend_fallback(iter); + v + } + + /// Creates a `SmallVec` value based on the contents of `slice`. + /// This will use the inline storage, not the heap. + /// + /// # Safety + /// + /// The caller must ensure that `slice.len() <= Self::inline_size()`. + unsafe fn from_slice_fallback(slice: &[T]) -> Self + where T: Clone { + let mut v = Self::new(); + + let src = slice.as_ptr(); + let len = slice.len(); + let dst = v.as_mut_ptr(); + + // SAFETY: The caller ensures that the slice length is smaller + // than or equal to the inline length. + unsafe { + let mut guard = DropGuard { + ptr: dst, + len: 0 + }; + for i in 0..len { + let val = (*src.add(i)).clone(); + dst.add(i).write(val); + guard.len += 1; + } + core::mem::forget(guard); + } + + // SAFETY: The elements were initialized in the loop above. + unsafe { + v.set_len(len); + } + + v + } +} + +impl SmallVec { fn extend_fallback(&mut self, iter: I) where I: IntoIterator { - struct SetLenOnDrop<'a, T, const N: usize> { - vec: &'a mut SmallVec, + struct SetLenOnDrop<'a, T, const N: usize, A: Allocator> { + vec: &'a mut SmallVec, len: usize } - impl Drop for SetLenOnDrop<'_, T, N> { + impl Drop for SetLenOnDrop<'_, T, N, A> { #[inline(always)] fn drop(&mut self) { // SAFETY: restores `len` only len of initialized items. @@ -2208,14 +2276,6 @@ impl SmallVec { } } - fn from_iter_fallback(iter: I) -> Self - where I: Iterator { - let (size, _) = iter.size_hint(); - let mut v = Self::with_capacity(size); - v.extend_fallback(iter); - v - } - fn clone_from_fallback(&mut self, source: &[T]) where T: Clone { // Inspired from `impl Clone for Vec`. @@ -2231,49 +2291,19 @@ impl SmallVec { self.clone_from_slice(init); self.extend(tail.iter().cloned()); } - - /// Creates a `SmallVec` value based on the contents of `slice`. - /// This will use the inline storage, not the heap. - /// - /// # Safety - /// - /// The caller must ensure that `slice.len() <= Self::inline_size()`. - unsafe fn from_slice_fallback(slice: &[T]) -> Self - where T: Clone { - let mut v = Self::new(); - - let src = slice.as_ptr(); - let len = slice.len(); - let dst = v.as_mut_ptr(); - - // SAFETY: The caller ensures that the slice length is smaller - // than or equal to the inline length. - unsafe { - let mut guard = DropGuard { - ptr: dst, - len: 0 - }; - for i in 0..len { - let val = (*src.add(i)).clone(); - dst.add(i).write(val); - guard.len += 1; - } - core::mem::forget(guard); - } - - // SAFETY: The elements were initialized in the loop above. - unsafe { - v.set_len(len); - } - - v - } } -impl Clone for SmallVec { +impl Clone for SmallVec { #[inline] - fn clone(&self) -> SmallVec { - SmallVec::from(self.as_slice()) + fn clone(&self) -> SmallVec { + let mut vec = SmallVec { + len: TaggedLen::new(0, false), + raw: RawSmallVec::new(self.raw.alloc.clone()) + }; + + vec.extend(self); + + vec } #[inline] @@ -2290,14 +2320,21 @@ impl Clone for SmallVec { } } -impl Clone for IntoIter { +impl Clone for IntoIter { #[inline] - fn clone(&self) -> IntoIter { - SmallVec::from(self.as_slice()).into_iter() + fn clone(&self) -> IntoIter { + let mut vec = SmallVec { + len: TaggedLen::new(0, false), + raw: RawSmallVec::new(self.raw.alloc.clone()) + }; + + vec.extend(self.as_slice()); + + vec.into_iter() } } -impl Extend for SmallVec { +impl Extend for SmallVec { #[inline] fn extend>(&mut self, iter: I) { #[cfg(feature = "specialization")] @@ -2312,7 +2349,7 @@ impl Extend for SmallVec { } } -impl<'a, T: Clone + 'a, const N: usize> Extend<&'a T> for SmallVec { +impl<'a, T: Clone + 'a, const N: usize, A: Allocator> Extend<&'a T> for SmallVec { #[inline] fn extend>(&mut self, iter: I) { #[cfg(feature = "specialization")] @@ -2342,8 +2379,8 @@ impl core::iter::FromIterator for SmallVec { } } -impl IntoIterator for SmallVec { - type IntoIter = IntoIter; +impl IntoIterator for SmallVec { + type IntoIter = IntoIter; type Item = T; fn into_iter(self) -> Self::IntoIter { @@ -2354,16 +2391,15 @@ impl IntoIterator for SmallVec { // the elements let this = ManuallyDrop::new(self); IntoIter { - raw: (&this.raw as *const RawSmallVec).read(), + raw: (&this.raw as *const RawSmallVec).read(), begin: 0, - end: this.len, - _marker: PhantomData + end: this.len } } } } -impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { +impl<'a, T, const N: usize, A: Allocator> IntoIterator for &'a SmallVec { type IntoIter = core::slice::Iter<'a, T>; type Item = &'a T; @@ -2372,7 +2408,7 @@ impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { } } -impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { +impl<'a, T, const N: usize, A: Allocator> IntoIterator for &'a mut SmallVec { type IntoIter = core::slice::IterMut<'a, T>; type Item = &'a mut T; @@ -2381,25 +2417,25 @@ impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { } } -impl Hash for SmallVec { +impl Hash for SmallVec { fn hash(&self, state: &mut H) { self.as_slice().hash(state) } } -impl Debug for SmallVec { +impl Debug for SmallVec { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_list().entries(self.iter()).finish() } } -impl Debug for IntoIter { +impl Debug for IntoIter { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_tuple("IntoIter").field(&self.as_slice()).finish() } } -impl Debug for Drain<'_, T, N> { +impl Debug for Drain<'_, T, N, A> { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() } diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index 4d279cd..1a130f5 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -1,6 +1,8 @@ use { super::{ + Allocator, CollectionAllocErr, + infallible, taggedlen::TaggedLen }, core::{ @@ -16,44 +18,48 @@ use { } }; +#[repr(C)] +pub(crate) union RawSmallVecInner { + pub(crate) inline: ManuallyDrop>, + pub(crate) heap: (NonNull, usize) +} + /// Either a stack array with `length <= N` or a heap array /// whose pointer and capacity are stored here. /// /// We store a `NonNull` instead of a `*mut T` so that type is covariant /// with respect to `T`, and since the heap pointer is never null. -#[repr(C)] -pub union RawSmallVec { - pub inline: ManuallyDrop>, - pub heap: (NonNull, usize) -} - -impl Default for RawSmallVec { - #[inline] - fn default() -> Self { - Self::new() - } +pub struct RawSmallVec { + pub(crate) inner: RawSmallVecInner, + pub(crate) alloc: A } -impl RawSmallVec { +impl RawSmallVec { pub const INLINE_CAP: usize = if Self::IS_ZST { usize::MAX } else { N }; const IS_ZST: bool = size_of::() == 0; #[inline] - pub const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) + pub const fn new(alloc: A) -> Self { + Self::new_inline(MaybeUninit::uninit(), alloc) } #[inline] - pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + pub const fn new_inline(inline: MaybeUninit<[T; N]>, alloc: A) -> Self { Self { - inline: ManuallyDrop::new(inline) + inner: RawSmallVecInner { + inline: ManuallyDrop::new(inline) + }, + alloc } } #[inline] - pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + pub const fn new_heap(ptr: NonNull, capacity: usize, alloc: A) -> Self { Self { - heap: (ptr, capacity) + inner: RawSmallVecInner { + heap: (ptr, capacity) + }, + alloc } } @@ -63,14 +69,14 @@ impl RawSmallVec { // a reference to it. reading it would be UB potentially, but // for that downstream unsafe is required #[allow(unused_unsafe, reason = "Unsafe in MSRV")] - (unsafe { &raw const self.inline }).cast() + (unsafe { &raw const self.inner.inline }).cast() } #[inline] pub const fn as_mut_ptr_inline(&mut self) -> *mut T { // SAFETY: same as above #[allow(unused_unsafe, reason = "Unsafe in MSRV")] - (unsafe { &raw mut self.inline }).cast() + (unsafe { &raw mut self.inner.inline }).cast() } /// # Safety @@ -79,7 +85,7 @@ impl RawSmallVec { #[inline(always)] pub const unsafe fn as_ptr(&self, on_heap: bool) -> *const T { if on_heap { - unsafe { self.heap.0.as_ptr() } + unsafe { self.inner.heap.0.as_ptr() } } else { self.as_ptr_inline() } @@ -91,7 +97,7 @@ impl RawSmallVec { #[inline(always)] pub const unsafe fn as_mut_ptr(&mut self, on_heap: bool) -> *mut T { if on_heap { - unsafe { self.heap.0.as_ptr() } + unsafe { self.inner.heap.0.as_ptr() } } else { self.as_mut_ptr_inline() } @@ -103,7 +109,7 @@ impl RawSmallVec { #[inline(always)] pub const unsafe fn capacity(&self, on_heap: bool) -> usize { if on_heap { - unsafe { self.heap.1 } + unsafe { self.inner.heap.1 } } else { Self::INLINE_CAP } @@ -118,10 +124,6 @@ impl RawSmallVec { len: TaggedLen, new_capacity: usize ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{ - alloc, - realloc - }; let (len, was_on_heap) = len.parts(); debug_assert!(!Self::IS_ZST); debug_assert!(new_capacity > 0 && new_capacity >= len); @@ -137,19 +139,26 @@ impl RawSmallVec { let new_ptr = if !was_on_heap { // get a fresh allocation - let new_ptr = unsafe { alloc(new_layout) } as *mut T; // `new_layout` has nonzero size. - let new_ptr = NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { - layout: new_layout - })?; + // `new_layout` has nonzero size. + let new_ptr = self + .alloc + .allocate(new_layout) + .map_err(|_| CollectionAllocErr::AllocErr { + layout: new_layout + })? + .cast(); unsafe { copy_nonoverlapping(ptr, new_ptr.as_ptr(), len) }; new_ptr } else { - // use realloc + // use grow // this can't overflow since we already constructed an equivalent // layout during the previous allocation let old_layout = unsafe { - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()) + Layout::from_size_align_unchecked( + self.inner.heap.1 * size_of::(), + align_of::() + ) }; // SAFETY: ptr was allocated with this allocator @@ -158,13 +167,56 @@ impl RawSmallVec { // than zero does not overflow when rounded up to // alignment. since it was constructed // with Layout::array - let new_ptr = - unsafe { realloc(ptr as *mut u8, old_layout, new_layout.size()) } as *mut T; - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { + unsafe { + self.alloc.grow( + NonNull::new(ptr as *mut u8).unwrap(), + old_layout, + new_layout + ) + } + .map_err(|_| CollectionAllocErr::AllocErr { layout: new_layout })? + .cast() }; - *self = Self::new_heap(new_ptr, new_capacity); + self.inner.heap = (new_ptr, new_capacity); Ok(()) } + + /// # Safety + /// + /// `new_capacity` must be non zero, and smaller or equal to the current + /// one. T must not be a ZST. Items must be stored on the heap. + pub unsafe fn shrink_to_raw(&mut self, target: usize) { + unsafe { + // this can't overflow since it's smaller than one we already made + let new_layout = + Layout::from_size_align_unchecked(target * size_of::(), align_of::()); + + self.inner.heap = ( + infallible( + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to + // allocate the previous + // memory block + self.alloc + .shrink( + NonNull::new(self.inner.heap.0.as_ptr() as *mut u8).unwrap(), + // this can't overflow since we already constructed an equivalent + // layout during the previous allocation + Layout::from_size_align_unchecked( + self.inner.heap.1 * size_of::(), + align_of::() + ), + new_layout + ) + .map_err(|_| CollectionAllocErr::AllocErr { + layout: new_layout + }) + ) + .cast(), + target + ); + } + } } diff --git a/src/references.rs b/src/references.rs index a266be4..0582ea1 100644 --- a/src/references.rs +++ b/src/references.rs @@ -1,5 +1,8 @@ use { - super::SmallVec, + super::{ + Allocator, + SmallVec + }, core::{ borrow::{ Borrow, @@ -12,33 +15,33 @@ use { } }; -impl Borrow<[T]> for SmallVec { +impl Borrow<[T]> for SmallVec { #[inline] fn borrow(&self) -> &[T] { self.as_slice() } } -impl BorrowMut<[T]> for SmallVec { +impl BorrowMut<[T]> for SmallVec { #[inline] fn borrow_mut(&mut self) -> &mut [T] { self.as_mut_slice() } } -impl AsRef<[T]> for SmallVec { +impl AsRef<[T]> for SmallVec { #[inline] fn as_ref(&self) -> &[T] { self.as_slice() } } -impl AsMut<[T]> for SmallVec { +impl AsMut<[T]> for SmallVec { #[inline] fn as_mut(&mut self) -> &mut [T] { self.as_mut_slice() } } -impl Deref for SmallVec { +impl Deref for SmallVec { type Target = [T]; #[inline] @@ -46,7 +49,7 @@ impl Deref for SmallVec { self.as_slice() } } -impl DerefMut for SmallVec { +impl DerefMut for SmallVec { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.as_mut_slice() diff --git a/src/specialization.rs b/src/specialization.rs index f02f23b..a489e1f 100644 --- a/src/specialization.rs +++ b/src/specialization.rs @@ -1,5 +1,6 @@ use { crate::{ + Allocator, DropGuard, IntoIter, SmallVec @@ -62,7 +63,7 @@ pub trait SpecExtend { fn spec_extend(&mut self, iter: I); } -impl SpecExtend for SmallVec +impl SpecExtend for SmallVec where I: Iterator { #[inline] @@ -71,7 +72,7 @@ where I: Iterator } } -impl SpecExtend for SmallVec +impl SpecExtend for SmallVec where I: core::iter::TrustedLen { fn spec_extend(&mut self, iter: I) { @@ -104,8 +105,10 @@ where I: core::iter::TrustedLen } } -impl SpecExtend> for SmallVec { - fn spec_extend(&mut self, mut iter: IntoIter) { +impl SpecExtend> + for SmallVec +{ + fn spec_extend(&mut self, mut iter: IntoIter) { let slice = iter.as_slice(); let len = slice.len(); let old_len = self.len(); @@ -130,7 +133,7 @@ impl SpecExtend> for SmallV } } -impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec +impl<'a, T: 'a, const N: usize, I, A: Allocator> SpecExtend<&'a T, I> for SmallVec where I: Iterator, T: Clone @@ -141,7 +144,8 @@ where } } -impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec +impl<'a, T: 'a, const N: usize, A: Allocator> SpecExtend<&'a T, core::slice::Iter<'a, T>> + for SmallVec where T: Copy { fn spec_extend(&mut self, iter: core::slice::Iter<'a, T>) { @@ -182,7 +186,7 @@ pub trait SpecExtendFromWithin { unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); } -impl SpecExtendFromWithin for SmallVec { +impl SpecExtendFromWithin for SmallVec { default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { // SAFETY: Safety conditions are identical. unsafe { @@ -191,7 +195,7 @@ impl SpecExtendFromWithin for SmallVec { } } -impl SpecExtendFromWithin for SmallVec { +impl SpecExtendFromWithin for SmallVec { unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { let old_len = self.len(); @@ -256,14 +260,14 @@ pub trait SpecCloneFrom { fn spec_clone_from(&mut self, source: &[T]); } -impl SpecCloneFrom for SmallVec { +impl SpecCloneFrom for SmallVec { #[inline] default fn spec_clone_from(&mut self, source: &[T]) { self.clone_from_fallback(source); } } -impl SpecCloneFrom for SmallVec { +impl SpecCloneFrom for SmallVec { fn spec_clone_from(&mut self, source: &[T]) { self.clear(); self.extend_from_slice(source);