diff --git a/src/lib.rs b/src/lib.rs index f907444..11fe180 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -615,8 +615,13 @@ impl IntoIter { // `self.begin..end` are all initialized. So the pointer arithmetic is // valid, and so is the construction of the slice unsafe { - let ptr = self.raw.as_ptr(on_heap); - core::slice::from_raw_parts(ptr.add(self.begin), end - self.begin) + let ptr = if on_heap { + self.raw.as_heap() + } else { + self.raw.as_inline() + } + .as_ptr(); + core::slice::from_raw_parts(ptr.add(self.begin).cast(), end - self.begin) } } @@ -625,8 +630,13 @@ impl IntoIter { let (end, on_heap) = self.end.parts(); // SAFETY: see above unsafe { - let ptr = self.raw.as_mut_ptr(on_heap); - core::slice::from_raw_parts_mut(ptr.add(self.begin), end - self.begin) + let ptr = if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + .as_mut_ptr(); + core::slice::from_raw_parts_mut(ptr.add(self.begin).cast(), end - self.begin) } } } @@ -642,8 +652,12 @@ impl Iterator for IntoIter { } else { // SAFETY: see above unsafe { - let ptr = self.raw.as_mut_ptr(on_heap); - let value = ptr.add(self.begin).read(); + let reference = if on_heap { + self.raw.as_heap() + } else { + self.raw.as_inline() + }; + let value = reference[self.begin].assume_init_read(); self.begin += 1; Some(value) } @@ -666,9 +680,13 @@ impl DoubleEndedIterator for IntoIter { } else { // SAFETY: see above unsafe { - let ptr = self.raw.as_mut_ptr(on_heap); + let reference = if on_heap { + self.raw.as_heap() + } else { + self.raw.as_inline() + }; self.end.sub(1); - let value = ptr.add(end - 1).read(); + let value = reference[end - 1].assume_init_read(); Some(value) } } @@ -713,7 +731,7 @@ impl SmallVec { // Although we create a new buffer, since S and N are known at compile // time, even with `-C opt-level=1`, it gets optimized as best // as it could be. (Checked with ) - let mut buf: MaybeUninit<[T; N]> = MaybeUninit::uninit(); + let mut buf: [MaybeUninit; N] = [const { MaybeUninit::uninit() }; N]; // SAFETY: buf and elements do not overlap, are aligned and have space // for at least S elements since S <= N. @@ -739,22 +757,16 @@ impl SmallVec { // SAFETY: all the members in 0..len are initialized let mut vec = Self { len: TaggedLen::new(len, false), - raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), + raw: RawSmallVec::new_inline(buf.map(MaybeUninit::new)), _marker: PhantomData }; // Deallocate the remaining elements so no memory is leaked. unsafe { - // SAFETY: both the input and output pointers are in range of the - // stack allocation - let remainder_ptr = vec.raw.as_mut_ptr_inline().add(len); - let remainder_len = N - len; - - // SAFETY: the values are initialized, so dropping them here is - // fine. - core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( - remainder_ptr, - remainder_len - )); + core::ptr::slice_from_raw_parts_mut( + vec.raw.as_mut_inline().as_mut_ptr().add(len), + N - len + ) + .drop_in_place(); } vec @@ -782,7 +794,7 @@ impl SmallVec { /// /// `len <= N`, and all the elements in `buf[..len]` must be initialized #[inline] - pub const unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit<[T; N]>, len: usize) -> Self { + pub const unsafe fn from_buf_and_len_unchecked(buf: [MaybeUninit; N], len: usize) -> Self { debug_assert!(len <= N); Self { len: TaggedLen::new(len, false), @@ -825,11 +837,16 @@ impl SmallVec { let cap = vec.capacity(); // SAFETY: vec.capacity is not `0` (checked above), so the pointer // can not dangle and thus specifically cannot be null. - let ptr = unsafe { NonNull::new_unchecked(vec.as_mut_ptr()) }; + let ptr = unsafe { + NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut( + vec.as_mut_ptr().cast(), + cap + )) + }; Self { len: TaggedLen::new(len, true), - raw: RawSmallVec::new_heap(ptr, cap), + raw: RawSmallVec::new_heap(ptr), _marker: PhantomData } } @@ -1197,26 +1214,25 @@ impl SmallVec { unsafe { self.set_on_heap() }; } result - } else { + } else if on_heap { // new_capacity <= Self::inline_size() - if on_heap { - unsafe { - // SAFETY: heap member is active - let (ptr, old_cap) = self.raw.heap; - // inline member is now active - - // SAFETY: len <= new_capacity <= Self::inline_size() - // so the copy is within bounds of the inline member - copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); - drop(DropDealloc { - ptr: ptr.cast(), - size_bytes: old_cap * size_of::(), - align: align_of::() - }); - self.set_inline(); - } + unsafe { + // SAFETY: heap member is active + let ptr = self.raw.heap; + // inline member is now active + // SAFETY: len <= new_capacity <= Self::inline_size() + // so the copy is within bounds of the inline member + copy_nonoverlapping(ptr.as_ptr().cast(), self.raw.as_mut_inline(), len); + drop(DropDealloc { + ptr: ptr.cast(), + size_bytes: ptr.len() * size_of::(), + align: align_of::() + }); + self.set_inline(); } Ok(()) + } else { + Ok(()) } } @@ -1283,13 +1299,14 @@ impl SmallVec { if len <= Self::inline_size() { // SAFETY: on_heap is true, so we're on the heap unsafe { - let (ptr, capacity) = self.raw.heap; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); - copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); + // let (ptr, capacity) = self.raw.heap; + let ptr = self.raw.heap; + self.raw = RawSmallVec::new(); + copy_nonoverlapping(ptr.as_ptr().cast(), self.raw.as_mut_inline(), len); self.set_inline(); alloc::alloc::dealloc( ptr.cast().as_ptr(), - Layout::from_size_align_unchecked(capacity * size_of::(), align_of::()) + Layout::from_size_align_unchecked(ptr.len() * size_of::(), align_of::()) ); } } else if len < self.capacity() { @@ -1307,25 +1324,22 @@ impl SmallVec { return; } // SAFETY: the vector is on the heap - let capacity = unsafe { self.raw.heap.1 }; - if capacity > min_capacity { + let ptr = unsafe { self.raw.heap }; + let cap = ptr.len(); + if cap > min_capacity { let target = core::cmp::max(len, min_capacity); if target <= Self::inline_size() { // SAFETY: on_heap is true, so we're on the heap unsafe { - let (ptr, capacity) = self.raw.heap; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); - copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); + self.raw = RawSmallVec::new(); + copy_nonoverlapping(ptr.as_ptr().cast(), self.raw.as_mut_inline(), len); self.set_inline(); alloc::alloc::dealloc( ptr.cast().as_ptr(), - Layout::from_size_align_unchecked( - capacity * size_of::(), - align_of::() - ) + Layout::from_size_align_unchecked(cap * size_of::(), align_of::()) ); } - } else if target < capacity { + } else if target < cap { // SAFETY: len > Self::inline_size() >= 0 // so new capacity is non zero, it is equal to the length // T can't be a ZST because SmallVec is never spilled. @@ -1476,26 +1490,59 @@ impl SmallVec { pub const fn as_slice(&self) -> &[T] { let (len, on_heap) = self.len.parts(); // SAFETY: all the elements in `..len` are initialized - unsafe { core::slice::from_raw_parts(self.raw.as_ptr(on_heap), len) } + // unsafe { core::slice::from_raw_parts(self.raw.as_ptr(on_heap), len) } + unsafe { + let ptr = if on_heap { + self.raw.as_heap() + } else { + self.raw.as_inline() + } + .as_ptr(); + core::slice::from_raw_parts(ptr.cast(), len) + } } #[inline] pub const fn as_mut_slice(&mut self) -> &mut [T] { let (len, on_heap) = self.len.parts(); // SAFETY: see above - unsafe { core::slice::from_raw_parts_mut(self.raw.as_mut_ptr(on_heap), len) } + unsafe { + let ptr = if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + .as_mut_ptr(); + core::slice::from_raw_parts_mut(ptr.cast(), len) + } } #[inline] pub const fn as_ptr(&self) -> *const T { // SAFETY: the tag tells which member is active - unsafe { self.raw.as_ptr(self.len.on_heap()) } + unsafe { + if self.spilled() { + self.raw.as_heap() + } else { + self.raw.as_inline() + } + .as_ptr() + .cast() + } } #[inline] pub const fn as_mut_ptr(&mut self) -> *mut T { // SAFETY: see above - unsafe { self.raw.as_mut_ptr(self.len.on_heap()) } + unsafe { + if self.spilled() { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + .as_mut_ptr() + .cast() + } } #[inline] @@ -1509,7 +1556,7 @@ impl SmallVec { // the length we don't drop the elements we previously // held unsafe { - copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); + copy_nonoverlapping(this.raw.as_inline().as_ptr().cast(), vec.as_mut_ptr(), len); vec.set_len(len); } vec @@ -1523,8 +1570,8 @@ impl SmallVec { // - the first `len` entries are proper `T`-values // - the allocation is not larger than `isize::MAX` unsafe { - let (ptr, cap) = this.raw.heap; - Vec::from_raw_parts(ptr.as_ptr(), len, cap) + let ptr = this.raw.heap; + Vec::from_raw_parts(ptr.as_ptr().cast(), len, ptr.len()) } } } @@ -1726,7 +1773,7 @@ impl SmallVec { ); } let mut me = ManuallyDrop::new(self); - unsafe { core::slice::from_raw_parts_mut(me.raw.as_mut_ptr(true), len) } + unsafe { core::slice::from_raw_parts_mut(me.raw.as_mut_heap().as_mut_ptr().cast(), len) } } /// Returns the remaining spare capacity of the vector as a slice of @@ -1740,10 +1787,13 @@ impl SmallVec { let (len, on_heap) = self.len.parts(); unsafe { let capacity = self.raw.capacity(on_heap); - core::slice::from_raw_parts_mut( - self.raw.as_mut_ptr(on_heap).add(len) as *mut MaybeUninit, - capacity - len - ) + let ptr = if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + .as_mut_ptr(); + core::slice::from_raw_parts_mut(ptr.add(len) as *mut MaybeUninit, capacity - len) } } @@ -1826,7 +1876,7 @@ impl SmallVec { SmallVec { len: TaggedLen::new(length, true), - raw: RawSmallVec::new_heap(ptr, capacity), + raw: RawSmallVec::new_heap(NonNull::slice_from_raw_parts(ptr.cast(), capacity)), _marker: PhantomData } } @@ -1991,12 +2041,20 @@ impl Drop for DropDealloc { unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { fn drop(&mut self) { let (len, on_heap) = self.len.parts(); - let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; + // let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; + let ptr = unsafe { + if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + } + .as_mut_ptr(); // SAFETY: we first drop the elements, then `_drop_dealloc` is dropped, // releasing memory we used to own unsafe { let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.heap.len(); Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), @@ -2015,11 +2073,18 @@ impl Drop for SmallVec { fn drop(&mut self) { let (len, on_heap) = self.len.parts(); // SAFETY: the tag tells which member is active - let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; + let ptr = unsafe { + if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + } + .as_mut_ptr(); // SAFETY: see above unsafe { let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.heap.len(); Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), @@ -2039,9 +2104,14 @@ impl Drop for IntoIter { unsafe { let (end, on_heap) = self.end.parts(); let begin = self.begin; - let ptr = self.raw.as_mut_ptr(on_heap); + let ptr = if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + .as_mut_ptr(); let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.heap.len(); Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), @@ -2094,7 +2164,7 @@ impl SmallVec { let mut result = Self::new(); if n > 0 { - let ptr = result.raw.as_mut_ptr_inline(); + let ptr = unsafe { result.raw.as_mut_inline().as_mut_ptr().cast::() }; let mut guard = DropGuard { ptr, len: 0 @@ -2483,7 +2553,15 @@ unsafe impl BufMut for SmallVec { let (len, on_heap) = self.len.parts(); // SAFETY: the tag tells which member is active let cap = unsafe { self.raw.capacity(on_heap) }; - let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; + let ptr = unsafe { + if on_heap { + self.raw.as_mut_heap() + } else { + self.raw.as_mut_inline() + } + } + .as_mut_ptr() + .cast::(); // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be // valid for `cap - len` bytes. The subtraction will not underflow since // `len <= cap`. diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index 4d279cd..c43fe81 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -3,6 +3,10 @@ use { CollectionAllocErr, taggedlen::TaggedLen }, + alloc::alloc::{ + alloc, + realloc + }, core::{ alloc::Layout, mem::{ @@ -23,8 +27,8 @@ use { /// with respect to `T`, and since the heap pointer is never null. #[repr(C)] pub union RawSmallVec { - pub inline: ManuallyDrop>, - pub heap: (NonNull, usize) + pub inline: ManuallyDrop<[MaybeUninit; N]>, + pub heap: NonNull<[MaybeUninit]> } impl Default for RawSmallVec { @@ -40,61 +44,71 @@ impl RawSmallVec { #[inline] pub const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) + Self::new_inline([const { MaybeUninit::uninit() }; N]) } #[inline] - pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + pub const fn new_inline(inline: [MaybeUninit; N]) -> Self { Self { inline: ManuallyDrop::new(inline) } } #[inline] - pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + pub const fn new_heap(ptr: NonNull<[MaybeUninit]>) -> Self { Self { - heap: (ptr, capacity) + heap: ptr } } + /// # Safety + /// + /// `inline` must be the active variant + /// otherwise it reads pointer as elements #[inline] - pub const fn as_ptr_inline(&self) -> *const T { + pub const unsafe fn as_inline(&self) -> &[MaybeUninit; N] { // SAFETY: it is safe because we aren't reading the value, just getting // a reference to it. reading it would be UB potentially, but // for that downstream unsafe is required - #[allow(unused_unsafe, reason = "Unsafe in MSRV")] - (unsafe { &raw const self.inline }).cast() + unsafe { + (&raw const self.inline) + .cast::<[MaybeUninit; N]>() + .as_ref() + .unwrap_unchecked() + } } + /// # Safety + /// + /// `inline` must be the active variant + /// otherwise it reads pointer as elements #[inline] - pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + pub const unsafe fn as_mut_inline(&mut self) -> &mut [MaybeUninit; N] { // SAFETY: same as above - #[allow(unused_unsafe, reason = "Unsafe in MSRV")] - (unsafe { &raw mut self.inline }).cast() + unsafe { + (&raw mut self.inline) + .cast::<[MaybeUninit; N]>() + .as_mut() + .unwrap_unchecked() + } } /// # Safety /// - /// `on_heap` must be true if and only if `self.heap` is the active member. - #[inline(always)] - pub const unsafe fn as_ptr(&self, on_heap: bool) -> *const T { - if on_heap { - unsafe { self.heap.0.as_ptr() } - } else { - self.as_ptr_inline() - } + /// `heap` must be the active variant + /// otherwise it reads inlined elements as pointer + #[inline] + pub const unsafe fn as_heap(&self) -> &[MaybeUninit] { + unsafe { self.heap.as_ref() } } /// # Safety /// - /// `on_heap` must be true if and only if `self.heap` is the active member. - #[inline(always)] - pub const unsafe fn as_mut_ptr(&mut self, on_heap: bool) -> *mut T { - if on_heap { - unsafe { self.heap.0.as_ptr() } - } else { - self.as_mut_ptr_inline() - } + /// `heap` must be the active variant + /// otherwise it reads inlined elements as pointer + #[inline] + pub const unsafe fn as_mut_heap(&mut self) -> &mut [MaybeUninit] { + unsafe { self.heap.as_mut() } } /// # Safety @@ -103,7 +117,7 @@ impl RawSmallVec { #[inline(always)] pub const unsafe fn capacity(&self, on_heap: bool) -> usize { if on_heap { - unsafe { self.heap.1 } + unsafe { self.as_heap().len() } } else { Self::INLINE_CAP } @@ -118,16 +132,17 @@ impl RawSmallVec { len: TaggedLen, new_capacity: usize ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{ - alloc, - realloc - }; let (len, was_on_heap) = len.parts(); debug_assert!(!Self::IS_ZST); debug_assert!(new_capacity > 0 && new_capacity >= len); // SAFETY: the tag tells which member is active - let ptr = unsafe { self.as_mut_ptr(was_on_heap) }; + let ptr = if was_on_heap { + unsafe { self.as_mut_heap() } + } else { + unsafe { self.as_mut_inline() } + } + .as_mut_ptr(); let new_layout = Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; @@ -137,11 +152,12 @@ impl RawSmallVec { let new_ptr = if !was_on_heap { // get a fresh allocation - let new_ptr = unsafe { alloc(new_layout) } as *mut T; // `new_layout` has nonzero size. + + let new_ptr = unsafe { alloc(new_layout) } as *mut MaybeUninit; let new_ptr = NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; - unsafe { copy_nonoverlapping(ptr, new_ptr.as_ptr(), len) }; + unsafe { copy_nonoverlapping(ptr.cast(), new_ptr.as_ptr(), len) }; new_ptr } else { // use realloc @@ -149,7 +165,7 @@ impl RawSmallVec { // this can't overflow since we already constructed an equivalent // layout during the previous allocation let old_layout = unsafe { - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()) + Layout::from_size_align_unchecked(self.heap.len() * size_of::(), align_of::()) }; // SAFETY: ptr was allocated with this allocator @@ -158,13 +174,13 @@ impl RawSmallVec { // than zero does not overflow when rounded up to // alignment. since it was constructed // with Layout::array - let new_ptr = - unsafe { realloc(ptr as *mut u8, old_layout, new_layout.size()) } as *mut T; + let new_ptr = unsafe { realloc(ptr.cast(), old_layout, new_layout.size()) } + as *mut MaybeUninit; NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? }; - *self = Self::new_heap(new_ptr, new_capacity); + *self = Self::new_heap(NonNull::slice_from_raw_parts(new_ptr, new_capacity)); Ok(()) } } diff --git a/src/specialization.rs b/src/specialization.rs index f02f23b..c9157c5 100644 --- a/src/specialization.rs +++ b/src/specialization.rs @@ -33,13 +33,14 @@ impl SpecFromElem for SmallVec { let mut result = Self::new(); if n > 0 { - let ptr = result.raw.as_mut_ptr_inline(); - - // SAFETY: The caller ensures that the first `n` - // is smaller than the inline size. - unsafe { - for i in 0..n { - ptr.add(i).write(elem); + // SAFETY: the active variant is `inline` + let inline = unsafe { result.raw.as_mut_inline() }; + + for i in 0..n { + // SAFETY: The caller ensures that the first `n` + // is smaller than the inline size. + unsafe { + inline.get_unchecked_mut(i).write(elem); } } }