diff --git a/.travis.yml b/.travis.yml index ee744e1..b676984 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,11 @@ language: rust rust: - - 1.20.0 + - 1.36.0 - nightly - beta - stable -script: | - cargo build --verbose && - cargo test --verbose && - cargo test --verbose --features serde && - ([ $TRAVIS_RUST_VERSION != nightly ] || cargo check --verbose --no-default-features) && - ([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --features union) && - ([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --all-features) && - ([ $TRAVIS_RUST_VERSION != nightly ] || cargo bench --verbose bench) +script: + - cargo test + - cargo test --features serde + - "[ $TRAVIS_RUST_VERSION != nightly ] || cargo test --all-features" + - "[ $TRAVIS_RUST_VERSION != nightly ] || cargo bench" diff --git a/Cargo.toml b/Cargo.toml index fcea83c..fca9deb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "smallvec" -version = "0.6.10" -authors = ["Simon Sapin "] +version = "1.0.0" +authors = ["The Servo Project Developers"] license = "MIT/Apache-2.0" repository = "https://github.com/servo/rust-smallvec" description = "'Small vector' optimization: store up to a small number of items on the stack" @@ -11,15 +11,10 @@ readme = "README.md" documentation = "https://doc.servo.org/smallvec/" [features] -std = [] -union = [] -default = ["std"] -specialization = [] may_dangle = [] [lib] name = "smallvec" -path = "lib.rs" [dependencies] serde = { version = "1", optional = true } diff --git a/src/array.rs b/src/array.rs new file mode 100644 index 0000000..b9aa691 --- /dev/null +++ b/src/array.rs @@ -0,0 +1,30 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +/// Types that can be used as the backing store for a SmallVec +pub unsafe trait Array { + /// The type of the array's elements. + type Item; + /// Returns the number of items the array can hold. + const LEN: usize; +} + +macro_rules! impl_array( + ($($len:expr),+) => { + $( + unsafe impl Array for [T; $len] { + type Item = T; + const LEN: usize = $len; + } + )+ + } +); + +impl_array!( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 32, 36, 0x40, 0x80, 0x100, + 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, + 0x100000 +); diff --git a/src/heap.rs b/src/heap.rs new file mode 100644 index 0000000..149d4fc --- /dev/null +++ b/src/heap.rs @@ -0,0 +1,59 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use self::TryReserveError::CapacityOverflow as CO; +use alloc::alloc::{self, Layout}; +use core::mem; +use core::ptr::NonNull; + +fn array_layout(capacity: usize) -> Result { + let size = mem::size_of::().checked_mul(capacity).ok_or(CO)?; + if size > crate::repr::MAX_LEN { + return Err(CO); + } + Layout::from_size_align(size, mem::align_of::()).map_err(|_| CO) +} + +pub(crate) unsafe fn alloc_array(capacity: usize) -> Result, TryReserveError> { + let layout = array_layout::(capacity)?; + let ptr = alloc::alloc(layout); + NonNull::new(ptr) + .map(NonNull::cast) + .ok_or(TryReserveError::AllocError { layout }) +} + +pub(crate) unsafe fn realloc_array( + ptr: NonNull, + old: usize, + new: usize, +) -> Result, TryReserveError> { + let old = array_layout::(old)?; + let new = array_layout::(new)?; + let ptr = alloc::realloc(ptr.cast().as_ptr(), old, new.size()); + NonNull::new(ptr) + .map(NonNull::cast) + .ok_or(TryReserveError::AllocError { layout: new }) +} + +pub(crate) unsafe fn dealloc_array(ptr: NonNull, capacity: usize) { + let layout = array_layout::(capacity).unwrap(); + alloc::dealloc(ptr.cast().as_ptr(), layout) +} + +#[derive(Debug)] +pub enum TryReserveError { + CapacityOverflow, + AllocError { layout: Layout }, +} + +impl TryReserveError { + pub(crate) fn bail(&self) -> ! { + match *self { + TryReserveError::CapacityOverflow => panic!("SmallVec capacity overflow"), + TryReserveError::AllocError { layout } => alloc::handle_alloc_error(layout), + } + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..295ad20 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,232 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![no_std] +#![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))] + +extern crate alloc; + +mod array; +mod heap; +mod repr; +mod tagged_union; + +pub use array::Array; +pub use heap::TryReserveError; +pub use repr::SmallVec; + +use alloc::vec::Vec; + +impl SmallVec { + const INLINE_CAPACITY: usize = A::LEN; + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn capacity(&self) -> usize { + match self.as_inline() { + Ok(_) => Self::INLINE_CAPACITY, + Err(heap) => heap.capacity, + } + } + + pub fn as_ptr(&self) -> *const A::Item { + match self.as_inline() { + Ok(inline) => inline.as_ptr(), + Err(heap) => heap.ptr.as_ptr(), + } + } + + pub fn as_mut_ptr(&mut self) -> *mut A::Item { + match self.as_inline_mut() { + Ok(inline) => inline.as_mut_ptr(), + Err(heap) => heap.ptr.as_ptr(), + } + } + + pub fn as_slice(&self) -> &[A::Item] { + unsafe { core::slice::from_raw_parts(self.as_ptr(), self.len()) } + } + + pub fn as_mut_slice(&mut self) -> &mut [A::Item] { + unsafe { core::slice::from_raw_parts_mut(self.as_mut_ptr(), self.len()) } + } + + pub fn shrink_to_fit(&mut self) { + self.shrink_to(0) + } + + pub fn shrink_to(&mut self, new_capacity: usize) { + let len = self.len(); + let new_capacity = new_capacity.max(len); + if let Err(heap) = self.as_inline_mut() { + if new_capacity <= Self::INLINE_CAPACITY { + let ptr = heap.ptr; + let capacity = heap.capacity; + // Do accesses to `heap` above here, so we can borrow `self` again below: + unsafe { + let inline = self.set_inline_tag(len); + core::ptr::copy_nonoverlapping(ptr.as_ptr(), inline.as_mut_ptr(), len); + heap::dealloc_array(ptr, capacity) + } + } else if new_capacity < heap.capacity { + let new_ptr = unsafe { + heap::realloc_array(heap.ptr, heap.capacity, new_capacity) + .unwrap_or_else(|err| err.bail()) + }; + heap.ptr = new_ptr; + heap.capacity = new_capacity; + } + } + } + + fn try_reserve_internal(&mut self, extra: usize, exact: bool) -> Result<(), TryReserveError> { + let len = self.len(); + let capacity = self.capacity(); + let requested_capacity = len + .checked_add(extra) + .ok_or(TryReserveError::CapacityOverflow)?; + if requested_capacity <= capacity { + return Ok(()); + } + let new_capacity = if exact { + requested_capacity + } else { + requested_capacity.max(capacity * 2) + }; + unsafe { + match self.as_inline_mut() { + Ok(inline) => { + let new_ptr = heap::alloc_array(new_capacity)?; + core::ptr::copy_nonoverlapping(inline.as_ptr(), new_ptr.as_ptr(), len); + let heap = self.set_heap_tag(len); + heap.ptr = new_ptr; + heap.capacity = new_capacity; + } + Err(heap) => { + let new_ptr = heap::realloc_array(heap.ptr, capacity, new_capacity)?; + heap.ptr = new_ptr; + heap.capacity = new_capacity; + } + } + } + Ok(()) + } + + pub fn reserve(&mut self, extra: usize) { + self.try_reserve_internal(extra, false) + .unwrap_or_else(|err| err.bail()) + } + + pub fn reserve_exact(&mut self, extra: usize) { + self.try_reserve_internal(extra, true) + .unwrap_or_else(|err| err.bail()) + } + + pub fn try_reserve(&mut self, extra: usize) -> Result<(), TryReserveError> { + self.try_reserve_internal(extra, false) + } + + pub fn try_reserve_exact(&mut self, extra: usize) -> Result<(), TryReserveError> { + self.try_reserve_internal(extra, true) + } + + pub fn push(&mut self, value: A::Item) { + self.reserve(1); + let len = self.len(); + unsafe { + self.as_mut_ptr().add(len).write(value); + self.set_len(len + 1) + } + } + + pub fn clear(&mut self) { + self.truncate(0) + } + + pub fn truncate(&mut self, new_len: usize) { + if let Some(to_drop) = self.get_mut(new_len..) { + let to_drop: *mut [A::Item] = to_drop; + unsafe { + self.set_len(new_len); + to_drop.drop_in_place() + } + } + } +} + +#[cfg(feature = "may_dangle")] +unsafe impl<#[may_dangle] A: Array> Drop for SmallVec { + fn drop(&mut self) { + drop_impl(self) + } +} + +#[cfg(not(feature = "may_dangle"))] +impl Drop for SmallVec { + fn drop(&mut self) { + drop_impl(self) + } +} + +fn drop_impl(s: &mut SmallVec) { + unsafe { + core::ptr::drop_in_place(s.as_mut_slice()); + match s.as_inline() { + Ok(_) => {} + Err(heap) => heap::dealloc_array(heap.ptr, heap.capacity), + } + } +} + +impl core::ops::Deref for SmallVec { + type Target = [A::Item]; + fn deref(&self) -> &[A::Item] { + self.as_slice() + } +} + +impl core::ops::DerefMut for SmallVec { + fn deref_mut(&mut self) -> &mut [A::Item] { + self.as_mut_slice() + } +} + +impl core::ops::Index for SmallVec +where + I: core::slice::SliceIndex<[A::Item]>, +{ + type Output = I::Output; + fn index(&self, index: I) -> &I::Output { + &self.as_slice()[index] + } +} + +impl core::ops::IndexMut for SmallVec +where + I: core::slice::SliceIndex<[A::Item]>, +{ + fn index_mut(&mut self, index: I) -> &mut I::Output { + &mut self.as_mut_slice()[index] + } +} + +impl From> for SmallVec { + fn from(mut vec: Vec) -> Self { + let ptr = vec.as_mut_ptr(); + let len = vec.len(); + let cap = vec.capacity(); + core::mem::forget(vec); + let mut s = Self::new(); + unsafe { + let heap = s.set_heap_tag(len); + heap.ptr = core::ptr::NonNull::new_unchecked(ptr); + heap.capacity = cap; + } + s + } +} diff --git a/src/repr.rs b/src/repr.rs new file mode 100644 index 0000000..8da73f9 --- /dev/null +++ b/src/repr.rs @@ -0,0 +1,91 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use crate::array::Array; +use crate::tagged_union::TaggedUnion2; +use core::mem::MaybeUninit; + +pub struct SmallVec { + tagged_union: TaggedUnion2, HeapData>, +} + +pub(crate) struct InlineData { + pub storage: MaybeUninit, +} + +pub(crate) struct HeapData { + pub capacity: usize, + pub ptr: core::ptr::NonNull, +} + +impl InlineData { + pub(crate) fn as_ptr(&self) -> *const A::Item { + self.storage.as_ptr() as _ + } + + pub(crate) fn as_mut_ptr(&mut self) -> *mut A::Item { + self.storage.as_mut_ptr() as _ + } +} + +const LEN_MASK: usize = core::usize::MAX >> 1; +const IS_HEAP_MASK: usize = !LEN_MASK; + +pub(crate) const MAX_LEN: usize = LEN_MASK; + +impl SmallVec { + pub fn new() -> Self { + let mut tagged_union = TaggedUnion2::new_uninit(); + tagged_union.set_tag(0); // len = 0, is_heap = false + Self { tagged_union } + } + + pub(crate) unsafe fn set_inline_tag(&mut self, len: usize) -> &mut InlineData { + assert!((len & LEN_MASK) == len, "overflow"); + self.tagged_union.set_tag(len); + &mut *self.tagged_union.payload_mut_unchecked::>() + } + + pub(crate) unsafe fn set_heap_tag(&mut self, len: usize) -> &mut HeapData { + assert!((len & LEN_MASK) == len, "overflow"); + self.tagged_union.set_tag(IS_HEAP_MASK | len); + &mut *self.tagged_union.payload_mut_unchecked::>() + } + + pub unsafe fn set_len(&mut self, new_len: usize) { + assert!((new_len & LEN_MASK) == new_len, "overflow"); + let is_heap = self.tagged_union.tag() & IS_HEAP_MASK; + self.tagged_union.set_tag(is_heap | new_len) + } + + pub fn len(&self) -> usize { + self.tagged_union.tag() & LEN_MASK + } + + fn is_inline(&self) -> bool { + (self.tagged_union.tag() & IS_HEAP_MASK) == 0 + } + + pub(crate) fn as_inline(&self) -> Result<&InlineData, &HeapData> { + unsafe { + if self.is_inline() { + Ok(&*self.tagged_union.payload_unchecked::>()) + } else { + Err(&*self.tagged_union.payload_unchecked::>()) + } + } + } + + pub(crate) fn as_inline_mut(&mut self) -> Result<&mut InlineData, &mut HeapData> { + unsafe { + if self.is_inline() { + Ok(&mut *self.tagged_union.payload_mut_unchecked::>()) + } else { + Err(&mut *self.tagged_union.payload_mut_unchecked::>()) + } + } + } +} diff --git a/src/tagged_union.rs b/src/tagged_union.rs new file mode 100644 index 0000000..1cd5dd3 --- /dev/null +++ b/src/tagged_union.rs @@ -0,0 +1,85 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use core::mem::MaybeUninit; + +/// A type that has the same memory layout as: +/// +/// ```ignore +/// #[repr(C)] +/// union TaggedUnion2 { +/// a: Variant, +/// b: Variant, +/// uninit: (), +/// } +/// +/// #[repr(C)] +/// struct Variant { +/// tag: usize, +/// payload: Payload, +/// } +/// ``` +/// +/// … but works on Rust versions where unions fields must be `Copy`: +/// +/// * https://github.com/rust-lang/rust/issues/32836 +/// * https://github.com/rust-lang/rust/issues/55149 +pub(crate) struct TaggedUnion2 { + invalid_enum: MaybeUninit>, +} + +/// We rely on [RFC 2195] to construct a type that has the desired memory layout. +/// However, we’re going to store bit patterns in the enum’s tag +/// that do not correspond to any actual variant of the enum, +/// therefore violating its [validity invariant][VI]. +/// To avoid undefined behavior, we never manipulate this enum type directly. +/// We store it in a `MaybeUninit`, +/// and cast pointers to it to other types before doing anything else. +/// +/// [RFC 2195]: https://rust-lang.github.io/rfcs/2195-really-tagged-unions.html +/// [VI]: https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#validity-and-safety-invariant +#[derive(Copy, Clone)] +#[repr(usize)] +enum InvalidEnum { + #[allow(unused)] + A(PayloadA), + #[allow(unused)] + B(PayloadB), +} + +#[repr(C)] +struct Variant { + tag: usize, + payload: Payload, +} + +impl TaggedUnion2 { + pub fn new_uninit() -> Self { + TaggedUnion2 { + invalid_enum: MaybeUninit::uninit(), + } + } + + pub fn tag(&self) -> usize { + let ptr = self.invalid_enum.as_ptr() as *const Variant<()>; + unsafe { (*ptr).tag } + } + + pub fn set_tag(&mut self, tag: usize) { + let ptr = self.invalid_enum.as_mut_ptr() as *mut Variant<()>; + unsafe { (*ptr).tag = tag } + } + + pub fn payload_unchecked(&self) -> *const Payload { + let ptr = self.invalid_enum.as_ptr() as *const Variant; + unsafe { &(*ptr).payload } + } + + pub fn payload_mut_unchecked(&mut self) -> *mut Payload { + let ptr = self.invalid_enum.as_mut_ptr() as *mut Variant; + unsafe { &mut (*ptr).payload } + } +}