#![allow(missing_docs)]
extern crate nodrop;
#[cfg(feature = "servo")] extern crate serde;
extern crate stable_deref_trait;
use nodrop::NoDrop;
#[cfg(feature = "servo")]
use serde::{Deserialize, Serialize};
use stable_deref_trait::{CloneStableDeref, StableDeref};
use std::{isize, usize};
use std::borrow;
use std::cmp::Ordering;
use std::convert::From;
use std::fmt;
use std::hash::{Hash, Hasher};
use std::iter::{ExactSizeIterator, Iterator};
use std::mem;
use std::ops::{Deref, DerefMut};
use std::os::raw::c_void;
use std::process;
use std::ptr;
use std::slice;
use std::sync::atomic;
use std::sync::atomic::Ordering::{Acquire, Relaxed, Release};
macro_rules! offset_of {
($container:path, $field:ident) => {{
let $container { $field: _, .. };
let invalid: $container = ::std::mem::uninitialized();
let offset = &invalid.$field as *const _ as usize - &invalid as *const _ as usize;
::std::mem::forget(invalid);
offset as isize
}};
}
const MAX_REFCOUNT: usize = (isize::MAX) as usize;
pub struct NonZeroPtrMut<T: ?Sized + 'static>(&'static mut T);
impl<T: ?Sized> NonZeroPtrMut<T> {
pub fn new(ptr: *mut T) -> Self {
assert!(!(ptr as *mut u8).is_null());
NonZeroPtrMut(unsafe { mem::transmute(ptr) })
}
pub fn ptr(&self) -> *mut T {
self.0 as *const T as *mut T
}
}
impl<T: ?Sized + 'static> Clone for NonZeroPtrMut<T> {
fn clone(&self) -> Self {
NonZeroPtrMut::new(self.ptr())
}
}
impl<T: ?Sized + 'static> fmt::Pointer for NonZeroPtrMut<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Pointer::fmt(&self.ptr(), f)
}
}
impl<T: ?Sized + 'static> fmt::Debug for NonZeroPtrMut<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
<Self as fmt::Pointer>::fmt(self, f)
}
}
impl<T: ?Sized + 'static> PartialEq for NonZeroPtrMut<T> {
fn eq(&self, other: &Self) -> bool {
self.ptr() == other.ptr()
}
}
impl<T: ?Sized + 'static> Eq for NonZeroPtrMut<T> {}
impl<T: Sized + 'static> Hash for NonZeroPtrMut<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.ptr().hash(state)
}
}
#[repr(C)]
pub struct Arc<T: ?Sized + 'static> {
p: NonZeroPtrMut<ArcInner<T>>,
}
pub struct UniqueArc<T: ?Sized + 'static>(Arc<T>);
impl<T> UniqueArc<T> {
#[inline]
pub fn new(data: T) -> Self {
UniqueArc(Arc::new(data))
}
#[inline]
pub fn shareable(self) -> Arc<T> {
self.0
}
}
impl<T> Deref for UniqueArc<T> {
type Target = T;
fn deref(&self) -> &T {
&*self.0
}
}
impl<T> DerefMut for UniqueArc<T> {
fn deref_mut(&mut self) -> &mut T {
unsafe { &mut (*self.0.ptr()).data }
}
}
unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
#[repr(C)]
struct ArcInner<T: ?Sized> {
count: atomic::AtomicUsize,
data: T,
}
unsafe impl<T: ?Sized + Sync + Send> Send for ArcInner<T> {}
unsafe impl<T: ?Sized + Sync + Send> Sync for ArcInner<T> {}
impl<T> Arc<T> {
#[inline]
pub fn new(data: T) -> Self {
let x = Box::new(ArcInner {
count: atomic::AtomicUsize::new(1),
data: data,
});
Arc { p: NonZeroPtrMut::new(Box::into_raw(x)) }
}
#[inline]
pub fn into_raw(this: Self) -> *const T {
let ptr = unsafe { &((*this.ptr()).data) as *const _ };
mem::forget(this);
ptr
}
#[inline]
unsafe fn from_raw(ptr: *const T) -> Self {
let ptr = (ptr as *const u8).offset(-offset_of!(ArcInner<T>, data));
Arc {
p: NonZeroPtrMut::new(ptr as *mut ArcInner<T>),
}
}
#[inline]
pub fn borrow_arc<'a>(&'a self) -> ArcBorrow<'a, T> {
ArcBorrow(&**self)
}
#[inline(always)]
pub fn with_raw_offset_arc<F, U>(&self, f: F) -> U
where F: FnOnce(&RawOffsetArc<T>) -> U
{
let transient = unsafe { NoDrop::new(Arc::into_raw_offset(ptr::read(self))) };
let result = f(&transient);
mem::forget(transient);
result
}
pub fn heap_ptr(&self) -> *const c_void {
self.p.ptr() as *const ArcInner<T> as *const c_void
}
}
impl<T: ?Sized> Arc<T> {
#[inline]
fn inner(&self) -> &ArcInner<T> {
unsafe { &*self.ptr() }
}
#[inline(never)]
unsafe fn drop_slow(&mut self) {
let _ = Box::from_raw(self.ptr());
}
#[inline]
pub fn ptr_eq(this: &Self, other: &Self) -> bool {
this.ptr() == other.ptr()
}
fn ptr(&self) -> *mut ArcInner<T> {
self.p.ptr()
}
}
impl<T: ?Sized> Clone for Arc<T> {
#[inline]
fn clone(&self) -> Self {
let old_size = self.inner().count.fetch_add(1, Relaxed);
if old_size > MAX_REFCOUNT {
process::abort();
}
Arc { p: NonZeroPtrMut::new(self.ptr()) }
}
}
impl<T: ?Sized> Deref for Arc<T> {
type Target = T;
#[inline]
fn deref(&self) -> &T {
&self.inner().data
}
}
impl<T: Clone> Arc<T> {
#[inline]
pub fn make_mut(this: &mut Self) -> &mut T {
if !this.is_unique() {
*this = Arc::new((**this).clone());
}
unsafe {
&mut (*this.ptr()).data
}
}
}
impl<T: ?Sized> Arc<T> {
#[inline]
pub fn get_mut(this: &mut Self) -> Option<&mut T> {
if this.is_unique() {
unsafe {
Some(&mut (*this.ptr()).data)
}
} else {
None
}
}
#[inline]
pub fn is_unique(&self) -> bool {
self.inner().count.load(Relaxed) == 1
}
}
impl<T: ?Sized> Drop for Arc<T> {
#[inline]
fn drop(&mut self) {
if self.inner().count.fetch_sub(1, Release) != 1 {
return;
}
self.inner().count.load(Acquire);
unsafe {
self.drop_slow();
}
}
}
impl<T: ?Sized + PartialEq> PartialEq for Arc<T> {
fn eq(&self, other: &Arc<T>) -> bool {
Self::ptr_eq(self, other) || *(*self) == *(*other)
}
fn ne(&self, other: &Arc<T>) -> bool {
!Self::ptr_eq(self, other) && *(*self) != *(*other)
}
}
impl<T: ?Sized + PartialOrd> PartialOrd for Arc<T> {
fn partial_cmp(&self, other: &Arc<T>) -> Option<Ordering> {
(**self).partial_cmp(&**other)
}
fn lt(&self, other: &Arc<T>) -> bool {
*(*self) < *(*other)
}
fn le(&self, other: &Arc<T>) -> bool {
*(*self) <= *(*other)
}
fn gt(&self, other: &Arc<T>) -> bool {
*(*self) > *(*other)
}
fn ge(&self, other: &Arc<T>) -> bool {
*(*self) >= *(*other)
}
}
impl<T: ?Sized + Ord> Ord for Arc<T> {
fn cmp(&self, other: &Arc<T>) -> Ordering {
(**self).cmp(&**other)
}
}
impl<T: ?Sized + Eq> Eq for Arc<T> {}
impl<T: ?Sized + fmt::Display> fmt::Display for Arc<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Display::fmt(&**self, f)
}
}
impl<T: ?Sized + fmt::Debug> fmt::Debug for Arc<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(&**self, f)
}
}
impl<T: ?Sized> fmt::Pointer for Arc<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Pointer::fmt(&self.ptr(), f)
}
}
impl<T: Default> Default for Arc<T> {
fn default() -> Arc<T> {
Arc::new(Default::default())
}
}
impl<T: ?Sized + Hash> Hash for Arc<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
(**self).hash(state)
}
}
impl<T> From<T> for Arc<T> {
#[inline]
fn from(t: T) -> Self {
Arc::new(t)
}
}
impl<T: ?Sized> borrow::Borrow<T> for Arc<T> {
#[inline]
fn borrow(&self) -> &T {
&**self
}
}
impl<T: ?Sized> AsRef<T> for Arc<T> {
#[inline]
fn as_ref(&self) -> &T {
&**self
}
}
unsafe impl<T: ?Sized> StableDeref for Arc<T> {}
unsafe impl<T: ?Sized> CloneStableDeref for Arc<T> {}
#[cfg(feature = "servo")]
impl<'de, T: Deserialize<'de>> Deserialize<'de> for Arc<T>
{
fn deserialize<D>(deserializer: D) -> Result<Arc<T>, D::Error>
where
D: ::serde::de::Deserializer<'de>,
{
T::deserialize(deserializer).map(Arc::new)
}
}
#[cfg(feature = "servo")]
impl<T: Serialize> Serialize for Arc<T>
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: ::serde::ser::Serializer,
{
(**self).serialize(serializer)
}
}
#[derive(Debug, Eq, PartialEq, PartialOrd)]
pub struct HeaderSlice<H, T: ?Sized> {
pub header: H,
pub slice: T,
}
#[inline(always)]
fn divide_rounding_up(dividend: usize, divisor: usize) -> usize {
(dividend + divisor - 1) / divisor
}
impl<H, T> Arc<HeaderSlice<H, [T]>> {
#[inline]
pub fn from_header_and_iter<I>(header: H, mut items: I) -> Self
where I: Iterator<Item=T> + ExactSizeIterator
{
use ::std::mem::size_of;
assert_ne!(size_of::<T>(), 0, "Need to think about ZST");
let num_items = items.len();
let size = {
let fake_slice_ptr_align: usize = mem::align_of::<ArcInner<HeaderSlice<H, [T; 1]>>>();
let fake_slice_ptr = fake_slice_ptr_align as *const T;
let fake_slice = unsafe { slice::from_raw_parts(fake_slice_ptr, num_items) };
let fake_ptr = fake_slice as *const [T] as *const ArcInner<HeaderSlice<H, [T]>>;
let fake_ref: &ArcInner<HeaderSlice<H, [T]>> = unsafe { &*fake_ptr };
mem::size_of_val(fake_ref)
};
let ptr: *mut ArcInner<HeaderSlice<H, [T]>>;
unsafe {
let buffer = if mem::align_of::<T>() <= mem::align_of::<usize>() {
Self::allocate_buffer::<usize>(size)
} else if mem::align_of::<T>() <= mem::align_of::<u64>() {
Self::allocate_buffer::<u64>(size)
} else {
panic!("Over-aligned type not handled");
};
let fake_slice: &mut [T] = slice::from_raw_parts_mut(buffer as *mut T, num_items);
ptr = fake_slice as *mut [T] as *mut ArcInner<HeaderSlice<H, [T]>>;
ptr::write(&mut ((*ptr).count), atomic::AtomicUsize::new(1));
ptr::write(&mut ((*ptr).data.header), header);
let mut current: *mut T = &mut (*ptr).data.slice[0];
for _ in 0..num_items {
ptr::write(current, items.next().expect("ExactSizeIterator over-reported length"));
current = current.offset(1);
}
assert!(items.next().is_none(), "ExactSizeIterator under-reported length");
debug_assert_eq!(current as *mut u8, buffer.offset(size as isize));
}
assert_eq!(size_of::<Self>(), size_of::<usize>() * 2, "The Arc will be fat");
Arc { p: NonZeroPtrMut::new(ptr) }
}
#[inline]
unsafe fn allocate_buffer<W>(size: usize) -> *mut u8 {
let words_to_allocate = divide_rounding_up(size, mem::size_of::<W>());
let mut vec = Vec::<W>::with_capacity(words_to_allocate);
vec.set_len(words_to_allocate);
Box::into_raw(vec.into_boxed_slice()) as *mut W as *mut u8
}
}
#[derive(Debug, Eq, PartialEq, PartialOrd)]
pub struct HeaderWithLength<H> {
pub header: H,
length: usize,
}
impl<H> HeaderWithLength<H> {
pub fn new(header: H, length: usize) -> Self {
HeaderWithLength {
header: header,
length: length,
}
}
}
type HeaderSliceWithLength<H, T> = HeaderSlice<HeaderWithLength<H>, T>;
pub struct ThinArc<H: 'static, T: 'static> {
ptr: *mut ArcInner<HeaderSliceWithLength<H, [T; 1]>>,
}
unsafe impl<H: Sync + Send, T: Sync + Send> Send for ThinArc<H, T> {}
unsafe impl<H: Sync + Send, T: Sync + Send> Sync for ThinArc<H, T> {}
fn thin_to_thick<H, T>(thin: *mut ArcInner<HeaderSliceWithLength<H, [T; 1]>>)
-> *mut ArcInner<HeaderSliceWithLength<H, [T]>>
{
let len = unsafe { (*thin).data.header.length };
let fake_slice: *mut [T] = unsafe {
slice::from_raw_parts_mut(thin as *mut T, len)
};
fake_slice as *mut ArcInner<HeaderSliceWithLength<H, [T]>>
}
impl<H: 'static, T: 'static> ThinArc<H, T> {
#[inline]
pub fn with_arc<F, U>(&self, f: F) -> U
where F: FnOnce(&Arc<HeaderSliceWithLength<H, [T]>>) -> U
{
let transient = NoDrop::new(Arc {
p: NonZeroPtrMut::new(thin_to_thick(self.ptr))
});
let result = f(&transient);
mem::forget(transient);
result
}
#[inline]
pub fn heap_ptr(&self) -> *const c_void {
self.ptr as *const ArcInner<T> as *const c_void
}
}
impl<H, T> Deref for ThinArc<H, T> {
type Target = HeaderSliceWithLength<H, [T]>;
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { &(*thin_to_thick(self.ptr)).data }
}
}
impl<H: 'static, T: 'static> Clone for ThinArc<H, T> {
#[inline]
fn clone(&self) -> Self {
ThinArc::with_arc(self, |a| Arc::into_thin(a.clone()))
}
}
impl<H: 'static, T: 'static> Drop for ThinArc<H, T> {
#[inline]
fn drop(&mut self) {
let _ = Arc::from_thin(ThinArc { ptr: self.ptr });
}
}
impl<H: 'static, T: 'static> Arc<HeaderSliceWithLength<H, [T]>> {
#[inline]
pub fn into_thin(a: Self) -> ThinArc<H, T> {
assert_eq!(a.header.length, a.slice.len(),
"Length needs to be correct for ThinArc to work");
let fat_ptr: *mut ArcInner<HeaderSliceWithLength<H, [T]>> = a.ptr();
mem::forget(a);
let thin_ptr = fat_ptr as *mut [usize] as *mut usize;
ThinArc {
ptr: thin_ptr as *mut ArcInner<HeaderSliceWithLength<H, [T; 1]>>
}
}
#[inline]
pub fn from_thin(a: ThinArc<H, T>) -> Self {
let ptr = thin_to_thick(a.ptr);
mem::forget(a);
Arc {
p: NonZeroPtrMut::new(ptr)
}
}
}
impl<H: PartialEq + 'static, T: PartialEq + 'static> PartialEq for ThinArc<H, T> {
#[inline]
fn eq(&self, other: &ThinArc<H, T>) -> bool {
ThinArc::with_arc(self, |a| {
ThinArc::with_arc(other, |b| {
*a == *b
})
})
}
}
impl<H: Eq + 'static, T: Eq + 'static> Eq for ThinArc<H, T> {}
#[derive(Eq)]
#[repr(C)]
pub struct RawOffsetArc<T: 'static> {
ptr: NonZeroPtrMut<T>,
}
unsafe impl<T: 'static + Sync + Send> Send for RawOffsetArc<T> {}
unsafe impl<T: 'static + Sync + Send> Sync for RawOffsetArc<T> {}
impl<T: 'static> Deref for RawOffsetArc<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { &*self.ptr.ptr() }
}
}
impl<T: 'static> Clone for RawOffsetArc<T> {
#[inline]
fn clone(&self) -> Self {
Arc::into_raw_offset(self.clone_arc())
}
}
impl<T: 'static> Drop for RawOffsetArc<T> {
fn drop(&mut self) {
let _ = Arc::from_raw_offset(RawOffsetArc { ptr: self.ptr.clone() });
}
}
impl<T: fmt::Debug + 'static> fmt::Debug for RawOffsetArc<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(&**self, f)
}
}
impl<T: PartialEq> PartialEq for RawOffsetArc<T> {
fn eq(&self, other: &RawOffsetArc<T>) -> bool {
*(*self) == *(*other)
}
fn ne(&self, other: &RawOffsetArc<T>) -> bool {
*(*self) != *(*other)
}
}
impl<T: 'static> RawOffsetArc<T> {
#[inline]
pub fn with_arc<F, U>(&self, f: F) -> U
where F: FnOnce(&Arc<T>) -> U
{
let transient = unsafe { NoDrop::new(Arc::from_raw(self.ptr.ptr())) };
let result = f(&transient);
mem::forget(transient);
result
}
#[inline]
pub fn make_mut(&mut self) -> &mut T where T: Clone {
unsafe {
let this = ptr::read(self);
let mut arc = Arc::from_raw_offset(this);
let ret = Arc::make_mut(&mut arc) as *mut _;
ptr::write(self, Arc::into_raw_offset(arc));
&mut *ret
}
}
#[inline]
pub fn clone_arc(&self) -> Arc<T> {
RawOffsetArc::with_arc(self, |a| a.clone())
}
#[inline]
pub fn borrow_arc<'a>(&'a self) -> ArcBorrow<'a, T> {
ArcBorrow(&**self)
}
}
impl<T: 'static> Arc<T> {
#[inline]
pub fn into_raw_offset(a: Self) -> RawOffsetArc<T> {
RawOffsetArc {
ptr: NonZeroPtrMut::new(Arc::into_raw(a) as *mut T),
}
}
#[inline]
pub fn from_raw_offset(a: RawOffsetArc<T>) -> Self {
let ptr = a.ptr.ptr();
mem::forget(a);
unsafe { Arc::from_raw(ptr) }
}
}
#[derive(Eq, PartialEq)]
pub struct ArcBorrow<'a, T: 'a>(&'a T);
impl<'a, T> Copy for ArcBorrow<'a, T> {}
impl<'a, T> Clone for ArcBorrow<'a, T> {
#[inline]
fn clone(&self) -> Self {
*self
}
}
impl<'a, T> ArcBorrow<'a, T> {
#[inline]
pub fn clone_arc(&self) -> Arc<T> {
let arc = unsafe { Arc::from_raw(self.0) };
mem::forget(arc.clone());
arc
}
#[inline]
pub unsafe fn from_ref(r: &'a T) -> Self {
ArcBorrow(r)
}
#[inline]
pub fn with_arc<F, U>(&self, f: F) -> U where F: FnOnce(&Arc<T>) -> U, T: 'static {
let transient = unsafe { NoDrop::new(Arc::from_raw(self.0)) };
let result = f(&transient);
mem::forget(transient);
result
}
}
impl<'a, T> Deref for ArcBorrow<'a, T> {
type Target = T;
#[inline]
fn deref(&self) -> &T {
&*self.0
}
}
#[cfg(test)]
mod tests {
use std::clone::Clone;
use std::ops::Drop;
use std::sync::atomic;
use std::sync::atomic::Ordering::{Acquire, SeqCst};
use super::{Arc, HeaderWithLength, ThinArc};
#[derive(PartialEq)]
struct Canary(*mut atomic::AtomicUsize);
impl Drop for Canary {
fn drop(&mut self) {
unsafe { (*self.0).fetch_add(1, SeqCst); }
}
}
#[test]
fn slices_and_thin() {
let mut canary = atomic::AtomicUsize::new(0);
let c = Canary(&mut canary as *mut atomic::AtomicUsize);
let v = vec![5, 6];
let header = HeaderWithLength::new(c, v.len());
{
let x = Arc::into_thin(Arc::from_header_and_iter(header, v.into_iter()));
let y = ThinArc::with_arc(&x, |q| q.clone());
let _ = y.clone();
let _ = x == x;
Arc::from_thin(x.clone());
}
assert_eq!(canary.load(Acquire), 1);
}
}