use std::{
collections::{BTreeSet, HashMap},
ops::Range,
ptr::NonNull,
thread,
};
use {
crate::{
allocator::{Allocator, Kind},
block::Block,
mapping::*,
memory::*,
util::*,
},
gfx_hal::{Backend, Device as _},
hibitset::{BitSet, BitSetLike as _},
};
#[derive(Debug)]
pub struct DynamicBlock<B: Backend> {
block_index: u32,
chunk_index: u32,
count: u32,
memory: *const Memory<B>,
ptr: Option<NonNull<u8>>,
range: Range<u64>,
relevant: relevant::Relevant,
}
unsafe impl<B> Send for DynamicBlock<B> where B: Backend {}
unsafe impl<B> Sync for DynamicBlock<B> where B: Backend {}
impl<B> DynamicBlock<B>
where
B: Backend,
{
fn shared_memory(&self) -> &Memory<B> {
unsafe { &*self.memory }
}
fn size(&self) -> u64 {
self.range.end - self.range.start
}
fn dispose(self) {
self.relevant.dispose();
}
}
impl<B> Block<B> for DynamicBlock<B>
where
B: Backend,
{
#[inline]
fn properties(&self) -> gfx_hal::memory::Properties {
self.shared_memory().properties()
}
#[inline]
fn memory(&self) -> &B::Memory {
self.shared_memory().raw()
}
#[inline]
fn range(&self) -> Range<u64> {
self.range.clone()
}
#[inline]
fn map<'a>(
&'a mut self,
_device: &B::Device,
range: Range<u64>,
) -> Result<MappedRange<'a, B>, gfx_hal::mapping::Error> {
debug_assert!(
range.start < range.end,
"Memory mapping region must have valid size"
);
if !self.shared_memory().host_visible() {
return Err(gfx_hal::mapping::Error::InvalidAccess);
}
if let Some(ptr) = self.ptr {
if let Some((ptr, range)) = mapped_sub_range(ptr, self.range.clone(), range) {
let mapping = unsafe { MappedRange::from_raw(self.shared_memory(), ptr, range) };
Ok(mapping)
} else {
Err(gfx_hal::mapping::Error::OutOfBounds)
}
} else {
Err(gfx_hal::mapping::Error::MappingFailed)
}
}
#[inline]
fn unmap(&mut self, _device: &B::Device) {}
}
#[derive(Clone, Copy, Debug)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct DynamicConfig {
pub block_size_granularity: u64,
pub max_chunk_size: u64,
pub min_device_allocation: u64,
}
#[derive(Debug)]
pub struct DynamicAllocator<B: Backend> {
memory_type: gfx_hal::MemoryTypeId,
memory_properties: gfx_hal::memory::Properties,
block_size_granularity: u64,
max_chunk_size: u64,
min_device_allocation: u64,
sizes: HashMap<u64, SizeEntry<B>>,
chunks: BTreeSet<u64>,
}
unsafe impl<B> Send for DynamicAllocator<B> where B: Backend {}
unsafe impl<B> Sync for DynamicAllocator<B> where B: Backend {}
#[derive(Debug)]
struct SizeEntry<B: Backend> {
total_blocks: u64,
ready_chunks: BitSet,
chunks: slab::Slab<Chunk<B>>,
}
impl<B> Default for SizeEntry<B>
where
B: Backend,
{
fn default() -> Self {
SizeEntry {
chunks: Default::default(),
total_blocks: 0,
ready_chunks: Default::default(),
}
}
}
const MAX_BLOCKS_PER_CHUNK: u32 = 64;
const MIN_BLOCKS_PER_CHUNK: u32 = 8;
impl<B> DynamicAllocator<B>
where
B: Backend,
{
pub fn new(
memory_type: gfx_hal::MemoryTypeId,
memory_properties: gfx_hal::memory::Properties,
config: DynamicConfig,
) -> Self {
log::trace!(
"Create new allocator: type: '{:?}', properties: '{:#?}' config: '{:#?}'",
memory_type,
memory_properties,
config
);
assert!(
config.block_size_granularity.is_power_of_two(),
"Allocation granularity must be power of two"
);
assert!(
config.max_chunk_size.is_power_of_two(),
"Max chunk size must be power of two"
);
assert!(
config.min_device_allocation.is_power_of_two(),
"Min device allocation must be power of two"
);
assert!(
config.min_device_allocation <= config.max_chunk_size,
"Min device allocation must be less than or equalt to max chunk size"
);
if memory_properties.contains(gfx_hal::memory::Properties::CPU_VISIBLE) {
debug_assert!(
fits_usize(config.max_chunk_size),
"Max chunk size must fit usize for mapping"
);
}
DynamicAllocator {
memory_type,
memory_properties,
block_size_granularity: config.block_size_granularity,
max_chunk_size: config.max_chunk_size,
min_device_allocation: config.min_device_allocation,
sizes: HashMap::new(),
chunks: BTreeSet::new(),
}
}
pub fn max_allocation(&self) -> u64 {
self.max_chunk_size / MIN_BLOCKS_PER_CHUNK as u64
}
fn alloc_chunk_from_device(
&self,
device: &B::Device,
block_size: u64,
chunk_size: u64,
) -> Result<Chunk<B>, gfx_hal::device::AllocationError> {
log::trace!(
"Allocate chunk of size: {} for blocks of size {} from device",
chunk_size,
block_size
);
let (memory, mapping) = unsafe {
let raw = device.allocate_memory(self.memory_type, chunk_size)?;
let mapping = if self
.memory_properties
.contains(gfx_hal::memory::Properties::CPU_VISIBLE)
{
log::trace!("Map new memory object");
match device.map_memory(&raw, 0..chunk_size) {
Ok(mapping) => Some(NonNull::new_unchecked(mapping)),
Err(gfx_hal::mapping::Error::OutOfMemory(error)) => {
device.free_memory(raw);
return Err(error.into());
}
Err(_) => panic!("Unexpected mapping failure"),
}
} else {
None
};
let memory = Memory::from_raw(raw, chunk_size, self.memory_properties);
(memory, mapping)
};
Ok(Chunk::from_memory(block_size, memory, mapping))
}
fn alloc_chunk(
&mut self,
device: &B::Device,
block_size: u64,
total_blocks: u64,
) -> Result<(Chunk<B>, u64), gfx_hal::device::AllocationError> {
log::trace!(
"Allocate chunk for blocks of size {} ({} total blocks allocated)",
block_size,
total_blocks
);
let min_chunk_size = MIN_BLOCKS_PER_CHUNK as u64 * block_size;
let min_size = min_chunk_size.min(total_blocks * block_size);
let max_chunk_size = MAX_BLOCKS_PER_CHUNK as u64 * block_size;
if min_size > self.max_allocation()
|| (total_blocks < MIN_BLOCKS_PER_CHUNK as u64
&& min_size >= self.min_device_allocation)
{
let chunk = self.alloc_chunk_from_device(device, block_size, min_size)?;
return Ok((chunk, min_size));
}
if let Some(&chunk_size) = self
.chunks
.range(min_chunk_size..=max_chunk_size)
.next_back()
{
let (block, allocated) = self.alloc_from_entry(device, chunk_size, 1)?;
Ok((Chunk::from_block(block_size, block), allocated))
} else {
let total_blocks = self.sizes[&block_size].total_blocks;
let chunk_size =
(max_chunk_size.min(min_chunk_size.max(total_blocks * block_size)) / 2 + 1)
.next_power_of_two();
let (block, allocated) = self.alloc_block(device, chunk_size)?;
Ok((Chunk::from_block(block_size, block), allocated))
}
}
fn alloc_from_chunk(
chunks: &mut slab::Slab<Chunk<B>>,
chunk_index: u32,
block_size: u64,
count: u32,
) -> Option<DynamicBlock<B>> {
log::trace!(
"Allocate {} consecutive blocks of size {} from chunk {}",
count,
block_size,
chunk_index
);
let ref mut chunk = chunks[chunk_index as usize];
let block_index = chunk.acquire_blocks(count)?;
let block_range = chunk.blocks_range(block_size, block_index, count);
debug_assert_eq!((block_range.end - block_range.start) % count as u64, 0);
Some(DynamicBlock {
range: block_range.clone(),
memory: chunk.shared_memory(),
block_index,
chunk_index,
count,
ptr: chunk.mapping_ptr().map(|ptr| {
mapped_fitting_range(ptr, chunk.range(), block_range)
.expect("Block must be sub-range of chunk")
}),
relevant: relevant::Relevant,
})
}
fn alloc_from_entry(
&mut self,
device: &B::Device,
block_size: u64,
count: u32,
) -> Result<(DynamicBlock<B>, u64), gfx_hal::device::AllocationError> {
log::trace!(
"Allocate {} consecutive blocks for size {} from the entry",
count,
block_size
);
debug_assert!(count < MIN_BLOCKS_PER_CHUNK);
let size_entry = self.sizes.entry(block_size).or_default();
for chunk_index in (&size_entry.ready_chunks).iter() {
if let Some(block) =
Self::alloc_from_chunk(&mut size_entry.chunks, chunk_index, block_size, count)
{
return Ok((block, 0));
}
}
if size_entry.chunks.vacant_entry().key() > max_chunks_per_size() {
return Err(gfx_hal::device::OutOfMemory::OutOfHostMemory.into());
}
let total_blocks = size_entry.total_blocks;
let (chunk, allocated) = self.alloc_chunk(device, block_size, total_blocks)?;
let size_entry = self.sizes.entry(block_size).or_default();
let chunk_index = size_entry.chunks.insert(chunk) as u32;
let block = Self::alloc_from_chunk(&mut size_entry.chunks, chunk_index, block_size, count)
.expect("New chunk should yield blocks");
if !size_entry.chunks[chunk_index as usize].is_exhausted() {
size_entry.ready_chunks.add(chunk_index);
}
Ok((block, allocated))
}
fn alloc_block(
&mut self,
device: &B::Device,
block_size: u64,
) -> Result<(DynamicBlock<B>, u64), gfx_hal::device::AllocationError> {
log::trace!("Allocate block of size {}", block_size);
debug_assert_eq!(block_size % self.block_size_granularity, 0);
let size_entry = self.sizes.entry(block_size).or_default();
size_entry.total_blocks += 1;
let overhead = (MIN_BLOCKS_PER_CHUNK as u64 - 1) / size_entry.total_blocks;
if overhead >= 1 {
if let Some(&size) = self
.chunks
.range(block_size / 4..block_size * overhead)
.next()
{
return self.alloc_from_entry(device, size, ((block_size - 1) / size + 1) as u32);
}
}
if size_entry.total_blocks == MIN_BLOCKS_PER_CHUNK as u64 {
self.chunks.insert(block_size);
}
self.alloc_from_entry(device, block_size, 1)
}
fn free_chunk(&mut self, device: &B::Device, chunk: Chunk<B>, block_size: u64) -> u64 {
log::trace!("Free chunk: {:#?}", chunk);
assert!(chunk.is_unused(block_size));
match chunk.flavor {
ChunkFlavor::Dedicated(boxed, _) => {
let size = boxed.size();
unsafe {
if self
.memory_properties
.contains(gfx_hal::memory::Properties::CPU_VISIBLE)
{
log::trace!("Unmap memory: {:#?}", boxed);
device.unmap_memory(boxed.raw());
}
device.free_memory(boxed.into_raw());
}
size
}
ChunkFlavor::Dynamic(dynamic_block) => self.free(device, dynamic_block),
}
}
fn free_block(&mut self, device: &B::Device, block: DynamicBlock<B>) -> u64 {
log::trace!("Free block: {:#?}", block);
let block_size = block.size() / block.count as u64;
let ref mut size_entry = self
.sizes
.get_mut(&block_size)
.expect("Unable to get size entry from which block was allocated");
let chunk_index = block.chunk_index;
let ref mut chunk = size_entry.chunks[chunk_index as usize];
let block_index = block.block_index;
let count = block.count;
block.dispose();
chunk.release_blocks(block_index, count);
if chunk.is_unused(block_size) {
size_entry.ready_chunks.remove(chunk_index);
let chunk = size_entry.chunks.remove(chunk_index as usize);
self.free_chunk(device, chunk, block_size)
} else {
size_entry.ready_chunks.add(chunk_index);
0
}
}
pub fn dispose(self) {
if !thread::panicking() {
for (index, size) in self.sizes {
assert_eq!(size.chunks.len(), 0, "SizeEntry({}) is still used", index);
}
} else {
for (index, size) in self.sizes {
if size.chunks.len() != 0 {
log::error!("Memory leak: SizeEntry({}) is still used", index);
}
}
}
}
}
impl<B> Allocator<B> for DynamicAllocator<B>
where
B: Backend,
{
type Block = DynamicBlock<B>;
fn kind() -> Kind {
Kind::Dynamic
}
fn alloc(
&mut self,
device: &B::Device,
size: u64,
align: u64,
) -> Result<(DynamicBlock<B>, u64), gfx_hal::device::AllocationError> {
debug_assert!(size <= self.max_allocation());
debug_assert!(align.is_power_of_two());
let aligned_size = ((size - 1) | (align - 1) | (self.block_size_granularity - 1)) + 1;
log::trace!(
"Allocate dynamic block: size: {}, align: {}, aligned size: {}, type: {}",
size,
align,
aligned_size,
self.memory_type.0
);
self.alloc_block(device, aligned_size)
}
fn free(&mut self, device: &B::Device, block: DynamicBlock<B>) -> u64 {
self.free_block(device, block)
}
}
#[derive(Debug)]
enum ChunkFlavor<B: Backend> {
Dedicated(Box<Memory<B>>, Option<NonNull<u8>>),
Dynamic(DynamicBlock<B>),
}
#[derive(Debug)]
struct Chunk<B: Backend> {
flavor: ChunkFlavor<B>,
blocks: u64,
}
impl<B> Chunk<B>
where
B: Backend,
{
fn from_memory(block_size: u64, memory: Memory<B>, mapping: Option<NonNull<u8>>) -> Self {
let blocks = memory.size() / block_size;
debug_assert!(blocks <= MAX_BLOCKS_PER_CHUNK as u64);
let high_bit = 1 << (blocks - 1);
Chunk {
flavor: ChunkFlavor::Dedicated(Box::new(memory), mapping),
blocks: (high_bit - 1) | high_bit,
}
}
fn from_block(block_size: u64, chunk_block: DynamicBlock<B>) -> Self {
let blocks = (chunk_block.size() / block_size).min(MAX_BLOCKS_PER_CHUNK as u64);
let high_bit = 1 << (blocks - 1);
Chunk {
flavor: ChunkFlavor::Dynamic(chunk_block),
blocks: (high_bit - 1) | high_bit,
}
}
fn shared_memory(&self) -> &Memory<B> {
match &self.flavor {
ChunkFlavor::Dedicated(boxed, _) => &*boxed,
ChunkFlavor::Dynamic(chunk_block) => chunk_block.shared_memory(),
}
}
fn range(&self) -> Range<u64> {
match &self.flavor {
ChunkFlavor::Dedicated(boxed, _) => 0..boxed.size(),
ChunkFlavor::Dynamic(chunk_block) => chunk_block.range(),
}
}
fn size(&self) -> u64 {
let range = self.range();
range.end - range.start
}
fn blocks_range(&self, block_size: u64, block_index: u32, count: u32) -> Range<u64> {
let range = self.range();
let start = range.start + block_size * block_index as u64;
let end = start + block_size * count as u64;
debug_assert!(end <= range.end);
start..end
}
fn is_unused(&self, block_size: u64) -> bool {
let blocks = (self.size() / block_size).min(MAX_BLOCKS_PER_CHUNK as u64);
let high_bit = 1 << (blocks - 1);
let mask = (high_bit - 1) | high_bit;
debug_assert!(self.blocks <= mask);
self.blocks == mask
}
fn is_exhausted(&self) -> bool {
self.blocks == 0
}
fn acquire_blocks(&mut self, count: u32) -> Option<u32> {
debug_assert!(count > 0 && count <= MAX_BLOCKS_PER_CHUNK);
let mut blocks = !0;
for i in 0..count {
blocks &= self.blocks >> i;
}
let index = blocks.trailing_zeros();
if index == MAX_BLOCKS_PER_CHUNK {
None
} else {
let mask = ((1 << count) - 1) << index;
self.blocks &= !mask;
Some(index)
}
}
fn release_blocks(&mut self, index: u32, count: u32) {
let mask = ((1 << count) - 1) << index;
debug_assert_eq!(self.blocks & mask, 0);
self.blocks |= mask;
}
fn mapping_ptr(&self) -> Option<NonNull<u8>> {
match &self.flavor {
ChunkFlavor::Dedicated(_, ptr) => *ptr,
ChunkFlavor::Dynamic(chunk_block) => chunk_block.ptr,
}
}
}
fn max_chunks_per_size() -> usize {
let value = (std::mem::size_of::<usize>() * 8).pow(4);
debug_assert!(fits_u32(value));
value
}