1#ifndef HALIDE_RUNTIME_BLOCK_ALLOCATOR_H
2#define HALIDE_RUNTIME_BLOCK_ALLOCATOR_H
4#include "../HalideRuntime.h"
61 bool collect(
void *user_context);
62 int release(
void *user_context);
63 int destroy(
void *user_context);
86 int destroy_region_allocator(
void *user_context,
RegionAllocator *region_allocator);
98 int release_block_entry(
void *user_context,
BlockEntry *block_entry);
101 int destroy_block_entry(
void *user_context,
BlockEntry *block_entry);
104 int alloc_memory_block(
void *user_context,
BlockResource *block);
107 int free_memory_block(
void *user_context,
BlockResource *block);
110 size_t constrain_requested_size(
size_t size)
const;
116 bool is_block_suitable_for_request(
void *user_context,
const BlockResource *block,
const MemoryProperties &properties,
size_t size,
bool dedicated)
const;
128 if (result ==
nullptr) {
129 error(user_context) <<
"BlockAllocator: Failed to create instance! Out of memory!\n";
133 result->initialize(user_context, cfg, allocators);
140 instance->
destroy(user_context);
145void BlockAllocator::initialize(
void *user_context,
const Config &cfg,
const MemoryAllocators &ma) {
155#ifdef DEBUG_RUNTIME_INTERNAL
156 debug(user_context) <<
"BlockAllocator: Reserve ("
157 <<
"user_context=" << (
void *)(user_context) <<
" "
160 <<
"dedicated=" << (request.
dedicated ?
"true" :
"false") <<
" "
166 if (block_entry ==
nullptr) {
167 error(user_context) <<
"BlockAllocator: Failed to allocate new empty block of requested size ("
177 if (result ==
nullptr) {
181 if (block_entry ==
nullptr) {
182 error(user_context) <<
"BlockAllocator: Out of memory! Failed to allocate empty block of size ("
189 block->
allocator = create_region_allocator(user_context, block);
192 result = reserve_memory_region(user_context, block->
allocator, request);
198 if (memory_region ==
nullptr) {
202 if (allocator ==
nullptr) {
205 return allocator->
release(user_context, memory_region);
209 if (memory_region ==
nullptr) {
213 if (allocator ==
nullptr) {
216 return allocator->
reclaim(user_context, memory_region);
220 if (memory_region ==
nullptr) {
224 if (allocator ==
nullptr) {
227 return allocator->
retain(user_context, memory_region);
233 while (block_entry !=
nullptr) {
237 block_entry = prev_entry;
241#ifdef DEBUG_RUNTIME_INTERNAL
247#ifdef DEBUG_RUNTIME_INTERNAL
248 debug(user_context) <<
"Collected block ("
249 <<
"block=" << (
void *)block <<
" "
256 destroy_block_entry(user_context, block_entry);
260 block_entry = prev_entry;
267 while (block_entry !=
nullptr) {
269 release_block_entry(user_context, block_entry);
270 block_entry = prev_entry;
277 while (block_entry !=
nullptr) {
279 destroy_block_entry(user_context, block_entry);
280 block_entry = prev_entry;
282 block_list.
destroy(user_context);
288 if (result ==
nullptr) {
289#ifdef DEBUG_RUNTIME_INTERNAL
290 debug(user_context) <<
"BlockAllocator: Failed to allocate region of size ("
295 if (allocator->
collect(user_context)) {
296 result = allocator->
reserve(user_context, request);
302bool BlockAllocator::is_block_suitable_for_request(
void *user_context,
const BlockResource *block,
const MemoryProperties &properties,
size_t size,
bool dedicated)
const {
303 if (!is_compatible_block(block, properties)) {
304#ifdef DEBUG_RUNTIME_INTERNAL
305 debug(user_context) <<
"BlockAllocator: skipping block ... incompatible properties!\n"
306 <<
" block_resource=" << (
void *)block <<
"\n"
307 <<
" block_size=" << (
uint32_t)block->memory.
size <<
"\n"
308 <<
" block_reserved=" << (
uint32_t)block->reserved <<
"\n"
312 debug(user_context) <<
" request_size=" << (
uint32_t)size <<
"\n"
321 if (dedicated && (block->reserved > 0)) {
322#ifdef DEBUG_RUNTIME_INTERNAL
323 debug(user_context) <<
"BlockAllocator: skipping block ... can be used for dedicated allocation!\n"
324 <<
" block_resource=" << (
void *)block <<
"\n"
325 <<
" block_size=" << (
uint32_t)block->memory.size <<
"\n"
326 <<
" block_reserved=" << (
uint32_t)block->reserved <<
"\n";
331 }
else if (block->memory.dedicated && (block->reserved > 0)) {
332#ifdef DEBUG_RUNTIME_INTERNAL
333 debug(user_context) <<
"BlockAllocator: skipping block ... already dedicated to an allocation!\n"
334 <<
" block_resource=" << (
void *)block <<
"\n"
335 <<
" block_size=" << (
uint32_t)block->memory.size <<
"\n"
336 <<
" block_reserved=" << (
uint32_t)block->reserved <<
"\n";
342 size_t available = (block->memory.size - block->reserved);
343 if (available >= size) {
350BlockAllocator::BlockEntry *
351BlockAllocator::find_block_entry(
void *user_context,
const MemoryProperties &properties,
size_t size,
bool dedicated) {
352 BlockEntry *block_entry = block_list.
back();
353 while (block_entry !=
nullptr) {
354 BlockEntry *prev_entry = block_entry->
prev_ptr;
355 const BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
356 if (is_block_suitable_for_request(user_context, block, properties, size, dedicated)) {
357#ifdef DEBUG_RUNTIME_INTERNAL
358 debug(user_context) <<
"BlockAllocator: found suitable block ...\n"
359 <<
" user_context=" << (
void *)(user_context) <<
"\n"
360 <<
" block_resource=" << (
void *)block <<
"\n"
361 <<
" block_size=" << (
uint32_t)block->memory.size <<
"\n"
362 <<
" block_reserved=" << (
uint32_t)block->reserved <<
"\n"
363 <<
" request_size=" << (
uint32_t)size <<
"\n"
364 <<
" dedicated=" << (dedicated ?
"true" :
"false") <<
"\n"
371 block_entry = prev_entry;
374 if (block_entry ==
nullptr) {
375#ifdef DEBUG_RUNTIME_INTERNAL
376 debug(user_context) <<
"BlockAllocator: couldn't find suitable block!\n"
377 <<
" user_context=" << (
void *)(user_context) <<
"\n"
378 <<
" request_size=" << (
uint32_t)size <<
"\n"
379 <<
" dedicated=" << (dedicated ?
"true" :
"false") <<
"\n"
388BlockAllocator::BlockEntry *
389BlockAllocator::reserve_block_entry(
void *user_context,
const MemoryProperties &properties,
size_t size,
bool dedicated) {
390#ifdef DEBUG_RUNTIME_INTERNAL
391 debug(user_context) <<
"BlockAllocator: reserving block ... !\n"
392 <<
" requested_size=" << (
uint32_t)size <<
"\n"
393 <<
" requested_is_dedicated=" << (dedicated ?
"true" :
"false") <<
"\n"
398 BlockEntry *block_entry = find_block_entry(user_context, properties, size, dedicated);
399 if (block_entry ==
nullptr) {
400#ifdef DEBUG_RUNTIME_INTERNAL
401 debug(user_context) <<
"BlockAllocator: creating block ... !\n"
402 <<
" requested_size=" << (
uint32_t)size <<
"\n"
403 <<
" requested_is_dedicated=" << (dedicated ?
"true" :
"false") <<
"\n"
408 block_entry = create_block_entry(user_context, properties, size, dedicated);
412 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
413 if (block->allocator ==
nullptr) {
414 block->allocator = create_region_allocator(user_context, block);
421BlockAllocator::create_region_allocator(
void *user_context, BlockResource *block) {
422#ifdef DEBUG_RUNTIME_INTERNAL
423 debug(user_context) <<
"BlockAllocator: Creating region allocator ("
424 <<
"user_context=" << (
void *)(user_context) <<
" "
425 <<
"block_resource=" << (
void *)(block) <<
")...\n";
429 user_context, block, {allocators.
system, allocators.
region});
431 if (region_allocator ==
nullptr) {
432 error(user_context) <<
"BlockAllocator: Failed to create new region allocator!\n";
436 return region_allocator;
439int BlockAllocator::destroy_region_allocator(
void *user_context, RegionAllocator *region_allocator) {
440#ifdef DEBUG_RUNTIME_INTERNAL
441 debug(user_context) <<
"BlockAllocator: Destroying region allocator ("
442 <<
"user_context=" << (
void *)(user_context) <<
" "
443 <<
"region_allocator=" << (
void *)(region_allocator) <<
")...\n";
445 if (region_allocator ==
nullptr) {
451BlockAllocator::BlockEntry *
452BlockAllocator::create_block_entry(
void *user_context,
const MemoryProperties &properties,
size_t size,
bool dedicated) {
454 error(user_context) <<
"BlockAllocator: No free blocks found! Maximum pool size reached ("
461 error(user_context) <<
"BlockAllocator: No free blocks found! Maximum block count reached ("
466 BlockEntry *block_entry = block_list.
append(user_context);
467 if (block_entry ==
nullptr) {
468 debug(user_context) <<
"BlockAllocator: Failed to allocate new block entry!\n";
472#ifdef DEBUG_RUNTIME_INTERNAL
473 debug(user_context) <<
"BlockAllocator: Creating block entry ("
474 <<
"block_entry=" << (
void *)(block_entry) <<
" "
475 <<
"block=" << (
void *)(block_entry->value) <<
" "
476 <<
"allocator=" << (
void *)(allocators.
block.
allocate) <<
")...\n";
479 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
480 block->memory.size = constrain_requested_size(size);
481 block->memory.handle =
nullptr;
482 block->memory.properties = properties;
483 block->memory.properties.nearest_multiple =
max(config.
nearest_multiple, properties.nearest_multiple);
484 block->memory.dedicated = dedicated;
486 block->allocator = create_region_allocator(user_context, block);
487 alloc_memory_block(user_context, block);
491int BlockAllocator::release_block_entry(
void *user_context, BlockAllocator::BlockEntry *block_entry) {
492#ifdef DEBUG_RUNTIME_INTERNAL
493 debug(user_context) <<
"BlockAllocator: Releasing block entry ("
494 <<
"block_entry=" << (
void *)(block_entry) <<
" "
495 <<
"block=" << (
void *)(block_entry->value) <<
")...\n";
497 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
498 if (block->allocator) {
499 return block->allocator->release(user_context);
504int BlockAllocator::destroy_block_entry(
void *user_context, BlockAllocator::BlockEntry *block_entry) {
505#ifdef DEBUG_RUNTIME_INTERNAL
506 debug(user_context) <<
"BlockAllocator: Destroying block entry ("
507 <<
"block_entry=" << (
void *)(block_entry) <<
" "
508 <<
"block=" << (
void *)(block_entry->value) <<
" "
509 <<
"deallocator=" << (
void *)(allocators.
block.
deallocate) <<
")...\n";
511 BlockResource *block =
static_cast<BlockResource *
>(block_entry->value);
512 if (block->allocator) {
513 destroy_region_allocator(user_context, block->allocator);
514 block->allocator =
nullptr;
516 free_memory_block(user_context, block);
517 block_list.
remove(user_context, block_entry);
521int BlockAllocator::alloc_memory_block(
void *user_context, BlockResource *block) {
522#ifdef DEBUG_RUNTIME_INTERNAL
523 debug(user_context) <<
"BlockAllocator: Allocating block (ptr=" << (
void *)block <<
" allocator=" << (
void *)allocators.
block.
allocate <<
")...\n";
526 MemoryBlock *memory_block = &(block->memory);
532int BlockAllocator::free_memory_block(
void *user_context, BlockResource *block) {
533#ifdef DEBUG_RUNTIME_INTERNAL
534 debug(user_context) <<
"BlockAllocator: Deallocating block (ptr=" << (
void *)block <<
" allocator=" << (
void *)allocators.
block.
deallocate <<
")...\n";
537 MemoryBlock *memory_block = &(block->memory);
539 memory_block->handle =
nullptr;
541 block->memory.size = 0;
545size_t BlockAllocator::constrain_requested_size(
size_t size)
const {
546 size_t actual_size = size;
564bool BlockAllocator::is_compatible_block(
const BlockResource *block,
const MemoryProperties &properties)
const {
566 if (properties.caching != block->memory.properties.caching) {
572 if (properties.visibility != block->memory.properties.visibility) {
578 if (properties.usage != block->memory.properties.usage) {
600 return block_list.
size();
604 size_t total_size = 0;
606 for (block_entry = block_list.
front(); block_entry !=
nullptr; block_entry = block_entry->
next_ptr) {
608 if (block !=
nullptr) {
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
BlockAllocator(const BlockAllocator &)=delete
const Config & default_config() const
bool collect(void *user_context)
size_t block_count() const
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
const Config & current_config() const
int retain(void *user_context, MemoryRegion *region)
static void destroy(void *user_context, BlockAllocator *block_allocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
int reclaim(void *user_context, MemoryRegion *region)
BlockAllocator & operator=(const BlockAllocator &)=delete
const MemoryAllocators & current_allocators() const
void initialize(void *user_context, uint32_t entry_size, uint32_t capacity=default_capacity, const SystemMemoryAllocatorFns &allocator=default_allocator())
EntryType * append(void *user_context)
void remove(void *user_context, EntryType *entry_ptr)
void destroy(void *user_context)
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
static int destroy(void *user_context, RegionAllocator *region_allocator)
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
int retain(void *user_context, MemoryRegion *memory_region)
bool collect(void *user_context)
int reclaim(void *user_context, MemoryRegion *memory_region)
int release(void *user_context, MemoryRegion *memory_region)
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr max(const FuncRef &a, const FuncRef &b)
unsigned __INT64_TYPE__ uint64_t
signed __INT32_TYPE__ int32_t
unsigned __INT32_TYPE__ uint32_t
#define halide_abort_if_false(user_context, cond)
size_t maximum_block_count
size_t minimum_block_size
size_t maximum_block_size
MemoryRegionAllocatorFns region
MemoryBlockAllocatorFns block
SystemMemoryAllocatorFns system
RegionAllocator * allocator
DeallocateBlockFn deallocate
MemoryVisibility visibility
MemoryProperties properties
DeallocateSystemFn deallocate
AllocateSystemFn allocate