package mem /* The Rollback Stack Allocator was designed for the test runner to be fast, able to grow, and respect the Tracking Allocator's requirement for individual frees. It is not overly concerned with fragmentation, however. It has support for expansion when configured with a block allocator and limited support for out-of-order frees. Allocation has constant-time best and usual case performance. At worst, it is linear according to the number of memory blocks. Allocation follows a first-fit strategy when there are multiple memory blocks. Freeing has constant-time best and usual case performance. At worst, it is linear according to the number of memory blocks and number of freed items preceding the last item in a block. Resizing has constant-time performance, if it's the last item in a block, or the new size is smaller. Naturally, this becomes linear-time if there are multiple blocks to search for the pointer's owning block. Otherwise, the allocator defaults to a combined alloc & free operation internally. Out-of-order freeing is accomplished by collapsing a run of freed items from the last allocation backwards. Each allocation has an overhead of 8 bytes and any extra bytes to satisfy the requested alignment. */ import "base:runtime" ROLLBACK_STACK_DEFAULT_BLOCK_SIZE :: 4 * Megabyte /* This limitation is due to the size of `prev_ptr`, but it is only for the head block; any allocation in excess of the allocator's `block_size` is valid, so long as the block allocator can handle it. This is because allocations over the block size are not split up if the item within is freed; they are immediately returned to the block allocator. */ ROLLBACK_STACK_MAX_HEAD_BLOCK_SIZE :: 2 * Gigabyte Rollback_Stack_Header :: bit_field u64 { prev_offset: uintptr | 32, is_free: bool | 1, prev_ptr: uintptr | 31, } Rollback_Stack_Block :: struct { next_block: ^Rollback_Stack_Block, last_alloc: rawptr, offset: uintptr, buffer: []byte, } Rollback_Stack :: struct { head: ^Rollback_Stack_Block, block_size: int, block_allocator: Allocator, } @(private="file", require_results) rb_ptr_in_bounds :: proc(block: ^Rollback_Stack_Block, ptr: rawptr) -> bool { start := raw_data(block.buffer) end := start[block.offset:] return start < ptr && ptr <= end } @(private="file", require_results) rb_find_ptr :: proc(stack: ^Rollback_Stack, ptr: rawptr) -> ( parent: ^Rollback_Stack_Block, block: ^Rollback_Stack_Block, header: ^Rollback_Stack_Header, err: Allocator_Error, ) { for block = stack.head; block != nil; block = block.next_block { if rb_ptr_in_bounds(block, ptr) { header = cast(^Rollback_Stack_Header)(cast(uintptr)ptr - size_of(Rollback_Stack_Header)) return } parent = block } return nil, nil, nil, .Invalid_Pointer } @(private="file", require_results) rb_find_last_alloc :: proc(stack: ^Rollback_Stack, ptr: rawptr) -> ( block: ^Rollback_Stack_Block, header: ^Rollback_Stack_Header, ok: bool, ) { for block = stack.head; block != nil; block = block.next_block { if block.last_alloc == ptr { header = cast(^Rollback_Stack_Header)(cast(uintptr)ptr - size_of(Rollback_Stack_Header)) return block, header, true } } return nil, nil, false } @(private="file") rb_rollback_block :: proc(block: ^Rollback_Stack_Block, header: ^Rollback_Stack_Header) { header := header for block.offset > 0 && header.is_free { block.offset = header.prev_offset block.last_alloc = raw_data(block.buffer)[header.prev_ptr:] header = cast(^Rollback_Stack_Header)(raw_data(block.buffer)[header.prev_ptr - size_of(Rollback_Stack_Header):]) } } @(private="file", require_results) rb_free :: proc(stack: ^Rollback_Stack, ptr: rawptr) -> Allocator_Error { parent, block, header := rb_find_ptr(stack, ptr) or_return if header.is_free { return .Invalid_Pointer } header.is_free = true if block.last_alloc == ptr { block.offset = header.prev_offset rb_rollback_block(block, header) } if parent != nil && block.offset == 0 { parent.next_block = block.next_block runtime.mem_free_with_size(block, size_of(Rollback_Stack_Block) + len(block.buffer), stack.block_allocator) } return nil } @(private="file") rb_free_all :: proc(stack: ^Rollback_Stack) { for block := stack.head.next_block; block != nil; /**/ { next_block := block.next_block runtime.mem_free_with_size(block, size_of(Rollback_Stack_Block) + len(block.buffer), stack.block_allocator) block = next_block } stack.head.next_block = nil stack.head.last_alloc = nil stack.head.offset = 0 } @(private="file", require_results) rb_resize :: proc(stack: ^Rollback_Stack, ptr: rawptr, old_size, size, alignment: int) -> (result: []byte, err: Allocator_Error) { if ptr != nil { if block, _, ok := rb_find_last_alloc(stack, ptr); ok { // `block.offset` should never underflow because it is contingent // on `old_size` in the first place, assuming sane arguments. assert(block.offset >= cast(uintptr)old_size, "Rollback Stack Allocator received invalid `old_size`.") if block.offset + cast(uintptr)size - cast(uintptr)old_size < cast(uintptr)len(block.buffer) { // Prevent singleton allocations from fragmenting by forbidding // them to shrink, removing the possibility of overflow bugs. if len(block.buffer) <= stack.block_size { block.offset += cast(uintptr)size - cast(uintptr)old_size } #no_bounds_check return (cast([^]byte)ptr)[:size], nil } } } result = rb_alloc(stack, size, alignment) or_return runtime.mem_copy_non_overlapping(raw_data(result), ptr, old_size) err = rb_free(stack, ptr) return } @(private="file", require_results) rb_alloc :: proc(stack: ^Rollback_Stack, size, alignment: int) -> (result: []byte, err: Allocator_Error) { parent: ^Rollback_Stack_Block for block := stack.head; /**/; block = block.next_block { when !ODIN_DISABLE_ASSERT { allocated_new_block: bool } if block == nil { if stack.block_allocator.procedure == nil { return nil, .Out_Of_Memory } minimum_size_required := size_of(Rollback_Stack_Header) + size + alignment - 1 new_block_size := max(minimum_size_required, stack.block_size) block = rb_make_block(new_block_size, stack.block_allocator) or_return parent.next_block = block when !ODIN_DISABLE_ASSERT { allocated_new_block = true } } start := raw_data(block.buffer)[block.offset:] padding := cast(uintptr)calc_padding_with_header(cast(uintptr)start, cast(uintptr)alignment, size_of(Rollback_Stack_Header)) if block.offset + padding + cast(uintptr)size > cast(uintptr)len(block.buffer) { when !ODIN_DISABLE_ASSERT { if allocated_new_block { panic("Rollback Stack Allocator allocated a new block but did not use it.") } } parent = block continue } header := cast(^Rollback_Stack_Header)(start[padding - size_of(Rollback_Stack_Header):]) ptr := start[padding:] header^ = { prev_offset = block.offset, prev_ptr = uintptr(0) if block.last_alloc == nil else cast(uintptr)block.last_alloc - cast(uintptr)raw_data(block.buffer), is_free = false, } block.last_alloc = ptr block.offset += padding + cast(uintptr)size if len(block.buffer) > stack.block_size { // This block exceeds the allocator's standard block size and is considered a singleton. // Prevent any further allocations on it. block.offset = cast(uintptr)len(block.buffer) } #no_bounds_check return ptr[:size], nil } return nil, .Out_Of_Memory } @(private="file", require_results) rb_make_block :: proc(size: int, allocator: Allocator) -> (block: ^Rollback_Stack_Block, err: Allocator_Error) { buffer := runtime.mem_alloc(size_of(Rollback_Stack_Block) + size, align_of(Rollback_Stack_Block), allocator) or_return block = cast(^Rollback_Stack_Block)raw_data(buffer) #no_bounds_check block.buffer = buffer[size_of(Rollback_Stack_Block):] return } rollback_stack_init_buffered :: proc(stack: ^Rollback_Stack, buffer: []byte, location := #caller_location) { MIN_SIZE :: size_of(Rollback_Stack_Block) + size_of(Rollback_Stack_Header) + size_of(rawptr) assert(len(buffer) >= MIN_SIZE, "User-provided buffer to Rollback Stack Allocator is too small.", location) block := cast(^Rollback_Stack_Block)raw_data(buffer) block^ = {} #no_bounds_check block.buffer = buffer[size_of(Rollback_Stack_Block):] stack^ = {} stack.head = block stack.block_size = len(block.buffer) } rollback_stack_init_dynamic :: proc( stack: ^Rollback_Stack, block_size : int = ROLLBACK_STACK_DEFAULT_BLOCK_SIZE, block_allocator := context.allocator, location := #caller_location, ) -> Allocator_Error { assert(block_size >= size_of(Rollback_Stack_Header) + size_of(rawptr), "Rollback Stack Allocator block size is too small.", location) when size_of(int) > 4 { // It's impossible to specify an argument in excess when your integer // size is insufficient; check only on platforms with big enough ints. assert(block_size <= ROLLBACK_STACK_MAX_HEAD_BLOCK_SIZE, "Rollback Stack Allocators cannot support head blocks larger than 2 gigabytes.", location) } block := rb_make_block(block_size, block_allocator) or_return stack^ = {} stack.head = block stack.block_size = block_size stack.block_allocator = block_allocator return nil } rollback_stack_init :: proc { rollback_stack_init_buffered, rollback_stack_init_dynamic, } rollback_stack_destroy :: proc(stack: ^Rollback_Stack) { if stack.block_allocator.procedure != nil { rb_free_all(stack) free(stack.head, stack.block_allocator) } stack^ = {} } @(require_results) rollback_stack_allocator :: proc(stack: ^Rollback_Stack) -> Allocator { return Allocator { data = stack, procedure = rollback_stack_allocator_proc, } } @(require_results) rollback_stack_allocator_proc :: proc( allocator_data: rawptr, mode: Allocator_Mode, size, alignment: int, old_memory: rawptr, old_size: int, location := #caller_location, ) -> (result: []byte, err: Allocator_Error) { stack := cast(^Rollback_Stack)allocator_data switch mode { case .Alloc, .Alloc_Non_Zeroed: assert(size >= 0, "Size must be positive or zero.", location) assert(is_power_of_two(cast(uintptr)alignment), "Alignment must be a power of two.", location) result = rb_alloc(stack, size, alignment) or_return if mode == .Alloc { zero_slice(result) } case .Free: err = rb_free(stack, old_memory) case .Free_All: rb_free_all(stack) case .Resize, .Resize_Non_Zeroed: assert(size >= 0, "Size must be positive or zero.", location) assert(old_size >= 0, "Old size must be positive or zero.", location) assert(is_power_of_two(cast(uintptr)alignment), "Alignment must be a power of two.", location) result = rb_resize(stack, old_memory, old_size, size, alignment) or_return #no_bounds_check if mode == .Resize && size > old_size { zero_slice(result[old_size:]) } case .Query_Features: set := (^Allocator_Mode_Set)(old_memory) if set != nil { set^ = {.Alloc, .Alloc_Non_Zeroed, .Free, .Free_All, .Resize, .Resize_Non_Zeroed} } return nil, nil case .Query_Info: return nil, .Mode_Not_Implemented } return }