From 58ba273dd17fbbad64b1625b6e5e0527f96be6d5 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 18 Oct 2025 20:46:06 -0400 Subject: [PATCH] code2: initial curation of virtual arena --- code2/grime/allocator_interface.odin | 7 +- code2/grime/fixed_arena.odin | 4 +- code2/grime/pkg_mappings.odin | 3 +- code2/grime/virtual_arena.odin | 276 ++++++++++++++++++++++++++- code2/host/host.odin | 2 + code2/sectr/engine/client_api.odin | 3 +- code2/sectr/input/events.odin | 14 +- 7 files changed, 287 insertions(+), 22 deletions(-) diff --git a/code2/grime/allocator_interface.odin b/code2/grime/allocator_interface.odin index 3395533..ca1ec96 100644 --- a/code2/grime/allocator_interface.odin +++ b/code2/grime/allocator_interface.odin @@ -115,7 +115,7 @@ AllocatorInfo :: struct { // Listing of every single allocator (used on hot-reloadable builds) AllocatorProcID :: enum uintptr { FArena, - // VArena, + VArena, // CArena, // Pool, // Slab, @@ -127,7 +127,7 @@ resolve_allocator_proc :: #force_inline proc "contextless" (procedure: $Allocato when ODIN_DEBUG { switch (transmute(AllocatorProcID)procedure) { case .FArena: return farena_allocator_proc - // case .VArena: return varena_allocaotr_proc + case .VArena: return varena_allocator_proc // case .CArena: return carena_allocator_proc // case .Pool: return pool_allocator_proc // case .Slab: return slab_allocator_proc @@ -145,7 +145,7 @@ resolve_odin_allocator :: #force_inline proc "contextless" (allocator: Odin_Allo when ODIN_DEBUG { switch (transmute(AllocatorProcID)allocator.procedure) { case .FArena: return { farena_odin_allocator_proc, allocator.data } - // case .VArena: return { varena_odin_allocaotr_proc, allocator.data } + case .VArena: return { varena_odin_allocator_proc, allocator.data } // case .CArena: return { carena_odin_allocator_proc, allocator.data } // case .Pool: return nil // pool_allocator_proc // case .Slab: return nil // slab_allocator_proc @@ -177,6 +177,7 @@ odin_allocator_mode_to_allocator_op :: #force_inline proc "contextless" (mode: O panic_contextless("Impossible path") } +// TODO(Ed): Change to DEFAULT_ALIGNMENT MEMORY_ALIGNMENT_DEFAULT :: 2 * size_of(rawptr) allocatorinfo :: #force_inline proc(ainfo := context.allocator) -> AllocatorInfo { return transmute(AllocatorInfo) ainfo } diff --git a/code2/grime/fixed_arena.odin b/code2/grime/fixed_arena.odin index b31b54f..2847810 100644 --- a/code2/grime/fixed_arena.odin +++ b/code2/grime/fixed_arena.odin @@ -172,6 +172,6 @@ when ODIN_DEBUG { farena_allocator :: #force_inline proc "contextless" (arena: ^FArena) -> Odin_Allocator { return transmute(Odin_Allocator) AllocatorInfo{proc_id = .FArena, data = arena} } } else { - farena_ainfo :: #force_inline proc "contextless" (arena: ^FArena) -> AllocatorInfo { return AllocatorInfo{procedure = farena_allocator_proc, data = arena} } - farena_allocator :: #force_inline proc "contextless" (arena: ^FArena) -> Odin_Allocator { return transmute(Odin_Allocator) AllocatorInfo{procedure = farena_allocator_proc, data = arena} } + farena_ainfo :: #force_inline proc "contextless" (arena: ^FArena) -> AllocatorInfo { return AllocatorInfo{procedure = farena_allocator_proc, data = arena} } + farena_allocator :: #force_inline proc "contextless" (arena: ^FArena) -> Odin_Allocator { return transmute(Odin_Allocator) AllocatorInfo{procedure = farena_odin_allocator_proc, data = arena} } } diff --git a/code2/grime/pkg_mappings.odin b/code2/grime/pkg_mappings.odin index a470618..a74af1f 100644 --- a/code2/grime/pkg_mappings.odin +++ b/code2/grime/pkg_mappings.odin @@ -6,6 +6,7 @@ import "base:builtin" import "base:intrinsics" atomic_thread_fence :: intrinsics.atomic_thread_fence mem_zero_volatile :: intrinsics.mem_zero_volatile + add_overflow :: intrinsics.overflow_add // mem_zero :: intrinsics.mem_zero // mem_copy :: intrinsics.mem_copy_non_overlapping // mem_copy_overlapping :: intrinsics.mem_copy @@ -140,7 +141,7 @@ copy :: proc { mem_copy, slice_copy, } -copy_non_overlaping :: proc { +copy_non_overlapping :: proc { mem_copy_non_overlapping, slice_copy_overlapping, } diff --git a/code2/grime/virtual_arena.odin b/code2/grime/virtual_arena.odin index 4ceb589..0ba77b2 100644 --- a/code2/grime/virtual_arena.odin +++ b/code2/grime/virtual_arena.odin @@ -1,4 +1,7 @@ package grime + +import "base:runtime" + /* So this is a virtual memory backed arena allocator designed to take advantage of one large contigous reserve of memory. @@ -11,7 +14,6 @@ No other part of the program will directly touch the vitual memory interface dir Thus for the scope of this prototype the Virtual Arena are the only interfaces to dynamic address spaces for the runtime of the client app. The host application as well ideally (although this may not be the case for a while) */ -VArena_GrowthPolicyProc :: #type proc( commit_used, committed, reserved, requested_size : uint ) -> uint VArena :: struct { using vmem: VirtualMemoryRegion, @@ -19,7 +21,277 @@ VArena :: struct { dbg_name: string, commit_used: uint, growth_policy: VArena_GrowthPolicyProc, - allow_any_resize: b32, + allow_any_resize: b64, mutex: Mutex, } +VArena_GrowthPolicyProc :: #type proc(commit_used, committed, reserved, requested_size: uint) -> uint +varena_default_growth_policy :: proc(commit_used, committed, reserved, requested_size: uint) -> uint { + @static commit_limit := uint(1 * Mega) + @static increment := uint(16 * Kilo) + page_size := uint(virtual_get_page_size()) + if increment < Giga && committed > commit_limit { + commit_limit *= 2 + increment *= 2 + increment = clamp(increment, Mega, Giga) + } + remaining_reserve := reserved - committed + growth_increment := max( increment, requested_size ) + growth_increment = clamp( growth_increment, page_size, remaining_reserve ) + next_commit_size := memory_align_formula( committed + growth_increment, page_size ) + return next_commit_size +} + +// Default growth_policy is varena_default_growth_policy +varena_init :: proc(base_address: uintptr, to_reserve, to_commit: uint, + growth_policy: VArena_GrowthPolicyProc = varena_default_growth_policy, + allow_any_resize: bool = false, + dbg_name: string = "", + enable_mem_tracking: bool = false, +) -> (arena: VArena, alloc_error: AllocatorError) +{ + page_size := uint(virtual_get_page_size()) + verify( page_size > size_of(VirtualMemoryRegion), "Make sure page size is not smaller than a VirtualMemoryRegion?") + verify( to_reserve >= page_size, "Attempted to reserve less than a page size" ) + verify( to_commit >= page_size, "Attempted to commit less than a page size") + verify( to_reserve >= to_commit, "Attempted to commit more than there is to reserve" ) + vmem : VirtualMemoryRegion + vmem, alloc_error = virtual_reserve_and_commit( base_address, to_reserve, to_commit ) + if ensure(vmem.base_address == nil || alloc_error != .None, "Failed to allocate requested virtual memory for virtual arena") { + return + } + arena.vmem = vmem + arena.commit_used = 0 + if growth_policy == nil do arena.growth_policy = varena_default_growth_policy + else do arena.growth_policy = growth_policy + arena.allow_any_resize = b64(allow_any_resize) + if Track_Memory && enable_mem_tracking { + memtracker_init( & arena.tracker, runtime.heap_allocator(), Kilo * 128, dbg_name) + } + return +} +varena_alloc :: proc(using self: ^VArena, + size: int, + alignment: int = MEMORY_ALIGNMENT_DEFAULT, + zero_memory := true, + location := #caller_location +) -> (data: []byte, alloc_error: AllocatorError) +{ + verify( alignment & (alignment - 1) == 0, "Non-power of two alignment", location = location ) + page_size := uint(virtual_get_page_size()) + requested_size := uint(size) + if ensure(requested_size == 0, "Requested 0 size") do return nil, .Invalid_Argument + // ensure( requested_size > page_size, "Requested less than a page size, going to allocate a page size") + // requested_size = max(requested_size, page_size) + + // TODO(Ed): Prevent multiple threads from entering here extrusively? + // sync.mutex_guard( & mutex ) + + alignment_offset := uint(0) + current_offset := uintptr(self.reserve_start) + uintptr(commit_used) + mask := uintptr(alignment - 1) + if (current_offset & mask != 0) do alignment_offset = uint(alignment) - uint(current_offset & mask) + + size_to_allocate, overflow_signal := add_overflow( requested_size, alignment_offset ) + if overflow_signal do return {}, .Out_Of_Memory + to_be_used : uint + to_be_used, overflow_signal = add_overflow( commit_used, size_to_allocate ) + if (overflow_signal || to_be_used > reserved) do return {}, .Out_Of_Memory + + header_offset := uint( uintptr(reserve_start) - uintptr(base_address) ) + commit_left := committed - commit_used - header_offset + needs_more_committed := commit_left < size_to_allocate + if needs_more_committed { + profile("VArena Growing") + next_commit_size := growth_policy( commit_used, committed, reserved, size_to_allocate ) + alloc_error = virtual_commit( vmem, next_commit_size ) + if alloc_error != .None do return + } + data_ptr := ([^]byte)(current_offset + uintptr(alignment_offset)) + data = slice( data_ptr, requested_size ) + commit_used += size_to_allocate + alloc_error = .None + // log_backing: [Kilobyte * 16]byte; backing_slice := log_backing[:] + // log( str_pfmt_buffer( backing_slice, "varena alloc - BASE: %p PTR: %X, SIZE: %d", cast(rawptr) self.base_address, & data[0], requested_size) ) + if zero_memory { + // log( str_pfmt_buffer( backing_slice, "Zeroring data (Range: %p to %p)", raw_data(data), cast(rawptr) (uintptr(raw_data(data)) + uintptr(requested_size)))) + // zero( data ) + mem_zero( data_ptr, int(requested_size) ) + } + if Track_Memory && self.tracker.entries.header != nil { + memtracker_register_auto_name( & tracker, & data[0], & data[len(data) - 1] ) + } + return +} +varena_grow :: #force_inline proc(self: ^VArena, old_memory: []byte, requested_size: int, alignment: int = MEMORY_ALIGNMENT_DEFAULT, should_zero := true, loc := #caller_location) -> (data: []byte, error: AllocatorError) +{ + if ensure(old_memory == nil, "Growing without old_memory?") { + data, error = varena_alloc(self, requested_size, alignment, should_zero, loc) + return + } + if ensure(requested_size == len(old_memory), "Requested grow when none needed") { + data = old_memory + return + } + alignment_offset := uintptr(cursor(old_memory)) & uintptr(alignment - 1) + if ensure(alignment_offset == 0 && requested_size < len(old_memory), "Requested a shrink from varena_grow") { + data = old_memory + return + } + old_memory_offset := cursor(old_memory)[len(old_memory):] + current_offset := self.reserve_start[self.commit_used:] + when false { + if old_size < page_size { + // We're dealing with an allocation that requested less than the minimum allocated on vmem. + // Provide them more of their actual memory + data = slice(transmute([^]byte)old_memory, size ) + return + } + } + verify( old_memory_offset == current_offset || bool(self.allow_any_resize), + "Cannot grow existing allocation in vitual arena to a larger size unless it was the last allocated" ) + + log_backing: [Kilo * 16]byte + backing_slice := log_backing[:] + if old_memory_offset != current_offset && self.allow_any_resize + { + // Give it new memory and copy the old over. Old memory is unrecoverable until clear. + new_region : []byte + new_region, error = varena_alloc( self, requested_size, alignment, should_zero, loc ) + if ensure(new_region == nil || error != .None, "Failed to grab new region") { + data = old_memory + if Track_Memory && self.tracker.entries.header != nil { + memtracker_register_auto_name( & self.tracker, & data[0], & data[len(data) - 1] ) + } + return + } + copy_non_overlapping( cursor(new_region), cursor(old_memory), len(old_memory) ) + data = new_region + // log_print_fmt("varena resize (new): old: %p %v new: %p %v", old_memory, old_size, (& data[0]), size) + if Track_Memory && self.tracker.entries.header != nil { + memtracker_register_auto_name( & self.tracker, & data[0], & data[len(data) - 1] ) + } + return + } + new_region : []byte + new_region, error = varena_alloc( self, requested_size - len(old_memory), alignment, should_zero, loc) + if ensure(new_region == nil || error != .None, "Failed to grab new region") { + data = old_memory + return + } + data = slice(cursor(old_memory), requested_size ) + // log_print_fmt("varena resize (expanded): old: %p %v new: %p %v", old_memory, old_size, (& data[0]), size) + if Track_Memory && self.tracker.entries.header != nil { + memtracker_register_auto_name( & self.tracker, & data[0], & data[len(data) - 1] ) + } + return +} +varena_shrink :: proc(self: ^VArena, memory: []byte, requested_size: int, loc := #caller_location) -> (data: []byte, error: AllocatorError) +{ + if requested_size == len(memory) { return memory, .None } + if ensure(memory == nil, "Shrinking without old_memory?") do return memory, .Invalid_Argument + current_offset := self.reserve_start[self.commit_used:] + shrink_amount := len(memory) - requested_size + if shrink_amount < 0 do return memory, .None + assert(cursor(memory) == current_offset) + self.commit_used -= uint(shrink_amount) + return memory[:requested_size], .None +} +varena_reset :: #force_inline proc(self: ^VArena) { + // TODO(Ed): Prevent multiple threads from entering here extrusively? + // sync.mutex_guard( & mutex ) + self.commit_used = 0 + if Track_Memory && self.tracker.entries.header != nil { + array_clear(self.tracker.entries) + } +} +varena_release :: #force_inline proc(self: ^VArena) { + // TODO(Ed): Prevent multiple threads from entering here extrusively? + // sync.mutex_guard( & mutex ) + virtual_release( self.vmem ) + self.commit_used = 0 +} +varena_rewind :: #force_inline proc(arena: ^VArena, save_point: AllocatorSP, loc := #caller_location) { + assert_contextless(save_point.type_sig == varena_allocator_proc) + assert_contextless(save_point.slot >= 0 && save_point.slot <= int(arena.commit_used)) + arena.commit_used = cast(uint) save_point.slot +} +varena_save :: #force_inline proc(arena: ^VArena) -> AllocatorSP { return AllocatorSP { type_sig = varena_allocator_proc, slot = cast(int) arena.commit_used }} + +varena_allocator_proc :: proc(input: AllocatorProc_In, output: ^AllocatorProc_Out) { + assert(output != nil) + assert(input.data != nil) + arena := transmute(^VArena) input.data + switch input.op { + case .Alloc, .Alloc_NoZero: + output.allocation, output.error = varena_alloc(arena, input.requested_size, input.alignment, input.op == .Alloc, input.loc) + return + case .Free: + output.error = .Mode_Not_Implemented + case .Reset: + varena_reset(arena) + case .Grow, .Grow_NoZero: + output.allocation, output.error = varena_grow(arena, input.old_allocation, input.requested_size, input.alignment, input.op == .Grow, input.loc) + case .Shrink: + output.allocation, output.error = varena_shrink(arena, input.old_allocation, input.requested_size) + case .Rewind: + varena_rewind(arena, input.save_point) + case .SavePoint: + output.save_point = varena_save(arena) + case .Query: + output.features = {.Alloc, .Reset, .Grow, .Shrink, .Rewind} + output.max_alloc = int(arena.reserved - arena.commit_used) + output.min_alloc = 0 + output.left = output.max_alloc + output.save_point = varena_save(arena) + } +} +varena_odin_allocator_proc :: proc( + allocator_data : rawptr, + mode : Odin_AllocatorMode, + size : int, + alignment : int, + old_memory : rawptr, + old_size : int, + location : SourceCodeLocation = #caller_location +) -> (data: []byte, alloc_error: AllocatorError) +{ + arena := transmute( ^VArena) allocator_data + page_size := uint(virtual_get_page_size()) + switch mode { + case .Alloc, .Alloc_Non_Zeroed: + data, alloc_error = varena_alloc( arena, size, alignment, (mode == .Alloc), location ) + return + case .Free: + alloc_error = .Mode_Not_Implemented + case .Free_All: + varena_reset( arena ) + case .Resize, .Resize_Non_Zeroed: + if size > old_size do varena_grow (arena, slice(cursor(old_memory), old_size), size, alignment, (mode == .Alloc), location) + else do varena_shrink(arena, slice(cursor(old_memory), old_size), size, location) + case .Query_Features: + set := cast( ^Odin_AllocatorModeSet) old_memory + if set != nil do (set ^) = {.Alloc, .Alloc_Non_Zeroed, .Free_All, .Resize, .Query_Features} + case .Query_Info: + info := (^Odin_AllocatorQueryInfo)(old_memory) + info.pointer = transmute(rawptr) varena_save(arena).slot + info.size = cast(int) arena.reserved + info.alignment = MEMORY_ALIGNMENT_DEFAULT + return to_bytes(info), nil + } + return +} + +varena_odin_allocator :: proc(arena: ^VArena) -> (allocator: Odin_Allocator) { + allocator.procedure = varena_odin_allocator_proc + allocator.data = arena + return +} +when ODIN_DEBUG { + varena_ainfo :: #force_inline proc "contextless" (arena: ^VArena) -> AllocatorInfo { return AllocatorInfo{proc_id = .VArena, data = arena} } + varena_allocator :: #force_inline proc "contextless" (arena: ^VArena) -> Odin_Allocator { return transmute(Odin_Allocator) AllocatorInfo{proc_id = .VArena, data = arena} } +} +else { + varena_ainfo :: #force_inline proc "contextless" (arena: ^VArena) -> AllocatorInfo { return AllocatorInfo{procedure = varena_allocator_proc, data = arena} } + varena_allocator :: #force_inline proc "contextless" (arena: ^VArena) -> Odin_Allocator { return transmute(Odin_Allocator) AllocatorInfo{procedure = varena_odin_allocator_proc, data = arena} } +} diff --git a/code2/host/host.odin b/code2/host/host.odin index b80875b..2af745b 100644 --- a/code2/host/host.odin +++ b/code2/host/host.odin @@ -151,6 +151,8 @@ main :: proc() if thread_memory.id == .Master_Prepper { thread_join_multiple(.. host_memory.threads[1:THREAD_TICK_LANES + THREAD_JOB_WORKERS]) } + + host_memory.client_api.shutdown(); unload_client_api( & host_memory.client_api ) diff --git a/code2/sectr/engine/client_api.odin b/code2/sectr/engine/client_api.odin index 2eb2273..b8f0b97 100644 --- a/code2/sectr/engine/client_api.odin +++ b/code2/sectr/engine/client_api.odin @@ -100,7 +100,8 @@ startup :: proc(host_mem: ^ProcessMemory, thread_mem: ^ThreadMemory) log_print_fmt("Startup time: %v ms", startup_ms) } -// For some reason odin's symbols conflict with native foreign symbols... +// NOTE(Ed): For some reason odin's symbols conflict with native foreign symbols... +// Called in host.main after all tick lane or job worker threads have joined. @export sectr_shutdown :: proc() { diff --git a/code2/sectr/input/events.odin b/code2/sectr/input/events.odin index 02a79f2..e59ae90 100644 --- a/code2/sectr/input/events.odin +++ b/code2/sectr/input/events.odin @@ -183,11 +183,9 @@ poll_input_events :: proc( input, prev_input : ^InputState, input_events : Input for prev_key, id in prev_input.keyboard.keys { input.keyboard.keys[id].ended_down = prev_key.ended_down } - for prev_btn, id in prev_input.mouse.btns { input.mouse.btns[id].ended_down = prev_btn.ended_down } - input.mouse.raw_pos = prev_input.mouse.raw_pos input.mouse.pos = prev_input.mouse.pos @@ -200,7 +198,6 @@ poll_input_events :: proc( input, prev_input : ^InputState, input_events : Input if events.num > 0 { last_frame = peek_back( events).frame_id } - // No new events, don't update if last_frame == prev_frame do return @@ -232,7 +229,6 @@ poll_input_events :: proc( input, prev_input : ^InputState, input_events : Input } } } - Iterate_Mouse_Events: { iter_obj := iterator( & mouse_events ); iter := & iter_obj @@ -241,17 +237,13 @@ poll_input_events :: proc( input, prev_input : ^InputState, input_events : Input if last_frame > event.frame_id { break } - process_digital_btn :: proc( btn : ^DigitalBtn, prev_btn : DigitalBtn, ended_down : b32 ) { first_transition := btn.half_transitions == 0 - btn.half_transitions += 1 btn.ended_down = ended_down } - - // logf("mouse event: %v", event) - + // log_print_fmt("mouse event: %v", event) #partial switch event.type { case .Mouse_Pressed: btn := & input.mouse.btns[event.btn] @@ -277,22 +269,18 @@ poll_input_events :: proc( input, prev_input : ^InputState, input_events : Input input.mouse.delta = event.delta * { 1, -1 } } } - prev_frame = last_frame } input_event_iter :: #force_inline proc () -> FRingBufferIterator(InputEvent) { return iterator_ringbuf_fixed( & memory.client_memory.input_events.events ) } - input_key_event_iter :: #force_inline proc() -> FRingBufferIterator(InputKeyEvent) { return iterator_ringbuf_fixed( & memory.client_memory.input_events.key_events ) } - input_mouse_event_iter :: #force_inline proc() -> FRingBufferIterator(InputMouseEvent) { return iterator_ringbuf_fixed( & memory.client_memory.input_events.mouse_events ) } - input_codes_pressed_slice :: #force_inline proc() -> []rune { return to_slice( memory.client_memory.input_events.codes_pressed ) }