Kinda have this memory alloation strat sorted...

.vscode/c_cpp_properties.json

@@ -14,7 +14,10 @@
             "windowsSdkVersion": "10.0.22621.0",
             "compilerPath": "cl.exe",
             "cStandard": "c11",
-            "cppStandard": "c++17"
+            "cppStandard": "c++17",
+            "compilerArgs": [
+                "/Zc:preprocessor"
+            ]
         }
     ],
     "version": 4

.vscode/settings.json

@@ -24,6 +24,31 @@
 		"xiosbase": "c",
 		"iterator": "c",
 		"memory": "c",
-		"format": "c"
+		"format": "c",
-	}
+		"compare": "cpp",
+		"ratio": "cpp",
+		"tuple": "cpp",
+		"system_error": "cpp",
+		"xlocmon": "cpp"
+	},
+	"workbench.colorCustomizations": {
+		"activityBar.activeBackground": "#713fb8",
+		"activityBar.background": "#713fb8",
+		"activityBar.foreground": "#e7e7e7",
+		"activityBar.inactiveForeground": "#e7e7e799",
+		"activityBarBadge.background": "#c68659",
+		"activityBarBadge.foreground": "#15202b",
+		"commandCenter.border": "#e7e7e799",
+		"sash.hoverBorder": "#713fb8",
+		"statusBar.background": "#5a3292",
+		"statusBar.foreground": "#e7e7e7",
+		"statusBarItem.hoverBackground": "#713fb8",
+		"statusBarItem.remoteBackground": "#5a3292",
+		"statusBarItem.remoteForeground": "#e7e7e7",
+		"titleBar.activeBackground": "#5a3292",
+		"titleBar.activeForeground": "#e7e7e7",
+		"titleBar.inactiveBackground": "#5a329299",
+		"titleBar.inactiveForeground": "#e7e7e799"
+	},
+	"peacock.color": "#5a3292"
 }

code/base/arena.c

@@ -14,21 +14,8 @@
 Arena*
 arena_alloc_(ArenaParams* params)
 {
-	B32 is_virtual = false;
-
-	// This composite arena has the ability to decide a default behavior for allocation 
-
 	void* base = alloc(params->backing, params->block_size);
 	  
-	// rjf: panic on arena creation failure
-	#if OS_FEATURE_GRAPHICAL
-		if(unlikely(base == 0))
-		{
-			os_graphical_message(1, str8_lit("Fatal Allocation Failure"), str8_lit("Unexpected memory allocation failure."));
-			os_abort(1);
-		}
-	#endif
-	  
 	// rjf: extract arena header & fill
 	Arena* arena      = (Arena*) base;
 	arena->current    = arena;

code/base/arena.h

@@ -20,10 +20,10 @@ typedef U32 ArenaFlags;
 enum
 {
 	// Don't chain this arena
-	ArenaFlag_NoChain      = (1 << 0), 
+	ArenaFlag_NoChain        = (1 << 0), 
-	// Only works if backing is virtual memory, will allocate a new backing VArena when the current block exhausts
+	// Only relevant if backing is virtual memory, will prevent allocating a new backing VArena when the current block exhausts
-	// Otherwise will assume backing can chain multiple block_size arenas until failure.
+	// Will assume backing can chain multiple block_size arenas however. If there is an allocation failure it will assert.
-	ArenaFlag_ChainVirtual = (1 << 1),
+	ArenaFlag_NoChainVirtual = (1 << 1),
 };
 
 typedef struct ArenaParams ArenaParams;
@@ -34,13 +34,16 @@ struct ArenaParams
 	U64           block_size; // If chaining VArenas set this to the reserve size
 };
 
-/* NOTE(Ed): This is a combination of several concepts into a single interface:
+/* NOTE(Ed): The original metadesk arena is a combination of several concepts into a single interface:
-	* A arena 'block' of memory with segmented chaining of the blocks
 	* An OS virtual memory allocation scheme
+	* A arena 'block' of memory with segmented chaining of the blocks
 	* A push/pop stack allocation interface for the arena
 
-	TODO(Ed): We need to lift the virtual memory tracking to its own data structure 
+	The virtual memory has been abstracted into a backing allocator,
-	and virtual memory interface utilizing memory_substrate.h
+	and chaining still supports reserving new virtual address regions .
+	(can be disabled with ArenaFlag_NoChainVirtual)
+
+	If large pages are desired, see VArena.
 */
 
 typedef struct Arena Arena;
@@ -49,8 +52,8 @@ struct Arena
 	Arena*        prev;    // previous arena in chain
 	Arena*        current; // current arena in chain
 	AllocatorInfo backing;
-	U64           pos;
+	SSIZE         pos;
-	U64           block_size;
+	SSIZE         block_size;
 	ArenaFlags    flags;
 };
 static_assert(size_of(Arena) <= ARENA_HEADER_SIZE, "sizeof(Arena) <= ARENA_HEADER_SIZE");
@@ -59,7 +62,7 @@ typedef struct TempArena TempArena;
 struct TempArena
 {
 	Arena* arena;
-	U64    pos;
+	SSIZE  pos;
 };
 
 ////////////////////////////////
@@ -67,23 +70,29 @@ struct TempArena
 
 MD_API void* arena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags);
 
+inline
+AllocatorInfo arena_allocator(Arena* arena) {
+	AllocatorInfo info = { arena_allocator_proc, arena};
+	return info;
+}
+
 //- rjf: arena creation/destruction
 
-MD_API Arena* arena_alloc_(ArenaParams *params);
+MD_API Arena* arena_alloc_(ArenaParams* params);
-#define       arena_alloc(...) arena_alloc_( & ( ArenaParams) { .backing = {}, .reserve_size = MB(64), .commit_size = KB(64), __VA_ARGS__ } )
+#define       arena_alloc(...) arena_alloc_( &(ArenaParams){ __VA_ARGS__ } )
 
 void arena_release(Arena *arena);
 
 //- rjf: arena push/pop/pos core functions
 
-internal void *arena_push  (Arena *arena, U64 size, U64 align);
+internal void *arena_push  (Arena *arena, SSIZE size, SSIZE align);
 internal U64   arena_pos   (Arena *arena);
-internal void  arena_pop_to(Arena *arena, U64 pos);
+internal void  arena_pop_to(Arena *arena, SSIZE pos);
 
 //- rjf: arena push/pop helpers
 
 internal void arena_clear(Arena *arena);
-internal void arena_pop  (Arena *arena, U64 amt);
+internal void arena_pop  (Arena *arena, SSIZE amt);
 
 //- rjf: temporary arena scopes
 
@@ -110,3 +119,20 @@ arena_release(Arena* arena)
 		alloc_free(arena->backing, n);
 	}
 }
+
+// DEFAULT_ALLOCATOR
+#ifndef MD_OVERRIDE_DEFAULT_ALLOCATOR
+// The default allocator for this base module is the Arena allocator with a VArena backing
+inline
+AllocatorInfo default_allocator()
+{
+	local_persist thread_local VArena* backing_vmem = nullptr;
+	local_persist thread_local Arena*  arena        = nullptr;
+	if (arena == nullptr) {
+		backing_vmem = varena_alloc(.flags = 0, .base_addr = 0x0, .reserve_size = VARENA_DEFAULT_RESERVE, .commit_size = VARENA_DEFAULT_COMMIT);
+		arena        = arena_alloc(.backing = varena_allocator(backing_vmem), .block_size = VARENA_DEFAULT_RESERVE);
+	}
+	AllocatorInfo info = { arena_allocator_proc, arena };
+	return info;
+}
+#endif

code/base/macros.h

@@ -4,16 +4,6 @@
 #	include "linkage.h"
 #endif
 
-#ifndef local_persist
-#define local_persist static
-#endif
-
-#if COMPILER_MSVC
-#	define thread_static __declspec(thread)
-#elif COMPILER_CLANG || COMPILER_GCC
-#	define thread_static __thread
-#endif
-
 ////////////////////////////////
 //~ rjf: Branch Predictor Hints
 
@@ -89,10 +79,10 @@
 #	endif
 #endif
 
-#if ! defined(MD_PARAM_DEFAULT) && LANG_CPP
+#if ! defined(PARAM_DEFAULT) && LANG_CPP
-#	define MD_PARAM_DEFAULT = {}
+#	define PARAM_DEFAULT = {}
 #else
-#	define MD_PARAM_DEFAULT
+#	define PARAM_DEFAULT
 #endif
 
 #if LANG_C

code/base/memory.h

@@ -672,3 +672,25 @@ void* mem_set( void* destination, U8 fill_byte, SSIZE byte_count )
 #		define read_only
 #	endif
 #endif
+
+#ifndef local_persist
+#define local_persist static
+#endif
+
+#if COMPILER_MSVC
+#	define thread_static __declspec(thread)
+#elif COMPILER_CLANG || COMPILER_GCC
+#	define thread_static __thread
+#endif
+
+#if COMPILER_CPP
+// Already Defined
+#elif COMPILER_C && __STDC_VERSION__ >= 201112L
+#	define thread_local _Thread_local
+#elif COMPILER_MSVC
+#	define thread_local __declspec(thread)
+#elif COMPILER_CLANG
+#	define thread_local __thread
+#else
+#	error "No thread local support"
+#endif

code/base/memory_substrate.c

@@ -87,7 +87,7 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
 	switch ( mode )
 	{
 	#if defined( COMPILER_MSVC ) || ( defined( COMPILER_GCC ) && defined( OS_WINDOWS ) ) || ( defined( COMPILER_TINYC ) && defined( OS_WINDOWS ) )
-		case AllocatorMode_ALLOC:
+		case AllocatorMode_Alloc:
 		{
 			ptr = _aligned_malloc( size, alignment );
 			if ( flags & ALLOCATOR_FLAG_CLEAR_TO_ZERO )
@@ -95,11 +95,11 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
 		}
 		break;
 
-		case AllocatorMode_FREE:
+		case AllocatorMode_Free:
 			_aligned_free( old_memory );
 		break;
 
-		case AllocatorMode_FREE_ALL:
+		case AllocatorMode_Reisze:
 		{
 			AllocatorInfo a = heap();
 			ptr             = default_resize_align( a, old_memory, old_size, size, alignment );
@@ -118,7 +118,7 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
 		}
 		break;
 
-		case EAllocation_FREE :
+		case EAllocation_Freee :
 		{
 			free( old_memory );
 		}
@@ -156,15 +156,20 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
 		break;
 	#endif
 
-		case AllocatorMode_FREE_ALL:
+		case AllocatorMode_FreeAll:
 			break;
-		
+
 		case AllocatorMode_QueryType:
-			return (void*) Heap
+			return (void*) AllocatorType_Heap;
+
+		case AllocatorMode_QuerySupport:
+			return (void*) (
+				AllocatorQuery_Alloc | AllocatorQuery_Free | AllocatorQuery_Reisze
+			);
 	}
 
 #ifdef GEN_HEAP_ANALYSIS
-	if ( type == EAllocation_ALLOC )
+	if ( type == AllocatorMode_Alloc )
 	{
 		_heap_alloc_info* alloc_info = rcast( _heap_alloc_info*, rcast( char*, ptr) + alloc_info_remainder );
 		zero_item( alloc_info );
@@ -179,46 +184,87 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
 	return ptr;
 }
 
-void* vm_allocator_proc(void* allocator_data, AllocType type, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
+VArena varena__alloc(VArenaParams params)
-{
-	void* result = nullptr;
-
-			// if(params->flags & ArenaFlag_LargePages)
-			// {
-			// 	base = os_reserve_large(reserve_size);
-			// 	os_commit_large(base, commit_size);
-			// }
-			// else
-			// {
-			// 	base = os_reserve(reserve_size);
-			// 	os_commit(base, commit_size);
-			// }
-
-	return result;
-}
-
-VMemory vm_alloc(VMemoryParams params)
 {
 	// rjf: round up reserve/commit sizes
-	U64 reserve_size = params->reserve_size;
+	U64 reserve_size = params.reserve_size;
-	U64 commit_size  = params->commit_size;
+	U64 commit_size  = params.commit_size;
 
-	if(params->flags & ArenaFlag_LargePages)
+	void* base = nullptr;
+	if (params.flags & VArenaFlag_LargePages)
 	{
 		reserve_size = align_pow2(reserve_size, os_get_system_info()->large_page_size);
 		commit_size  = align_pow2(commit_size,  os_get_system_info()->large_page_size);
+
+		base = os_reserve_large(reserve_size);
+		os_commit_large(base, commit_size);
+		asan_poison_memory_region(base, params.commit_size);
 	}
 	else
 	{
 		reserve_size = align_pow2(reserve_size, os_get_system_info()->page_size);
 		commit_size  = align_pow2(commit_size,  os_get_system_info()->page_size);
+
+		base = os_reserve(reserve_size);
+		os_commit(base, commit_size);
+		asan_poison_memory_region(base, params.commit_size);
 	}
 
-	// Allocate virtual memory
+	// NOTE(Ed): Panic on varena creation failure
-	is_virtual = true;
+	#if OS_FEATURE_GRAPHICAL
+		if(unlikely(base == 0))
+		{
+			os_graphical_message(1, str8_lit("Fatal Allocation Failure"), str8_lit("Unexpected memory allocation failure."));
+			os_abort(1);
+		}
+	#endif
 
-	VMemory vm = ;
+	SPTR header_size = size_of(VArena);
 
-	// TODO(Ed): Move this to vmem?
+	VArena* vm = (VArena* ) base;
-	asan_poison_memory_region(base, params->commit_size);
+	vm->reserve       = reserve_size;
+	vm->committed     = commit_size;
+	vm->reserve_start = (SPTR)base + header_size;
+	vm->flags         = params.flags;
+	vm->commit_used   = 0;
+	asan_unpoison_memory_region(vm, header_size);
 }
+
+void Varena_release(VArena* arena)
+{
+
+
+	arena = nullptr;
+}
+
+void* varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
+{
+	void* result = nullptr;
+
+	switch (mode)
+	{
+		case AllocatorMode_Alloc:
+
+		break;
+
+		case AllocatorMode_Free:
+		break;
+
+		case AllocatorMode_FreeAll:
+		break;
+
+		case AllocatorMode_Reisze:
+		break;
+
+		case AllocatorMode_QueryType:
+		break;
+
+		case AllocatorMode_QuerySupport:
+		break;
+	}
+
+
+
+	return result;
+}
+

code/base/memory_substrate.h

@@ -20,7 +20,7 @@
 #endif
 
 // Return value of allocator_type
-typedef U32 AllocatorType;
+typedef U64 AllocatorType;
 enum
 {
 	AllocatorType_Heap   = 0, // Genreal heap allocator
@@ -30,13 +30,22 @@ enum
 };
 
 typedef U32 AllocatorMode;
-enum AllocatorMode enum_underlying(U32)
+enum AllocatorMode
 {
-	AllocatorMode_ALLOC,
+	AllocatorMode_Alloc,
-	AllocatorMode_FREE,
+	AllocatorMode_Free,
-	AllocatorMode_FREE_ALL,
+	AllocatorMode_FreeAll,
-	AllocatorMode_RESIZE,
+	AllocatorMode_Reisze,
 	AllocatorMode_QueryType,
+	AllocatorMode_QuerySupport,
+};
+typedef U64 AllocatorQueryFlags;
+enum
+{
+	AllocatorQuery_Alloc     = (1 << 0),
+	AllocatorQuery_Free      = (1 << 1),
+	AllocatorQuery_FreeAll   = (1 << 2),
+	AllocatorQuery_Reisze    = (1 << 3),
 };
 
 typedef void*(AllocatorProc)( void* allocator_data, AllocatorMode type, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags );
@@ -48,6 +57,9 @@ struct AllocatorInfo
 	void*          data;
 };
 
+// Overridable by the user by defining MD_OVERRIDE_DEFAULT_ALLOCATOR
+AllocatorInfo default_allocator();
+
 enum AllocFlag
 {
 	ALLOCATOR_FLAG_CLEAR_TO_ZERO = 0,
@@ -61,40 +73,42 @@ enum AllocFlag
 #	define MD_DEFAULT_ALLOCATOR_FLAGS ( ALLOCATOR_FLAG_CLEAR_TO_ZERO )
 #endif
 
-// Allows the user to retrieve which type of allocator is being used.
+// Retrieve which type of allocator
 AllocatorType allocator_type(AllocatorInfo a);
+// Retreive which modes the allocator supports
+AllocatorQueryFlags allocator_query_support(AllocatorInfo a);
 
-//! Allocate memory with default alignment.
+// Allocate memory with default alignment.
 void* alloc( AllocatorInfo a, SSIZE size );
 
-//! Allocate memory with specified alignment.
+// Allocate memory with specified alignment.
 void* alloc_align( AllocatorInfo a, SSIZE size, SSIZE alignment );
 
-//! Free allocated memory.
+// Free allocated memory.
 void alloc_free( AllocatorInfo a, void* ptr );
 
-//! Free all memory allocated by an allocator.
+// Free all memory allocated by an allocator.
 void free_all( AllocatorInfo a );
 
-//! Resize an allocated memory.
+// Resize an allocated memory.
 void* resize( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size );
 
-//! Resize an allocated memory with specified alignment.
+// Resize an allocated memory with specified alignment.
 void* resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size, SSIZE alignment );
 
 #ifndef alloc_item
-//! Allocate memory for an item.
+// Allocate memory for an item.
 #define alloc_item( allocator_, Type ) ( Type* )alloc( allocator_, size_of( Type ) )
 #endif
 
 #ifndef alloc_array
-//! Allocate memory for an array of items.
+// Allocate memory for an array of items.
 #define alloc_array( allocator_, Type, count ) ( Type* )alloc( allocator_, size_of( Type ) * ( count ) )
 #endif
 
-//! Allocate/Resize memory using default options.
+// Allocate/Resize memory using default options.
 
-//! Use this if you don't need a "fancy" resize allocation
+// Use this if you don't need a "fancy" resize allocation
 void* default_resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size, SSIZE alignment );
 
 #ifdef MD_HEAP_ANALYSIS
@@ -112,17 +126,17 @@ MD_API void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZ
 
 #ifndef heap
 //! The heap allocator backed by operating system's memory manager.
-#define heap() (AllocatorInfo){ AllocatorInfo allocator = { heap_allocator_proc, nullptr }; return allocator; }
+#define heap() (AllocatorInfo){ heap_allocator_proc, nullptr }
 #endif
 
-#ifndef malloc
+#ifndef md_malloc
 //! Helper to allocate memory using heap allocator.
-#define malloc( sz ) alloc( heap(), sz )
+#define md_malloc( sz ) alloc( heap(), sz )
 #endif
 
-#ifndef mfree
+#ifndef md_free
 //! Helper to free memory allocated by heap allocator.
-#define mfree( ptr ) free( heap(), ptr )
+#define md_free( ptr ) alloc_free( heap(), ptr )
 #endif
 
 /* Virtual Memory Arena
@@ -135,6 +149,13 @@ MD_API void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZ
 	Like with the composite Arena, the VArena has its struct as the header of the reserve of memory.
 */
 
+#ifndef VARENA_DEFUALT_RESERVE
+#define VARENA_DEFAULT_RESERVE MB(64)
+#endif
+#ifndef VARENA_DEFUALT_COMMIT
+#define VARENA_DEFAULT_COMMIT  KB(64)
+#endif
+
 typedef U32 VArenaFlags;
 enum
 {
@@ -154,22 +175,22 @@ typedef struct VArena VArena;
 struct VArena
 {
 	VArenaFlags flags;
-	U64 base_pos;
+	SSIZE reserve_start;
-	U64 cmt;
+	SSIZE reserve;
-	U64 res;
+	SSIZE committed;
-	U64 res_size;
+	SSIZE commit_used;
-	U64 cmt_size;
 };
 
 AllocatorInfo vm_allocator(VArena* vm) {
-	AllocatorInfo info = { vm_allocator_proc, vm };
+	AllocatorInfo info = { varena_allocator_proc, vm };
 	return info;
 }
 
-VArena*      varena_alloc    (VArenaParams params);
+VArena* varena__alloc(VArenaParams params PARAM_DEFAULT);
+#define varena_alloc(...) varena__alloc( (VArenaParams){__VA_ARGS__} )
+
 void         varena_commit   (VArena vm, SSIZE commit_size);
-VArenaParams varena_free     (VArena vm);
+VArenaParams varena_release  (VArena vm);
-SSIZE        varena_page_size(SSIZE* alignment_out);
 
 MD_API void* varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags);
 
@@ -185,7 +206,7 @@ typedef struct FArena FArena;
 struct FArena
 {
 	ByteSlice slice;
-	SSIZE     pos;
+	SSIZE     used;
 };
 
 FArena farena_from_byteslice(ByteSlice slice);
@@ -199,6 +220,11 @@ AllocatorType allocator_type(AllocatorInfo a) {
 	return (AllocatorType) a.proc(a.data, AllocatorMode_QueryType, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS);
 }
 
+inline
+AllocatorQueryFlags allocator_query_support(AllocatorInfo a) {
+	return (AllocatorType) a.proc(a.data, AllocatorMode_QuerySupport, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS);
+}
+
 inline
 void* alloc_align( AllocatorInfo a, SSIZE size, SSIZE alignment ) {
 	return a.proc( a.data, AllocatorMode_ALLOC, size, alignment, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS );

code/mdesk/mdesk.h

@@ -2,7 +2,7 @@
 #pragma once
 #include "base/base_types.h"
 #include "base/math.h"
-#include "base/rarena.h"
+#include "base/arena.h"
 #include "base/string.h"
 #endif
 
@@ -299,6 +299,8 @@ internal B32 md_node_match(MD_Node *a, MD_Node *b, StringMatchFlags flags);
 
 //- rjf: tree duplication
 internal MD_Node *md_tree_copy(Arena *arena, MD_Node *src_root);
+// MD_Node* tree_copy_arena(MD_Node* src_root, Arena* arena);
+// MD_Node* tree_copy_ainfo(MD_Node* src_root, AllocatorInfo info);
 
 ////////////////////////////////
 //~ rjf: Text -> Tokens Functions

code/metadesk.h

@@ -4,10 +4,6 @@
 
 // metadesk header: intended for "As-Is" library usage
 
-// os
-
-
-
 // base
 
 #include "base/context_cracking.h"