Alot (see description)

- Made a better global allocator for the process.
- Some small fixes to gen.hpp, removed clear_code_memory as I'm designing this library to for now never free any memory.
- Fixes to memory usage for cached strings
- Added missing verification for attributes in some upfront constructors. Added attribute param for def_type procedure.
- Started to use internal and global keywords in gen.cpp for associated definitions
- Progress toward getting the parsing constructors to support operator definitions.
- There was an *attempt* to get parse_type to support parsing function types. Its not tested yet....
  - Its not an nice setup, there is no validation of parameters, problably will add that in the future.

project/Bloat.cpp

@@ -6,33 +6,100 @@ namespace Memory
 {
 	using namespace zpl;
 
-	Arena Global_Arena {};
+	global AllocatorInfo GlobalAllocator;
+
+	global Array(Arena) Global_AllocatorBuckets;
+
+	void* Global_Allocator_Proc( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags )
+	{
+		Arena* last = & array_back( Global_AllocatorBuckets );
+
+		switch ( type )
+		{
+			case EAllocationALLOC:
+			{
+				if ( last->total_allocated + size > last->total_size )
+				{
+					Arena bucket;
+					arena_init_from_allocator( & bucket, heap(), BucketSize );
+
+					if ( bucket.physical_start == nullptr )
+						fatal( "Failed to create bucket for Global_AllocatorBuckets");
+
+					if ( ! array_append( Global_AllocatorBuckets, bucket ) )
+						fatal( "Failed to append bucket to Global_AllocatorBuckets");
+
+					last = & array_back( Global_AllocatorBuckets );
+				}
+
+				return alloc_align( arena_allocator( last), size, alignment );
+			}
+			case EAllocationFREE:
+			{
+				// Doesn't recycle.
+			}
+			case EAllocationFREE_ALL:
+			{
+				// Memory::cleanup instead.
+			}
+			case EAllocationRESIZE:
+			{
+				if ( last->total_allocated + size > last->total_size )
+				{
+					Arena bucket;
+					arena_init_from_allocator( & bucket, heap(), BucketSize );
+
+					if ( bucket.physical_start == nullptr )
+						fatal( "Failed to create bucket for Global_AllocatorBuckets");
+
+					if ( ! array_append( Global_AllocatorBuckets, bucket ) )
+						fatal( "Failed to append bucket to Global_AllocatorBuckets");
+
+					last = & array_back( Global_AllocatorBuckets );
+				}
+
+				void* result = alloc_align( arena_allocator( last), size, alignment );
+
+				if ( result != nullptr && old_memory != nullptr )
+				{
+					mem_copy( result, old_memory, size );
+				}
+
+				return result;
+			}
+		}
+
+		return nullptr;
+	}
 
 	void setup()
 	{
-		arena_init_from_allocator( & Global_Arena, heap(), Initial_Reserve );
+		GlobalAllocator = AllocatorInfo { & Global_Allocator_Proc, nullptr };
 
-		if ( Global_Arena.total_size == 0 )
-		{
-			assert_crash( "Failed to reserve memory for Tests:: Global_Arena" );
-		}
-	}
+		if ( ! array_init_reserve( Global_AllocatorBuckets, heap(), 128 ) )
+			fatal( "Failed to reserve memory for Global_AllocatorBuckets");
 
-	void resize( uw new_size )
-	{
-		void* new_memory = resize( heap(), Global_Arena.physical_start, Global_Arena.total_size, new_size );
+		Arena bucket;
+		arena_init_from_allocator( & bucket, heap(), BucketSize );
 
-		if ( new_memory == nullptr )
-		{
-			fatal("Failed to resize global arena!");
-		}
+		if ( bucket.physical_start == nullptr )
+			fatal( "Failed to create first bucket for Global_AllocatorBuckets");
 
-		Global_Arena.physical_start = new_memory;
-		Global_Arena.total_size     = new_size;
+		array_append( Global_AllocatorBuckets, bucket );
 	}
 
 	void cleanup()
 	{
-		arena_free( & Global_Arena);
+		s32 index = 0;
+		s32 left  = array_count( Global_AllocatorBuckets );
+		do
+		{
+			Arena* bucket = & Global_AllocatorBuckets[ index ];
+			arena_free( bucket );
+			index++;
+		}
+		while ( left--, left );
+
+		array_free( Global_AllocatorBuckets );
 	}
 }

project/Bloat.hpp

@@ -61,6 +61,7 @@ using zpl::EFileMode_WRITE;
 using zpl::EFileError_NONE;
 
 using zpl::alloc;
+using zpl::alloc_align;
 using zpl::arena_allocator;
 using zpl::arena_init_from_memory;
 using zpl::arena_init_from_allocator;
@@ -70,6 +71,8 @@ using zpl::str_fmt_buf;
 using zpl::char_first_occurence;
 using zpl::char_is_alpha;
 using zpl::char_is_alphanumeric;
+using zpl::char_is_digit;
+using zpl::char_is_hex_digit;
 using zpl::char_is_space;
 using zpl::crc32;
 using zpl::free_all;
@@ -564,16 +567,17 @@ char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED";
 
 namespace Memory
 {
-	constexpr uw Initial_Reserve = megabytes(10);
+	// NOTE: This limits the size of the string that can be read from a file or generated to 10 megs.
+	// If you are generating a string larger than this, increase the size of the bucket here.
+	constexpr uw BucketSize = megabytes(10);
 
-	extern Arena Global_Arena;
-	// #define g_allocator arena_allocator( & Memory::Global_Arena)
+	// Global allocator used for data with process lifetime.
+	extern AllocatorInfo GlobalAllocator;
 
 	// Heap allocator is being used for now to isolate errors from being memory related (tech debt till ready to address)
-	#define g_allocator heap()
+	// #define g_allocator heap()
 
 	void setup();
-	void resize( uw new_size );
 	void cleanup();
 }
 

project/Readme.md

@@ -33,7 +33,6 @@ While getting fleshed out, all feature macros are defined on the top of the head
 These macros are:
 
 * `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage
-* `GEN_FEATURE_INCREMENTAL` : Defines the incremental constructors
 * `GEN_FEATURE_PARSING` : Defines the parse constructors
 * `GEN_FEATURE_EDITOR` : Defines the file editing features for changing definitions based on ASTs
 * `GEN_FEATURE_SCANNER` : Defines the file scanning features for generating ASTs

project/gen.hpp

@@ -548,8 +548,8 @@ namespace gen
 			- sizeof(ModuleFlag)   // ModuleFlags
 			- sizeof(AccessSpec)   // ParentAccess
 			- sizeof(u32) 		   // StaticIndex
-			- sizeof(bool) * 1     // DynamicEntries
-			- sizeof(u8) * 2 )     // _Align_Pad
+			- sizeof(bool)         // DynamicEntries
+			- sizeof(u8) * 3 )     // _Align_Pad
 		/ sizeof(AST*);
 
 		constexpr static
@@ -713,13 +713,6 @@ namespace gen
 	// However on Windows at least, it doesn't need to occur as the OS will clean up after the process.
 	void deinit();
 
-	/*
-		Use this only if you know you generated the code you needed to a file.
-		And rather get rid of current code asts instead of growing the pool memory.
-		TODO: Need to put permanent ASTs into a separate set of memory. (I might just remove this tbh as it might be useless)
-	*/
-	void clear_code_memory();
-
 	// Used internally to retrive or make string allocations.
 	// Strings are stored in a series of string arenas of fixed size (SizePer_StringArena)
 	StringCached get_cached_string( StrC str );
@@ -787,7 +780,7 @@ namespace gen
 		, Code       attributes = NoCode
 		, ModuleFlag mflags     = ModuleFlag::None );
 
-	Code def_type   ( StrC name, Code arrayexpr = NoCode, Code specifiers = NoCode );
+	Code def_type   ( StrC name, Code arrayexpr = NoCode, Code specifiers = NoCode, Code attributes = NoCode );
 	Code def_typedef( StrC name, Code type, Code attributes = NoCode, ModuleFlag mflags = ModuleFlag::None );
 
 	Code def_union( StrC name, Code body, Code attributes = NoCode, ModuleFlag mflags = ModuleFlag::None );
@@ -830,6 +823,7 @@ namespace gen
 #	ifdef GEN_FEATURE_PARSING
 	Code parse_class       ( StrC class_def     );
 	Code parse_enum        ( StrC enum_def      );
+	Code parse_export_body ( StrC export_def    );
 	Code parse_extern_link ( StrC exten_link_def);
 	Code parse_friend      ( StrC friend_def    );
 	Code parse_function    ( StrC fn_def        );
@@ -1030,7 +1024,7 @@ namespace gen
 
 namespace gen
 {
-	// These constexprs are used for allocation heavior of data structurs
+	// These constexprs are used for allocation behavior of data structures
 	// or string handling while constructing or serializing.
 	// Change them to suit your needs.