Fixes to memory mangment, library is much faster now.

Readme.md

@@ -23,6 +23,9 @@ These build up a code AST to then serialize with a file builder.
 
 The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases.
 
+Note: Do not trying to do any large generations with this (at least not without changing the serialization implementation).
+It does not resue any memory yet for dynamic strings and thus any signficant size memory will result in massive consumption. 
+
 The project has no external dependencies beyond:
 
 * `stdarg.h`
@@ -287,6 +290,12 @@ Data Notes:
   * Memory within the buckets is not resused, so its inherently wasteful (most likely will give non-cached strings their own tailored allocator later)
 * Linked lists used children nodes on bodies, and parameters.
 * Its intended to generate the AST in one go and serialize after. The contructors and serializer are designed to be a "one pass, front to back" setup.
+* When benchmarking, the three most significant var to tune are:
+  * `Memory::Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
+  * `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
+  * `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
+  * The default values can handled generating for a stirng up to a size of ~650 kbs (bottleneck is serialization).
+  * Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
 
 Two generic templated containers are used throughout the library:
 

project/gen.cpp

@@ -386,6 +386,10 @@ namespace gen
 		local_persist thread_local
 		char SerializationLevel = 0;
 
+	#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION)
+		u64 time_start = time_rel_ms();
+	#endif
+
 		// TODO : Need to refactor so that intermeidate strings are freed conviently.
 		String result = String::make( Memory::GlobalAllocator, "" );
 
@@ -980,8 +984,10 @@ namespace gen
 			break;
 		}
 
+	#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION)
+		log_fmt("AST::to_string() time taken: %llu for: %s\n", time_rel_ms() - time_start, result );
+	#endif
 		return result;
-
 	#undef ProcessModuleFlags
 	}
 
@@ -1282,19 +1288,21 @@ namespace gen
 	{
 		using namespace StaticData;
 
-		Arena& last = StringArenas.back();
+		Arena* last = & StringArenas.back();
 
-		if ( last.TotalUsed + str_length > last.TotalSize )
+		uw size_req = str_length + sizeof(String::Header) + sizeof(char*);
+
+		if ( last->TotalUsed + size_req > last->TotalSize )
 		{
 			Arena new_arena = Arena::init_from_allocator( Allocator_StringArena, SizePer_StringArena );
 
 			if ( ! StringArenas.append( new_arena ) )
 				fatal( "gen::get_string_allocator: Failed to allocate a new string arena" );
 
-			last = StringArenas.back();
+			last = & StringArenas.back();
 		}
 
-		return last;
+		return * last;
 	}
 
 	// Will either make or retrive a code string.
@@ -1303,7 +1311,7 @@ namespace gen
 		using namespace StaticData;
 
 		s32 hash_length = str.Len > kilobytes(1) ? kilobytes(1) : str.Len;
-		u32 key         = crc32( str.Ptr, hash_length );
+		u64 key         = crc32( str.Ptr, hash_length );
 		{
 			StringCached* result = StringCache.get( key );
 
@@ -1325,22 +1333,8 @@ namespace gen
 	{
 		using namespace StaticData;
 
-		AllocatorInfo allocator = CodePools.back();
+		Pool* allocator = & CodePools.back();
-
+		if ( allocator->FreeList == nullptr )
-		s32 index = 0;
-		s32 left  = CodePools.num();
-		do
-		{
-			if ( CodePools[index].FreeList != nullptr  )
-			{
-				allocator = CodePools[index];
-				break;
-			}
-			index++;
-		}
-		while ( left--, left );
-
-		if ( allocator.Data == nullptr )
 		{
 			Pool code_pool = Pool::init( Allocator_CodePool, CodePool_NumBlocks, sizeof(AST) );
 
@@ -1350,10 +1344,10 @@ namespace gen
 			if ( ! CodePools.append( code_pool ) )
 				fatal( "gen::make_code: Failed to allocate a new code pool - CodePools failed to append new pool." );
 
-			allocator = * CodePools;
+			allocator = & CodePools.back();
 		}
 
-		Code result { rcast( AST*, alloc( allocator, sizeof(AST) )) };
+		Code result { rcast( AST*, alloc( * allocator, sizeof(AST) )) };
 
 		result->Content         = { nullptr };
 		result->Prev            = { nullptr };
@@ -3426,16 +3420,6 @@ namespace gen
 				return { 0, nullptr };                                       \
 			}
 
-			// do_once_start
-			// 	LexAllocator = Arena::init_from_allocator( heap(), megabytes(10) );
-
-			// 	if ( LexAllocator.PhysicalStart == nullptr )
-			// 	{
-			// 		log_failure( "gen::lex: failed to allocate memory for parsing constructor's lexer");
-			// 		return { { nullptr }, 0  };
-			// 	}
-			// do_once_end
-
 			local_persist thread_local
 			Array<Token> Tokens = { nullptr };
 
@@ -6380,7 +6364,7 @@ namespace gen
 			return false;
 		}
 
-		Buffer = String::make( Memory::GlobalAllocator, "" );
+		Buffer = String::make_reserve( Memory::GlobalAllocator, Builder_StrBufferReserve );
 
 		return true;
 	}
@@ -6393,6 +6377,7 @@ namespace gen
 			log_failure("gen::File::write - Failed to write to file: %s", file_name( & File ) );
 
 		file_close( & File );
+		Buffer.free();
 	}
 #pragma endregion Builder
 

project/gen.hpp

@@ -8,10 +8,6 @@
 */
 #pragma once
 
-#define GEN_FEATURE_PARSING
-#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
-#define GEN_ENFORCE_STRONG_CODE_TYPES
-
 #ifdef gen_time
 //! If its desired to roll your own dependencies, define GENCPP_PROVIDE_DEPENDENCIES before including this file.
 // Dependencies are derived from the c-zpl library: https://github.com/zpl-c/zpl
@@ -1659,8 +1655,8 @@ namespace gen
 	constexpr s32 InitSize_DataArrays  = 16;
 	constexpr s32 InitSize_StringTable = megabytes(4);
 
-	constexpr s32 CodePool_NumBlocks        = 4096;
+	constexpr s32 CodePool_NumBlocks        = kilobytes(4);
-	constexpr s32 SizePer_StringArena       = megabytes(32);
+	constexpr s32 SizePer_StringArena       = megabytes(1);
 
 	constexpr s32 MaxCommentLineLength      = 1024;
 	constexpr s32 MaxNameLength             = 128;
@@ -1668,6 +1664,7 @@ namespace gen
 	constexpr s32 StringTable_MaxHashLength = kilobytes(1);
 	constexpr s32 TokenFmt_TokenMap_MemSize	= kilobytes(4);
 	constexpr s32 LexAllocator_Size         = megabytes(10);
+	constexpr s32 Builder_StrBufferReserve  = megabytes(1);
 
 	extern CodeType t_auto;
 	extern CodeType t_void;
@@ -2053,3 +2050,30 @@ namespace gen
 // end: gen_time
 #endif
 
+#ifdef GEN_EXPOSE_BACKEND
+namespace gen
+{
+	namespace Memory
+	{
+		extern Array<Arena> Global_AllocatorBuckets;
+	}
+
+	namespace StaticData
+	{
+		extern Array< Pool >  CodePools;
+		extern Array< Arena > StringArenas;
+
+		extern StringTable StringCache;
+
+		extern Arena LexArena;
+
+		extern AllocatorInfo Allocator_DataArrays;
+		extern AllocatorInfo Allocator_CodePool;
+		extern AllocatorInfo Allocator_Lexer;
+		extern AllocatorInfo Allocator_StringArena;
+		extern AllocatorInfo Allocator_StringTable;
+		extern AllocatorInfo Allocator_TypeTable;
+	}
+}
+
+#endif

project/gen_dep.cpp

@@ -2,6 +2,8 @@
 
 #ifdef gen_time
 
+#ifdef GEN_BENCHMARK
+
 // NOTE: Ensure we use standard methods for these calls if we use GEN_PICO
 #pragma region Macros
 #	include <stdio.h>
@@ -41,6 +43,27 @@
 #	endif
 #pragma endregion Macros
 
+#if defined( GEN_SYSTEM_MACOS ) || GEN_SYSTEM_UNIX
+#	include <time.h>
+#	include <sys/time.h>
+#endif
+
+#if defined( GEN_SYSTEM_MACOS )
+#	include <mach/mach.h>
+#	include <mach/mach_time.h>
+#	include <mach/clock.h>
+#endif
+
+#if defined( GEN_SYSTEM_EMSCRIPTEN )
+#	include <emscripten.h>
+#endif
+
+#if defined( GEN_SYSTEM_WINDOWS )
+#	include <timezoneapi.h>
+#endif
+#endif
+
+
 namespace gen
 {
 #pragma region Debug
@@ -425,6 +448,7 @@ namespace gen
 					if ( arena->TotalUsed + total_size > (sw) arena->TotalSize )
 					{
 						// zpl__printf_err("%s", "Arena out of memory\n");
+						fatal("Arena out of memory! (Possibly could not fit for the largest size Arena!!)");
 						return nullptr;
 					}
 
@@ -537,6 +561,8 @@ namespace gen
 		void *data, *curr;
 		uptr* end;
 
+		zero_item( &pool );
+
 		pool.Backing     = backing;
 		pool.BlockSize   = block_size;
 		pool.BlockAlign  = block_align;
@@ -557,7 +583,7 @@ namespace gen
 		}
 
 		end  =  ( uptr* ) curr;
-		*end =  ( uptr )  0;
+		*end =  ( uptr )  NULL;
 
 		pool.PhysicalStart = data;
 		pool.FreeList      = data;
@@ -1151,6 +1177,56 @@ namespace gen
 			result = ( result >> 8 ) ^ ( _crc32_table[ ( result ^ *c ) & 0xff ] );
 		return ~result;
 	}
+
+	global u64 const _crc64_table[ 256 ] = {
+	0x0000000000000000ull, 0x7ad870c830358979ull, 0xf5b0e190606b12f2ull, 0x8f689158505e9b8bull, 0xc038e5739841b68full, 0xbae095bba8743ff6ull, 0x358804e3f82aa47dull,
+	0x4f50742bc81f2d04ull, 0xab28ecb46814fe75ull, 0xd1f09c7c5821770cull, 0x5e980d24087fec87ull, 0x24407dec384a65feull, 0x6b1009c7f05548faull, 0x11c8790fc060c183ull,
+	0x9ea0e857903e5a08ull, 0xe478989fa00bd371ull, 0x7d08ff3b88be6f81ull, 0x07d08ff3b88be6f8ull, 0x88b81eabe8d57d73ull, 0xf2606e63d8e0f40aull, 0xbd301a4810ffd90eull,
+	0xc7e86a8020ca5077ull, 0x4880fbd87094cbfcull, 0x32588b1040a14285ull, 0xd620138fe0aa91f4ull, 0xacf86347d09f188dull, 0x2390f21f80c18306ull, 0x594882d7b0f40a7full,
+	0x1618f6fc78eb277bull, 0x6cc0863448deae02ull, 0xe3a8176c18803589ull, 0x997067a428b5bcf0ull, 0xfa11fe77117cdf02ull, 0x80c98ebf2149567bull, 0x0fa11fe77117cdf0ull,
+	0x75796f2f41224489ull, 0x3a291b04893d698dull, 0x40f16bccb908e0f4ull, 0xcf99fa94e9567b7full, 0xb5418a5cd963f206ull, 0x513912c379682177ull, 0x2be1620b495da80eull,
+	0xa489f35319033385ull, 0xde51839b2936bafcull, 0x9101f7b0e12997f8ull, 0xebd98778d11c1e81ull, 0x64b116208142850aull, 0x1e6966e8b1770c73ull, 0x8719014c99c2b083ull,
+	0xfdc17184a9f739faull, 0x72a9e0dcf9a9a271ull, 0x08719014c99c2b08ull, 0x4721e43f0183060cull, 0x3df994f731b68f75ull, 0xb29105af61e814feull, 0xc849756751dd9d87ull,
+	0x2c31edf8f1d64ef6ull, 0x56e99d30c1e3c78full, 0xd9810c6891bd5c04ull, 0xa3597ca0a188d57dull, 0xec09088b6997f879ull, 0x96d1784359a27100ull, 0x19b9e91b09fcea8bull,
+	0x636199d339c963f2ull, 0xdf7adabd7a6e2d6full, 0xa5a2aa754a5ba416ull, 0x2aca3b2d1a053f9dull, 0x50124be52a30b6e4ull, 0x1f423fcee22f9be0ull, 0x659a4f06d21a1299ull,
+	0xeaf2de5e82448912ull, 0x902aae96b271006bull, 0x74523609127ad31aull, 0x0e8a46c1224f5a63ull, 0x81e2d7997211c1e8ull, 0xfb3aa75142244891ull, 0xb46ad37a8a3b6595ull,
+	0xceb2a3b2ba0eececull, 0x41da32eaea507767ull, 0x3b024222da65fe1eull, 0xa2722586f2d042eeull, 0xd8aa554ec2e5cb97ull, 0x57c2c41692bb501cull, 0x2d1ab4dea28ed965ull,
+	0x624ac0f56a91f461ull, 0x1892b03d5aa47d18ull, 0x97fa21650afae693ull, 0xed2251ad3acf6feaull, 0x095ac9329ac4bc9bull, 0x7382b9faaaf135e2ull, 0xfcea28a2faafae69ull,
+	0x8632586aca9a2710ull, 0xc9622c4102850a14ull, 0xb3ba5c8932b0836dull, 0x3cd2cdd162ee18e6ull, 0x460abd1952db919full, 0x256b24ca6b12f26dull, 0x5fb354025b277b14ull,
+	0xd0dbc55a0b79e09full, 0xaa03b5923b4c69e6ull, 0xe553c1b9f35344e2ull, 0x9f8bb171c366cd9bull, 0x10e3202993385610ull, 0x6a3b50e1a30ddf69ull, 0x8e43c87e03060c18ull,
+	0xf49bb8b633338561ull, 0x7bf329ee636d1eeaull, 0x012b592653589793ull, 0x4e7b2d0d9b47ba97ull, 0x34a35dc5ab7233eeull, 0xbbcbcc9dfb2ca865ull, 0xc113bc55cb19211cull,
+	0x5863dbf1e3ac9decull, 0x22bbab39d3991495ull, 0xadd33a6183c78f1eull, 0xd70b4aa9b3f20667ull, 0x985b3e827bed2b63ull, 0xe2834e4a4bd8a21aull, 0x6debdf121b863991ull,
+	0x1733afda2bb3b0e8ull, 0xf34b37458bb86399ull, 0x8993478dbb8deae0ull, 0x06fbd6d5ebd3716bull, 0x7c23a61ddbe6f812ull, 0x3373d23613f9d516ull, 0x49aba2fe23cc5c6full,
+	0xc6c333a67392c7e4ull, 0xbc1b436e43a74e9dull, 0x95ac9329ac4bc9b5ull, 0xef74e3e19c7e40ccull, 0x601c72b9cc20db47ull, 0x1ac40271fc15523eull, 0x5594765a340a7f3aull,
+	0x2f4c0692043ff643ull, 0xa02497ca54616dc8ull, 0xdafce7026454e4b1ull, 0x3e847f9dc45f37c0ull, 0x445c0f55f46abeb9ull, 0xcb349e0da4342532ull, 0xb1eceec59401ac4bull,
+	0xfebc9aee5c1e814full, 0x8464ea266c2b0836ull, 0x0b0c7b7e3c7593bdull, 0x71d40bb60c401ac4ull, 0xe8a46c1224f5a634ull, 0x927c1cda14c02f4dull, 0x1d148d82449eb4c6ull,
+	0x67ccfd4a74ab3dbfull, 0x289c8961bcb410bbull, 0x5244f9a98c8199c2ull, 0xdd2c68f1dcdf0249ull, 0xa7f41839ecea8b30ull, 0x438c80a64ce15841ull, 0x3954f06e7cd4d138ull,
+	0xb63c61362c8a4ab3ull, 0xcce411fe1cbfc3caull, 0x83b465d5d4a0eeceull, 0xf96c151de49567b7ull, 0x76048445b4cbfc3cull, 0x0cdcf48d84fe7545ull, 0x6fbd6d5ebd3716b7ull,
+	0x15651d968d029fceull, 0x9a0d8ccedd5c0445ull, 0xe0d5fc06ed698d3cull, 0xaf85882d2576a038ull, 0xd55df8e515432941ull, 0x5a3569bd451db2caull, 0x20ed197575283bb3ull,
+	0xc49581ead523e8c2ull, 0xbe4df122e51661bbull, 0x3125607ab548fa30ull, 0x4bfd10b2857d7349ull, 0x04ad64994d625e4dull, 0x7e7514517d57d734ull, 0xf11d85092d094cbfull,
+	0x8bc5f5c11d3cc5c6ull, 0x12b5926535897936ull, 0x686de2ad05bcf04full, 0xe70573f555e26bc4ull, 0x9ddd033d65d7e2bdull, 0xd28d7716adc8cfb9ull, 0xa85507de9dfd46c0ull,
+	0x273d9686cda3dd4bull, 0x5de5e64efd965432ull, 0xb99d7ed15d9d8743ull, 0xc3450e196da80e3aull, 0x4c2d9f413df695b1ull, 0x36f5ef890dc31cc8ull, 0x79a59ba2c5dc31ccull,
+	0x037deb6af5e9b8b5ull, 0x8c157a32a5b7233eull, 0xf6cd0afa9582aa47ull, 0x4ad64994d625e4daull, 0x300e395ce6106da3ull, 0xbf66a804b64ef628ull, 0xc5bed8cc867b7f51ull,
+	0x8aeeace74e645255ull, 0xf036dc2f7e51db2cull, 0x7f5e4d772e0f40a7ull, 0x05863dbf1e3ac9deull, 0xe1fea520be311aafull, 0x9b26d5e88e0493d6ull, 0x144e44b0de5a085dull,
+	0x6e963478ee6f8124ull, 0x21c640532670ac20ull, 0x5b1e309b16452559ull, 0xd476a1c3461bbed2ull, 0xaeaed10b762e37abull, 0x37deb6af5e9b8b5bull, 0x4d06c6676eae0222ull,
+	0xc26e573f3ef099a9ull, 0xb8b627f70ec510d0ull, 0xf7e653dcc6da3dd4ull, 0x8d3e2314f6efb4adull, 0x0256b24ca6b12f26ull, 0x788ec2849684a65full, 0x9cf65a1b368f752eull,
+	0xe62e2ad306bafc57ull, 0x6946bb8b56e467dcull, 0x139ecb4366d1eea5ull, 0x5ccebf68aecec3a1ull, 0x2616cfa09efb4ad8ull, 0xa97e5ef8cea5d153ull, 0xd3a62e30fe90582aull,
+	0xb0c7b7e3c7593bd8ull, 0xca1fc72bf76cb2a1ull, 0x45775673a732292aull, 0x3faf26bb9707a053ull, 0x70ff52905f188d57ull, 0x0a2722586f2d042eull, 0x854fb3003f739fa5ull,
+	0xff97c3c80f4616dcull, 0x1bef5b57af4dc5adull, 0x61372b9f9f784cd4ull, 0xee5fbac7cf26d75full, 0x9487ca0fff135e26ull, 0xdbd7be24370c7322ull, 0xa10fceec0739fa5bull,
+	0x2e675fb4576761d0ull, 0x54bf2f7c6752e8a9ull, 0xcdcf48d84fe75459ull, 0xb71738107fd2dd20ull, 0x387fa9482f8c46abull, 0x42a7d9801fb9cfd2ull, 0x0df7adabd7a6e2d6ull,
+	0x772fdd63e7936bafull, 0xf8474c3bb7cdf024ull, 0x829f3cf387f8795dull, 0x66e7a46c27f3aa2cull, 0x1c3fd4a417c62355ull, 0x935745fc4798b8deull, 0xe98f353477ad31a7ull,
+	0xa6df411fbfb21ca3ull, 0xdc0731d78f8795daull, 0x536fa08fdfd90e51ull, 0x29b7d047efec8728ull,
+};
+
+	u64 crc64( void const* data, sw len )
+	{
+		sw        remaining;
+		u64       result = ( zpl_cast( u64 ) 0 );
+		u8 const* c      = zpl_cast( u8 const* ) data;
+		for ( remaining = len; remaining--; c++ )
+			result = ( result >> 8 ) ^ ( _crc64_table[ ( result ^ *c ) & 0xff ] );
+		return result;
+	}
 #pragma endregion Hashing
 
 #pragma region File Handling
@@ -1591,6 +1667,167 @@ namespace gen
 	}
 #pragma endregion String
 
+#ifdef GEN_BENCHMARK
+#pragma region Timing
+	#if defined( GEN_COMPILER_MSVC ) && ! defined( __clang__ )
+	u64 read_cpu_time_stamp_counter( void )
+	{
+		return __rdtsc();
+	}
+	#elif defined( __i386__ )
+	u64 read_cpu_time_stamp_counter( void )
+	{
+		u64 x;
+		__asm__ volatile( ".byte 0x0f, 0x31" : "=A"( x ) );
+		return x;
+	}
+	#elif defined( __x86_64__ )
+	u64 read_cpu_time_stamp_counter( void )
+	{
+		u32 hi, lo;
+		__asm__ __volatile__( "rdtsc" : "=a"( lo ), "=d"( hi ) );
+		return ( zpl_cast( u64 ) lo ) | ( ( zpl_cast( u64 ) hi ) << 32 );
+	}
+	#elif defined( __powerpc__ )
+	u64 read_cpu_time_stamp_counter( void )
+	{
+		u64 result = 0;
+		u32 upper, lower, tmp;
+		__asm__ volatile(
+			"0:                   \n"
+			"\tmftbu   %0         \n"
+			"\tmftb    %1         \n"
+			"\tmftbu   %2         \n"
+			"\tcmpw    %2,%0      \n"
+			"\tbne     0b         \n"
+			: "=r"( upper ), "=r"( lower ), "=r"( tmp )
+		);
+		result = upper;
+		result = result << 32;
+		result = result | lower;
+
+		return result;
+	}
+	#elif defined( GEN_SYSTEM_EMSCRIPTEN )
+	u64 read_cpu_time_stamp_counter( void )
+	{
+		return ( u64 )( emscripten_get_now() * 1e+6 );
+	}
+	#elif defined( GEN_CPU_ARM ) && ! defined( GEN_COMPILER_TINYC )
+	u64 read_cpu_time_stamp_counter( void )
+	{
+	#	if defined( __aarch64__ )
+		int64_t r = 0;
+		asm volatile( "mrs %0, cntvct_el0" : "=r"( r ) );
+	#	elif ( __ARM_ARCH >= 6 )
+		uint32_t r = 0;
+		uint32_t pmccntr;
+		uint32_t pmuseren;
+		uint32_t pmcntenset;
+
+		// Read the user mode perf monitor counter access permissions.
+		asm volatile( "mrc p15, 0, %0, c9, c14, 0" : "=r"( pmuseren ) );
+		if ( pmuseren & 1 )
+		{    // Allows reading perfmon counters for user mode code.
+			asm volatile( "mrc p15, 0, %0, c9, c12, 1" : "=r"( pmcntenset ) );
+			if ( pmcntenset & 0x80000000ul )
+			{    // Is it counting?
+				asm volatile( "mrc p15, 0, %0, c9, c13, 0" : "=r"( pmccntr ) );
+				// The counter is set up to count every 64th cycle
+				return ( ( int64_t )pmccntr ) * 64;    // Should optimize to << 6
+			}
+		}
+	#	else
+	#		error "No suitable method for read_cpu_time_stamp_counter for this cpu type"
+	#	endif
+
+		return r;
+	}
+	#else
+	u64 read_cpu_time_stamp_counter( void )
+	{
+		GEN_PANIC( "read_cpu_time_stamp_counter is not supported on this particular setup" );
+		return -0;
+	}
+	#endif
+
+	#if defined( GEN_SYSTEM_WINDOWS ) || defined( GEN_SYSTEM_CYGWIN )
+
+	u64 time_rel_ms( void )
+	{
+		local_persist LARGE_INTEGER win32_perf_count_freq = {};
+		u64                         result;
+		LARGE_INTEGER               counter;
+		local_persist LARGE_INTEGER win32_perf_counter = {};
+		if ( ! win32_perf_count_freq.QuadPart )
+		{
+			QueryPerformanceFrequency( &win32_perf_count_freq );
+			GEN_ASSERT( win32_perf_count_freq.QuadPart != 0 );
+			QueryPerformanceCounter( &win32_perf_counter );
+		}
+
+		QueryPerformanceCounter( &counter );
+
+		result = ( counter.QuadPart - win32_perf_counter.QuadPart ) * 1000 / ( win32_perf_count_freq.QuadPart );
+		return result;
+	}
+
+	#else
+
+	#	if defined( GEN_SYSTEM_LINUX ) || defined( GEN_SYSTEM_FREEBSD ) || defined( GEN_SYSTEM_OPENBSD ) || defined( GEN_SYSTEM_EMSCRIPTEN )
+	u64 _unix_gettime( void )
+	{
+		struct timespec t;
+		u64             result;
+
+		clock_gettime( 1 /*CLOCK_MONOTONIC*/, &t );
+		result = 1000 * t.tv_sec + 1.0e-6 * t.tv_nsec;
+		return result;
+	}
+	#	endif
+
+	u64 time_rel_ms( void )
+	{
+	#	if defined( GEN_SYSTEM_OSX )
+		u64 result;
+
+		local_persist u64 timebase  = 0;
+		local_persist u64 timestart = 0;
+
+		if ( ! timestart )
+		{
+			mach_timebase_info_data_t tb = { 0 };
+			mach_timebase_info( &tb );
+			timebase   = tb.numer;
+			timebase  /= tb.denom;
+			timestart  = mach_absolute_time();
+		}
+
+		// NOTE: mach_absolute_time() returns things in nanoseconds
+		result = 1.0e-6 * ( mach_absolute_time() - timestart ) * timebase;
+		return result;
+	#	else
+		local_persist u64 unix_timestart = 0.0;
+
+		if ( ! unix_timestart )
+		{
+			unix_timestart = _unix_gettime();
+		}
+
+		u64 now = _unix_gettime();
+
+		return ( now - unix_timestart );
+	#	endif
+	}
+	#endif
+
+	f64 time_rel( void )
+	{
+		return ( f64 )( time_rel_ms() * 1e-3 );
+	}
+#pragma endregion Timing
+#endif
+
 	namespace Memory
 	{
 		global AllocatorInfo GlobalAllocator;
@@ -1606,7 +1843,7 @@ namespace gen
 				{
 					if ( last.TotalUsed + size > last.TotalSize )
 					{
-						Arena bucket = Arena::init_from_allocator( heap(), BucketSize );
+						Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize );
 
 						if ( bucket.PhysicalStart == nullptr )
 							fatal( "Failed to create bucket for Global_AllocatorBuckets");
@@ -1633,7 +1870,7 @@ namespace gen
 				{
 					if ( last.TotalUsed + size > last.TotalSize )
 					{
-						Arena bucket = Arena::init_from_allocator( heap(), BucketSize );
+						Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize );
 
 						if ( bucket.PhysicalStart == nullptr )
 							fatal( "Failed to create bucket for Global_AllocatorBuckets");
@@ -1667,7 +1904,7 @@ namespace gen
 			if ( Global_AllocatorBuckets == nullptr )
 				fatal( "Failed to reserve memory for Global_AllocatorBuckets");
 
-			Arena bucket = Arena::init_from_allocator( heap(), BucketSize );
+			Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize );
 
 			if ( bucket.PhysicalStart == nullptr )
 				fatal( "Failed to create first bucket for Global_AllocatorBuckets");

project/gen_dep.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #if gen_time
-
+#define GEN_BENCHMARK
 #if __clang__
 #	pragma clang diagnostic ignored "-Wunused-const-variable"
 #	pragma clang diagnostic ignored "-Wswitch"
@@ -1340,9 +1340,10 @@ namespace gen
 		void destroy( void )
 		{
 			if ( Hashes && Hashes.get_header()->Capacity )
+			{
 				Hashes.free();
-			if ( Entries && Hashes.get_header()->Capacity )
 				Entries.free();
+			}
 		}
 
 		Type* get( u64 key )
@@ -1547,6 +1548,7 @@ namespace gen
 	#pragma region Hashing
 
 	u32 crc32( void const* data, sw len );
+	u64 crc64( void const* data, sw len );
 
 	#pragma endregion Hashing
 
@@ -1587,6 +1589,13 @@ namespace gen
 				sw            Capacity;
 			};
 
+			static
+			uw grow_formula( uw value )
+			{
+				// Using a very aggressive growth formula to reduce time mem_copying with recursive calls to append in this library.
+				return 4 * value + 8;
+			}
+
 			static
 			String make( AllocatorInfo allocator, char const* str )
 			{
@@ -1634,9 +1643,6 @@ namespace gen
 				if ( allocation == nullptr )
 					return { nullptr };
 
-				if ( ! str )
-					mem_set( allocation, 0, alloc_size );
-
 				Header&
 				header = * rcast(Header*, allocation);
 				header = { allocator, length, length };
@@ -1645,6 +1651,8 @@ namespace gen
 
 				if ( length && str )
 					mem_copy( result, str, length );
+				else
+					mem_set( result, 0, alloc_size - header_size );
 
 				result[ length ] = '\0';
 
@@ -1686,70 +1694,14 @@ namespace gen
 				return true;
 			}
 
-			bool make_space_for( char const* str, sw add_len )
+			bool make_space_for( char const* str, sw add_len );
-			{
-				sw available = avail_space();
-
-				// NOTE: Return if there is enough space left
-				if ( available >= add_len )
-				{
-					return true;
-				}
-				else
-				{
-					sw new_len, old_size, new_size;
-
-					void* ptr;
-					void* new_ptr;
-
-					AllocatorInfo allocator = get_header().Allocator;
-					Header*       header	= nullptr;
-
-					new_len  = length() + add_len;
-					ptr      = & get_header();
-					old_size = size_of( Header ) + length() + 1;
-					new_size = size_of( Header ) + new_len + 1;
-
-					new_ptr = resize( allocator, ptr, old_size, new_size );
-
-					if ( new_ptr == nullptr )
-						return false;
-
-					header            = zpl_cast( Header* ) new_ptr;
-					header->Allocator = allocator;
-					header->Capacity  = new_len;
-
-					Data = rcast( char*, header + 1 );
-
-					return str;
-				}
-			}
 
 			bool append( char const* str )
 			{
 				return append( str, str_len( str ) );
 			}
 
-			bool append( char const* str, sw length )
+			bool append( char const* str, sw length );
-			{
-				if ( sptr(str) > 0 )
-				{
-					sw curr_len = this->length();
-
-					if ( ! make_space_for( str, length ) )
-						return false;
-
-					Header& header = get_header();
-
-					mem_copy( Data + curr_len, str, length );
-
-					Data[ curr_len + length ] = '\0';
-
-					header.Length = curr_len + length;
-				}
-
-				return str;
-			}
 
 			bool append( StrC str)
 			{
@@ -2169,11 +2121,23 @@ namespace gen
 	sw    str_fmt_file_va( FileInfo* f, char const* fmt, va_list va );
 	#pragma endregion Printing
 
+#ifdef GEN_BENCHMARK
+	//! Return CPU timestamp.
+	u64 read_cpu_time_stamp_counter( void );
+
+	//! Return relative time (in seconds) since the application start.
+	f64 time_rel( void );
+
+	//! Return relative time since the application start.
+	u64 time_rel_ms( void );
+#endif
+
 	namespace Memory
 	{
-		// NOTE: This limits the size of the string that can be read from a file or generated to 10 megs.
+		// NOTE: This limits the maximum size of an allocation
 		// If you are generating a string larger than this, increase the size of the bucket here.
-		constexpr uw BucketSize = megabytes(10);
+		constexpr uw Global_BucketSize = megabytes(10);
+
 
 		// Global allocator used for data with process lifetime.
 		extern AllocatorInfo GlobalAllocator;
@@ -2226,6 +2190,66 @@ namespace gen
 	#endif
 	}
 
+	bool String::make_space_for( char const* str, sw add_len )
+	{
+		sw available = avail_space();
+
+		// NOTE: Return if there is enough space left
+		if ( available >= add_len )
+		{
+			return true;
+		}
+		else
+		{
+			sw new_len, old_size, new_size;
+
+			void* ptr;
+			void* new_ptr;
+
+			AllocatorInfo allocator = get_header().Allocator;
+			Header*       header	= nullptr;
+
+			new_len  = grow_formula( length() + add_len );
+			ptr      = & get_header();
+			old_size = size_of( Header ) + length() + 1;
+			new_size = size_of( Header ) + new_len + 1;
+
+			new_ptr = resize( allocator, ptr, old_size, new_size );
+
+			if ( new_ptr == nullptr )
+				return false;
+
+			header            = zpl_cast( Header* ) new_ptr;
+			header->Allocator = allocator;
+			header->Capacity  = new_len;
+
+			Data = rcast( char*, header + 1 );
+
+			return str;
+		}
+	}
+
+	bool String::append( char const* str, sw length )
+	{
+		u64 time_start = time_rel_ms();
+		if ( sptr(str) > 0 )
+		{
+			sw curr_len = this->length();
+
+			if ( ! make_space_for( str, length ) )
+				return false;
+
+			Header& header = get_header();
+
+			mem_copy( Data + curr_len, str, length );
+
+			Data[ curr_len + length ] = '\0';
+
+			header.Length = curr_len + length;
+		}
+		return str;
+	}
+
 // gen namespace
 }
 

scripts/build.ci.ps1

@@ -48,7 +48,7 @@ Push-location $path_gen
 # Run meta-program
 	$gencpp = Join-Path $path_gen_build gencpp.exe
 
-	Write-Host `nGenerating files -- Parsed...
+	Write-Host `nRunning tests...
 	& $gencpp
 
 # Format generated files

scripts/genccp.natstepfilter

@@ -24,4 +24,7 @@
 		<Name>gen::Code.*::to_string</Name>
 		<Action>NoStepInto</Action>
 	</Function>
+	<Function>
+		<Name>gen::String::operator .*</Name>
+	</Function>
 </StepFilter>

scripts/gencpp.natvis

@@ -5,6 +5,18 @@
 		<DisplayString>Data:{Data} Proc:{Proc}</DisplayString>
 	</Type>
 
+	<Type Name="gen::Pool">
+		<DisplayString>NumBlocks: {NumBlocks} TotalSize: {TotalSize}</DisplayString>
+		<Expand>
+			<LinkedListItems>
+				<Size>NumBlocks</Size>
+				<HeadPointer>FreeList</HeadPointer>
+				<NextPointer>FreeList</NextPointer>
+				<ValueNode>PhysicalStart</ValueNode>
+			</LinkedListItems>
+		</Expand>
+	</Type>
+
 	<Type Name="gen::Array&lt;*&gt;">
 		<DisplayString>Num:{((Header*)((char*)Data - sizeof(Header)))->Num},
 			Capacity:{((Header*)((char*)Data - sizeof(Header)))->Capacity}</DisplayString>
@@ -70,7 +82,8 @@
 			<Item Name="Content">ast->Content</Item>
 			<Item Name="Body">ast->Body</Item>
 			<Item Name="Parent">ast->Parent</Item>
-			<Item Name="ModuleFlags" Condition="ast->ModuleFlags != ModuleFlag::Invalid">ast->ModuleFlags</Item>
+			<Item Name="ModuleFlags" Condition="ast->ModuleFlags != ModuleFlag::Invalid">
+				ast->ModuleFlags</Item>
 			<Item Name="ArrSpecs" Condition="ast->ArrSpecs[0] &lt; 18">ast->ArrSpecs</Item>
 			<Item Name="Prev">ast->Prev</Item>
 			<Item Name="Next">ast->Next</Item>

test/Parsed/Array.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 
 using namespace gen;

test/Parsed/Buffer.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 
 using namespace gen;

test/Parsed/HashTable.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 #include "Array.Parsed.hpp"
 

test/Parsed/Ring.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 #include "Buffer.Parsed.hpp"
 

test/Parsed/Sanity.Parsed.hpp

@@ -3,6 +3,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 
 using namespace gen;
@@ -289,7 +291,7 @@ u32 gen_sanity()
 
 		));
 
-		CodeUsingNamespace npspace_using = (CodeUsingNamespace) parse_using( code(
+		CodeUsing npspace_using = parse_using( code(
 			using namespace TestNamespace;
 		));
 

test/sanity.cpp

@@ -0,0 +1,92 @@
+// Testing to make sure backend of library is operating properly.
+
+#ifdef gen_time
+#define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
+#include "gen.hpp"
+
+void check_sanity()
+{
+	using namespace gen;
+	gen::init();
+	log_fmt("\ncheck_sanity:\n");
+
+	// Test string caching:
+	CodeType t_int_dupe = def_type( name(int) );
+
+	if ( t_int_dupe->Name != t_int->Name )
+		fatal("check_sanity: String caching failed!");
+
+
+	// Purposefully use an excessive amount of memory to make so the the memory backend doesn't break.
+	// This has been tested with num_iterations set to 15000000 (generates 15 million lines of code), the Global_BlockSize, along with CodePool_NumBlocks, and SizePer_StringArena
+	// must be adjusted to gigabytes(2), kilobytes(512), and gigabyte(1) for good performance without crashing.
+	/*
+		Typical usage (megabytes(10), kilobytes(4), megabytes(1), for 650000 (the limit of 10 meg partition buckets in global arena) )
+		Num Global Arenas       : 2 TotalSize: 4294967296 !
+		Num Code Pools          : 144 TotalSize: 9663676416 !
+		Num String Cache Arenas : 2 TotalSize: 2147483648 !
+		Num String Cache        : 30000025
+
+		Memory usage to expect at 15 mil file:
+		Num Global Arenas       : 2 TotalSize: 4294967296 !
+		Num Code Pools          : 144 TotalSize: 9663676416 !
+		Num String Cache Arenas : 2 TotalSize: 2147483648 !
+		Num String Cache        : 30000025
+	*/
+	constexpr
+	s32 num_iterations = 650000;
+
+	Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( Memory::GlobalAllocator, num_iterations * 2 );
+
+	s32 idx = num_iterations;
+	while( --idx )
+	{
+		// Stress testing string allocation
+		String type_name = String::fmt_buf( Memory::GlobalAllocator, "type_%d", idx );
+		String typedef_name = String::fmt_buf( Memory::GlobalAllocator, "typedef_%d", idx );
+
+		CodeTypedef type_as_int = def_typedef( type_name, t_int );
+		CodeType    type        = def_type( type_name );
+		CodeTypedef type_def    = def_typedef( typedef_name, type );
+
+		typedefs.append( type_as_int );
+		typedefs.append( type_def );
+	}
+
+	log_fmt("\nMemory before builder:\n");
+	log_fmt("Num Global Arenas       : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize);
+	log_fmt("Num Code Pools          : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize);
+	log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena);
+	log_fmt("Num String Cache        : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache);
+
+	Builder builder;
+	builder.open( "sanity.gen.hpp" );
+
+	idx = num_iterations;
+#ifdef GEN_BENCHMARK
+	u64 time_start     = time_rel_ms();
+#endif
+	while( --idx )
+	{
+		builder.print( typedefs[idx] );
+	}
+
+	builder.write();
+#ifdef GEN_BENCHMARK
+	log_fmt("\n\nBuilder finished writting. Time taken: %llu ms\n", time_rel_ms() - time_start);
+#endif
+
+	log_fmt("\nMemory after builder:\n");
+	log_fmt("Num Global Arenas       : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize);
+	log_fmt("Num Code Pools          : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize);
+	log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena);
+	log_fmt("Num String Cache        : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache);
+
+	log_fmt("\nSanity passed!\n");
+	gen::deinit();
+}
+#endif

test/test.Upfront.cpp

@@ -1,6 +1,9 @@
 #ifdef gen_time
-#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.cpp"
 #include "Upfront\Array.Upfront.hpp"
 #include "Upfront\Buffer.Upfront.hpp"

test/test.cpp

@@ -1,73 +1,25 @@
-#ifdef gen_time
-#define GEN_FEATURE_PARSING
-#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
-#define GEN_ENFORCE_STRONG_CODE_TYPES
-#include "Parsed\Array.Parsed.hpp"
-#include "Parsed\Buffer.Parsed.hpp"
-#include "Parsed\HashTable.Parsed.hpp"
-#include "Parsed\Ring.Parsed.hpp"
-#include "Parsed\Sanity.Parsed.hpp"
-#include "SOA.hpp"
 #include "gen.cpp"
+#include "sanity.cpp"
 
-using namespace gen;
+#if gen_time
-
-// TODO : Need to make a more robust test suite
-
 int gen_main()
 {
-	gen::init();
+	using namespace gen;
+	log_fmt("\ngen_time:");
 
-	gen_sanity();
+	check_sanity();
 
-	gen_array( u8 );
-	gen_array( sw );
-
-	gen_buffer( u8 );
-
-	gen_hashtable( u32 );
-
-	gen_ring( s16 );
-	gen_ring( uw );
-
-	gen_array_file();
-	gen_buffer_file();
-	gen_hashtable_file();
-	gen_ring_file();
-
-	Builder soa_test; soa_test.open( "SOA.gen.hpp" );
-
-	soa_test.print( parse_using( code(
-		using u16 = unsigned short;
-	)));
-
-	soa_test.print( def_include( txt_StrC("gen.hpp")));
-
-	soa_test.print( def_using_namespace( name(gen) ) );
-
-	soa_test.print( gen_SOA(
-		parse_struct( code(
-			struct TestStruct
-			{
-				u8  A;
-				u16 B;
-				u32 C;
-				u64 D;
-			};
-		))
-	));
-
-	soa_test.write();
-
-	gen::deinit();
 	return 0;
 }
 #endif
 
 
-#ifdef runtime
+// This only has to be done if symbol conflicts occur.
+#ifndef gen_time
 int main()
 {
+
+
 	return 0;
 }
 #endif

test/test.parsing.cpp

@@ -0,0 +1,75 @@
+#ifdef gen_time
+#define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
+#include "Parsed\Array.Parsed.hpp"
+#include "Parsed\Buffer.Parsed.hpp"
+#include "Parsed\HashTable.Parsed.hpp"
+#include "Parsed\Ring.Parsed.hpp"
+#include "Parsed\Sanity.Parsed.hpp"
+#include "SOA.hpp"
+#include "gen.cpp"
+
+using namespace gen;
+
+// TODO : Need to make a more robust test suite
+
+int gen_main()
+{
+	gen::init();
+
+	gen_sanity();
+
+	gen_array( u8 );
+	gen_array( sw );
+
+	gen_buffer( u8 );
+
+	gen_hashtable( u32 );
+
+	gen_ring( s16 );
+	gen_ring( uw );
+
+	gen_array_file();
+	gen_buffer_file();
+	gen_hashtable_file();
+	gen_ring_file();
+
+	Builder soa_test; soa_test.open( "SOA.gen.hpp" );
+
+	soa_test.print( parse_using( code(
+		using u16 = unsigned short;
+	)));
+
+	soa_test.print( def_include( txt_StrC("gen.hpp")));
+
+	soa_test.print( def_using_namespace( name(gen) ) );
+
+	soa_test.print( gen_SOA(
+		parse_struct( code(
+			struct TestStruct
+			{
+				u8  A;
+				u16 B;
+				u32 C;
+				u64 D;
+			};
+		))
+	));
+
+	soa_test.write();
+
+	gen::deinit();
+	return 0;
+}
+#endif
+
+
+#ifdef runtime
+int main()
+{
+	return 0;
+}
+#endif

test/upfront.hpp

@@ -0,0 +1,21 @@
+#ifdef gen_time
+#define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
+#include "gen.hpp"
+
+void check_upfront()
+{
+	using namespace gen;
+	log_fmt("\nupfront: ");
+	gen::init();
+
+
+
+	gen::deinit();
+	log_fmt("Passed!\n");
+}
+
+#endif