Small correction to test comment.

Its easier to manage, I'm sticking with generating the single header so it wont matter, its easy to refactor back if desired.

Readme.md

@@ -23,6 +23,9 @@ These build up a code AST to then serialize with a file builder.
 
 The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases.
 
+Note: Do not trying to do any large generations with this (at least not without changing the serialization implementation).
+It does not resue any memory yet for dynamic strings and thus any signficant size memory will result in massive consumption. 
+
 The project has no external dependencies beyond:
 
 * `stdarg.h`
@@ -287,6 +290,12 @@ Data Notes:
   * Memory within the buckets is not resused, so its inherently wasteful (most likely will give non-cached strings their own tailored allocator later)
 * Linked lists used children nodes on bodies, and parameters.
 * Its intended to generate the AST in one go and serialize after. The contructors and serializer are designed to be a "one pass, front to back" setup.
+* When benchmarking, the three most significant var to tune are:
+  * `Memory::Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
+  * `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
+  * `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
+  * The default values can handled generating for a stirng up to a size of ~650 kbs (bottleneck is serialization).
+  * Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
 
 Two generic templated containers are used throughout the library:
 

scripts/build.ci.ps1

@@ -48,7 +48,7 @@ Push-location $path_gen
 # Run meta-program
 	$gencpp = Join-Path $path_gen_build gencpp.exe
 
-	Write-Host `nGenerating files -- Parsed...
+	Write-Host `nRunning tests...
 	& $gencpp
 
 # Format generated files

scripts/genccp.natstepfilter

@@ -24,4 +24,7 @@
 		<Name>gen::Code.*::to_string</Name>
 		<Action>NoStepInto</Action>
 	</Function>
+	<Function>
+		<Name>gen::String::operator .*</Name>
+	</Function>
 </StepFilter>

scripts/gencpp.natvis

@@ -5,6 +5,18 @@
 		<DisplayString>Data:{Data} Proc:{Proc}</DisplayString>
 	</Type>
 
+	<Type Name="gen::Pool">
+		<DisplayString>NumBlocks: {NumBlocks} TotalSize: {TotalSize}</DisplayString>
+		<Expand>
+			<LinkedListItems>
+				<Size>NumBlocks</Size>
+				<HeadPointer>FreeList</HeadPointer>
+				<NextPointer>FreeList</NextPointer>
+				<ValueNode>PhysicalStart</ValueNode>
+			</LinkedListItems>
+		</Expand>
+	</Type>
+
 	<Type Name="gen::Array&lt;*&gt;">
 		<DisplayString>Num:{((Header*)((char*)Data - sizeof(Header)))->Num},
 			Capacity:{((Header*)((char*)Data - sizeof(Header)))->Capacity}</DisplayString>
@@ -70,7 +82,8 @@
 			<Item Name="Content">ast->Content</Item>
 			<Item Name="Body">ast->Body</Item>
 			<Item Name="Parent">ast->Parent</Item>
-			<Item Name="ModuleFlags" Condition="ast->ModuleFlags != ModuleFlag::Invalid">ast->ModuleFlags</Item>
+			<Item Name="ModuleFlags" Condition="ast->ModuleFlags != ModuleFlag::Invalid">
+				ast->ModuleFlags</Item>
 			<Item Name="ArrSpecs" Condition="ast->ArrSpecs[0] &lt; 18">ast->ArrSpecs</Item>
 			<Item Name="Prev">ast->Prev</Item>
 			<Item Name="Next">ast->Next</Item>
@@ -665,4 +678,4 @@
 		<DisplayString>Current[ { Arr[Idx] } ] Idx:{ Idx }</DisplayString>
 	</Type>
 
-</AutoVisualizer>
+</AutoVisualizer>

test/Parsed/Array.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 
 using namespace gen;

test/Parsed/Buffer.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 
 using namespace gen;

test/Parsed/HashTable.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 #include "Array.Parsed.hpp"
 

test/Parsed/Ring.Parsed.hpp

@@ -4,6 +4,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 #include "Buffer.Parsed.hpp"
 

test/Parsed/Sanity.Parsed.hpp

@@ -3,6 +3,8 @@
 #define GEN_FEATURE_PARSING
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.hpp"
 
 using namespace gen;
@@ -289,7 +291,7 @@ u32 gen_sanity()
 
 		));
 
-		CodeUsingNamespace npspace_using = (CodeUsingNamespace) parse_using( code(
+		CodeUsing npspace_using = parse_using( code(
 			using namespace TestNamespace;
 		));
 

test/sanity.cpp

@@ -0,0 +1,93 @@
+// Testing to make sure backend of library is operating properly.
+
+#ifdef gen_time
+#define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
+#include "gen.hpp"
+
+void check_sanity()
+{
+	using namespace gen;
+	gen::init();
+	log_fmt("\ncheck_sanity:\n");
+
+	// Test string caching:
+	CodeType t_int_dupe = def_type( name(int) );
+
+	if ( t_int_dupe->Name != t_int->Name )
+		fatal("check_sanity: String caching failed!");
+
+
+	// Purposefully use an excessive amount of memory to make so the the memory backend doesn't break.
+	// This has been tested with num_iterations set to 15000000 (generates 15 million lines of code), the Global_BlockSize, along with CodePool_NumBlocks, and SizePer_StringArena
+	// must be adjusted to gigabytes(2), kilobytes(512), and gigabyte(1) for good performance without crashing.
+	/*
+		Typical usage (megabytes(10), kilobytes(4), megabytes(1), for 650000 (the limit of 10 meg partition buckets in global arena) )
+		Memory after builder:
+		Num Global Arenas       : 14 TotalSize: 146800640 !
+		Num Code Pools          : 794 TotalSize: 416284672 !
+		Num String Cache Arenas : 60 TotalSize: 62914560 !
+		Num String Cache        : 1300007
+
+		Memory usage to expect at 15 mil file:
+		Num Global Arenas       : 2 TotalSize: 4294967296 !
+		Num Code Pools          : 144 TotalSize: 9663676416 !
+		Num String Cache Arenas : 2 TotalSize: 2147483648 !
+		Num String Cache        : 30000025
+	*/
+	constexpr
+	s32 num_iterations = 650000;
+
+	Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( Memory::GlobalAllocator, num_iterations * 2 );
+
+	s32 idx = num_iterations;
+	while( --idx )
+	{
+		// Stress testing string allocation
+		String type_name = String::fmt_buf( Memory::GlobalAllocator, "type_%d", idx );
+		String typedef_name = String::fmt_buf( Memory::GlobalAllocator, "typedef_%d", idx );
+
+		CodeTypedef type_as_int = def_typedef( type_name, t_int );
+		CodeType    type        = def_type( type_name );
+		CodeTypedef type_def    = def_typedef( typedef_name, type );
+
+		typedefs.append( type_as_int );
+		typedefs.append( type_def );
+	}
+
+	log_fmt("\nMemory before builder:\n");
+	log_fmt("Num Global Arenas       : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize);
+	log_fmt("Num Code Pools          : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize);
+	log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena);
+	log_fmt("Num String Cache        : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache);
+
+	Builder builder;
+	builder.open( "sanity.gen.hpp" );
+
+	idx = num_iterations;
+#ifdef GEN_BENCHMARK
+	u64 time_start     = time_rel_ms();
+#endif
+	while( --idx )
+	{
+		builder.print( typedefs[idx] );
+	}
+
+	builder.write();
+#ifdef GEN_BENCHMARK
+	log_fmt("\n\nBuilder finished writting. Time taken: %llu ms\n", time_rel_ms() - time_start);
+#endif
+
+	log_fmt("\nMemory after builder:\n");
+	log_fmt("Num Global Arenas       : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize);
+	log_fmt("Num Code Pools          : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize);
+	log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena);
+	log_fmt("Num String Cache        : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache);
+
+	log_fmt("\nSanity passed!\n");
+	gen::deinit();
+}
+#endif

test/test.Upfront.cpp

@@ -1,6 +1,9 @@
 #ifdef gen_time
-#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
 #include "gen.cpp"
 #include "Upfront\Array.Upfront.hpp"
 #include "Upfront\Buffer.Upfront.hpp"

test/test.cpp

@@ -1,73 +1,25 @@
-#ifdef gen_time
-#define GEN_FEATURE_PARSING
-#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
-#define GEN_ENFORCE_STRONG_CODE_TYPES
-#include "Parsed\Array.Parsed.hpp"
-#include "Parsed\Buffer.Parsed.hpp"
-#include "Parsed\HashTable.Parsed.hpp"
-#include "Parsed\Ring.Parsed.hpp"
-#include "Parsed\Sanity.Parsed.hpp"
-#include "SOA.hpp"
 #include "gen.cpp"
+#include "sanity.cpp"
 
-using namespace gen;
-
-// TODO : Need to make a more robust test suite
-
+#if gen_time
 int gen_main()
 {
-	gen::init();
+	using namespace gen;
+	log_fmt("\ngen_time:");
 
-	gen_sanity();
+	check_sanity();
 
-	gen_array( u8 );
-	gen_array( sw );
-
-	gen_buffer( u8 );
-
-	gen_hashtable( u32 );
-
-	gen_ring( s16 );
-	gen_ring( uw );
-
-	gen_array_file();
-	gen_buffer_file();
-	gen_hashtable_file();
-	gen_ring_file();
-
-	Builder soa_test; soa_test.open( "SOA.gen.hpp" );
-
-	soa_test.print( parse_using( code(
-		using u16 = unsigned short;
-	)));
-
-	soa_test.print( def_include( txt_StrC("gen.hpp")));
-
-	soa_test.print( def_using_namespace( name(gen) ) );
-
-	soa_test.print( gen_SOA(
-		parse_struct( code(
-			struct TestStruct
-			{
-				u8  A;
-				u16 B;
-				u32 C;
-				u64 D;
-			};
-		))
-	));
-
-	soa_test.write();
-
-	gen::deinit();
 	return 0;
 }
 #endif
 
 
-#ifdef runtime
+// This only has to be done if symbol conflicts occur.
+#ifndef gen_time
 int main()
 {
+
+
 	return 0;
 }
 #endif

test/test.parsing.cpp

@@ -0,0 +1,75 @@
+#ifdef gen_time
+#define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
+#include "Parsed\Array.Parsed.hpp"
+#include "Parsed\Buffer.Parsed.hpp"
+#include "Parsed\HashTable.Parsed.hpp"
+#include "Parsed\Ring.Parsed.hpp"
+#include "Parsed\Sanity.Parsed.hpp"
+#include "SOA.hpp"
+#include "gen.cpp"
+
+using namespace gen;
+
+// TODO : Need to make a more robust test suite
+
+int gen_main()
+{
+	gen::init();
+
+	gen_sanity();
+
+	gen_array( u8 );
+	gen_array( sw );
+
+	gen_buffer( u8 );
+
+	gen_hashtable( u32 );
+
+	gen_ring( s16 );
+	gen_ring( uw );
+
+	gen_array_file();
+	gen_buffer_file();
+	gen_hashtable_file();
+	gen_ring_file();
+
+	Builder soa_test; soa_test.open( "SOA.gen.hpp" );
+
+	soa_test.print( parse_using( code(
+		using u16 = unsigned short;
+	)));
+
+	soa_test.print( def_include( txt_StrC("gen.hpp")));
+
+	soa_test.print( def_using_namespace( name(gen) ) );
+
+	soa_test.print( gen_SOA(
+		parse_struct( code(
+			struct TestStruct
+			{
+				u8  A;
+				u16 B;
+				u32 C;
+				u64 D;
+			};
+		))
+	));
+
+	soa_test.write();
+
+	gen::deinit();
+	return 0;
+}
+#endif
+
+
+#ifdef runtime
+int main()
+{
+	return 0;
+}
+#endif

test/upfront.hpp

@@ -0,0 +1,21 @@
+#ifdef gen_time
+#define GEN_FEATURE_PARSING
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_BENCHMARK
+#include "gen.hpp"
+
+void check_upfront()
+{
+	using namespace gen;
+	log_fmt("\nupfront: ");
+	gen::init();
+
+
+
+	gen::deinit();
+	log_fmt("Passed!\n");
+}
+
+#endif