Adding tuning macros for memory usage of the library, fixes.

Reduced the sanity test case to half its original iterations.

Readme.md

@@ -12,7 +12,7 @@ These build up a code AST to then serialize with a file builder.
 * [Building](#notes)
 * [Outline](#outline)
 * [What is not provided](#what-is-not-provided)
-* [The four constructors](#there-are-four-sets-of-interfaces-for-code-ast-generation-the-library-provides)
+* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides)
 * [Predefined Codes](#predefined-codes)
 * [Code generation and modification](#code-generation-and-modification)
 * [On multithreading](#on-multi-threading)
@@ -291,12 +291,16 @@ Data Notes:
   * I will be augmenting the single arena with a simple slag allocator.
 * Linked lists used children nodes on bodies, and parameters.
 * Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup.
-* When benchmarking, the three most significant var to tune are:
+* Allocations can be tuned by defining the folloiwng macros:
-  * `Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
+  * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator
-  * `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
+  * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator
-  * `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
+  * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching.
-  * The default values can handled generating for a string up to a size of ~650 kbs (bottleneck is serialization).
+  * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line.
-  * Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
+  * `GEN_MAX_NAME_LENGTH` : Max length of any identifier.
+  * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code.
+  * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable.
+  * `GEN_LEX_ALLOCATOR_SIZE`
+  * `GEN_BUILDER_STR_BUFFER_RESERVE`
 
 Two generic templated containers are used throughout the library:
 

project/gen.cpp

@@ -4115,6 +4115,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
 	using namespace Parser;
 
 	Token start;
+	s32 len = 0;
 
 	if ( check(TokType::Attribute_Open) )
 	{
@@ -4126,6 +4127,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
 		}
 
 		eat( TokType::Attribute_Close );
+
+		s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
 	}
 
 	else if ( check(TokType::Decl_GNU_Attribute) )
@@ -4140,6 +4143,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
 
 		eat(TokType::BraceCurly_Close);
 		eat(TokType::BraceCurly_Close);
+
+		s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
 	}
 
 	else if ( check(TokType::Decl_MSVC_Attribute) )
@@ -4153,15 +4158,16 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
 		}
 
 		eat(TokType::BraceCurly_Close);
+
+		s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
 	}
 
 	else if ( tok_is_attribute( currtok ) )
 	{
 		eat(currtok.Type);
+		s32 len = start.Length;
 	}
 
-	s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
-
 	if ( len > 0 )
 	{
 		StrC attribute_txt = { len, start.Text };
@@ -6237,7 +6243,7 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
 {
 	using namespace Parser;
 
-	SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
+	SpecifierT specs_found[16] { ESpecifier::Invalid };
 	s32        num_specifiers = 0;
 
 	Token    name       = { nullptr, 0, TokType::Invalid };
@@ -6416,11 +6422,13 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va )
 	char const* buf_begin = buf;
 	sw          remaining = buf_size;
 
-	static Arena tok_map_arena;
+	local_persist
+	Arena tok_map_arena;
 
 	HashTable<StrC> tok_map;
 	{
-		static char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
+		local_persist
+		char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
 
 		tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) );
 

project/gen.hpp

@@ -1888,26 +1888,52 @@ StrC token_fmt_impl( sw num, ... )
 	extern CodeType t_f64;
 #endif
 
+#ifndef GEN_GLOBAL_BUCKET_SIZE
+#	define GEN_GLOBAL_BUCKET_SIZE megabytes(10)
+#endif
+#ifndef GEN_CODEPOOL_NUM_BLOCKS
+#	define GEN_CODEPOOL_NUM_BLOCKS kilobytes(64)
+#endif
+#ifndef GEN_SIZE_PER_STRING_ARENA
+#	define GEN_SIZE_PER_STRING_ARENA megabytes(1)
+#endif
+#ifndef GEN_MAX_COMMENT_LINE_LENGTH
+#	define GEN_MAX_COMMENT_LINE_LENGTH 1024
+#endif
+#ifndef GEN_MAX_NAME_LENGTH
+#	define GEN_MAX_NAME_LENGTH 128
+#endif
+#ifndef GEN_MAX_UNTYPED_STR_LENGTH
+#	define GEN_MAX_UNTYPED_STR_LENGTH kilobytes(640)
+#endif
+#ifndef GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE
+#	define GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE kilobytes(4)
+#endif
+#ifndef GEN_LEX_ALLOCATOR_SIZE
+#	define GEN_LEX_ALLOCATOR_SIZE megabytes(10)
+#endif
+#ifndef GEN_BUILDER_STR_BUFFER_RESERVE
+#	define GEN_BUILDER_STR_BUFFER_RESERVE megabytes(1)
+#endif
+
 	// These constexprs are used for allocation behavior of data structures
 	// or string handling while constructing or serializing.
 	// Change them to suit your needs.
 
-	constexpr s32 InitSize_DataArrays  = 16;
+	constexpr s32 InitSize_DataArrays = 16;
-	constexpr s32 InitSize_StringTable = megabytes(4);
 
 	// NOTE: This limits the maximum size of an allocation
 	// If you are generating a string larger than this, increase the size of the bucket here.
-	constexpr uw  Global_BucketSize         = megabytes(10);
+	constexpr uw  Global_BucketSize         = GEN_GLOBAL_BUCKET_SIZE;
-	constexpr s32 CodePool_NumBlocks        = kilobytes(64);
+	constexpr s32 CodePool_NumBlocks        = GEN_CODEPOOL_NUM_BLOCKS;
-	constexpr s32 SizePer_StringArena       = megabytes(1);
+	constexpr s32 SizePer_StringArena       = GEN_SIZE_PER_STRING_ARENA;
 
-	constexpr s32 MaxCommentLineLength      = 1024;
+	constexpr s32 MaxCommentLineLength      = GEN_MAX_COMMENT_LINE_LENGTH;
-	constexpr s32 MaxNameLength             = 128;
+	constexpr s32 MaxNameLength             = GEN_MAX_NAME_LENGTH;
-	constexpr s32 MaxUntypedStrLength       = kilobytes(640);
+	constexpr s32 MaxUntypedStrLength       = GEN_MAX_UNTYPED_STR_LENGTH;
-	constexpr s32 StringTable_MaxHashLength = kilobytes(1);
+	constexpr s32 TokenFmt_TokenMap_MemSize	= GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE;
-	constexpr s32 TokenFmt_TokenMap_MemSize	= kilobytes(4);
+	constexpr s32 LexAllocator_Size         = GEN_LEX_ALLOCATOR_SIZE;
-	constexpr s32 LexAllocator_Size         = megabytes(10);
+	constexpr s32 Builder_StrBufferReserve  = GEN_BUILDER_STR_BUFFER_RESERVE;
-	constexpr s32 Builder_StrBufferReserve  = megabytes(1);
 
 	extern CodeType t_auto;
 	extern CodeType t_void;

test/sanity.cpp

@@ -39,7 +39,7 @@ void check_sanity()
 		Num String Cache        : 30000025
 	*/
 	constexpr
-	s32 num_iterations = 650000;
+	s32 num_iterations = 325000;
 
 	Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( GlobalAllocator, num_iterations * 2 );