diff --git a/Readme.md b/Readme.md index 2f09e76..2cdd0db 100644 --- a/Readme.md +++ b/Readme.md @@ -12,7 +12,7 @@ These build up a code AST to then serialize with a file builder. * [Building](#notes) * [Outline](#outline) * [What is not provided](#what-is-not-provided) -* [The four constructors](#there-are-four-sets-of-interfaces-for-code-ast-generation-the-library-provides) +* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides) * [Predefined Codes](#predefined-codes) * [Code generation and modification](#code-generation-and-modification) * [On multithreading](#on-multi-threading) @@ -291,12 +291,16 @@ Data Notes: * I will be augmenting the single arena with a simple slag allocator. * Linked lists used children nodes on bodies, and parameters. * Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup. -* When benchmarking, the three most significant var to tune are: - * `Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena. - * `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings. - * `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs. - * The default values can handled generating for a string up to a size of ~650 kbs (bottleneck is serialization). - * Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it) +* Allocations can be tuned by defining the folloiwng macros: + * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator + * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator + * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching. + * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line. + * `GEN_MAX_NAME_LENGTH` : Max length of any identifier. + * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code. + * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable. + * `GEN_LEX_ALLOCATOR_SIZE` + * `GEN_BUILDER_STR_BUFFER_RESERVE` Two generic templated containers are used throughout the library: diff --git a/project/gen.cpp b/project/gen.cpp index ccd0190..bb5ff91 100644 --- a/project/gen.cpp +++ b/project/gen.cpp @@ -4115,6 +4115,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) using namespace Parser; Token start; + s32 len = 0; if ( check(TokType::Attribute_Open) ) { @@ -4126,6 +4127,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) } eat( TokType::Attribute_Close ); + + s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; } else if ( check(TokType::Decl_GNU_Attribute) ) @@ -4140,6 +4143,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) eat(TokType::BraceCurly_Close); eat(TokType::BraceCurly_Close); + + s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; } else if ( check(TokType::Decl_MSVC_Attribute) ) @@ -4153,15 +4158,16 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) } eat(TokType::BraceCurly_Close); + + s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; } else if ( tok_is_attribute( currtok ) ) { eat(currtok.Type); + s32 len = start.Length; } - s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; - if ( len > 0 ) { StrC attribute_txt = { len, start.Text }; @@ -6237,7 +6243,7 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context ) { using namespace Parser; - SpecifierT specs_found[16] { ESpecifier::Num_Specifiers }; + SpecifierT specs_found[16] { ESpecifier::Invalid }; s32 num_specifiers = 0; Token name = { nullptr, 0, TokType::Invalid }; @@ -6416,11 +6422,13 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va ) char const* buf_begin = buf; sw remaining = buf_size; - static Arena tok_map_arena; + local_persist + Arena tok_map_arena; HashTable tok_map; { - static char tok_map_mem[ TokenFmt_TokenMap_MemSize ]; + local_persist + char tok_map_mem[ TokenFmt_TokenMap_MemSize ]; tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) ); diff --git a/project/gen.hpp b/project/gen.hpp index ecac2b3..529e00a 100644 --- a/project/gen.hpp +++ b/project/gen.hpp @@ -1888,26 +1888,52 @@ StrC token_fmt_impl( sw num, ... ) extern CodeType t_f64; #endif +#ifndef GEN_GLOBAL_BUCKET_SIZE +# define GEN_GLOBAL_BUCKET_SIZE megabytes(10) +#endif +#ifndef GEN_CODEPOOL_NUM_BLOCKS +# define GEN_CODEPOOL_NUM_BLOCKS kilobytes(64) +#endif +#ifndef GEN_SIZE_PER_STRING_ARENA +# define GEN_SIZE_PER_STRING_ARENA megabytes(1) +#endif +#ifndef GEN_MAX_COMMENT_LINE_LENGTH +# define GEN_MAX_COMMENT_LINE_LENGTH 1024 +#endif +#ifndef GEN_MAX_NAME_LENGTH +# define GEN_MAX_NAME_LENGTH 128 +#endif +#ifndef GEN_MAX_UNTYPED_STR_LENGTH +# define GEN_MAX_UNTYPED_STR_LENGTH kilobytes(640) +#endif +#ifndef GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE +# define GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE kilobytes(4) +#endif +#ifndef GEN_LEX_ALLOCATOR_SIZE +# define GEN_LEX_ALLOCATOR_SIZE megabytes(10) +#endif +#ifndef GEN_BUILDER_STR_BUFFER_RESERVE +# define GEN_BUILDER_STR_BUFFER_RESERVE megabytes(1) +#endif + // These constexprs are used for allocation behavior of data structures // or string handling while constructing or serializing. // Change them to suit your needs. - constexpr s32 InitSize_DataArrays = 16; - constexpr s32 InitSize_StringTable = megabytes(4); + constexpr s32 InitSize_DataArrays = 16; // NOTE: This limits the maximum size of an allocation // If you are generating a string larger than this, increase the size of the bucket here. - constexpr uw Global_BucketSize = megabytes(10); - constexpr s32 CodePool_NumBlocks = kilobytes(64); - constexpr s32 SizePer_StringArena = megabytes(1); + constexpr uw Global_BucketSize = GEN_GLOBAL_BUCKET_SIZE; + constexpr s32 CodePool_NumBlocks = GEN_CODEPOOL_NUM_BLOCKS; + constexpr s32 SizePer_StringArena = GEN_SIZE_PER_STRING_ARENA; - constexpr s32 MaxCommentLineLength = 1024; - constexpr s32 MaxNameLength = 128; - constexpr s32 MaxUntypedStrLength = kilobytes(640); - constexpr s32 StringTable_MaxHashLength = kilobytes(1); - constexpr s32 TokenFmt_TokenMap_MemSize = kilobytes(4); - constexpr s32 LexAllocator_Size = megabytes(10); - constexpr s32 Builder_StrBufferReserve = megabytes(1); + constexpr s32 MaxCommentLineLength = GEN_MAX_COMMENT_LINE_LENGTH; + constexpr s32 MaxNameLength = GEN_MAX_NAME_LENGTH; + constexpr s32 MaxUntypedStrLength = GEN_MAX_UNTYPED_STR_LENGTH; + constexpr s32 TokenFmt_TokenMap_MemSize = GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE; + constexpr s32 LexAllocator_Size = GEN_LEX_ALLOCATOR_SIZE; + constexpr s32 Builder_StrBufferReserve = GEN_BUILDER_STR_BUFFER_RESERVE; extern CodeType t_auto; extern CodeType t_void; diff --git a/test/sanity.cpp b/test/sanity.cpp index 90fdfe5..8ba3378 100644 --- a/test/sanity.cpp +++ b/test/sanity.cpp @@ -39,7 +39,7 @@ void check_sanity() Num String Cache : 30000025 */ constexpr - s32 num_iterations = 650000; + s32 num_iterations = 325000; Array typedefs = Array::init_reserve( GlobalAllocator, num_iterations * 2 );