Adding tuning macros for memory usage of the library, fixes.

Reduced the sanity test case to half its original iterations.
This commit is contained in:
Edward R. Gonzalez 2023-07-23 23:11:53 -04:00
parent 5ce8bfa259
commit 80b5c9768d
4 changed files with 63 additions and 25 deletions

View File

@ -12,7 +12,7 @@ These build up a code AST to then serialize with a file builder.
* [Building](#notes)
* [Outline](#outline)
* [What is not provided](#what-is-not-provided)
* [The four constructors](#there-are-four-sets-of-interfaces-for-code-ast-generation-the-library-provides)
* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides)
* [Predefined Codes](#predefined-codes)
* [Code generation and modification](#code-generation-and-modification)
* [On multithreading](#on-multi-threading)
@ -291,12 +291,16 @@ Data Notes:
* I will be augmenting the single arena with a simple slag allocator.
* Linked lists used children nodes on bodies, and parameters.
* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup.
* When benchmarking, the three most significant var to tune are:
* `Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
* `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
* `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
* The default values can handled generating for a string up to a size of ~650 kbs (bottleneck is serialization).
* Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
* Allocations can be tuned by defining the folloiwng macros:
* `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator
* `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator
* `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching.
* `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line.
* `GEN_MAX_NAME_LENGTH` : Max length of any identifier.
* `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code.
* `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable.
* `GEN_LEX_ALLOCATOR_SIZE`
* `GEN_BUILDER_STR_BUFFER_RESERVE`
Two generic templated containers are used throughout the library:

View File

@ -4115,6 +4115,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
using namespace Parser;
Token start;
s32 len = 0;
if ( check(TokType::Attribute_Open) )
{
@ -4126,6 +4127,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
}
eat( TokType::Attribute_Close );
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
}
else if ( check(TokType::Decl_GNU_Attribute) )
@ -4140,6 +4143,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
eat(TokType::BraceCurly_Close);
eat(TokType::BraceCurly_Close);
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
}
else if ( check(TokType::Decl_MSVC_Attribute) )
@ -4153,15 +4158,16 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
}
eat(TokType::BraceCurly_Close);
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
}
else if ( tok_is_attribute( currtok ) )
{
eat(currtok.Type);
s32 len = start.Length;
}
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
if ( len > 0 )
{
StrC attribute_txt = { len, start.Text };
@ -6237,7 +6243,7 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
{
using namespace Parser;
SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
SpecifierT specs_found[16] { ESpecifier::Invalid };
s32 num_specifiers = 0;
Token name = { nullptr, 0, TokType::Invalid };
@ -6416,11 +6422,13 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va )
char const* buf_begin = buf;
sw remaining = buf_size;
static Arena tok_map_arena;
local_persist
Arena tok_map_arena;
HashTable<StrC> tok_map;
{
static char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
local_persist
char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) );

View File

@ -1888,26 +1888,52 @@ StrC token_fmt_impl( sw num, ... )
extern CodeType t_f64;
#endif
#ifndef GEN_GLOBAL_BUCKET_SIZE
# define GEN_GLOBAL_BUCKET_SIZE megabytes(10)
#endif
#ifndef GEN_CODEPOOL_NUM_BLOCKS
# define GEN_CODEPOOL_NUM_BLOCKS kilobytes(64)
#endif
#ifndef GEN_SIZE_PER_STRING_ARENA
# define GEN_SIZE_PER_STRING_ARENA megabytes(1)
#endif
#ifndef GEN_MAX_COMMENT_LINE_LENGTH
# define GEN_MAX_COMMENT_LINE_LENGTH 1024
#endif
#ifndef GEN_MAX_NAME_LENGTH
# define GEN_MAX_NAME_LENGTH 128
#endif
#ifndef GEN_MAX_UNTYPED_STR_LENGTH
# define GEN_MAX_UNTYPED_STR_LENGTH kilobytes(640)
#endif
#ifndef GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE
# define GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE kilobytes(4)
#endif
#ifndef GEN_LEX_ALLOCATOR_SIZE
# define GEN_LEX_ALLOCATOR_SIZE megabytes(10)
#endif
#ifndef GEN_BUILDER_STR_BUFFER_RESERVE
# define GEN_BUILDER_STR_BUFFER_RESERVE megabytes(1)
#endif
// These constexprs are used for allocation behavior of data structures
// or string handling while constructing or serializing.
// Change them to suit your needs.
constexpr s32 InitSize_DataArrays = 16;
constexpr s32 InitSize_StringTable = megabytes(4);
constexpr s32 InitSize_DataArrays = 16;
// NOTE: This limits the maximum size of an allocation
// If you are generating a string larger than this, increase the size of the bucket here.
constexpr uw Global_BucketSize = megabytes(10);
constexpr s32 CodePool_NumBlocks = kilobytes(64);
constexpr s32 SizePer_StringArena = megabytes(1);
constexpr uw Global_BucketSize = GEN_GLOBAL_BUCKET_SIZE;
constexpr s32 CodePool_NumBlocks = GEN_CODEPOOL_NUM_BLOCKS;
constexpr s32 SizePer_StringArena = GEN_SIZE_PER_STRING_ARENA;
constexpr s32 MaxCommentLineLength = 1024;
constexpr s32 MaxNameLength = 128;
constexpr s32 MaxUntypedStrLength = kilobytes(640);
constexpr s32 StringTable_MaxHashLength = kilobytes(1);
constexpr s32 TokenFmt_TokenMap_MemSize = kilobytes(4);
constexpr s32 LexAllocator_Size = megabytes(10);
constexpr s32 Builder_StrBufferReserve = megabytes(1);
constexpr s32 MaxCommentLineLength = GEN_MAX_COMMENT_LINE_LENGTH;
constexpr s32 MaxNameLength = GEN_MAX_NAME_LENGTH;
constexpr s32 MaxUntypedStrLength = GEN_MAX_UNTYPED_STR_LENGTH;
constexpr s32 TokenFmt_TokenMap_MemSize = GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE;
constexpr s32 LexAllocator_Size = GEN_LEX_ALLOCATOR_SIZE;
constexpr s32 Builder_StrBufferReserve = GEN_BUILDER_STR_BUFFER_RESERVE;
extern CodeType t_auto;
extern CodeType t_void;

View File

@ -39,7 +39,7 @@ void check_sanity()
Num String Cache : 30000025
*/
constexpr
s32 num_iterations = 650000;
s32 num_iterations = 325000;
Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( GlobalAllocator, num_iterations * 2 );