mirror of
https://github.com/Ed94/gencpp.git
synced 2024-11-10 02:54:53 -08:00
Adding tuning macros for memory usage of the library, fixes.
Reduced the sanity test case to half its original iterations.
This commit is contained in:
parent
5ce8bfa259
commit
80b5c9768d
18
Readme.md
18
Readme.md
@ -12,7 +12,7 @@ These build up a code AST to then serialize with a file builder.
|
||||
* [Building](#notes)
|
||||
* [Outline](#outline)
|
||||
* [What is not provided](#what-is-not-provided)
|
||||
* [The four constructors](#there-are-four-sets-of-interfaces-for-code-ast-generation-the-library-provides)
|
||||
* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides)
|
||||
* [Predefined Codes](#predefined-codes)
|
||||
* [Code generation and modification](#code-generation-and-modification)
|
||||
* [On multithreading](#on-multi-threading)
|
||||
@ -291,12 +291,16 @@ Data Notes:
|
||||
* I will be augmenting the single arena with a simple slag allocator.
|
||||
* Linked lists used children nodes on bodies, and parameters.
|
||||
* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup.
|
||||
* When benchmarking, the three most significant var to tune are:
|
||||
* `Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
|
||||
* `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
|
||||
* `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
|
||||
* The default values can handled generating for a string up to a size of ~650 kbs (bottleneck is serialization).
|
||||
* Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
|
||||
* Allocations can be tuned by defining the folloiwng macros:
|
||||
* `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator
|
||||
* `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator
|
||||
* `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching.
|
||||
* `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line.
|
||||
* `GEN_MAX_NAME_LENGTH` : Max length of any identifier.
|
||||
* `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code.
|
||||
* `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable.
|
||||
* `GEN_LEX_ALLOCATOR_SIZE`
|
||||
* `GEN_BUILDER_STR_BUFFER_RESERVE`
|
||||
|
||||
Two generic templated containers are used throughout the library:
|
||||
|
||||
|
||||
@ -4115,6 +4115,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
|
||||
using namespace Parser;
|
||||
|
||||
Token start;
|
||||
s32 len = 0;
|
||||
|
||||
if ( check(TokType::Attribute_Open) )
|
||||
{
|
||||
@ -4126,6 +4127,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
|
||||
}
|
||||
|
||||
eat( TokType::Attribute_Close );
|
||||
|
||||
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
|
||||
}
|
||||
|
||||
else if ( check(TokType::Decl_GNU_Attribute) )
|
||||
@ -4140,6 +4143,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
|
||||
|
||||
eat(TokType::BraceCurly_Close);
|
||||
eat(TokType::BraceCurly_Close);
|
||||
|
||||
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
|
||||
}
|
||||
|
||||
else if ( check(TokType::Decl_MSVC_Attribute) )
|
||||
@ -4153,15 +4158,16 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
|
||||
}
|
||||
|
||||
eat(TokType::BraceCurly_Close);
|
||||
|
||||
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
|
||||
}
|
||||
|
||||
else if ( tok_is_attribute( currtok ) )
|
||||
{
|
||||
eat(currtok.Type);
|
||||
s32 len = start.Length;
|
||||
}
|
||||
|
||||
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
|
||||
|
||||
if ( len > 0 )
|
||||
{
|
||||
StrC attribute_txt = { len, start.Text };
|
||||
@ -6237,7 +6243,7 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
|
||||
{
|
||||
using namespace Parser;
|
||||
|
||||
SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
|
||||
SpecifierT specs_found[16] { ESpecifier::Invalid };
|
||||
s32 num_specifiers = 0;
|
||||
|
||||
Token name = { nullptr, 0, TokType::Invalid };
|
||||
@ -6416,11 +6422,13 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va )
|
||||
char const* buf_begin = buf;
|
||||
sw remaining = buf_size;
|
||||
|
||||
static Arena tok_map_arena;
|
||||
local_persist
|
||||
Arena tok_map_arena;
|
||||
|
||||
HashTable<StrC> tok_map;
|
||||
{
|
||||
static char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
|
||||
local_persist
|
||||
char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
|
||||
|
||||
tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) );
|
||||
|
||||
|
||||
@ -1888,26 +1888,52 @@ StrC token_fmt_impl( sw num, ... )
|
||||
extern CodeType t_f64;
|
||||
#endif
|
||||
|
||||
#ifndef GEN_GLOBAL_BUCKET_SIZE
|
||||
# define GEN_GLOBAL_BUCKET_SIZE megabytes(10)
|
||||
#endif
|
||||
#ifndef GEN_CODEPOOL_NUM_BLOCKS
|
||||
# define GEN_CODEPOOL_NUM_BLOCKS kilobytes(64)
|
||||
#endif
|
||||
#ifndef GEN_SIZE_PER_STRING_ARENA
|
||||
# define GEN_SIZE_PER_STRING_ARENA megabytes(1)
|
||||
#endif
|
||||
#ifndef GEN_MAX_COMMENT_LINE_LENGTH
|
||||
# define GEN_MAX_COMMENT_LINE_LENGTH 1024
|
||||
#endif
|
||||
#ifndef GEN_MAX_NAME_LENGTH
|
||||
# define GEN_MAX_NAME_LENGTH 128
|
||||
#endif
|
||||
#ifndef GEN_MAX_UNTYPED_STR_LENGTH
|
||||
# define GEN_MAX_UNTYPED_STR_LENGTH kilobytes(640)
|
||||
#endif
|
||||
#ifndef GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE
|
||||
# define GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE kilobytes(4)
|
||||
#endif
|
||||
#ifndef GEN_LEX_ALLOCATOR_SIZE
|
||||
# define GEN_LEX_ALLOCATOR_SIZE megabytes(10)
|
||||
#endif
|
||||
#ifndef GEN_BUILDER_STR_BUFFER_RESERVE
|
||||
# define GEN_BUILDER_STR_BUFFER_RESERVE megabytes(1)
|
||||
#endif
|
||||
|
||||
// These constexprs are used for allocation behavior of data structures
|
||||
// or string handling while constructing or serializing.
|
||||
// Change them to suit your needs.
|
||||
|
||||
constexpr s32 InitSize_DataArrays = 16;
|
||||
constexpr s32 InitSize_StringTable = megabytes(4);
|
||||
|
||||
// NOTE: This limits the maximum size of an allocation
|
||||
// If you are generating a string larger than this, increase the size of the bucket here.
|
||||
constexpr uw Global_BucketSize = megabytes(10);
|
||||
constexpr s32 CodePool_NumBlocks = kilobytes(64);
|
||||
constexpr s32 SizePer_StringArena = megabytes(1);
|
||||
constexpr uw Global_BucketSize = GEN_GLOBAL_BUCKET_SIZE;
|
||||
constexpr s32 CodePool_NumBlocks = GEN_CODEPOOL_NUM_BLOCKS;
|
||||
constexpr s32 SizePer_StringArena = GEN_SIZE_PER_STRING_ARENA;
|
||||
|
||||
constexpr s32 MaxCommentLineLength = 1024;
|
||||
constexpr s32 MaxNameLength = 128;
|
||||
constexpr s32 MaxUntypedStrLength = kilobytes(640);
|
||||
constexpr s32 StringTable_MaxHashLength = kilobytes(1);
|
||||
constexpr s32 TokenFmt_TokenMap_MemSize = kilobytes(4);
|
||||
constexpr s32 LexAllocator_Size = megabytes(10);
|
||||
constexpr s32 Builder_StrBufferReserve = megabytes(1);
|
||||
constexpr s32 MaxCommentLineLength = GEN_MAX_COMMENT_LINE_LENGTH;
|
||||
constexpr s32 MaxNameLength = GEN_MAX_NAME_LENGTH;
|
||||
constexpr s32 MaxUntypedStrLength = GEN_MAX_UNTYPED_STR_LENGTH;
|
||||
constexpr s32 TokenFmt_TokenMap_MemSize = GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE;
|
||||
constexpr s32 LexAllocator_Size = GEN_LEX_ALLOCATOR_SIZE;
|
||||
constexpr s32 Builder_StrBufferReserve = GEN_BUILDER_STR_BUFFER_RESERVE;
|
||||
|
||||
extern CodeType t_auto;
|
||||
extern CodeType t_void;
|
||||
|
||||
@ -39,7 +39,7 @@ void check_sanity()
|
||||
Num String Cache : 30000025
|
||||
*/
|
||||
constexpr
|
||||
s32 num_iterations = 650000;
|
||||
s32 num_iterations = 325000;
|
||||
|
||||
Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( GlobalAllocator, num_iterations * 2 );
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user