ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2025-10-30 14:30:53 -07:00
Code Issues Packages Projects Releases Wiki Activity

Adding tuning macros for memory usage of the library, fixes.

Browse Source 
Reduced the sanity test case to half its original iterations.
This commit is contained in:
Edward R. Gonzalez
2023-07-23 23:11:53 -04:00
parent 5ce8bfa259
commit 80b5c9768d
4 changed files with 63 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Readme.md
View File

@@ -12,7 +12,7 @@ These build up a code AST to then serialize with a file builder.
* [Building](#notes)
* [Outline](#outline)
* [What is not provided](#what-is-not-provided)
* [The four constructors](#there-are-four-sets-of-interfaces-for-code-ast-generation-the-library-provides)
* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides)
* [Predefined Codes](#predefined-codes)
* [Code generation and modification](#code-generation-and-modification)
* [On multithreading](#on-multi-threading)
@@ -291,12 +291,16 @@ Data Notes:
* I will be augmenting the single arena with a simple slag allocator.
* Linked lists used children nodes on bodies, and parameters.
* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup.
* When benchmarking, the three most significant var to tune are:
* `Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
* `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
* `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
* The default values can handled generating for a string up to a size of ~650 kbs (bottleneck is serialization).
* Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
* Allocations can be tuned by defining the folloiwng macros:
* `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator
* `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator
* `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching.
* `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line.
* `GEN_MAX_NAME_LENGTH` : Max length of any identifier.
* `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code.
* `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable.
* `GEN_LEX_ALLOCATOR_SIZE`
* `GEN_BUILDER_STR_BUFFER_RESERVE`
Two generic templated containers are used throughout the library:

18
project/gen.cpp
View File

@@ -4115,6 +4115,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
using namespace Parser;
Token start;
s32 len = 0;
if ( check(TokType::Attribute_Open) )
{
@@ -4126,6 +4127,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
}
eat( TokType::Attribute_Close );
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
}
else if ( check(TokType::Decl_GNU_Attribute) )
@@ -4140,6 +4143,8 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
eat(TokType::BraceCurly_Close);
eat(TokType::BraceCurly_Close);
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
}
else if ( check(TokType::Decl_MSVC_Attribute) )
@@ -4153,15 +4158,16 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
}
eat(TokType::BraceCurly_Close);
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
}
else if ( tok_is_attribute( currtok ) )
{
eat(currtok.Type);
s32 len = start.Length;
}
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
if ( len > 0 )
{
StrC attribute_txt = { len, start.Text };
@@ -6237,7 +6243,7 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
{
using namespace Parser;
SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
SpecifierT specs_found[16] { ESpecifier::Invalid };
s32 num_specifiers = 0;
Token name = { nullptr, 0, TokType::Invalid };
@@ -6416,11 +6422,13 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va )
char const* buf_begin = buf;
sw remaining = buf_size;
static Arena tok_map_arena;
local_persist
Arena tok_map_arena;
HashTable<StrC> tok_map;
{
static char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
local_persist
char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) );

48
project/gen.hpp
View File

@@ -1888,26 +1888,52 @@ StrC token_fmt_impl( sw num, ... )
extern CodeType t_f64;
#endif
#ifndef GEN_GLOBAL_BUCKET_SIZE
# define GEN_GLOBAL_BUCKET_SIZE megabytes(10)
#endif
#ifndef GEN_CODEPOOL_NUM_BLOCKS
# define GEN_CODEPOOL_NUM_BLOCKS kilobytes(64)
#endif
#ifndef GEN_SIZE_PER_STRING_ARENA
# define GEN_SIZE_PER_STRING_ARENA megabytes(1)
#endif
#ifndef GEN_MAX_COMMENT_LINE_LENGTH
# define GEN_MAX_COMMENT_LINE_LENGTH 1024
#endif
#ifndef GEN_MAX_NAME_LENGTH
# define GEN_MAX_NAME_LENGTH 128
#endif
#ifndef GEN_MAX_UNTYPED_STR_LENGTH
# define GEN_MAX_UNTYPED_STR_LENGTH kilobytes(640)
#endif
#ifndef GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE
# define GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE kilobytes(4)
#endif
#ifndef GEN_LEX_ALLOCATOR_SIZE
# define GEN_LEX_ALLOCATOR_SIZE megabytes(10)
#endif
#ifndef GEN_BUILDER_STR_BUFFER_RESERVE
# define GEN_BUILDER_STR_BUFFER_RESERVE megabytes(1)
#endif
// These constexprs are used for allocation behavior of data structures
// or string handling while constructing or serializing.
// Change them to suit your needs.
constexpr s32 InitSize_DataArrays = 16;
constexpr s32 InitSize_StringTable = megabytes(4);
// NOTE: This limits the maximum size of an allocation
// If you are generating a string larger than this, increase the size of the bucket here.
constexpr uw Global_BucketSize = megabytes(10);
constexpr s32 CodePool_NumBlocks = kilobytes(64);
constexpr s32 SizePer_StringArena = megabytes(1);
constexpr uw Global_BucketSize = GEN_GLOBAL_BUCKET_SIZE;
constexpr s32 CodePool_NumBlocks = GEN_CODEPOOL_NUM_BLOCKS;
constexpr s32 SizePer_StringArena = GEN_SIZE_PER_STRING_ARENA;
constexpr s32 MaxCommentLineLength = 1024;
constexpr s32 MaxNameLength = 128;
constexpr s32 MaxUntypedStrLength = kilobytes(640);
constexpr s32 StringTable_MaxHashLength = kilobytes(1);
constexpr s32 TokenFmt_TokenMap_MemSize = kilobytes(4);
constexpr s32 LexAllocator_Size = megabytes(10);
constexpr s32 Builder_StrBufferReserve = megabytes(1);
constexpr s32 MaxCommentLineLength = GEN_MAX_COMMENT_LINE_LENGTH;
constexpr s32 MaxNameLength = GEN_MAX_NAME_LENGTH;
constexpr s32 MaxUntypedStrLength = GEN_MAX_UNTYPED_STR_LENGTH;
constexpr s32 TokenFmt_TokenMap_MemSize = GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE;
constexpr s32 LexAllocator_Size = GEN_LEX_ALLOCATOR_SIZE;
constexpr s32 Builder_StrBufferReserve = GEN_BUILDER_STR_BUFFER_RESERVE;
extern CodeType t_auto;
extern CodeType t_void;

2
test/sanity.cpp
View File

@@ -39,7 +39,7 @@ void check_sanity()
Num String Cache : 30000025
*/
constexpr
s32 num_iterations = 650000;
s32 num_iterations = 325000;
Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( GlobalAllocator, num_iterations * 2 );