From 7634aeb34c81f9f30db4271ac25af58de2033c72 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 16 Jul 2023 18:00:07 -0400 Subject: [PATCH] Fixes to memory mangment, library is much faster now. --- Readme.md | 9 ++ project/gen.cpp | 55 +++---- project/gen.hpp | 36 ++++- project/gen_dep.cpp | 247 ++++++++++++++++++++++++++++++- project/gen_dep.hpp | 154 +++++++++++-------- scripts/build.ci.ps1 | 2 +- scripts/genccp.natstepfilter | 3 + scripts/gencpp.natvis | 17 ++- test/Parsed/Array.Parsed.hpp | 2 + test/Parsed/Buffer.Parsed.hpp | 2 + test/Parsed/HashTable.Parsed.hpp | 2 + test/Parsed/Ring.Parsed.hpp | 2 + test/Parsed/Sanity.Parsed.hpp | 4 +- test/parsing.cpp | 0 test/parsing.hpp | 0 test/sanity.cpp | 92 ++++++++++++ test/test.Upfront.cpp | 5 +- test/test.cpp | 66 ++------- test/test.parsing.cpp | 75 ++++++++++ test/upfront.cpp | 0 test/upfront.hpp | 21 +++ 21 files changed, 621 insertions(+), 173 deletions(-) create mode 100644 test/parsing.cpp create mode 100644 test/parsing.hpp create mode 100644 test/sanity.cpp create mode 100644 test/test.parsing.cpp create mode 100644 test/upfront.cpp create mode 100644 test/upfront.hpp diff --git a/Readme.md b/Readme.md index d54f0a0..710e439 100644 --- a/Readme.md +++ b/Readme.md @@ -23,6 +23,9 @@ These build up a code AST to then serialize with a file builder. The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases. +Note: Do not trying to do any large generations with this (at least not without changing the serialization implementation). +It does not resue any memory yet for dynamic strings and thus any signficant size memory will result in massive consumption. + The project has no external dependencies beyond: * `stdarg.h` @@ -287,6 +290,12 @@ Data Notes: * Memory within the buckets is not resused, so its inherently wasteful (most likely will give non-cached strings their own tailored allocator later) * Linked lists used children nodes on bodies, and parameters. * Its intended to generate the AST in one go and serialize after. The contructors and serializer are designed to be a "one pass, front to back" setup. +* When benchmarking, the three most significant var to tune are: + * `Memory::Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena. + * `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings. + * `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs. + * The default values can handled generating for a stirng up to a size of ~650 kbs (bottleneck is serialization). + * Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it) Two generic templated containers are used throughout the library: diff --git a/project/gen.cpp b/project/gen.cpp index 59c3465..3ac8b88 100644 --- a/project/gen.cpp +++ b/project/gen.cpp @@ -386,6 +386,10 @@ namespace gen local_persist thread_local char SerializationLevel = 0; + #if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION) + u64 time_start = time_rel_ms(); + #endif + // TODO : Need to refactor so that intermeidate strings are freed conviently. String result = String::make( Memory::GlobalAllocator, "" ); @@ -980,8 +984,10 @@ namespace gen break; } + #if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION) + log_fmt("AST::to_string() time taken: %llu for: %s\n", time_rel_ms() - time_start, result ); + #endif return result; - #undef ProcessModuleFlags } @@ -1282,19 +1288,21 @@ namespace gen { using namespace StaticData; - Arena& last = StringArenas.back(); + Arena* last = & StringArenas.back(); - if ( last.TotalUsed + str_length > last.TotalSize ) + uw size_req = str_length + sizeof(String::Header) + sizeof(char*); + + if ( last->TotalUsed + size_req > last->TotalSize ) { Arena new_arena = Arena::init_from_allocator( Allocator_StringArena, SizePer_StringArena ); if ( ! StringArenas.append( new_arena ) ) fatal( "gen::get_string_allocator: Failed to allocate a new string arena" ); - last = StringArenas.back(); + last = & StringArenas.back(); } - return last; + return * last; } // Will either make or retrive a code string. @@ -1303,7 +1311,7 @@ namespace gen using namespace StaticData; s32 hash_length = str.Len > kilobytes(1) ? kilobytes(1) : str.Len; - u32 key = crc32( str.Ptr, hash_length ); + u64 key = crc32( str.Ptr, hash_length ); { StringCached* result = StringCache.get( key ); @@ -1325,22 +1333,8 @@ namespace gen { using namespace StaticData; - AllocatorInfo allocator = CodePools.back(); - - s32 index = 0; - s32 left = CodePools.num(); - do - { - if ( CodePools[index].FreeList != nullptr ) - { - allocator = CodePools[index]; - break; - } - index++; - } - while ( left--, left ); - - if ( allocator.Data == nullptr ) + Pool* allocator = & CodePools.back(); + if ( allocator->FreeList == nullptr ) { Pool code_pool = Pool::init( Allocator_CodePool, CodePool_NumBlocks, sizeof(AST) ); @@ -1350,10 +1344,10 @@ namespace gen if ( ! CodePools.append( code_pool ) ) fatal( "gen::make_code: Failed to allocate a new code pool - CodePools failed to append new pool." ); - allocator = * CodePools; + allocator = & CodePools.back(); } - Code result { rcast( AST*, alloc( allocator, sizeof(AST) )) }; + Code result { rcast( AST*, alloc( * allocator, sizeof(AST) )) }; result->Content = { nullptr }; result->Prev = { nullptr }; @@ -3426,16 +3420,6 @@ namespace gen return { 0, nullptr }; \ } - // do_once_start - // LexAllocator = Arena::init_from_allocator( heap(), megabytes(10) ); - - // if ( LexAllocator.PhysicalStart == nullptr ) - // { - // log_failure( "gen::lex: failed to allocate memory for parsing constructor's lexer"); - // return { { nullptr }, 0 }; - // } - // do_once_end - local_persist thread_local Array Tokens = { nullptr }; @@ -6380,7 +6364,7 @@ namespace gen return false; } - Buffer = String::make( Memory::GlobalAllocator, "" ); + Buffer = String::make_reserve( Memory::GlobalAllocator, Builder_StrBufferReserve ); return true; } @@ -6393,6 +6377,7 @@ namespace gen log_failure("gen::File::write - Failed to write to file: %s", file_name( & File ) ); file_close( & File ); + Buffer.free(); } #pragma endregion Builder diff --git a/project/gen.hpp b/project/gen.hpp index 1adad33..66325c2 100644 --- a/project/gen.hpp +++ b/project/gen.hpp @@ -8,10 +8,6 @@ */ #pragma once -#define GEN_FEATURE_PARSING -#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS -#define GEN_ENFORCE_STRONG_CODE_TYPES - #ifdef gen_time //! If its desired to roll your own dependencies, define GENCPP_PROVIDE_DEPENDENCIES before including this file. // Dependencies are derived from the c-zpl library: https://github.com/zpl-c/zpl @@ -1659,8 +1655,8 @@ namespace gen constexpr s32 InitSize_DataArrays = 16; constexpr s32 InitSize_StringTable = megabytes(4); - constexpr s32 CodePool_NumBlocks = 4096; - constexpr s32 SizePer_StringArena = megabytes(32); + constexpr s32 CodePool_NumBlocks = kilobytes(4); + constexpr s32 SizePer_StringArena = megabytes(1); constexpr s32 MaxCommentLineLength = 1024; constexpr s32 MaxNameLength = 128; @@ -1668,6 +1664,7 @@ namespace gen constexpr s32 StringTable_MaxHashLength = kilobytes(1); constexpr s32 TokenFmt_TokenMap_MemSize = kilobytes(4); constexpr s32 LexAllocator_Size = megabytes(10); + constexpr s32 Builder_StrBufferReserve = megabytes(1); extern CodeType t_auto; extern CodeType t_void; @@ -2053,3 +2050,30 @@ namespace gen // end: gen_time #endif +#ifdef GEN_EXPOSE_BACKEND +namespace gen +{ + namespace Memory + { + extern Array Global_AllocatorBuckets; + } + + namespace StaticData + { + extern Array< Pool > CodePools; + extern Array< Arena > StringArenas; + + extern StringTable StringCache; + + extern Arena LexArena; + + extern AllocatorInfo Allocator_DataArrays; + extern AllocatorInfo Allocator_CodePool; + extern AllocatorInfo Allocator_Lexer; + extern AllocatorInfo Allocator_StringArena; + extern AllocatorInfo Allocator_StringTable; + extern AllocatorInfo Allocator_TypeTable; + } +} + +#endif \ No newline at end of file diff --git a/project/gen_dep.cpp b/project/gen_dep.cpp index 63730ff..3339fe5 100644 --- a/project/gen_dep.cpp +++ b/project/gen_dep.cpp @@ -2,6 +2,8 @@ #ifdef gen_time +#ifdef GEN_BENCHMARK + // NOTE: Ensure we use standard methods for these calls if we use GEN_PICO #pragma region Macros # include @@ -41,6 +43,27 @@ # endif #pragma endregion Macros +#if defined( GEN_SYSTEM_MACOS ) || GEN_SYSTEM_UNIX +# include +# include +#endif + +#if defined( GEN_SYSTEM_MACOS ) +# include +# include +# include +#endif + +#if defined( GEN_SYSTEM_EMSCRIPTEN ) +# include +#endif + +#if defined( GEN_SYSTEM_WINDOWS ) +# include +#endif +#endif + + namespace gen { #pragma region Debug @@ -425,6 +448,7 @@ namespace gen if ( arena->TotalUsed + total_size > (sw) arena->TotalSize ) { // zpl__printf_err("%s", "Arena out of memory\n"); + fatal("Arena out of memory! (Possibly could not fit for the largest size Arena!!)"); return nullptr; } @@ -537,6 +561,8 @@ namespace gen void *data, *curr; uptr* end; + zero_item( &pool ); + pool.Backing = backing; pool.BlockSize = block_size; pool.BlockAlign = block_align; @@ -557,7 +583,7 @@ namespace gen } end = ( uptr* ) curr; - *end = ( uptr ) 0; + *end = ( uptr ) NULL; pool.PhysicalStart = data; pool.FreeList = data; @@ -1151,6 +1177,56 @@ namespace gen result = ( result >> 8 ) ^ ( _crc32_table[ ( result ^ *c ) & 0xff ] ); return ~result; } + + global u64 const _crc64_table[ 256 ] = { + 0x0000000000000000ull, 0x7ad870c830358979ull, 0xf5b0e190606b12f2ull, 0x8f689158505e9b8bull, 0xc038e5739841b68full, 0xbae095bba8743ff6ull, 0x358804e3f82aa47dull, + 0x4f50742bc81f2d04ull, 0xab28ecb46814fe75ull, 0xd1f09c7c5821770cull, 0x5e980d24087fec87ull, 0x24407dec384a65feull, 0x6b1009c7f05548faull, 0x11c8790fc060c183ull, + 0x9ea0e857903e5a08ull, 0xe478989fa00bd371ull, 0x7d08ff3b88be6f81ull, 0x07d08ff3b88be6f8ull, 0x88b81eabe8d57d73ull, 0xf2606e63d8e0f40aull, 0xbd301a4810ffd90eull, + 0xc7e86a8020ca5077ull, 0x4880fbd87094cbfcull, 0x32588b1040a14285ull, 0xd620138fe0aa91f4ull, 0xacf86347d09f188dull, 0x2390f21f80c18306ull, 0x594882d7b0f40a7full, + 0x1618f6fc78eb277bull, 0x6cc0863448deae02ull, 0xe3a8176c18803589ull, 0x997067a428b5bcf0ull, 0xfa11fe77117cdf02ull, 0x80c98ebf2149567bull, 0x0fa11fe77117cdf0ull, + 0x75796f2f41224489ull, 0x3a291b04893d698dull, 0x40f16bccb908e0f4ull, 0xcf99fa94e9567b7full, 0xb5418a5cd963f206ull, 0x513912c379682177ull, 0x2be1620b495da80eull, + 0xa489f35319033385ull, 0xde51839b2936bafcull, 0x9101f7b0e12997f8ull, 0xebd98778d11c1e81ull, 0x64b116208142850aull, 0x1e6966e8b1770c73ull, 0x8719014c99c2b083ull, + 0xfdc17184a9f739faull, 0x72a9e0dcf9a9a271ull, 0x08719014c99c2b08ull, 0x4721e43f0183060cull, 0x3df994f731b68f75ull, 0xb29105af61e814feull, 0xc849756751dd9d87ull, + 0x2c31edf8f1d64ef6ull, 0x56e99d30c1e3c78full, 0xd9810c6891bd5c04ull, 0xa3597ca0a188d57dull, 0xec09088b6997f879ull, 0x96d1784359a27100ull, 0x19b9e91b09fcea8bull, + 0x636199d339c963f2ull, 0xdf7adabd7a6e2d6full, 0xa5a2aa754a5ba416ull, 0x2aca3b2d1a053f9dull, 0x50124be52a30b6e4ull, 0x1f423fcee22f9be0ull, 0x659a4f06d21a1299ull, + 0xeaf2de5e82448912ull, 0x902aae96b271006bull, 0x74523609127ad31aull, 0x0e8a46c1224f5a63ull, 0x81e2d7997211c1e8ull, 0xfb3aa75142244891ull, 0xb46ad37a8a3b6595ull, + 0xceb2a3b2ba0eececull, 0x41da32eaea507767ull, 0x3b024222da65fe1eull, 0xa2722586f2d042eeull, 0xd8aa554ec2e5cb97ull, 0x57c2c41692bb501cull, 0x2d1ab4dea28ed965ull, + 0x624ac0f56a91f461ull, 0x1892b03d5aa47d18ull, 0x97fa21650afae693ull, 0xed2251ad3acf6feaull, 0x095ac9329ac4bc9bull, 0x7382b9faaaf135e2ull, 0xfcea28a2faafae69ull, + 0x8632586aca9a2710ull, 0xc9622c4102850a14ull, 0xb3ba5c8932b0836dull, 0x3cd2cdd162ee18e6ull, 0x460abd1952db919full, 0x256b24ca6b12f26dull, 0x5fb354025b277b14ull, + 0xd0dbc55a0b79e09full, 0xaa03b5923b4c69e6ull, 0xe553c1b9f35344e2ull, 0x9f8bb171c366cd9bull, 0x10e3202993385610ull, 0x6a3b50e1a30ddf69ull, 0x8e43c87e03060c18ull, + 0xf49bb8b633338561ull, 0x7bf329ee636d1eeaull, 0x012b592653589793ull, 0x4e7b2d0d9b47ba97ull, 0x34a35dc5ab7233eeull, 0xbbcbcc9dfb2ca865ull, 0xc113bc55cb19211cull, + 0x5863dbf1e3ac9decull, 0x22bbab39d3991495ull, 0xadd33a6183c78f1eull, 0xd70b4aa9b3f20667ull, 0x985b3e827bed2b63ull, 0xe2834e4a4bd8a21aull, 0x6debdf121b863991ull, + 0x1733afda2bb3b0e8ull, 0xf34b37458bb86399ull, 0x8993478dbb8deae0ull, 0x06fbd6d5ebd3716bull, 0x7c23a61ddbe6f812ull, 0x3373d23613f9d516ull, 0x49aba2fe23cc5c6full, + 0xc6c333a67392c7e4ull, 0xbc1b436e43a74e9dull, 0x95ac9329ac4bc9b5ull, 0xef74e3e19c7e40ccull, 0x601c72b9cc20db47ull, 0x1ac40271fc15523eull, 0x5594765a340a7f3aull, + 0x2f4c0692043ff643ull, 0xa02497ca54616dc8ull, 0xdafce7026454e4b1ull, 0x3e847f9dc45f37c0ull, 0x445c0f55f46abeb9ull, 0xcb349e0da4342532ull, 0xb1eceec59401ac4bull, + 0xfebc9aee5c1e814full, 0x8464ea266c2b0836ull, 0x0b0c7b7e3c7593bdull, 0x71d40bb60c401ac4ull, 0xe8a46c1224f5a634ull, 0x927c1cda14c02f4dull, 0x1d148d82449eb4c6ull, + 0x67ccfd4a74ab3dbfull, 0x289c8961bcb410bbull, 0x5244f9a98c8199c2ull, 0xdd2c68f1dcdf0249ull, 0xa7f41839ecea8b30ull, 0x438c80a64ce15841ull, 0x3954f06e7cd4d138ull, + 0xb63c61362c8a4ab3ull, 0xcce411fe1cbfc3caull, 0x83b465d5d4a0eeceull, 0xf96c151de49567b7ull, 0x76048445b4cbfc3cull, 0x0cdcf48d84fe7545ull, 0x6fbd6d5ebd3716b7ull, + 0x15651d968d029fceull, 0x9a0d8ccedd5c0445ull, 0xe0d5fc06ed698d3cull, 0xaf85882d2576a038ull, 0xd55df8e515432941ull, 0x5a3569bd451db2caull, 0x20ed197575283bb3ull, + 0xc49581ead523e8c2ull, 0xbe4df122e51661bbull, 0x3125607ab548fa30ull, 0x4bfd10b2857d7349ull, 0x04ad64994d625e4dull, 0x7e7514517d57d734ull, 0xf11d85092d094cbfull, + 0x8bc5f5c11d3cc5c6ull, 0x12b5926535897936ull, 0x686de2ad05bcf04full, 0xe70573f555e26bc4ull, 0x9ddd033d65d7e2bdull, 0xd28d7716adc8cfb9ull, 0xa85507de9dfd46c0ull, + 0x273d9686cda3dd4bull, 0x5de5e64efd965432ull, 0xb99d7ed15d9d8743ull, 0xc3450e196da80e3aull, 0x4c2d9f413df695b1ull, 0x36f5ef890dc31cc8ull, 0x79a59ba2c5dc31ccull, + 0x037deb6af5e9b8b5ull, 0x8c157a32a5b7233eull, 0xf6cd0afa9582aa47ull, 0x4ad64994d625e4daull, 0x300e395ce6106da3ull, 0xbf66a804b64ef628ull, 0xc5bed8cc867b7f51ull, + 0x8aeeace74e645255ull, 0xf036dc2f7e51db2cull, 0x7f5e4d772e0f40a7ull, 0x05863dbf1e3ac9deull, 0xe1fea520be311aafull, 0x9b26d5e88e0493d6ull, 0x144e44b0de5a085dull, + 0x6e963478ee6f8124ull, 0x21c640532670ac20ull, 0x5b1e309b16452559ull, 0xd476a1c3461bbed2ull, 0xaeaed10b762e37abull, 0x37deb6af5e9b8b5bull, 0x4d06c6676eae0222ull, + 0xc26e573f3ef099a9ull, 0xb8b627f70ec510d0ull, 0xf7e653dcc6da3dd4ull, 0x8d3e2314f6efb4adull, 0x0256b24ca6b12f26ull, 0x788ec2849684a65full, 0x9cf65a1b368f752eull, + 0xe62e2ad306bafc57ull, 0x6946bb8b56e467dcull, 0x139ecb4366d1eea5ull, 0x5ccebf68aecec3a1ull, 0x2616cfa09efb4ad8ull, 0xa97e5ef8cea5d153ull, 0xd3a62e30fe90582aull, + 0xb0c7b7e3c7593bd8ull, 0xca1fc72bf76cb2a1ull, 0x45775673a732292aull, 0x3faf26bb9707a053ull, 0x70ff52905f188d57ull, 0x0a2722586f2d042eull, 0x854fb3003f739fa5ull, + 0xff97c3c80f4616dcull, 0x1bef5b57af4dc5adull, 0x61372b9f9f784cd4ull, 0xee5fbac7cf26d75full, 0x9487ca0fff135e26ull, 0xdbd7be24370c7322ull, 0xa10fceec0739fa5bull, + 0x2e675fb4576761d0ull, 0x54bf2f7c6752e8a9ull, 0xcdcf48d84fe75459ull, 0xb71738107fd2dd20ull, 0x387fa9482f8c46abull, 0x42a7d9801fb9cfd2ull, 0x0df7adabd7a6e2d6ull, + 0x772fdd63e7936bafull, 0xf8474c3bb7cdf024ull, 0x829f3cf387f8795dull, 0x66e7a46c27f3aa2cull, 0x1c3fd4a417c62355ull, 0x935745fc4798b8deull, 0xe98f353477ad31a7ull, + 0xa6df411fbfb21ca3ull, 0xdc0731d78f8795daull, 0x536fa08fdfd90e51ull, 0x29b7d047efec8728ull, +}; + + u64 crc64( void const* data, sw len ) + { + sw remaining; + u64 result = ( zpl_cast( u64 ) 0 ); + u8 const* c = zpl_cast( u8 const* ) data; + for ( remaining = len; remaining--; c++ ) + result = ( result >> 8 ) ^ ( _crc64_table[ ( result ^ *c ) & 0xff ] ); + return result; + } #pragma endregion Hashing #pragma region File Handling @@ -1591,6 +1667,167 @@ namespace gen } #pragma endregion String +#ifdef GEN_BENCHMARK +#pragma region Timing + #if defined( GEN_COMPILER_MSVC ) && ! defined( __clang__ ) + u64 read_cpu_time_stamp_counter( void ) + { + return __rdtsc(); + } + #elif defined( __i386__ ) + u64 read_cpu_time_stamp_counter( void ) + { + u64 x; + __asm__ volatile( ".byte 0x0f, 0x31" : "=A"( x ) ); + return x; + } + #elif defined( __x86_64__ ) + u64 read_cpu_time_stamp_counter( void ) + { + u32 hi, lo; + __asm__ __volatile__( "rdtsc" : "=a"( lo ), "=d"( hi ) ); + return ( zpl_cast( u64 ) lo ) | ( ( zpl_cast( u64 ) hi ) << 32 ); + } + #elif defined( __powerpc__ ) + u64 read_cpu_time_stamp_counter( void ) + { + u64 result = 0; + u32 upper, lower, tmp; + __asm__ volatile( + "0: \n" + "\tmftbu %0 \n" + "\tmftb %1 \n" + "\tmftbu %2 \n" + "\tcmpw %2,%0 \n" + "\tbne 0b \n" + : "=r"( upper ), "=r"( lower ), "=r"( tmp ) + ); + result = upper; + result = result << 32; + result = result | lower; + + return result; + } + #elif defined( GEN_SYSTEM_EMSCRIPTEN ) + u64 read_cpu_time_stamp_counter( void ) + { + return ( u64 )( emscripten_get_now() * 1e+6 ); + } + #elif defined( GEN_CPU_ARM ) && ! defined( GEN_COMPILER_TINYC ) + u64 read_cpu_time_stamp_counter( void ) + { + # if defined( __aarch64__ ) + int64_t r = 0; + asm volatile( "mrs %0, cntvct_el0" : "=r"( r ) ); + # elif ( __ARM_ARCH >= 6 ) + uint32_t r = 0; + uint32_t pmccntr; + uint32_t pmuseren; + uint32_t pmcntenset; + + // Read the user mode perf monitor counter access permissions. + asm volatile( "mrc p15, 0, %0, c9, c14, 0" : "=r"( pmuseren ) ); + if ( pmuseren & 1 ) + { // Allows reading perfmon counters for user mode code. + asm volatile( "mrc p15, 0, %0, c9, c12, 1" : "=r"( pmcntenset ) ); + if ( pmcntenset & 0x80000000ul ) + { // Is it counting? + asm volatile( "mrc p15, 0, %0, c9, c13, 0" : "=r"( pmccntr ) ); + // The counter is set up to count every 64th cycle + return ( ( int64_t )pmccntr ) * 64; // Should optimize to << 6 + } + } + # else + # error "No suitable method for read_cpu_time_stamp_counter for this cpu type" + # endif + + return r; + } + #else + u64 read_cpu_time_stamp_counter( void ) + { + GEN_PANIC( "read_cpu_time_stamp_counter is not supported on this particular setup" ); + return -0; + } + #endif + + #if defined( GEN_SYSTEM_WINDOWS ) || defined( GEN_SYSTEM_CYGWIN ) + + u64 time_rel_ms( void ) + { + local_persist LARGE_INTEGER win32_perf_count_freq = {}; + u64 result; + LARGE_INTEGER counter; + local_persist LARGE_INTEGER win32_perf_counter = {}; + if ( ! win32_perf_count_freq.QuadPart ) + { + QueryPerformanceFrequency( &win32_perf_count_freq ); + GEN_ASSERT( win32_perf_count_freq.QuadPart != 0 ); + QueryPerformanceCounter( &win32_perf_counter ); + } + + QueryPerformanceCounter( &counter ); + + result = ( counter.QuadPart - win32_perf_counter.QuadPart ) * 1000 / ( win32_perf_count_freq.QuadPart ); + return result; + } + + #else + + # if defined( GEN_SYSTEM_LINUX ) || defined( GEN_SYSTEM_FREEBSD ) || defined( GEN_SYSTEM_OPENBSD ) || defined( GEN_SYSTEM_EMSCRIPTEN ) + u64 _unix_gettime( void ) + { + struct timespec t; + u64 result; + + clock_gettime( 1 /*CLOCK_MONOTONIC*/, &t ); + result = 1000 * t.tv_sec + 1.0e-6 * t.tv_nsec; + return result; + } + # endif + + u64 time_rel_ms( void ) + { + # if defined( GEN_SYSTEM_OSX ) + u64 result; + + local_persist u64 timebase = 0; + local_persist u64 timestart = 0; + + if ( ! timestart ) + { + mach_timebase_info_data_t tb = { 0 }; + mach_timebase_info( &tb ); + timebase = tb.numer; + timebase /= tb.denom; + timestart = mach_absolute_time(); + } + + // NOTE: mach_absolute_time() returns things in nanoseconds + result = 1.0e-6 * ( mach_absolute_time() - timestart ) * timebase; + return result; + # else + local_persist u64 unix_timestart = 0.0; + + if ( ! unix_timestart ) + { + unix_timestart = _unix_gettime(); + } + + u64 now = _unix_gettime(); + + return ( now - unix_timestart ); + # endif + } + #endif + + f64 time_rel( void ) + { + return ( f64 )( time_rel_ms() * 1e-3 ); + } +#pragma endregion Timing +#endif + namespace Memory { global AllocatorInfo GlobalAllocator; @@ -1606,7 +1843,7 @@ namespace gen { if ( last.TotalUsed + size > last.TotalSize ) { - Arena bucket = Arena::init_from_allocator( heap(), BucketSize ); + Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize ); if ( bucket.PhysicalStart == nullptr ) fatal( "Failed to create bucket for Global_AllocatorBuckets"); @@ -1633,7 +1870,7 @@ namespace gen { if ( last.TotalUsed + size > last.TotalSize ) { - Arena bucket = Arena::init_from_allocator( heap(), BucketSize ); + Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize ); if ( bucket.PhysicalStart == nullptr ) fatal( "Failed to create bucket for Global_AllocatorBuckets"); @@ -1667,7 +1904,7 @@ namespace gen if ( Global_AllocatorBuckets == nullptr ) fatal( "Failed to reserve memory for Global_AllocatorBuckets"); - Arena bucket = Arena::init_from_allocator( heap(), BucketSize ); + Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize ); if ( bucket.PhysicalStart == nullptr ) fatal( "Failed to create first bucket for Global_AllocatorBuckets"); @@ -1696,4 +1933,4 @@ namespace gen } // gen_time -#endif \ No newline at end of file +#endif diff --git a/project/gen_dep.hpp b/project/gen_dep.hpp index 74b85a0..d211463 100644 --- a/project/gen_dep.hpp +++ b/project/gen_dep.hpp @@ -1,7 +1,7 @@ #pragma once #if gen_time - +#define GEN_BENCHMARK #if __clang__ # pragma clang diagnostic ignored "-Wunused-const-variable" # pragma clang diagnostic ignored "-Wswitch" @@ -1340,9 +1340,10 @@ namespace gen void destroy( void ) { if ( Hashes && Hashes.get_header()->Capacity ) + { Hashes.free(); - if ( Entries && Hashes.get_header()->Capacity ) Entries.free(); + } } Type* get( u64 key ) @@ -1547,6 +1548,7 @@ namespace gen #pragma region Hashing u32 crc32( void const* data, sw len ); + u64 crc64( void const* data, sw len ); #pragma endregion Hashing @@ -1587,6 +1589,13 @@ namespace gen sw Capacity; }; + static + uw grow_formula( uw value ) + { + // Using a very aggressive growth formula to reduce time mem_copying with recursive calls to append in this library. + return 4 * value + 8; + } + static String make( AllocatorInfo allocator, char const* str ) { @@ -1634,9 +1643,6 @@ namespace gen if ( allocation == nullptr ) return { nullptr }; - if ( ! str ) - mem_set( allocation, 0, alloc_size ); - Header& header = * rcast(Header*, allocation); header = { allocator, length, length }; @@ -1645,6 +1651,8 @@ namespace gen if ( length && str ) mem_copy( result, str, length ); + else + mem_set( result, 0, alloc_size - header_size ); result[ length ] = '\0'; @@ -1686,70 +1694,14 @@ namespace gen return true; } - bool make_space_for( char const* str, sw add_len ) - { - sw available = avail_space(); - - // NOTE: Return if there is enough space left - if ( available >= add_len ) - { - return true; - } - else - { - sw new_len, old_size, new_size; - - void* ptr; - void* new_ptr; - - AllocatorInfo allocator = get_header().Allocator; - Header* header = nullptr; - - new_len = length() + add_len; - ptr = & get_header(); - old_size = size_of( Header ) + length() + 1; - new_size = size_of( Header ) + new_len + 1; - - new_ptr = resize( allocator, ptr, old_size, new_size ); - - if ( new_ptr == nullptr ) - return false; - - header = zpl_cast( Header* ) new_ptr; - header->Allocator = allocator; - header->Capacity = new_len; - - Data = rcast( char*, header + 1 ); - - return str; - } - } + bool make_space_for( char const* str, sw add_len ); bool append( char const* str ) { return append( str, str_len( str ) ); } - bool append( char const* str, sw length ) - { - if ( sptr(str) > 0 ) - { - sw curr_len = this->length(); - - if ( ! make_space_for( str, length ) ) - return false; - - Header& header = get_header(); - - mem_copy( Data + curr_len, str, length ); - - Data[ curr_len + length ] = '\0'; - - header.Length = curr_len + length; - } - - return str; - } + bool append( char const* str, sw length ); bool append( StrC str) { @@ -2169,11 +2121,23 @@ namespace gen sw str_fmt_file_va( FileInfo* f, char const* fmt, va_list va ); #pragma endregion Printing +#ifdef GEN_BENCHMARK + //! Return CPU timestamp. + u64 read_cpu_time_stamp_counter( void ); + + //! Return relative time (in seconds) since the application start. + f64 time_rel( void ); + + //! Return relative time since the application start. + u64 time_rel_ms( void ); +#endif + namespace Memory { - // NOTE: This limits the size of the string that can be read from a file or generated to 10 megs. + // NOTE: This limits the maximum size of an allocation // If you are generating a string larger than this, increase the size of the bucket here. - constexpr uw BucketSize = megabytes(10); + constexpr uw Global_BucketSize = megabytes(10); + // Global allocator used for data with process lifetime. extern AllocatorInfo GlobalAllocator; @@ -2226,6 +2190,66 @@ namespace gen #endif } + bool String::make_space_for( char const* str, sw add_len ) + { + sw available = avail_space(); + + // NOTE: Return if there is enough space left + if ( available >= add_len ) + { + return true; + } + else + { + sw new_len, old_size, new_size; + + void* ptr; + void* new_ptr; + + AllocatorInfo allocator = get_header().Allocator; + Header* header = nullptr; + + new_len = grow_formula( length() + add_len ); + ptr = & get_header(); + old_size = size_of( Header ) + length() + 1; + new_size = size_of( Header ) + new_len + 1; + + new_ptr = resize( allocator, ptr, old_size, new_size ); + + if ( new_ptr == nullptr ) + return false; + + header = zpl_cast( Header* ) new_ptr; + header->Allocator = allocator; + header->Capacity = new_len; + + Data = rcast( char*, header + 1 ); + + return str; + } + } + + bool String::append( char const* str, sw length ) + { + u64 time_start = time_rel_ms(); + if ( sptr(str) > 0 ) + { + sw curr_len = this->length(); + + if ( ! make_space_for( str, length ) ) + return false; + + Header& header = get_header(); + + mem_copy( Data + curr_len, str, length ); + + Data[ curr_len + length ] = '\0'; + + header.Length = curr_len + length; + } + return str; + } + // gen namespace } diff --git a/scripts/build.ci.ps1 b/scripts/build.ci.ps1 index c3fb459..48807fc 100644 --- a/scripts/build.ci.ps1 +++ b/scripts/build.ci.ps1 @@ -48,7 +48,7 @@ Push-location $path_gen # Run meta-program $gencpp = Join-Path $path_gen_build gencpp.exe - Write-Host `nGenerating files -- Parsed... + Write-Host `nRunning tests... & $gencpp # Format generated files diff --git a/scripts/genccp.natstepfilter b/scripts/genccp.natstepfilter index f3d23b0..403ae58 100644 --- a/scripts/genccp.natstepfilter +++ b/scripts/genccp.natstepfilter @@ -24,4 +24,7 @@ gen::Code.*::to_string NoStepInto + + gen::String::operator .* + diff --git a/scripts/gencpp.natvis b/scripts/gencpp.natvis index 88a8a55..29fb7e2 100644 --- a/scripts/gencpp.natvis +++ b/scripts/gencpp.natvis @@ -5,6 +5,18 @@ Data:{Data} Proc:{Proc} + + NumBlocks: {NumBlocks} TotalSize: {TotalSize} + + + NumBlocks + FreeList + FreeList + PhysicalStart + + + + Num:{((Header*)((char*)Data - sizeof(Header)))->Num}, Capacity:{((Header*)((char*)Data - sizeof(Header)))->Capacity} @@ -70,7 +82,8 @@ ast->Content ast->Body ast->Parent - ast->ModuleFlags + + ast->ModuleFlags ast->ArrSpecs ast->Prev ast->Next @@ -665,4 +678,4 @@ Current[ { Arr[Idx] } ] Idx:{ Idx } - + \ No newline at end of file diff --git a/test/Parsed/Array.Parsed.hpp b/test/Parsed/Array.Parsed.hpp index b40e337..941412a 100644 --- a/test/Parsed/Array.Parsed.hpp +++ b/test/Parsed/Array.Parsed.hpp @@ -4,6 +4,8 @@ #define GEN_FEATURE_PARSING #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK #include "gen.hpp" using namespace gen; diff --git a/test/Parsed/Buffer.Parsed.hpp b/test/Parsed/Buffer.Parsed.hpp index 70059dc..b28c6a8 100644 --- a/test/Parsed/Buffer.Parsed.hpp +++ b/test/Parsed/Buffer.Parsed.hpp @@ -4,6 +4,8 @@ #define GEN_FEATURE_PARSING #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK #include "gen.hpp" using namespace gen; diff --git a/test/Parsed/HashTable.Parsed.hpp b/test/Parsed/HashTable.Parsed.hpp index 678bf60..ec6a562 100644 --- a/test/Parsed/HashTable.Parsed.hpp +++ b/test/Parsed/HashTable.Parsed.hpp @@ -4,6 +4,8 @@ #define GEN_FEATURE_PARSING #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK #include "gen.hpp" #include "Array.Parsed.hpp" diff --git a/test/Parsed/Ring.Parsed.hpp b/test/Parsed/Ring.Parsed.hpp index c744ccc..9729fb7 100644 --- a/test/Parsed/Ring.Parsed.hpp +++ b/test/Parsed/Ring.Parsed.hpp @@ -4,6 +4,8 @@ #define GEN_FEATURE_PARSING #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK #include "gen.hpp" #include "Buffer.Parsed.hpp" diff --git a/test/Parsed/Sanity.Parsed.hpp b/test/Parsed/Sanity.Parsed.hpp index 1824343..083e9ac 100644 --- a/test/Parsed/Sanity.Parsed.hpp +++ b/test/Parsed/Sanity.Parsed.hpp @@ -3,6 +3,8 @@ #define GEN_FEATURE_PARSING #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK #include "gen.hpp" using namespace gen; @@ -289,7 +291,7 @@ u32 gen_sanity() )); - CodeUsingNamespace npspace_using = (CodeUsingNamespace) parse_using( code( + CodeUsing npspace_using = parse_using( code( using namespace TestNamespace; )); diff --git a/test/parsing.cpp b/test/parsing.cpp new file mode 100644 index 0000000..e69de29 diff --git a/test/parsing.hpp b/test/parsing.hpp new file mode 100644 index 0000000..e69de29 diff --git a/test/sanity.cpp b/test/sanity.cpp new file mode 100644 index 0000000..c39369a --- /dev/null +++ b/test/sanity.cpp @@ -0,0 +1,92 @@ +// Testing to make sure backend of library is operating properly. + +#ifdef gen_time +#define GEN_FEATURE_PARSING +#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS +#define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK +#include "gen.hpp" + +void check_sanity() +{ + using namespace gen; + gen::init(); + log_fmt("\ncheck_sanity:\n"); + + // Test string caching: + CodeType t_int_dupe = def_type( name(int) ); + + if ( t_int_dupe->Name != t_int->Name ) + fatal("check_sanity: String caching failed!"); + + + // Purposefully use an excessive amount of memory to make so the the memory backend doesn't break. + // This has been tested with num_iterations set to 15000000 (generates 15 million lines of code), the Global_BlockSize, along with CodePool_NumBlocks, and SizePer_StringArena + // must be adjusted to gigabytes(2), kilobytes(512), and gigabyte(1) for good performance without crashing. + /* + Typical usage (megabytes(10), kilobytes(4), megabytes(1), for 650000 (the limit of 10 meg partition buckets in global arena) ) + Num Global Arenas : 2 TotalSize: 4294967296 ! + Num Code Pools : 144 TotalSize: 9663676416 ! + Num String Cache Arenas : 2 TotalSize: 2147483648 ! + Num String Cache : 30000025 + + Memory usage to expect at 15 mil file: + Num Global Arenas : 2 TotalSize: 4294967296 ! + Num Code Pools : 144 TotalSize: 9663676416 ! + Num String Cache Arenas : 2 TotalSize: 2147483648 ! + Num String Cache : 30000025 + */ + constexpr + s32 num_iterations = 650000; + + Array typedefs = Array::init_reserve( Memory::GlobalAllocator, num_iterations * 2 ); + + s32 idx = num_iterations; + while( --idx ) + { + // Stress testing string allocation + String type_name = String::fmt_buf( Memory::GlobalAllocator, "type_%d", idx ); + String typedef_name = String::fmt_buf( Memory::GlobalAllocator, "typedef_%d", idx ); + + CodeTypedef type_as_int = def_typedef( type_name, t_int ); + CodeType type = def_type( type_name ); + CodeTypedef type_def = def_typedef( typedef_name, type ); + + typedefs.append( type_as_int ); + typedefs.append( type_def ); + } + + log_fmt("\nMemory before builder:\n"); + log_fmt("Num Global Arenas : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize); + log_fmt("Num Code Pools : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize); + log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena); + log_fmt("Num String Cache : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache); + + Builder builder; + builder.open( "sanity.gen.hpp" ); + + idx = num_iterations; +#ifdef GEN_BENCHMARK + u64 time_start = time_rel_ms(); +#endif + while( --idx ) + { + builder.print( typedefs[idx] ); + } + + builder.write(); +#ifdef GEN_BENCHMARK + log_fmt("\n\nBuilder finished writting. Time taken: %llu ms\n", time_rel_ms() - time_start); +#endif + + log_fmt("\nMemory after builder:\n"); + log_fmt("Num Global Arenas : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize); + log_fmt("Num Code Pools : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize); + log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena); + log_fmt("Num String Cache : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache); + + log_fmt("\nSanity passed!\n"); + gen::deinit(); +} +#endif diff --git a/test/test.Upfront.cpp b/test/test.Upfront.cpp index 8778c1f..8701b76 100644 --- a/test/test.Upfront.cpp +++ b/test/test.Upfront.cpp @@ -1,6 +1,9 @@ #ifdef gen_time -#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_FEATURE_PARSING +#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS +#define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK #include "gen.cpp" #include "Upfront\Array.Upfront.hpp" #include "Upfront\Buffer.Upfront.hpp" diff --git a/test/test.cpp b/test/test.cpp index 6213cdf..11b8d4c 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -1,73 +1,25 @@ -#ifdef gen_time -#define GEN_FEATURE_PARSING -#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS -#define GEN_ENFORCE_STRONG_CODE_TYPES -#include "Parsed\Array.Parsed.hpp" -#include "Parsed\Buffer.Parsed.hpp" -#include "Parsed\HashTable.Parsed.hpp" -#include "Parsed\Ring.Parsed.hpp" -#include "Parsed\Sanity.Parsed.hpp" -#include "SOA.hpp" #include "gen.cpp" +#include "sanity.cpp" -using namespace gen; - -// TODO : Need to make a more robust test suite - +#if gen_time int gen_main() { - gen::init(); + using namespace gen; + log_fmt("\ngen_time:"); - gen_sanity(); + check_sanity(); - gen_array( u8 ); - gen_array( sw ); - - gen_buffer( u8 ); - - gen_hashtable( u32 ); - - gen_ring( s16 ); - gen_ring( uw ); - - gen_array_file(); - gen_buffer_file(); - gen_hashtable_file(); - gen_ring_file(); - - Builder soa_test; soa_test.open( "SOA.gen.hpp" ); - - soa_test.print( parse_using( code( - using u16 = unsigned short; - ))); - - soa_test.print( def_include( txt_StrC("gen.hpp"))); - - soa_test.print( def_using_namespace( name(gen) ) ); - - soa_test.print( gen_SOA( - parse_struct( code( - struct TestStruct - { - u8 A; - u16 B; - u32 C; - u64 D; - }; - )) - )); - - soa_test.write(); - - gen::deinit(); return 0; } #endif -#ifdef runtime +// This only has to be done if symbol conflicts occur. +#ifndef gen_time int main() { + + return 0; } #endif diff --git a/test/test.parsing.cpp b/test/test.parsing.cpp new file mode 100644 index 0000000..4c4a1d3 --- /dev/null +++ b/test/test.parsing.cpp @@ -0,0 +1,75 @@ +#ifdef gen_time +#define GEN_FEATURE_PARSING +#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS +#define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK +#include "Parsed\Array.Parsed.hpp" +#include "Parsed\Buffer.Parsed.hpp" +#include "Parsed\HashTable.Parsed.hpp" +#include "Parsed\Ring.Parsed.hpp" +#include "Parsed\Sanity.Parsed.hpp" +#include "SOA.hpp" +#include "gen.cpp" + +using namespace gen; + +// TODO : Need to make a more robust test suite + +int gen_main() +{ + gen::init(); + + gen_sanity(); + + gen_array( u8 ); + gen_array( sw ); + + gen_buffer( u8 ); + + gen_hashtable( u32 ); + + gen_ring( s16 ); + gen_ring( uw ); + + gen_array_file(); + gen_buffer_file(); + gen_hashtable_file(); + gen_ring_file(); + + Builder soa_test; soa_test.open( "SOA.gen.hpp" ); + + soa_test.print( parse_using( code( + using u16 = unsigned short; + ))); + + soa_test.print( def_include( txt_StrC("gen.hpp"))); + + soa_test.print( def_using_namespace( name(gen) ) ); + + soa_test.print( gen_SOA( + parse_struct( code( + struct TestStruct + { + u8 A; + u16 B; + u32 C; + u64 D; + }; + )) + )); + + soa_test.write(); + + gen::deinit(); + return 0; +} +#endif + + +#ifdef runtime +int main() +{ + return 0; +} +#endif diff --git a/test/upfront.cpp b/test/upfront.cpp new file mode 100644 index 0000000..e69de29 diff --git a/test/upfront.hpp b/test/upfront.hpp new file mode 100644 index 0000000..e150f23 --- /dev/null +++ b/test/upfront.hpp @@ -0,0 +1,21 @@ +#ifdef gen_time +#define GEN_FEATURE_PARSING +#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS +#define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK +#include "gen.hpp" + +void check_upfront() +{ + using namespace gen; + log_fmt("\nupfront: "); + gen::init(); + + + + gen::deinit(); + log_fmt("Passed!\n"); +} + +#endif \ No newline at end of file