mirror of
https://github.com/Ed94/gencpp.git
synced 2025-06-15 03:01:47 -07:00
Compare commits
3 Commits
V0.2-Alpha
...
V0.3-Alpha
Author | SHA1 | Date | |
---|---|---|---|
98b776d66e | |||
7634aeb34c | |||
b544f24015 |
@ -23,6 +23,9 @@ These build up a code AST to then serialize with a file builder.
|
||||
|
||||
The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases.
|
||||
|
||||
Note: Do not trying to do any large generations with this (at least not without changing the serialization implementation).
|
||||
It does not resue any memory yet for dynamic strings and thus any signficant size memory will result in massive consumption.
|
||||
|
||||
The project has no external dependencies beyond:
|
||||
|
||||
* `stdarg.h`
|
||||
@ -287,6 +290,12 @@ Data Notes:
|
||||
* Memory within the buckets is not resused, so its inherently wasteful (most likely will give non-cached strings their own tailored allocator later)
|
||||
* Linked lists used children nodes on bodies, and parameters.
|
||||
* Its intended to generate the AST in one go and serialize after. The contructors and serializer are designed to be a "one pass, front to back" setup.
|
||||
* When benchmarking, the three most significant var to tune are:
|
||||
* `Memory::Global_BlockSize` (found gen_dep.hpp) : Used by the GlobalAllocator for the size of each global arena.
|
||||
* `SizePer_StringArena` (found in gen.hpp under the constants region) : Used by the string cache to store strings.
|
||||
* `CodePool_NumBlocks` (found in gen.hpp under constants region) : Used by code pool to store ASTs.
|
||||
* The default values can handled generating for a stirng up to a size of ~650 kbs (bottleneck is serialization).
|
||||
* Increasing the values can generate files upwards of over a million lines without issue (the formatter will most likely run slower than it)
|
||||
|
||||
Two generic templated containers are used throughout the library:
|
||||
|
||||
|
1811
project/gen.cpp
1811
project/gen.cpp
File diff suppressed because it is too large
Load Diff
2290
project/gen.hpp
2290
project/gen.hpp
File diff suppressed because it is too large
Load Diff
1936
project/gen_dep.cpp
Normal file
1936
project/gen_dep.cpp
Normal file
File diff suppressed because it is too large
Load Diff
2257
project/gen_dep.hpp
Normal file
2257
project/gen_dep.hpp
Normal file
File diff suppressed because it is too large
Load Diff
@ -48,7 +48,7 @@ Push-location $path_gen
|
||||
# Run meta-program
|
||||
$gencpp = Join-Path $path_gen_build gencpp.exe
|
||||
|
||||
Write-Host `nGenerating files -- Parsed...
|
||||
Write-Host `nRunning tests...
|
||||
& $gencpp
|
||||
|
||||
# Format generated files
|
||||
|
@ -24,4 +24,7 @@
|
||||
<Name>gen::Code.*::to_string</Name>
|
||||
<Action>NoStepInto</Action>
|
||||
</Function>
|
||||
<Function>
|
||||
<Name>gen::String::operator .*</Name>
|
||||
</Function>
|
||||
</StepFilter>
|
||||
|
@ -5,6 +5,18 @@
|
||||
<DisplayString>Data:{Data} Proc:{Proc}</DisplayString>
|
||||
</Type>
|
||||
|
||||
<Type Name="gen::Pool">
|
||||
<DisplayString>NumBlocks: {NumBlocks} TotalSize: {TotalSize}</DisplayString>
|
||||
<Expand>
|
||||
<LinkedListItems>
|
||||
<Size>NumBlocks</Size>
|
||||
<HeadPointer>FreeList</HeadPointer>
|
||||
<NextPointer>FreeList</NextPointer>
|
||||
<ValueNode>PhysicalStart</ValueNode>
|
||||
</LinkedListItems>
|
||||
</Expand>
|
||||
</Type>
|
||||
|
||||
<Type Name="gen::Array<*>">
|
||||
<DisplayString>Num:{((Header*)((char*)Data - sizeof(Header)))->Num},
|
||||
Capacity:{((Header*)((char*)Data - sizeof(Header)))->Capacity}</DisplayString>
|
||||
@ -70,7 +82,8 @@
|
||||
<Item Name="Content">ast->Content</Item>
|
||||
<Item Name="Body">ast->Body</Item>
|
||||
<Item Name="Parent">ast->Parent</Item>
|
||||
<Item Name="ModuleFlags" Condition="ast->ModuleFlags != ModuleFlag::Invalid">ast->ModuleFlags</Item>
|
||||
<Item Name="ModuleFlags" Condition="ast->ModuleFlags != ModuleFlag::Invalid">
|
||||
ast->ModuleFlags</Item>
|
||||
<Item Name="ArrSpecs" Condition="ast->ArrSpecs[0] < 18">ast->ArrSpecs</Item>
|
||||
<Item Name="Prev">ast->Prev</Item>
|
||||
<Item Name="Next">ast->Next</Item>
|
||||
@ -665,4 +678,4 @@
|
||||
<DisplayString>Current[ { Arr[Idx] } ] Idx:{ Idx }</DisplayString>
|
||||
</Type>
|
||||
|
||||
</AutoVisualizer>
|
||||
</AutoVisualizer>
|
@ -4,6 +4,8 @@
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
|
||||
using namespace gen;
|
||||
|
@ -4,6 +4,8 @@
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
|
||||
using namespace gen;
|
||||
|
@ -4,6 +4,8 @@
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
#include "Array.Parsed.hpp"
|
||||
|
||||
|
@ -4,6 +4,8 @@
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
#include "Buffer.Parsed.hpp"
|
||||
|
||||
|
@ -3,6 +3,8 @@
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
|
||||
using namespace gen;
|
||||
@ -289,7 +291,7 @@ u32 gen_sanity()
|
||||
|
||||
));
|
||||
|
||||
CodeUsingNamespace npspace_using = (CodeUsingNamespace) parse_using( code(
|
||||
CodeUsing npspace_using = parse_using( code(
|
||||
using namespace TestNamespace;
|
||||
));
|
||||
|
||||
|
0
test/parsing.cpp
Normal file
0
test/parsing.cpp
Normal file
0
test/parsing.hpp
Normal file
0
test/parsing.hpp
Normal file
93
test/sanity.cpp
Normal file
93
test/sanity.cpp
Normal file
@ -0,0 +1,93 @@
|
||||
// Testing to make sure backend of library is operating properly.
|
||||
|
||||
#ifdef gen_time
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
|
||||
void check_sanity()
|
||||
{
|
||||
using namespace gen;
|
||||
gen::init();
|
||||
log_fmt("\ncheck_sanity:\n");
|
||||
|
||||
// Test string caching:
|
||||
CodeType t_int_dupe = def_type( name(int) );
|
||||
|
||||
if ( t_int_dupe->Name != t_int->Name )
|
||||
fatal("check_sanity: String caching failed!");
|
||||
|
||||
|
||||
// Purposefully use an excessive amount of memory to make so the the memory backend doesn't break.
|
||||
// This has been tested with num_iterations set to 15000000 (generates 15 million lines of code), the Global_BlockSize, along with CodePool_NumBlocks, and SizePer_StringArena
|
||||
// must be adjusted to gigabytes(2), kilobytes(512), and gigabyte(1) for good performance without crashing.
|
||||
/*
|
||||
Typical usage (megabytes(10), kilobytes(4), megabytes(1), for 650000 (the limit of 10 meg partition buckets in global arena) )
|
||||
Memory after builder:
|
||||
Num Global Arenas : 14 TotalSize: 146800640 !
|
||||
Num Code Pools : 794 TotalSize: 416284672 !
|
||||
Num String Cache Arenas : 60 TotalSize: 62914560 !
|
||||
Num String Cache : 1300007
|
||||
|
||||
Memory usage to expect at 15 mil file:
|
||||
Num Global Arenas : 2 TotalSize: 4294967296 !
|
||||
Num Code Pools : 144 TotalSize: 9663676416 !
|
||||
Num String Cache Arenas : 2 TotalSize: 2147483648 !
|
||||
Num String Cache : 30000025
|
||||
*/
|
||||
constexpr
|
||||
s32 num_iterations = 650000;
|
||||
|
||||
Array<CodeTypedef> typedefs = Array<CodeTypedef>::init_reserve( Memory::GlobalAllocator, num_iterations * 2 );
|
||||
|
||||
s32 idx = num_iterations;
|
||||
while( --idx )
|
||||
{
|
||||
// Stress testing string allocation
|
||||
String type_name = String::fmt_buf( Memory::GlobalAllocator, "type_%d", idx );
|
||||
String typedef_name = String::fmt_buf( Memory::GlobalAllocator, "typedef_%d", idx );
|
||||
|
||||
CodeTypedef type_as_int = def_typedef( type_name, t_int );
|
||||
CodeType type = def_type( type_name );
|
||||
CodeTypedef type_def = def_typedef( typedef_name, type );
|
||||
|
||||
typedefs.append( type_as_int );
|
||||
typedefs.append( type_def );
|
||||
}
|
||||
|
||||
log_fmt("\nMemory before builder:\n");
|
||||
log_fmt("Num Global Arenas : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize);
|
||||
log_fmt("Num Code Pools : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize);
|
||||
log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena);
|
||||
log_fmt("Num String Cache : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache);
|
||||
|
||||
Builder builder;
|
||||
builder.open( "sanity.gen.hpp" );
|
||||
|
||||
idx = num_iterations;
|
||||
#ifdef GEN_BENCHMARK
|
||||
u64 time_start = time_rel_ms();
|
||||
#endif
|
||||
while( --idx )
|
||||
{
|
||||
builder.print( typedefs[idx] );
|
||||
}
|
||||
|
||||
builder.write();
|
||||
#ifdef GEN_BENCHMARK
|
||||
log_fmt("\n\nBuilder finished writting. Time taken: %llu ms\n", time_rel_ms() - time_start);
|
||||
#endif
|
||||
|
||||
log_fmt("\nMemory after builder:\n");
|
||||
log_fmt("Num Global Arenas : %llu TotalSize: %llu !\n", Memory::Global_AllocatorBuckets.num(), Memory::Global_AllocatorBuckets.num() * Memory::Global_BucketSize);
|
||||
log_fmt("Num Code Pools : %llu TotalSize: %llu !\n", StaticData::CodePools.num(), StaticData::CodePools.num() * CodePool_NumBlocks * StaticData::CodePools.back().BlockSize);
|
||||
log_fmt("Num String Cache Arenas : %llu TotalSize: %llu !\n", StaticData::StringArenas.num(), StaticData::StringArenas.num() * SizePer_StringArena);
|
||||
log_fmt("Num String Cache : %llu\n", StaticData::StringCache.Entries.num(), StaticData::StringCache);
|
||||
|
||||
log_fmt("\nSanity passed!\n");
|
||||
gen::deinit();
|
||||
}
|
||||
#endif
|
@ -1,6 +1,9 @@
|
||||
#ifdef gen_time
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.cpp"
|
||||
#include "Upfront\Array.Upfront.hpp"
|
||||
#include "Upfront\Buffer.Upfront.hpp"
|
||||
|
@ -1,73 +1,25 @@
|
||||
#ifdef gen_time
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#include "Parsed\Array.Parsed.hpp"
|
||||
#include "Parsed\Buffer.Parsed.hpp"
|
||||
#include "Parsed\HashTable.Parsed.hpp"
|
||||
#include "Parsed\Ring.Parsed.hpp"
|
||||
#include "Parsed\Sanity.Parsed.hpp"
|
||||
#include "SOA.hpp"
|
||||
#include "gen.cpp"
|
||||
#include "sanity.cpp"
|
||||
|
||||
using namespace gen;
|
||||
|
||||
// TODO : Need to make a more robust test suite
|
||||
|
||||
#if gen_time
|
||||
int gen_main()
|
||||
{
|
||||
gen::init();
|
||||
using namespace gen;
|
||||
log_fmt("\ngen_time:");
|
||||
|
||||
gen_sanity();
|
||||
check_sanity();
|
||||
|
||||
gen_array( u8 );
|
||||
gen_array( sw );
|
||||
|
||||
gen_buffer( u8 );
|
||||
|
||||
gen_hashtable( u32 );
|
||||
|
||||
gen_ring( s16 );
|
||||
gen_ring( uw );
|
||||
|
||||
gen_array_file();
|
||||
gen_buffer_file();
|
||||
gen_hashtable_file();
|
||||
gen_ring_file();
|
||||
|
||||
Builder soa_test; soa_test.open( "SOA.gen.hpp" );
|
||||
|
||||
soa_test.print( parse_using( code(
|
||||
using u16 = unsigned short;
|
||||
)));
|
||||
|
||||
soa_test.print( def_include( txt_StrC("gen.hpp")));
|
||||
|
||||
soa_test.print( def_using_namespace( name(gen) ) );
|
||||
|
||||
soa_test.print( gen_SOA(
|
||||
parse_struct( code(
|
||||
struct TestStruct
|
||||
{
|
||||
u8 A;
|
||||
u16 B;
|
||||
u32 C;
|
||||
u64 D;
|
||||
};
|
||||
))
|
||||
));
|
||||
|
||||
soa_test.write();
|
||||
|
||||
gen::deinit();
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef runtime
|
||||
// This only has to be done if symbol conflicts occur.
|
||||
#ifndef gen_time
|
||||
int main()
|
||||
{
|
||||
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
75
test/test.parsing.cpp
Normal file
75
test/test.parsing.cpp
Normal file
@ -0,0 +1,75 @@
|
||||
#ifdef gen_time
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "Parsed\Array.Parsed.hpp"
|
||||
#include "Parsed\Buffer.Parsed.hpp"
|
||||
#include "Parsed\HashTable.Parsed.hpp"
|
||||
#include "Parsed\Ring.Parsed.hpp"
|
||||
#include "Parsed\Sanity.Parsed.hpp"
|
||||
#include "SOA.hpp"
|
||||
#include "gen.cpp"
|
||||
|
||||
using namespace gen;
|
||||
|
||||
// TODO : Need to make a more robust test suite
|
||||
|
||||
int gen_main()
|
||||
{
|
||||
gen::init();
|
||||
|
||||
gen_sanity();
|
||||
|
||||
gen_array( u8 );
|
||||
gen_array( sw );
|
||||
|
||||
gen_buffer( u8 );
|
||||
|
||||
gen_hashtable( u32 );
|
||||
|
||||
gen_ring( s16 );
|
||||
gen_ring( uw );
|
||||
|
||||
gen_array_file();
|
||||
gen_buffer_file();
|
||||
gen_hashtable_file();
|
||||
gen_ring_file();
|
||||
|
||||
Builder soa_test; soa_test.open( "SOA.gen.hpp" );
|
||||
|
||||
soa_test.print( parse_using( code(
|
||||
using u16 = unsigned short;
|
||||
)));
|
||||
|
||||
soa_test.print( def_include( txt_StrC("gen.hpp")));
|
||||
|
||||
soa_test.print( def_using_namespace( name(gen) ) );
|
||||
|
||||
soa_test.print( gen_SOA(
|
||||
parse_struct( code(
|
||||
struct TestStruct
|
||||
{
|
||||
u8 A;
|
||||
u16 B;
|
||||
u32 C;
|
||||
u64 D;
|
||||
};
|
||||
))
|
||||
));
|
||||
|
||||
soa_test.write();
|
||||
|
||||
gen::deinit();
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef runtime
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
0
test/upfront.cpp
Normal file
0
test/upfront.cpp
Normal file
21
test/upfront.hpp
Normal file
21
test/upfront.hpp
Normal file
@ -0,0 +1,21 @@
|
||||
#ifdef gen_time
|
||||
#define GEN_FEATURE_PARSING
|
||||
#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
|
||||
#define GEN_ENFORCE_STRONG_CODE_TYPES
|
||||
#define GEN_EXPOSE_BACKEND
|
||||
#define GEN_BENCHMARK
|
||||
#include "gen.hpp"
|
||||
|
||||
void check_upfront()
|
||||
{
|
||||
using namespace gen;
|
||||
log_fmt("\nupfront: ");
|
||||
gen::init();
|
||||
|
||||
|
||||
|
||||
gen::deinit();
|
||||
log_fmt("Passed!\n");
|
||||
}
|
||||
|
||||
#endif
|
Reference in New Issue
Block a user